update
This commit is contained in:
toom1996
@ -50,4 +50,30 @@ type BrandRunwayImage struct {
|
||||
// TableName 指定图片明细表名
|
||||
func (BrandRunwayImage) TableName() string {
|
||||
return "brand_runway_images"
|
||||
}
|
||||
|
||||
// Article The Impression 街拍文章表（对应 PHP 的 articles 模型）
|
||||
// 字段与 PHP 的 ArticleModuleEnum::STREET 文章结构保持一致
|
||||
type Article struct {
|
||||
ID uint `gorm:"primaryKey;column:id;autoIncrement" json:"id"`
|
||||
Title string `gorm:"column:title" json:"title"`
|
||||
Platform string `gorm:"column:platform" json:"platform"`
|
||||
Module string `gorm:"column:module" json:"module"`
|
||||
Year uint16 `gorm:"column:year" json:"year"`
|
||||
Cover string `gorm:"column:cover" json:"cover"`
|
||||
Images string `gorm:"column:images" json:"images"`
|
||||
SourceURL string `gorm:"column:source_url" json:"source_url"`
|
||||
Brand uint `gorm:"column:brand;default:0" json:"brand"`
|
||||
CreatedAt int64 `gorm:"column:created_at" json:"created_at"`
|
||||
UpdatedAt int64 `gorm:"column:updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
// TableName 指定文章主表名
|
||||
func (Article) TableName() string {
|
||||
return "articles"
|
||||
}
|
||||
|
||||
// ImageItem 图片条目，序列化进 Article.Images(JSON 数组)
|
||||
type ImageItem struct {
|
||||
Src string `json:"src"`
|
||||
}
|
||||
285
internal/spider/theimpression.go
Normal file
285
internal/spider/theimpression.go
Normal file
@ -0,0 +1,285 @@
|
||||
package spider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"my-spiders/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
TheImpressionBaseURL = "https://theimpression.com"
|
||||
TheImpressionPlatform = "theimpression-street"
|
||||
// TheImpressionModule 对应 PHP 的 ArticleModuleEnum::STREET->value
|
||||
// 若你服务端枚举值不是 "street"，请在此修改为实际值
|
||||
TheImpressionModule = "street"
|
||||
)
|
||||
|
||||
// TheImpressionSpider The Impression 街拍采集器（对应 PHP 的 TheImpressionStreetCommand）
|
||||
type TheImpressionSpider struct {
|
||||
Client *http.Client
|
||||
DB *gorm.DB
|
||||
MaxCo int // 最大并发数
|
||||
Debug bool // Debug 模式：仅输出不入库
|
||||
ForceUpdate bool // 强制更新已存在的记录
|
||||
}
|
||||
|
||||
func NewTheImpressionSpider(db *gorm.DB, maxCo int) *TheImpressionSpider {
|
||||
if maxCo <= 0 {
|
||||
maxCo = 5
|
||||
}
|
||||
return &TheImpressionSpider{
|
||||
DB: db,
|
||||
MaxCo: maxCo,
|
||||
Client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// streetTask 单个待抓取的文章任务
|
||||
type streetTask struct {
|
||||
URL string
|
||||
Title string
|
||||
}
|
||||
|
||||
// Run 爬虫总入口
|
||||
func (s *TheImpressionSpider) Run() {
|
||||
var tasks []streetTask
|
||||
|
||||
// 1. 抓取 street-style 首页 banner（.parallax .mask-overlay）
|
||||
bannerURL := TheImpressionBaseURL + "/street-style"
|
||||
body, httpCode := s.request(bannerURL)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[错误] %s 请求失败, code: %d", bannerURL, httpCode)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("[错误] 解析 %s 失败: %v", bannerURL, err)
|
||||
return
|
||||
}
|
||||
|
||||
doc.Find(".parallax .mask-overlay").Each(func(_ int, node *goquery.Selection) {
|
||||
href, _ := node.Attr("href")
|
||||
text := strings.TrimSpace(node.AttrOr("aria-label", ""))
|
||||
log.Printf("标题: %s", text)
|
||||
log.Printf("链接: %s", href)
|
||||
if href != "" && text != "" {
|
||||
tasks = append(tasks, streetTask{URL: href, Title: text})
|
||||
}
|
||||
})
|
||||
|
||||
// 2. 抓取 WP JSON 接口的文章列表（对应 PHP 的 wp-json/codetipi-zeen 分页）
|
||||
// 注意：原 PHP 写为 for ($i = 1; $i < 2; $i++)，即仅抓取第 1 页。
|
||||
// 如需翻页，将此处 2 改为目标页数 +1 即可（原文注释为“前五十页”）。
|
||||
for i := 1; i < 2; i++ {
|
||||
listURL := fmt.Sprintf(
|
||||
"%s/wp-json/codetipi-zeen/v1/block?paged=%d&type=1&data%%5Bargs%%5D%%5Bcat%%5D=1",
|
||||
TheImpressionBaseURL, i,
|
||||
)
|
||||
body, httpCode := s.request(listURL)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[错误] %s 请求失败, code: %d", listURL, httpCode)
|
||||
return
|
||||
}
|
||||
|
||||
// 接口返回的是 JSON 数组，第 2 个元素（index 1）是包含文章的 HTML 片段
|
||||
var arr []interface{}
|
||||
if err := json.Unmarshal([]byte(body), &arr); err != nil {
|
||||
log.Printf("[错误] %s JSON 解析失败: %v", listURL, err)
|
||||
continue
|
||||
}
|
||||
if len(arr) < 2 {
|
||||
log.Printf("[Info] %s 未返回足够的数组元素，跳过.", listURL)
|
||||
continue
|
||||
}
|
||||
htmlStr, ok := arr[1].(string)
|
||||
if !ok || htmlStr == "" {
|
||||
log.Printf("[Info] %s 第 2 个元素不是 HTML 字符串，跳过.", listURL)
|
||||
continue
|
||||
}
|
||||
|
||||
listDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
log.Printf("[错误] 解析 %s 文章列表 HTML 失败: %v", listURL, err)
|
||||
continue
|
||||
}
|
||||
|
||||
listDoc.Find("article").Each(func(_ int, node *goquery.Selection) {
|
||||
href, _ := node.Find(".mask-img").Attr("href")
|
||||
title := strings.TrimSpace(node.Find(".title-wrap").Text())
|
||||
if href == "" || title == "" {
|
||||
log.Printf("找不到标题或链接.")
|
||||
return
|
||||
}
|
||||
tasks = append(tasks, streetTask{URL: href, Title: title})
|
||||
})
|
||||
}
|
||||
|
||||
log.Printf("[Info] 共收集到 %d 个待抓取任务，开始执行...", len(tasks))
|
||||
|
||||
// 3. 并发抓取详情页
|
||||
sem := make(chan struct{}, s.MaxCo)
|
||||
var wg sync.WaitGroup
|
||||
for _, t := range tasks {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(tk streetTask) {
|
||||
defer func() {
|
||||
<-sem
|
||||
wg.Done()
|
||||
}()
|
||||
s.getDetail(tk)
|
||||
time.Sleep(200 * time.Millisecond) // 频控，防封 IP
|
||||
}(t)
|
||||
}
|
||||
wg.Wait()
|
||||
log.Println("[Info] The Impression 所有抓取任务已完成！")
|
||||
}
|
||||
|
||||
// getDetail 抓取并解析单个文章详情（对应 PHP 的 getDetail）
|
||||
func (s *TheImpressionSpider) getDetail(tk streetTask) {
|
||||
url := tk.URL
|
||||
title := tk.Title
|
||||
if url == "" || title == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// 判重：按 title + platform 查询是否已存在
|
||||
var exist model.Article
|
||||
if !s.Debug && s.DB != nil {
|
||||
s.DB.Where("title = ? AND platform = ?", title, TheImpressionPlatform).First(&exist)
|
||||
if exist.ID > 0 && !s.ForceUpdate {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
body, httpCode := s.request(url)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[错误] %s 请求失败, code: %d", url, httpCode)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("[错误] 解析 %s 失败: %v", url, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 采集 figure a img 的图片（去重）
|
||||
imagesMap := make(map[string]model.ImageItem)
|
||||
doc.Find("figure a img").Each(func(_ int, node *goquery.Selection) {
|
||||
src, _ := node.Attr("src")
|
||||
if src != "" {
|
||||
if _, ok := imagesMap[src]; !ok {
|
||||
log.Printf("采集图片: %s", src)
|
||||
imagesMap[src] = model.ImageItem{Src: src}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if len(imagesMap) == 0 {
|
||||
log.Printf("[Warning] %s 未采集到图片，跳过保存.", url)
|
||||
return
|
||||
}
|
||||
|
||||
cover := ""
|
||||
var imageList []model.ImageItem
|
||||
for _, v := range imagesMap {
|
||||
if cover == "" {
|
||||
cover = v.Src
|
||||
}
|
||||
imageList = append(imageList, v)
|
||||
}
|
||||
|
||||
imagesJSON, _ := json.Marshal(imageList)
|
||||
now := time.Now().Unix()
|
||||
|
||||
article := model.Article{
|
||||
Title: title,
|
||||
Platform: TheImpressionPlatform,
|
||||
Module: TheImpressionModule,
|
||||
Year: uint16(s.parseYear(title)),
|
||||
Cover: cover,
|
||||
Images: string(imagesJSON),
|
||||
SourceURL: url,
|
||||
Brand: 0,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
// Debug 模式或无数据库连接时，仅控制台输出
|
||||
if s.Debug || s.DB == nil {
|
||||
out, _ := json.MarshalIndent(article, "", " ")
|
||||
log.Printf("\n================ [DEBUG OUTPUT] ================\n%s\n提取图片总数: %d 张\n================================================",
|
||||
string(out), len(imageList))
|
||||
return
|
||||
}
|
||||
|
||||
// 入库（存在则更新，不存在则新增）
|
||||
err = s.DB.Transaction(func(tx *gorm.DB) error {
|
||||
if exist.ID > 0 {
|
||||
article.ID = exist.ID
|
||||
if err := tx.Save(&article).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := tx.Create(&article).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("[错误] 数据入库失败 [%s]: %v", title, err)
|
||||
} else {
|
||||
log.Printf("[成功] 保存文章: %s (共 %d 张图片)", title, len(imageList))
|
||||
}
|
||||
}
|
||||
|
||||
// request HTTP 请求封装（带浏览器 UA）
|
||||
func (s *TheImpressionSpider) request(url string) (string, int) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/json,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
|
||||
resp, err := s.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", resp.StatusCode
|
||||
}
|
||||
return string(bodyBytes), resp.StatusCode
|
||||
}
|
||||
|
||||
// parseYear 从标题中提取 4 位年份数字（对应 PHP 的 AppHelper::getYear）
|
||||
func (s *TheImpressionSpider) parseYear(title string) int {
|
||||
re := regexp.MustCompile(`\b(19|20)\d{2}\b`)
|
||||
match := re.FindString(title)
|
||||
if match != "" {
|
||||
y, _ := strconv.Atoi(match)
|
||||
return y
|
||||
}
|
||||
return time.Now().Year()
|
||||
}
|
||||
@ -220,7 +220,7 @@ func (s *VogueSpider) getDetail(brandID int64, info gjson.Result) {
|
||||
xxlURL := img.Get("image.sources.xxl.url").String()
|
||||
if xxlURL != "" {
|
||||
saveURL = append(saveURL, ImageItem{Src: xxlURL})
|
||||
log.Printf(xxlURL)
|
||||
log.Printf("%s", xxlURL)
|
||||
}
|
||||
|
||||
// 详情图片提取
|
||||
|
||||
Reference in New Issue
Block a user