update
This commit is contained in:
toom1996
52
internal/config/config.go
Normal file
52
internal/config/config.go
Normal file
@ -0,0 +1,52 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// GlobalConfig 全局配置实例
|
||||
var GlobalConfig Config
|
||||
|
||||
type Config struct {
|
||||
App AppConfig `mapstructure:"app"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
}
|
||||
|
||||
type AppConfig struct {
|
||||
MaxCo int `mapstructure:"max_co"`
|
||||
Debug bool `mapstructure:"debug"` // 新增 Debug 字段
|
||||
}
|
||||
|
||||
type DatabaseConfig struct {
|
||||
DSN string `mapstructure:"dsn"`
|
||||
MaxOpenConns int `mapstructure:"max_open_conns"`
|
||||
MaxIdleConns int `mapstructure:"max_idle_conns"`
|
||||
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
|
||||
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
|
||||
}
|
||||
|
||||
// InitConfig 初始化并读取 YAML 配置文件
|
||||
func InitConfig(cfgFile string) error {
|
||||
if cfgFile != "" {
|
||||
// 如果命令行指定了配置文件路径
|
||||
viper.SetConfigFile(cfgFile)
|
||||
} else {
|
||||
// 默认在当前目录下查找 config.yaml
|
||||
viper.AddConfigPath(".")
|
||||
viper.SetConfigType("yaml")
|
||||
viper.SetConfigName("config")
|
||||
}
|
||||
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
return fmt.Errorf("读取配置文件失败: %w", err)
|
||||
}
|
||||
|
||||
// 解析到结构体
|
||||
if err := viper.Unmarshal(&GlobalConfig); err != nil {
|
||||
return fmt.Errorf("解析配置文件结构失败: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
29
internal/model/models.go
Normal file
29
internal/model/models.go
Normal file
@ -0,0 +1,29 @@
|
||||
package model
|
||||
|
||||
// AppBrand 品牌数据表模型
|
||||
type AppBrand struct {
|
||||
ID int64 `gorm:"primaryKey;column:id"`
|
||||
Name string `gorm:"column:name"`
|
||||
SpiderOrigin string `gorm:"column:spider_origin"`
|
||||
}
|
||||
|
||||
func (AppBrand) TableName() string {
|
||||
return "app_brands" // 根据实际表名调整
|
||||
}
|
||||
|
||||
// Article 文章/发布会数据表模型
|
||||
type Article struct {
|
||||
ID int64 `gorm:"primaryKey;column:id"`
|
||||
Title string `gorm:"column:title"`
|
||||
Images string `gorm:"column:images"`
|
||||
Platform string `gorm:"column:platform"`
|
||||
Brand int64 `gorm:"column:brand"`
|
||||
Module int `gorm:"column:module"`
|
||||
Year int `gorm:"column:year"`
|
||||
SourceURL string `gorm:"column:source_url"`
|
||||
Cover string `gorm:"column:cover"`
|
||||
}
|
||||
|
||||
func (Article) TableName() string {
|
||||
return "articles" // 根据实际表名调整
|
||||
}
|
||||
301
internal/spider/vogue.go
Normal file
301
internal/spider/vogue.go
Normal file
@ -0,0 +1,301 @@
|
||||
package spider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"my-spiders/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
VogueBaseURL = "https://www.vogue.com"
|
||||
VoguePlatform = "vogue"
|
||||
)
|
||||
|
||||
type VogueSpider struct {
|
||||
DB *gorm.DB
|
||||
Client *http.Client
|
||||
MaxCo int
|
||||
BrandID int64
|
||||
ForceUpdate bool
|
||||
OnlyPlatform bool
|
||||
Debug bool // 新增 Debug 标志
|
||||
}
|
||||
|
||||
func NewVogueSpider(db *gorm.DB, maxCo int) *VogueSpider {
|
||||
return &VogueSpider{
|
||||
DB: db,
|
||||
MaxCo: maxCo,
|
||||
Client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Execute 任务主入口（对应 execute）
|
||||
func (s *VogueSpider) Execute() {
|
||||
tasks, err := s.getTask()
|
||||
if err != nil {
|
||||
log.Printf("[Error] 获取品牌任务失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 最大查询的品牌数量, 控制并发数 ($maxBrandExecuteCount = $this->maxCo / 2)
|
||||
maxBrandExecuteCount := s.MaxCo / 2
|
||||
if maxBrandExecuteCount < 1 {
|
||||
maxBrandExecuteCount = 1
|
||||
}
|
||||
|
||||
// 使用 Channel 信号量精准控制并发品牌数，无需 while(true) sleep
|
||||
sem := make(chan struct{}, maxBrandExecuteCount)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, task := range tasks {
|
||||
sem <- struct{}{} // 占用信号量槽位
|
||||
wg.Add(1)
|
||||
|
||||
go func(t model.AppBrand) {
|
||||
defer func() {
|
||||
<-sem // 释放槽位
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
s.SpiderStart(t)
|
||||
}(task)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
log.Println("[Command] Vogue 所有抓取任务已完成。")
|
||||
}
|
||||
|
||||
// _getTask 获取任务列表（对应 _getTask）
|
||||
func (s *VogueSpider) getTask() ([]model.AppBrand, error) {
|
||||
|
||||
// 如果没有数据库连接（如本地调试场景）
|
||||
if s.DB == nil {
|
||||
log.Println("[Debug] 未连接数据库，启动 Mock 测试数据模式")
|
||||
|
||||
// 如果命令行指定了 -b 参数，使用指定的品牌 ID，否则默认给一个示例品牌
|
||||
brandID := s.BrandID
|
||||
if brandID == 0 {
|
||||
brandID = 108
|
||||
}
|
||||
|
||||
return []model.AppBrand{
|
||||
{ID: brandID, Name: "chanel", SpiderOrigin: VoguePlatform},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var brands []model.AppBrand
|
||||
query := s.DB.Model(&model.AppBrand{})
|
||||
|
||||
if s.BrandID > 0 {
|
||||
query = query.Where("id = ?", s.BrandID)
|
||||
} else {
|
||||
query = query.Where("id > ?", 1)
|
||||
if s.OnlyPlatform {
|
||||
query = query.Where("spider_origin = ?", VoguePlatform)
|
||||
}
|
||||
query = query.Order("id asc")
|
||||
}
|
||||
|
||||
err := query.Find(&brands).Error
|
||||
return brands, err
|
||||
}
|
||||
|
||||
// getTaskName 品牌名称格式化（对应 getTaskName）
|
||||
func (s *VogueSpider) getTaskName(name string) string {
|
||||
r := strings.NewReplacer(".", "-", " ", "-", "&", "")
|
||||
return strings.ToLower(r.Replace(name))
|
||||
}
|
||||
|
||||
// SpiderStart 针对单个品牌开始采集
|
||||
func (s *VogueSpider) SpiderStart(task model.AppBrand) {
|
||||
brandName := s.getTaskName(task.Name)
|
||||
url := VogueBaseURL + "/fashion-shows/designer/" + brandName
|
||||
log.Printf("[Command] brandName: %s; spiderUrl: %s", brandName, url)
|
||||
|
||||
showsList := s.getShowsList(url)
|
||||
|
||||
// 1. 核心修复：限制详情页的最大并发数，受 s.MaxCo 控制
|
||||
detailSem := make(chan struct{}, s.MaxCo)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, list := range showsList {
|
||||
detailSem <- struct{}{} // 达到 MaxCo 上限时自动阻塞等待
|
||||
wg.Add(1)
|
||||
|
||||
go func(info gjson.Result) {
|
||||
defer func() {
|
||||
<-detailSem // 释放槽位
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
s.getDetail(task.ID, info)
|
||||
|
||||
// 2. 建议：每个请求完成后稍微休眠 200ms，既省带宽又能防封 IP
|
||||
// time.Sleep(200 * time.Millisecond)
|
||||
}(list)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// getShowsList 获取发布会列表（对应 getShowsList）
|
||||
func (s *VogueSpider) getShowsList(url string) []gjson.Result {
|
||||
body, httpCode := s.request(url)
|
||||
|
||||
if httpCode == 200 && body != "" {
|
||||
re := regexp.MustCompile(`(?s)window\.__PRELOADED_STATE__\s*=\s*(.*?);</script>`)
|
||||
matches := re.FindStringSubmatch(body)
|
||||
if len(matches) > 1 {
|
||||
collections := gjson.Get(matches[1], "transformed.runwayDesignerContent.designerCollections")
|
||||
return collections.Array()
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[Info] %s 未找到数据.", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// getDetail 获取发布会详情并保存（对应 getDetail）
|
||||
func (s *VogueSpider) getDetail(brandID int64, info gjson.Result) {
|
||||
hed := info.Get("hed").String()
|
||||
|
||||
var modelArticle model.Article
|
||||
|
||||
// 如果处于非 Debug 模式，进行数据库查询判重
|
||||
if !s.Debug && s.DB != nil {
|
||||
s.DB.Where("brand = ? AND title = ?", brandID, hed).First(&modelArticle)
|
||||
if modelArticle.ID > 0 && !s.ForceUpdate {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 如果非 forceUpdate 且记录已存在，跳过更新
|
||||
if modelArticle.ID > 0 && !s.ForceUpdate {
|
||||
return
|
||||
}
|
||||
|
||||
modelArticle.Title = hed
|
||||
modelArticle.Images = "[]"
|
||||
modelArticle.Platform = VoguePlatform
|
||||
modelArticle.Brand = brandID
|
||||
modelArticle.Module = 0
|
||||
modelArticle.Year = s.parseYear(hed)
|
||||
|
||||
// 获取图片
|
||||
pageURI := info.Get("url").String()
|
||||
requestURL := VogueBaseURL + pageURI + "/slideshow/collection"
|
||||
log.Printf("正在匹配发布会详情 %s", requestURL)
|
||||
modelArticle.SourceURL = requestURL
|
||||
|
||||
body, httpCode := s.request(requestURL)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[Warning] %s 请求失败.", requestURL)
|
||||
return
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(`(?s)window\.__PRELOADED_STATE__\s*=\s*(.*?);<`)
|
||||
matches := re.FindStringSubmatch(body)
|
||||
|
||||
if len(matches) > 1 {
|
||||
imagesResult := gjson.Get(matches[1], "transformed.runwayGalleries.galleries.0.items")
|
||||
|
||||
if !imagesResult.Exists() {
|
||||
log.Printf("[Warning] %s 获取图片失败.", requestURL)
|
||||
return
|
||||
}
|
||||
|
||||
type ImageItem struct {
|
||||
Src string `json:"src"`
|
||||
}
|
||||
|
||||
var saveURL []ImageItem
|
||||
var detailURL []ImageItem
|
||||
|
||||
for _, img := range imagesResult.Array() {
|
||||
xxlURL := img.Get("image.sources.xxl.url").String()
|
||||
if xxlURL != "" {
|
||||
saveURL = append(saveURL, ImageItem{Src: xxlURL})
|
||||
log.Printf(xxlURL)
|
||||
}
|
||||
|
||||
// 详情图片提取
|
||||
for _, detail := range img.Get("details").Array() {
|
||||
detailXxlURL := detail.Get("image.sources.xxl.url").String()
|
||||
if detailXxlURL != "" {
|
||||
detailURL = append(detailURL, ImageItem{Src: detailXxlURL})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(saveURL) > 0 {
|
||||
jsonBytes, _ := json.Marshal(saveURL)
|
||||
modelArticle.Images = string(jsonBytes)
|
||||
modelArticle.Cover = saveURL[0].Src
|
||||
}
|
||||
}
|
||||
|
||||
s.saveArticle(&modelArticle)
|
||||
}
|
||||
|
||||
// saveArticle 数据持久化或 Debug 输出控制
|
||||
func (s *VogueSpider) saveArticle(article *model.Article) {
|
||||
// Debug 模式：只打印结果，不入库
|
||||
if s.Debug {
|
||||
indentJSON, _ := json.MarshalIndent(article, "", " ")
|
||||
log.Printf("\n================ [DEBUG OUTPUT] ================\n%s\n================================================\n", string(indentJSON))
|
||||
return
|
||||
}
|
||||
|
||||
// 生产模式：保存到数据库
|
||||
if s.DB != nil {
|
||||
s.DB.Save(article)
|
||||
log.Printf("[Success] 入库成功: %s", article.SourceURL)
|
||||
}
|
||||
}
|
||||
|
||||
// 提取年份 (替换 PHP 中的 AppHelper::getYear)
|
||||
func (s *VogueSpider) parseYear(title string) int {
|
||||
re := regexp.MustCompile(`\b(19|20)\d{2}\b`)
|
||||
match := re.FindString(title)
|
||||
if match != "" {
|
||||
year, _ := strconv.Atoi(match)
|
||||
return year
|
||||
}
|
||||
return time.Now().Year()
|
||||
}
|
||||
|
||||
// HTTP 请求封装
|
||||
func (s *VogueSpider) request(url string) (string, int) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := s.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", resp.StatusCode
|
||||
}
|
||||
|
||||
return string(bodyBytes), resp.StatusCode
|
||||
}
|
||||
Reference in New Issue
Block a user