From cab1023ba15a5699cb6356f8c22a31cb2f358f19 Mon Sep 17 00:00:00 2001 From: toom1996 Date: Mon, 10 Aug 2026 20:47:34 +0800 Subject: [PATCH] update --- cmd/root.go | 91 ++++++++++++ cmd/vogue.go | 44 ++++++ config.yml | 12 ++ go.mod | 30 ++++ go.sum | 55 +++++++ internal/config/config.go | 52 +++++++ internal/model/models.go | 29 ++++ internal/spider/vogue.go | 301 ++++++++++++++++++++++++++++++++++++++ main.go | 7 + 9 files changed, 621 insertions(+) create mode 100644 cmd/root.go create mode 100644 cmd/vogue.go create mode 100644 config.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/model/models.go create mode 100644 internal/spider/vogue.go create mode 100644 main.go diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..c85f9f9 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,91 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "time" + + "github.com/spf13/cobra" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "my-spiders/internal/config" +) + +var ( + DB *gorm.DB + cfgFile string + debug bool // 命令行 debug 标志 + + rootCmd = &cobra.Command{ + Use: "spider-cli", + Short: "自动化多平台爬虫集成系统", + } +) + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + // 支持通过 --config 参数指定其他的配置文件路径 + rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "配置文件路径 (默认是 ./config.yaml)") + + // 新增全局 -d / --debug 参数 + rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "调试模式 (仅输出不入库)") + + cobra.OnInitialize(initConfigAndDB) +} + +func initConfigAndDB() { + // 1. 初始化配置文件 + if err := config.InitConfig(cfgFile); err != nil { + log.Fatalf("[致命错误] %v", err) + } + + // 命令行 -d 参数优先于 YAML 配置 + if debug { + config.GlobalConfig.App.Debug = true + } + + isDebug := config.GlobalConfig.App.Debug + + // 2. 初始化数据库连接 + dbCfg := config.GlobalConfig.Database + var err error + DB, err = gorm.Open(mysql.Open(dbCfg.DSN), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Warn), + }) + + if err != nil { + if isDebug { + log.Println("[Warning] 数据库连接失败，当前处于 Debug 模式，将跳过数据库操作...") + DB = nil + return + } + log.Fatalf("[致命错误] 数据库连接失败: %v", err) + } + + // 3. 设置连接池参数 + sqlDB, err := DB.DB() + if err != nil || sqlDB.Ping() != nil { + if isDebug { + log.Println("[Warning] 数据库 Ping 失败，当前处于 Debug 模式，将跳过数据库操作...") + DB = nil + return + } + log.Fatalf("[致命错误] 数据库不可用: %v", err) + } + + sqlDB.SetMaxOpenConns(dbCfg.MaxOpenConns) + sqlDB.SetMaxIdleConns(dbCfg.MaxIdleConns) + sqlDB.SetConnMaxLifetime(time.Duration(dbCfg.ConnMaxLifetime) * time.Minute) + sqlDB.SetConnMaxIdleTime(time.Duration(dbCfg.ConnMaxIdleTime) * time.Minute) + + log.Printf("[Info] MySQL 连接池初始化完成 | MaxOpen: %d | MaxIdle: %d", dbCfg.MaxOpenConns, dbCfg.MaxIdleConns) +} diff --git a/cmd/vogue.go b/cmd/vogue.go new file mode 100644 index 0000000..5257df7 --- /dev/null +++ b/cmd/vogue.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "log" + + "github.com/spf13/cobra" + + "my-spiders/internal/config" + "my-spiders/internal/spider" +) + +var ( + brandID int64 + forceUpdate bool + onlyPlatform bool +) + +var vogueCmd = &cobra.Command{ + Use: "vogue", + Short: "自动采集 vogue.com 网站发布会数据", + Run: func(cmd *cobra.Command, args []string) { + log.Println("[Command] 启动 Vogue 采集指令...") + + // 从全局 YAML 配置中获取并发数 + maxCo := config.GlobalConfig.App.MaxCo + + vogueSpider := spider.NewVogueSpider(DB, maxCo) + vogueSpider.BrandID = brandID + vogueSpider.ForceUpdate = forceUpdate + vogueSpider.OnlyPlatform = onlyPlatform + vogueSpider.Debug = config.GlobalConfig.App.Debug // 赋值 Debug 状态 + + vogueSpider.Execute() + }, +} + +func init() { + rootCmd.AddCommand(vogueCmd) + + vogueCmd.Flags().Int64VarP(&brandID, "brandId", "b", 0, "指定的品牌id.") + vogueCmd.Flags().BoolVarP(&forceUpdate, "forceUpdate", "f", false, "是否对已经保存的数据进行强制更新.") + vogueCmd.Flags().BoolVarP(&onlyPlatform, "onlyPlatform", "o", false, "是否只对当前平台品牌更新.") + +} diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..973c60a --- /dev/null +++ b/config.yml @@ -0,0 +1,12 @@ +# 爬虫框架通用配置 +app: + max_co: 3 # 默认最大并发协程数 + debug: true # 默认为 true（开发调试），生产环境中设为 false + +# MySQL 数据库及连接池配置 +database: + dsn: "root:123456@tcp(127.0.0.1:3306)/database?charset=utf8mb4&parseTime=True&loc=Local" + max_open_conns: 50 # 连接池最大打开连接数 + max_idle_conns: 10 # 连接池最大空闲连接数 + conn_max_lifetime: 30 # 连接可复用的最长时间(分钟) + conn_max_idle_time: 10 # 连接空闲最大存活时间(分钟) \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3ce05a0 --- /dev/null +++ b/go.mod @@ -0,0 +1,30 @@ +module my-spiders + +go 1.24.0 + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.21.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tidwall/gjson v1.19.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.28.0 // indirect + gorm.io/driver/mysql v1.6.0 // indirect + gorm.io/gorm v1.31.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6dc6f90 --- /dev/null +++ b/go.sum @@ -0,0 +1,55 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= +gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..459561d --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,52 @@ +package config + +import ( + "fmt" + + "github.com/spf13/viper" +) + +// GlobalConfig 全局配置实例 +var GlobalConfig Config + +type Config struct { + App AppConfig `mapstructure:"app"` + Database DatabaseConfig `mapstructure:"database"` +} + +type AppConfig struct { + MaxCo int `mapstructure:"max_co"` + Debug bool `mapstructure:"debug"` // 新增 Debug 字段 +} + +type DatabaseConfig struct { + DSN string `mapstructure:"dsn"` + MaxOpenConns int `mapstructure:"max_open_conns"` + MaxIdleConns int `mapstructure:"max_idle_conns"` + ConnMaxLifetime int `mapstructure:"conn_max_lifetime"` + ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"` +} + +// InitConfig 初始化并读取 YAML 配置文件 +func InitConfig(cfgFile string) error { + if cfgFile != "" { + // 如果命令行指定了配置文件路径 + viper.SetConfigFile(cfgFile) + } else { + // 默认在当前目录下查找 config.yaml + viper.AddConfigPath(".") + viper.SetConfigType("yaml") + viper.SetConfigName("config") + } + + if err := viper.ReadInConfig(); err != nil { + return fmt.Errorf("读取配置文件失败: %w", err) + } + + // 解析到结构体 + if err := viper.Unmarshal(&GlobalConfig); err != nil { + return fmt.Errorf("解析配置文件结构失败: %w", err) + } + + return nil +} diff --git a/internal/model/models.go b/internal/model/models.go new file mode 100644 index 0000000..7a6f4ae --- /dev/null +++ b/internal/model/models.go @@ -0,0 +1,29 @@ +package model + +// AppBrand 品牌数据表模型 +type AppBrand struct { + ID int64 `gorm:"primaryKey;column:id"` + Name string `gorm:"column:name"` + SpiderOrigin string `gorm:"column:spider_origin"` +} + +func (AppBrand) TableName() string { + return "app_brands" // 根据实际表名调整 +} + +// Article 文章/发布会数据表模型 +type Article struct { + ID int64 `gorm:"primaryKey;column:id"` + Title string `gorm:"column:title"` + Images string `gorm:"column:images"` + Platform string `gorm:"column:platform"` + Brand int64 `gorm:"column:brand"` + Module int `gorm:"column:module"` + Year int `gorm:"column:year"` + SourceURL string `gorm:"column:source_url"` + Cover string `gorm:"column:cover"` +} + +func (Article) TableName() string { + return "articles" // 根据实际表名调整 +} diff --git a/internal/spider/vogue.go b/internal/spider/vogue.go new file mode 100644 index 0000000..b730234 --- /dev/null +++ b/internal/spider/vogue.go @@ -0,0 +1,301 @@ +package spider + +import ( + "encoding/json" + "io" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/tidwall/gjson" + "gorm.io/gorm" + + "my-spiders/internal/model" +) + +const ( + VogueBaseURL = "https://www.vogue.com" + VoguePlatform = "vogue" +) + +type VogueSpider struct { + DB *gorm.DB + Client *http.Client + MaxCo int + BrandID int64 + ForceUpdate bool + OnlyPlatform bool + Debug bool // 新增 Debug 标志 +} + +func NewVogueSpider(db *gorm.DB, maxCo int) *VogueSpider { + return &VogueSpider{ + DB: db, + MaxCo: maxCo, + Client: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// Execute 任务主入口（对应 execute） +func (s *VogueSpider) Execute() { + tasks, err := s.getTask() + if err != nil { + log.Printf("[Error] 获取品牌任务失败: %v", err) + return + } + + // 最大查询的品牌数量, 控制并发数 ($maxBrandExecuteCount = $this->maxCo / 2) + maxBrandExecuteCount := s.MaxCo / 2 + if maxBrandExecuteCount < 1 { + maxBrandExecuteCount = 1 + } + + // 使用 Channel 信号量精准控制并发品牌数，无需 while(true) sleep + sem := make(chan struct{}, maxBrandExecuteCount) + var wg sync.WaitGroup + + for _, task := range tasks { + sem <- struct{}{} // 占用信号量槽位 + wg.Add(1) + + go func(t model.AppBrand) { + defer func() { + <-sem // 释放槽位 + wg.Done() + }() + + s.SpiderStart(t) + }(task) + } + + wg.Wait() + log.Println("[Command] Vogue 所有抓取任务已完成。") +} + +// _getTask 获取任务列表（对应 _getTask） +func (s *VogueSpider) getTask() ([]model.AppBrand, error) { + + // 如果没有数据库连接（如本地调试场景） + if s.DB == nil { + log.Println("[Debug] 未连接数据库，启动 Mock 测试数据模式") + + // 如果命令行指定了 -b 参数，使用指定的品牌 ID，否则默认给一个示例品牌 + brandID := s.BrandID + if brandID == 0 { + brandID = 108 + } + + return []model.AppBrand{ + {ID: brandID, Name: "chanel", SpiderOrigin: VoguePlatform}, + }, nil + } + + var brands []model.AppBrand + query := s.DB.Model(&model.AppBrand{}) + + if s.BrandID > 0 { + query = query.Where("id = ?", s.BrandID) + } else { + query = query.Where("id > ?", 1) + if s.OnlyPlatform { + query = query.Where("spider_origin = ?", VoguePlatform) + } + query = query.Order("id asc") + } + + err := query.Find(&brands).Error + return brands, err +} + +// getTaskName 品牌名称格式化（对应 getTaskName） +func (s *VogueSpider) getTaskName(name string) string { + r := strings.NewReplacer(".", "-", " ", "-", "&", "") + return strings.ToLower(r.Replace(name)) +} + +// SpiderStart 针对单个品牌开始采集 +func (s *VogueSpider) SpiderStart(task model.AppBrand) { + brandName := s.getTaskName(task.Name) + url := VogueBaseURL + "/fashion-shows/designer/" + brandName + log.Printf("[Command] brandName: %s; spiderUrl: %s", brandName, url) + + showsList := s.getShowsList(url) + + // 1. 核心修复：限制详情页的最大并发数，受 s.MaxCo 控制 + detailSem := make(chan struct{}, s.MaxCo) + var wg sync.WaitGroup + + for _, list := range showsList { + detailSem <- struct{}{} // 达到 MaxCo 上限时自动阻塞等待 + wg.Add(1) + + go func(info gjson.Result) { + defer func() { + <-detailSem // 释放槽位 + wg.Done() + }() + + s.getDetail(task.ID, info) + + // 2. 建议：每个请求完成后稍微休眠 200ms，既省带宽又能防封 IP + // time.Sleep(200 * time.Millisecond) + }(list) + } + + wg.Wait() +} + +// getShowsList 获取发布会列表（对应 getShowsList） +func (s *VogueSpider) getShowsList(url string) []gjson.Result { + body, httpCode := s.request(url) + + if httpCode == 200 && body != "" { + re := regexp.MustCompile(`(?s)window\.__PRELOADED_STATE__\s*=\s*(.*?);`) + matches := re.FindStringSubmatch(body) + if len(matches) > 1 { + collections := gjson.Get(matches[1], "transformed.runwayDesignerContent.designerCollections") + return collections.Array() + } + } + + log.Printf("[Info] %s 未找到数据.", url) + return nil +} + +// getDetail 获取发布会详情并保存（对应 getDetail） +func (s *VogueSpider) getDetail(brandID int64, info gjson.Result) { + hed := info.Get("hed").String() + + var modelArticle model.Article + + // 如果处于非 Debug 模式，进行数据库查询判重 + if !s.Debug && s.DB != nil { + s.DB.Where("brand = ? AND title = ?", brandID, hed).First(&modelArticle) + if modelArticle.ID > 0 && !s.ForceUpdate { + return + } + } + + // 如果非 forceUpdate 且记录已存在，跳过更新 + if modelArticle.ID > 0 && !s.ForceUpdate { + return + } + + modelArticle.Title = hed + modelArticle.Images = "[]" + modelArticle.Platform = VoguePlatform + modelArticle.Brand = brandID + modelArticle.Module = 0 + modelArticle.Year = s.parseYear(hed) + + // 获取图片 + pageURI := info.Get("url").String() + requestURL := VogueBaseURL + pageURI + "/slideshow/collection" + log.Printf("正在匹配发布会详情 %s", requestURL) + modelArticle.SourceURL = requestURL + + body, httpCode := s.request(requestURL) + if httpCode != 200 || body == "" { + log.Printf("[Warning] %s 请求失败.", requestURL) + return + } + + re := regexp.MustCompile(`(?s)window\.__PRELOADED_STATE__\s*=\s*(.*?);<`) + matches := re.FindStringSubmatch(body) + + if len(matches) > 1 { + imagesResult := gjson.Get(matches[1], "transformed.runwayGalleries.galleries.0.items") + + if !imagesResult.Exists() { + log.Printf("[Warning] %s 获取图片失败.", requestURL) + return + } + + type ImageItem struct { + Src string `json:"src"` + } + + var saveURL []ImageItem + var detailURL []ImageItem + + for _, img := range imagesResult.Array() { + xxlURL := img.Get("image.sources.xxl.url").String() + if xxlURL != "" { + saveURL = append(saveURL, ImageItem{Src: xxlURL}) + log.Printf(xxlURL) + } + + // 详情图片提取 + for _, detail := range img.Get("details").Array() { + detailXxlURL := detail.Get("image.sources.xxl.url").String() + if detailXxlURL != "" { + detailURL = append(detailURL, ImageItem{Src: detailXxlURL}) + } + } + } + + if len(saveURL) > 0 { + jsonBytes, _ := json.Marshal(saveURL) + modelArticle.Images = string(jsonBytes) + modelArticle.Cover = saveURL[0].Src + } + } + + s.saveArticle(&modelArticle) +} + +// saveArticle 数据持久化或 Debug 输出控制 +func (s *VogueSpider) saveArticle(article *model.Article) { + // Debug 模式：只打印结果，不入库 + if s.Debug { + indentJSON, _ := json.MarshalIndent(article, "", " ") + log.Printf("\n================ [DEBUG OUTPUT] ================\n%s\n================================================\n", string(indentJSON)) + return + } + + // 生产模式：保存到数据库 + if s.DB != nil { + s.DB.Save(article) + log.Printf("[Success] 入库成功: %s", article.SourceURL) + } +} + +// 提取年份 (替换 PHP 中的 AppHelper::getYear) +func (s *VogueSpider) parseYear(title string) int { + re := regexp.MustCompile(`\b(19|20)\d{2}\b`) + match := re.FindString(title) + if match != "" { + year, _ := strconv.Atoi(match) + return year + } + return time.Now().Year() +} + +// HTTP 请求封装 +func (s *VogueSpider) request(url string) (string, int) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", 0 + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + + resp, err := s.Client.Do(req) + if err != nil { + return "", 0 + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", resp.StatusCode + } + + return string(bodyBytes), resp.StatusCode +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..3222ac8 --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "my-spiders/cmd" + +func main() { + cmd.Execute() +}