update
This commit is contained in:
toom1996
91
cmd/root.go
Normal file
91
cmd/root.go
Normal file
@ -0,0 +1,91 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"my-spiders/internal/config"
|
||||
)
|
||||
|
||||
var (
|
||||
DB *gorm.DB
|
||||
cfgFile string
|
||||
debug bool // 命令行 debug 标志
|
||||
|
||||
rootCmd = &cobra.Command{
|
||||
Use: "spider-cli",
|
||||
Short: "自动化多平台爬虫集成系统",
|
||||
}
|
||||
)
|
||||
|
||||
func Execute() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
// 支持通过 --config 参数指定其他的配置文件路径
|
||||
rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "配置文件路径 (默认是 ./config.yaml)")
|
||||
|
||||
// 新增全局 -d / --debug 参数
|
||||
rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "调试模式 (仅输出不入库)")
|
||||
|
||||
cobra.OnInitialize(initConfigAndDB)
|
||||
}
|
||||
|
||||
func initConfigAndDB() {
|
||||
// 1. 初始化配置文件
|
||||
if err := config.InitConfig(cfgFile); err != nil {
|
||||
log.Fatalf("[致命错误] %v", err)
|
||||
}
|
||||
|
||||
// 命令行 -d 参数优先于 YAML 配置
|
||||
if debug {
|
||||
config.GlobalConfig.App.Debug = true
|
||||
}
|
||||
|
||||
isDebug := config.GlobalConfig.App.Debug
|
||||
|
||||
// 2. 初始化数据库连接
|
||||
dbCfg := config.GlobalConfig.Database
|
||||
var err error
|
||||
DB, err = gorm.Open(mysql.Open(dbCfg.DSN), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Warn),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if isDebug {
|
||||
log.Println("[Warning] 数据库连接失败，当前处于 Debug 模式，将跳过数据库操作...")
|
||||
DB = nil
|
||||
return
|
||||
}
|
||||
log.Fatalf("[致命错误] 数据库连接失败: %v", err)
|
||||
}
|
||||
|
||||
// 3. 设置连接池参数
|
||||
sqlDB, err := DB.DB()
|
||||
if err != nil || sqlDB.Ping() != nil {
|
||||
if isDebug {
|
||||
log.Println("[Warning] 数据库 Ping 失败，当前处于 Debug 模式，将跳过数据库操作...")
|
||||
DB = nil
|
||||
return
|
||||
}
|
||||
log.Fatalf("[致命错误] 数据库不可用: %v", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxOpenConns(dbCfg.MaxOpenConns)
|
||||
sqlDB.SetMaxIdleConns(dbCfg.MaxIdleConns)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(dbCfg.ConnMaxLifetime) * time.Minute)
|
||||
sqlDB.SetConnMaxIdleTime(time.Duration(dbCfg.ConnMaxIdleTime) * time.Minute)
|
||||
|
||||
log.Printf("[Info] MySQL 连接池初始化完成 | MaxOpen: %d | MaxIdle: %d", dbCfg.MaxOpenConns, dbCfg.MaxIdleConns)
|
||||
}
|
||||
44
cmd/vogue.go
Normal file
44
cmd/vogue.go
Normal file
@ -0,0 +1,44 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"my-spiders/internal/config"
|
||||
"my-spiders/internal/spider"
|
||||
)
|
||||
|
||||
var (
|
||||
brandID int64
|
||||
forceUpdate bool
|
||||
onlyPlatform bool
|
||||
)
|
||||
|
||||
var vogueCmd = &cobra.Command{
|
||||
Use: "vogue",
|
||||
Short: "自动采集 vogue.com 网站发布会数据",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
log.Println("[Command] 启动 Vogue 采集指令...")
|
||||
|
||||
// 从全局 YAML 配置中获取并发数
|
||||
maxCo := config.GlobalConfig.App.MaxCo
|
||||
|
||||
vogueSpider := spider.NewVogueSpider(DB, maxCo)
|
||||
vogueSpider.BrandID = brandID
|
||||
vogueSpider.ForceUpdate = forceUpdate
|
||||
vogueSpider.OnlyPlatform = onlyPlatform
|
||||
vogueSpider.Debug = config.GlobalConfig.App.Debug // 赋值 Debug 状态
|
||||
|
||||
vogueSpider.Execute()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(vogueCmd)
|
||||
|
||||
vogueCmd.Flags().Int64VarP(&brandID, "brandId", "b", 0, "指定的品牌id.")
|
||||
vogueCmd.Flags().BoolVarP(&forceUpdate, "forceUpdate", "f", false, "是否对已经保存的数据进行强制更新.")
|
||||
vogueCmd.Flags().BoolVarP(&onlyPlatform, "onlyPlatform", "o", false, "是否只对当前平台品牌更新.")
|
||||
|
||||
}
|
||||
12
config.yml
Normal file
12
config.yml
Normal file
@ -0,0 +1,12 @@
|
||||
# 爬虫框架通用配置
|
||||
app:
|
||||
max_co: 3 # 默认最大并发协程数
|
||||
debug: true # 默认为 true（开发调试），生产环境中设为 false
|
||||
|
||||
# MySQL 数据库及连接池配置
|
||||
database:
|
||||
dsn: "root:123456@tcp(127.0.0.1:3306)/database?charset=utf8mb4&parseTime=True&loc=Local"
|
||||
max_open_conns: 50 # 连接池最大打开连接数
|
||||
max_idle_conns: 10 # 连接池最大空闲连接数
|
||||
conn_max_lifetime: 30 # 连接可复用的最长时间(分钟)
|
||||
conn_max_idle_time: 10 # 连接空闲最大存活时间(分钟)
|
||||
30
go.mod
Normal file
30
go.mod
Normal file
@ -0,0 +1,30 @@
|
||||
module my-spiders
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.8.1 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/cobra v1.10.2 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spf13/viper v1.21.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tidwall/gjson v1.19.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/sys v0.29.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
gorm.io/driver/mysql v1.6.0 // indirect
|
||||
gorm.io/gorm v1.31.2 // indirect
|
||||
)
|
||||
55
go.sum
Normal file
55
go.sum
Normal file
@ -0,0 +1,55 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU=
|
||||
github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
|
||||
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
|
||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
|
||||
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
|
||||
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
|
||||
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
52
internal/config/config.go
Normal file
52
internal/config/config.go
Normal file
@ -0,0 +1,52 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// GlobalConfig 全局配置实例
|
||||
var GlobalConfig Config
|
||||
|
||||
type Config struct {
|
||||
App AppConfig `mapstructure:"app"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
}
|
||||
|
||||
type AppConfig struct {
|
||||
MaxCo int `mapstructure:"max_co"`
|
||||
Debug bool `mapstructure:"debug"` // 新增 Debug 字段
|
||||
}
|
||||
|
||||
type DatabaseConfig struct {
|
||||
DSN string `mapstructure:"dsn"`
|
||||
MaxOpenConns int `mapstructure:"max_open_conns"`
|
||||
MaxIdleConns int `mapstructure:"max_idle_conns"`
|
||||
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
|
||||
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
|
||||
}
|
||||
|
||||
// InitConfig 初始化并读取 YAML 配置文件
|
||||
func InitConfig(cfgFile string) error {
|
||||
if cfgFile != "" {
|
||||
// 如果命令行指定了配置文件路径
|
||||
viper.SetConfigFile(cfgFile)
|
||||
} else {
|
||||
// 默认在当前目录下查找 config.yaml
|
||||
viper.AddConfigPath(".")
|
||||
viper.SetConfigType("yaml")
|
||||
viper.SetConfigName("config")
|
||||
}
|
||||
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
return fmt.Errorf("读取配置文件失败: %w", err)
|
||||
}
|
||||
|
||||
// 解析到结构体
|
||||
if err := viper.Unmarshal(&GlobalConfig); err != nil {
|
||||
return fmt.Errorf("解析配置文件结构失败: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
29
internal/model/models.go
Normal file
29
internal/model/models.go
Normal file
@ -0,0 +1,29 @@
|
||||
package model
|
||||
|
||||
// AppBrand 品牌数据表模型
|
||||
type AppBrand struct {
|
||||
ID int64 `gorm:"primaryKey;column:id"`
|
||||
Name string `gorm:"column:name"`
|
||||
SpiderOrigin string `gorm:"column:spider_origin"`
|
||||
}
|
||||
|
||||
func (AppBrand) TableName() string {
|
||||
return "app_brands" // 根据实际表名调整
|
||||
}
|
||||
|
||||
// Article 文章/发布会数据表模型
|
||||
type Article struct {
|
||||
ID int64 `gorm:"primaryKey;column:id"`
|
||||
Title string `gorm:"column:title"`
|
||||
Images string `gorm:"column:images"`
|
||||
Platform string `gorm:"column:platform"`
|
||||
Brand int64 `gorm:"column:brand"`
|
||||
Module int `gorm:"column:module"`
|
||||
Year int `gorm:"column:year"`
|
||||
SourceURL string `gorm:"column:source_url"`
|
||||
Cover string `gorm:"column:cover"`
|
||||
}
|
||||
|
||||
func (Article) TableName() string {
|
||||
return "articles" // 根据实际表名调整
|
||||
}
|
||||
301
internal/spider/vogue.go
Normal file
301
internal/spider/vogue.go
Normal file
@ -0,0 +1,301 @@
|
||||
package spider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"my-spiders/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
VogueBaseURL = "https://www.vogue.com"
|
||||
VoguePlatform = "vogue"
|
||||
)
|
||||
|
||||
type VogueSpider struct {
|
||||
DB *gorm.DB
|
||||
Client *http.Client
|
||||
MaxCo int
|
||||
BrandID int64
|
||||
ForceUpdate bool
|
||||
OnlyPlatform bool
|
||||
Debug bool // 新增 Debug 标志
|
||||
}
|
||||
|
||||
func NewVogueSpider(db *gorm.DB, maxCo int) *VogueSpider {
|
||||
return &VogueSpider{
|
||||
DB: db,
|
||||
MaxCo: maxCo,
|
||||
Client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Execute 任务主入口（对应 execute）
|
||||
func (s *VogueSpider) Execute() {
|
||||
tasks, err := s.getTask()
|
||||
if err != nil {
|
||||
log.Printf("[Error] 获取品牌任务失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 最大查询的品牌数量, 控制并发数 ($maxBrandExecuteCount = $this->maxCo / 2)
|
||||
maxBrandExecuteCount := s.MaxCo / 2
|
||||
if maxBrandExecuteCount < 1 {
|
||||
maxBrandExecuteCount = 1
|
||||
}
|
||||
|
||||
// 使用 Channel 信号量精准控制并发品牌数，无需 while(true) sleep
|
||||
sem := make(chan struct{}, maxBrandExecuteCount)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, task := range tasks {
|
||||
sem <- struct{}{} // 占用信号量槽位
|
||||
wg.Add(1)
|
||||
|
||||
go func(t model.AppBrand) {
|
||||
defer func() {
|
||||
<-sem // 释放槽位
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
s.SpiderStart(t)
|
||||
}(task)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
log.Println("[Command] Vogue 所有抓取任务已完成。")
|
||||
}
|
||||
|
||||
// _getTask 获取任务列表（对应 _getTask）
|
||||
func (s *VogueSpider) getTask() ([]model.AppBrand, error) {
|
||||
|
||||
// 如果没有数据库连接（如本地调试场景）
|
||||
if s.DB == nil {
|
||||
log.Println("[Debug] 未连接数据库，启动 Mock 测试数据模式")
|
||||
|
||||
// 如果命令行指定了 -b 参数，使用指定的品牌 ID，否则默认给一个示例品牌
|
||||
brandID := s.BrandID
|
||||
if brandID == 0 {
|
||||
brandID = 108
|
||||
}
|
||||
|
||||
return []model.AppBrand{
|
||||
{ID: brandID, Name: "chanel", SpiderOrigin: VoguePlatform},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var brands []model.AppBrand
|
||||
query := s.DB.Model(&model.AppBrand{})
|
||||
|
||||
if s.BrandID > 0 {
|
||||
query = query.Where("id = ?", s.BrandID)
|
||||
} else {
|
||||
query = query.Where("id > ?", 1)
|
||||
if s.OnlyPlatform {
|
||||
query = query.Where("spider_origin = ?", VoguePlatform)
|
||||
}
|
||||
query = query.Order("id asc")
|
||||
}
|
||||
|
||||
err := query.Find(&brands).Error
|
||||
return brands, err
|
||||
}
|
||||
|
||||
// getTaskName 品牌名称格式化（对应 getTaskName）
|
||||
func (s *VogueSpider) getTaskName(name string) string {
|
||||
r := strings.NewReplacer(".", "-", " ", "-", "&", "")
|
||||
return strings.ToLower(r.Replace(name))
|
||||
}
|
||||
|
||||
// SpiderStart 针对单个品牌开始采集
|
||||
func (s *VogueSpider) SpiderStart(task model.AppBrand) {
|
||||
brandName := s.getTaskName(task.Name)
|
||||
url := VogueBaseURL + "/fashion-shows/designer/" + brandName
|
||||
log.Printf("[Command] brandName: %s; spiderUrl: %s", brandName, url)
|
||||
|
||||
showsList := s.getShowsList(url)
|
||||
|
||||
// 1. 核心修复：限制详情页的最大并发数，受 s.MaxCo 控制
|
||||
detailSem := make(chan struct{}, s.MaxCo)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, list := range showsList {
|
||||
detailSem <- struct{}{} // 达到 MaxCo 上限时自动阻塞等待
|
||||
wg.Add(1)
|
||||
|
||||
go func(info gjson.Result) {
|
||||
defer func() {
|
||||
<-detailSem // 释放槽位
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
s.getDetail(task.ID, info)
|
||||
|
||||
// 2. 建议：每个请求完成后稍微休眠 200ms，既省带宽又能防封 IP
|
||||
// time.Sleep(200 * time.Millisecond)
|
||||
}(list)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// getShowsList 获取发布会列表（对应 getShowsList）
|
||||
func (s *VogueSpider) getShowsList(url string) []gjson.Result {
|
||||
body, httpCode := s.request(url)
|
||||
|
||||
if httpCode == 200 && body != "" {
|
||||
re := regexp.MustCompile(`(?s)window\.__PRELOADED_STATE__\s*=\s*(.*?);</script>`)
|
||||
matches := re.FindStringSubmatch(body)
|
||||
if len(matches) > 1 {
|
||||
collections := gjson.Get(matches[1], "transformed.runwayDesignerContent.designerCollections")
|
||||
return collections.Array()
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[Info] %s 未找到数据.", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// getDetail 获取发布会详情并保存（对应 getDetail）
|
||||
func (s *VogueSpider) getDetail(brandID int64, info gjson.Result) {
|
||||
hed := info.Get("hed").String()
|
||||
|
||||
var modelArticle model.Article
|
||||
|
||||
// 如果处于非 Debug 模式，进行数据库查询判重
|
||||
if !s.Debug && s.DB != nil {
|
||||
s.DB.Where("brand = ? AND title = ?", brandID, hed).First(&modelArticle)
|
||||
if modelArticle.ID > 0 && !s.ForceUpdate {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 如果非 forceUpdate 且记录已存在，跳过更新
|
||||
if modelArticle.ID > 0 && !s.ForceUpdate {
|
||||
return
|
||||
}
|
||||
|
||||
modelArticle.Title = hed
|
||||
modelArticle.Images = "[]"
|
||||
modelArticle.Platform = VoguePlatform
|
||||
modelArticle.Brand = brandID
|
||||
modelArticle.Module = 0
|
||||
modelArticle.Year = s.parseYear(hed)
|
||||
|
||||
// 获取图片
|
||||
pageURI := info.Get("url").String()
|
||||
requestURL := VogueBaseURL + pageURI + "/slideshow/collection"
|
||||
log.Printf("正在匹配发布会详情 %s", requestURL)
|
||||
modelArticle.SourceURL = requestURL
|
||||
|
||||
body, httpCode := s.request(requestURL)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[Warning] %s 请求失败.", requestURL)
|
||||
return
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(`(?s)window\.__PRELOADED_STATE__\s*=\s*(.*?);<`)
|
||||
matches := re.FindStringSubmatch(body)
|
||||
|
||||
if len(matches) > 1 {
|
||||
imagesResult := gjson.Get(matches[1], "transformed.runwayGalleries.galleries.0.items")
|
||||
|
||||
if !imagesResult.Exists() {
|
||||
log.Printf("[Warning] %s 获取图片失败.", requestURL)
|
||||
return
|
||||
}
|
||||
|
||||
type ImageItem struct {
|
||||
Src string `json:"src"`
|
||||
}
|
||||
|
||||
var saveURL []ImageItem
|
||||
var detailURL []ImageItem
|
||||
|
||||
for _, img := range imagesResult.Array() {
|
||||
xxlURL := img.Get("image.sources.xxl.url").String()
|
||||
if xxlURL != "" {
|
||||
saveURL = append(saveURL, ImageItem{Src: xxlURL})
|
||||
log.Printf(xxlURL)
|
||||
}
|
||||
|
||||
// 详情图片提取
|
||||
for _, detail := range img.Get("details").Array() {
|
||||
detailXxlURL := detail.Get("image.sources.xxl.url").String()
|
||||
if detailXxlURL != "" {
|
||||
detailURL = append(detailURL, ImageItem{Src: detailXxlURL})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(saveURL) > 0 {
|
||||
jsonBytes, _ := json.Marshal(saveURL)
|
||||
modelArticle.Images = string(jsonBytes)
|
||||
modelArticle.Cover = saveURL[0].Src
|
||||
}
|
||||
}
|
||||
|
||||
s.saveArticle(&modelArticle)
|
||||
}
|
||||
|
||||
// saveArticle 数据持久化或 Debug 输出控制
|
||||
func (s *VogueSpider) saveArticle(article *model.Article) {
|
||||
// Debug 模式：只打印结果，不入库
|
||||
if s.Debug {
|
||||
indentJSON, _ := json.MarshalIndent(article, "", " ")
|
||||
log.Printf("\n================ [DEBUG OUTPUT] ================\n%s\n================================================\n", string(indentJSON))
|
||||
return
|
||||
}
|
||||
|
||||
// 生产模式：保存到数据库
|
||||
if s.DB != nil {
|
||||
s.DB.Save(article)
|
||||
log.Printf("[Success] 入库成功: %s", article.SourceURL)
|
||||
}
|
||||
}
|
||||
|
||||
// 提取年份 (替换 PHP 中的 AppHelper::getYear)
|
||||
func (s *VogueSpider) parseYear(title string) int {
|
||||
re := regexp.MustCompile(`\b(19|20)\d{2}\b`)
|
||||
match := re.FindString(title)
|
||||
if match != "" {
|
||||
year, _ := strconv.Atoi(match)
|
||||
return year
|
||||
}
|
||||
return time.Now().Year()
|
||||
}
|
||||
|
||||
// HTTP 请求封装
|
||||
func (s *VogueSpider) request(url string) (string, int) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := s.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", resp.StatusCode
|
||||
}
|
||||
|
||||
return string(bodyBytes), resp.StatusCode
|
||||
}
|
||||
Reference in New Issue
Block a user