diff --git a/cmd/theimpression.go b/cmd/theimpression.go new file mode 100644 index 0000000..8421652 --- /dev/null +++ b/cmd/theimpression.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + "my-spiders/internal/spider" +) + +var theImpressionMaxCo int + +var theImpressionCmd = &cobra.Command{ + Use: "theimpression", + Short: "启动 The Impression 街拍采集器 (street-style)", + Run: func(cmd *cobra.Command, args []string) { + if theImpressionMaxCo <= 0 { + theImpressionMaxCo = 5 + } + // 复用全局 -d/--debug 标志 + isDebug := debug + s := spider.NewTheImpressionSpider(DB, theImpressionMaxCo) + s.Debug = isDebug + s.Run() + }, +} + +func init() { + rootCmd.AddCommand(theImpressionCmd) + theImpressionCmd.Flags().IntVarP(&theImpressionMaxCo, "max-co", "m", 5, "最大并发数限制") +} diff --git a/go.mod b/go.mod index 3ce05a0..ffe7a72 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,18 @@ module my-spiders go 1.24.0 +require ( + github.com/PuerkitoBio/goquery v1.9.2 + github.com/spf13/cobra v1.10.2 + github.com/spf13/viper v1.21.0 + github.com/tidwall/gjson v1.19.0 + gorm.io/driver/mysql v1.6.0 + gorm.io/gorm v1.31.2 +) + require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect @@ -15,16 +25,12 @@ require ( github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/spf13/viper v1.21.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tidwall/gjson v1.19.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.24.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.28.0 // indirect - gorm.io/driver/mysql v1.6.0 // indirect - gorm.io/gorm v1.31.2 // indirect ) diff --git a/go.sum b/go.sum index 6dc6f90..d149529 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,40 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= @@ -26,12 +46,13 @@ github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= @@ -40,16 +61,56 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= diff --git a/internal/model/models.go b/internal/model/models.go index 15765d8..20816ee 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -50,4 +50,30 @@ type BrandRunwayImage struct { // TableName 指定图片明细表名 func (BrandRunwayImage) TableName() string { return "brand_runway_images" +} + +// Article The Impression 街拍文章表（对应 PHP 的 articles 模型） +// 字段与 PHP 的 ArticleModuleEnum::STREET 文章结构保持一致 +type Article struct { + ID uint `gorm:"primaryKey;column:id;autoIncrement" json:"id"` + Title string `gorm:"column:title" json:"title"` + Platform string `gorm:"column:platform" json:"platform"` + Module string `gorm:"column:module" json:"module"` + Year uint16 `gorm:"column:year" json:"year"` + Cover string `gorm:"column:cover" json:"cover"` + Images string `gorm:"column:images" json:"images"` + SourceURL string `gorm:"column:source_url" json:"source_url"` + Brand uint `gorm:"column:brand;default:0" json:"brand"` + CreatedAt int64 `gorm:"column:created_at" json:"created_at"` + UpdatedAt int64 `gorm:"column:updated_at" json:"updated_at"` +} + +// TableName 指定文章主表名 +func (Article) TableName() string { + return "articles" +} + +// ImageItem 图片条目，序列化进 Article.Images(JSON 数组) +type ImageItem struct { + Src string `json:"src"` } \ No newline at end of file diff --git a/internal/spider/theimpression.go b/internal/spider/theimpression.go new file mode 100644 index 0000000..c326799 --- /dev/null +++ b/internal/spider/theimpression.go @@ -0,0 +1,285 @@ +package spider + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/PuerkitoBio/goquery" + "gorm.io/gorm" + + "my-spiders/internal/model" +) + +const ( + TheImpressionBaseURL = "https://theimpression.com" + TheImpressionPlatform = "theimpression-street" + // TheImpressionModule 对应 PHP 的 ArticleModuleEnum::STREET->value + // 若你服务端枚举值不是 "street"，请在此修改为实际值 + TheImpressionModule = "street" +) + +// TheImpressionSpider The Impression 街拍采集器（对应 PHP 的 TheImpressionStreetCommand） +type TheImpressionSpider struct { + Client *http.Client + DB *gorm.DB + MaxCo int // 最大并发数 + Debug bool // Debug 模式：仅输出不入库 + ForceUpdate bool // 强制更新已存在的记录 +} + +func NewTheImpressionSpider(db *gorm.DB, maxCo int) *TheImpressionSpider { + if maxCo <= 0 { + maxCo = 5 + } + return &TheImpressionSpider{ + DB: db, + MaxCo: maxCo, + Client: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// streetTask 单个待抓取的文章任务 +type streetTask struct { + URL string + Title string +} + +// Run 爬虫总入口 +func (s *TheImpressionSpider) Run() { + var tasks []streetTask + + // 1. 抓取 street-style 首页 banner（.parallax .mask-overlay） + bannerURL := TheImpressionBaseURL + "/street-style" + body, httpCode := s.request(bannerURL) + if httpCode != 200 || body == "" { + log.Printf("[错误] %s 请求失败, code: %d", bannerURL, httpCode) + return + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + log.Printf("[错误] 解析 %s 失败: %v", bannerURL, err) + return + } + + doc.Find(".parallax .mask-overlay").Each(func(_ int, node *goquery.Selection) { + href, _ := node.Attr("href") + text := strings.TrimSpace(node.AttrOr("aria-label", "")) + log.Printf("标题: %s", text) + log.Printf("链接: %s", href) + if href != "" && text != "" { + tasks = append(tasks, streetTask{URL: href, Title: text}) + } + }) + + // 2. 抓取 WP JSON 接口的文章列表（对应 PHP 的 wp-json/codetipi-zeen 分页） + // 注意：原 PHP 写为 for ($i = 1; $i < 2; $i++)，即仅抓取第 1 页。 + // 如需翻页，将此处 2 改为目标页数 +1 即可（原文注释为“前五十页”）。 + for i := 1; i < 2; i++ { + listURL := fmt.Sprintf( + "%s/wp-json/codetipi-zeen/v1/block?paged=%d&type=1&data%%5Bargs%%5D%%5Bcat%%5D=1", + TheImpressionBaseURL, i, + ) + body, httpCode := s.request(listURL) + if httpCode != 200 || body == "" { + log.Printf("[错误] %s 请求失败, code: %d", listURL, httpCode) + return + } + + // 接口返回的是 JSON 数组，第 2 个元素（index 1）是包含文章的 HTML 片段 + var arr []interface{} + if err := json.Unmarshal([]byte(body), &arr); err != nil { + log.Printf("[错误] %s JSON 解析失败: %v", listURL, err) + continue + } + if len(arr) < 2 { + log.Printf("[Info] %s 未返回足够的数组元素，跳过.", listURL) + continue + } + htmlStr, ok := arr[1].(string) + if !ok || htmlStr == "" { + log.Printf("[Info] %s 第 2 个元素不是 HTML 字符串，跳过.", listURL) + continue + } + + listDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlStr)) + if err != nil { + log.Printf("[错误] 解析 %s 文章列表 HTML 失败: %v", listURL, err) + continue + } + + listDoc.Find("article").Each(func(_ int, node *goquery.Selection) { + href, _ := node.Find(".mask-img").Attr("href") + title := strings.TrimSpace(node.Find(".title-wrap").Text()) + if href == "" || title == "" { + log.Printf("找不到标题或链接.") + return + } + tasks = append(tasks, streetTask{URL: href, Title: title}) + }) + } + + log.Printf("[Info] 共收集到 %d 个待抓取任务，开始执行...", len(tasks)) + + // 3. 并发抓取详情页 + sem := make(chan struct{}, s.MaxCo) + var wg sync.WaitGroup + for _, t := range tasks { + sem <- struct{}{} + wg.Add(1) + go func(tk streetTask) { + defer func() { + <-sem + wg.Done() + }() + s.getDetail(tk) + time.Sleep(200 * time.Millisecond) // 频控，防封 IP + }(t) + } + wg.Wait() + log.Println("[Info] The Impression 所有抓取任务已完成！") +} + +// getDetail 抓取并解析单个文章详情（对应 PHP 的 getDetail） +func (s *TheImpressionSpider) getDetail(tk streetTask) { + url := tk.URL + title := tk.Title + if url == "" || title == "" { + return + } + + // 判重：按 title + platform 查询是否已存在 + var exist model.Article + if !s.Debug && s.DB != nil { + s.DB.Where("title = ? AND platform = ?", title, TheImpressionPlatform).First(&exist) + if exist.ID > 0 && !s.ForceUpdate { + return + } + } + + body, httpCode := s.request(url) + if httpCode != 200 || body == "" { + log.Printf("[错误] %s 请求失败, code: %d", url, httpCode) + return + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + log.Printf("[错误] 解析 %s 失败: %v", url, err) + return + } + + // 采集 figure a img 的图片（去重） + imagesMap := make(map[string]model.ImageItem) + doc.Find("figure a img").Each(func(_ int, node *goquery.Selection) { + src, _ := node.Attr("src") + if src != "" { + if _, ok := imagesMap[src]; !ok { + log.Printf("采集图片: %s", src) + imagesMap[src] = model.ImageItem{Src: src} + } + } + }) + + if len(imagesMap) == 0 { + log.Printf("[Warning] %s 未采集到图片，跳过保存.", url) + return + } + + cover := "" + var imageList []model.ImageItem + for _, v := range imagesMap { + if cover == "" { + cover = v.Src + } + imageList = append(imageList, v) + } + + imagesJSON, _ := json.Marshal(imageList) + now := time.Now().Unix() + + article := model.Article{ + Title: title, + Platform: TheImpressionPlatform, + Module: TheImpressionModule, + Year: uint16(s.parseYear(title)), + Cover: cover, + Images: string(imagesJSON), + SourceURL: url, + Brand: 0, + CreatedAt: now, + UpdatedAt: now, + } + + // Debug 模式或无数据库连接时，仅控制台输出 + if s.Debug || s.DB == nil { + out, _ := json.MarshalIndent(article, "", " ") + log.Printf("\n================ [DEBUG OUTPUT] ================\n%s\n提取图片总数: %d 张\n================================================", + string(out), len(imageList)) + return + } + + // 入库（存在则更新，不存在则新增） + err = s.DB.Transaction(func(tx *gorm.DB) error { + if exist.ID > 0 { + article.ID = exist.ID + if err := tx.Save(&article).Error; err != nil { + return err + } + } else { + if err := tx.Create(&article).Error; err != nil { + return err + } + } + return nil + }) + + if err != nil { + log.Printf("[错误] 数据入库失败 [%s]: %v", title, err) + } else { + log.Printf("[成功] 保存文章: %s (共 %d 张图片)", title, len(imageList)) + } +} + +// request HTTP 请求封装（带浏览器 UA） +func (s *TheImpressionSpider) request(url string) (string, int) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", 0 + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/json,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + resp, err := s.Client.Do(req) + if err != nil { + return "", 0 + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", resp.StatusCode + } + return string(bodyBytes), resp.StatusCode +} + +// parseYear 从标题中提取 4 位年份数字（对应 PHP 的 AppHelper::getYear） +func (s *TheImpressionSpider) parseYear(title string) int { + re := regexp.MustCompile(`\b(19|20)\d{2}\b`) + match := re.FindString(title) + if match != "" { + y, _ := strconv.Atoi(match) + return y + } + return time.Now().Year() +} diff --git a/internal/spider/vogue.go b/internal/spider/vogue.go index c8f4bf1..76bf8c2 100644 --- a/internal/spider/vogue.go +++ b/internal/spider/vogue.go @@ -220,7 +220,7 @@ func (s *VogueSpider) getDetail(brandID int64, info gjson.Result) { xxlURL := img.Get("image.sources.xxl.url").String() if xxlURL != "" { saveURL = append(saveURL, ImageItem{Src: xxlURL}) - log.Printf(xxlURL) + log.Printf("%s", xxlURL) } // 详情图片提取