update
This commit is contained in:
toom1996
29
cmd/theimpression.go
Normal file
29
cmd/theimpression.go
Normal file
@ -0,0 +1,29 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"my-spiders/internal/spider"
|
||||
)
|
||||
|
||||
var theImpressionMaxCo int
|
||||
|
||||
var theImpressionCmd = &cobra.Command{
|
||||
Use: "theimpression",
|
||||
Short: "启动 The Impression 街拍采集器 (street-style)",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if theImpressionMaxCo <= 0 {
|
||||
theImpressionMaxCo = 5
|
||||
}
|
||||
// 复用全局 -d/--debug 标志
|
||||
isDebug := debug
|
||||
s := spider.NewTheImpressionSpider(DB, theImpressionMaxCo)
|
||||
s.Debug = isDebug
|
||||
s.Run()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(theImpressionCmd)
|
||||
theImpressionCmd.Flags().IntVarP(&theImpressionMaxCo, "max-co", "m", 5, "最大并发数限制")
|
||||
}
|
||||
16
go.mod
16
go.mod
@ -2,8 +2,18 @@ module my-spiders
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/tidwall/gjson v1.19.0
|
||||
gorm.io/driver/mysql v1.6.0
|
||||
gorm.io/gorm v1.31.2
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.8.1 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
@ -15,16 +25,12 @@ require (
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/cobra v1.10.2 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spf13/viper v1.21.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tidwall/gjson v1.19.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/net v0.24.0 // indirect
|
||||
golang.org/x/sys v0.29.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
gorm.io/driver/mysql v1.6.0 // indirect
|
||||
gorm.io/gorm v1.31.2 // indirect
|
||||
)
|
||||
|
||||
67
go.sum
67
go.sum
@ -1,20 +1,40 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
@ -26,12 +46,13 @@ github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU=
|
||||
@ -40,16 +61,56 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
|
||||
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
|
||||
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
|
||||
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
|
||||
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
|
||||
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
|
||||
@ -50,4 +50,30 @@ type BrandRunwayImage struct {
|
||||
// TableName 指定图片明细表名
|
||||
func (BrandRunwayImage) TableName() string {
|
||||
return "brand_runway_images"
|
||||
}
|
||||
|
||||
// Article The Impression 街拍文章表（对应 PHP 的 articles 模型）
|
||||
// 字段与 PHP 的 ArticleModuleEnum::STREET 文章结构保持一致
|
||||
type Article struct {
|
||||
ID uint `gorm:"primaryKey;column:id;autoIncrement" json:"id"`
|
||||
Title string `gorm:"column:title" json:"title"`
|
||||
Platform string `gorm:"column:platform" json:"platform"`
|
||||
Module string `gorm:"column:module" json:"module"`
|
||||
Year uint16 `gorm:"column:year" json:"year"`
|
||||
Cover string `gorm:"column:cover" json:"cover"`
|
||||
Images string `gorm:"column:images" json:"images"`
|
||||
SourceURL string `gorm:"column:source_url" json:"source_url"`
|
||||
Brand uint `gorm:"column:brand;default:0" json:"brand"`
|
||||
CreatedAt int64 `gorm:"column:created_at" json:"created_at"`
|
||||
UpdatedAt int64 `gorm:"column:updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
// TableName 指定文章主表名
|
||||
func (Article) TableName() string {
|
||||
return "articles"
|
||||
}
|
||||
|
||||
// ImageItem 图片条目，序列化进 Article.Images(JSON 数组)
|
||||
type ImageItem struct {
|
||||
Src string `json:"src"`
|
||||
}
|
||||
285
internal/spider/theimpression.go
Normal file
285
internal/spider/theimpression.go
Normal file
@ -0,0 +1,285 @@
|
||||
package spider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"my-spiders/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
TheImpressionBaseURL = "https://theimpression.com"
|
||||
TheImpressionPlatform = "theimpression-street"
|
||||
// TheImpressionModule 对应 PHP 的 ArticleModuleEnum::STREET->value
|
||||
// 若你服务端枚举值不是 "street"，请在此修改为实际值
|
||||
TheImpressionModule = "street"
|
||||
)
|
||||
|
||||
// TheImpressionSpider The Impression 街拍采集器（对应 PHP 的 TheImpressionStreetCommand）
|
||||
type TheImpressionSpider struct {
|
||||
Client *http.Client
|
||||
DB *gorm.DB
|
||||
MaxCo int // 最大并发数
|
||||
Debug bool // Debug 模式：仅输出不入库
|
||||
ForceUpdate bool // 强制更新已存在的记录
|
||||
}
|
||||
|
||||
func NewTheImpressionSpider(db *gorm.DB, maxCo int) *TheImpressionSpider {
|
||||
if maxCo <= 0 {
|
||||
maxCo = 5
|
||||
}
|
||||
return &TheImpressionSpider{
|
||||
DB: db,
|
||||
MaxCo: maxCo,
|
||||
Client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// streetTask 单个待抓取的文章任务
|
||||
type streetTask struct {
|
||||
URL string
|
||||
Title string
|
||||
}
|
||||
|
||||
// Run 爬虫总入口
|
||||
func (s *TheImpressionSpider) Run() {
|
||||
var tasks []streetTask
|
||||
|
||||
// 1. 抓取 street-style 首页 banner（.parallax .mask-overlay）
|
||||
bannerURL := TheImpressionBaseURL + "/street-style"
|
||||
body, httpCode := s.request(bannerURL)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[错误] %s 请求失败, code: %d", bannerURL, httpCode)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("[错误] 解析 %s 失败: %v", bannerURL, err)
|
||||
return
|
||||
}
|
||||
|
||||
doc.Find(".parallax .mask-overlay").Each(func(_ int, node *goquery.Selection) {
|
||||
href, _ := node.Attr("href")
|
||||
text := strings.TrimSpace(node.AttrOr("aria-label", ""))
|
||||
log.Printf("标题: %s", text)
|
||||
log.Printf("链接: %s", href)
|
||||
if href != "" && text != "" {
|
||||
tasks = append(tasks, streetTask{URL: href, Title: text})
|
||||
}
|
||||
})
|
||||
|
||||
// 2. 抓取 WP JSON 接口的文章列表（对应 PHP 的 wp-json/codetipi-zeen 分页）
|
||||
// 注意：原 PHP 写为 for ($i = 1; $i < 2; $i++)，即仅抓取第 1 页。
|
||||
// 如需翻页，将此处 2 改为目标页数 +1 即可（原文注释为“前五十页”）。
|
||||
for i := 1; i < 2; i++ {
|
||||
listURL := fmt.Sprintf(
|
||||
"%s/wp-json/codetipi-zeen/v1/block?paged=%d&type=1&data%%5Bargs%%5D%%5Bcat%%5D=1",
|
||||
TheImpressionBaseURL, i,
|
||||
)
|
||||
body, httpCode := s.request(listURL)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[错误] %s 请求失败, code: %d", listURL, httpCode)
|
||||
return
|
||||
}
|
||||
|
||||
// 接口返回的是 JSON 数组，第 2 个元素（index 1）是包含文章的 HTML 片段
|
||||
var arr []interface{}
|
||||
if err := json.Unmarshal([]byte(body), &arr); err != nil {
|
||||
log.Printf("[错误] %s JSON 解析失败: %v", listURL, err)
|
||||
continue
|
||||
}
|
||||
if len(arr) < 2 {
|
||||
log.Printf("[Info] %s 未返回足够的数组元素，跳过.", listURL)
|
||||
continue
|
||||
}
|
||||
htmlStr, ok := arr[1].(string)
|
||||
if !ok || htmlStr == "" {
|
||||
log.Printf("[Info] %s 第 2 个元素不是 HTML 字符串，跳过.", listURL)
|
||||
continue
|
||||
}
|
||||
|
||||
listDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
log.Printf("[错误] 解析 %s 文章列表 HTML 失败: %v", listURL, err)
|
||||
continue
|
||||
}
|
||||
|
||||
listDoc.Find("article").Each(func(_ int, node *goquery.Selection) {
|
||||
href, _ := node.Find(".mask-img").Attr("href")
|
||||
title := strings.TrimSpace(node.Find(".title-wrap").Text())
|
||||
if href == "" || title == "" {
|
||||
log.Printf("找不到标题或链接.")
|
||||
return
|
||||
}
|
||||
tasks = append(tasks, streetTask{URL: href, Title: title})
|
||||
})
|
||||
}
|
||||
|
||||
log.Printf("[Info] 共收集到 %d 个待抓取任务，开始执行...", len(tasks))
|
||||
|
||||
// 3. 并发抓取详情页
|
||||
sem := make(chan struct{}, s.MaxCo)
|
||||
var wg sync.WaitGroup
|
||||
for _, t := range tasks {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(tk streetTask) {
|
||||
defer func() {
|
||||
<-sem
|
||||
wg.Done()
|
||||
}()
|
||||
s.getDetail(tk)
|
||||
time.Sleep(200 * time.Millisecond) // 频控，防封 IP
|
||||
}(t)
|
||||
}
|
||||
wg.Wait()
|
||||
log.Println("[Info] The Impression 所有抓取任务已完成！")
|
||||
}
|
||||
|
||||
// getDetail 抓取并解析单个文章详情（对应 PHP 的 getDetail）
|
||||
func (s *TheImpressionSpider) getDetail(tk streetTask) {
|
||||
url := tk.URL
|
||||
title := tk.Title
|
||||
if url == "" || title == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// 判重：按 title + platform 查询是否已存在
|
||||
var exist model.Article
|
||||
if !s.Debug && s.DB != nil {
|
||||
s.DB.Where("title = ? AND platform = ?", title, TheImpressionPlatform).First(&exist)
|
||||
if exist.ID > 0 && !s.ForceUpdate {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
body, httpCode := s.request(url)
|
||||
if httpCode != 200 || body == "" {
|
||||
log.Printf("[错误] %s 请求失败, code: %d", url, httpCode)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("[错误] 解析 %s 失败: %v", url, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 采集 figure a img 的图片（去重）
|
||||
imagesMap := make(map[string]model.ImageItem)
|
||||
doc.Find("figure a img").Each(func(_ int, node *goquery.Selection) {
|
||||
src, _ := node.Attr("src")
|
||||
if src != "" {
|
||||
if _, ok := imagesMap[src]; !ok {
|
||||
log.Printf("采集图片: %s", src)
|
||||
imagesMap[src] = model.ImageItem{Src: src}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if len(imagesMap) == 0 {
|
||||
log.Printf("[Warning] %s 未采集到图片，跳过保存.", url)
|
||||
return
|
||||
}
|
||||
|
||||
cover := ""
|
||||
var imageList []model.ImageItem
|
||||
for _, v := range imagesMap {
|
||||
if cover == "" {
|
||||
cover = v.Src
|
||||
}
|
||||
imageList = append(imageList, v)
|
||||
}
|
||||
|
||||
imagesJSON, _ := json.Marshal(imageList)
|
||||
now := time.Now().Unix()
|
||||
|
||||
article := model.Article{
|
||||
Title: title,
|
||||
Platform: TheImpressionPlatform,
|
||||
Module: TheImpressionModule,
|
||||
Year: uint16(s.parseYear(title)),
|
||||
Cover: cover,
|
||||
Images: string(imagesJSON),
|
||||
SourceURL: url,
|
||||
Brand: 0,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
// Debug 模式或无数据库连接时，仅控制台输出
|
||||
if s.Debug || s.DB == nil {
|
||||
out, _ := json.MarshalIndent(article, "", " ")
|
||||
log.Printf("\n================ [DEBUG OUTPUT] ================\n%s\n提取图片总数: %d 张\n================================================",
|
||||
string(out), len(imageList))
|
||||
return
|
||||
}
|
||||
|
||||
// 入库（存在则更新，不存在则新增）
|
||||
err = s.DB.Transaction(func(tx *gorm.DB) error {
|
||||
if exist.ID > 0 {
|
||||
article.ID = exist.ID
|
||||
if err := tx.Save(&article).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := tx.Create(&article).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("[错误] 数据入库失败 [%s]: %v", title, err)
|
||||
} else {
|
||||
log.Printf("[成功] 保存文章: %s (共 %d 张图片)", title, len(imageList))
|
||||
}
|
||||
}
|
||||
|
||||
// request HTTP 请求封装（带浏览器 UA）
|
||||
func (s *TheImpressionSpider) request(url string) (string, int) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/json,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
|
||||
resp, err := s.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", resp.StatusCode
|
||||
}
|
||||
return string(bodyBytes), resp.StatusCode
|
||||
}
|
||||
|
||||
// parseYear 从标题中提取 4 位年份数字（对应 PHP 的 AppHelper::getYear）
|
||||
func (s *TheImpressionSpider) parseYear(title string) int {
|
||||
re := regexp.MustCompile(`\b(19|20)\d{2}\b`)
|
||||
match := re.FindString(title)
|
||||
if match != "" {
|
||||
y, _ := strconv.Atoi(match)
|
||||
return y
|
||||
}
|
||||
return time.Now().Year()
|
||||
}
|
||||
@ -220,7 +220,7 @@ func (s *VogueSpider) getDetail(brandID int64, info gjson.Result) {
|
||||
xxlURL := img.Get("image.sources.xxl.url").String()
|
||||
if xxlURL != "" {
|
||||
saveURL = append(saveURL, ImageItem{Src: xxlURL})
|
||||
log.Printf(xxlURL)
|
||||
log.Printf("%s", xxlURL)
|
||||
}
|
||||
|
||||
// 详情图片提取
|
||||
|
||||
Reference in New Issue
Block a user