feat: 媒体机直传 + 向量回填
- media: spider 下载图片后按内容 sha1 生成 key，经后端预签名直传 S4，上报只带 key （spider 不持有 AK/SK；单张失败自动回退为上报 URL） - embed: 新增 embed 子命令，从后端领取缺向量的图片→下载→本地 CLIP 编码→回传 （媒体拆分后后端拿不到图片字节，向量只能靠回填补齐） - serve: 服务化常驻模式（领取 crawl 任务 → 抓取 → 回报） - config: 新增 media.upload/concurrency、vision.endpoint/concurrency
This commit is contained in:
toom1996
113
cmd/embed.go
Normal file
113
cmd/embed.go
Normal file
@ -0,0 +1,113 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"my-spiders/internal/config"
|
||||
"my-spiders/internal/embed"
|
||||
)
|
||||
|
||||
var (
|
||||
embedKinds string
|
||||
embedLimit int
|
||||
embedOnce bool
|
||||
embedSleep int
|
||||
embedVision string
|
||||
embedCo int
|
||||
)
|
||||
|
||||
var embedCmd = &cobra.Command{
|
||||
Use: "embed",
|
||||
Short: "向量回填：拉取缺向量的图片，本地 CLIP 编码后回传后端",
|
||||
Long: `媒体拆分后由 spider 直接传图，后端拿不到图片字节、算不出向量。
|
||||
本命令跑在有 GPU 的机器上，把缺向量的图补上：
|
||||
|
||||
后端领取（缺向量的图） → 从存储下载 → 本地 CLIP 编码 → 回传向量
|
||||
|
||||
常驻运行（默认）：spider embed
|
||||
只跑一轮： spider embed --once`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if IngestClient == nil {
|
||||
log.Fatal("[embed] 入库管线未启用（config.ingest.enabled=false），无法领取回填任务")
|
||||
}
|
||||
vision := embedVision
|
||||
if vision == "" {
|
||||
vision = config.GlobalConfig.Vision.Endpoint
|
||||
}
|
||||
if vision == "" {
|
||||
vision = "http://127.0.0.1:8900"
|
||||
}
|
||||
// 视觉服务没起来就白跑：先探活，明确报错比反复超时好排查。
|
||||
if err := visionReady(vision); err != nil {
|
||||
log.Fatalf("[embed] 视觉服务不可用（%s）: %v\n 先把 CLIP 服务起来：app-cli\\vision\\vision.ps1 start", vision, err)
|
||||
}
|
||||
|
||||
kinds := []string{"runway", "street"}
|
||||
if strings.TrimSpace(embedKinds) != "" {
|
||||
kinds = nil
|
||||
for _, k := range strings.Split(embedKinds, ",") {
|
||||
if k = strings.TrimSpace(k); k != "" {
|
||||
kinds = append(kinds, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
conc := embedCo
|
||||
if conc <= 0 {
|
||||
conc = config.GlobalConfig.Vision.Concurrency
|
||||
}
|
||||
w := embed.New(IngestClient, vision, conc, 8)
|
||||
log.Printf("[embed] 启动：视觉=%s 类别=%v 单批=%d 并发=%d", vision, kinds, embedLimit, conc)
|
||||
|
||||
ctx := context.Background()
|
||||
for {
|
||||
idle := 0
|
||||
for _, kind := range kinds {
|
||||
st, err := w.RunOnce(ctx, kind, embedLimit)
|
||||
if err != nil {
|
||||
log.Printf("[embed] %s 本轮失败: %v", kind, err)
|
||||
continue
|
||||
}
|
||||
if st.Fetched == 0 {
|
||||
idle++
|
||||
}
|
||||
}
|
||||
if embedOnce {
|
||||
return
|
||||
}
|
||||
if idle == len(kinds) {
|
||||
// 没有待办：睡一会儿再看，避免空转打后端
|
||||
time.Sleep(time.Duration(embedSleep) * time.Second)
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// visionReady 探活视觉服务（/health），失败返回错误。
|
||||
func visionReady(endpoint string) error {
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := client.Get(endpoint + "/health")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(embedCmd)
|
||||
embedCmd.Flags().StringVar(&embedKinds, "kind", "", "只处理指定类别（runway,street），默认两类都做")
|
||||
embedCmd.Flags().IntVar(&embedLimit, "limit", 32, "每轮向后端领取的张数（1~200）")
|
||||
embedCmd.Flags().BoolVar(&embedOnce, "once", false, "只跑一轮就退出（用于验证/定时任务）")
|
||||
embedCmd.Flags().IntVar(&embedSleep, "sleep", 30, "空闲时休眠秒数（常驻模式）")
|
||||
embedCmd.Flags().StringVar(&embedVision, "vision", "", "视觉服务地址（默认取 config.vision.endpoint，再兜底 127.0.0.1:8900）")
|
||||
embedCmd.Flags().IntVar(&embedCo, "concurrency", 0, "下载并发（默认取 config.vision.concurrency）")
|
||||
}
|
||||
153
cmd/serve.go
Normal file
153
cmd/serve.go
Normal file
@ -0,0 +1,153 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"my-spiders/internal/config"
|
||||
"my-spiders/internal/media"
|
||||
"my-spiders/internal/spider"
|
||||
)
|
||||
|
||||
var (
|
||||
serveWorkers int
|
||||
serveInterval int
|
||||
serveMaxCo int
|
||||
)
|
||||
|
||||
// serveCmd 把爬虫变成「常驻服务」：不再由人决定爬谁，而是从后台领取任务。
|
||||
//
|
||||
// 与 vogue 子命令的区别：
|
||||
// - vogue：一次性的，参数指定品牌（--brand），跑完退出；
|
||||
// - serve：常驻，循环「领任务 → 执行 → 回报」，没有任务就空转等待。
|
||||
//
|
||||
// 这样后台（/admin/crawl）就能控制爬哪些品牌，爬虫部署在哪台机器都不影响。
|
||||
var serveCmd = &cobra.Command{
|
||||
Use: "serve",
|
||||
Short: "以服务方式运行：从后台领取品牌抓取任务并执行",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if IngestClient == nil {
|
||||
log.Fatal("[致命] 未启用入库管线：config.yml 里 ingest.enabled 必须为 true（serve 模式依赖后台接口领取任务）")
|
||||
}
|
||||
|
||||
host, _ := os.Hostname()
|
||||
if strings.TrimSpace(host) == "" {
|
||||
host = "spider"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Ctrl+C / SIGTERM 优雅退出：不再领新任务，等在跑的品牌结束。
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
log.Printf("[serve] 启动 %d 个 worker，轮询间隔 %ds，单品牌并发 %d", serveWorkers, serveInterval, serveMaxCo)
|
||||
|
||||
for i := 0; i < serveWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
workerID := fmt.Sprintf("%s-%d-%d", host, os.Getpid(), idx)
|
||||
runServeWorker(ctx, workerID)
|
||||
}(i)
|
||||
}
|
||||
|
||||
<-sig
|
||||
log.Println("[serve] 收到退出信号，等待进行中的任务结束…")
|
||||
cancel()
|
||||
wg.Wait()
|
||||
log.Println("[serve] 已退出")
|
||||
},
|
||||
}
|
||||
|
||||
func runServeWorker(ctx context.Context, workerID string) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
task, err := IngestClient.FetchTask(ctx, workerID)
|
||||
if err != nil {
|
||||
log.Printf("[serve] %s 领取任务失败: %v", workerID, err)
|
||||
sleepOrDone(ctx, serveInterval)
|
||||
continue
|
||||
}
|
||||
if task == nil {
|
||||
// 队列空：退避轮询，避免打满后台接口。
|
||||
sleepOrDone(ctx, serveInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[serve] %s 领取任务 task=%d 品牌=%s(uid=%s) 场次上限=%d",
|
||||
workerID, task.ID, task.BrandName, task.BrandUID, task.MaxShows)
|
||||
|
||||
start := time.Now()
|
||||
shows, errMsg := executeBrand(ctx, task.BrandUID, task.BrandName, int(task.MaxShows))
|
||||
elapsed := time.Since(start).Round(time.Second)
|
||||
|
||||
status := "done"
|
||||
if errMsg != "" {
|
||||
status = "failed"
|
||||
}
|
||||
if err := IngestClient.ReportTask(ctx, task.ID, status, shows, errMsg); err != nil {
|
||||
log.Printf("[serve] %s 回报任务 %d 失败: %v", workerID, task.ID, err)
|
||||
// 回报失败不要紧：租约（CrawlStuckTimeoutSec）到期后后台会把任务重新放回队列。
|
||||
sleepOrDone(ctx, serveInterval)
|
||||
continue
|
||||
}
|
||||
log.Printf("[serve] %s 任务 %d 完成：status=%s 提交 %d 场，耗时 %s", workerID, task.ID, status, shows, elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// executeBrand 执行单个品牌，异常（panic）也转成错误返回，避免一个坏品牌把整个 serve 进程带崩。
|
||||
func executeBrand(ctx context.Context, brandUID, brandName string, maxShows int) (shows int, errMsg string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
shows = 0
|
||||
errMsg = fmt.Sprintf("panic: %v", r)
|
||||
log.Printf("[serve] 执行品牌 %s 时 panic: %v", brandName, r)
|
||||
}
|
||||
}()
|
||||
|
||||
s := spider.NewVogueSpider(serveMaxCo, 0, maxShows, IngestClient)
|
||||
s.Debug = debug
|
||||
// 媒体机直传：spider 自己下载并上传图片，后端只写库。
|
||||
if config.GlobalConfig.Media.Upload && IngestClient != nil {
|
||||
s.Media = media.New(IngestClient, "runway", config.GlobalConfig.Media.Concurrency)
|
||||
}
|
||||
shows = s.RunBrand(brandUID, brandName, maxShows)
|
||||
return shows, ""
|
||||
}
|
||||
|
||||
// sleepOrDone 可被 ctx 取消打断的 sleep。
|
||||
func sleepOrDone(ctx context.Context, seconds int) {
|
||||
if seconds < 1 {
|
||||
seconds = 5
|
||||
}
|
||||
t := time.NewTimer(time.Duration(seconds) * time.Second)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(serveCmd)
|
||||
serveCmd.Flags().IntVarP(&serveWorkers, "workers", "w", 1, "并发执行的品牌数（每路一个 goroutine）")
|
||||
serveCmd.Flags().IntVarP(&serveInterval, "interval", "i", 5, "队列为空时的轮询间隔（秒）")
|
||||
serveCmd.Flags().IntVarP(&serveMaxCo, "max-co", "m", 5, "单个品牌内部的抓取并发数")
|
||||
}
|
||||
@ -3,6 +3,8 @@ package cmd
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"my-spiders/internal/config"
|
||||
"my-spiders/internal/media"
|
||||
"my-spiders/internal/spider"
|
||||
)
|
||||
|
||||
@ -18,6 +20,10 @@ var vogueCmd = &cobra.Command{
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// 1. 修复参数数量 & 类型强转 uint(brandID)
|
||||
vogueSpider := spider.NewVogueSpider(maxCo, uint(brandID), maxShows, IngestClient)
|
||||
// 媒体机直传：由 spider 下载并上传图片，后端只写库（config.media.upload 控制）。
|
||||
if config.GlobalConfig.Media.Upload && IngestClient != nil {
|
||||
vogueSpider.Media = media.New(IngestClient, "runway", config.GlobalConfig.Media.Concurrency)
|
||||
}
|
||||
|
||||
// 2. 修复方法名：将 Execute() 改为 Run()
|
||||
vogueSpider.Run()
|
||||
|
||||
Reference in New Issue
Block a user