package cmd import ( "context" "log" "net/http" "strings" "time" "github.com/spf13/cobra" "my-spiders/internal/config" "my-spiders/internal/embed" ) var ( embedKinds string embedLimit int embedOnce bool embedSleep int embedVision string embedCo int ) var embedCmd = &cobra.Command{ Use: "embed", Short: "向量回填:拉取缺向量的图片,本地 CLIP 编码后回传后端", Long: `媒体拆分后由 spider 直接传图,后端拿不到图片字节、算不出向量。 本命令跑在有 GPU 的机器上,把缺向量的图补上: 后端领取(缺向量的图) → 从存储下载 → 本地 CLIP 编码 → 回传向量 常驻运行(默认):spider embed 只跑一轮: spider embed --once`, Run: func(cmd *cobra.Command, args []string) { if IngestClient == nil { log.Fatal("[embed] 入库管线未启用(config.ingest.enabled=false),无法领取回填任务") } vision := embedVision if vision == "" { vision = config.GlobalConfig.Vision.Endpoint } if vision == "" { vision = "http://127.0.0.1:8900" } // 视觉服务没起来就白跑:先探活,明确报错比反复超时好排查。 if err := visionReady(vision); err != nil { log.Fatalf("[embed] 视觉服务不可用(%s): %v\n 先把 CLIP 服务起来:app-cli\\vision\\vision.ps1 start", vision, err) } kinds := []string{"runway", "street"} if strings.TrimSpace(embedKinds) != "" { kinds = nil for _, k := range strings.Split(embedKinds, ",") { if k = strings.TrimSpace(k); k != "" { kinds = append(kinds, k) } } } conc := embedCo if conc <= 0 { conc = config.GlobalConfig.Vision.Concurrency } w := embed.New(IngestClient, vision, conc, 8) log.Printf("[embed] 启动:视觉=%s 类别=%v 单批=%d 并发=%d", vision, kinds, embedLimit, conc) ctx := context.Background() for { idle := 0 for _, kind := range kinds { st, err := w.RunOnce(ctx, kind, embedLimit) if err != nil { log.Printf("[embed] %s 本轮失败: %v", kind, err) continue } if st.Fetched == 0 { idle++ } } if embedOnce { return } if idle == len(kinds) { // 没有待办:睡一会儿再看,避免空转打后端 time.Sleep(time.Duration(embedSleep) * time.Second) } } }, } // visionReady 探活视觉服务(/health),失败返回错误。 func visionReady(endpoint string) error { client := &http.Client{Timeout: 5 * time.Second} resp, err := client.Get(endpoint + "/health") if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return err } return nil } func init() { rootCmd.AddCommand(embedCmd) embedCmd.Flags().StringVar(&embedKinds, "kind", "", "只处理指定类别(runway,street),默认两类都做") embedCmd.Flags().IntVar(&embedLimit, "limit", 32, "每轮向后端领取的张数(1~200)") embedCmd.Flags().BoolVar(&embedOnce, "once", false, "只跑一轮就退出(用于验证/定时任务)") embedCmd.Flags().IntVar(&embedSleep, "sleep", 30, "空闲时休眠秒数(常驻模式)") embedCmd.Flags().StringVar(&embedVision, "vision", "", "视觉服务地址(默认取 config.vision.endpoint,再兜底 127.0.0.1:8900)") embedCmd.Flags().IntVar(&embedCo, "concurrency", 0, "下载并发(默认取 config.vision.concurrency)") }