feat: 媒体机直传 + 向量回填

- media: spider 下载图片后按内容 sha1 生成 key,经后端预签名直传 S4,上报只带 key
  (spider 不持有 AK/SK;单张失败自动回退为上报 URL)
- embed: 新增 embed 子命令,从后端领取缺向量的图片→下载→本地 CLIP 编码→回传
  (媒体拆分后后端拿不到图片字节,向量只能靠回填补齐)
- serve: 服务化常驻模式(领取 crawl 任务 → 抓取 → 回报)
- config: 新增 media.upload/concurrency、vision.endpoint/concurrency
This commit is contained in:
toom1996
2026-10-25 00:15:22 +08:00
parent 6020426815
commit 3fe0b86e11
11 changed files with 1305 additions and 2 deletions

View File

@ -3,6 +3,8 @@ package cmd
import (
"github.com/spf13/cobra"
"my-spiders/internal/config"
"my-spiders/internal/media"
"my-spiders/internal/spider"
)
@ -18,6 +20,10 @@ var vogueCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
// 1. 修复参数数量 & 类型强转 uint(brandID)
vogueSpider := spider.NewVogueSpider(maxCo, uint(brandID), maxShows, IngestClient)
// 媒体机直传:由 spider 下载并上传图片,后端只写库(config.media.upload 控制)。
if config.GlobalConfig.Media.Upload && IngestClient != nil {
vogueSpider.Media = media.New(IngestClient, "runway", config.GlobalConfig.Media.Concurrency)
}
// 2. 修复方法名:将 Execute() 改为 Run()
vogueSpider.Run()