fix: 抓取前预检已入库的秀,避免重复下载(重爬 62 分钟 0 新增)
问题:图片 key 是内容 sha1,不下载就无法判断是否已存在; 后端又没存 source_url,无法做 URL 级判重。 于是每次重爬都把几百张已下过的图再下一遍 —— 线上实测 Celine 跑了 62 分钟、约 600MB,入库 0 新增。 修复: - 后端新增 GET /admin/internal/shows/covered?brand_uid= 按 (年份, 季节, 类型) 聚合已入库图片张数 - spider 抓取前预检,整场已入库(>=10 张)就跳过,一张图都不下载 - 加 --force 可强制重抓(默认跳过) - 媒体直传并发改为全进程共享信号量:此前秀级 2 × 每场 6 = 12 路 同时打 Vogue,被限流后单张从 9.8s 掉到 ~90s - 批次下载失败时打印第一条错误(此前只统计个数,查不出原因)
This commit is contained in:
@ -195,6 +195,63 @@ type EmbeddingFailure struct {
|
||||
}
|
||||
|
||||
// FetchPendingEmbeddings 拉一批缺向量的图片。remaining 为后端统计的剩余量(-1 = 统计失败)。
|
||||
// ShowCoverageKey 生成「秀覆盖」预检键:年份|季节|类型,与后端的实体键一一对应。
|
||||
func ShowCoverageKey(year int, season, collectionType string) string {
|
||||
return strconv.Itoa(year) + "|" + season + "|" + collectionType
|
||||
}
|
||||
|
||||
// CoveredShows 取某品牌已入库的秀,返回「预检键 → 已有图片张数」。
|
||||
//
|
||||
// 用途:抓取前整场跳过已入库的秀。图片 key 是内容 sha1,不下载就无法知道
|
||||
// 一张图是否已经存在,所以「跳过」只能在场这一级做。
|
||||
func (c *Client) CoveredShows(ctx context.Context, brandUID string) (map[string]int, error) {
|
||||
if c == nil || c.Endpoint == "" {
|
||||
return nil, fmt.Errorf("ingest client 未配置")
|
||||
}
|
||||
url := c.internalBase() + "/shows/covered?brand_uid=" + neturl.QueryEscape(brandUID)
|
||||
ts := NowTS()
|
||||
nonce, err := newNonce()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("X-Signature", Sign(c.Secret, ts, nonce, ""))
|
||||
req.Header.Set("X-Timestamp", ts)
|
||||
req.Header.Set("X-Nonce", nonce)
|
||||
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("shows/covered http %d: %s", resp.StatusCode, string(raw))
|
||||
}
|
||||
var out struct {
|
||||
Items []ShowCoverage `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("shows/covered 解析失败: %w", err)
|
||||
}
|
||||
covered := make(map[string]int, len(out.Items))
|
||||
for _, it := range out.Items {
|
||||
covered[ShowCoverageKey(int(it.Year), it.Season, it.CollectionType)] = it.Images
|
||||
}
|
||||
return covered, nil
|
||||
}
|
||||
|
||||
// ShowCoverage 一场秀在库里的覆盖情况。
|
||||
type ShowCoverage struct {
|
||||
Year uint16 `json:"year"`
|
||||
Season string `json:"season"`
|
||||
CollectionType string `json:"collection_type"`
|
||||
Images int `json:"images"`
|
||||
}
|
||||
|
||||
func (c *Client) FetchPendingEmbeddings(ctx context.Context, kind string, limit int) ([]PendingEmbedding, int, error) {
|
||||
if c == nil || c.Endpoint == "" {
|
||||
return nil, 0, fmt.Errorf("ingest client 未配置")
|
||||
|
||||
Reference in New Issue
Block a user