feat: 抓取任务队列 + 媒体机直传 + 向量回填

crawl(品牌抓取任务队列)
- /admin/internal/crawl/tasks 领取 / :id/result 回报,spider 可常驻拉任务
- 后台品牌列表支持「加入抓取队列」,队列页可看状态与重试
- 租约超时的 running 自动回收为 queued,避免 worker 崩溃后任务卡死

media(媒体机直传)
- ingest 载荷新增 main_key/detail_keys/main_size/detail_sizes:
  有 key 就直接落库,跳过下载与上传;与 URL 形态可逐图混用
- 细节图按 Details/DetailKeys 两者较长者遍历(只给 key 时不再丢图)
- 媒体机上传的 key 不进失败回滚清单(内容寻址、可能与他人共享)
- /admin/internal/upload-url 签发 S4 PUT 预签名(spider 不持有 AK/SK)

embeddings(向量回填)
- GET /admin/internal/embeddings/pending 派发缺向量的图(含可直接下载的 url)
- POST /admin/internal/embeddings 回写向量并重算近邻;失败项计入
  image_embedding_failures,连续失败到上限后退出队列(坏图不占位)
- 近邻重算改为异步串行:同步会让 100 张的批次 POST 超时(实测 >60s)
- 新增 key 分支单测:混用 key/url、细节图只给 key、字节数、回滚清单、向量占位
This commit is contained in:
toom1996
2026-10-25 00:37:13 +08:00
parent e18c5a0815
commit 151e09ab30
22 changed files with 1535 additions and 28 deletions

View File

@ -58,7 +58,24 @@ type DupStat struct {
//
// 近邻结果走 image_similar_pairs 预计算表:入库时(或 dbtool similarbackfill 回填时)
// 算一次存表,后台页面只读结果。原因见 ReplaceSimilarPairs 注释。
// PendingEmbedding 是一张「缺向量」的图片,供回填 worker 拉取。
// 带 Key 而不是完整 URL:worker 自己按存储基址拼(后端可能换了域名,不该把地址写死在队列里)。
type PendingEmbedding struct {
Kind string // runway | street
ImageID uint32
Key string // 存储 key(S4)或本地相对路径(/uploads/...)
}
type EmbeddingRepository interface {
// ListPendingEmbeddings 取一批「还活着、没有向量、失败次数未达上限」的图片,按 id 升序。
//
// 媒体拆分后这条查询是向量回填的唯一入口:媒体机直传只写 key,后端不碰图片字节,
// 向量必须靠外部 worker 回传,而 worker 需要知道「还有哪些图缺向量」。
ListPendingEmbeddings(ctx context.Context, kind string, limit, maxAttempts int) ([]PendingEmbedding, error)
// CountPendingEmbeddings 统计剩余待回填数量(worker 据此决定是否继续拉)。
CountPendingEmbeddings(ctx context.Context, kind string, maxAttempts int) (int, error)
// RecordEmbeddingFailure 记一次失败(attempts+1)。达到 maxAttempts 后该图不再进队列。
RecordEmbeddingFailure(ctx context.Context, kind string, imageID uint32, reason string) error
// UpsertEmbedding 写入或更新一张图的向量((kind, image_id) 唯一)。
UpsertEmbedding(ctx context.Context, kind string, imageID uint32, vec []float32) error
// BulkUpsertEmbedding 批量写入;vecs[i] 对应 ids[i],vecs[i] 为 nil 则跳过该行。
@ -175,6 +192,85 @@ func (r *embeddingRepository) UpsertEmbedding(ctx context.Context, kind string,
return r.BulkUpsertEmbedding(ctx, kind, []uint32{imageID}, [][]float32{vec})
}
// ListPendingEmbeddings 取缺失向量的图片。
//
// 三个 LEFT JOIN 各有明确目的:
// - e 判「已有没有向量」;
// - f 判「失败次数是否已到上限」——没有它,一张永远编不出来的坏图会让每轮回填都空转;
// - i.is_deleted = 0 跳过软删图,避免为已删内容白算向量。
//
// 表名/列名来自 ownerOf 常量,kind 走参数绑定,无注入面。
func (r *embeddingRepository) ListPendingEmbeddings(ctx context.Context, kind string, limit, maxAttempts int) ([]PendingEmbedding, error) {
owner, ok := ownerOf(kind)
if !ok {
return nil, nil
}
if limit <= 0 {
limit = 50
}
if maxAttempts <= 0 {
maxAttempts = 5
}
var rows []struct {
ImageID uint32 `gorm:"column:image_id"`
Image string `gorm:"column:image"`
}
if err := r.db.WithContext(ctx).Raw(
"SELECT i.id AS image_id, i.image AS image "+
"FROM "+owner.table+" AS i "+
"LEFT JOIN image_embeddings AS e ON e.kind = ? AND e.image_id = i.id "+
"LEFT JOIN image_embedding_failures AS f ON f.kind = ? AND f.image_id = i.id "+
"WHERE i.is_deleted = 0 AND i.image <> '' AND e.image_id IS NULL "+
" AND coalesce(f.attempts, 0) < ? "+
"ORDER BY i.id LIMIT ?",
kind, kind, maxAttempts, limit).Scan(&rows).Error; err != nil {
return nil, err
}
out := make([]PendingEmbedding, 0, len(rows))
for _, row := range rows {
out = append(out, PendingEmbedding{Kind: kind, ImageID: row.ImageID, Key: row.Image})
}
return out, nil
}
// CountPendingEmbeddings 统计剩余量(与 ListPendingEmbeddings 同口径,避免两边不一致)。
func (r *embeddingRepository) CountPendingEmbeddings(ctx context.Context, kind string, maxAttempts int) (int, error) {
owner, ok := ownerOf(kind)
if !ok {
return 0, nil
}
if maxAttempts <= 0 {
maxAttempts = 5
}
var n int
if err := r.db.WithContext(ctx).Raw(
"SELECT count(*) FROM "+owner.table+" AS i "+
"LEFT JOIN image_embeddings AS e ON e.kind = ? AND e.image_id = i.id "+
"LEFT JOIN image_embedding_failures AS f ON f.kind = ? AND f.image_id = i.id "+
"WHERE i.is_deleted = 0 AND i.image <> '' AND e.image_id IS NULL "+
" AND coalesce(f.attempts, 0) < ?",
kind, kind, maxAttempts).Scan(&n).Error; err != nil {
return 0, err
}
return n, nil
}
// RecordEmbeddingFailure 累加失败次数。
// 用 upsert 而不是先查后写:worker 可能并发回传,两步写法会丢计数。
func (r *embeddingRepository) RecordEmbeddingFailure(ctx context.Context, kind string, imageID uint32, reason string) error {
if len(reason) > 300 {
reason = reason[:300]
}
return r.db.WithContext(ctx).Exec(`
INSERT INTO image_embedding_failures (kind, image_id, attempts, last_error, updated_at)
VALUES (?, ?, 1, ?, ?)
ON CONFLICT (kind, image_id) DO UPDATE
SET attempts = image_embedding_failures.attempts + 1,
last_error = EXCLUDED.last_error,
updated_at = EXCLUDED.updated_at`,
kind, imageID, reason, uint32(time.Now().Unix())).Error
}
// BulkUpsertEmbedding 逐行写入而非 unnest 批量:日入库仅几百张,量级极小,
// 换来的是可读性与低出错概率。ON CONFLICT 保证同一图重复入库不会写出多行。
func (r *embeddingRepository) BulkUpsertEmbedding(ctx context.Context, kind string, ids []uint32, vecs [][]float32) error {