feat: 抓取任务队列 + 媒体机直传 + 向量回填
crawl(品牌抓取任务队列) - /admin/internal/crawl/tasks 领取 / :id/result 回报,spider 可常驻拉任务 - 后台品牌列表支持「加入抓取队列」,队列页可看状态与重试 - 租约超时的 running 自动回收为 queued,避免 worker 崩溃后任务卡死 media(媒体机直传) - ingest 载荷新增 main_key/detail_keys/main_size/detail_sizes: 有 key 就直接落库,跳过下载与上传;与 URL 形态可逐图混用 - 细节图按 Details/DetailKeys 两者较长者遍历(只给 key 时不再丢图) - 媒体机上传的 key 不进失败回滚清单(内容寻址、可能与他人共享) - /admin/internal/upload-url 签发 S4 PUT 预签名(spider 不持有 AK/SK) embeddings(向量回填) - GET /admin/internal/embeddings/pending 派发缺向量的图(含可直接下载的 url) - POST /admin/internal/embeddings 回写向量并重算近邻;失败项计入 image_embedding_failures,连续失败到上限后退出队列(坏图不占位) - 近邻重算改为异步串行:同步会让 100 张的批次 POST 超时(实测 >60s) - 新增 key 分支单测:混用 key/url、细节图只给 key、字节数、回滚清单、向量占位
This commit is contained in:
@ -58,7 +58,24 @@ type DupStat struct {
|
||||
//
|
||||
// 近邻结果走 image_similar_pairs 预计算表:入库时(或 dbtool similarbackfill 回填时)
|
||||
// 算一次存表,后台页面只读结果。原因见 ReplaceSimilarPairs 注释。
|
||||
// PendingEmbedding 是一张「缺向量」的图片,供回填 worker 拉取。
|
||||
// 带 Key 而不是完整 URL:worker 自己按存储基址拼(后端可能换了域名,不该把地址写死在队列里)。
|
||||
type PendingEmbedding struct {
|
||||
Kind string // runway | street
|
||||
ImageID uint32
|
||||
Key string // 存储 key(S4)或本地相对路径(/uploads/...)
|
||||
}
|
||||
|
||||
type EmbeddingRepository interface {
|
||||
// ListPendingEmbeddings 取一批「还活着、没有向量、失败次数未达上限」的图片,按 id 升序。
|
||||
//
|
||||
// 媒体拆分后这条查询是向量回填的唯一入口:媒体机直传只写 key,后端不碰图片字节,
|
||||
// 向量必须靠外部 worker 回传,而 worker 需要知道「还有哪些图缺向量」。
|
||||
ListPendingEmbeddings(ctx context.Context, kind string, limit, maxAttempts int) ([]PendingEmbedding, error)
|
||||
// CountPendingEmbeddings 统计剩余待回填数量(worker 据此决定是否继续拉)。
|
||||
CountPendingEmbeddings(ctx context.Context, kind string, maxAttempts int) (int, error)
|
||||
// RecordEmbeddingFailure 记一次失败(attempts+1)。达到 maxAttempts 后该图不再进队列。
|
||||
RecordEmbeddingFailure(ctx context.Context, kind string, imageID uint32, reason string) error
|
||||
// UpsertEmbedding 写入或更新一张图的向量((kind, image_id) 唯一)。
|
||||
UpsertEmbedding(ctx context.Context, kind string, imageID uint32, vec []float32) error
|
||||
// BulkUpsertEmbedding 批量写入;vecs[i] 对应 ids[i],vecs[i] 为 nil 则跳过该行。
|
||||
@ -175,6 +192,85 @@ func (r *embeddingRepository) UpsertEmbedding(ctx context.Context, kind string,
|
||||
return r.BulkUpsertEmbedding(ctx, kind, []uint32{imageID}, [][]float32{vec})
|
||||
}
|
||||
|
||||
// ListPendingEmbeddings 取缺失向量的图片。
|
||||
//
|
||||
// 三个 LEFT JOIN 各有明确目的:
|
||||
// - e 判「已有没有向量」;
|
||||
// - f 判「失败次数是否已到上限」——没有它,一张永远编不出来的坏图会让每轮回填都空转;
|
||||
// - i.is_deleted = 0 跳过软删图,避免为已删内容白算向量。
|
||||
//
|
||||
// 表名/列名来自 ownerOf 常量,kind 走参数绑定,无注入面。
|
||||
func (r *embeddingRepository) ListPendingEmbeddings(ctx context.Context, kind string, limit, maxAttempts int) ([]PendingEmbedding, error) {
|
||||
owner, ok := ownerOf(kind)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = 5
|
||||
}
|
||||
var rows []struct {
|
||||
ImageID uint32 `gorm:"column:image_id"`
|
||||
Image string `gorm:"column:image"`
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
"SELECT i.id AS image_id, i.image AS image "+
|
||||
"FROM "+owner.table+" AS i "+
|
||||
"LEFT JOIN image_embeddings AS e ON e.kind = ? AND e.image_id = i.id "+
|
||||
"LEFT JOIN image_embedding_failures AS f ON f.kind = ? AND f.image_id = i.id "+
|
||||
"WHERE i.is_deleted = 0 AND i.image <> '' AND e.image_id IS NULL "+
|
||||
" AND coalesce(f.attempts, 0) < ? "+
|
||||
"ORDER BY i.id LIMIT ?",
|
||||
kind, kind, maxAttempts, limit).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]PendingEmbedding, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, PendingEmbedding{Kind: kind, ImageID: row.ImageID, Key: row.Image})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CountPendingEmbeddings 统计剩余量(与 ListPendingEmbeddings 同口径,避免两边不一致)。
|
||||
func (r *embeddingRepository) CountPendingEmbeddings(ctx context.Context, kind string, maxAttempts int) (int, error) {
|
||||
owner, ok := ownerOf(kind)
|
||||
if !ok {
|
||||
return 0, nil
|
||||
}
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = 5
|
||||
}
|
||||
var n int
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
"SELECT count(*) FROM "+owner.table+" AS i "+
|
||||
"LEFT JOIN image_embeddings AS e ON e.kind = ? AND e.image_id = i.id "+
|
||||
"LEFT JOIN image_embedding_failures AS f ON f.kind = ? AND f.image_id = i.id "+
|
||||
"WHERE i.is_deleted = 0 AND i.image <> '' AND e.image_id IS NULL "+
|
||||
" AND coalesce(f.attempts, 0) < ?",
|
||||
kind, kind, maxAttempts).Scan(&n).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// RecordEmbeddingFailure 累加失败次数。
|
||||
// 用 upsert 而不是先查后写:worker 可能并发回传,两步写法会丢计数。
|
||||
func (r *embeddingRepository) RecordEmbeddingFailure(ctx context.Context, kind string, imageID uint32, reason string) error {
|
||||
if len(reason) > 300 {
|
||||
reason = reason[:300]
|
||||
}
|
||||
return r.db.WithContext(ctx).Exec(`
|
||||
INSERT INTO image_embedding_failures (kind, image_id, attempts, last_error, updated_at)
|
||||
VALUES (?, ?, 1, ?, ?)
|
||||
ON CONFLICT (kind, image_id) DO UPDATE
|
||||
SET attempts = image_embedding_failures.attempts + 1,
|
||||
last_error = EXCLUDED.last_error,
|
||||
updated_at = EXCLUDED.updated_at`,
|
||||
kind, imageID, reason, uint32(time.Now().Unix())).Error
|
||||
}
|
||||
|
||||
// BulkUpsertEmbedding 逐行写入而非 unnest 批量:日入库仅几百张,量级极小,
|
||||
// 换来的是可读性与低出错概率。ON CONFLICT 保证同一图重复入库不会写出多行。
|
||||
func (r *embeddingRepository) BulkUpsertEmbedding(ctx context.Context, kind string, ids []uint32, vecs [][]float32) error {
|
||||
|
||||
Reference in New Issue
Block a user