feat: 入库结果落库 + 品牌详情展示更新记录
背景:任务只有 status=done,分不清「新建」「已存在跳过」「已删除跳过」, 品牌更新历史只能在进程日志里翻,日志一滚就没了。 - ingest_jobs 新增 outcome / note / brand_id(迁移 2026-10-25-01) outcome 记结果类型,note 记人话细节(新建 N 张图 / 已存在(status=published)) - 提交任务时解出 brand_id 落库,品牌详情页按 (brand_id, id DESC) 直接查 - 处理时优先用 job.brand_id,payload 里的 hashid 只作老任务兜底: 避免「别的实例盐不同 → 解出错品牌 → 整场 brand not found」再发生 - 品牌编辑页新增「更新记录」区块:时间 / 场次 / 结果 / 明细(含失败原因与尝试次数) - 测试:fake 仓储补齐新接口,capture 仓储记录 outcome 供断言
This commit is contained in:
@ -117,13 +117,26 @@ func (s *IngestService) Submit(ctx context.Context, payload dto.RunwayIngest) (u
|
||||
if payload.Kind == dto.IngestKindRunway && payload.BrandUID == "" {
|
||||
return 0, NewError(http.StatusBadRequest, "brand_uid required")
|
||||
}
|
||||
job := &model.IngestJob{Payload: string(raw), Kind: model.IngestKindCrawl}
|
||||
// brand_id 提交时就解出来:品牌详情页按它直接查更新记录,
|
||||
// 不必每次把 payload 里的 hashid 解一遍(也避免与查询时的盐不一致)。
|
||||
var brandID uint32
|
||||
if payload.BrandUID != "" {
|
||||
if id, derr := hashid.Decode(payload.BrandUID); derr == nil {
|
||||
brandID = id
|
||||
}
|
||||
}
|
||||
job := &model.IngestJob{Payload: string(raw), Kind: model.IngestKindCrawl, BrandID: brandID}
|
||||
if err := s.repo.Enqueue(ctx, job); err != nil {
|
||||
return 0, internalErr(err.Error())
|
||||
}
|
||||
return job.ID, nil
|
||||
}
|
||||
|
||||
// ListBrandRecords 取某品牌的入库记录(品牌详情页的「更新记录」)。
|
||||
func (s *IngestService) ListBrandRecords(ctx context.Context, brandID uint32, limit int) ([]repository.BrandIngestRecord, error) {
|
||||
return s.repo.ListByBrand(ctx, brandID, limit)
|
||||
}
|
||||
|
||||
// ListJobs 分页列出入库任务(后台监控页用)。返回本页任务、总条数与错误。
|
||||
func (s *IngestService) ListJobs(ctx context.Context, offset, limit int) ([]model.IngestJob, int64, error) {
|
||||
jobs, err := s.repo.ListJobs(ctx, offset, limit)
|
||||
@ -245,11 +258,11 @@ func (s *IngestService) processMediaCleanup(ctx context.Context, job model.Inges
|
||||
return
|
||||
}
|
||||
if len(p.Keys) == 0 {
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeNoop, "媒体清理:无需处理")
|
||||
return
|
||||
}
|
||||
purgeOrphanImages(ctx, s.media, s.del, p.Keys)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeCleaned, "媒体清理完成")
|
||||
}
|
||||
|
||||
// failOrRetry 把临时失败(如网络抖动 / 单图下载失败)按指数退避自动重试:
|
||||
@ -266,10 +279,22 @@ func (s *IngestService) failOrRetry(ctx context.Context, id uint32, errMsg strin
|
||||
// processRunway 走秀入库:品牌校验 → 实体键分三支(新建 / 放弃 / 复用驳回)→ 下载图 → 直写 brand_runways。
|
||||
func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob, p dto.RunwayIngest) {
|
||||
// 1) 品牌必须存在(爬虫负责先建/复用品牌)
|
||||
brandID, err := hashid.Decode(p.BrandUID)
|
||||
if err != nil {
|
||||
_ = s.repo.MarkFailed(ctx, job.ID, "unknown brand_uid")
|
||||
return
|
||||
//
|
||||
// 优先用「提交任务时就解好的 brand_id」:它由接收上报的那个实例解出,盐一定正确。
|
||||
// 只有当 brand_id 缺省(本次改动前的老任务)才回落到解 payload 里的 hashid。
|
||||
//
|
||||
// 为什么必须这样:hashid 的解码依赖 HASHID_SECRET。线上曾出现另一台机器上的
|
||||
// 旧后端连同一套库抢任务,它的盐与线上不同 → 同一个 uid 解出别的 id → 整场被判
|
||||
// brand not found 丢掉(AMBUSH 那批 8 场就是这么没的)。brand_id 落库后,
|
||||
// 即便 worker 与提交方不是同一个实例,也不再有这个分歧。
|
||||
brandID := job.BrandID
|
||||
if brandID == 0 {
|
||||
id, derr := hashid.Decode(p.BrandUID)
|
||||
if derr != nil {
|
||||
_ = s.repo.MarkFailed(ctx, job.ID, "unknown brand_uid")
|
||||
return
|
||||
}
|
||||
brandID = id
|
||||
}
|
||||
if _, err := s.brandRepo.FindByID(ctx, brandID); err != nil {
|
||||
// 必须区分「品牌真不存在」与「查询本身失败」:前者永久失败,后者应退避重试。
|
||||
@ -304,7 +329,7 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
// 删除即永久:已软删的实体键不再因重爬复活(原设计靠部分唯一索引释放键、允许重爬,已改为禁止)。
|
||||
if exist && isDeleted {
|
||||
log.Printf("[ingest] job=%d runway 实体已删除(is_deleted=1),跳过重爬", job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedDeleted, "实体已删除(删除即永久,不重爬)")
|
||||
return
|
||||
}
|
||||
if !exist && p.Year == 0 {
|
||||
@ -319,7 +344,7 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
reuseID = rid
|
||||
if exist && isDeleted {
|
||||
log.Printf("[ingest] job=%d runway 实体已删除(is_deleted=1),跳过重爬", job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedDeleted, "实体已删除(删除即永久,不重爬)")
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -333,7 +358,7 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedExisting, "已存在(status="+existStatus+"),跳过重下")
|
||||
return
|
||||
}
|
||||
|
||||
@ -439,7 +464,11 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
}
|
||||
log.Printf("[ingest] job=%d runway 写正式表 耗时=%v 图片行=%d 复用驳回=%v",
|
||||
job.ID, time.Since(writeStart).Round(time.Millisecond), len(images), action == ingestReuseRejected)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
outcome, note := model.IngestOutcomeCreated, fmt.Sprintf("新建 %d 张图", len(images))
|
||||
if action == ingestReuseRejected {
|
||||
outcome, note = model.IngestOutcomeReused, fmt.Sprintf("复用驳回记录,覆盖 %d 张图", len(images))
|
||||
}
|
||||
_ = s.repo.MarkDone(ctx, job.ID, outcome, note)
|
||||
}
|
||||
|
||||
// fetchLookImages 按 Looks 结构下载主图+细节图并上传到存储,返回可直接落正式图片表的
|
||||
@ -581,13 +610,13 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
// 删除即永久:已软删的实体键不再因重爬复活(与 runway 同源)。
|
||||
if exist && isDeleted {
|
||||
log.Printf("[ingest] job=%d street 实体已删除(is_deleted=1),跳过重爬", job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedDeleted, "街拍实体已删除(删除即永久,不重爬)")
|
||||
return
|
||||
}
|
||||
action := decideIngestAction(exist, existStatus)
|
||||
if action == ingestSkip {
|
||||
log.Printf("[ingest] job=%d street 实体已存在(status=%s),跳过", job.ID, existStatus)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedExisting, "街拍已存在(status="+existStatus+"),跳过重下")
|
||||
return
|
||||
}
|
||||
|
||||
@ -655,7 +684,11 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
s.precomputeSimilar(ctx, dto.IngestKindStreet, embedIDs)
|
||||
log.Printf("[ingest] job=%d street 写正式表 耗时=%v 图片行=%d 复用驳回=%v",
|
||||
job.ID, time.Since(writeStart).Round(time.Millisecond), len(rows), action == ingestReuseRejected)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
outcome, note := model.IngestOutcomeCreated, fmt.Sprintf("新建街拍 %d 张图", len(rows))
|
||||
if action == ingestReuseRejected {
|
||||
outcome, note = model.IngestOutcomeReused, fmt.Sprintf("复用驳回的街拍,覆盖 %d 张图", len(rows))
|
||||
}
|
||||
_ = s.repo.MarkDone(ctx, job.ID, outcome, note)
|
||||
}
|
||||
|
||||
// fetchImages 下载图片并上传到存储,返回 (cover 地址, 已下载图结构, 已成功上传对象的 key 列表, 是否有任意一张失败)。
|
||||
|
||||
Reference in New Issue
Block a user