feat(publish): 爬虫入库直写正式表，驳回行可复用重审
This commit is contained in:
toom1996
@ -229,7 +229,7 @@ func (s *IngestService) failOrRetry(ctx context.Context, id uint32, errMsg strin
|
||||
}
|
||||
}
|
||||
|
||||
// processRunway 走秀入库：品牌校验 → 去重 → 补季节码 → 下载图 → 写 brand_runway_drafts。
|
||||
// processRunway 走秀入库：品牌校验 → 实体键分三支（新建 / 放弃 / 复用驳回）→ 下载图 → 直写 brand_runways。
|
||||
func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob, p dto.RunwayIngest) {
|
||||
// 1) 品牌必须存在（爬虫负责先建/复用品牌）
|
||||
brandID, err := hashid.Decode(p.BrandUID)
|
||||
@ -242,38 +242,35 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
return
|
||||
}
|
||||
|
||||
// 2) 按实体键（brand_id + season_code + collection_type）去重：正式表已存在该秀则跳过，
|
||||
// 避免重爬重复下载。注意：只判正式表、不判 pending 草稿——否则多来源（Vogue + theImpression）
|
||||
// 爬同一场秀时第二个来源会被误判重复而丢弃，破坏晋升阶段的图片聚合。
|
||||
// 2) 实体键查重并决定分支（单表模型）
|
||||
seasonCode := season.Derive(p.Year, p.CollectionType, p.Season)
|
||||
dedupStart := time.Now()
|
||||
if _, found, err := s.repo.RunwayIDByEntity(ctx, brandID, seasonCode, p.CollectionType); err == nil && found {
|
||||
log.Printf("[ingest] job=%d runway 实体去重命中(season=%s type=%s)，跳过 耗时=%v",
|
||||
job.ID, seasonCode, p.CollectionType, time.Since(dedupStart).Round(time.Millisecond))
|
||||
reuseID, existStatus, exist, _ := s.repo.RunwayEntityState(ctx, brandID, seasonCode, p.CollectionType)
|
||||
action := decideIngestAction(exist, existStatus)
|
||||
if action == ingestSkip {
|
||||
log.Printf("[ingest] job=%d runway 实体已存在(status=%s)，跳过", job.ID, existStatus)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
return
|
||||
}
|
||||
log.Printf("[ingest] job=%d runway 实体去重 耗时=%v（未命中，继续下载）", job.ID, time.Since(dedupStart).Round(time.Millisecond))
|
||||
|
||||
// 3) season_code 已在去重前补齐（vogue.go 历史漏填的 bug，统一在此兜底）
|
||||
|
||||
// 4) 下载图片并上传到存储（结构化 Looks 优先：主图+细节图分组；否则回退 Images 全部视为主图）。
|
||||
// 内容哈希（sha1）key 保证重爬不产生孤儿文件：失败回滚删本批 key 即可。
|
||||
var cover string
|
||||
var draftImages []model.BrandRunwayDraftImage
|
||||
var images []model.BrandRunwayImage
|
||||
var keys []string
|
||||
var imgFailed bool
|
||||
var imageCount uint16
|
||||
var timing *fetchTiming
|
||||
if len(p.Looks) > 0 {
|
||||
cover, draftImages, keys, imgFailed, timing = s.fetchLookImages(ctx, p.Looks, "runway")
|
||||
cover, images, keys, imgFailed, timing = s.fetchLookImages(ctx, p.Looks, "runway")
|
||||
imageCount = uint16(len(p.Looks))
|
||||
} else {
|
||||
var fimgs []fetchedImage
|
||||
cover, fimgs, keys, imgFailed, timing = s.fetchImages(ctx, p.Images, "runway", dto.IngestKindRunway)
|
||||
imageCount = uint16(len(fimgs))
|
||||
for i, fi := range fimgs {
|
||||
draftImages = append(draftImages, model.BrandRunwayDraftImage{
|
||||
images = append(images, model.BrandRunwayImage{
|
||||
Image: fi.url,
|
||||
Name: fmt.Sprintf("Look %d", i+1),
|
||||
SortOrder: uint32(i + 1),
|
||||
@ -294,9 +291,9 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
return
|
||||
}
|
||||
|
||||
// 5) 写草稿表（status=pending），等待后台审核通过后再晋升正式表
|
||||
// 5) 直写正式表（status=pending），等待后台审核通过（审核只改状态，不重建图片）
|
||||
writeStart := time.Now()
|
||||
draft := &model.BrandRunwayDraft{
|
||||
rw := &model.BrandRunway{
|
||||
JobID: job.ID,
|
||||
BrandID: brandID,
|
||||
TitleEn: p.TitleEn,
|
||||
@ -309,26 +306,28 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
SeasonCode: seasonCode,
|
||||
Cover: cover,
|
||||
ImageCount: imageCount,
|
||||
Status: model.DraftStatusPending,
|
||||
Status: model.StatusPending,
|
||||
}
|
||||
id, err := s.repo.CreateRunwayDraft(ctx, draft)
|
||||
if err != nil {
|
||||
s.failOrRetry(ctx, job.ID, "create draft: "+err.Error())
|
||||
for i := range images {
|
||||
images[i].BrandID = brandID
|
||||
}
|
||||
var saveErr error
|
||||
if action == ingestReuseRejected {
|
||||
saveErr = s.repo.ReuseRejectedRunway(ctx, reuseID, rw, images)
|
||||
} else {
|
||||
_, saveErr = s.repo.CreateRunwayWithImages(ctx, rw, images)
|
||||
}
|
||||
if saveErr != nil {
|
||||
s.failOrRetry(ctx, job.ID, "save runway: "+saveErr.Error())
|
||||
return
|
||||
}
|
||||
for i := range draftImages {
|
||||
draftImages[i].DraftID = id
|
||||
}
|
||||
if err := s.repo.CreateRunwayDraftImages(ctx, draftImages); err != nil {
|
||||
s.failOrRetry(ctx, job.ID, "create draft images: "+err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("[ingest] job=%d runway 写草稿 耗时=%v 图片行=%d", job.ID, time.Since(writeStart).Round(time.Millisecond), len(draftImages))
|
||||
log.Printf("[ingest] job=%d runway 写正式表 耗时=%v 图片行=%d 复用驳回=%v",
|
||||
job.ID, time.Since(writeStart).Round(time.Millisecond), len(images), action == ingestReuseRejected)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
}
|
||||
|
||||
// fetchLookImages 按 Looks 结构下载主图+细节图并上传到存储，返回可直接落草稿的
|
||||
// BrandRunwayDraftImage 行（带 look_index / is_detail 分组）。任意一张下载/上传失败即把
|
||||
// fetchLookImages 按 Looks 结构下载主图+细节图并上传到存储，返回可直接落正式图片表的
|
||||
// BrandRunwayImage 行（带 look_index / is_detail 分组）。任意一张下载/上传失败即把
|
||||
// failed 置 true，调用方据此把整条任务判失败并回滚本批已上传的 key，符合「单图失败=整任务失败」策略。
|
||||
// cover 取首个成功下载的主图；image_count（主图数）由调用方按 len(Looks) 计，不在此返回。
|
||||
// 每张图入库前做去重：命中 dHash 近重复则仍入库但标记留痕。
|
||||
@ -336,11 +335,11 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
// 实现：先把全部「主图 + 细节图」按原始顺序摊平成任务列表，用 concurrentFetch 并发完成
|
||||
// 「下载 → phash → 上传」这段 IO 密集操作；随后再按下标顺序串行去重与组装。
|
||||
// 这样既拿到并发收益，又保证 cover / sort_order / 批次内去重与原串行实现一致。
|
||||
func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayLook, prefix string) (string, []model.BrandRunwayDraftImage, []string, bool, *fetchTiming) {
|
||||
func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayLook, prefix string) (string, []model.BrandRunwayImage, []string, bool, *fetchTiming) {
|
||||
start := time.Now()
|
||||
timing := &fetchTiming{}
|
||||
cover := ""
|
||||
rows := make([]model.BrandRunwayDraftImage, 0)
|
||||
rows := make([]model.BrandRunwayImage, 0)
|
||||
keys := make([]string, 0)
|
||||
seen := make(map[string]bool)
|
||||
failed := false
|
||||
@ -392,7 +391,7 @@ func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayL
|
||||
name = fmt.Sprintf("Look %d — Detail %d", task.lookIdx, task.detailNo)
|
||||
isDetail = 1
|
||||
}
|
||||
rows = append(rows, model.BrandRunwayDraftImage{
|
||||
rows = append(rows, model.BrandRunwayImage{
|
||||
Image: r.url,
|
||||
Name: name,
|
||||
SortOrder: uint32(order),
|
||||
@ -408,18 +407,16 @@ func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayL
|
||||
return cover, rows, keys, failed, timing
|
||||
}
|
||||
|
||||
// processStreet 街拍入库：去重 → 下载图 → 写 street_snap_draft（无品牌）。
|
||||
// processStreet 街拍入库：实体键分三支（新建 / 放弃 / 复用驳回）→ 下载图 → 直写 street_snaps（无品牌）。
|
||||
func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob, p dto.RunwayIngest) {
|
||||
// 1) 按实体键（city + year）去重：正式表已存在该街拍则跳过，避免重爬重复下载。
|
||||
// 只判正式表、不判 pending 草稿，保留多来源街拍图片在晋升阶段聚合。
|
||||
dedupStart := time.Now()
|
||||
if _, found, err := s.repo.StreetSnapIDByEntity(ctx, p.City, p.Year); err == nil && found {
|
||||
log.Printf("[ingest] job=%d street 实体去重命中(city=%s year=%d)，跳过 耗时=%v",
|
||||
job.ID, p.City, p.Year, time.Since(dedupStart).Round(time.Millisecond))
|
||||
// 1) 实体键查重并决定分支（单表模型，同 runway）
|
||||
reuseID, existStatus, exist, _ := s.repo.StreetSnapEntityState(ctx, p.City, p.Year)
|
||||
action := decideIngestAction(exist, existStatus)
|
||||
if action == ingestSkip {
|
||||
log.Printf("[ingest] job=%d street 实体已存在(status=%s)，跳过", job.ID, existStatus)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
return
|
||||
}
|
||||
log.Printf("[ingest] job=%d street 实体去重 耗时=%v（未命中，继续下载）", job.ID, time.Since(dedupStart).Round(time.Millisecond))
|
||||
|
||||
// 2) 下载图片并上传到存储（S4优先，失败兜底本地）。
|
||||
cover, fimgs, keys, imgFailed, timing := s.fetchImages(ctx, p.Images, "street", dto.IngestKindStreet)
|
||||
@ -432,26 +429,20 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
return
|
||||
}
|
||||
|
||||
// 3) 写草稿表（status=pending），等待后台审核通过后再晋升 street_snaps 正式表
|
||||
// 3) 直写正式表（status=pending）
|
||||
writeStart := time.Now()
|
||||
draft := &model.StreetSnapDraft{
|
||||
snap := &model.StreetSnap{
|
||||
JobID: job.ID,
|
||||
Title: p.TitleEn, // 街拍单标题，爬虫优先填 title_en
|
||||
Year: p.Year,
|
||||
City: p.City,
|
||||
Cover: cover,
|
||||
ImageCount: uint16(len(fimgs)),
|
||||
Status: model.DraftStatusPending,
|
||||
Status: model.StatusPending,
|
||||
}
|
||||
id, err := s.repo.CreateStreetSnapDraft(ctx, draft)
|
||||
if err != nil {
|
||||
s.failOrRetry(ctx, job.ID, "create street draft: "+err.Error())
|
||||
return
|
||||
}
|
||||
rows := make([]model.StreetSnapDraftImage, 0, len(fimgs))
|
||||
rows := make([]model.StreetSnapImage, 0, len(fimgs))
|
||||
for i, fi := range fimgs {
|
||||
rows = append(rows, model.StreetSnapDraftImage{
|
||||
DraftID: id,
|
||||
rows = append(rows, model.StreetSnapImage{
|
||||
Image: fi.url,
|
||||
Name: fmt.Sprintf("Look %d", i+1),
|
||||
SortOrder: uint32(i + 1),
|
||||
@ -460,12 +451,18 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
DupOf: strconv.FormatUint(uint64(fi.dupID), 10),
|
||||
})
|
||||
}
|
||||
|
||||
if err := s.repo.CreateStreetSnapDraftImages(ctx, rows); err != nil {
|
||||
s.failOrRetry(ctx, job.ID, "create street draft images: "+err.Error())
|
||||
var saveErr error
|
||||
if action == ingestReuseRejected {
|
||||
saveErr = s.repo.ReuseRejectedStreetSnap(ctx, reuseID, snap, rows)
|
||||
} else {
|
||||
_, saveErr = s.repo.CreateStreetSnapWithImages(ctx, snap, rows)
|
||||
}
|
||||
if saveErr != nil {
|
||||
s.failOrRetry(ctx, job.ID, "save street snap: "+saveErr.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("[ingest] job=%d street 写草稿 耗时=%v 图片行=%d", job.ID, time.Since(writeStart).Round(time.Millisecond), len(rows))
|
||||
log.Printf("[ingest] job=%d street 写正式表 耗时=%v 图片行=%d 复用驳回=%v",
|
||||
job.ID, time.Since(writeStart).Round(time.Millisecond), len(rows), action == ingestReuseRejected)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
}
|
||||
|
||||
@ -606,9 +603,9 @@ func (s *IngestService) dedupImage(ctx context.Context, kind, phashBits string,
|
||||
var tables []string
|
||||
switch kind {
|
||||
case dto.IngestKindRunway:
|
||||
tables = []string{"brand_runway_draft_images", "brand_runway_images"}
|
||||
tables = []string{"brand_runway_images"}
|
||||
case dto.IngestKindStreet:
|
||||
tables = []string{"street_snap_draft_images", "street_snap_images"}
|
||||
tables = []string{"street_snap_images"}
|
||||
default:
|
||||
return 0, 0
|
||||
}
|
||||
@ -742,3 +739,25 @@ func (s *IngestService) downloadAndUpload(ctx context.Context, u, prefix string)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// ingestActionKind 入库分支决策结果（单表发布模型）。
|
||||
type ingestActionKind int
|
||||
|
||||
const (
|
||||
ingestCreate ingestActionKind = iota // 实体键未命中 → 新建记录
|
||||
ingestSkip // 命中 pending/published → 放弃本次入库
|
||||
ingestReuseRejected // 命中 rejected → 复用该行重审
|
||||
)
|
||||
|
||||
// decideIngestAction 按实体键命中情况与既有行状态决定入库分支。
|
||||
// 抽成纯函数，是为了让「驳回不是永久黑名单」这条规则有独立的、不需要数据库的测试。
|
||||
func decideIngestAction(found bool, status string) ingestActionKind {
|
||||
switch {
|
||||
case !found:
|
||||
return ingestCreate
|
||||
case status == model.StatusRejected:
|
||||
return ingestReuseRejected
|
||||
default:
|
||||
return ingestSkip
|
||||
}
|
||||
}
|
||||
|
||||
@ -84,3 +84,26 @@ func TestIngestRetryBackoff(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecideIngestAction 入库三分支决策：未命中→新建；命中 pending/published→放弃；命中 rejected→复用重审。
|
||||
// 这是「驳回不是永久黑名单」这条规则的直接证据。
|
||||
func TestDecideIngestAction(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
found bool
|
||||
status string
|
||||
want ingestActionKind
|
||||
}{
|
||||
{"未命中→新建", false, "", ingestCreate},
|
||||
{"命中 pending→放弃", true, model.StatusPending, ingestSkip},
|
||||
{"命中 published→放弃", true, model.StatusPublished, ingestSkip},
|
||||
{"命中 rejected→复用重审", true, model.StatusRejected, ingestReuseRejected},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := decideIngestAction(c.found, c.status); got != c.want {
|
||||
t.Fatalf("decideIngestAction(%v, %q) = %v, want %v", c.found, c.status, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user