feat(publish): 爬虫入库直写正式表，驳回行可复用重审
This commit is contained in:
toom1996
@ -14,9 +14,9 @@ import (
|
||||
// IngestRepository 爬虫入库管线专属仓储：任务队列（ingest_jobs）+ nonce 防重放
|
||||
// （ingest_nonces）+ 走秀正式表写入（brand_runways / brand_runway_images）。
|
||||
//
|
||||
// 去重由 worker 按「实体键」完成：走秀 = brand_id + season_code + collection_type，
|
||||
// 街拍 = city + year（与 review_repository 的 SaveRunwayFromDraft / SaveStreetSnapFromDraft
|
||||
// 晋升时的幂等键一致），因此一个秀/街拍只会在正式表留一行，多来源图片在晋升阶段聚合。
|
||||
// 实体键：走秀 = brand_id + season_code + collection_type，街拍 = city + year。
|
||||
// 单表发布模型下入库直写正式表（status=pending），实体键命中后按既有行状态分三支：
|
||||
// pending/published → 放弃本次入库；rejected → 复用该行重审。因此一个秀/街拍始终只留一行。
|
||||
//
|
||||
// 入队与领取用同一张 ingest_jobs 表，领取靠 PostgreSQL 的 FOR UPDATE SKIP LOCKED
|
||||
// 实现「多 worker 安全并发」——同一条任务只会被一个 worker 拿到，其它 worker 跳过它。
|
||||
@ -32,19 +32,20 @@ type IngestRepository interface {
|
||||
MarkFailed(ctx context.Context, id uint32, errMsg string) error
|
||||
// ReserveNonce 写入一次性随机串；若已存在（重放）返回 ok=false。
|
||||
ReserveNonce(ctx context.Context, nonce string) (ok bool, err error)
|
||||
// RunwayIDByEntity 按实体键（brand_id + season_code + collection_type）查是否已存在走秀正式表；
|
||||
// 返回 (id, found)。worker 据此跳过已晋升秀的重爬，避免重复下载。
|
||||
RunwayIDByEntity(ctx context.Context, brandID uint32, seasonCode, collectionType string) (uint32, bool, error)
|
||||
// CreateRunwayDraft 插入走秀草稿行（status=pending），返回自增主键。
|
||||
CreateRunwayDraft(ctx context.Context, d *model.BrandRunwayDraft) (uint32, error)
|
||||
// CreateRunwayDraftImages 批量插入草稿图片行。
|
||||
CreateRunwayDraftImages(ctx context.Context, imgs []model.BrandRunwayDraftImage) error
|
||||
// StreetSnapIDByEntity 按实体键（city + year）查是否已存在街拍正式表；返回 (id, found)。
|
||||
StreetSnapIDByEntity(ctx context.Context, city string, year uint16) (uint32, bool, error)
|
||||
// CreateStreetSnapDraft 插入街拍草稿行（status=pending），返回自增主键。
|
||||
CreateStreetSnapDraft(ctx context.Context, d *model.StreetSnapDraft) (uint32, error)
|
||||
// CreateStreetSnapDraftImages 批量插入街拍草稿图片行。
|
||||
CreateStreetSnapDraftImages(ctx context.Context, imgs []model.StreetSnapDraftImage) error
|
||||
// RunwayEntityState 按实体键（brand_id + season_code + collection_type）查正式表既有行，返回其状态。
|
||||
// worker 据此分三支：pending/published → 直接放弃本次入库；rejected → 复用该行重审；未命中 → 新建。
|
||||
RunwayEntityState(ctx context.Context, brandID uint32, seasonCode, collectionType string) (id uint32, status string, found bool, err error)
|
||||
// CreateRunwayWithImages 新建走秀正式行（status=pending）并写入图片，返回记录主键（事务内完成）。
|
||||
CreateRunwayWithImages(ctx context.Context, rw *model.BrandRunway, imgs []model.BrandRunwayImage) (uint32, error)
|
||||
// ReuseRejectedRunway 复用一条 rejected 走秀：覆盖内容字段、软删旧图、写入新图、置回 pending（事务内完成）。
|
||||
// 驳回因此不是永久黑名单：重爬同一实体即重新送审，且始终「一个实体一行」。
|
||||
ReuseRejectedRunway(ctx context.Context, id uint32, rw *model.BrandRunway, imgs []model.BrandRunwayImage) error
|
||||
// StreetSnapEntityState 按实体键（city + year）查正式表既有行，返回其状态。
|
||||
StreetSnapEntityState(ctx context.Context, city string, year uint16) (id uint32, status string, found bool, err error)
|
||||
// CreateStreetSnapWithImages 新建街拍正式行（status=pending）并写入图片（事务内完成）。
|
||||
CreateStreetSnapWithImages(ctx context.Context, snap *model.StreetSnap, imgs []model.StreetSnapImage) (uint32, error)
|
||||
// ReuseRejectedStreetSnap 复用一条 rejected 街拍记录（事务内完成）。
|
||||
ReuseRejectedStreetSnap(ctx context.Context, id uint32, snap *model.StreetSnap, imgs []model.StreetSnapImage) error
|
||||
// FindNearDuplicateImage 在给定图片表中按 dHash 汉明距离检索近重复，返回命中行 id（仅取最近一条）。
|
||||
FindNearDuplicateImage(ctx context.Context, tables []string, phashBits string, threshold int) (dupID uint32, found bool, err error)
|
||||
// ListJobs 按 id 倒序分页列出入库任务（用于后台监控页）。offset/limit 控制分页。
|
||||
@ -239,94 +240,167 @@ func (r *ingestRepository) ReserveNonce(ctx context.Context, nonce string) (bool
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// RunwayIDByEntity 按实体键查走秀正式表是否已存在（与 SaveRunwayFromDraft 晋升键一致）。
|
||||
func (r *ingestRepository) RunwayIDByEntity(ctx context.Context, brandID uint32, seasonCode, collectionType string) (uint32, bool, error) {
|
||||
// ---- 入库直写正式表（单表发布模型）----
|
||||
|
||||
func (r *ingestRepository) RunwayEntityState(ctx context.Context, brandID uint32, seasonCode, collectionType string) (uint32, string, bool, error) {
|
||||
var row struct {
|
||||
ID uint32 `gorm:"column:id"`
|
||||
ID uint32 `gorm:"column:id"`
|
||||
Status string `gorm:"column:status"`
|
||||
}
|
||||
err := r.db.WithContext(ctx).
|
||||
Model(&model.BrandRunway{}).
|
||||
Select("id").
|
||||
Select("id, status").
|
||||
Where("brand_id = ? AND season_code = ? AND collection_type = ? AND is_deleted = 0", brandID, seasonCode, collectionType).
|
||||
Limit(1).
|
||||
Scan(&row).Error
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
return 0, "", false, err
|
||||
}
|
||||
if row.ID == 0 {
|
||||
return 0, false, nil
|
||||
return 0, "", false, nil
|
||||
}
|
||||
return row.ID, true, nil
|
||||
return row.ID, row.Status, true, nil
|
||||
}
|
||||
|
||||
func (r *ingestRepository) CreateRunwayDraft(ctx context.Context, d *model.BrandRunwayDraft) (uint32, error) {
|
||||
func (r *ingestRepository) CreateRunwayWithImages(ctx context.Context, rw *model.BrandRunway, imgs []model.BrandRunwayImage) (uint32, error) {
|
||||
now := uint32(time.Now().Unix())
|
||||
d.CreatedAt = now
|
||||
d.UpdatedAt = now
|
||||
if d.Status == "" {
|
||||
d.Status = model.DraftStatusPending
|
||||
rw.CreatedAt, rw.UpdatedAt = now, now
|
||||
if rw.Status == "" {
|
||||
rw.Status = model.StatusPending
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Create(d).Error; err != nil {
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if cErr := tx.Create(rw).Error; cErr != nil {
|
||||
return cErr
|
||||
}
|
||||
if len(imgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := range imgs {
|
||||
imgs[i].RunwayID = rw.ID
|
||||
imgs[i].CreatedAt, imgs[i].UpdatedAt = now, now
|
||||
}
|
||||
return tx.Create(&imgs).Error
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return d.ID, nil
|
||||
return rw.ID, nil
|
||||
}
|
||||
|
||||
func (r *ingestRepository) CreateRunwayDraftImages(ctx context.Context, imgs []model.BrandRunwayDraftImage) error {
|
||||
if len(imgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
func (r *ingestRepository) ReuseRejectedRunway(ctx context.Context, id uint32, rw *model.BrandRunway, imgs []model.BrandRunwayImage) error {
|
||||
now := uint32(time.Now().Unix())
|
||||
for i := range imgs {
|
||||
imgs[i].CreatedAt = now
|
||||
imgs[i].UpdatedAt = now
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(&imgs).Error
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if uErr := tx.Model(&model.BrandRunway{}).Where("id = ?", id).Updates(map[string]any{
|
||||
"title_en": rw.TitleEn,
|
||||
"title_cn": rw.TitleCn,
|
||||
"description_en": rw.DescriptionEn,
|
||||
"description_cn": rw.DescriptionCn,
|
||||
"year": rw.Year,
|
||||
"season": rw.Season,
|
||||
"collection_type": rw.CollectionType,
|
||||
"season_code": rw.SeasonCode,
|
||||
"cover": rw.Cover,
|
||||
"image_count": rw.ImageCount,
|
||||
"job_id": rw.JobID,
|
||||
"status": model.StatusPending,
|
||||
"reviewer": "",
|
||||
"reject_reason": "",
|
||||
"updated_at": now,
|
||||
}).Error; uErr != nil {
|
||||
return uErr
|
||||
}
|
||||
if dErr := tx.Model(&model.BrandRunwayImage{}).
|
||||
Where("runway_id = ? AND is_deleted = 0", id).
|
||||
Updates(map[string]any{"is_deleted": 1, "updated_at": now}).Error; dErr != nil {
|
||||
return dErr
|
||||
}
|
||||
if len(imgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := range imgs {
|
||||
imgs[i].RunwayID = id
|
||||
imgs[i].CreatedAt, imgs[i].UpdatedAt = now, now
|
||||
}
|
||||
return tx.Create(&imgs).Error
|
||||
})
|
||||
}
|
||||
|
||||
// StreetSnapIDByEntity 按实体键（city + year）查街拍正式表是否已存在（与 SaveStreetSnapFromDraft 晋升键一致）。
|
||||
func (r *ingestRepository) StreetSnapIDByEntity(ctx context.Context, city string, year uint16) (uint32, bool, error) {
|
||||
func (r *ingestRepository) StreetSnapEntityState(ctx context.Context, city string, year uint16) (uint32, string, bool, error) {
|
||||
var row struct {
|
||||
ID uint32 `gorm:"column:id"`
|
||||
ID uint32 `gorm:"column:id"`
|
||||
Status string `gorm:"column:status"`
|
||||
}
|
||||
err := r.db.WithContext(ctx).
|
||||
Model(&model.StreetSnap{}).
|
||||
Select("id").
|
||||
Select("id, status").
|
||||
Where("city = ? AND year = ? AND is_deleted = 0", city, year).
|
||||
Limit(1).
|
||||
Scan(&row).Error
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
return 0, "", false, err
|
||||
}
|
||||
if row.ID == 0 {
|
||||
return 0, false, nil
|
||||
return 0, "", false, nil
|
||||
}
|
||||
return row.ID, true, nil
|
||||
return row.ID, row.Status, true, nil
|
||||
}
|
||||
|
||||
func (r *ingestRepository) CreateStreetSnapDraft(ctx context.Context, d *model.StreetSnapDraft) (uint32, error) {
|
||||
func (r *ingestRepository) CreateStreetSnapWithImages(ctx context.Context, snap *model.StreetSnap, imgs []model.StreetSnapImage) (uint32, error) {
|
||||
now := uint32(time.Now().Unix())
|
||||
d.CreatedAt = now
|
||||
d.UpdatedAt = now
|
||||
if d.Status == "" {
|
||||
d.Status = model.DraftStatusPending
|
||||
snap.CreatedAt, snap.UpdatedAt = now, now
|
||||
if snap.Status == "" {
|
||||
snap.Status = model.StatusPending
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Create(d).Error; err != nil {
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if cErr := tx.Create(snap).Error; cErr != nil {
|
||||
return cErr
|
||||
}
|
||||
if len(imgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := range imgs {
|
||||
imgs[i].SnapID = snap.ID
|
||||
imgs[i].CreatedAt, imgs[i].UpdatedAt = now, now
|
||||
}
|
||||
return tx.Create(&imgs).Error
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return d.ID, nil
|
||||
return snap.ID, nil
|
||||
}
|
||||
|
||||
func (r *ingestRepository) CreateStreetSnapDraftImages(ctx context.Context, imgs []model.StreetSnapDraftImage) error {
|
||||
if len(imgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
func (r *ingestRepository) ReuseRejectedStreetSnap(ctx context.Context, id uint32, snap *model.StreetSnap, imgs []model.StreetSnapImage) error {
|
||||
now := uint32(time.Now().Unix())
|
||||
for i := range imgs {
|
||||
imgs[i].CreatedAt = now
|
||||
imgs[i].UpdatedAt = now
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(&imgs).Error
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if uErr := tx.Model(&model.StreetSnap{}).Where("id = ?", id).Updates(map[string]any{
|
||||
"title": snap.Title,
|
||||
"year": snap.Year,
|
||||
"city": snap.City,
|
||||
"cover": snap.Cover,
|
||||
"image_count": snap.ImageCount,
|
||||
"job_id": snap.JobID,
|
||||
"status": model.StatusPending,
|
||||
"reviewer": "",
|
||||
"reject_reason": "",
|
||||
"updated_at": now,
|
||||
}).Error; uErr != nil {
|
||||
return uErr
|
||||
}
|
||||
if dErr := tx.Model(&model.StreetSnapImage{}).
|
||||
Where("snap_id = ? AND is_deleted = 0", id).
|
||||
Updates(map[string]any{"is_deleted": 1, "updated_at": now}).Error; dErr != nil {
|
||||
return dErr
|
||||
}
|
||||
if len(imgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := range imgs {
|
||||
imgs[i].SnapID = id
|
||||
imgs[i].CreatedAt, imgs[i].UpdatedAt = now, now
|
||||
}
|
||||
return tx.Create(&imgs).Error
|
||||
})
|
||||
}
|
||||
|
||||
// ListJobs 按 id 倒序分页列出入库任务（后台监控页用）。offset/limit 控制分页区间。
|
||||
|
||||
Reference in New Issue
Block a user