feat: 上报抓取进度(后台显示 3/25 场 · 正在抓什么)
一次品牌抓取可能跑几十分钟,此前后台只有 running/done 两个状态, 看不到「在干什么」,也分不清「慢」和「卡死」。 - getDetail 进/出一场各打一次点(含 404 等失败场次,避免进度永远差几场) - 总数在「预检过滤 + 场次上限」之后定格,与实际要抓的场次一致 - 只对 TaskID != 0(serve 模式)上报,单机 vogue 模式不上报
This commit is contained in:
@ -95,7 +95,7 @@ func runServeWorker(ctx context.Context, workerID string) {
|
||||
workerID, task.ID, task.BrandName, task.BrandUID, task.MaxShows)
|
||||
|
||||
start := time.Now()
|
||||
shows, errMsg := executeBrand(ctx, task.BrandUID, task.BrandName, int(task.MaxShows))
|
||||
shows, errMsg := executeBrand(ctx, task.ID, task.BrandUID, task.BrandName, int(task.MaxShows))
|
||||
elapsed := time.Since(start).Round(time.Second)
|
||||
|
||||
status := "done"
|
||||
@ -113,7 +113,7 @@ func runServeWorker(ctx context.Context, workerID string) {
|
||||
}
|
||||
|
||||
// executeBrand 执行单个品牌,异常(panic)也转成错误返回,避免一个坏品牌把整个 serve 进程带崩。
|
||||
func executeBrand(ctx context.Context, brandUID, brandName string, maxShows int) (shows int, errMsg string) {
|
||||
func executeBrand(ctx context.Context, taskID uint32, brandUID, brandName string, maxShows int) (shows int, errMsg string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
shows = 0
|
||||
@ -127,6 +127,8 @@ func executeBrand(ctx context.Context, brandUID, brandName string, maxShows int)
|
||||
// 默认跳过已入库的秀:图片 key 是内容 sha1,不下载无法判重,
|
||||
// 场次级预检是唯一能在下载前省掉整场带宽的手段。
|
||||
s.Force = forceAll
|
||||
// 任务 ID 用于回报执行进度(后台「进度」列)。
|
||||
s.TaskID = taskID
|
||||
// 媒体机直传:spider 自己下载并上传图片,后端只写库。
|
||||
if config.GlobalConfig.Media.Upload && IngestClient != nil {
|
||||
s.Media = media.New(IngestClient, "runway", config.GlobalConfig.Media.Concurrency)
|
||||
|
||||
Reference in New Issue
Block a user