feat: 上报抓取进度(后台显示 3/25 场 · 正在抓什么)
一次品牌抓取可能跑几十分钟,此前后台只有 running/done 两个状态, 看不到「在干什么」,也分不清「慢」和「卡死」。 - getDetail 进/出一场各打一次点(含 404 等失败场次,避免进度永远差几场) - 总数在「预检过滤 + 场次上限」之后定格,与实际要抓的场次一致 - 只对 TaskID != 0(serve 模式)上报,单机 vogue 模式不上报
This commit is contained in:
@ -244,6 +244,44 @@ func (c *Client) CoveredShows(ctx context.Context, brandUID string) (map[string]
|
||||
return covered, nil
|
||||
}
|
||||
|
||||
// ReportProgress 上报执行进度(每处理完一场秀调用一次)。
|
||||
//
|
||||
// 失败只是观测信息丢失:调用方记日志即可,不要中断抓取。
|
||||
func (c *Client) ReportProgress(ctx context.Context, taskID uint32, done, total int, text string) error {
|
||||
if c == nil || c.Endpoint == "" || taskID == 0 {
|
||||
return nil
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{"done": done, "total": total, "text": text})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ts := NowTS()
|
||||
nonce, err := newNonce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
url := fmt.Sprintf("%s/crawl/tasks/%d/progress", c.internalBase(), taskID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Signature", Sign(c.Secret, ts, nonce, string(body)))
|
||||
req.Header.Set("X-Timestamp", ts)
|
||||
req.Header.Set("X-Nonce", nonce)
|
||||
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("progress http %d: %s", resp.StatusCode, string(raw))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShowCoverage 一场秀在库里的覆盖情况。
|
||||
type ShowCoverage struct {
|
||||
Year uint16 `json:"year"`
|
||||
|
||||
Reference in New Issue
Block a user