fix: 爬虫页队列数被查询 LIMIT 截断(显示 199,实际 1620)

队列数字原本取 len(查询结果),而 ListActive 带 LIMIT 200:
1 个执行中 + 199 个排队刚好凑满,页面就永远显示 199,
看起来像「队列被卡住了、只有这么点任务」。

- 数字改用 SELECT count(*) GROUP BY status,不受 LIMIT 影响
- 表格仍只列前 50 个(渲染上千行会拖慢页面),
  并在标题下注明「共 N 个,下面只列前 50 个」
- 渲染测试新增一条:截断时必须出现提示文案
This commit is contained in:
toom1996
2026-10-27 15:10:53 +08:00
parent e930ac41c8
commit bc0fce4edd
4 changed files with 41 additions and 10 deletions

View File

@ -74,7 +74,8 @@ func TestBackstageTemplatesRender(t *testing.T) {
// 列表数据
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
// 爬虫页(spider.html):状态条 + 正在抓 + 待抓队列 + 最近入库结果
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0, "PendingIngest": int64(0),
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0,
"QueuedShown": 0, "QueuedPreview": 50, "PendingIngest": int64(0),
"Running": []spiderRunView{}, "Queued": []spiderQueueView{}, "Results": []spiderResultView{},
// 品牌页的「抓取状态 / 上次抓取」列(按 hashid 索引)
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,

View File

@ -88,15 +88,29 @@ func (h *BackstageHandler) SpiderBoard(c *gin.Context) {
pending = 0
}
// 队列总数必须走 Counts(SELECT count(*) GROUP BY status),不能用 len(queues):
// ListActive 有 LIMIT,队列上千时 len 会永远停在 200 附近,
// 看起来像「一直抓不完」,其实是数字被截断了。
counts, _ := h.crawl.Counts(ctx)
// 表格只铺前 50 个,避免一次渲染几百行拖慢页面;真实总数用 count 展示。
const queuedPreview = 50
shown := queues
if len(shown) > queuedPreview {
shown = shown[:queuedPreview]
}
state, class := spiderState(active, now)
renderPage(c, http.StatusOK, "spider", gin.H{
"State": state,
"StateClass": class,
"RunningCount": len(runs),
"QueuedCount": len(queues),
"RunningCount": int(counts[model.CrawlRunning]),
"QueuedCount": int(counts[model.CrawlQueued]),
"QueuedShown": len(shown),
"QueuedPreview": queuedPreview,
"PendingIngest": pending,
"Running": runs,
"Queued": queues,
"Queued": shown,
"Results": results,
})
}
@ -170,7 +184,7 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": "查询失败"})
return
}
runs, queues := splitActive(active, now)
runs, _ := splitActive(active, now)
state, class := spiderState(active, now)
items := make([]gin.H, 0, len(runs))
@ -188,11 +202,13 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
if n, pErr := h.ingest.CountPendingJobs(ctx); pErr == nil {
pending = n
}
// 同 SpiderBoard:队列数走 count(*),不要用 len(queues)(受 LIMIT 截断)。
counts, _ := h.crawl.Counts(ctx)
c.JSON(http.StatusOK, gin.H{
"state": state,
"state_class": class,
"running": items,
"queued": len(queues),
"queued": counts[model.CrawlQueued],
"pending_ingest": pending,
})
}

View File

@ -25,6 +25,8 @@ func TestSpiderPageRenders(t *testing.T) {
"StateClass": "failed",
"RunningCount": 1,
"QueuedCount": 2,
"QueuedShown": 2,
"QueuedPreview": 50,
"PendingIngest": 3,
"Running": []spiderRunView{{
ID: 85,
@ -73,4 +75,15 @@ func TestSpiderPageRenders(t *testing.T) {
if !strings.Contains(out, `class="badge done"`) {
t.Error("正常结果没有 done 徽标")
}
// 队列被截断时要说清楚:否则「队列 1620 个」配一张 50 行的表会让人以为数据错了。
data["QueuedCount"] = 1620
data["QueuedShown"] = 50
var buf2 bytes.Buffer
if err := tpl.ExecuteTemplate(&buf2, "layout", data); err != nil {
t.Fatalf("渲染截断提示失败: %v", err)
}
if !strings.Contains(buf2.String(), "只列前 50 个") {
t.Error("队列被截断时缺少提示")
}
}

View File

@ -37,6 +37,7 @@
</table>
<h2>待抓队列({{.QueuedCount}})</h2>
{{if gt .QueuedCount .QueuedShown}}<p class="muted">队列里共 {{.QueuedCount}} 个,下面只列前 {{.QueuedShown}} 个。</p>{{end}}
<table>
<thead><tr><th>品牌</th><th>入队时间</th><th>已等待</th></tr></thead>
<tbody>