fix: 爬虫页队列数被查询 LIMIT 截断(显示 199,实际 1620)
队列数字原本取 len(查询结果),而 ListActive 带 LIMIT 200: 1 个执行中 + 199 个排队刚好凑满,页面就永远显示 199, 看起来像「队列被卡住了、只有这么点任务」。 - 数字改用 SELECT count(*) GROUP BY status,不受 LIMIT 影响 - 表格仍只列前 50 个(渲染上千行会拖慢页面), 并在标题下注明「共 N 个,下面只列前 50 个」 - 渲染测试新增一条:截断时必须出现提示文案
This commit is contained in:
@ -74,7 +74,8 @@ func TestBackstageTemplatesRender(t *testing.T) {
|
||||
// 列表数据
|
||||
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
|
||||
// 爬虫页(spider.html):状态条 + 正在抓 + 待抓队列 + 最近入库结果
|
||||
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0, "PendingIngest": int64(0),
|
||||
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0,
|
||||
"QueuedShown": 0, "QueuedPreview": 50, "PendingIngest": int64(0),
|
||||
"Running": []spiderRunView{}, "Queued": []spiderQueueView{}, "Results": []spiderResultView{},
|
||||
// 品牌页的「抓取状态 / 上次抓取」列(按 hashid 索引)
|
||||
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,
|
||||
|
||||
@ -88,16 +88,30 @@ func (h *BackstageHandler) SpiderBoard(c *gin.Context) {
|
||||
pending = 0
|
||||
}
|
||||
|
||||
// 队列总数必须走 Counts(SELECT count(*) GROUP BY status),不能用 len(queues):
|
||||
// ListActive 有 LIMIT,队列上千时 len 会永远停在 200 附近,
|
||||
// 看起来像「一直抓不完」,其实是数字被截断了。
|
||||
counts, _ := h.crawl.Counts(ctx)
|
||||
|
||||
// 表格只铺前 50 个,避免一次渲染几百行拖慢页面;真实总数用 count 展示。
|
||||
const queuedPreview = 50
|
||||
shown := queues
|
||||
if len(shown) > queuedPreview {
|
||||
shown = shown[:queuedPreview]
|
||||
}
|
||||
|
||||
state, class := spiderState(active, now)
|
||||
renderPage(c, http.StatusOK, "spider", gin.H{
|
||||
"State": state,
|
||||
"StateClass": class,
|
||||
"RunningCount": len(runs),
|
||||
"QueuedCount": len(queues),
|
||||
"State": state,
|
||||
"StateClass": class,
|
||||
"RunningCount": int(counts[model.CrawlRunning]),
|
||||
"QueuedCount": int(counts[model.CrawlQueued]),
|
||||
"QueuedShown": len(shown),
|
||||
"QueuedPreview": queuedPreview,
|
||||
"PendingIngest": pending,
|
||||
"Running": runs,
|
||||
"Queued": queues,
|
||||
"Results": results,
|
||||
"Running": runs,
|
||||
"Queued": shown,
|
||||
"Results": results,
|
||||
})
|
||||
}
|
||||
|
||||
@ -170,7 +184,7 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "查询失败"})
|
||||
return
|
||||
}
|
||||
runs, queues := splitActive(active, now)
|
||||
runs, _ := splitActive(active, now)
|
||||
state, class := spiderState(active, now)
|
||||
|
||||
items := make([]gin.H, 0, len(runs))
|
||||
@ -188,11 +202,13 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
|
||||
if n, pErr := h.ingest.CountPendingJobs(ctx); pErr == nil {
|
||||
pending = n
|
||||
}
|
||||
// 同 SpiderBoard:队列数走 count(*),不要用 len(queues)(受 LIMIT 截断)。
|
||||
counts, _ := h.crawl.Counts(ctx)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"state": state,
|
||||
"state_class": class,
|
||||
"running": items,
|
||||
"queued": len(queues),
|
||||
"queued": counts[model.CrawlQueued],
|
||||
"pending_ingest": pending,
|
||||
})
|
||||
}
|
||||
|
||||
@ -25,6 +25,8 @@ func TestSpiderPageRenders(t *testing.T) {
|
||||
"StateClass": "failed",
|
||||
"RunningCount": 1,
|
||||
"QueuedCount": 2,
|
||||
"QueuedShown": 2,
|
||||
"QueuedPreview": 50,
|
||||
"PendingIngest": 3,
|
||||
"Running": []spiderRunView{{
|
||||
ID: 85,
|
||||
@ -73,4 +75,15 @@ func TestSpiderPageRenders(t *testing.T) {
|
||||
if !strings.Contains(out, `class="badge done"`) {
|
||||
t.Error("正常结果没有 done 徽标")
|
||||
}
|
||||
|
||||
// 队列被截断时要说清楚:否则「队列 1620 个」配一张 50 行的表会让人以为数据错了。
|
||||
data["QueuedCount"] = 1620
|
||||
data["QueuedShown"] = 50
|
||||
var buf2 bytes.Buffer
|
||||
if err := tpl.ExecuteTemplate(&buf2, "layout", data); err != nil {
|
||||
t.Fatalf("渲染截断提示失败: %v", err)
|
||||
}
|
||||
if !strings.Contains(buf2.String(), "只列前 50 个") {
|
||||
t.Error("队列被截断时缺少提示")
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,6 +37,7 @@
|
||||
</table>
|
||||
|
||||
<h2>待抓队列({{.QueuedCount}})</h2>
|
||||
{{if gt .QueuedCount .QueuedShown}}<p class="muted">队列里共 {{.QueuedCount}} 个,下面只列前 {{.QueuedShown}} 个。</p>{{end}}
|
||||
<table>
|
||||
<thead><tr><th>品牌</th><th>入队时间</th><th>已等待</th></tr></thead>
|
||||
<tbody>
|
||||
|
||||
Reference in New Issue
Block a user