fix: 爬虫页队列数被查询 LIMIT 截断(显示 199,实际 1620)
队列数字原本取 len(查询结果),而 ListActive 带 LIMIT 200: 1 个执行中 + 199 个排队刚好凑满,页面就永远显示 199, 看起来像「队列被卡住了、只有这么点任务」。 - 数字改用 SELECT count(*) GROUP BY status,不受 LIMIT 影响 - 表格仍只列前 50 个(渲染上千行会拖慢页面), 并在标题下注明「共 N 个,下面只列前 50 个」 - 渲染测试新增一条:截断时必须出现提示文案
This commit is contained in:
@ -74,7 +74,8 @@ func TestBackstageTemplatesRender(t *testing.T) {
|
|||||||
// 列表数据
|
// 列表数据
|
||||||
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
|
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
|
||||||
// 爬虫页(spider.html):状态条 + 正在抓 + 待抓队列 + 最近入库结果
|
// 爬虫页(spider.html):状态条 + 正在抓 + 待抓队列 + 最近入库结果
|
||||||
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0, "PendingIngest": int64(0),
|
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0,
|
||||||
|
"QueuedShown": 0, "QueuedPreview": 50, "PendingIngest": int64(0),
|
||||||
"Running": []spiderRunView{}, "Queued": []spiderQueueView{}, "Results": []spiderResultView{},
|
"Running": []spiderRunView{}, "Queued": []spiderQueueView{}, "Results": []spiderResultView{},
|
||||||
// 品牌页的「抓取状态 / 上次抓取」列(按 hashid 索引)
|
// 品牌页的「抓取状态 / 上次抓取」列(按 hashid 索引)
|
||||||
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,
|
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,
|
||||||
|
|||||||
@ -88,15 +88,29 @@ func (h *BackstageHandler) SpiderBoard(c *gin.Context) {
|
|||||||
pending = 0
|
pending = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 队列总数必须走 Counts(SELECT count(*) GROUP BY status),不能用 len(queues):
|
||||||
|
// ListActive 有 LIMIT,队列上千时 len 会永远停在 200 附近,
|
||||||
|
// 看起来像「一直抓不完」,其实是数字被截断了。
|
||||||
|
counts, _ := h.crawl.Counts(ctx)
|
||||||
|
|
||||||
|
// 表格只铺前 50 个,避免一次渲染几百行拖慢页面;真实总数用 count 展示。
|
||||||
|
const queuedPreview = 50
|
||||||
|
shown := queues
|
||||||
|
if len(shown) > queuedPreview {
|
||||||
|
shown = shown[:queuedPreview]
|
||||||
|
}
|
||||||
|
|
||||||
state, class := spiderState(active, now)
|
state, class := spiderState(active, now)
|
||||||
renderPage(c, http.StatusOK, "spider", gin.H{
|
renderPage(c, http.StatusOK, "spider", gin.H{
|
||||||
"State": state,
|
"State": state,
|
||||||
"StateClass": class,
|
"StateClass": class,
|
||||||
"RunningCount": len(runs),
|
"RunningCount": int(counts[model.CrawlRunning]),
|
||||||
"QueuedCount": len(queues),
|
"QueuedCount": int(counts[model.CrawlQueued]),
|
||||||
|
"QueuedShown": len(shown),
|
||||||
|
"QueuedPreview": queuedPreview,
|
||||||
"PendingIngest": pending,
|
"PendingIngest": pending,
|
||||||
"Running": runs,
|
"Running": runs,
|
||||||
"Queued": queues,
|
"Queued": shown,
|
||||||
"Results": results,
|
"Results": results,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -170,7 +184,7 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
|
|||||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "查询失败"})
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "查询失败"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
runs, queues := splitActive(active, now)
|
runs, _ := splitActive(active, now)
|
||||||
state, class := spiderState(active, now)
|
state, class := spiderState(active, now)
|
||||||
|
|
||||||
items := make([]gin.H, 0, len(runs))
|
items := make([]gin.H, 0, len(runs))
|
||||||
@ -188,11 +202,13 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
|
|||||||
if n, pErr := h.ingest.CountPendingJobs(ctx); pErr == nil {
|
if n, pErr := h.ingest.CountPendingJobs(ctx); pErr == nil {
|
||||||
pending = n
|
pending = n
|
||||||
}
|
}
|
||||||
|
// 同 SpiderBoard:队列数走 count(*),不要用 len(queues)(受 LIMIT 截断)。
|
||||||
|
counts, _ := h.crawl.Counts(ctx)
|
||||||
c.JSON(http.StatusOK, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"state": state,
|
"state": state,
|
||||||
"state_class": class,
|
"state_class": class,
|
||||||
"running": items,
|
"running": items,
|
||||||
"queued": len(queues),
|
"queued": counts[model.CrawlQueued],
|
||||||
"pending_ingest": pending,
|
"pending_ingest": pending,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@ -25,6 +25,8 @@ func TestSpiderPageRenders(t *testing.T) {
|
|||||||
"StateClass": "failed",
|
"StateClass": "failed",
|
||||||
"RunningCount": 1,
|
"RunningCount": 1,
|
||||||
"QueuedCount": 2,
|
"QueuedCount": 2,
|
||||||
|
"QueuedShown": 2,
|
||||||
|
"QueuedPreview": 50,
|
||||||
"PendingIngest": 3,
|
"PendingIngest": 3,
|
||||||
"Running": []spiderRunView{{
|
"Running": []spiderRunView{{
|
||||||
ID: 85,
|
ID: 85,
|
||||||
@ -73,4 +75,15 @@ func TestSpiderPageRenders(t *testing.T) {
|
|||||||
if !strings.Contains(out, `class="badge done"`) {
|
if !strings.Contains(out, `class="badge done"`) {
|
||||||
t.Error("正常结果没有 done 徽标")
|
t.Error("正常结果没有 done 徽标")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 队列被截断时要说清楚:否则「队列 1620 个」配一张 50 行的表会让人以为数据错了。
|
||||||
|
data["QueuedCount"] = 1620
|
||||||
|
data["QueuedShown"] = 50
|
||||||
|
var buf2 bytes.Buffer
|
||||||
|
if err := tpl.ExecuteTemplate(&buf2, "layout", data); err != nil {
|
||||||
|
t.Fatalf("渲染截断提示失败: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(buf2.String(), "只列前 50 个") {
|
||||||
|
t.Error("队列被截断时缺少提示")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -37,6 +37,7 @@
|
|||||||
</table>
|
</table>
|
||||||
|
|
||||||
<h2>待抓队列({{.QueuedCount}})</h2>
|
<h2>待抓队列({{.QueuedCount}})</h2>
|
||||||
|
{{if gt .QueuedCount .QueuedShown}}<p class="muted">队列里共 {{.QueuedCount}} 个,下面只列前 {{.QueuedShown}} 个。</p>{{end}}
|
||||||
<table>
|
<table>
|
||||||
<thead><tr><th>品牌</th><th>入队时间</th><th>已等待</th></tr></thead>
|
<thead><tr><th>品牌</th><th>入队时间</th><th>已等待</th></tr></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
|
|||||||
Reference in New Issue
Block a user