From bc0fce4edde9f16c9049701be87cf29b9c60fcb1 Mon Sep 17 00:00:00 2001
From: toom1996 <23cm.cn@gmail.com>
Date: Tue, 27 Oct 2026 15:10:53 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E7=88=AC=E8=99=AB=E9=A1=B5=E9=98=9F?=
=?UTF-8?q?=E5=88=97=E6=95=B0=E8=A2=AB=E6=9F=A5=E8=AF=A2=20LIMIT=20?=
=?UTF-8?q?=E6=88=AA=E6=96=AD=EF=BC=88=E6=98=BE=E7=A4=BA=20199=EF=BC=8C?=
=?UTF-8?q?=E5=AE=9E=E9=99=85=201620=EF=BC=89?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
队列数字原本取 len(查询结果)，而 ListActive 带 LIMIT 200：
1 个执行中 + 199 个排队刚好凑满，页面就永远显示 199，
看起来像「队列被卡住了、只有这么点任务」。
- 数字改用 SELECT count(*) GROUP BY status，不受 LIMIT 影响
- 表格仍只列前 50 个（渲染上千行会拖慢页面），
并在标题下注明「共 N 个，下面只列前 50 个」
- 渲染测试新增一条：截断时必须出现提示文案
---
internal/handler/backstage_handler_test.go | 3 +-
internal/handler/backstage_spider.go | 34 ++++++++++++++------
internal/handler/spider_render_test.go | 13 ++++++++
internal/handler/templates/pages/spider.html | 1 +
4 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/internal/handler/backstage_handler_test.go b/internal/handler/backstage_handler_test.go
index 53a04c1..52ea593 100644
--- a/internal/handler/backstage_handler_test.go
+++ b/internal/handler/backstage_handler_test.go
@@ -74,7 +74,8 @@ func TestBackstageTemplatesRender(t *testing.T) {
// 列表数据
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
// 爬虫页（spider.html）：状态条 + 正在抓 + 待抓队列 + 最近入库结果
- "State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0, "PendingIngest": int64(0),
+ "State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0,
+ "QueuedShown": 0, "QueuedPreview": 50, "PendingIngest": int64(0),
"Running": []spiderRunView{}, "Queued": []spiderQueueView{}, "Results": []spiderResultView{},
// 品牌页的「抓取状态 / 上次抓取」列（按 hashid 索引）
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,
diff --git a/internal/handler/backstage_spider.go b/internal/handler/backstage_spider.go
index b092784..e46b83b 100644
--- a/internal/handler/backstage_spider.go
+++ b/internal/handler/backstage_spider.go
@@ -88,16 +88,30 @@ func (h *BackstageHandler) SpiderBoard(c *gin.Context) {
pending = 0
}
+ // 队列总数必须走 Counts（SELECT count(*) GROUP BY status），不能用 len(queues)：
+ // ListActive 有 LIMIT，队列上千时 len 会永远停在 200 附近，
+ // 看起来像「一直抓不完」，其实是数字被截断了。
+ counts, _ := h.crawl.Counts(ctx)
+
+ // 表格只铺前 50 个，避免一次渲染几百行拖慢页面；真实总数用 count 展示。
+ const queuedPreview = 50
+ shown := queues
+ if len(shown) > queuedPreview {
+ shown = shown[:queuedPreview]
+ }
+
state, class := spiderState(active, now)
renderPage(c, http.StatusOK, "spider", gin.H{
- "State": state,
- "StateClass": class,
- "RunningCount": len(runs),
- "QueuedCount": len(queues),
+ "State": state,
+ "StateClass": class,
+ "RunningCount": int(counts[model.CrawlRunning]),
+ "QueuedCount": int(counts[model.CrawlQueued]),
+ "QueuedShown": len(shown),
+ "QueuedPreview": queuedPreview,
"PendingIngest": pending,
- "Running": runs,
- "Queued": queues,
- "Results": results,
+ "Running": runs,
+ "Queued": shown,
+ "Results": results,
})
}
@@ -170,7 +184,7 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": "查询失败"})
return
}
- runs, queues := splitActive(active, now)
+ runs, _ := splitActive(active, now)
state, class := spiderState(active, now)
items := make([]gin.H, 0, len(runs))
@@ -188,11 +202,13 @@ func (h *BackstageHandler) SpiderLive(c *gin.Context) {
if n, pErr := h.ingest.CountPendingJobs(ctx); pErr == nil {
pending = n
}
+ // 同 SpiderBoard：队列数走 count(*)，不要用 len(queues)（受 LIMIT 截断）。
+ counts, _ := h.crawl.Counts(ctx)
c.JSON(http.StatusOK, gin.H{
"state": state,
"state_class": class,
"running": items,
- "queued": len(queues),
+ "queued": counts[model.CrawlQueued],
"pending_ingest": pending,
})
}
diff --git a/internal/handler/spider_render_test.go b/internal/handler/spider_render_test.go
index 921d467..6851f7a 100644
--- a/internal/handler/spider_render_test.go
+++ b/internal/handler/spider_render_test.go
@@ -25,6 +25,8 @@ func TestSpiderPageRenders(t *testing.T) {
"StateClass": "failed",
"RunningCount": 1,
"QueuedCount": 2,
+ "QueuedShown": 2,
+ "QueuedPreview": 50,
"PendingIngest": 3,
"Running": []spiderRunView{{
ID: 85,
@@ -73,4 +75,15 @@ func TestSpiderPageRenders(t *testing.T) {
if !strings.Contains(out, `class="badge done"`) {
t.Error("正常结果没有 done 徽标")
}
+
+ // 队列被截断时要说清楚：否则「队列 1620 个」配一张 50 行的表会让人以为数据错了。
+ data["QueuedCount"] = 1620
+ data["QueuedShown"] = 50
+ var buf2 bytes.Buffer
+ if err := tpl.ExecuteTemplate(&buf2, "layout", data); err != nil {
+ t.Fatalf("渲染截断提示失败: %v", err)
+ }
+ if !strings.Contains(buf2.String(), "只列前 50 个") {
+ t.Error("队列被截断时缺少提示")
+ }
}
diff --git a/internal/handler/templates/pages/spider.html b/internal/handler/templates/pages/spider.html
index 7bcb579..1301f55 100644
--- a/internal/handler/templates/pages/spider.html
+++ b/internal/handler/templates/pages/spider.html
@@ -37,6 +37,7 @@
待抓队列（{{.QueuedCount}}）
+{{if gt .QueuedCount .QueuedShown}}队列里共 {{.QueuedCount}} 个，下面只列前 {{.QueuedShown}} 个。
{{end}}