feat: 抓取任务队列 + 媒体机直传 + 向量回填
crawl（品牌抓取任务队列） - /admin/internal/crawl/tasks 领取 / :id/result 回报，spider 可常驻拉任务 - 后台品牌列表支持「加入抓取队列」，队列页可看状态与重试 - 租约超时的 running 自动回收为 queued，避免 worker 崩溃后任务卡死 media（媒体机直传） - ingest 载荷新增 main_key/detail_keys/main_size/detail_sizes： 有 key 就直接落库，跳过下载与上传；与 URL 形态可逐图混用 - 细节图按 Details/DetailKeys 两者较长者遍历（只给 key 时不再丢图） - 媒体机上传的 key 不进失败回滚清单（内容寻址、可能与他人共享） - /admin/internal/upload-url 签发 S4 PUT 预签名（spider 不持有 AK/SK） embeddings（向量回填） - GET /admin/internal/embeddings/pending 派发缺向量的图（含可直接下载的 url） - POST /admin/internal/embeddings 回写向量并重算近邻；失败项计入 image_embedding_failures，连续失败到上限后退出队列（坏图不占位） - 近邻重算改为异步串行：同步会让 100 张的批次 POST 超时（实测 >60s） - 新增 key 分支单测：混用 key/url、细节图只给 key、字节数、回滚清单、向量占位
This commit is contained in:
toom1996
347
internal/handler/backstage_crawl.go
Normal file
347
internal/handler/backstage_crawl.go
Normal file
@ -0,0 +1,347 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"fashionapi/internal/model"
|
||||
"fashionapi/internal/pkg/hashid"
|
||||
"fashionapi/internal/pkg/pagination"
|
||||
)
|
||||
|
||||
// 本文件收口「品牌抓取任务」的控制面：
|
||||
// - 后台页面：/admin/crawl（列表 + 批量入队 + 取消/重试/删除）
|
||||
// - 品牌页行内动作：/admin/crawl/brand/:uid（把单个品牌加入队列）
|
||||
// - spider 服务接口（HMAC 验签，不走 cookie）：
|
||||
// GET /admin/internal/crawl/tasks?worker=xxx → 领取一个任务
|
||||
// POST /admin/internal/crawl/tasks/:id/result → 回报执行结果
|
||||
//
|
||||
// 与入库任务（ingest_jobs）的分工见 model.CrawlTask 的注释。
|
||||
|
||||
// crawlRowView 列表展示结构（时间已格式化为字符串，模板里没有日期函数）。
|
||||
type crawlRowView struct {
|
||||
ID uint32
|
||||
BrandUID string
|
||||
BrandName string
|
||||
Status string
|
||||
StatusLabel string
|
||||
MaxShows int16
|
||||
Shows int32
|
||||
Attempts int16
|
||||
Worker string
|
||||
UpdatedAt string
|
||||
LastError string
|
||||
CanRetry bool
|
||||
CanCancel bool
|
||||
}
|
||||
|
||||
// crawlStatusClass 把抓取状态映射到后台已有的徽标配色类（admin.css 只有
|
||||
// pending / processing / done / failed 四套配色，这里做一次语义映射）。
|
||||
func crawlStatusClass(s string) string {
|
||||
switch s {
|
||||
case model.CrawlQueued:
|
||||
return "pending"
|
||||
case model.CrawlRunning:
|
||||
return "processing"
|
||||
case model.CrawlDone:
|
||||
return "done"
|
||||
case model.CrawlFailed:
|
||||
return "failed"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func crawlStatusLabel(s string) string {
|
||||
switch s {
|
||||
case model.CrawlQueued:
|
||||
return "待领取"
|
||||
case model.CrawlRunning:
|
||||
return "执行中"
|
||||
case model.CrawlDone:
|
||||
return "已完成"
|
||||
case model.CrawlFailed:
|
||||
return "失败"
|
||||
case model.CrawlCanceled:
|
||||
return "已取消"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// CrawlList 渲染抓取任务列表页。
|
||||
func (h *BackstageHandler) CrawlList(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": "抓取任务未装配（crawl repository 为空）"})
|
||||
return
|
||||
}
|
||||
p := pagination.Parse(c, "page", "pageSize", 1, 30, 200)
|
||||
rows, total, err := h.crawl.List(c.Request.Context(), p.Offset(), p.PageSize)
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
views := make([]crawlRowView, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
ua := "-"
|
||||
if r.UpdatedAt > 0 {
|
||||
ua = time.Unix(int64(r.UpdatedAt), 0).Format("2006-01-02 15:04")
|
||||
}
|
||||
views = append(views, crawlRowView{
|
||||
ID: r.ID,
|
||||
BrandUID: hashid.Encode(r.BrandID),
|
||||
BrandName: r.BrandName,
|
||||
Status: r.Status,
|
||||
StatusLabel: crawlStatusLabel(r.Status),
|
||||
MaxShows: r.MaxShows,
|
||||
Shows: r.ShowsSubmitted,
|
||||
Attempts: r.Attempts,
|
||||
Worker: r.Worker,
|
||||
UpdatedAt: ua,
|
||||
LastError: r.LastError,
|
||||
CanRetry: r.Status == model.CrawlFailed || r.Status == model.CrawlDone || r.Status == model.CrawlCanceled,
|
||||
CanCancel: r.Status == model.CrawlQueued || r.Status == model.CrawlRunning,
|
||||
})
|
||||
}
|
||||
|
||||
counts, _ := h.crawl.Counts(c.Request.Context())
|
||||
meta := p.NewMeta(total)
|
||||
renderPage(c, http.StatusOK, "crawl", gin.H{
|
||||
"Rows": views,
|
||||
"Total": total,
|
||||
"Pager": pagination.Pager{
|
||||
PrevURL: pagination.PageLinkWith(c, meta.PrevPage, "pageSize", p.PageSize),
|
||||
NextURL: pagination.PageLinkWith(c, meta.NextPage, "pageSize", p.PageSize),
|
||||
Label: fmt.Sprintf("第 %d / %d 页", meta.Page, meta.TotalPages),
|
||||
HasPrev: meta.HasPrev,
|
||||
HasNext: meta.HasNext,
|
||||
},
|
||||
"Queued": counts[model.CrawlQueued],
|
||||
"Running": counts[model.CrawlRunning],
|
||||
"Done": counts[model.CrawlDone],
|
||||
"Failed": counts[model.CrawlFailed],
|
||||
// 批量入队后回显「本次加入了多少个」
|
||||
"Added": atoiDefault(c.Query("added"), 0),
|
||||
})
|
||||
}
|
||||
|
||||
// CrawlEnqueue 把单个品牌加入抓取队列（品牌列表页的行内按钮）。
|
||||
// 品牌 id 用 hashid（与品牌页其它动作一致），避免暴露自增主键。
|
||||
func (h *BackstageHandler) CrawlEnqueue(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": "抓取任务未装配"})
|
||||
return
|
||||
}
|
||||
brandID, err := hashid.Decode(c.Param("uid"))
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "品牌 ID 非法"})
|
||||
return
|
||||
}
|
||||
maxShows := atoiDefault(c.PostForm("maxShows"), 0)
|
||||
if _, err := h.crawl.Enqueue(c.Request.Context(), brandID, maxShows, 10); err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.Redirect(http.StatusFound, c.PostForm("back"))
|
||||
}
|
||||
|
||||
// CrawlBulk 把「品牌管理页勾选的品牌」批量加入抓取队列。
|
||||
//
|
||||
// 这是批量操作的唯一入口（原先的「按品牌名前缀批量」已下线）：
|
||||
// 勾选哪几个就加哪几个，作用域明确，不会因为前缀写错而误抓几千个品牌。
|
||||
func (h *BackstageHandler) CrawlBulk(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": "抓取任务未装配"})
|
||||
return
|
||||
}
|
||||
uids := c.PostFormArray("uid")
|
||||
if len(uids) == 0 {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "没有勾选任何品牌"})
|
||||
return
|
||||
}
|
||||
maxShows := atoiDefault(c.PostForm("maxShows"), 3)
|
||||
|
||||
added := 0
|
||||
for _, uid := range uids {
|
||||
id, derr := hashid.Decode(strings.TrimSpace(uid))
|
||||
if derr != nil {
|
||||
continue
|
||||
}
|
||||
ok, err := h.crawl.Enqueue(c.Request.Context(), id, maxShows, 5)
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
added++
|
||||
}
|
||||
}
|
||||
c.Redirect(http.StatusFound, fmt.Sprintf("/admin/brands?added=%d", added))
|
||||
}
|
||||
|
||||
// CrawlRetry 把任务重置为待领取（失败 / 取消 / 已完成后想再跑一遍）。
|
||||
func (h *BackstageHandler) CrawlRetry(c *gin.Context) {
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id <= 0 || h.crawl == nil {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "任务 ID 非法"})
|
||||
return
|
||||
}
|
||||
if err := h.crawl.SetStatus(c.Request.Context(), uint32(id), model.CrawlQueued); err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.Redirect(http.StatusFound, "/admin/crawl")
|
||||
}
|
||||
|
||||
// CrawlCancel 取消任务（执行中的任务无法真正打断 spider，只保证不再被领取）。
|
||||
func (h *BackstageHandler) CrawlCancel(c *gin.Context) {
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id <= 0 || h.crawl == nil {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "任务 ID 非法"})
|
||||
return
|
||||
}
|
||||
if err := h.crawl.SetStatus(c.Request.Context(), uint32(id), model.CrawlCanceled); err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.Redirect(http.StatusFound, "/admin/crawl")
|
||||
}
|
||||
|
||||
// CrawlDelete 删除任务记录（已入库的数据不受影响）。
|
||||
func (h *BackstageHandler) CrawlDelete(c *gin.Context) {
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id <= 0 || h.crawl == nil {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "任务 ID 非法"})
|
||||
return
|
||||
}
|
||||
if err := h.crawl.Delete(c.Request.Context(), uint32(id)); err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.Redirect(http.StatusFound, "/admin/crawl")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// spider 服务接口（HMAC 验签，由 router 的 /admin/internal 组挂载）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// crawlTaskPayload 领取接口返回给 spider 的最小信息：品牌 uid（直接用于上报）+ 英文名（拼 Vogue URL）+ 场次上限。
|
||||
type crawlTaskPayload struct {
|
||||
ID uint32 `json:"id"`
|
||||
BrandUID string `json:"brand_uid"`
|
||||
BrandName string `json:"brand_name"`
|
||||
MaxShows int16 `json:"max_shows"`
|
||||
}
|
||||
|
||||
// WorkerTask 领取一个抓取任务；无任务时返回 {"task": null}（spider 据此退避轮询）。
|
||||
func (h *BackstageHandler) WorkerTask(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "crawl repository not configured"})
|
||||
return
|
||||
}
|
||||
worker := c.Query("worker")
|
||||
if worker == "" {
|
||||
worker = "spider"
|
||||
}
|
||||
row, err := h.crawl.Claim(c.Request.Context(), worker)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "claim failed"})
|
||||
return
|
||||
}
|
||||
if row == nil {
|
||||
c.JSON(http.StatusOK, gin.H{"task": nil})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"task": crawlTaskPayload{
|
||||
ID: row.ID,
|
||||
BrandUID: hashid.Encode(row.BrandID),
|
||||
BrandName: row.BrandName,
|
||||
MaxShows: row.MaxShows,
|
||||
}})
|
||||
}
|
||||
|
||||
// WorkerUploadURL 给媒体机签发 S4 直传 URL（PUT 预签名）。
|
||||
//
|
||||
// 设计取舍：不在 spider 里配 AK/SK、也不引入 S3 SDK —— 密钥只留在后端，
|
||||
// 媒体机拿到有时效的 URL 直接 PUT 即可。上传成功后把 key 一起上送 ingest。
|
||||
// 请求：{"items":[{"key":"runway/ab12.jpg","content_type":"image/jpeg"}]}
|
||||
// 响应：{"items":[{"key":"...","url":"https://..."}],"expires_in":900}
|
||||
func (h *BackstageHandler) WorkerUploadURL(c *gin.Context) {
|
||||
if h.uploader == nil || !h.uploader.Enabled() {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "S4 未启用，无法签发直传 URL"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Items []struct {
|
||||
Key string `json:"key"`
|
||||
ContentType string `json:"content_type"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if len(body.Items) == 0 || len(body.Items) > 200 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "items 数量应为 1~200"})
|
||||
return
|
||||
}
|
||||
|
||||
const ttl = 15 * time.Minute
|
||||
out := make([]gin.H, 0, len(body.Items))
|
||||
for _, it := range body.Items {
|
||||
// key 由调用方生成（内容 sha1），这里只做前缀与路径穿越校验，避免被拿来写任意对象。
|
||||
if !strings.HasPrefix(it.Key, "runway/") && !strings.HasPrefix(it.Key, "street/") {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "key 必须以 runway/ 或 street/ 开头: " + it.Key})
|
||||
return
|
||||
}
|
||||
if strings.Contains(it.Key, "..") {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "key 非法: " + it.Key})
|
||||
return
|
||||
}
|
||||
u, err := h.uploader.PresignPut(c.Request.Context(), it.Key, it.ContentType, ttl)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
out = append(out, gin.H{"key": it.Key, "url": u})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": out, "expires_in": int(ttl.Seconds())})
|
||||
}
|
||||
|
||||
// WorkerReport spider 回报执行结果。
|
||||
// status: done（执行完成，哪怕 Vogue 上没有该品牌的秀）/ failed（进程级失败，可重试）。
|
||||
func (h *BackstageHandler) WorkerReport(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "crawl repository not configured"})
|
||||
return
|
||||
}
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id <= 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid task id"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Status string `json:"status"`
|
||||
ShowsSubmitted int `json:"shows_submitted"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if body.Status != model.CrawlDone && body.Status != model.CrawlFailed {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "status must be done or failed"})
|
||||
return
|
||||
}
|
||||
if body.Error != "" && len(body.Error) > 1000 {
|
||||
body.Error = body.Error[:1000]
|
||||
}
|
||||
if err := h.crawl.Report(c.Request.Context(), uint32(id), body.Status, body.ShowsSubmitted, body.Error); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "report failed"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
170
internal/handler/backstage_embeddings.go
Normal file
170
internal/handler/backstage_embeddings.go
Normal file
@ -0,0 +1,170 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"fashionapi/internal/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// pairMu 串行化「近邻重算」：见 WorkerEmbeddings 里的说明。
|
||||
var pairMu sync.Mutex
|
||||
|
||||
// precomputePairsAsync 在后台重算近邻，失败只记日志（向量已写入，重算可由 similarbackfill 补）。
|
||||
func precomputePairsAsync(similar *service.SimilarService, kind string, ids []uint32) {
|
||||
pairMu.Lock()
|
||||
defer pairMu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
if err := similar.PrecomputeForGallery(ctx, kind, ids); err != nil {
|
||||
log.Printf("[similar] 回填后近邻重算失败 kind=%s 数量=%d err=%v", kind, len(ids), err)
|
||||
}
|
||||
}
|
||||
|
||||
// embeddingMaxAttempts 同一张图最多回填几次。
|
||||
//
|
||||
// 为什么必须有上限：坏图（文件损坏 / 格式不支持 / 已从存储删掉）是**永久失败**，
|
||||
// 没有计数的话它每轮都会重新出现在队列头部，把后面的好图永远挤在后面。
|
||||
// 5 次足够覆盖网络抖动与视觉服务重启，又不至于让坏图长期占位。
|
||||
const embeddingMaxAttempts = 5
|
||||
|
||||
// embeddingMaxDim 允许的最大向量维度（FashionSigLIP 是 768，留足余量防误配）。
|
||||
const embeddingMaxDim = 4096
|
||||
|
||||
// WorkerPendingEmbeddings 给回填 worker 派发「缺向量」的图片。
|
||||
//
|
||||
// GET /admin/internal/embeddings/pending?kind=runway&limit=32
|
||||
// → {"items":[{"kind":"runway","image_id":123,"key":"runway/ab.jpg","url":"https://..."}],
|
||||
// "remaining": 812}
|
||||
//
|
||||
// 返回 url 而不只是 key：worker 直接下载，不必知道存储域名怎么拼
|
||||
// （域名换了、换了存储、本地兜底，都在后端一处收敛）。
|
||||
func (h *BackstageHandler) WorkerPendingEmbeddings(c *gin.Context) {
|
||||
if h.emb == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "embedding repository not configured"})
|
||||
return
|
||||
}
|
||||
kind := c.DefaultQuery("kind", "runway")
|
||||
if kind != "runway" && kind != "street" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "kind 只能是 runway / street"})
|
||||
return
|
||||
}
|
||||
limit := 32
|
||||
if v := c.Query("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n <= 0 || n > 200 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "limit 应为 1~200"})
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
|
||||
rows, err := h.emb.ListPendingEmbeddings(c.Request.Context(), kind, limit, embeddingMaxAttempts)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "query pending failed: " + err.Error()})
|
||||
return
|
||||
}
|
||||
items := make([]gin.H, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
items = append(items, gin.H{
|
||||
"kind": r.Kind,
|
||||
"image_id": r.ImageID,
|
||||
"key": r.Key,
|
||||
"url": imgComposer.Compose(r.Key),
|
||||
})
|
||||
}
|
||||
// remaining 用独立 count：worker 靠它判断「还有没有活儿」，不必取到空批次才知道。
|
||||
remaining, err := h.emb.CountPendingEmbeddings(c.Request.Context(), kind, embeddingMaxAttempts)
|
||||
if err != nil {
|
||||
remaining = -1 // -1 = 统计失败，worker 应继续拉取后续批次
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items, "remaining": remaining})
|
||||
}
|
||||
|
||||
// WorkerEmbeddings 接收回填结果：成功的写向量并重算近邻，失败的累加失败计数。
|
||||
//
|
||||
// POST /admin/internal/embeddings
|
||||
// {"items":[{"kind":"runway","image_id":123,"vector":[...768 个数...]}],
|
||||
// "failures":[{"kind":"runway","image_id":124,"error":"下载 404"}]}
|
||||
//
|
||||
// 为什么近邻重算放在这里：image_similar_pairs 是入库时预计算的（页面只读结果）。
|
||||
// 回填补上的向量如果不重算，后台「疑似同款」就永远看不到新入库的图。
|
||||
func (h *BackstageHandler) WorkerEmbeddings(c *gin.Context) {
|
||||
if h.emb == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "embedding repository not configured"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Items []struct {
|
||||
Kind string `json:"kind"`
|
||||
ImageID uint32 `json:"image_id"`
|
||||
Vector []float32 `json:"vector"`
|
||||
} `json:"items"`
|
||||
Failures []struct {
|
||||
Kind string `json:"kind"`
|
||||
ImageID uint32 `json:"image_id"`
|
||||
Error string `json:"error"`
|
||||
} `json:"failures"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if len(body.Items) == 0 && len(body.Failures) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "items 与 failures 不能同时为空"})
|
||||
return
|
||||
}
|
||||
|
||||
// 按 kind 分组批量写：一次往返写一批，避免 32 张图 32 次事务。
|
||||
idsByKind := map[string][]uint32{}
|
||||
vecsByKind := map[string][][]float32{}
|
||||
failed := len(body.Failures)
|
||||
|
||||
for _, it := range body.Items {
|
||||
kind := strings.TrimSpace(it.Kind)
|
||||
if kind != "runway" && kind != "street" {
|
||||
failed++
|
||||
_ = h.emb.RecordEmbeddingFailure(c.Request.Context(), kind, it.ImageID, "kind 非法: "+it.Kind)
|
||||
continue
|
||||
}
|
||||
if n := len(it.Vector); n == 0 || n > embeddingMaxDim {
|
||||
// 维度异常一律拒写：写进去会污染向量列，让近邻检索结果全错。
|
||||
failed++
|
||||
_ = h.emb.RecordEmbeddingFailure(c.Request.Context(), kind, it.ImageID, "向量维度异常: "+strconv.Itoa(n))
|
||||
continue
|
||||
}
|
||||
idsByKind[kind] = append(idsByKind[kind], it.ImageID)
|
||||
vecsByKind[kind] = append(vecsByKind[kind], it.Vector)
|
||||
}
|
||||
|
||||
saved := 0
|
||||
for kind, ids := range idsByKind {
|
||||
if err := h.emb.BulkUpsertEmbedding(c.Request.Context(), kind, ids, vecsByKind[kind]); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "write embedding failed: " + err.Error()})
|
||||
return
|
||||
}
|
||||
saved += len(ids)
|
||||
if h.similar != nil {
|
||||
// 近邻重算**必须异步**：它要对每张图各跑一次向量近邻查询（20KB payload/次），
|
||||
// 一批 100 张就是 100 次往返。同步做会让 worker 的 POST 直接超时
|
||||
// （实测：库在远端时 100 张超过 60s），于是向量其实已经写入、worker 却判失败并重试。
|
||||
//
|
||||
// 用互斥串行而不是每请求一个 goroutine：多个回填 worker 并发时，
|
||||
// 并发跑近邻查询只会互相拖慢，还容易把连接池占满。
|
||||
go precomputePairsAsync(h.similar, kind, ids)
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range body.Failures {
|
||||
_ = h.emb.RecordEmbeddingFailure(c.Request.Context(), f.Kind, f.ImageID, f.Error)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"saved": saved, "failed": failed})
|
||||
}
|
||||
@ -16,6 +16,7 @@ import (
|
||||
"fashionapi/internal/pkg/hashid"
|
||||
"fashionapi/internal/pkg/pagination"
|
||||
"fashionapi/internal/pkg/season"
|
||||
"fashionapi/internal/pkg/storage"
|
||||
"fashionapi/internal/repository"
|
||||
"fashionapi/internal/service"
|
||||
|
||||
@ -45,8 +46,25 @@ type BackstageHandler struct {
|
||||
// manual 后台「手工新建文章」（可为 nil：未装配时新建入口提示不可用）。
|
||||
// 用 SetManual 注入，避免改构造签名（同 SetSimilar 的取舍）。
|
||||
manual *service.ManualArticleService
|
||||
// crawl 品牌抓取任务（crawl_tasks）。用 SetCrawl 注入，避免改构造签名。
|
||||
crawl repository.CrawlRepository
|
||||
// uploader S4 上传器（可为 nil：未配置 S4 时不提供预签名直传）。
|
||||
// 媒体机（spider）用它拿一次性 PUT URL，自己上传图片，不必持有 AK/SK。
|
||||
uploader *storage.S4Uploader
|
||||
// emb 向量仓储（可为 nil：未注入时回填接口返回 503）。
|
||||
// 媒体机直传后后端拿不到图片字节，向量只能靠回填 worker 从存储拉图编码再回传。
|
||||
emb repository.EmbeddingRepository
|
||||
}
|
||||
|
||||
// SetUploader 注入 S4 上传器（装配期调用），用于给媒体机签发直传 URL。
|
||||
func (h *BackstageHandler) SetUploader(up *storage.S4Uploader) { h.uploader = up }
|
||||
|
||||
// SetEmbeddingRepo 注入向量仓储（装配期调用），用于向量回填的取件与回写。
|
||||
func (h *BackstageHandler) SetEmbeddingRepo(repo repository.EmbeddingRepository) { h.emb = repo }
|
||||
|
||||
// SetCrawl 注入抓取任务仓储（装配期调用；未注入时 /admin/crawl 提示不可用）。
|
||||
func (h *BackstageHandler) SetCrawl(repo repository.CrawlRepository) { h.crawl = repo }
|
||||
|
||||
// SetManual 注入手工建稿服务（装配期调用；未注入时新建页提示不可用）。
|
||||
func (h *BackstageHandler) SetManual(svc *service.ManualArticleService) { h.manual = svc }
|
||||
|
||||
@ -145,6 +163,8 @@ func (h *BackstageHandler) Brands(c *gin.Context) {
|
||||
All: c.Query("all") == "1",
|
||||
Page: atoiDefault(c.Query("page"), 1),
|
||||
Locale: "en",
|
||||
// 后台可切换「按最近更新排序」（走秀或抓取任务的最新时间）
|
||||
Sort: c.Query("sort"),
|
||||
}
|
||||
items, total, err := h.brand.AdminList(c.Request.Context(), q)
|
||||
if err != nil {
|
||||
@ -157,13 +177,53 @@ func (h *BackstageHandler) Brands(c *gin.Context) {
|
||||
p.Page = 1
|
||||
}
|
||||
meta := p.NewMeta(total)
|
||||
|
||||
// 当前页品牌的数字 id：下面的抓取状态与数据量统计都要用。
|
||||
ids := make([]uint32, 0, len(items))
|
||||
for _, it := range items {
|
||||
if id, derr := hashid.Decode(it.UID); derr == nil {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
|
||||
// 装饰「抓取状态 / 上次抓取」：从 crawl_tasks 取每个品牌最近一条任务。
|
||||
// 没有任务记录的品牌不在 map 里，模板显示「—」（= 从没加入过抓取队列）。
|
||||
crawlStates := map[string]gin.H{}
|
||||
if h.crawl != nil {
|
||||
if m, lerr := h.crawl.BrandCrawlStates(c.Request.Context(), ids); lerr == nil {
|
||||
for id, st := range m {
|
||||
last := ""
|
||||
if !st.UpdatedAt.IsZero() {
|
||||
last = st.UpdatedAt.Format("2006-01-02 15:04")
|
||||
}
|
||||
crawlStates[hashid.Encode(id)] = gin.H{
|
||||
"Label": crawlStatusLabel(st.Status),
|
||||
"Class": crawlStatusClass(st.Status),
|
||||
"LastAt": last,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 数据量：图集数 / 图片数（只查当前页的 30 个品牌）
|
||||
stats := map[string]gin.H{}
|
||||
if m, serr := h.brand.StatsByIDs(c.Request.Context(), ids); serr == nil {
|
||||
for id, st := range m {
|
||||
stats[hashid.Encode(id)] = gin.H{"Runways": st.Runways, "Images": st.Images}
|
||||
}
|
||||
}
|
||||
|
||||
renderPage(c, http.StatusOK, "brands", gin.H{
|
||||
"Items": items,
|
||||
"Total": total,
|
||||
"Keyword": q.Keyword,
|
||||
"Letter": q.Letter,
|
||||
"All": q.All,
|
||||
"Letters": lettersAtoZ(),
|
||||
"Items": items,
|
||||
"Total": total,
|
||||
"Keyword": q.Keyword,
|
||||
"Letter": q.Letter,
|
||||
"All": q.All,
|
||||
"Sort": q.Sort,
|
||||
"Letters": lettersAtoZ(),
|
||||
"CrawlStates": crawlStates,
|
||||
"Stats": stats,
|
||||
"Added": atoiDefault(c.Query("added"), 0),
|
||||
"Pager": pagination.Pager{
|
||||
PrevURL: pagination.PageLink(c, meta.PrevPage),
|
||||
NextURL: pagination.PageLink(c, meta.NextPage),
|
||||
|
||||
@ -28,6 +28,7 @@ func TestBackstageTemplatesParsed(t *testing.T) {
|
||||
"articles",
|
||||
"brand-form",
|
||||
"brands",
|
||||
"crawl",
|
||||
"dashboard",
|
||||
"error",
|
||||
"ingest-jobs",
|
||||
@ -73,6 +74,10 @@ func TestBackstageTemplatesRender(t *testing.T) {
|
||||
"Images": []any{},
|
||||
// 列表数据
|
||||
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
|
||||
// 抓取任务页（crawl.html）：列表 + 四个状态计数
|
||||
"Rows": []any{}, "Queued": int64(0), "Running": int64(0), "Done": int64(0), "Failed": int64(0),
|
||||
// 品牌页的「抓取状态 / 上次抓取」列（按 hashid 索引）
|
||||
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,
|
||||
// 详情页实体（.Snap 在两个页面里类型不同，见下面的 perPage 覆盖）
|
||||
"Article": &dto.PublicArticleDetail{UID: "abc123"},
|
||||
"Snap": &dto.PublicStreetSnapDetail{UID: "abc123"},
|
||||
|
||||
@ -8,21 +8,69 @@
|
||||
<form class="toolbar" method="get" action="/admin/brands">
|
||||
<input name="keyword" placeholder="搜索品牌名…" value="{{.Keyword}}">
|
||||
{{if .All}}<input type="hidden" name="all" value="1">{{end}}
|
||||
{{if .Sort}}<input type="hidden" name="sort" value="{{.Sort}}">{{end}}
|
||||
<button type="submit">搜索</button>
|
||||
{{if .Keyword}}<a href="/admin/brands{{if .All}}?all=1{{end}}">清除</a>{{end}}
|
||||
<span class="spacer"></span>
|
||||
{{if .All}}<a class="view" href="/admin/brands?keyword={{.Keyword}}">只看有图集的品牌</a>{{else}}<a class="view" href="/admin/brands?keyword={{.Keyword}}&all=1">显示全部品牌（含无图集）</a>{{end}}
|
||||
</form>
|
||||
<p class="muted">默认只显示「已有图集」的品牌；品牌目录共 2 万余条，其余请用上面的搜索或开关查看。</p>
|
||||
|
||||
<p class="muted">
|
||||
默认只显示「已有图集」的品牌；品牌目录共 2 万余条，其余请用上面的搜索或开关查看。
|
||||
排序：
|
||||
{{if eq .Sort "updated"}}<strong>最近更新</strong>{{else}}<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}&sort=updated">最近更新</a>{{end}}
|
||||
·
|
||||
{{if eq .Sort "updated"}}<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}">按名字</a>{{else}}<strong>按名字</strong>{{end}}
|
||||
</p>
|
||||
|
||||
{{if .Added}}<p class="muted">✅ 本次已加入抓取队列 <strong>{{.Added}}</strong> 个品牌（已在队列中的不会重复添加）。</p>{{end}}
|
||||
|
||||
<div class="idx">
|
||||
<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}" class="{{if eq .Letter ""}}on{{end}}">全部</a>
|
||||
<a href="/admin/brands?letter=OTHER&keyword={{.Keyword}}{{if .All}}&all=1{{end}}" class="{{if eq .Letter "OTHER"}}on{{end}}">中文</a>
|
||||
{{range $l := .Letters}}<a href="/admin/brands?letter={{$l}}&keyword={{$.Keyword}}{{if $.All}}&all=1{{end}}" class="{{if eq $l $.Letter}}on{{end}}">{{$l}}</a>{{end}}
|
||||
<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}{{if .Sort}}&sort={{.Sort}}{{end}}" class="{{if eq .Letter ""}}on{{end}}">全部</a>
|
||||
<a href="/admin/brands?letter=OTHER&keyword={{.Keyword}}{{if .All}}&all=1{{end}}{{if .Sort}}&sort={{.Sort}}{{end}}" class="{{if eq .Letter "OTHER"}}on{{end}}">中文</a>
|
||||
{{range $l := .Letters}}<a href="/admin/brands?letter={{$l}}&keyword={{$.Keyword}}{{if $.All}}&all=1{{end}}{{if $.Sort}}&sort={{$.Sort}}{{end}}" class="{{if eq $l $.Letter}}on{{end}}">{{$l}}</a>{{end}}
|
||||
</div>
|
||||
|
||||
{{/* 批量操作表单：勾选框用 form="bulk" 关联，避免与行内表单嵌套 */}}
|
||||
<form id="bulk" method="post" action="/admin/crawl/bulk"></form>
|
||||
<div class="ops" style="margin:10px 0">
|
||||
<label class="muted">每品牌场次上限
|
||||
<select name="maxShows" form="bulk">
|
||||
<option value="1">1 场（最快探有无数据）</option>
|
||||
<option value="3" selected>3 场（最近，推荐）</option>
|
||||
<option value="0">全量（所有历史场次）</option>
|
||||
</select>
|
||||
</label>
|
||||
<button type="submit" form="bulk" class="op">把勾选的品牌加入抓取队列</button>
|
||||
<span class="muted">（勾选本页任意行，先点复选框列头可全选本页）</span>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<thead><tr><th>编码 ID</th><th>英文名</th><th>中文名</th><th>操作</th></tr></thead>
|
||||
<thead><tr>
|
||||
<th><input type="checkbox" title="全选本页" onclick="document.querySelectorAll('input[name=uid]').forEach(function(c){c.checked=this.checked}.bind(this))"></th>
|
||||
<th>编码 ID</th><th>英文名</th><th>中文名</th><th>图集</th><th>图片</th><th>抓取状态</th><th>上次抓取</th><th>操作</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{{range .Items}}<tr><td><code>{{.UID}}</code></td><td>{{.NameEn}}</td><td>{{.NameCn}}</td><td><a class="view" href="/admin/brands/{{.UID}}">编辑</a><form class="inline" method="post" action="/admin/brands/{{.UID}}/delete" onsubmit="return confirm('确认删除品牌「{{.NameEn}}」？该品牌下若还有图集会拒绝删除。')"><button type="submit" class="op op-warn">删除</button></form></td></tr>{{end}}
|
||||
{{range .Items}}{{$cs := index $.CrawlStates .UID}}{{$st := index $.Stats .UID}}<tr>
|
||||
<td><input type="checkbox" name="uid" value="{{.UID}}" form="bulk"></td>
|
||||
<td><code>{{.UID}}</code></td>
|
||||
<td>{{.NameEn}}</td>
|
||||
<td>{{.NameCn}}</td>
|
||||
<td>{{if $st}}{{$st.Runways}}{{else}}0{{end}}</td>
|
||||
<td>{{if $st}}{{$st.Images}}{{else}}0{{end}}</td>
|
||||
<td>{{if $cs}}<span class="badge {{$cs.Class}}">{{$cs.Label}}</span>{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td class="muted">{{if $cs}}{{$cs.LastAt}}{{else}}—{{end}}</td>
|
||||
<td>
|
||||
<a class="view" href="/admin/brands/{{.UID}}">编辑</a>
|
||||
<form class="inline" method="post" action="/admin/crawl/brands/{{.UID}}" onsubmit="return confirm('把「{{.NameEn}}」加入抓取队列？默认只抓最近 3 场。')">
|
||||
<input type="hidden" name="back" value="/admin/brands"><input type="hidden" name="maxShows" value="3">
|
||||
<button type="submit" class="op">加入队列</button>
|
||||
</form>
|
||||
<form class="inline" method="post" action="/admin/brands/{{.UID}}/delete" onsubmit="return confirm('确认删除品牌「{{.NameEn}}」？该品牌下若还有图集会拒绝删除。')">
|
||||
<button type="submit" class="op op-warn">删除</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{template "pager" .Pager}}
|
||||
|
||||
44
internal/handler/templates/pages/crawl.html
Normal file
44
internal/handler/templates/pages/crawl.html
Normal file
@ -0,0 +1,44 @@
|
||||
{{define "title"}}抓取任务 · 管理后台{{end}}
|
||||
{{define "barLabel"}}管理后台 · 品牌抓取任务{{end}}
|
||||
{{define "barRight"}}<a href="/admin/brands">去品牌管理加任务</a> · <a href="/admin/logout">退出</a>{{end}}
|
||||
{{define "content"}}
|
||||
{{template "bar" .}}
|
||||
<div class="main">
|
||||
<h1>抓取任务</h1>
|
||||
<p class="muted">
|
||||
任务是「从品牌管理页加入」的：勾选品牌 → 选场次上限 → 加入抓取队列。
|
||||
这里看的是执行状态与失败原因。真正干活的是 spider 服务（可在另一台机器）。
|
||||
</p>
|
||||
<p class="muted">
|
||||
待领取 {{.Queued}} · 执行中 {{.Running}} · 已完成 {{.Done}} · 失败 {{.Failed}}，共 {{.Total}} 条。
|
||||
<strong>「已提交 0 场」不是错误</strong>——Vogue 上很多品牌没有走秀页面（实测 A 开头里约 75%）。
|
||||
</p>
|
||||
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>ID</th><th>品牌</th><th>状态</th><th>场次上限</th><th>已提交</th><th>尝试</th><th>Worker</th><th>更新时间</th><th>错误</th><th>操作</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{{range .Rows}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td><a class="view" href="/admin/brands/{{.BrandUID}}">{{.BrandName}}</a></td>
|
||||
<td><span class="badge">{{.StatusLabel}}</span></td>
|
||||
<td>{{if eq .MaxShows 0}}<strong>全量</strong>{{else}}{{.MaxShows}}{{end}}</td>
|
||||
<td>{{.Shows}}</td>
|
||||
<td>{{.Attempts}}</td>
|
||||
<td class="muted">{{if .Worker}}{{.Worker}}{{else}}—{{end}}</td>
|
||||
<td class="muted">{{.UpdatedAt}}</td>
|
||||
<td class="err break">{{.LastError}}</td>
|
||||
<td>
|
||||
<div class="ops">
|
||||
{{if .CanRetry}}<form class="inline" method="post" action="/admin/crawl/tasks/{{.ID}}/retry"><button class="op" type="submit">重试</button></form>{{end}}
|
||||
{{if .CanCancel}}<form class="inline" method="post" action="/admin/crawl/tasks/{{.ID}}/cancel"><button class="op" type="submit">取消</button></form>{{end}}
|
||||
<form class="inline" method="post" action="/admin/crawl/tasks/{{.ID}}/delete" onsubmit="return confirm('删除任务记录？已入库的数据不受影响。')"><button class="op" type="submit">删除</button></form>
|
||||
</div>
|
||||
</td>
|
||||
</tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{template "pager" .Pager}}
|
||||
</div>
|
||||
{{end}}
|
||||
@ -18,6 +18,7 @@
|
||||
<a href="/admin/" class="{{if eq .NavKey "dashboard"}}on{{end}}">首页</a>
|
||||
<a href="/admin/articles" class="{{if or (eq .NavKey "articles") (eq .NavKey "runway-edit") (eq .NavKey "snap-edit") (eq .NavKey "runway-detail") (eq .NavKey "street-snap-detail")}}on{{end}}">文章{{if gt .PendingReviews 0}}<span class="bar-badge">{{.PendingReviews}}</span>{{end}}</a>
|
||||
<a href="/admin/brands" class="{{if or (eq .NavKey "brands") (eq .NavKey "brand-form")}}on{{end}}">品牌</a>
|
||||
<a href="/admin/crawl" class="{{if eq .NavKey "crawl"}}on{{end}}">抓取任务</a>
|
||||
<a href="/admin/ingest-jobs" class="{{if eq .NavKey "ingest-jobs"}}on{{end}}">入库任务</a>
|
||||
<a href="/admin/users" class="{{if eq .NavKey "users"}}on{{end}}">用户</a>
|
||||
<span class="sep"></span>
|
||||
|
||||
Reference in New Issue
Block a user