3fe0b86e11
- media: spider 下载图片后按内容 sha1 生成 key，经后端预签名直传 S4，上报只带 key （spider 不持有 AK/SK；单张失败自动回退为上报 URL） - embed: 新增 embed 子命令，从后端领取缺向量的图片→下载→本地 CLIP 编码→回传 （媒体拆分后后端拿不到图片字节，向量只能靠回填补齐） - serve: 服务化常驻模式（领取 crawl 任务 → 抓取 → 回报） - config: 新增 media.upload/concurrency、vision.endpoint/concurrency
410 lines
12 KiB
Go
410 lines
12 KiB
Go
package ingest
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/rand"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
neturl "net/url"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// Client 向后台 ingest 接口发送 HMAC 签名上报的客户端。
|
||
type Client struct {
|
||
Endpoint string
|
||
Secret string
|
||
HTTPClient *http.Client
|
||
}
|
||
|
||
// NewClient 创建上报客户端。endpoint 形如 http://host:8092/admin/internal/ingest。
|
||
func NewClient(endpoint, secret string) *Client {
|
||
return &Client{
|
||
Endpoint: endpoint,
|
||
Secret: secret,
|
||
HTTPClient: &http.Client{Timeout: 30 * time.Second},
|
||
}
|
||
}
|
||
|
||
// Submit 把一条走秀上报签名后 POST 到后台，返回 job_id（202 入队即返回）。
|
||
// 失败返回 error（网络/校验/服务端拒绝），调用方应记日志并继续，不中断整轮抓取。
|
||
func (c *Client) Submit(ctx context.Context, p RunwayIngest) (uint32, error) {
|
||
if c == nil || c.Endpoint == "" {
|
||
return 0, fmt.Errorf("ingest client 未配置")
|
||
}
|
||
body, err := json.Marshal(p)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
ts := strconv.FormatInt(time.Now().Unix(), 10)
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
sig := Sign(c.Secret, ts, nonce, string(body))
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.Endpoint, bytes.NewReader(body))
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("X-Signature", sig)
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusAccepted {
|
||
return 0, fmt.Errorf("ingest http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
var out struct {
|
||
JobID uint32 `json:"job_id"`
|
||
Status string `json:"status"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
// 202 但响应结构非预期：任务已入队，记日志后当作成功
|
||
log.Printf("[Ingest] 响应解析失败但已被接受: %s", string(raw))
|
||
return 0, nil
|
||
}
|
||
return out.JobID, nil
|
||
}
|
||
|
||
// newNonce 生成 16 字节随机十六进制串，作为一次性 X-Nonce 防重放。
|
||
func newNonce() (string, error) {
|
||
b := make([]byte, 16)
|
||
if _, err := rand.Read(b); err != nil {
|
||
return "", err
|
||
}
|
||
return hex.EncodeToString(b), nil
|
||
}
|
||
|
||
// CrawlBrand 后端抓取任务接口返回的单个品牌（brand_uid 已编码，name 为英文名）。
|
||
type CrawlBrand struct {
|
||
BrandUID string `json:"brand_uid"`
|
||
Name string `json:"name"`
|
||
}
|
||
|
||
// crawlBrandsURL 由 ingest endpoint 推导抓取任务接口地址（把末尾 /ingest 换成 /crawl/brands）。
|
||
func (c *Client) crawlBrandsURL() string {
|
||
return c.internalBase() + "/crawl/brands"
|
||
}
|
||
|
||
// crawlTasksURL 服务化模式的任务接口：领取 / 回报。
|
||
func (c *Client) crawlTasksURL() string {
|
||
return c.internalBase() + "/crawl/tasks"
|
||
}
|
||
|
||
// internalBase 把 endpoint 的结尾 /ingest 去掉，得到内部接口的公共前缀。
|
||
func (c *Client) internalBase() string {
|
||
base := c.Endpoint
|
||
if i := strings.LastIndex(base, "/ingest"); i != -1 {
|
||
base = base[:i]
|
||
}
|
||
return strings.TrimRight(base, "/")
|
||
}
|
||
|
||
// UploadItem / UploadURL 媒体机直传用：向后端要一次性 PUT 地址（媒体机不持有 S4 密钥）。
|
||
type UploadItem struct {
|
||
Key string `json:"key"`
|
||
ContentType string `json:"content_type"`
|
||
}
|
||
|
||
// UploadURL 后端签发的预签名地址（有效期默认 15 分钟）。
|
||
type UploadURL struct {
|
||
Key string `json:"key"`
|
||
URL string `json:"url"`
|
||
}
|
||
|
||
// RequestUploadURLs 批量申请预签名 PUT 地址。单次上限 200（后端限制），调用方自行分片。
|
||
func (c *Client) RequestUploadURLs(ctx context.Context, items []UploadItem) ([]UploadURL, error) {
|
||
if c == nil || c.Endpoint == "" {
|
||
return nil, fmt.Errorf("ingest client 未配置")
|
||
}
|
||
if len(items) == 0 {
|
||
return nil, nil
|
||
}
|
||
body, err := json.Marshal(map[string]any{"items": items})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
ts := NowTS()
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
sig := Sign(c.Secret, ts, nonce, string(body))
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.internalBase()+"/upload-url", bytes.NewReader(body))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("X-Signature", sig)
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("upload-url http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
var out struct {
|
||
Items []UploadURL `json:"items"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
return nil, fmt.Errorf("upload-url 解析失败: %w", err)
|
||
}
|
||
return out.Items, nil
|
||
}
|
||
|
||
// PendingEmbedding 一张「缺向量」的图片（向量回填用）。
|
||
// URL 由后端拼好，worker 不需要知道存储域名怎么配。
|
||
type PendingEmbedding struct {
|
||
Kind string `json:"kind"`
|
||
ImageID uint32 `json:"image_id"`
|
||
Key string `json:"key"`
|
||
URL string `json:"url"`
|
||
}
|
||
|
||
// EmbeddingVector 回传成功编码的向量。
|
||
type EmbeddingVector struct {
|
||
Kind string `json:"kind"`
|
||
ImageID uint32 `json:"image_id"`
|
||
Vector []float32 `json:"vector"`
|
||
}
|
||
|
||
// EmbeddingFailure 回传失败项；后端累加计数，连续失败到上限后该图不再进队列。
|
||
type EmbeddingFailure struct {
|
||
Kind string `json:"kind"`
|
||
ImageID uint32 `json:"image_id"`
|
||
Error string `json:"error"`
|
||
}
|
||
|
||
// FetchPendingEmbeddings 拉一批缺向量的图片。remaining 为后端统计的剩余量（-1 = 统计失败）。
|
||
func (c *Client) FetchPendingEmbeddings(ctx context.Context, kind string, limit int) ([]PendingEmbedding, int, error) {
|
||
if c == nil || c.Endpoint == "" {
|
||
return nil, 0, fmt.Errorf("ingest client 未配置")
|
||
}
|
||
url := fmt.Sprintf("%s/embeddings/pending?kind=%s&limit=%d", c.internalBase(), neturl.QueryEscape(kind), limit)
|
||
ts := NowTS()
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return nil, 0, err
|
||
}
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return nil, 0, err
|
||
}
|
||
req.Header.Set("X-Signature", Sign(c.Secret, ts, nonce, ""))
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return nil, 0, err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusOK {
|
||
return nil, 0, fmt.Errorf("pending embeddings http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
var out struct {
|
||
Items []PendingEmbedding `json:"items"`
|
||
Remaining int `json:"remaining"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
return nil, 0, fmt.Errorf("pending embeddings 解析失败: %w", err)
|
||
}
|
||
return out.Items, out.Remaining, nil
|
||
}
|
||
|
||
// SubmitEmbeddings 回传向量与失败项，返回后端实际写入与失败计数。
|
||
func (c *Client) SubmitEmbeddings(ctx context.Context, items []EmbeddingVector, failures []EmbeddingFailure) (int, int, error) {
|
||
if c == nil || c.Endpoint == "" {
|
||
return 0, 0, fmt.Errorf("ingest client 未配置")
|
||
}
|
||
body, err := json.Marshal(map[string]any{"items": items, "failures": failures})
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
ts := NowTS()
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.internalBase()+"/embeddings", bytes.NewReader(body))
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("X-Signature", Sign(c.Secret, ts, nonce, string(body)))
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusOK {
|
||
return 0, 0, fmt.Errorf("submit embeddings http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
var out struct {
|
||
Saved int `json:"saved"`
|
||
Failed int `json:"failed"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
return 0, 0, fmt.Errorf("submit embeddings 解析失败: %w", err)
|
||
}
|
||
return out.Saved, out.Failed, nil
|
||
}
|
||
|
||
// CrawlTask 后台派发的品牌抓取任务（服务化模式）。
|
||
// brand_uid 是后端已编码好的串，直接原样上送 ingest 即可；brand_name 用于拼 Vogue 抓取 URL。
|
||
type CrawlTask struct {
|
||
ID uint32 `json:"id"`
|
||
BrandUID string `json:"brand_uid"`
|
||
BrandName string `json:"brand_name"`
|
||
MaxShows int16 `json:"max_shows"`
|
||
}
|
||
|
||
// FetchTask 领取一个抓取任务；队列为空时返回 nil, nil（调用方据此退避轮询）。
|
||
func (c *Client) FetchTask(ctx context.Context, worker string) (*CrawlTask, error) {
|
||
if c == nil || c.Endpoint == "" {
|
||
return nil, fmt.Errorf("ingest client 未配置")
|
||
}
|
||
url := c.crawlTasksURL() + "?worker=" + neturl.QueryEscape(worker)
|
||
ts := NowTS()
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
sig := Sign(c.Secret, ts, nonce, "")
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("X-Signature", sig)
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("fetch task http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
var out struct {
|
||
Task *CrawlTask `json:"task"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
return nil, fmt.Errorf("fetch task 解析失败: %w", err)
|
||
}
|
||
return out.Task, nil
|
||
}
|
||
|
||
// ReportTask 回报任务结果：status 取 done（执行完毕，哪怕 Vogue 上没有这个品牌）/ failed（进程级失败，后端会自动重试）。
|
||
func (c *Client) ReportTask(ctx context.Context, id uint32, status string, shows int, errMsg string) error {
|
||
if c == nil || c.Endpoint == "" {
|
||
return fmt.Errorf("ingest client 未配置")
|
||
}
|
||
body, err := json.Marshal(map[string]any{
|
||
"status": status,
|
||
"shows_submitted": shows,
|
||
"error": errMsg,
|
||
})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
url := fmt.Sprintf("%s/%d/result", c.crawlTasksURL(), id)
|
||
ts := NowTS()
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
sig := Sign(c.Secret, ts, nonce, string(body))
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("X-Signature", sig)
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusOK {
|
||
return fmt.Errorf("report task http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// GetCrawlBrands 拉取爬虫要抓取的品牌任务列表（HMAC 签名 GET，复用与上报相同的验签算法）。
|
||
// brandID>0 时只取该品牌（对应 --brand 单品牌调试）。返回 brand_uid（直接上送 ingest）+ 英文名。
|
||
func (c *Client) GetCrawlBrands(ctx context.Context, brandID uint) ([]CrawlBrand, error) {
|
||
if c == nil || c.Endpoint == "" {
|
||
return nil, fmt.Errorf("ingest client 未配置")
|
||
}
|
||
url := c.crawlBrandsURL()
|
||
if brandID > 0 {
|
||
url += fmt.Sprintf("?brand=%d", brandID)
|
||
}
|
||
// GET 无请求体，签名串的 body 部分为空（与后台 IngestAuth 验签一致）。
|
||
ts := NowTS()
|
||
nonce, err := newNonce()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
sig := Sign(c.Secret, ts, nonce, "")
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("X-Signature", sig)
|
||
req.Header.Set("X-Timestamp", ts)
|
||
req.Header.Set("X-Nonce", nonce)
|
||
|
||
resp, err := c.HTTPClient.Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("crawl brands http %d: %s", resp.StatusCode, string(raw))
|
||
}
|
||
var out struct {
|
||
Brands []CrawlBrand `json:"brands"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
return nil, fmt.Errorf("crawl brands 解析失败: %w", err)
|
||
}
|
||
return out.Brands, nil
|
||
}
|