update
This commit is contained in:
toom1996
171
internal/ingest/client.go
Normal file
171
internal/ingest/client.go
Normal file
@ -0,0 +1,171 @@
|
||||
// Package ingest 爬虫 → 后台 ingest 管线的客户端。
|
||||
//
|
||||
// 设计（与 backend_v2/internal/middleware/ingest_auth.go + handler/ingest_handler.go 对齐）：
|
||||
// - spider 不再直连数据库：品牌任务经 GET /admin/internal/crawl/brands 拉取，
|
||||
// 抓取结果经 POST /admin/internal/ingest 上报，由后台 worker 异步下载图、补 season_code、写正式表。
|
||||
// - 两个接口都走 HMAC-SHA256 验签（X-Signature / X-Timestamp / X-Nonce），
|
||||
// 签名串 = HMAC_SHA256(secret, timestamp + "." + nonce + "." + bodyRaw)。
|
||||
// - 时间戳容忍窗口默认 ±300s；nonce 由后台一次性去重防重放。
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Kind 取值（与 backend_v2/internal/dto/ingest.go 一致）。
|
||||
const (
|
||||
KindRunway = "runway" // 走秀（默认，需 brand_uid）
|
||||
KindStreet = "street" // 街拍（无品牌，需 city/title）
|
||||
)
|
||||
|
||||
// RunwayIngest 上送后端的单场载荷（JSON 字段与 dto.RunwayIngest 完全对齐）。
|
||||
type RunwayIngest struct {
|
||||
Kind string `json:"kind"` // runway | street，缺省 runway
|
||||
BrandUID string `json:"brand_uid"` // 品牌编码 id（hashid），runway 必填
|
||||
TitleEn string `json:"title_en"` // 英文标题（runway 优先；street 作为单标题）
|
||||
TitleCn string `json:"title_cn"` // 中文标题（可空）
|
||||
DescriptionEn string `json:"description_en"` // 英文描述（可空）
|
||||
DescriptionCn string `json:"description_cn"` // 中文描述（可空）
|
||||
Year uint16 `json:"year"` // 年份，如 2026
|
||||
Season string `json:"season"` // spring / fall
|
||||
CollectionType string `json:"collection_type"` // rtw / menswear / couture / resort / pre_fall
|
||||
City string `json:"city"` // 地区/城市（street 用）
|
||||
Images []string `json:"images"` // 铺平的主图 URL 列表（回退用）
|
||||
Looks []RunwayLook `json:"looks,omitempty"` // 结构化「主图 + 细节图」分组（优先）
|
||||
}
|
||||
|
||||
// RunwayLook 一场秀中的一个 look：1 张主图 + 0..N 张细节图。
|
||||
type RunwayLook struct {
|
||||
Main string `json:"main"` // 主图（look）原始 URL
|
||||
Details []string `json:"details"` // 细节图原始 URL 列表
|
||||
}
|
||||
|
||||
// CrawlBrand 爬虫取任务接口返回的单个品牌。
|
||||
type CrawlBrand struct {
|
||||
BrandUID string `json:"brand_uid"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Client 入库管线客户端。
|
||||
type Client struct {
|
||||
endpoint string
|
||||
secret string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// NewClient 创建入库管线客户端。endpoint 形如 http://localhost:8092/admin/internal/ingest。
|
||||
func NewClient(endpoint, secret string) *Client {
|
||||
return &Client{
|
||||
endpoint: endpoint,
|
||||
secret: secret,
|
||||
http: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// Submit 把一场走秀/街拍元数据 HMAC 签名后 POST 到后台 ingest，返回 job_id（202 Accepted）。
|
||||
func (c *Client) Submit(ctx context.Context, p RunwayIngest) (string, error) {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("new request: %w", err)
|
||||
}
|
||||
c.signRequest(req, body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
return "", fmt.Errorf("ingest submit unexpected status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var out struct {
|
||||
JobID string `json:"job_id"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
_ = json.Unmarshal(respBody, &out)
|
||||
return out.JobID, nil
|
||||
}
|
||||
|
||||
// GetCrawlBrands 从后台拉取可抓取品牌任务（GET /admin/internal/crawl/brands）。
|
||||
// brandID>0 时仅返回该品牌（单品牌调试）。
|
||||
func (c *Client) GetCrawlBrands(ctx context.Context, brandID uint) ([]CrawlBrand, error) {
|
||||
url := strings.Replace(c.endpoint, "/ingest", "/crawl/brands", 1)
|
||||
if brandID > 0 {
|
||||
url += "?brand=" + strconv.FormatUint(uint64(brandID), 10)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new request: %w", err)
|
||||
}
|
||||
c.signRequest(req, nil)
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("crawl brands unexpected status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var out struct {
|
||||
Brands []CrawlBrand `json:"brands"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode crawl brands: %w", err)
|
||||
}
|
||||
return out.Brands, nil
|
||||
}
|
||||
|
||||
// signRequest 对请求体做 HMAC-SHA256 签名并写入验签头。body 为 nil 时按空串处理（GET）。
|
||||
func (c *Client) signRequest(req *http.Request, body []byte) {
|
||||
ts := strconv.FormatInt(time.Now().Unix(), 10)
|
||||
nonce := newNonce()
|
||||
sig := sign(c.secret, ts, nonce, string(body))
|
||||
|
||||
req.Header.Set("X-Signature", sig)
|
||||
req.Header.Set("X-Timestamp", ts)
|
||||
req.Header.Set("X-Nonce", nonce)
|
||||
}
|
||||
|
||||
// sign 复刻 backend_v2/internal/pkg/hmac.Sign：HMAC_SHA256(secret, ts + "." + nonce + "." + body)。
|
||||
func sign(secret, ts, nonce, body string) string {
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
mac.Write([]byte(ts))
|
||||
mac.Write([]byte("."))
|
||||
mac.Write([]byte(nonce))
|
||||
mac.Write([]byte("."))
|
||||
mac.Write([]byte(body))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// newNonce 生成 16 字节随机十六进制串作为一次性 nonce。
|
||||
func newNonce() string {
|
||||
b := make([]byte, 16)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
87
internal/ingest/hashid.go
Normal file
87
internal/ingest/hashid.go
Normal file
@ -0,0 +1,87 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// 本文件是 backend_v2/internal/pkg/hashid 中「无类型编码（品牌等）」的忠实移植，
|
||||
// 仅用于 spider 离线 Mock 模式下生成 brand_uid（真实上报路径下 brand_uid 直接来自后台接口，无需自编码）。
|
||||
// 算法：SHA256(盐) 派生 4 个子密钥 → 32-bit 平衡 Feistel 网络 → base62 定长(minLen=8) 序列化。
|
||||
// 默认盐与后台一致：hashid_secret 为空时后台走 "fashion-archive-default-salt-change-me"。
|
||||
|
||||
const (
|
||||
hidAlphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
hidBase = 62
|
||||
hidMinLen = 8
|
||||
)
|
||||
|
||||
var hidKeys [4]uint32
|
||||
|
||||
// InitHashID 用部署级盐值初始化混淆密钥。盐为空时使用与后台相同的内置默认盐。
|
||||
func InitHashID(secret string) {
|
||||
if secret == "" {
|
||||
secret = "fashion-archive-default-salt-change-me"
|
||||
}
|
||||
h := sha256.Sum256([]byte(secret))
|
||||
for i := 0; i < 4; i++ {
|
||||
hidKeys[i] = binary.BigEndian.Uint32(h[i*4 : i*4+4])
|
||||
}
|
||||
}
|
||||
|
||||
// feistel 32-bit 平衡 Feistel 网络（全局密钥版，品牌等无类型实体用）。
|
||||
func feistel(v uint32, encrypt bool) uint32 {
|
||||
return feistelK(v, encrypt, hidKeys)
|
||||
}
|
||||
|
||||
func feistelK(v uint32, encrypt bool, k [4]uint32) uint32 {
|
||||
const rounds = 8
|
||||
l, r := uint16(v>>16), uint16(v&0xffff)
|
||||
for i := 0; i < rounds; i++ {
|
||||
idx := i
|
||||
if !encrypt {
|
||||
idx = rounds - 1 - i
|
||||
}
|
||||
round := func(h uint16) uint16 {
|
||||
f := uint32(h)*0x9E3779B1 + k[idx%4]
|
||||
return uint16((f ^ (f >> 16)) & 0xffff)
|
||||
}
|
||||
if encrypt {
|
||||
nl := r
|
||||
nr := l ^ round(r)
|
||||
l, r = nl, nr
|
||||
} else {
|
||||
nl := r ^ round(l)
|
||||
nr := l
|
||||
l, r = nl, nr
|
||||
}
|
||||
}
|
||||
return uint32(l)<<16 | uint32(r)
|
||||
}
|
||||
|
||||
// numToBase62 把 32-bit 值序列化为定长（minLen）base62 串，高位在前。
|
||||
func numToBase62(x uint32) string {
|
||||
var sb strings.Builder
|
||||
for x > 0 {
|
||||
sb.WriteByte(hidAlphabet[x%hidBase])
|
||||
x /= hidBase
|
||||
}
|
||||
if sb.Len() == 0 {
|
||||
sb.WriteByte(hidAlphabet[0])
|
||||
}
|
||||
runes := []rune(sb.String())
|
||||
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
|
||||
runes[i], runes[j] = runes[j], runes[i]
|
||||
}
|
||||
out := string(runes)
|
||||
if len(out) < hidMinLen {
|
||||
out = strings.Repeat("0", hidMinLen-len(out)) + out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// EncodeBrand 把数字品牌主键编码为对外 hashid 串（与后台 hashid.Encode 同算法、同盐）。
|
||||
func EncodeBrand(id uint32) string {
|
||||
return numToBase62(feistel(id, true))
|
||||
}
|
||||
45
internal/ingest/parse.go
Normal file
45
internal/ingest/parse.go
Normal file
@ -0,0 +1,45 @@
|
||||
package ingest
|
||||
|
||||
import "strings"
|
||||
|
||||
// ParseCollection 从走秀标题解析出 collection_type 与 season。
|
||||
//
|
||||
// 仅输出后端白名单枚举（与 backend_v2/internal/pkg/season.Derive 一致）：
|
||||
// collection_type ∈ {rtw, menswear, couture, resort, pre_fall}
|
||||
// season ∈ {spring, fall}（resort/pre_fall 无 spring/fall 词，season 留空，由 Derive 按 collection 前缀补码）
|
||||
//
|
||||
// 常见 Vogue 标题形态：
|
||||
// "Fall 2024 Ready-to-Wear" → (rtw, fall)
|
||||
// "Spring 2024 Menswear" → (menswear, spring)
|
||||
// "Fall 2024 Couture" → (couture, fall)
|
||||
// "Resort 2024" → (resort, "")
|
||||
// "Pre-Fall 2024" → (pre_fall, "")
|
||||
// "Cruise" 视作 resort 的同义表述（Vogue 两词混用）；未识别到系列词时兜底为 rtw（成衣）。
|
||||
func ParseCollection(title string) (collectionType, season string) {
|
||||
t := strings.ToLower(title)
|
||||
|
||||
// season：仅 spring / fall 两种
|
||||
switch {
|
||||
case strings.Contains(t, "spring"):
|
||||
season = "spring"
|
||||
case strings.Contains(t, "fall"):
|
||||
season = "fall"
|
||||
}
|
||||
|
||||
// collection_type：按关键字优先级匹配，最后兜底 rtw
|
||||
switch {
|
||||
case strings.Contains(t, "menswear"), strings.Contains(t, "men's"):
|
||||
collectionType = "menswear"
|
||||
case strings.Contains(t, "couture"):
|
||||
collectionType = "couture"
|
||||
case strings.Contains(t, "resort"), strings.Contains(t, "cruise"):
|
||||
collectionType = "resort"
|
||||
case strings.Contains(t, "pre-fall"), strings.Contains(t, "pre fall"), strings.Contains(t, "prefall"):
|
||||
collectionType = "pre_fall"
|
||||
case strings.Contains(t, "ready-to-wear"), strings.Contains(t, "ready to wear"), strings.Contains(t, "rtw"):
|
||||
collectionType = "rtw"
|
||||
default:
|
||||
collectionType = "rtw"
|
||||
}
|
||||
return collectionType, season
|
||||
}
|
||||
Reference in New Issue
Block a user