This commit is contained in:
toom1996
2026-09-29 19:59:30 +08:00
parent d15d2a4701
commit 361ffc01af
28 changed files with 896 additions and 47 deletions

View File

@ -22,6 +22,8 @@ type Config struct {
Upload UploadConfig `yaml:"upload"`
CORS CORSConfig `yaml:"cors"`
ClientSign ClientSignConfig `yaml:"client_sign"`
// RateLimit 按客户端 IP 的速率限制(兜底防刷,与 client_sign 配合)。
RateLimit RateLimitConfig `yaml:"rate_limit"`
// Ingest 爬虫上报接口的 HMAC 验签配置(服务端到服务端,密钥不下发前端)。
Ingest IngestConfig `yaml:"ingest"`
// S4 缤纷云对象存储配置(S3 兼容,爬虫入库图片上传目标)。
@ -171,6 +173,13 @@ type ClientSignConfig struct {
TTLSeconds int `yaml:"ttl_seconds"`
}
// RateLimitConfig 按客户端 IP 的速率限制配置(令牌桶)。
type RateLimitConfig struct {
Enabled bool `yaml:"enabled"`
RPS int `yaml:"rps"` // 每秒补充令牌数(平均允许 QPS)
Burst int `yaml:"burst"` // 突发容量(瞬间允许的最大请求数)
}
// defaultConfig 返回内置默认值,保证 yml 缺字段时服务仍可启动。
func defaultConfig() *Config {
return &Config{
@ -215,6 +224,11 @@ func defaultConfig() *Config {
Secret: "",
TTLSeconds: 30,
},
RateLimit: RateLimitConfig{
Enabled: false,
RPS: 20,
Burst: 40,
},
Ingest: IngestConfig{
Secret: "",
TTLSeconds: 300,
@ -332,6 +346,20 @@ func (c *Config) applyEnv() {
envStr("CLIENT_SIGN_SECRET", &c.ClientSign.Secret)
envInt("CLIENT_SIGN_TTL", &c.ClientSign.TTLSeconds)
// CORS 允许的来源(逗号分隔),覆盖 yml 的 allow_origins,便于容器注入具体域名收紧跨域。
if v := strings.TrimSpace(os.Getenv("CORS_ALLOW_ORIGINS")); v != "" {
parts := strings.Split(v, ",")
c.CORS.AllowOrigins = make([]string, 0, len(parts))
for _, p := range parts {
if p = strings.TrimSpace(p); p != "" {
c.CORS.AllowOrigins = append(c.CORS.AllowOrigins, p)
}
}
}
envBool("RATE_LIMIT_ENABLED", &c.RateLimit.Enabled)
envInt("RATE_LIMIT_RPS", &c.RateLimit.RPS)
envInt("RATE_LIMIT_BURST", &c.RateLimit.Burst)
envStr("UPLOAD_DIR", &c.Upload.Dir)
envStr("UPLOAD_URL_PREFIX", &c.Upload.URLPrefix)
@ -383,6 +411,12 @@ func (c *Config) normalize() {
if c.JWT.RefreshExpireHours <= 0 {
c.JWT.RefreshExpireHours = 720
}
if c.RateLimit.RPS <= 0 {
c.RateLimit.RPS = 20
}
if c.RateLimit.Burst <= 0 {
c.RateLimit.Burst = 40
}
if c.Upload.Dir == "" {
c.Upload.Dir = "./uploads"
}

212
internal/config/prompt.go Normal file
View File

@ -0,0 +1,212 @@
package config
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/goccy/go-yaml"
"github.com/mattn/go-isatty"
)
// requiredSpec 描述一个「必填 / 推荐」敏感配置项,以及取值、赋值与回写路径。
//
// 解析顺序:环境变量 > config.yml > config.local.yml > 交互提示(仅 TTY)> 非 TTY 缺失则 fatal。
// 这样关键配置无需硬编码进仓库:仓库里的 config.yml 把这些项留空,
// 本地开发在终端里交互填写(自动写回 config.local.yml,已被 .gitignore 忽略),
// 容器 / CI 等无终端环境则通过环境变量注入,缺失必填项直接报错退出。
type requiredSpec struct {
Name string // 展示名
EnvKey string // 对应环境变量名(非 TTY 报错提示 + 文档)
Secret bool // 是否为密钥(仅影响提示文案)
Optional bool // true=缺失只告警不阻断;false=缺失且非 TTY 时 fatal
Get func(c *Config) string
Set func(c *Config, v string)
YAMLPath []string // 写回 config.local.yml 的嵌套路径
Cond func(c *Config) bool // 是否参与校验(如 S4 启用才要求 ak/sk)
}
func requiredSpecs() []requiredSpec {
return []requiredSpec{
{
Name: "JWT 签名密钥",
EnvKey: "JWT_SECRET",
Secret: true,
Optional: false,
Get: func(c *Config) string { return c.JWT.Secret },
Set: func(c *Config, v string) { c.JWT.Secret = v },
YAMLPath: []string{"jwt", "secret"},
},
{
Name: "数据库密码",
EnvKey: "DB_PASSWORD",
Secret: true,
Optional: false,
Get: func(c *Config) string { return c.Database.Password },
Set: func(c *Config, v string) { c.Database.Password = v },
YAMLPath: []string{"database", "password"},
},
{
Name: "爬虫上报 HMAC 密钥",
EnvKey: "INGEST_SECRET",
Secret: true,
Optional: true,
Get: func(c *Config) string { return c.Ingest.Secret },
Set: func(c *Config, v string) { c.Ingest.Secret = v },
YAMLPath: []string{"ingest", "secret"},
},
{
Name: "公开 ID 混淆盐",
EnvKey: "HASHID_SECRET",
Secret: true,
Optional: true,
Get: func(c *Config) string { return c.Server.HashIDSecret },
Set: func(c *Config, v string) { c.Server.HashIDSecret = v },
YAMLPath: []string{"server", "hashid_secret"},
},
{
Name: "S4 对象存储 AccessKey",
EnvKey: "S4_AK",
Secret: true,
Optional: false,
Cond: func(c *Config) bool { return c.S4.Enabled },
Get: func(c *Config) string { return c.S4.AK },
Set: func(c *Config, v string) { c.S4.AK = v },
YAMLPath: []string{"s4", "ak"},
},
{
Name: "S4 对象存储 SecretKey",
EnvKey: "S4_SK",
Secret: true,
Optional: false,
Cond: func(c *Config) bool { return c.S4.Enabled },
Get: func(c *Config) string { return c.S4.SK },
Set: func(c *Config, v string) { c.S4.SK = v },
YAMLPath: []string{"s4", "sk"},
},
}
}
// PromptMissing 在配置加载完成后,对缺失的必填/推荐敏感配置做交互式补填。
// 交互补填的值会写回 config.local.yml(已被 .gitignore 忽略),下次启动免填。
// 非交互环境(容器 / CI,无终端输入)下,缺失的必填项直接返回 error,由调用方 fatal 退出,
// 并打印应注入的环境变量名。
func PromptMissing(cfg *Config) error {
var missing []requiredSpec
for _, s := range requiredSpecs() {
if s.Cond != nil && !s.Cond(cfg) {
continue
}
if strings.TrimSpace(s.Get(cfg)) == "" {
missing = append(missing, s)
}
}
if len(missing) == 0 {
return nil
}
if !isTerminal(os.Stdin) {
var fatalLines, warnLines []string
for _, s := range missing {
if s.Optional {
warnLines = append(warnLines, fmt.Sprintf(" ! %s 未设置(建议通过环境变量 %s 注入)", s.Name, s.EnvKey))
} else {
fatalLines = append(fatalLines, fmt.Sprintf(" - %s(请设置环境变量 %s)", s.Name, s.EnvKey))
}
}
for _, l := range warnLines {
fmt.Println(l)
}
if len(fatalLines) > 0 {
return fmt.Errorf("缺少必填配置,无法启动:\n%s", strings.Join(fatalLines, "\n"))
}
return nil
}
fmt.Println("以下必填/推荐配置缺失,请交互式填写(将写入 config.local.yml,已被 .gitignore 忽略):")
reader := bufio.NewReader(os.Stdin)
changed := map[string]string{}
for _, s := range missing {
val, err := promptLine(reader, s)
if err != nil {
return err
}
s.Set(cfg, val)
changed[strings.Join(s.YAMLPath, ".")] = val
}
return writeLocalOverlay(changed)
}
func promptLine(r *bufio.Reader, s requiredSpec) (string, error) {
tag := ""
if s.Secret {
tag = "(密钥,输入不会回显保存,请自行保密)"
}
fmt.Printf(" %s [%s]%s: ", s.Name, s.EnvKey, tag)
val, err := r.ReadString('\n')
if err != nil && val == "" {
return "", fmt.Errorf("读取 %s 失败: %w", s.Name, err)
}
val = strings.TrimRight(val, "\r\n")
if strings.TrimSpace(val) == "" {
return "", fmt.Errorf("%s 不能为空", s.Name)
}
return val, nil
}
// writeLocalOverlay 把交互补填的值合并写回 config.local.yml(已 gitignore)。
// 若文件已存在,保留其中未被覆盖的其它键(如 download_proxy)。
func writeLocalOverlay(changed map[string]string) error {
candidates := []string{
"configs/config.local.yml",
"config.local.yml",
filepath.Join("..", "..", "configs", "config.local.yml"),
}
target := candidates[0]
var m map[string]interface{}
for _, p := range candidates {
data, err := os.ReadFile(p)
if err == nil {
_ = yaml.Unmarshal(data, &m)
target = p
break
}
}
if m == nil {
m = map[string]interface{}{}
}
for dotted, val := range changed {
setNested(m, strings.Split(dotted, "."), val)
}
out, err := yaml.Marshal(m)
if err != nil {
return fmt.Errorf("序列化 config.local.yml 失败: %w", err)
}
if err := os.WriteFile(target, out, 0o600); err != nil {
return fmt.Errorf("写入 %s 失败: %w", target, err)
}
fmt.Printf("✓ 已写入本地私有配置: %s(已被 .gitignore 忽略,不会进版本库)\n", target)
return nil
}
func setNested(m map[string]interface{}, path []string, val string) {
for i := 0; i < len(path)-1; i++ {
k := path[i]
child, ok := m[k].(map[string]interface{})
if !ok {
child = map[string]interface{}{}
m[k] = child
}
m = child
}
m[path[len(path)-1]] = val
}
// isTerminal 判断文件描述符是否为真实终端,用于区分交互式启动与容器/CI 无输入环境。
// 使用 mattn/go-isatty 而非 Stat().Mode()&ModeCharDevice,因为后者会把 /dev/null、管道
// 等字符设备误判为终端,导致容器里错误地进入交互提示分支。
func isTerminal(f *os.File) bool {
return isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd())
}

View File

@ -249,7 +249,8 @@ func (h *BackstageHandler) RunwayDetail(c *gin.Context) {
renderPage(c, http.StatusNotFound, "error", gin.H{"Error": "走秀不存在或已删除: " + raw})
return
}
renderPage(c, http.StatusOK, "runway-detail", gin.H{"Article": art})
total, _ := h.article.SumImageSize(c.Request.Context(), numeric)
renderPage(c, http.StatusOK, "runway-detail", gin.H{"Article": art, "ImageTotalSize": total})
}
// StreetSnaps 街拍管理列表:按年份/城市/排序筛选,复用 StreetSnapService.AdminList(读基表,含待审)。
@ -305,7 +306,8 @@ func (h *BackstageHandler) StreetSnapDetail(c *gin.Context) {
renderPage(c, http.StatusNotFound, "error", gin.H{"Error": "街拍不存在或已删除: " + raw})
return
}
renderPage(c, http.StatusOK, "street-snap-detail", gin.H{"Snap": snap})
total, _ := h.street.SumImageSize(c.Request.Context(), numeric)
renderPage(c, http.StatusOK, "street-snap-detail", gin.H{"Snap": snap, "ImageTotalSize": total})
}
// ── 文章管理(走秀 + 街拍混合列表,替代原审核页与分类型列表)─────────────
@ -327,6 +329,8 @@ func (h *BackstageHandler) Articles(c *gin.Context) {
}
// 批量统计每篇图集的重复率(基于 is_duplicate 列,阈值 phash.DefaultThreshold=4)。
dupStats := h.dupStatsByCards(c.Request.Context(), cards)
// 批量统计每篇图集的原图总大小(基于 image_size 列,worker 下载时写入源文件字节数)。
imageSizes := h.imageSizeByCards(c.Request.Context(), cards)
items := make([]gin.H, 0, len(cards))
for _, card := range cards {
var t byte
@ -347,6 +351,7 @@ func (h *BackstageHandler) Articles(c *gin.Context) {
"Title": card.Title,
"Subtitle": card.Subtitle,
"ImageCount": card.ImageCount,
"ImageTotalSize": imageSizes[card.ID],
"Status": card.Status,
"DupCount": ds.Dup,
"DupPct": dupPct,
@ -407,6 +412,36 @@ func (h *BackstageHandler) dupStatsByCards(ctx context.Context, cards []reposito
return out
}
// imageSizeByCards 按卡片类型批量取每篇图集的原图字节数之和(image_size 列)。
// 按 kind 分别查对应图片表,避免跨表混算;查询失败时该 kind 整体回落为「无大小」(不阻断列表)。
func (h *BackstageHandler) imageSizeByCards(ctx context.Context, cards []repository.AdminRecordCard) map[uint32]int64 {
out := make(map[uint32]int64)
runwayIDs := make([]uint32, 0, len(cards))
streetIDs := make([]uint32, 0, len(cards))
for _, c := range cards {
if c.Kind == "runway" {
runwayIDs = append(runwayIDs, c.ID)
} else {
streetIDs = append(streetIDs, c.ID)
}
}
if len(runwayIDs) > 0 {
if m, err := h.adminRepo.ImageSizeByRecords(ctx, "runway", runwayIDs); err == nil {
for id, s := range m {
out[id] = s
}
}
}
if len(streetIDs) > 0 {
if m, err := h.adminRepo.ImageSizeByRecords(ctx, "street", streetIDs); err == nil {
for id, s := range m {
out[id] = s
}
}
}
return out
}
// ArticleApprove 通过:把记录状态置为已发布(单表模型:仅改状态,不重建图片)。
// 路由标识用 hashid,与编辑页一致;已发布 / 已驳回记录调用仍是幂等的。
func (h *BackstageHandler) ArticleApprove(c *gin.Context) {
@ -684,12 +719,17 @@ func (h *BackstageHandler) RunwayEdit(c *gin.Context) {
dupPct = float64(dupCount) / float64(n) * 100
}
dupSources := h.buildDupSources(c.Request.Context(), "runway", cells)
var totalSize int64
for _, im := range imgs {
totalSize += im.ImageSize
}
renderPage(c, http.StatusOK, "runway-edit", gin.H{
"UID": raw,
"Kind": "runway",
"RecordID": rw.ID,
"Status": rw.Status,
"Runway": rw,
"ImageTotalSize": totalSize,
"Images": imgs,
"Cells": cells,
"BrandName": brandName,
@ -869,12 +909,17 @@ func (h *BackstageHandler) StreetSnapEdit(c *gin.Context) {
dupPct = float64(dupCount) / float64(n) * 100
}
dupSources := h.buildDupSources(c.Request.Context(), "street", cells)
var totalSize int64
for _, im := range imgs {
totalSize += im.ImageSize
}
renderPage(c, http.StatusOK, "snap-edit", gin.H{
"UID": raw,
"Kind": "street",
"RecordID": snap.ID,
"Status": snap.Status,
"Snap": snap,
"ImageTotalSize": totalSize,
"Images": imgs,
"Cells": cells,
"IsDown": snap.IsDeleted == 1,

View File

@ -19,6 +19,7 @@ import (
"bytes"
"context"
"embed"
"fmt"
"html/template"
"io/fs"
"net/http"
@ -41,6 +42,25 @@ func SetImgComposer(c *imgurl.Composer) {
}
}
// formatBytes 把字节数格式化为人类可读串(B/KB/MB/GB/TB);0 或负数返回「—」。
func formatBytes(n int64) string {
if n <= 0 {
return "—"
}
const unit = 1024
if n < unit {
return fmt.Sprintf("%d B", n)
}
val := float64(n)
for _, u := range []string{"KB", "MB", "GB", "TB"} {
val /= unit
if val < unit {
return fmt.Sprintf("%.1f %s", val, u)
}
}
return fmt.Sprintf("%.1f PB", val)
}
//go:embed templates/layout.html templates/partials.html templates/pages/*.html assets/*
var backstageFS embed.FS
@ -60,6 +80,7 @@ func init() {
// imgThumb 同但走列表缩略图样式(小尺寸场景省流量)。两者兜底逻辑见 imgurl.Composer。
"imgURL": func(stored string) string { return imgComposer.Compose(stored) },
"imgThumb": func(stored string) string { return imgComposer.ComposeThumb(stored) },
"formatBytes": formatBytes,
}
base, err := template.New("layout.html").Funcs(funcs).

View File

@ -21,7 +21,7 @@
<table>
<thead><tr>
<th style="width:30px"><input type="checkbox" id="sel-all" aria-label="全选本页"></th>
<th>类型</th><th>ID</th><th>标题</th><th>副标题</th><th>图数</th><th>重复率</th><th>状态</th><th>操作</th>
<th>类型</th><th>ID</th><th>标题</th><th>副标题</th><th>图数</th><th>原图总大小</th><th>重复率</th><th>状态</th><th>操作</th>
</tr></thead>
<tbody>
{{range .Items}}<tr data-kind="{{.Kind}}" data-id="{{.ID}}">
@ -30,7 +30,7 @@
<td><code>{{.ID}}</code></td>
<td>{{.Title}}</td>
<td class="muted">{{.Subtitle}}</td>
<td>{{.ImageCount}}</td>
<td>{{.ImageCount}}</td><td>{{formatBytes .ImageTotalSize}}</td>
<td>{{if .DupCount}}<span class="badge dup">{{printf "%.0f" .DupPct}}%({{.DupCount}})</span>{{else}}<span class="muted">—</span>{{end}}</td>
<td><span class="badge {{.Status}}">{{.Status}}</span></td>
<td>

View File

@ -6,6 +6,7 @@
<div class="main">
<p class="muted">编码 {{.Article.UID}} · 品牌 {{.Article.BrandName}}</p>
<h1>{{.Article.Title}}</h1>
<p class="muted">原图总大小:{{formatBytes .ImageTotalSize}}</p>
<div class="grid">
{{range .Article.Images}}<div class="cell"><button class="thumb" type="button" data-zoom><img src="{{imgURL .Image}}" loading="lazy"></button><span>{{.Name}}</span></div>{{range .Detail}}<div class="cell det"><button class="thumb" type="button" data-zoom><img src="{{imgURL .Image}}" loading="lazy"></button><span>{{.Name}} · 副图</span></div>{{end}}{{end}}
</div>

View File

@ -5,6 +5,7 @@
{{template "bar" .}}
<div class="main">
<p class="muted">品牌 {{.BrandName}}{{if .IsDown}} · <span class="down">已下架</span>{{end}}</p>
<p class="muted">原图总大小:{{formatBytes .ImageTotalSize}}</p>
<form method="post" action="/admin/runways/{{.UID}}/edit">
<div class="row"><div><label>英文标题</label><input name="title_en" value="{{.Runway.TitleEn}}"></div><div><label>中文标题</label><input name="title_cn" value="{{.Runway.TitleCn}}"></div></div>
<div class="row"><div><label>英文描述</label><textarea name="description_en">{{.Runway.DescriptionEn}}</textarea></div><div><label>中文描述</label><textarea name="description_cn">{{.Runway.DescriptionCn}}</textarea></div></div>

View File

@ -5,6 +5,7 @@
{{template "bar" .}}
<div class="main">
<p class="muted">{{if .IsDown}}<span class="down">已下架</span>{{end}}</p>
<p class="muted">原图总大小:{{formatBytes .ImageTotalSize}}</p>
<form method="post" action="/admin/street-snaps/{{.UID}}/edit">
<div class="row"><div><label>英文标题</label><input name="title" value="{{.Snap.Title}}"></div><div><label>中文标题</label><input name="title_cn" value="{{.Snap.TitleCn}}"></div></div>
<div class="row"><div><label>年份</label><input name="year" value="{{.Snap.Year}}"></div><div><label>城市</label><input name="city" value="{{.Snap.City}}"></div></div>

View File

@ -6,6 +6,7 @@
<div class="main">
<p class="muted">编码 {{.Snap.UID}}</p>
<h1>{{.Snap.Title}}</h1>
<p class="muted">原图总大小:{{formatBytes .ImageTotalSize}}</p>
<div class="grid">
{{range .Snap.Images}}<div class="cell"><button class="thumb" type="button" data-zoom><img src="{{imgURL .Image}}" loading="lazy"></button><span>{{.Name}}</span></div>{{range .Detail}}<div class="cell det"><button class="thumb" type="button" data-zoom><img src="{{imgURL .Image}}" loading="lazy"></button><span>{{.Name}} · 副图</span></div>{{end}}{{end}}
</div>

View File

@ -0,0 +1,25 @@
package handler
import "testing"
// TestFormatBytes 校验后台「图集原图总大小」的字节格式化(0/负数为「—」)。
func TestFormatBytes(t *testing.T) {
cases := []struct {
n int64
want string
}{
{0, "—"},
{-1, "—"},
{500, "500 B"},
{1024, "1.0 KB"},
{1536, "1.5 KB"},
{1024 * 1024, "1.0 MB"},
{int64(1.5 * 1024 * 1024), "1.5 MB"},
{int64(1.5 * 1024 * 1024 * 1024), "1.5 GB"},
}
for _, c := range cases {
if got := formatBytes(c.n); got != c.want {
t.Errorf("formatBytes(%d) = %q, want %q", c.n, got, c.want)
}
}
}

View File

@ -0,0 +1,53 @@
package middleware
import (
"time"
"github.com/gin-gonic/gin"
"log/slog"
)
// GinLogger 用结构化日志(slog,由 main 设为 JSON handler)记录每个请求,
// 替代 gin 默认的 text logger,便于生产环境集中采集与检索。
func GinLogger() gin.HandlerFunc {
return func(c *gin.Context) {
start := time.Now()
path := c.Request.URL.Path
raw := c.Request.URL.RawQuery
c.Next()
// 跳过健康检查等高频噪声路径
if path == "/api/health" || path == "/api/v1/health" {
return
}
latency := time.Since(start)
status := c.Writer.Status()
clientIP := c.ClientIP()
method := c.Request.Method
if raw != "" {
path = path + "?" + raw
}
attrs := []slog.Attr{
slog.String("method", method),
slog.String("path", path),
slog.Int("status", status),
slog.String("ip", clientIP),
slog.Duration("latency", latency),
}
if len(c.Errors) > 0 {
attrs = append(attrs, slog.String("error", c.Errors.ByType(gin.ErrorTypePrivate).String()))
}
switch {
case status >= 500:
slog.LogAttrs(c.Request.Context(), slog.LevelError, "request", attrs...)
case status >= 400:
slog.LogAttrs(c.Request.Context(), slog.LevelWarn, "request", attrs...)
default:
slog.LogAttrs(c.Request.Context(), slog.LevelInfo, "request", attrs...)
}
}
}

View File

@ -0,0 +1,97 @@
package middleware
import (
"net/http"
"sync"
"time"
"fashionapi/internal/config"
"github.com/gin-gonic/gin"
)
// rateLimiter 基于令牌桶的按 IP 限流(进程内,单实例足够;多实例需换集中式如 redis)。
//
// 设计取舍:公开接口「首页公开、翻页需登录」已天然抬高批量枚举门槛;本中间件作为
// 兜底,按客户端 IP 限制总量,兜住 casual 爬虫与突发流量,避免单 IP 打垮服务。
type rateLimiter struct {
mu sync.Mutex
visitors map[string]*visitor
rate float64 // 每秒补充令牌数
burst float64 // 桶容量
lastSeen map[string]time.Time
}
type visitor struct {
tokens float64
last time.Time
}
func newRateLimiter(rps, burst int) *rateLimiter {
rl := &rateLimiter{
visitors: make(map[string]*visitor),
lastSeen: make(map[string]time.Time),
rate: float64(rps),
burst: float64(burst),
}
go rl.cleanup(10 * time.Minute)
return rl
}
// cleanup 周期清理长时间无活动的客户端,避免 map 无限增长。
func (rl *rateLimiter) cleanup(interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
rl.mu.Lock()
for ip, t := range rl.lastSeen {
if time.Since(t) > 3*interval {
delete(rl.visitors, ip)
delete(rl.lastSeen, ip)
}
}
rl.mu.Unlock()
}
}
func (rl *rateLimiter) allow(ip string) bool {
rl.mu.Lock()
defer rl.mu.Unlock()
now := time.Now()
v, ok := rl.visitors[ip]
if !ok {
// 首次访问直接发放 burst-1 个令牌(允许一个突发请求),后续按速率补充
rl.visitors[ip] = &visitor{tokens: rl.burst - 1, last: now}
rl.lastSeen[ip] = now
return true
}
elapsed := now.Sub(v.last).Seconds()
v.tokens += elapsed * rl.rate
if v.tokens > rl.burst {
v.tokens = rl.burst
}
v.last = now
rl.lastSeen[ip] = now
if v.tokens >= 1 {
v.tokens--
return true
}
return false
}
// RateLimit 按 IP 的速率限制中间件。Enabled=false 时为 noop(便于灰度与本地开发)。
func RateLimit(cfg config.RateLimitConfig) gin.HandlerFunc {
if !cfg.Enabled || cfg.RPS <= 0 {
return func(c *gin.Context) { c.Next() }
}
rl := newRateLimiter(cfg.RPS, cfg.Burst)
return func(c *gin.Context) {
if !rl.allow(c.ClientIP()) {
c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{
"error": "too many requests",
})
return
}
c.Next()
}
}

View File

@ -20,6 +20,10 @@ type BrandRunwayImage struct {
// =1 为细节图,parent_image_id 指向同 runway 内主图的行 id。Look 序号对产品无意义,已移除。
ParentImageID uint32 `gorm:"column:parent_image_id;not null;default:0" json:"parent_image_id"`
// 原图大小(字节):worker 下载图片时记录源文件字节数,供后台「图集原图总大小」展示。
// 历史已入库图片为 0(未重爬、无源数据),后台按「—」处理;重爬后自动补齐。
ImageSize int64 `gorm:"column:image_size;not null;default:0" json:"image_size"`
// 去重字段(近重复 + 语义 embedding 留痕):
// phash 为 dHash 的 pgvector 二进制向量(vector(64)),NULL 表示无法解码(如 webp),不参与检索;
// is_duplicate / dup_of 仅在命中近重复时标记,供人工审核留痕,不阻断入库。

View File

@ -50,6 +50,9 @@ type StreetSnapImage struct {
// is_detail=0 为主图(parent_image_id=0);=1 为副图,parent_image_id 指向同专辑内主图的行 id。
IsDetail uint8 `gorm:"column:is_detail;not null;default:0" json:"is_detail"`
ParentImageID uint32 `gorm:"column:parent_image_id;not null;default:0" json:"parent_image_id"`
// 原图大小(字节):同 BrandRunwayImage.ImageSize(见该字段注释)。
ImageSize int64 `gorm:"column:image_size;not null;default:0" json:"image_size"`
}
// TableName 指定图片明细表名。

View File

@ -39,6 +39,9 @@ type AdminRepository interface {
// DupRateByRecords 批量统计若干记录的图片去重情况(仅存活行)。
// 返回 owner id → {Total 存活图片总数, Dup is_duplicate=1 张数};不存在的 id 不出现在 map 中。
DupRateByRecords(ctx context.Context, kind string, ids []uint32) (map[uint32]DupStat, error)
// ImageSizeByRecords 批量返回若干记录的原图字节数之和(按 kind 分别查对应图片表);
// 返回 owner id → 字节数。供文章管理混合列表展示「图集原图总大小」。
ImageSizeByRecords(ctx context.Context, kind string, ids []uint32) (map[uint32]int64, error)
}
type adminRepository struct {
@ -148,6 +151,41 @@ func (r *adminRepository) DupRateByRecords(ctx context.Context, kind string, ids
return out, nil
}
// ImageSizeByRecords 批量返回若干记录的原图字节数之和(按 kind 分别查对应图片表)。
// 返回 owner id → 字节数;不存在的 id 不出现在 map 中。供文章管理混合列表展示「图集原图总大小」。
func (r *adminRepository) ImageSizeByRecords(ctx context.Context, kind string, ids []uint32) (map[uint32]int64, error) {
out := make(map[uint32]int64)
if len(ids) == 0 {
return out, nil
}
var table, fk string
switch kind {
case "runway":
table, fk = "brand_runway_images", "runway_id"
case "street":
table, fk = "street_snap_images", "snap_id"
default:
return out, nil
}
type agg struct {
Owner uint32 `gorm:"column:owner"`
Total int64 `gorm:"column:total"`
}
var rows []agg
if err := r.db.WithContext(ctx).
Table(table).
Select(fk + " AS owner, COALESCE(SUM(image_size), 0) AS total").
Where(fk+" IN ? AND is_deleted = 0", ids).
Group(fk).
Scan(&rows).Error; err != nil {
return nil, err
}
for _, row := range rows {
out[row.Owner] = row.Total
}
return out, nil
}
func (r *adminRepository) CountMixed(ctx context.Context, status, kind string) (int64, error) {
rw, rargs := legWhere("r", status)
sw, sargs := legWhere("s", status)

View File

@ -52,6 +52,8 @@ type ArticleRepository interface {
SetRunwayCover(ctx context.Context, runwayID uint32, image string) error
// SetRunwayDeleted 上下架:deleted=1 下架(对外不可见),0 上架。
SetRunwayDeleted(ctx context.Context, id uint32, deleted uint8) error
// SumImageSize 返回某条走秀全部存活图片的原图字节数之和(供后台「图集原图总大小」展示)。
SumImageSize(ctx context.Context, runwayID uint32) (int64, error)
}
type articleRepository struct {
@ -565,3 +567,14 @@ func (r *articleRepository) SetRunwayDeleted(ctx context.Context, id uint32, del
Where("id = ?", id).
Updates(map[string]any{"is_deleted": deleted, "updated_at": uint32(time.Now().Unix())}).Error
}
// SumImageSize 返回某条走秀全部存活图片的原图字节数之和(image_size 由 worker 下载时写入源文件字节数)。
func (r *articleRepository) SumImageSize(ctx context.Context, runwayID uint32) (int64, error) {
var total int64
err := r.db.WithContext(ctx).
Model(&model.BrandRunwayImage{}).
Where("runway_id = ? AND is_deleted = 0", runwayID).
Select("COALESCE(SUM(image_size), 0)").
Scan(&total).Error
return total, err
}

View File

@ -54,6 +54,8 @@ type StreetSnapRepository interface {
SetSnapCover(ctx context.Context, snapID uint32, image string) error
// SetSnapDeleted 上下架:deleted=1 下架,0 上架。
SetSnapDeleted(ctx context.Context, id uint32, deleted uint8) error
// SumImageSize 返回某条街拍全部存活图片的原图字节数之和(供后台「图集原图总大小」展示)。
SumImageSize(ctx context.Context, snapID uint32) (int64, error)
}
type streetSnapRepository struct {
@ -498,3 +500,14 @@ func (r *streetSnapRepository) SetSnapDeleted(ctx context.Context, id uint32, de
Where("id = ?", id).
Updates(map[string]any{"is_deleted": deleted, "updated_at": uint32(time.Now().Unix())}).Error
}
// SumImageSize 返回某条街拍全部存活图片的原图字节数之和(image_size 由 worker 下载时写入源文件字节数)。
func (r *streetSnapRepository) SumImageSize(ctx context.Context, snapID uint32) (int64, error) {
var total int64
err := r.db.WithContext(ctx).
Model(&model.StreetSnapImage{}).
Where("snap_id = ? AND is_deleted = 0", snapID).
Select("COALESCE(SUM(image_size), 0)").
Scan(&total).Error
return total, err
}

View File

@ -40,8 +40,11 @@ type Options struct {
// 注意:SSG 内部接口(/api/internal/ssg/*)不在这里注册,而是由 NewSSG 挂在独立的内部端口上,
// 二者物理隔离,确保构建期全量数据不会从对外公开端口泄露。
func New(opt Options) *gin.Engine {
r := gin.Default()
r := gin.New()
r.Use(gin.Recovery())
r.Use(middleware.GinLogger())
r.Use(middleware.CORS(opt.Config.CORS))
r.Use(middleware.RateLimit(opt.Config.RateLimit))
// 上传图片的静态服务。
// 接口返回的 cover / image 是相对路径(如 /uploads/2026/08/xx.jpg),

View File

@ -40,6 +40,8 @@ type ArticleService interface {
SetCover(ctx context.Context, runwayID uint32, image string) error
// SetDeleted 上下架:deleted=1 下架(对外不可见),0 上架。
SetDeleted(ctx context.Context, id uint32, deleted uint8) error
// SumImageSize 返回某条走秀全部存活图片的原图字节数之和(供后台「图集原图总大小」展示)。
SumImageSize(ctx context.Context, runwayID uint32) (int64, error)
// PurgeGallery 彻底下架图集:级联软删图片 + 下架 + 异步入队清理S4孤儿图。
PurgeGallery(ctx context.Context, id uint32) error
}
@ -305,6 +307,11 @@ func (s *articleService) SetCover(ctx context.Context, runwayID uint32, image st
return s.articles.SetRunwayCover(ctx, runwayID, image)
}
// SumImageSize 返回某条走秀全部存活图片的原图字节数之和(供后台「图集原图总大小」展示)。
func (s *articleService) SumImageSize(ctx context.Context, runwayID uint32) (int64, error) {
return s.articles.SumImageSize(ctx, runwayID)
}
func (s *articleService) SetDeleted(ctx context.Context, id uint32, deleted uint8) error {
if deleted == 1 {
// 下架图集:软删图片 + 置下架 + 异步入队清理S4孤儿图(见 PurgeGallery)。

View File

@ -0,0 +1,43 @@
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
)
// fakeSizeUploader 仅用于单测:返回固定 URL,不真正写存储。
type fakeSizeUploader struct{}
func (fakeSizeUploader) Upload(_ context.Context, key string, _ []byte, _ string) (string, error) {
return "http://example/" + key, nil
}
func (fakeSizeUploader) Enabled() bool { return true }
// TestDownloadAndUploadRecordsOriginalSize 校验 worker 在下载阶段记录源文件字节数
// (即 image_size 的来源:io.ReadAll 读到的原图响应体长度)。
func TestDownloadAndUploadRecordsOriginalSize(t *testing.T) {
body := []byte("fake-original-image-bytes-payload-of-known-length-1234567890")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/jpeg")
_, _ = w.Write(body)
}))
defer srv.Close()
s := &IngestService{
uploader: fakeSizeUploader{},
httpClient: &http.Client{},
}
res := s.downloadAndUpload(context.Background(), srv.URL, "runway")
if res.err != nil {
t.Fatalf("downloadAndUpload 失败: %v", res.err)
}
if res.size != int64(len(body)) {
t.Fatalf("res.size = %d, want %d(源文件字节数)", res.size, len(body))
}
if res.url == "" {
t.Fatalf("res.url 不应为空(上传应成功)")
}
}

View File

@ -341,6 +341,7 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
Phash: sqlNull(fi.phash),
IsDuplicate: fi.isDup,
DupOf: strconv.FormatUint(uint64(fi.dupID), 10),
ImageSize: fi.size,
})
}
}
@ -471,6 +472,7 @@ func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayL
Phash: sqlNull(r.phashBits),
IsDuplicate: isDup,
DupOf: strconv.FormatUint(uint64(dupID), 10),
ImageSize: r.size,
})
}
timing.images = len(rows)
@ -547,6 +549,7 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
Phash: sqlNull(fi.phash),
IsDuplicate: fi.isDup,
DupOf: strconv.FormatUint(uint64(fi.dupID), 10),
ImageSize: fi.size,
})
}
var saveErr error
@ -609,6 +612,7 @@ func (s *IngestService) fetchImages(ctx context.Context, urls []string, prefix,
phash: r.phashBits,
dupID: dupID,
isDup: isDup,
size: r.size,
})
}
timing.images = len(out)
@ -642,6 +646,7 @@ type fetchedImage struct {
phash string // dHash 的 pgvector 二进制向量串,空串表示无法解码
dupID uint32 // 命中近重复时的参考图 id
isDup uint8 // 是否标记为近重复(供审核留痕)
size int64 // 原图源文件字节数,随结果带回供落库写入 image_size
}
// slowImageThreshold 单张图各阶段合计超过该阈值时单独打一条告警,便于从大量图里定位异常慢图。
@ -731,6 +736,7 @@ type downloadResult struct {
download time.Duration // 本张下载耗时
phash time.Duration // 本张指纹计算耗时
upload time.Duration // 本张上传耗时
size int64 // 原图源文件字节数(io.ReadAll 读到的下载响应体长度)
err error // 下载或上传失败原因(非空即该张失败)
}
@ -796,6 +802,7 @@ func (s *IngestService) downloadAndUpload(ctx context.Context, u, prefix string)
return res
}
res.download = time.Since(dlStart)
res.size = int64(len(data))
ext := path.Ext(u)
if ext == "" || len(ext) > 5 {

View File

@ -37,6 +37,8 @@ type StreetSnapService interface {
SetCover(ctx context.Context, snapID uint32, image string) error
// SetDeleted 上下架:deleted=1 下架,0 上架。
SetDeleted(ctx context.Context, id uint32, deleted uint8) error
// SumImageSize 返回某条街拍全部存活图片的原图字节数之和(供后台「图集原图总大小」展示)。
SumImageSize(ctx context.Context, snapID uint32) (int64, error)
// PurgeGallery 彻底下架街拍:级联软删图片 + 下架 + 异步入队清理S4孤儿图。
PurgeGallery(ctx context.Context, id uint32) error
}
@ -406,6 +408,11 @@ func (s *streetSnapService) SetCover(ctx context.Context, snapID uint32, image s
return s.repo.SetSnapCover(ctx, snapID, image)
}
// SumImageSize 返回某条街拍全部存活图片的原图字节数之和(供后台「图集原图总大小」展示)。
func (s *streetSnapService) SumImageSize(ctx context.Context, snapID uint32) (int64, error) {
return s.repo.SumImageSize(ctx, snapID)
}
func (s *streetSnapService) SetDeleted(ctx context.Context, id uint32, deleted uint8) error {
if deleted == 1 {
// 下架街拍:软删图片 + 置下架 + 异步入队清理S4孤儿图(见 PurgeGallery)。