Compare commits
18 Commits
818562ebdd
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
| bc0fce4edd | |||
| e930ac41c8 | |||
| b30f18fc20 | |||
| 51e3d13383 | |||
| db3e3efab9 | |||
| d6ed96624a | |||
| 64b264ee73 | |||
| c8a70760cc | |||
| c483018cc8 | |||
| 5f1c69ce98 | |||
| 151e09ab30 | |||
| e18c5a0815 | |||
| 49f8ea9358 | |||
| 2e76507d37 | |||
| 822dd69911 | |||
| acc0928401 | |||
| 98284c59e8 | |||
| 93edad28e5 |
15
.env.example
15
.env.example
@ -5,13 +5,24 @@
|
||||
# ============================================================
|
||||
|
||||
# ---- 数据库 ----
|
||||
# 生产复用宿主机上的 Postgres(如 1Panel 自带),容器内经 host-gateway 访问
|
||||
# 下列 DB_* 同时用于容器与 init-db.sh / backup-db.sh(脚本走 --network host)
|
||||
DB_USER=fashion
|
||||
DB_NAME=fashion
|
||||
# 上线必改:不要沿用开发库口令
|
||||
DB_PASSWORD=
|
||||
# 容器内访问宿主的固定写法;若数据库在另一台机器,改成那台的地址
|
||||
DB_HOST=host.docker.internal
|
||||
DB_PORT=5432
|
||||
# 1Panel 管理的 Postgres 容器名(init-db.sh / backup-db.sh 直接 docker exec 进去)
|
||||
# 查法:docker ps --format '{{.Names}}' | grep -i postgres
|
||||
PG_CONTAINER=1Panel-postgresql-4Ndp
|
||||
|
||||
# ---- 后端密钥(必填,缺了容器拒绝启动)----
|
||||
# 公开 ID 混淆盐:上线后不要再改,改了所有旧链接失效
|
||||
# ⚠️ 本地开发后端若连的是生产库(同一套 DB),这个值必须与线上完全一致。
|
||||
# 不一致时本地后端会把 crawl/ingest 里线上编码的 brand_uid 解码成另一个 id,
|
||||
# 任务被判「brand not found」——本地机器于是把生产队列里的任务做成失败。
|
||||
HASHID_SECRET=
|
||||
JWT_SECRET=
|
||||
# 爬虫上报 HMAC 密钥:须与爬虫端 config.yml 的 ingest.secret 完全一致
|
||||
@ -31,8 +42,8 @@ S4_REGION=cn-east-1
|
||||
S4_BASE_URL=
|
||||
|
||||
# ---- 目录与访问 ----
|
||||
# 前端仓库位置(服务器 /opt/fashion 下默认 ../test)
|
||||
WEB_DIR=../test
|
||||
# 前端仓库位置(服务器 /opt/fashion 下是 ../frontend_v2)
|
||||
WEB_DIR=../frontend_v2
|
||||
# 前端 nginx 监听端口(有宿主机 nginx/宝塔占用 80 时改成 8080 再由外层反代)
|
||||
HTTP_PORT=80
|
||||
CORS_ALLOW_ORIGINS=https://xisoa.com,https://www.xisoa.com
|
||||
|
||||
11
.gitattributes
vendored
Normal file
11
.gitattributes
vendored
Normal file
@ -0,0 +1,11 @@
|
||||
# Shell 脚本必须保持 LF:CRLF 会让 Linux 上的 bash 直接报语法错误
|
||||
# (典型报错:syntax error near unexpected token `$'do\r'')
|
||||
*.sh text eol=lf
|
||||
|
||||
# Dockerfile 同理(CRLF 会导致某些解析器把续行符后的 \r 当成参数)
|
||||
Dockerfile text eol=lf
|
||||
Dockerfile.* text eol=lf
|
||||
|
||||
# compose / nginx 配置也统一 LF
|
||||
*.yml text eol=lf
|
||||
*.conf text eol=lf
|
||||
39
.gitea/workflows/deploy.yml
Normal file
39
.gitea/workflows/deploy.yml
Normal file
@ -0,0 +1,39 @@
|
||||
name: deploy
|
||||
on:
|
||||
<<<<<<< HEAD
|
||||
# 手动触发:在 Gitea 仓库 Actions 页面点「Run workflow」才部署,
|
||||
# 避免每次 push 到 main 的半成品代码被自动上线。
|
||||
=======
|
||||
>>>>>>> 98284c59e89856adf5a09ec675185b157e98d442
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: docker
|
||||
steps:
|
||||
<<<<<<< HEAD
|
||||
- name: SSH 到服务器执行上线脚本
|
||||
uses: appleboy/ssh-action@v1
|
||||
with:
|
||||
host: ${{ secrets.DEPLOY_HOST }}
|
||||
username: ${{ secrets.DEPLOY_USER }}
|
||||
key: ${{ secrets.DEPLOY_KEY }}
|
||||
script: |
|
||||
cd /opt/fashion/test
|
||||
./deploy.sh
|
||||
=======
|
||||
- name: SSH deploy
|
||||
shell: sh
|
||||
env:
|
||||
KEY_B64: ${{ secrets.DEPLOY_KEY_B64 }}
|
||||
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
||||
run: |
|
||||
set -e
|
||||
apk add --no-cache openssh-client >/dev/null 2>&1 || true
|
||||
mkdir -p "$HOME/.ssh"
|
||||
echo "$KEY_B64" | base64 -d > "$HOME/.ssh/id"
|
||||
chmod 600 "$HOME/.ssh/id"
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$HOME/.ssh/id" \
|
||||
"$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/fashion/backend_v2 && ./deploy.sh backend"
|
||||
>>>>>>> 98284c59e89856adf5a09ec675185b157e98d442
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -51,4 +51,7 @@ admin_cookies.txt
|
||||
|
||||
# 部署状态与备份(deploy.sh 生成,不进版本库)
|
||||
.deploy/
|
||||
backups/
|
||||
backups/
|
||||
|
||||
# 本地预构建产物(scripts/push-build.sh 生成)
|
||||
/server-linux-amd64
|
||||
422
DEPLOY.md
422
DEPLOY.md
@ -1,175 +1,365 @@
|
||||
# 部署手册(Docker · 蓝绿零停机)
|
||||
# 部署手册(1Panel 单机 · Docker 蓝绿 · Gitea Actions)
|
||||
|
||||
> 本文记录**当前实际线上环境**的部署方式。与老模板的差异见文末「与默认模板的差异」。
|
||||
>
|
||||
> 最后更新:2026-10-20
|
||||
|
||||
---
|
||||
|
||||
## 0. 一图看懂
|
||||
|
||||
```
|
||||
浏览器 ──▶ proxy(nginx:80,常驻,发布时从不重建)
|
||||
├─ /api ──▶ backend_a 或 backend_b
|
||||
├─ / ──▶ web_a 或 web_b
|
||||
├─ 18090(仅容器网络)SSR 取数 → 当前活跃后端
|
||||
├─ 127.0.0.1:18091 前端构建期取 SSG
|
||||
└─ 127.0.0.1:18092 管理后台(SSH 隧道)
|
||||
浏览器
|
||||
│ https://fashion.23cm.cn (443, Let's Encrypt)
|
||||
▼
|
||||
1Panel OpenResty (80/443,宿主机,已服务 git.23cm.cn 等站点)
|
||||
│ proxy_pass 127.0.0.1:8080
|
||||
▼
|
||||
proxy 容器 (nginx:1.27-alpine,常驻,发布时不重建)
|
||||
├─ /api/ ──▶ backend_a 或 backend_b :8090
|
||||
├─ / ──▶ web_a 或 web_b :3000
|
||||
├─ :18090(仅容器网络)SSR 取数 → 当前活跃后端
|
||||
├─ 宿主 127.0.0.1:18091 → 后端 SSG :8091(构建期取数)
|
||||
└─ 宿主 127.0.0.1:18092 → 后端管理后台 :8092
|
||||
│
|
||||
▼
|
||||
1Panel PostgreSQL 容器 (5432, 库 fashion, 用户 user_JA83HF)
|
||||
```
|
||||
|
||||
发布时只重建**空闲槽**,等它健康后改写切换文件并 `nginx -s reload`,在途请求由旧槽跑完。
|
||||
镜像以 `RELEASE`(`时间戳-后端提交号`)打标签,**不可变**,历史版本不会被覆盖。
|
||||
发布时只重建**空闲槽**,等它 healthy 后改写切换文件并 `nginx -s reload`,在途请求由旧槽跑完。
|
||||
镜像以 `RELEASE`(`时间戳-后端提交号`)打标签,不可变,历史版本不被覆盖。
|
||||
|
||||
部署入口在本仓库(后端仓),脚本会自动把前端仓也拉到位。
|
||||
部署入口在本仓库(后端仓 `deploy.sh`),脚本会同时同步前端仓。
|
||||
|
||||
---
|
||||
|
||||
## 1. 服务器准备(只做一次)
|
||||
## 1. 服务器现状(只做一次,已就绪)
|
||||
|
||||
```bash
|
||||
# Ubuntu 22.04 / 24.04,2 核 4G 起
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
apt install -y docker-compose-plugin git
|
||||
|
||||
mkdir -p /opt/fashion && cd /opt/fashion
|
||||
git clone <后端仓库> backend
|
||||
git clone <前端仓库> test # 必须与 backend 同级
|
||||
|
||||
cd backend
|
||||
cp .env.example .env && vi .env
|
||||
chmod +x deploy.sh init-db.sh
|
||||
```
|
||||
|
||||
私有仓库需要配好拉取凭据:SSH deploy key,或 HTTPS token + `git config credential.helper store`。
|
||||
|
||||
---
|
||||
|
||||
## 2. 首次部署(只做一次)
|
||||
|
||||
```bash
|
||||
./init-db.sh /path/to/your.sql # 建库(传入与当前代码匹配的 SQL)
|
||||
./deploy.sh all # 启动全部服务
|
||||
curl http://127.0.0.1/api/health # {"status":"ok"}
|
||||
```
|
||||
|
||||
`.env` 必填 6 项:
|
||||
|
||||
| 变量 | 说明 |
|
||||
| 项 | 值 |
|
||||
| --- | --- |
|
||||
| `DB_PASSWORD` | 数据库口令,别沿用开发库的 |
|
||||
| `HASHID_SECRET` | 公开 ID 混淆盐,**上线后不要改**(改了所有旧链接失效) |
|
||||
| `JWT_SECRET` | 登录令牌密钥 |
|
||||
| `INGEST_SECRET` | 爬虫上报密钥,须与爬虫端一致 |
|
||||
| `S4_AK` / `S4_SK` | 对象存储密钥,缺了图片全 404 |
|
||||
| 主机 | `43.161.253.59`,Ubuntu 24.04,**2 核 / 3.6G 内存 / 1.9G swap** |
|
||||
| 面板 | 1Panel v2.0.3(已装 Docker 28.3 + Compose v2.38) |
|
||||
| Gitea | 容器 `1Panel-gitea-1fEW`(EE 24.2.3,Web `:3000` / SSH `:222`,域名 `git.23cm.cn`) |
|
||||
| Gitea Runner | 容器 `gitea-runner`(act_runner,label `docker`,job 镜像 `node:22-alpine`) |
|
||||
| 数据库 | 容器 `1Panel-postgresql-4Ndp`(postgres:18.6,`0.0.0.0:5432`,库 `fashion`,用户 `user_JA83HF`) |
|
||||
| 反向代理 | 容器 `1Panel-openresty-2mkA`(占 80/443,站点配置在 `/opt/1panel/www/conf.d`) |
|
||||
| 其它 | MySQL / MariaDB / Redis / PHP-FPM×2(供其它站点用,勿动) |
|
||||
|
||||
### 目录
|
||||
|
||||
```
|
||||
/opt/fashion/
|
||||
├── backend_v2/ # 后端仓(部署入口,含 deploy.sh / docker-compose.yml / nginx/)
|
||||
├── frontend_v2/ # 前端仓(Astro)
|
||||
└── spider/ # 爬虫仓(Go CLI,当前暂不部署)
|
||||
```
|
||||
|
||||
三个仓库都以 **`dev`** 分支为准(不是 main)。
|
||||
|
||||
### 初始化命令(仅重建机器时参考)
|
||||
|
||||
```bash
|
||||
mkdir -p /opt/fashion && cd /opt/fashion
|
||||
git clone -b dev https://git.23cm.cn/toom1996/backend_v2.git backend_v2
|
||||
git clone -b dev https://git.23cm.cn/toom1996/frontend_v2.git frontend_v2
|
||||
git clone -b dev https://git.23cm.cn/toom1996/spider.git spider
|
||||
|
||||
cd backend_v2
|
||||
cp .env.example .env # 按第 3 节填写
|
||||
chmod +x deploy.sh
|
||||
```
|
||||
|
||||
> 数据库已存在且有数据,**不需要** `init-db.sh` / 导入 dump。
|
||||
|
||||
---
|
||||
|
||||
## 3. 日常发布
|
||||
## 2. 关键配置(已改好,重建机器时照抄)
|
||||
|
||||
```bash
|
||||
./deploy.sh # 拉 main + 只更新后端(默认)
|
||||
./deploy.sh all # 拉 main + 前后端都更新
|
||||
./deploy.sh web # 只更新前端
|
||||
./deploy.sh all --no-pull # 不拉代码,用服务器现有代码构建
|
||||
GIT_BRANCH=dev ./deploy.sh all # 临时指定分支(优先于 .env)
|
||||
### 2.1 docker-compose.yml
|
||||
|
||||
相对上游模板做了 3 处适配:
|
||||
|
||||
1. **删掉自带 `db` 服务**:改用 1Panel 现有 Postgres(不再另起 pgvector)。
|
||||
2. **后端连库走宿主网关**:
|
||||
```yaml
|
||||
DB_HOST: host.docker.internal
|
||||
# backend-base 下:
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
```
|
||||
3. **proxy 只绑宿主回环**(80/443 已被 OpenResty 占用):
|
||||
```yaml
|
||||
ports:
|
||||
- "127.0.0.1:${HTTP_PORT:-8080}:80"
|
||||
- "127.0.0.1:18091:18091"
|
||||
- "127.0.0.1:18092:18092"
|
||||
```
|
||||
|
||||
### 2.2 nginx/default.conf(重要)
|
||||
|
||||
容器内的 18091 / 18092 **必须监听 0.0.0.0**,不能监听 `127.0.0.1`:
|
||||
|
||||
```nginx
|
||||
listen 18091; # 容器内回环监听会导致 docker 端口发布不可达
|
||||
listen 18092;
|
||||
```
|
||||
|
||||
**拉代码**(`.env` 的 `GIT_BRANCH`,默认 `main`):
|
||||
原因:Docker 端口发布是转发到**容器网卡**,不是容器回环。监听 `127.0.0.1` 时,
|
||||
宿主机 `127.0.0.1:18091` 会连接被重置,导致**前端构建期 SSG 取数为空、首页无数据**。
|
||||
宿主机侧仍只发布到 `127.0.0.1`,不会外泄。
|
||||
|
||||
- 后端仓 `.` 和前端仓 `$WEB_DIR`(默认 `../test`)都会 `fetch` → 必要时 `checkout` → `pull --ff-only`
|
||||
- **不能快进就报错退出**,不会硬覆盖(有本地改动时会提示 `reset --hard` 命令)
|
||||
- 每次打印实际提交:`后端:main @ a1b2c3d fix: xxx`
|
||||
### 2.3 deploy.sh(相对模板新增)
|
||||
|
||||
**发布过程**:
|
||||
- 首次部署顺序修复:先 `render_active` 再 `--no-deps` 起代理(否则 map 为空 nginx 起不来)。
|
||||
- 新增 `--no-cache`:强制前端不用构建缓存(见第 5 节 SSG)。
|
||||
|
||||
---
|
||||
|
||||
## 3. `.env`(敏感值只放服务器,600 权限)
|
||||
|
||||
路径:`/opt/fashion/backend_v2/.env`(已 gitignore,**绝不入库**)。
|
||||
|
||||
```ini
|
||||
# 数据库(复用 1Panel 现有 Postgres)
|
||||
DB_USER=user_JA83HF
|
||||
DB_NAME=fashion
|
||||
DB_PASSWORD=<真实口令,见下方说明>
|
||||
|
||||
# 密钥
|
||||
HASHID_SECRET= # 留空=内置默认盐,必须与爬虫 ingest.hashid_secret 一致
|
||||
JWT_SECRET=<随机长串>
|
||||
INGEST_SECRET=<与爬虫 ingest.secret 一致>
|
||||
SSG_TOKEN=<随机串,构建期取 SSG 用>
|
||||
|
||||
# 对象存储(当前关闭;现有图片走 CDN 公开桶,仍可正常显示)
|
||||
S4_ENABLED=false
|
||||
S4_AK=disabled
|
||||
S4_SK=disabled
|
||||
|
||||
# 目录 / 端口
|
||||
WEB_DIR=../frontend_v2
|
||||
HTTP_PORT=8080
|
||||
CORS_ALLOW_ORIGINS=https://fashion.23cm.cn,https://test.23cm.cn
|
||||
|
||||
# 其它
|
||||
GIT_BRANCH=dev
|
||||
GOPROXY=https://goproxy.cn,direct
|
||||
```
|
||||
|
||||
> **DB 密码从哪来**:以 Postgres 容器的环境变量为准,而不是旧 `config.yml`:
|
||||
> ```bash
|
||||
> docker exec 1Panel-postgresql-4Ndp printenv POSTGRES_PASSWORD
|
||||
> ```
|
||||
> 旧 `/opt/fashionapi/config.yml` 里的密码被行尾 `#` 注释截断过,是错的(28P01 的根因)。
|
||||
|
||||
---
|
||||
|
||||
## 4. 数据库
|
||||
|
||||
- 复用 1Panel 的 `postgres:18.6`,库 `fashion`,用户 `user_JA83HF`。
|
||||
- 已开启 pgvector(`vector` 扩展)。
|
||||
- 主要表:`brands` / `brand_runways` / `brand_runway_images` / `street_snaps` / `street_snap_images` / `users` / `favorites` / `histories` / `ingest_jobs` …
|
||||
- 后端不再自动迁移,表结构变更走 `cmd/dbtool`。
|
||||
|
||||
---
|
||||
|
||||
## 5. 部署方式 A:Gitea Actions(手动触发)
|
||||
|
||||
三个仓库的 `.gitea/workflows/deploy.yml` 都是 `workflow_dispatch`,**不会 push 自动上线**。
|
||||
|
||||
### 使用
|
||||
|
||||
1. 打开 `https://git.23cm.cn/toom1996/<repo>`
|
||||
2. 顶部 **Actions → deploy → Run workflow**
|
||||
3. 分支选 **`dev`** → Run
|
||||
|
||||
各仓行为:
|
||||
|
||||
| 仓库 | 实际执行 |
|
||||
| --- | --- |
|
||||
| `backend_v2` | `cd /opt/fashion/backend_v2 && ./deploy.sh backend` |
|
||||
| `frontend_v2` | `cd /opt/fashion/backend_v2 && ./deploy.sh web --no-cache` |
|
||||
| `spider` | `git pull` + `go build -o bin/spider .` |
|
||||
|
||||
### 原理与 Secrets
|
||||
|
||||
- Runner 与服务器同机,job 容器用 `node:22-alpine`,通过 SSH 连 `127.0.0.1` 执行部署。
|
||||
- 仓库 Secrets:`DEPLOY_HOST=127.0.0.1`、`DEPLOY_USER=root`、`DEPLOY_KEY_B64`(SSH 私钥 base64)。
|
||||
- 注意:**不要用公网 IP 43.161.253.59 做 DEPLOY_HOST**,云主机常拦截「访问自己公网 IP」的 hairpin,会卡住。
|
||||
|
||||
---
|
||||
|
||||
## 6. 部署方式 B:手动 SSH
|
||||
|
||||
```bash
|
||||
ssh -i ~/.ssh/id_rsa_hongkong root@43.161.253.59
|
||||
cd /opt/fashion/backend_v2
|
||||
|
||||
./deploy.sh # 只更新后端(默认)
|
||||
./deploy.sh web # 只更新前端
|
||||
./deploy.sh all # 前后端都更新
|
||||
./deploy.sh all --no-pull
|
||||
./deploy.sh web --no-cache # 强制重建前端(刷新首页 SSG 数据)
|
||||
./deploy.sh --rollback # 秒级回滚到上一槽
|
||||
./deploy.sh --prune # 清理旧镜像(默认保留最近 5 个)
|
||||
GIT_BRANCH=dev ./deploy.sh all
|
||||
```
|
||||
|
||||
**发布流程**:
|
||||
|
||||
| 步 | 动作 | 用户感受 |
|
||||
| --- | --- | --- |
|
||||
| 1 | 生成 `RELEASE`,拉代码 | 无感 |
|
||||
| 1 | 生成 `RELEASE`,`git pull --ff-only`(后端 + 前端) | 无感 |
|
||||
| 2 | `docker compose build` 构建 `fashion-*:$RELEASE` | 无感(旧槽在跑) |
|
||||
| 3 | 重建空闲槽 | 无感 |
|
||||
| 4 | 轮询 health,未通过则**直接退出** | 旧槽继续服务 |
|
||||
| 4 | 轮询 health,未通过则退出 | 旧槽继续服务 |
|
||||
| 5 | 改切换文件 → `nginx -t` → `nginx -s reload` | 在途请求不丢 |
|
||||
| 6 | 写 `.deploy/releases.log` | — |
|
||||
|
||||
---
|
||||
|
||||
## 4. 回滚
|
||||
## 7. 域名与 HTTPS
|
||||
|
||||
| 域名 | 指向 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `fashion.23cm.cn` | 127.0.0.1:8080 | **正式入口**,Let's Encrypt 证书,80/443 都通 |
|
||||
| `test.23cm.cn` | 127.0.0.1:8080 | 同一应用;原为旧静态站,旧配置备份在 `test.23cm.cn.conf.bak` |
|
||||
|
||||
- OpenResty 站点配置:`/opt/1panel/www/conf.d/<domain>.conf`
|
||||
- 证书目录:`/opt/1panel/www/sites/<domain>/ssl/{fullchain.pem,privkey.pem}`
|
||||
- `fashion.23cm.cn` 证书由 **certbot(webroot)** 签发,自动续期;续期 hook:
|
||||
`/etc/letsencrypt/renewal-hooks/deploy/fashion-23cm.sh`(复制证书 + `nginx -s reload`)
|
||||
- 验证:`curl -I https://fashion.23cm.cn/`
|
||||
|
||||
> 注意:1Panel 面板里的证书是自签(`CN=localhost`),与 certbot 那张不是同一张;面板「证书」列表里看不到 fashion 的证书属正常。
|
||||
|
||||
还原 `test.23cm.cn` 旧静态站:
|
||||
|
||||
```bash
|
||||
./deploy.sh --rollback
|
||||
# 回滚 backend:b → a(该槽镜像 fashion-backend:20261018-2210-9f3e1a2)
|
||||
```
|
||||
|
||||
不动代码、不重建容器,只改 nginx 指向 + reload,秒级生效。
|
||||
前提是该槽容器还在(正常交替发布时,非活跃槽就是上一版)。
|
||||
|
||||
---
|
||||
|
||||
## 5. 版本与镜像管理
|
||||
|
||||
```bash
|
||||
cat .deploy/releases.log # 发布历史
|
||||
docker inspect -f '{{.Config.Image}}' $(docker compose ps -q backend_a) # 线上版本
|
||||
./deploy.sh --prune # 清理旧镜像
|
||||
KEEP_IMAGES=3 ./deploy.sh --prune # 只留 3 个
|
||||
```
|
||||
|
||||
镜像标签形如 `fashion-backend:20261019-153012-a1b2c3d`(时间 + 提交号),不会被自动清理。
|
||||
`latest` 只是手动执行 compose 命令的别名,`deploy.sh` 从不依赖它。
|
||||
|
||||
---
|
||||
|
||||
## 6. 必须记住的规则
|
||||
|
||||
1. **默认发布 `main`,日常开发在 `dev`** —— 直接 `./deploy.sh all` 会部署 main 上的旧代码。要么先 merge,要么 `GIT_BRANCH=dev ./deploy.sh all`。
|
||||
2. **改表要向后兼容** —— 切换的几十秒里新旧代码同时在跑。加字段/加表可以;改类型、删字段要拆两次发布。
|
||||
3. **`HASHID_SECRET` 上线后别动**。
|
||||
4. **镜像会占磁盘**(前端约 250MB/版),每 5 版跑一次 `./deploy.sh --prune`。
|
||||
|
||||
---
|
||||
|
||||
## 7. 管理后台
|
||||
|
||||
```bash
|
||||
ssh -L 18092:127.0.0.1:18092 user@server
|
||||
# 本机浏览器打开 http://localhost:18092/admin
|
||||
cp /opt/1panel/www/conf.d/test.23cm.cn.conf.bak /opt/1panel/www/conf.d/test.23cm.cn.conf
|
||||
docker exec 1Panel-openresty-2mkA nginx -s reload
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 域名 / HTTPS
|
||||
## 8. 爬虫(当前暂不部署)
|
||||
|
||||
前端只监听 80,二选一:
|
||||
- 已构建:`/opt/fashion/spider/bin/spider`
|
||||
- 生产配置:`/opt/fashion/spider/config.prod.yml`(ingest 指向 `http://127.0.0.1:18092`)
|
||||
- 是 **CLI**,不是守护进程,跑完即退出:
|
||||
|
||||
- **宿主机 nginx / 宝塔**:`.env` 里 `HTTP_PORT=8080`,外层反代到 `127.0.0.1:8080`,证书在外层做
|
||||
- **Caddy(自动证书)**:同 compose 加 caddy 容器,`reverse_proxy 127.0.0.1:80`
|
||||
```bash
|
||||
cd /opt/fashion/spider
|
||||
./bin/spider -c config.prod.yml vogue -b 1
|
||||
./bin/spider -c config.prod.yml theimpression -n 5
|
||||
```
|
||||
|
||||
- 若要定时执行,可加 systemd timer(需要时再配)。
|
||||
|
||||
---
|
||||
|
||||
## 9. 备份
|
||||
|
||||
数据库在 1Panel Postgres 容器里,备份命令:
|
||||
|
||||
```bash
|
||||
docker compose exec -T db pg_dump -U fashion fashion | gzip > backups/db-$(date +%F).sql.gz
|
||||
docker exec 1Panel-postgresql-4Ndp pg_dump -U user_JA83HF fashion \
|
||||
| gzip > /opt/fashion/backups/db-$(date +%F).sql.gz
|
||||
```
|
||||
|
||||
每天 4 点自动备份并保留 7 天:
|
||||
每日 4 点、保留 7 天(示例):
|
||||
|
||||
```
|
||||
0 4 * * * cd /opt/fashion/backend && docker compose exec -T db pg_dump -U fashion fashion | gzip > backups/db-$(date +\%F).sql.gz && find backups -name 'db-*.sql.gz' -mtime +7 -delete
|
||||
```cron
|
||||
0 4 * * * docker exec 1Panel-postgresql-4Ndp pg_dump -U user_JA83HF fashion | gzip > /opt/fashion/backups/db-$(date +\%F).sql.gz && find /opt/fashion/backups -name 'db-*.sql.gz' -mtime +7 -delete
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. 排障
|
||||
## 10. 本地构建发布(前后端都不在服务器编译)
|
||||
|
||||
服务器资源紧张时用这条路径:**构建全在本机完成**,服务器只负责打镜像层 + 蓝绿切换。
|
||||
|
||||
```bash
|
||||
docker compose ps # 各容器状态与健康
|
||||
docker compose logs -f proxy # 切换失败先看这里
|
||||
docker compose logs -f backend_a
|
||||
# 本机(后端仓)执行,一条命令跑完
|
||||
SERVER=user@server ./scripts/push-build.sh
|
||||
```
|
||||
|
||||
脚本做五件事:
|
||||
|
||||
| 步 | 动作 | 说明 |
|
||||
|---|---|---|
|
||||
| 1 | 建 SSH 隧道 `本地18091 → 服务器 127.0.0.1:18091` | 前端构建期靠它读**线上** SSG 数据,公网零暴露 |
|
||||
| 2 | 本机 `npm run build` | 静态页内容与线上一致,不吃服务器内存 |
|
||||
| 3 | 本机 `CGO_ENABLED=0 GOOS=linux go build` | 产出 35MB 的 `server-linux-amd64` |
|
||||
| 4 | `tar`/`scp` 上传 `dist/` 与二进制 | 只传产物,不传源码 |
|
||||
| 5 | 远程 `./deploy.sh all --prebuilt` | 用 `Dockerfile.prebuilt` 打层,蓝绿切换 |
|
||||
|
||||
前置条件:
|
||||
|
||||
- 本机有 `node` / `npm` / `go` / `ssh` / `scp` / `tar`
|
||||
- 本机后端仓放一份 `.env`(复制服务器那份,用于取 `SSG_TOKEN`)
|
||||
- 服务器上 `proxy` 正在跑(18091 是它的端口)
|
||||
|
||||
与「服务器构建」的区别:
|
||||
|
||||
| | 服务器构建 | 本地构建(push-build) |
|
||||
|---|---|---|
|
||||
| 服务器 CPU/内存 | 前端 2~4 分钟 + 1GB 内存 | 只 `COPY` 产物,秒级 |
|
||||
| 构建期数据 | 线上(天然一致) | 线上(走隧道,同样一致) |
|
||||
| git 拉取 | 会拉两个仓库 | **跳过**(`--prebuilt` 自动 `--no-pull`) |
|
||||
| 发布号 | 服务器仓库的 sha | 本机 sha,通过 `RELEASE` 传入 |
|
||||
|
||||
两种方式可以混用:后端改动小、想走服务器构建,就 `./deploy.sh backend`;想省资源就 `push-build.sh`。
|
||||
|
||||
## 11. 排障(真实案例)
|
||||
|
||||
```bash
|
||||
cd /opt/fashion/backend_v2
|
||||
docker compose ps
|
||||
docker compose logs --tail 100 proxy
|
||||
docker compose logs --tail 100 backend_a
|
||||
docker compose logs --tail 100 web_b
|
||||
```
|
||||
|
||||
| 现象 | 原因 |
|
||||
| 现象 | 原因 / 处理 |
|
||||
| --- | --- |
|
||||
| 连库报 `28P01 password authentication failed` | 密码不对。以容器 `POSTGRES_PASSWORD` 为准;旧 config 里的可能被 `#` 截断 |
|
||||
| 首页只有 `Loading`、没有图集 | 构建期 SSG 取数失败。确认 nginx 内部 `listen 18091`(不是 `127.0.0.1:18091`)、proxy 在跑、`curl 127.0.0.1:18091/api/internal/ssg/home/runway?token=…` 有数据;再 `./deploy.sh web --no-cache` |
|
||||
| 改了内容但首页没变 | 首页是构建期 SSG,需重新部署前端(workflow 已带 `--no-cache`) |
|
||||
| `nginx: bind() to 0.0.0.0:18091 failed (Address in use)` | 旧进程还占着回环端口,`docker compose restart proxy` |
|
||||
| 全站接口 404 | `nginx/default.conf` 的 `proxy_pass` 带了 URI/斜杠(会剥掉 `/api` 前缀) |
|
||||
| 详情页 500 | 前端 `BASE_API_SSR` 必须是 `http://proxy:18090`(容器里的 localhost 是它自己) |
|
||||
| 图片全 404 | `S4_AK/S4_SK` 没填或填错 |
|
||||
| 前端构建失败 | 构建期要连 18091 取 SSG:确认 `proxy` 在跑、`backend_a` healthy |
|
||||
| 发布后仍是旧版本 | 分支不对(默认 `main`,日常在 `dev`);或 `docker compose logs proxy` 里 reload 报错 |
|
||||
| 回滚提示"槽不可用" | 该槽被更早的发布重建过,回滚只能退一格 |
|
||||
| 登录后跳回登录页 | `JWT_SECRET` 改过,旧 token 失效 |
|
||||
| 磁盘涨得快 | `./deploy.sh --prune` |
|
||||
| 图片全 404 | S4 关闭但现有图片走 CDN 正常;新入库图片需补 `S4_AK/S4_SK` 再开 `S4_ENABLED` |
|
||||
| 发布后仍是旧版本 | 分支不对(用 `dev`);或看 `docker compose logs proxy` 的 reload 报错 |
|
||||
| 回滚提示「槽不可用」 | 该槽被更早发布重建过,回滚只能退一格 |
|
||||
| 磁盘涨得快 | `./deploy.sh --prune`;`docker builder prune -f` |
|
||||
|
||||
---
|
||||
|
||||
## 11. 性能说明(当前瓶颈)
|
||||
|
||||
服务器 **2 核 / 3.6G**,且已常驻 Gitea、Postgres、MySQL、MariaDB、Redis、PHP×2、OpenResty、两个 Go 后端、Node 前端。体检时 **swap 1.9G 已用 1.7G**,构建期容易换页,`npm ci + astro build` 明显慢。
|
||||
|
||||
建议(按性价比):
|
||||
|
||||
1. **长期(推荐)**:构建移到本地 **WSL2 + Docker Engine**,推送到 Gitea 容器仓库,服务器只 `pull + up`。
|
||||
- 不要把构建目录放在 `/mnt/d/...`(9p 极慢),放 WSL 自己的 `~/project`。
|
||||
2. **短期(零成本)**:
|
||||
- 前端用 `--no-cache` 只用于刷新 SSG;进一步可改 `CACHEBUST` 构建参数,只重跑 `npm run build`,`npm ci` 走缓存。
|
||||
- Dockerfile 加 BuildKit cache mount(npm/go 依赖缓存)。
|
||||
- `docker builder prune -f` 回收构建缓存;`./deploy.sh --prune` 控制镜像数。
|
||||
- 可再加 4G swap 降低 OOM 风险。
|
||||
|
||||
---
|
||||
|
||||
## 12. 与默认模板的差异(为什么这么改)
|
||||
|
||||
| 模板假设 | 实际 |
|
||||
| --- | --- |
|
||||
| 自带 `db` 服务 + `init-db.sh` | 复用 1Panel 现有 Postgres,库已有数据 |
|
||||
| 目录 `backend` / `test` | `backend_v2` / `frontend_v2` |
|
||||
| 默认分支 `main` | `dev` |
|
||||
| 外层 Caddy / 宿主机 nginx 空着 | 1Panel OpenResty 已占 80/443,应用 proxy 改绑 `127.0.0.1:8080` |
|
||||
| `S4_AK/S4_SK` 必填 | 当前 `S4_ENABLED=false`,图片走 CDN 公开桶 |
|
||||
| push 自动部署 | 改为 `workflow_dispatch` 手动触发 |
|
||||
| nginx 内部 `listen 127.0.0.1:18091` | 容器内必须 `listen 18091`(0.0.0.0),否则端口发布不可达 |
|
||||
|
||||
27
Dockerfile.prebuilt
Normal file
27
Dockerfile.prebuilt
Normal file
@ -0,0 +1,27 @@
|
||||
# 预构建镜像:二进制在本机(GOOS=linux)编好,服务器只负责打层。
|
||||
# 用途:服务器资源紧张时,跳过容器内的 go build。
|
||||
# 由 scripts/push-build.sh 与 deploy.sh --prebuilt 使用;构建上下文里的
|
||||
# server-linux-amd64 由本机交叉编译产出(已在 .gitignore / .dockerignore 中)。
|
||||
|
||||
FROM alpine:3.20
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
|
||||
WORKDIR /app
|
||||
COPY server-linux-amd64 /app/server
|
||||
# 从 Windows 传上来的文件没有执行位(644),必须显式补上,否则容器启动报
|
||||
# exec: "/app/server": permission denied
|
||||
RUN chmod +x /app/server
|
||||
COPY configs ./configs
|
||||
|
||||
RUN mkdir -p /app/uploads
|
||||
|
||||
ENV CONFIG_PATH=/app/configs/config.yml \
|
||||
GIN_MODE=release \
|
||||
UPLOAD_DIR=/app/uploads
|
||||
|
||||
EXPOSE 8090 8091 8092
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD wget -qO- http://127.0.0.1:8090/api/health || exit 1
|
||||
|
||||
CMD ["/app/server"]
|
||||
12
README.md
12
README.md
@ -212,3 +212,15 @@ cp .env.example .env # 填 6 个必填密钥
|
||||
镜像按 `RELEASE=时间戳-提交号` 打不可变标签,历史版本不被覆盖。
|
||||
|
||||
完整步骤、发布流程、镜像管理、数据库兼容约束与排障见 **[`DEPLOY.md`](./DEPLOY.md)**。
|
||||
|
||||
### 本地构建(服务器不编译)
|
||||
|
||||
服务器资源紧张时,前后端都可以在本机构建,只把产物传上去:
|
||||
|
||||
```bash
|
||||
SERVER=user@server ./scripts/push-build.sh
|
||||
```
|
||||
|
||||
它会建 SSH 隧道让前端读**线上** SSG 数据 → 本机 `astro build` → 本机交叉编译后端 →
|
||||
上传 `dist/` 与 `server-linux-amd64` → 远程 `deploy.sh all --prebuilt`。
|
||||
详见 [`DEPLOY.md` 第 10 节](./DEPLOY.md)。
|
||||
|
||||
142
cmd/s4migrate/main.go
Normal file
142
cmd/s4migrate/main.go
Normal file
@ -0,0 +1,142 @@
|
||||
// Command s4migrate 把历史上兜底存在本地的图片(库里的 `/uploads/...` 路径)
|
||||
// 迁移到 S4,并把库里的路径改写为对象 key。
|
||||
//
|
||||
// 背景:S4 未启用(或上传失败)时,入库会兜底写本地磁盘,库里留下 `/uploads/runway/xxx.jpg`。
|
||||
// 这类路径只有在「nginx 把 /uploads 转发给后端」时才可访问;本项目的原则是
|
||||
// **图片一律进 S4**,所以这里做一次性的归并,不保留本地兜底访问通道。
|
||||
//
|
||||
// 用法(在本机,先把服务器上的容器卷拷下来):
|
||||
//
|
||||
// s4migrate -config configs/config.local.yml -dir ./uploads -dry-run
|
||||
// s4migrate -config configs/config.local.yml -dir ./uploads
|
||||
//
|
||||
// 幂等:已经迁移过的行不再是 `/uploads/` 前缀,重复执行只会处理剩余的。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"fashionapi/internal/config"
|
||||
"fashionapi/internal/database"
|
||||
"fashionapi/internal/pkg/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfgPath := flag.String("config", "", "配置文件路径(默认 configs/config.yml + config.local.yml 叠加)")
|
||||
dir := flag.String("dir", "", "本地 uploads 目录(其下应有 runway/ 与 street/)")
|
||||
dryRun := flag.Bool("dry-run", false, "只打印将要做什么,不写库不上传")
|
||||
flag.Parse()
|
||||
|
||||
if *dir == "" {
|
||||
log.Fatal("必须指定 -dir(本地 uploads 目录)")
|
||||
}
|
||||
|
||||
cfg, err := config.Load(*cfgPath)
|
||||
if err != nil {
|
||||
log.Fatalf("加载配置失败: %v", err)
|
||||
}
|
||||
if !cfg.S4.Enabled || cfg.S4.AK == "" {
|
||||
log.Fatal("配置里 S4 未启用或缺少 AK/SK,无法迁移")
|
||||
}
|
||||
db, err := database.New(cfg.Database)
|
||||
if err != nil {
|
||||
log.Fatalf("连接数据库失败: %v", err)
|
||||
}
|
||||
defer func() { _ = database.Close(db) }()
|
||||
|
||||
up := storage.NewS4Uploader(cfg.S4.AK, cfg.S4.SK, cfg.S4.Bucket, cfg.S4.Endpoint, cfg.S4.BaseURL, cfg.S4.Region)
|
||||
ctx := context.Background()
|
||||
|
||||
type target struct {
|
||||
table string
|
||||
id uint32
|
||||
image string
|
||||
}
|
||||
var targets []target
|
||||
for _, tbl := range []string{"brand_runway_images", "street_snap_images"} {
|
||||
var rows []struct {
|
||||
ID uint32
|
||||
Image string
|
||||
}
|
||||
if err := db.Raw("SELECT id, image FROM " + tbl +
|
||||
" WHERE is_deleted = 0 AND image LIKE '/uploads/%' ORDER BY id").Scan(&rows).Error; err != nil {
|
||||
log.Fatalf("查询 %s 失败: %v", tbl, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
targets = append(targets, target{table: tbl, id: r.ID, image: r.Image})
|
||||
}
|
||||
}
|
||||
log.Printf("待迁移图片 %d 张(来源 %s)", len(targets), *dir)
|
||||
|
||||
var okCount, failCount, missing int
|
||||
start := time.Now()
|
||||
for i, t := range targets {
|
||||
rel := strings.TrimPrefix(t.image, "/uploads/") // runway/ab12.jpg
|
||||
local := filepath.Join(*dir, filepath.FromSlash(rel))
|
||||
data, rerr := os.ReadFile(local)
|
||||
if rerr != nil {
|
||||
missing++
|
||||
log.Printf(" [缺失] %s/%d %s: %v", t.table, t.id, rel, rerr)
|
||||
continue
|
||||
}
|
||||
if *dryRun {
|
||||
log.Printf(" [dry] %s/%d %s (%d 字节)", t.table, t.id, rel, len(data))
|
||||
continue
|
||||
}
|
||||
// 文件名本身就是内容 sha1(LocalUploader 的命名),所以 key 与本地相对路径一致。
|
||||
if _, uerr := up.Upload(ctx, rel, data, contentTypeOf(rel)); uerr != nil {
|
||||
failCount++
|
||||
log.Printf(" [失败] %s/%d %s: %v", t.table, t.id, rel, uerr)
|
||||
continue
|
||||
}
|
||||
if err := db.Exec("UPDATE "+t.table+" SET image = ? WHERE id = ?", rel, t.id).Error; err != nil {
|
||||
failCount++
|
||||
log.Printf(" [写库失败] %s/%d %s: %v", t.table, t.id, rel, err)
|
||||
continue
|
||||
}
|
||||
okCount++
|
||||
if (i+1)%20 == 0 {
|
||||
log.Printf(" 进度 %d/%d", i+1, len(targets))
|
||||
}
|
||||
}
|
||||
|
||||
if *dryRun {
|
||||
log.Printf("dry-run 结束:%d 张可迁移,%d 张本地文件缺失", len(targets)-missing, missing)
|
||||
return
|
||||
}
|
||||
|
||||
// 封面同样可能指向 /uploads/:一并改写成 key(否则列表封面仍是死链)。
|
||||
for _, sql := range []string{
|
||||
"UPDATE brand_runways SET cover = replace(cover, '/uploads/', '') WHERE cover LIKE '/uploads/%'",
|
||||
"UPDATE street_snaps SET cover = replace(cover, '/uploads/', '') WHERE cover LIKE '/uploads/%'",
|
||||
} {
|
||||
if err := db.Exec(sql).Error; err != nil {
|
||||
log.Printf(" [封面改写失败] %s: %v", sql, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("完成:成功 %d,失败 %d,本地文件缺失 %d,耗时 %v",
|
||||
okCount, failCount, missing, time.Since(start).Round(time.Second))
|
||||
fmt.Println("提示:若仍有残留(缺失文件),用 SELECT 复核后再手工处理。")
|
||||
}
|
||||
|
||||
// contentTypeOf 按扩展名给 Content-Type,与上传时保持一致。
|
||||
func contentTypeOf(key string) string {
|
||||
switch strings.ToLower(filepath.Ext(key)) {
|
||||
case ".png":
|
||||
return "image/png"
|
||||
case ".gif":
|
||||
return "image/gif"
|
||||
case ".webp":
|
||||
return "image/webp"
|
||||
default:
|
||||
return "image/jpeg"
|
||||
}
|
||||
}
|
||||
@ -13,6 +13,8 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@ -217,6 +219,12 @@ func main() {
|
||||
// 后台页面因此只需读一次小查询(原先是页面内对每张图实时检索,一个 116 张图的图集要 5 秒)。
|
||||
ingestSvc.SetSimilar(similarSvc)
|
||||
backstageHandler := handler.NewBackstageHandler(authSvc, brandSvc, articleSvc, snapSvc, reviewSvc, ingestSvc, adminRepo, translateSvc, similarSvc)
|
||||
// 品牌抓取任务(crawl_tasks):后台决定爬哪些品牌,spider 服务(可跨机)领取执行。
|
||||
backstageHandler.SetCrawl(repository.NewCrawlRepository(db))
|
||||
// 向量回填接口的仓储(媒体机直传后,向量只能由外部 worker 回传)。
|
||||
backstageHandler.SetEmbeddingRepo(embedRepo)
|
||||
// S4 上传器:给媒体机签发直传 URL(密钥只留在后端)。
|
||||
backstageHandler.SetUploader(s4Up)
|
||||
// 手工建稿:复用入库仓储的写入方法(Create*WithImages,事务内完成)与同一套存储上传
|
||||
// (imgUp = S4/本地主上传器,sha1 内容寻址),保证与爬虫入库产出的 key / 行结构一致。
|
||||
backstageHandler.SetManual(service.NewManualArticleService(ingestRepo, brandRepo, imgUp, localUp, visionClient, embedRepo))
|
||||
@ -275,8 +283,18 @@ func main() {
|
||||
// 7.6 启动爬虫入库 worker(异步处理 ingest_jobs 队列)。
|
||||
// 用独立可取消 ctx,优雅关闭时随主流程一起退出。
|
||||
workerCtx, workerCancel := context.WithCancel(context.Background())
|
||||
go ingestSvc.Run(workerCtx, 2, 3*time.Second)
|
||||
slog.Info("入库 worker 已启动", "queue", "ingest_jobs")
|
||||
// 并发数可配:批量重爬时单进程 2 个 worker 是吞吐瓶颈(配合 INGEST_IMAGE_CONCURRENCY 决定总并发)。
|
||||
// 默认仍是 2,行为与之前一致;1~16 之外的值忽略并回落默认。
|
||||
ingestWorkers := 2
|
||||
if v := strings.TrimSpace(os.Getenv("INGEST_WORKERS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n >= 1 && n <= 16 {
|
||||
ingestWorkers = n
|
||||
} else {
|
||||
slog.Warn("INGEST_WORKERS 非法(应为 1~16 的整数),回退默认 2", "value", v)
|
||||
}
|
||||
}
|
||||
go ingestSvc.Run(workerCtx, ingestWorkers, 3*time.Second)
|
||||
slog.Info("入库 worker 已启动", "queue", "ingest_jobs", "workers", ingestWorkers)
|
||||
srvBackstage := &http.Server{
|
||||
Addr: ":" + cfg.Server.BackstagePort,
|
||||
Handler: backstageEngine,
|
||||
|
||||
@ -125,7 +125,9 @@ s4:
|
||||
bucket: "toomstudio" # 你的缤纷云 S4 桶名
|
||||
endpoint: "https://s3.bitiful.net"
|
||||
region: "cn-east-1"
|
||||
base_url: "" # 留空自动推导 https://toomstudio.s3.bitiful.net
|
||||
base_url: "http://bitiful.23cm.cn" # CDN 自定义域名(留空则自动推导 https://toomstudio.s3.bitiful.net)
|
||||
# 注意:该域名目前只有 HTTP 证书可用,HTTPS 仍返回网宿默认证书,
|
||||
# 等 CDN 侧签发证书后改成 https://bitiful.23cm.cn 即可(只改这一行)。
|
||||
style_display: "high" # 高清公开样式名(须在缤纷云控制台建 public 样式 high);或填查询串 w=1080&q=80&fmt=webp
|
||||
style_thumb: "thumb" # 列表缩略图公开样式名(控制台建 public 样式 thumb);仅用于列表/卡片等小尺寸场景,详情页与大图仍用 style_display。留空则回落 style_display
|
||||
|
||||
|
||||
23
db/migrations/2026-10-24-01-crawl-tasks.sql
Normal file
23
db/migrations/2026-10-24-01-crawl-tasks.sql
Normal file
@ -0,0 +1,23 @@
|
||||
-- 品牌抓取任务表:后台表达「要爬哪些品牌」,spider 服务来领取执行。
|
||||
-- 与 ingest_jobs 分层:crawl_tasks 对外(跨机器),ingest_jobs 对内(后端 worker)。
|
||||
CREATE TABLE IF NOT EXISTS crawl_tasks (
|
||||
id bigserial PRIMARY KEY,
|
||||
brand_id bigint NOT NULL,
|
||||
status varchar(16) NOT NULL DEFAULT 'queued',
|
||||
priority smallint NOT NULL DEFAULT 0,
|
||||
max_shows smallint NOT NULL DEFAULT 0,
|
||||
worker varchar(64) NOT NULL DEFAULT '',
|
||||
locked_at bigint NOT NULL DEFAULT 0,
|
||||
attempts smallint NOT NULL DEFAULT 0,
|
||||
shows_submitted integer NOT NULL DEFAULT 0,
|
||||
last_error text NOT NULL DEFAULT '',
|
||||
created_at bigint NOT NULL DEFAULT 0,
|
||||
updated_at bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- 一个品牌同时只允许一条任务:后台反复点「加入队列」不会产生重复任务,
|
||||
-- 而是把已完成/失败的记录重置为 queued。
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_crawl_tasks_brand ON crawl_tasks (brand_id);
|
||||
|
||||
-- 领取用索引:按优先级 + 入队顺序。
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_tasks_pick ON crawl_tasks (status, priority DESC, id);
|
||||
16
db/migrations/2026-10-24-01-embedding-failures.sql
Normal file
16
db/migrations/2026-10-24-01-embedding-failures.sql
Normal file
@ -0,0 +1,16 @@
|
||||
-- 向量回填的失败计数表(媒体拆分后新增)。
|
||||
--
|
||||
-- 背景:媒体机直传后,后端拿到的是 key 而不是图片字节,向量只能由「回填 worker」
|
||||
-- (有 GPU 的机器)从存储拉图、编码、再回传。若某张图永远编不出向量
|
||||
-- (文件损坏、格式不支持、被删),没有计数就会每次回填都重试,把队列堵死。
|
||||
--
|
||||
-- 规则:同一 (kind, image_id) 连续失败 attempts 次后不再出现在待回填队列里;
|
||||
-- 回填成功时删除该行(这张图已经不需要再试了)。
|
||||
CREATE TABLE IF NOT EXISTS image_embedding_failures (
|
||||
kind varchar(16) NOT NULL,
|
||||
image_id bigint NOT NULL,
|
||||
attempts smallint NOT NULL DEFAULT 0,
|
||||
last_error text NOT NULL DEFAULT '',
|
||||
updated_at bigint NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (kind, image_id)
|
||||
);
|
||||
19
db/migrations/2026-10-25-01-crawl-progress.sql
Normal file
19
db/migrations/2026-10-25-01-crawl-progress.sql
Normal file
@ -0,0 +1,19 @@
|
||||
-- 抓取任务的执行进度。
|
||||
--
|
||||
-- 为什么需要:spider 一次抓取的品牌可能有几十场秀、每场几百张图,
|
||||
-- 跑起来几分钟到几小时。此前只有 queued/running/done 三个状态,
|
||||
-- 后台看到「执行中」就再无信息,既判断不了「在干什么」,
|
||||
-- 也判断不了「是慢还是卡死」。
|
||||
--
|
||||
-- progress_at 与 updated_at 分开记:updated_at 会在领取/回报时变,
|
||||
-- 而 progress_at 只有 spider 真的推进了才变——用它判卡死更准。
|
||||
ALTER TABLE crawl_tasks
|
||||
ADD COLUMN IF NOT EXISTS progress_done smallint NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS progress_total smallint NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS progress_text varchar(160) NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS progress_at bigint NOT NULL DEFAULT 0;
|
||||
|
||||
COMMENT ON COLUMN crawl_tasks.progress_done IS '已处理的秀数量(spider 上报)';
|
||||
COMMENT ON COLUMN crawl_tasks.progress_total IS '本任务计划处理的秀数量(预检过滤后)';
|
||||
COMMENT ON COLUMN crawl_tasks.progress_text IS '当前/最近一场秀的标题,供后台显示「正在抓什么」';
|
||||
COMMENT ON COLUMN crawl_tasks.progress_at IS '最后一次进度心跳的 Unix 秒;用于判断任务是否卡死';
|
||||
15
db/migrations/2026-10-25-01-ingest-outcome.sql
Normal file
15
db/migrations/2026-10-25-01-ingest-outcome.sql
Normal file
@ -0,0 +1,15 @@
|
||||
-- 入库任务的「结果类型」与品牌归属(品牌详情页的更新记录用)。
|
||||
--
|
||||
-- 背景:原来任务只有 status=done,分不清「新建入库」「实体已存在跳过」「实体已删除跳过」,
|
||||
-- 于是「这个品牌这次爬了到底干了什么」只能去翻进程日志,日志一滚就没了。
|
||||
--
|
||||
-- outcome 记录结果类型,note 记录人可读的细节(新建几张图 / 已存在的状态是什么)。
|
||||
ALTER TABLE ingest_jobs ADD COLUMN IF NOT EXISTS outcome varchar(24) NOT NULL DEFAULT '';
|
||||
ALTER TABLE ingest_jobs ADD COLUMN IF NOT EXISTS note text NOT NULL DEFAULT '';
|
||||
|
||||
-- brand_id:提交任务时就解出品牌,品牌详情页按它直接查,不必每次解 payload 里的 hashid。
|
||||
-- 0 表示无品牌(街拍任务)。
|
||||
ALTER TABLE ingest_jobs ADD COLUMN IF NOT EXISTS brand_id bigint NOT NULL DEFAULT 0;
|
||||
|
||||
-- 品牌详情页的查询是「某品牌最近 N 条」,按 (brand_id, id DESC) 走索引。
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_jobs_brand ON ingest_jobs (brand_id, id DESC);
|
||||
26
db/migrations/2026-10-25-02-brand-crawl-logs.sql
Normal file
26
db/migrations/2026-10-25-02-brand-crawl-logs.sql
Normal file
@ -0,0 +1,26 @@
|
||||
-- 品牌抓取历史(append-only)。
|
||||
--
|
||||
-- 为什么另建一张表:crawl_tasks 每品牌只有一行(uq_crawl_tasks_brand),
|
||||
-- 入队/完成都是覆盖,答不了「这个品牌什么时候爬过、爬了几次、哪次没抓到」。
|
||||
-- 这里每次 worker 回报写一条,与 ingest_jobs(每场秀一条)拼成完整链路:
|
||||
-- 抓取一次 → 上报 N 场 → 每场入库结果。
|
||||
--
|
||||
-- 量级:每品牌每次抓取 1 行,行宽约 160B(含索引约 200B)。
|
||||
-- 全品牌跑一遍 23237 行 ≈ 4.6MB;按月重爬一年 ≈ 56MB。
|
||||
-- 为防止无限增长,仓库层在写入后只保留每品牌最近 30 条(见 CrawlLogKeepPerBrand)。
|
||||
CREATE TABLE IF NOT EXISTS brand_crawl_logs (
|
||||
id bigserial PRIMARY KEY,
|
||||
brand_id bigint NOT NULL,
|
||||
task_id bigint NOT NULL DEFAULT 0,
|
||||
status varchar(16) NOT NULL DEFAULT '', -- 回报的原始状态:done / failed
|
||||
attempt smallint NOT NULL DEFAULT 1, -- 第几次尝试(含自动重试)
|
||||
shows_requested smallint NOT NULL DEFAULT 0, -- 任务上的场次上限
|
||||
shows_submitted integer NOT NULL DEFAULT 0, -- 实际上报了几场
|
||||
message text NOT NULL DEFAULT '', -- spider 回报的错误/说明
|
||||
worker varchar(64) NOT NULL DEFAULT '',
|
||||
started_at bigint NOT NULL DEFAULT 0, -- 领取任务的时间
|
||||
finished_at bigint NOT NULL DEFAULT 0 -- 回报时间
|
||||
);
|
||||
|
||||
-- 品牌详情页:某品牌最近 N 条
|
||||
CREATE INDEX IF NOT EXISTS idx_brand_crawl_logs_brand ON brand_crawl_logs (brand_id, id DESC);
|
||||
11
db/migrations/2026-10-25-02-runway-expected-images.sql
Normal file
11
db/migrations/2026-10-25-02-runway-expected-images.sql
Normal file
@ -0,0 +1,11 @@
|
||||
-- 一场秀「应该有多少张图」。
|
||||
--
|
||||
-- 用途:spider 重爬前预检,判断这场秀是不是真的抓全了。
|
||||
-- 此前按「已入库 ≥10 张」就整场跳过,遇到当初只成功抓了十几张的场次,
|
||||
-- 剩下的图永远补不上(缺的部分没有任何记录能说明它缺)。
|
||||
--
|
||||
-- 0 表示未知(改动之前入库的历史数据),判定时退化为按张数阈值。
|
||||
ALTER TABLE brand_runways
|
||||
ADD COLUMN IF NOT EXISTS expected_images int NOT NULL DEFAULT 0;
|
||||
|
||||
COMMENT ON COLUMN brand_runways.expected_images IS '该场秀应有图片数(spider 上报);0=未知,判定时回落阈值';
|
||||
15
db/migrations/2026-10-25-03-menswear-season-code.sql
Normal file
15
db/migrations/2026-10-25-03-menswear-season-code.sql
Normal file
@ -0,0 +1,15 @@
|
||||
-- 修正男装的 season_code:春/夏男装应为 SSyy,秋/冬男装为 FWyy。
|
||||
--
|
||||
-- 原实现里 menswear 一律返回 FW,于是同年的 Spring/Fall 男装共用 FWyy,
|
||||
-- 而实体键是(brand_id + season_code + collection_type)→ 后到的一场被判「已存在」跳过。
|
||||
-- 实测 AMBUSH:Vogue 25 场,入库 21 场,缺的 4 场全是男装。
|
||||
--
|
||||
-- 只改「春夏男装」(标题以 Spring 开头)的历史行;秋冬男装本来就是 FWyy,保持不变。
|
||||
-- 安全性:同一品牌的春夏男装此前必然与秋冬男装撞键而未入库,
|
||||
-- 因此改成 SSyy 不会与现有行冲突(唯一索引 (brand_id, season_code, collection_type))。
|
||||
UPDATE brand_runways
|
||||
SET season_code = 'SS' || right(year::text, 2)
|
||||
WHERE collection_type = 'menswear'
|
||||
AND season_code LIKE 'FW%'
|
||||
AND year >= 2000
|
||||
AND (title_en LIKE 'Spring%' OR title_en LIKE 'Resort%');
|
||||
59
deploy.sh
Normal file → Executable file
59
deploy.sh
Normal file → Executable file
@ -6,7 +6,9 @@
|
||||
# ./deploy.sh web 只更新前端
|
||||
# ./deploy.sh --rollback 切回上一个槽(不动代码,秒级)
|
||||
# ./deploy.sh all --no-pull 用服务器上现有代码构建
|
||||
# ./deploy.sh web --no-cache 强制不用构建缓存(前端 SSG 依赖后端数据,数据变了要强制重建)
|
||||
# ./deploy.sh --prune 只清理旧镜像(保留最近 KEEP_IMAGES 个,默认 5)
|
||||
# ./deploy.sh all --prebuilt 用已上传的预构建产物(不拉代码、不在容器内编译)
|
||||
# GIT_BRANCH=dev ./deploy.sh all 临时指定分支(优先于 .env)
|
||||
#
|
||||
# 每次发布生成 RELEASE=<时间戳>-<后端提交号>,镜像以此打标签(不可变,不覆盖历史)。
|
||||
@ -16,14 +18,18 @@ cd "$(dirname "$0")"
|
||||
|
||||
TARGET="backend"
|
||||
NO_PULL=0
|
||||
NO_CACHE=0
|
||||
PRUNE_ONLY=0
|
||||
PREBUILT=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
backend|web|all) TARGET="$arg" ;;
|
||||
--rollback) TARGET="rollback" ;;
|
||||
--no-pull) NO_PULL=1 ;;
|
||||
--no-cache) NO_CACHE=1 ;;
|
||||
--prune) PRUNE_ONLY=1 ;;
|
||||
*) echo "用法: ./deploy.sh [backend|web|all|--rollback|--prune] [--no-pull]"; exit 1 ;;
|
||||
--prebuilt) PREBUILT=1 ;;
|
||||
*) echo "用法: ./deploy.sh [backend|web|all|--rollback|--prune] [--no-pull] [--no-cache] [--prebuilt]"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
@ -35,7 +41,7 @@ set -a; . ./.env; set +a
|
||||
[ -n "$CLI_GIT_BRANCH" ] && GIT_BRANCH="$CLI_GIT_BRANCH"
|
||||
GIT_BRANCH="${GIT_BRANCH:-main}"
|
||||
KEEP_IMAGES="${KEEP_IMAGES:-5}"
|
||||
WEB_DIR="${WEB_DIR:-../test}"
|
||||
WEB_DIR="${WEB_DIR:-../frontend_v2}"
|
||||
|
||||
STATE_DIR=".deploy"
|
||||
ACTIVE_CONF="$STATE_DIR/nginx/active.conf"
|
||||
@ -148,7 +154,9 @@ roll() { # $1=component(backend|web)
|
||||
old=$(cur_slot "$comp"); idle=$(other_slot "$old")
|
||||
echo "==> $comp:活跃槽 $old,更新空闲槽 $idle(镜像 $RELEASE)"
|
||||
|
||||
docker compose build "${comp}_a" >/dev/null
|
||||
local build_flag=""
|
||||
[ "$NO_CACHE" = "1" ] && build_flag="--no-cache"
|
||||
docker compose build $build_flag "${comp}_a" >/dev/null
|
||||
# latest 只是给手动执行 compose 命令用的便利别名,deploy 自己从不依赖它
|
||||
docker tag "fashion-${comp}:${RELEASE}" "fashion-${comp}:latest" 2>/dev/null || true
|
||||
|
||||
@ -196,12 +204,45 @@ prune_images() {
|
||||
done
|
||||
}
|
||||
|
||||
# ---- 预构建产物校验 ----
|
||||
# 本地构建 + 上传的路径:镜像只负责打包,不在服务器上编译。
|
||||
# 同时强制 --no-pull:代码已在本地构建完毕,服务器仓库不需要(也不应该)再更新。
|
||||
use_prebuilt() {
|
||||
PREBUILT=1
|
||||
NO_PULL=1
|
||||
export BACKEND_DOCKERFILE=Dockerfile.prebuilt
|
||||
export WEB_DOCKERFILE=Dockerfile.prebuilt
|
||||
echo "==> 使用预构建产物(不在服务器编译)"
|
||||
case "$TARGET" in
|
||||
backend|all)
|
||||
[ -f server-linux-amd64 ] || {
|
||||
echo "!! 缺少 server-linux-amd64:先在本机执行 scripts/push-build.sh"
|
||||
exit 1
|
||||
}
|
||||
echo " 后端二进制:$(du -h server-linux-amd64 | cut -f1)"
|
||||
;;
|
||||
esac
|
||||
case "$TARGET" in
|
||||
web|all)
|
||||
[ -f "$WEB_DIR/dist/server/entry.mjs" ] || {
|
||||
echo "!! 缺少 $WEB_DIR/dist/server/entry.mjs:前端还没构建或没上传"
|
||||
exit 1
|
||||
}
|
||||
echo " 前端产物:$WEB_DIR/dist"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---- 主流程 ----
|
||||
if [ "$PRUNE_ONLY" = "1" ]; then
|
||||
prune_images
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$PREBUILT" = "1" ] && [ "$TARGET" != "rollback" ]; then
|
||||
use_prebuilt
|
||||
fi
|
||||
|
||||
if [ "$TARGET" != "rollback" ]; then
|
||||
export RELEASE="${RELEASE:-$(new_release)}"
|
||||
echo "==> 发布号 $RELEASE"
|
||||
@ -214,12 +255,14 @@ else
|
||||
export RELEASE="${RELEASE:-latest}"
|
||||
fi
|
||||
|
||||
# 首次部署先让代理起来(前端构建期要用 18091 取 SSG 数据)
|
||||
if ! healthy proxy; then
|
||||
echo "==> 启动代理(首次会一并拉起 db 与 a 槽)"
|
||||
docker compose up -d proxy
|
||||
fi
|
||||
# 首次部署:先写切换点,再只启动代理。
|
||||
# - render_active 必须先于 proxy 启动,否则 include 的 map 为空,nginx 会因未知变量启动失败。
|
||||
# - --no-deps 避免首次起代理时顺带构建前端(此时 18091 尚未就绪,SSG 取数会失败)。
|
||||
render_active
|
||||
if ! healthy proxy; then
|
||||
echo "==> 启动代理(首次只起代理,槽位由 roll 流程构建)"
|
||||
docker compose up -d --no-deps proxy
|
||||
fi
|
||||
|
||||
if [ "$TARGET" = "rollback" ]; then
|
||||
do_rollback
|
||||
|
||||
@ -26,7 +26,7 @@ x-backend-base: &backend-base
|
||||
BACKSTAGE_PORT: 8092
|
||||
GIN_MODE: release
|
||||
|
||||
HASHID_SECRET: ${HASHID_SECRET:?HASHID_SECRET 未设置:复制 .env.example 为 .env 并填写}
|
||||
HASHID_SECRET: ${HASHID_SECRET:-}
|
||||
JWT_SECRET: ${JWT_SECRET:?JWT_SECRET 未设置}
|
||||
SSG_TOKEN: ${SSG_TOKEN:-}
|
||||
CORS_ALLOW_ORIGINS: ${CORS_ALLOW_ORIGINS:-https://xisoa.com,https://www.xisoa.com}
|
||||
@ -35,7 +35,8 @@ x-backend-base: &backend-base
|
||||
RATE_LIMIT_RPS: ${RATE_LIMIT_RPS:-20}
|
||||
RATE_LIMIT_BURST: ${RATE_LIMIT_BURST:-40}
|
||||
|
||||
DB_HOST: db
|
||||
# 复用 1Panel 现有 Postgres(宿主 5432)。host-gateway 让容器经宿主访问它。
|
||||
DB_HOST: host.docker.internal
|
||||
DB_PORT: 5432
|
||||
DB_USER: ${DB_USER:-fashion}
|
||||
DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD 未设置}
|
||||
@ -60,9 +61,8 @@ x-backend-base: &backend-base
|
||||
UPLOAD_DIR: /app/uploads
|
||||
volumes:
|
||||
- uploads:/app/uploads
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8090/api/health"]
|
||||
interval: 10s
|
||||
@ -84,29 +84,15 @@ x-web-base: &web-base
|
||||
start_period: 15s
|
||||
|
||||
services:
|
||||
# ---- 数据库:PostgreSQL 16 + pgvector ----
|
||||
db:
|
||||
image: pgvector/pgvector:pg16
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: ${DB_USER:-fashion}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD 未设置}
|
||||
POSTGRES_DB: ${DB_NAME:-fashion}
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- db_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-fashion} -d ${DB_NAME:-fashion}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
# ---- 数据库:复用 1Panel 现有 Postgres(连接信息见 .env 的 DB_HOST/DB_PORT/DB_USER/DB_PASSWORD)----
|
||||
|
||||
# ---- 常驻代理:整个发布过程中不重建,切换只靠 reload ----
|
||||
proxy:
|
||||
image: nginx:1.27-alpine
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${HTTP_PORT:-80}:80"
|
||||
# 只绑宿主回环:80/443 由 1Panel OpenResty 占用,外层反代到 127.0.0.1:${HTTP_PORT}
|
||||
- "127.0.0.1:${HTTP_PORT:-8080}:80"
|
||||
- "127.0.0.1:18091:18091" # 前端构建期取 SSG
|
||||
- "127.0.0.1:18092:18092" # 管理后台(SSH 隧道)
|
||||
volumes:
|
||||
@ -130,6 +116,8 @@ services:
|
||||
<<: *backend-base
|
||||
build:
|
||||
context: .
|
||||
# 默认在容器内编译;本地预构建时由 deploy.sh --prebuilt 切到 Dockerfile.prebuilt
|
||||
dockerfile: ${BACKEND_DOCKERFILE:-Dockerfile}
|
||||
args:
|
||||
GOPROXY: ${GOPROXY:-https://goproxy.cn,direct}
|
||||
|
||||
@ -141,6 +129,8 @@ services:
|
||||
<<: *web-base
|
||||
build:
|
||||
context: ${WEB_DIR:-../test}
|
||||
# 默认在容器内 npm ci + astro build;本地预构建时切到 Dockerfile.prebuilt
|
||||
dockerfile: ${WEB_DOCKERFILE:-Dockerfile}
|
||||
args:
|
||||
# 走宿主回环的代理 18091,由代理转发到活跃后端槽
|
||||
BASE_API_SSG: http://127.0.0.1:18091
|
||||
@ -151,5 +141,4 @@ services:
|
||||
<<: *web-base
|
||||
|
||||
volumes:
|
||||
db_data:
|
||||
uploads:
|
||||
|
||||
137
docs/superpowers/specs/2026-10-25-spider-console-design.md
Normal file
137
docs/superpowers/specs/2026-10-25-spider-console-design.md
Normal file
@ -0,0 +1,137 @@
|
||||
# 爬虫管理台(/admin/spider)设计
|
||||
|
||||
日期:2026-10-25
|
||||
状态:已与需求方确认,待实现
|
||||
|
||||
## 背景
|
||||
|
||||
后台把「抓取任务」和「入库任务」拆成两个页面,但它们其实是同一条流水线的上下游:
|
||||
|
||||
```
|
||||
spider 抓图 → POST /ingest → ingest_jobs 队列 → 后端 worker 下载/写库 → brand_runways + images
|
||||
↑ 抓取进度写 crawl_tasks ↑ 入库结果写 ingest_jobs.outcome/note
|
||||
```
|
||||
|
||||
分开放导致排障时要在两页之间跳,且任何一页都看不全「现在到底在干什么」。
|
||||
|
||||
同时存在两个问题:
|
||||
|
||||
1. **小批量模式**(`max_shows=3`)让「入队」这件事语义含糊,实际使用时总是想全量。
|
||||
2. **跳过判定太粗**:只要某场秀已入库 ≥10 张就整场跳过。如果当初只成功抓到 12 张(其余失败),
|
||||
剩下的图永远补不上。
|
||||
|
||||
## 目标
|
||||
|
||||
1. 入队只有一个语义:**全量**。
|
||||
2. 跳过判定改为**覆盖率**:入库张数接近应有张数才跳过,缺得多就补抓。
|
||||
3. 用一页只读的「爬虫管理」替代原两个页面,能看清当前在抓什么、队列有多长、最近入库结果。
|
||||
4. 卡死任务 5 分钟自动回收,不依赖人工点「重试」。
|
||||
|
||||
## 决策
|
||||
|
||||
| 决策 | 结论 | 理由 |
|
||||
|---|---|---|
|
||||
| 页面形态 | 一条流水线看全(状态条 + 正在抓 + 待抓队列 + 最近入库) | 两表本就是一条链路,合并不需要改数据模型 |
|
||||
| 操作能力 | 纯只读 | 抓取卡死由租约自动回收;入库失败多为不可恢复的数据问题 |
|
||||
| 场次上限 | 后台入口全部移除,入队恒为全量 | 用户明确只保留全量模式 |
|
||||
| 跳过判定 | 覆盖率(允许缺 5 张) | 个别图在 Vogue 侧永久 404,要求一张不差会导致整场无限重抓 |
|
||||
| 抓取租约 | 900s → 300s,且「有心跳就不回收」 | 单纯缩短会误回收长任务;心跳是现成的存活信号 |
|
||||
|
||||
## 设计
|
||||
|
||||
### 1. 全量抓取
|
||||
|
||||
- `brands.html`:删除批量入队的「每品牌场次上限」下拉、删除单品牌按钮里的 `hidden maxShows=3`。
|
||||
- `CrawlBulk` / `CrawlEnqueue` 不再读取 `maxShows`,入队固定传 0。
|
||||
- `crawl_tasks.max_shows` 列保留:历史任务仍按原值渲染,新任务恒为 0。
|
||||
- spider `serve` 用任务里的 `max_shows`(恒 0 = 全量);`vogue` 子命令的 `-s` 仅留作本地调试。
|
||||
|
||||
### 2. 覆盖率判定
|
||||
|
||||
**数据**
|
||||
|
||||
- `brand_runways` 新增 `expected_images int not null default 0`:本场秀应有图片数。
|
||||
- `RunwayIngest` 新增 `expected_images`(spider 侧已知,日志里的「共 N 张图」)。
|
||||
- ingest 处理时写入:`expected_images = max(旧值, 载荷值)`,避免重抓变少时退化。
|
||||
|
||||
**接口**
|
||||
|
||||
`GET /admin/internal/shows/covered` 每项返回 `{year, season, collection_type, images, expected}`。
|
||||
|
||||
**判定(spider 侧)**
|
||||
|
||||
```
|
||||
if expected > 0: 跳过条件 = images >= expected - 5
|
||||
else: 跳过条件 = images >= 10 (历史数据无 expected)
|
||||
```
|
||||
|
||||
- 余量 5:缺 1~5 张视为完整。少了再补抓是整场重下(内容寻址无法只补缺的),代价高。
|
||||
- 缺超过 5 张才补抓,补抓后 `expected_images` 可能更新,逐步收敛。
|
||||
|
||||
### 3. 页面 `/admin/spider`
|
||||
|
||||
**① 状态条**
|
||||
|
||||
```
|
||||
爬虫 在线 · 正在抓 1 个 · 队列 3 个 · 待入库 2 条
|
||||
```
|
||||
|
||||
- 在线判定:存在 running 任务且最近心跳 < 10 分钟 → 在线;
|
||||
running 但心跳 ≥ 10 分钟 → `可能卡住`;无 running → `空闲`。
|
||||
|
||||
**② 正在抓**(`crawl_tasks.status = running`)
|
||||
|
||||
| 品牌 | 进度 | 当前场次 | 心跳 |
|
||||
|---|---|---|---|
|
||||
| Amiri | 12/47 场 + 进度条 | Fall 2021 Menswear(24/38 张) | 8 秒前 |
|
||||
|
||||
心跳 > 10 分钟整行标黄。
|
||||
|
||||
**③ 待抓队列**(`status = queued`):品牌 / 入队时间 / 已等待。
|
||||
|
||||
**④ 最近入库结果**(`ingest_jobs` 最近 50 条,`kind='crawl'`)
|
||||
|
||||
| 时间 | 品牌 | 场次 | 结果 |
|
||||
|---|---|---|---|
|
||||
| 23:12 | Amiri | Fall 2019 Menswear | 新建 34 张图 |
|
||||
| 23:10 | Celine | Spring 2020 RTW | 已存在跳过 |
|
||||
| 23:08 | Gucci | Fall 2020 RTW | 失败:图片下载超时 |
|
||||
|
||||
场次标题直接取 `payload::json->>'title_en'`,结果取 `outcome` + `note`,无需改表。
|
||||
|
||||
**轮询**:`GET /admin/spider/live` 每 4 秒,返回 ①②③ 三块的结构化数据;④ 不轮询(历史不会变)。
|
||||
|
||||
### 4. 租约改 300 秒 + 心跳存活
|
||||
|
||||
```sql
|
||||
WHERE status = 'running' AND locked_at > 0
|
||||
AND locked_at < ?
|
||||
AND coalesce(progress_at, 0) < ?
|
||||
```
|
||||
|
||||
两个时间点都取 `now - 300`。心不断 → 永不回收;真卡住 → 5 分钟回收。
|
||||
`CrawlStuckTimeoutSec` 由 900 改为 300。
|
||||
|
||||
### 5. 删除项
|
||||
|
||||
- 页面与模板:`/admin/crawl`、`/admin/ingest-jobs`
|
||||
- 路由:`CrawlList`、`CrawlLive`、`CrawlBulk`、`CrawlRetry`、`CrawlCancel`、`CrawlDelete`、
|
||||
`IngestJobs`、`IngestJobRetry`
|
||||
- 导航:两项合并为一个「爬虫」
|
||||
- 入队接口 `/admin/crawl/brands/:uid` → 改名 `/admin/spider/brands/:uid`(品牌页按钮同步改)
|
||||
|
||||
保留:品牌页的「抓取状态 / 上次抓取 / 更新记录」(品牌维度,不是爬虫维度)。
|
||||
|
||||
## 测试
|
||||
|
||||
- 渲染测试:`/admin/spider` 三区块、轮询锚点、卡住样式。
|
||||
- 单元测试:
|
||||
- 覆盖率判定:`expected>0 缺≤5 跳过`、`缺>5 补抓`、`expected=0 回落 10 张`
|
||||
- 租约回收 SQL:有心跳不回收(集成测试落地时验证)
|
||||
- 手动验证:Celine 全量重跑应只补抓缺图场次;新品牌全量抓取进度可见。
|
||||
|
||||
## 风险
|
||||
|
||||
- 全量抓取一个品牌可能数小时,若 spider 进程被杀,已入库的场次会被跳过,只重抓当前场次——可接受。
|
||||
- 覆盖率依赖 spider 上报;老数据 `expected_images = 0` 时退回 10 张阈值,行为与现在一致。
|
||||
- 只读页面意味着入库 `failed` 任务无法在界面上恢复,需改库处理(已确认接受)。
|
||||
52
init-db.sh
52
init-db.sh
@ -1,16 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# 首次建库:把导出的 SQL 灌进空的 PostgreSQL 容器
|
||||
# 首次建库:把导出的 SQL 灌进 PostgreSQL
|
||||
#
|
||||
# ./init-db.sh /path/to/dump.sql
|
||||
#
|
||||
# 前置:在开发机导出最新结构与数据(含 pgvector 扩展):
|
||||
# go run ./cmd/dbtool dump -with-extension -out db/backups/db_dump.sql
|
||||
# 数据库由 1Panel 管理(独立的 postgres 容器,端口不对外发布),所以直接
|
||||
# docker exec 进那个容器跑 psql —— 服务器上不需要装 psql,也不依赖宿主端口。
|
||||
#
|
||||
# ⚠️ 仓库里 db/backups/db_dump.sql 是 2026-09-21 导出的,比当前代码少了之后两个月
|
||||
# 的新增迁移(向量表 / 近邻对表 / 品牌重编号 / published_at / image_size 等),
|
||||
# 直接拿它建库会导致后端启动后字段缺失——所以这里必须显式传入文件路径。
|
||||
# 容器名在 .env 的 PG_CONTAINER 里;不确定就看:
|
||||
# docker ps --format '{{.Names}}' | grep -i postgres
|
||||
#
|
||||
# 注意:只在库为空时执行一次;生产库已有数据时重跑会与现有行冲突。
|
||||
# 注意:只在库为空时执行一次;已有数据时重跑会与现有行冲突。
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
@ -22,24 +21,39 @@ if [ -z "$DUMP" ]; then
|
||||
echo "用法:./init-db.sh <dump.sql>"
|
||||
echo ""
|
||||
echo " 请传入一份与当前代码匹配的 SQL(含全部新增迁移)。"
|
||||
echo " 导出命令:go run ./cmd/dbtool dump -with-extension -out db/backups/db_dump.sql"
|
||||
echo " 导出命令(开发机):go run ./cmd/dbtool dump -with-extension -out db/backups/db_dump.sql"
|
||||
exit 1
|
||||
fi
|
||||
[ -f "$DUMP" ] || { echo "!! 找不到 dump 文件:$DUMP"; exit 1; }
|
||||
|
||||
echo "==> 启动数据库"
|
||||
docker compose up -d db
|
||||
PG_CONTAINER="${PG_CONTAINER:?PG_CONTAINER 未设置:填 .env 里那个 postgres 容器名}"
|
||||
if ! docker inspect "$PG_CONTAINER" >/dev/null 2>&1; then
|
||||
echo "!! 容器不存在:$PG_CONTAINER"
|
||||
echo " docker ps --format '{{.Names}}' | grep -i postgres"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> 等待数据库就绪"
|
||||
for i in $(seq 1 30); do
|
||||
if docker compose exec -T db pg_isready -U "${DB_USER:-fashion}" -d "${DB_NAME:-fashion}" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
[ "$i" -eq 30 ] && { echo "!! 数据库 60 秒未就绪"; docker compose logs --tail 30 db; exit 1; }
|
||||
sleep 2
|
||||
done
|
||||
run_psql() {
|
||||
docker exec -i -e PGPASSWORD="${DB_PASSWORD:?DB_PASSWORD 未设置}" "$PG_CONTAINER" \
|
||||
psql -U "${DB_USER:-fashion}" -d "${DB_NAME:-fashion}" "$@"
|
||||
}
|
||||
|
||||
echo "==> 检查连接(容器 $PG_CONTAINER)"
|
||||
if ! run_psql -c 'select 1' >/dev/null 2>&1; then
|
||||
echo "!! 连不上。检查 DB_USER / DB_NAME / DB_PASSWORD 是否与该容器一致"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> 检查 pgvector"
|
||||
if run_psql -tAc "select 1 from pg_available_extensions where name = 'vector'" 2>/dev/null | grep -q 1; then
|
||||
echo " pgvector 可用"
|
||||
else
|
||||
echo " !! 该 Postgres 没有 pgvector:dump 里的 vector 类型会导入失败"
|
||||
echo " 换用带 pgvector 的镜像,或先安装扩展后重试"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> 导入 $DUMP"
|
||||
docker compose exec -T db psql -U "${DB_USER:-fashion}" -d "${DB_NAME:-fashion}" -v ON_ERROR_STOP=1 < "$DUMP"
|
||||
run_psql -v ON_ERROR_STOP=1 < "$DUMP"
|
||||
|
||||
echo "==> 完成。可以执行 ./deploy.sh all 启动全部服务"
|
||||
|
||||
@ -16,6 +16,11 @@ type BrandQuery struct {
|
||||
// Featured 仅保留"代表品牌"精选集合(按指标排名的前 N 个)。
|
||||
Featured bool
|
||||
Metric string // images(默认)| shows
|
||||
|
||||
// Sort 后台列表排序:空 / "name"(默认,按英文名);"updated" 按「最近有动静」倒序
|
||||
// (取该品牌下走秀图集的最大 updated_at 与抓取任务的最大 updated_at,两者取大)。
|
||||
// 公开侧忽略此字段,避免改变前台 A-Z 索引的既定顺序。
|
||||
Sort string
|
||||
Limit int // 精选集合大小,默认 200
|
||||
|
||||
// Locale 语言:cn | en。空时由 service 回落默认 en。
|
||||
|
||||
@ -29,6 +29,9 @@ type RunwayIngest struct {
|
||||
CollectionType string `json:"collection_type"` // rtw / menswear / couture / resort / pre_fall
|
||||
City string `json:"city"` // 地区/城市(street 用)
|
||||
PublishedAt uint32 `json:"published_at"` // Vogue 详情页 datePublished 的 Unix 时间戳(秒);0=未抽取。标题缺年份时用于矫正 year
|
||||
// ExpectedImages 本场秀应有的图片总数(spider 侧已知,含细节图)。
|
||||
// 用于重爬前的覆盖率判定:实际入库张数接近它才算抓全了;0=未知(旧爬虫不上报)。
|
||||
ExpectedImages int `json:"expected_images,omitempty"`
|
||||
Images []string `json:"images"` // 兼容旧 worker:铺平的主图 URL 列表(无细节分组时回退用)
|
||||
// Looks 结构化「主图 + 细节图」分组(Vogue 一个 gallery item = 1 主图 + N 细节图)。
|
||||
// 爬虫结构化上报时优先用 Looks;为空时 worker 回退到 Images(全部视为主图,无细节)。
|
||||
@ -37,7 +40,22 @@ type RunwayIngest struct {
|
||||
|
||||
// RunwayLook 一场秀中的一个 look:1 张主图 + 0..N 张细节图。
|
||||
// 主图即对外默认展示的走秀图;细节图挂靠到对应主图,由「细节入口」按需展开。
|
||||
//
|
||||
// 图片有两种来源,二选一(同一条 payload 内可混用):
|
||||
//
|
||||
// ① 旧路径:Main / Details 给原始 URL,worker 自己下载 + 上传存储(默认行为);
|
||||
// ② 新路径(媒体机直传):MainKey / DetailKeys 给「已上传到 S4 的对象 key」,
|
||||
// worker 直接落库,不再下载、不再上传。
|
||||
//
|
||||
// 新路径的意义:抓取与入库分离后,下载/上传这类带宽密集的活由部署在快网络里的
|
||||
// spider 完成,后端只做业务写库,吞吐不再被后端机器的出口带宽卡住。
|
||||
// key 由 spider 按内容 sha1 生成(runway/<sha1>.jpg),因此重复上传天然幂等。
|
||||
type RunwayLook struct {
|
||||
Main string `json:"main"` // 主图(look)原始 URL
|
||||
Details []string `json:"details"` // 细节图原始 URL 列表
|
||||
|
||||
MainKey string `json:"main_key,omitempty"` // 主图已上传的存储 key(新路径)
|
||||
DetailKeys []string `json:"detail_keys,omitempty"` // 细节图已上传的存储 key(新路径)
|
||||
MainSize int64 `json:"main_size,omitempty"` // 主图字节数(新路径,0=未知)
|
||||
DetailSizes []int64 `json:"detail_sizes,omitempty"` // 细节图字节数(与 DetailKeys 对齐,0=未知)
|
||||
}
|
||||
|
||||
@ -540,3 +540,8 @@ tbody tr:last-child td { border-bottom: 0; }
|
||||
}
|
||||
/* 近邻若落在默认隐藏的副图(.cell.det)上,锚点定位后必须可见,否则跳过去是一片空白。 */
|
||||
.cell.det:target { display: flex; }
|
||||
|
||||
/* 抓取进度:正常更新的进度用灰字;心跳停了(可能卡住)才用警示色。
|
||||
用颜色而不是弹窗,是因为轮询每 4 秒一次,弹窗会变成骚扰。 */
|
||||
.progress { color: #666; font-size: 12px; }
|
||||
.progress.stale { color: #b45309; font-weight: 600; }
|
||||
|
||||
150
internal/handler/backstage_brand_records.go
Normal file
150
internal/handler/backstage_brand_records.go
Normal file
@ -0,0 +1,150 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"fashionapi/internal/model"
|
||||
"fashionapi/internal/repository"
|
||||
)
|
||||
|
||||
// brandRecordView 是品牌详情页「更新记录」的一行(已格式化成可直接渲染的字符串)。
|
||||
//
|
||||
// 为什么不把原始结构丢给模板:outcome / status 是给机器看的枚举,
|
||||
// 模板里再写一堆 if 既啰嗦又容易漏;这里一次性把「结果」翻译成人话。
|
||||
type brandRecordView struct {
|
||||
TimeText string // 时间(MM-DD HH:MM)
|
||||
Title string // 场次标题(如 Resort 2027)
|
||||
OutcomeText string // 结果(新建 / 已存在跳过 / 失败 / 排队中…)
|
||||
Detail string // 明细(新建 N 张图 / 错误原因)
|
||||
}
|
||||
|
||||
// outcomeText 把结果枚举翻译成中文。空值时回落到任务状态。
|
||||
func outcomeText(outcome, status string) string {
|
||||
switch outcome {
|
||||
case model.IngestOutcomeCreated:
|
||||
return "新建"
|
||||
case model.IngestOutcomeReused:
|
||||
return "复用驳回"
|
||||
case model.IngestOutcomeSkippedExisting:
|
||||
return "已存在·跳过"
|
||||
case model.IngestOutcomeSkippedDeleted:
|
||||
return "已删除·跳过"
|
||||
case model.IngestOutcomeCleaned:
|
||||
return "媒体清理"
|
||||
case model.IngestOutcomeNoop:
|
||||
return "无操作"
|
||||
}
|
||||
// 老数据没有 outcome:只能按状态说清楚
|
||||
switch status {
|
||||
case model.IngestStatusDone:
|
||||
return "完成"
|
||||
case model.IngestStatusFailed:
|
||||
return "失败"
|
||||
case model.IngestStatusProcessing:
|
||||
return "处理中"
|
||||
default:
|
||||
return "排队中"
|
||||
}
|
||||
}
|
||||
|
||||
// crawlStatusText 抓取任务状态 → 中文。
|
||||
func crawlStatusText(s string) string {
|
||||
switch s {
|
||||
case "queued":
|
||||
return "排队中"
|
||||
case "running":
|
||||
return "抓取中"
|
||||
case "done":
|
||||
return "已完成"
|
||||
case "failed":
|
||||
return "失败"
|
||||
case "canceled":
|
||||
return "已取消"
|
||||
default:
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
// buildBrandRecordViews 把仓储记录转成模板行;失败的行优先显示错误原因(比 note 更有用)。
|
||||
func buildBrandRecordViews(recs []repository.BrandIngestRecord) []brandRecordView {
|
||||
out := make([]brandRecordView, 0, len(recs))
|
||||
for _, r := range recs {
|
||||
v := brandRecordView{
|
||||
TimeText: time.Unix(int64(r.UpdatedAt), 0).Format("01-02 15:04"),
|
||||
Title: r.Title,
|
||||
OutcomeText: outcomeText(r.Outcome, r.Status),
|
||||
}
|
||||
switch {
|
||||
case r.Status == model.IngestStatusFailed && r.LastError != "":
|
||||
v.Detail = "错误:" + truncateRunes(r.LastError, 120)
|
||||
case r.Note != "":
|
||||
v.Detail = r.Note
|
||||
case r.Status == model.IngestStatusPending:
|
||||
v.Detail = "等待 worker 处理"
|
||||
default:
|
||||
v.Detail = "-"
|
||||
}
|
||||
// 重试过的任务标出来:同一场反复失败时,这一列能直接看出「试了几次」
|
||||
if r.Attempts > 0 && r.Status != model.IngestStatusDone {
|
||||
v.Detail += fmt.Sprintf("(第 %d 次尝试)", r.Attempts+1)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// crawlLogView 是品牌详情页「抓取历史」的一行。
|
||||
type crawlLogView struct {
|
||||
TimeText string // 结束时间
|
||||
StatusText string // 完成 / 失败
|
||||
Detail string // 上报几场(上限几场)/ 错误原因 / 第几次尝试
|
||||
Worker string // 谁跑的(主机名-进程号),排障时能看出是不是野实例
|
||||
Elapsed string // 耗时(有开始时间才算)
|
||||
}
|
||||
|
||||
// buildCrawlLogViews 把抓取历史转成模板行。
|
||||
func buildCrawlLogViews(logs []model.CrawlLog) []crawlLogView {
|
||||
out := make([]crawlLogView, 0, len(logs))
|
||||
for _, l := range logs {
|
||||
v := crawlLogView{
|
||||
TimeText: time.Unix(int64(l.FinishedAt), 0).Format("01-02 15:04"),
|
||||
StatusText: crawlStatusText(l.Status),
|
||||
Worker: l.Worker,
|
||||
}
|
||||
if l.Status == model.CrawlFailed {
|
||||
v.StatusText = "失败"
|
||||
if l.Message != "" {
|
||||
v.Detail = "错误:" + truncateRunes(l.Message, 100)
|
||||
} else {
|
||||
v.Detail = "错误:(无详情)"
|
||||
}
|
||||
} else {
|
||||
v.Detail = fmt.Sprintf("上报 %d 场", l.ShowsSubmitted)
|
||||
if l.ShowsRequested > 0 {
|
||||
v.Detail += fmt.Sprintf("(上限 %d)", l.ShowsRequested)
|
||||
}
|
||||
if l.Message != "" {
|
||||
v.Detail += " · " + truncateRunes(l.Message, 60)
|
||||
}
|
||||
}
|
||||
if l.Attempt > 1 {
|
||||
v.Detail += fmt.Sprintf(" · 第 %d 次尝试", l.Attempt)
|
||||
}
|
||||
if l.StartedAt > 0 && l.FinishedAt > l.StartedAt {
|
||||
v.Elapsed = fmt.Sprintf("%d 秒", l.FinishedAt-l.StartedAt)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// truncateRunes 按字符(不是字节)截断,避免把中文切成半个字。
|
||||
func truncateRunes(s string, n int) string {
|
||||
r := []rune(strings.TrimSpace(s))
|
||||
if len(r) <= n {
|
||||
return string(r)
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
294
internal/handler/backstage_crawl.go
Normal file
294
internal/handler/backstage_crawl.go
Normal file
@ -0,0 +1,294 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"fashionapi/internal/model"
|
||||
"fashionapi/internal/pkg/hashid"
|
||||
)
|
||||
|
||||
// 本文件收口「品牌抓取任务」的控制面:
|
||||
// - 后台页面:/admin/crawl(列表 + 批量入队 + 取消/重试/删除)
|
||||
// - 品牌页行内动作:/admin/crawl/brand/:uid(把单个品牌加入队列)
|
||||
// - spider 服务接口(HMAC 验签,不走 cookie):
|
||||
// GET /admin/internal/crawl/tasks?worker=xxx → 领取一个任务
|
||||
// POST /admin/internal/crawl/tasks/:id/result → 回报执行结果
|
||||
//
|
||||
// 与入库任务(ingest_jobs)的分工见 model.CrawlTask 的注释。
|
||||
|
||||
// 本文件保留:进度格式化(progressLabel / progressStale)、状态文案(crawlStatusLabel /
|
||||
// crawlStatusClass,品牌页的「抓取状态」列在用),以及 spider 侧的全部接口。
|
||||
// 原先的列表页视图 crawlRowView 与 CrawlList 已随「抓取任务」页一起下线,
|
||||
// 页面改由 backstage_spider.go 的「爬虫」页承担。
|
||||
|
||||
// progressStaleSec 进度心跳超过这个时长没更新,就认为可能卡住了。
|
||||
// 一场秀的图片下载在正常情况下是分钟级;10 分钟没有心跳基本可以断定卡住。
|
||||
const progressStaleSec = 600
|
||||
|
||||
// progressLabel 拼出「3/25 场 · 秀标题」。
|
||||
//
|
||||
// 只显示数字没意义(不知道在处理哪一场),只显示标题也没意义(不知道还剩多少),
|
||||
// 所以两者拼在一起。
|
||||
func progressLabel(t model.CrawlTask) string {
|
||||
out := ""
|
||||
switch {
|
||||
case t.ProgressTotal > 0:
|
||||
out = fmt.Sprintf("%d/%d 场", t.ProgressDone, t.ProgressTotal)
|
||||
case t.ProgressDone > 0:
|
||||
out = fmt.Sprintf("%d 场", t.ProgressDone)
|
||||
}
|
||||
if t.ProgressText != "" {
|
||||
if out != "" {
|
||||
out += " · "
|
||||
}
|
||||
out += t.ProgressText
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// progressStale 判断「执行中但心跳停了」。
|
||||
//
|
||||
// 用 progress_at 而不是 updated_at:updated_at 在领取、回报时都会变,
|
||||
// 用它判断会把「刚领取、还没开始抓」误判成卡死。
|
||||
// 还没上报过进度时退化为 locked_at(领取时间)。
|
||||
func progressStale(t model.CrawlTask, now int64) bool {
|
||||
if t.Status != model.CrawlRunning {
|
||||
return false
|
||||
}
|
||||
beat := int64(t.ProgressAt)
|
||||
if beat == 0 {
|
||||
beat = int64(t.LockedAt)
|
||||
}
|
||||
return beat > 0 && now-beat > progressStaleSec
|
||||
}
|
||||
|
||||
// crawlStatusClass 把抓取状态映射到后台已有的徽标配色类(admin.css 只有
|
||||
// pending / processing / done / failed 四套配色,这里做一次语义映射)。
|
||||
func crawlStatusClass(s string) string {
|
||||
switch s {
|
||||
case model.CrawlQueued:
|
||||
return "pending"
|
||||
case model.CrawlRunning:
|
||||
return "processing"
|
||||
case model.CrawlDone:
|
||||
return "done"
|
||||
case model.CrawlFailed:
|
||||
return "failed"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func crawlStatusLabel(s string) string {
|
||||
switch s {
|
||||
case model.CrawlQueued:
|
||||
return "待领取"
|
||||
case model.CrawlRunning:
|
||||
return "执行中"
|
||||
case model.CrawlDone:
|
||||
return "已完成"
|
||||
case model.CrawlFailed:
|
||||
return "失败"
|
||||
case model.CrawlCanceled:
|
||||
return "已取消"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// spider 服务接口(HMAC 验签,由 router 的 /admin/internal 组挂载)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// crawlTaskPayload 领取接口返回给 spider 的最小信息:品牌 uid(直接用于上报)+ 英文名(拼 Vogue URL)+ 场次上限。
|
||||
type crawlTaskPayload struct {
|
||||
ID uint32 `json:"id"`
|
||||
BrandUID string `json:"brand_uid"`
|
||||
BrandName string `json:"brand_name"`
|
||||
MaxShows int16 `json:"max_shows"`
|
||||
}
|
||||
|
||||
// WorkerTask 领取一个抓取任务;无任务时返回 {"task": null}(spider 据此退避轮询)。
|
||||
func (h *BackstageHandler) WorkerTask(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "crawl repository not configured"})
|
||||
return
|
||||
}
|
||||
worker := c.Query("worker")
|
||||
if worker == "" {
|
||||
worker = "spider"
|
||||
}
|
||||
row, err := h.crawl.Claim(c.Request.Context(), worker)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "claim failed"})
|
||||
return
|
||||
}
|
||||
if row == nil {
|
||||
c.JSON(http.StatusOK, gin.H{"task": nil})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"task": crawlTaskPayload{
|
||||
ID: row.ID,
|
||||
BrandUID: hashid.Encode(row.BrandID),
|
||||
BrandName: row.BrandName,
|
||||
MaxShows: row.MaxShows,
|
||||
}})
|
||||
}
|
||||
|
||||
// WorkerShowsCovered 返回某品牌「已入库的秀 → 已有图片张数」,供 spider 抓取前预检。
|
||||
//
|
||||
// 为什么需要它:图片 key 是内容 sha1,**不下载就不知道一张图是否已存在**。
|
||||
// 于是重爬会把几百张已下过的图再下一遍(线上实测 Celine 62 分钟、约 600MB,0 新增)。
|
||||
// 这个接口让 spider 在抓之前就知道「这场我已经有了」,整场跳过,一张都不下。
|
||||
//
|
||||
// 请求:GET /admin/internal/shows/covered?brand_uid=<hashid>
|
||||
// 响应:{"items":[{"year":2020,"season":"fall","collection_type":"rtw","images":415}]}
|
||||
func (h *BackstageHandler) WorkerShowsCovered(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "crawl repository not configured"})
|
||||
return
|
||||
}
|
||||
uid := strings.TrimSpace(c.Query("brand_uid"))
|
||||
brandID, err := hashid.Decode(uid)
|
||||
if err != nil || brandID == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "brand_uid 非法"})
|
||||
return
|
||||
}
|
||||
coverage, err := h.crawl.ShowCoverage(c.Request.Context(), brandID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "query coverage failed"})
|
||||
return
|
||||
}
|
||||
items := make([]gin.H, 0, len(coverage))
|
||||
for k, st := range coverage {
|
||||
items = append(items, gin.H{
|
||||
"year": k.Year,
|
||||
"season": k.Season,
|
||||
"collection_type": k.CollectionType,
|
||||
"images": st.Images,
|
||||
// expected=0 表示历史数据没有记录应有张数,spider 会退化为按张数阈值判断。
|
||||
"expected": st.Expected,
|
||||
})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
}
|
||||
|
||||
// WorkerProgress 接收 spider 的执行进度(每处理完一场秀上报一次)。
|
||||
//
|
||||
// 请求:POST /admin/internal/crawl/tasks/:id/progress
|
||||
//
|
||||
// {"done": 3, "total": 25, "text": "Fall 2020 Ready-to-Wear"}
|
||||
//
|
||||
// 回报失败不影响抓取:进度是观测信息,丢了只是后台显示旧一点。
|
||||
func (h *BackstageHandler) WorkerProgress(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "crawl repository not configured"})
|
||||
return
|
||||
}
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id <= 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "任务 ID 非法"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Done int `json:"done"`
|
||||
Total int `json:"total"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if err := h.crawl.UpdateProgress(c.Request.Context(), uint32(id), body.Done, body.Total, body.Text); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "update progress failed"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
|
||||
// WorkerUploadURL 给媒体机签发 S4 直传 URL(PUT 预签名)。
|
||||
//
|
||||
// 设计取舍:不在 spider 里配 AK/SK、也不引入 S3 SDK —— 密钥只留在后端,
|
||||
// 媒体机拿到有时效的 URL 直接 PUT 即可。上传成功后把 key 一起上送 ingest。
|
||||
// 请求:{"items":[{"key":"runway/ab12.jpg","content_type":"image/jpeg"}]}
|
||||
// 响应:{"items":[{"key":"...","url":"https://..."}],"expires_in":900}
|
||||
func (h *BackstageHandler) WorkerUploadURL(c *gin.Context) {
|
||||
if h.uploader == nil || !h.uploader.Enabled() {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "S4 未启用,无法签发直传 URL"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Items []struct {
|
||||
Key string `json:"key"`
|
||||
ContentType string `json:"content_type"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if len(body.Items) == 0 || len(body.Items) > 200 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "items 数量应为 1~200"})
|
||||
return
|
||||
}
|
||||
|
||||
const ttl = 15 * time.Minute
|
||||
out := make([]gin.H, 0, len(body.Items))
|
||||
for _, it := range body.Items {
|
||||
// key 由调用方生成(内容 sha1),这里只做前缀与路径穿越校验,避免被拿来写任意对象。
|
||||
if !strings.HasPrefix(it.Key, "runway/") && !strings.HasPrefix(it.Key, "street/") {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "key 必须以 runway/ 或 street/ 开头: " + it.Key})
|
||||
return
|
||||
}
|
||||
if strings.Contains(it.Key, "..") {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "key 非法: " + it.Key})
|
||||
return
|
||||
}
|
||||
u, err := h.uploader.PresignPut(c.Request.Context(), it.Key, it.ContentType, ttl)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
out = append(out, gin.H{"key": it.Key, "url": u})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": out, "expires_in": int(ttl.Seconds())})
|
||||
}
|
||||
|
||||
// WorkerReport spider 回报执行结果。
|
||||
// status: done(执行完成,哪怕 Vogue 上没有该品牌的秀)/ failed(进程级失败,可重试)。
|
||||
func (h *BackstageHandler) WorkerReport(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "crawl repository not configured"})
|
||||
return
|
||||
}
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id <= 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid task id"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Status string `json:"status"`
|
||||
ShowsSubmitted int `json:"shows_submitted"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if body.Status != model.CrawlDone && body.Status != model.CrawlFailed {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "status must be done or failed"})
|
||||
return
|
||||
}
|
||||
if body.Error != "" && len(body.Error) > 1000 {
|
||||
body.Error = body.Error[:1000]
|
||||
}
|
||||
if err := h.crawl.Report(c.Request.Context(), uint32(id), body.Status, body.ShowsSubmitted, body.Error); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "report failed"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
170
internal/handler/backstage_embeddings.go
Normal file
170
internal/handler/backstage_embeddings.go
Normal file
@ -0,0 +1,170 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"fashionapi/internal/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// pairMu 串行化「近邻重算」:见 WorkerEmbeddings 里的说明。
|
||||
var pairMu sync.Mutex
|
||||
|
||||
// precomputePairsAsync 在后台重算近邻,失败只记日志(向量已写入,重算可由 similarbackfill 补)。
|
||||
func precomputePairsAsync(similar *service.SimilarService, kind string, ids []uint32) {
|
||||
pairMu.Lock()
|
||||
defer pairMu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
if err := similar.PrecomputeForGallery(ctx, kind, ids); err != nil {
|
||||
log.Printf("[similar] 回填后近邻重算失败 kind=%s 数量=%d err=%v", kind, len(ids), err)
|
||||
}
|
||||
}
|
||||
|
||||
// embeddingMaxAttempts 同一张图最多回填几次。
|
||||
//
|
||||
// 为什么必须有上限:坏图(文件损坏 / 格式不支持 / 已从存储删掉)是**永久失败**,
|
||||
// 没有计数的话它每轮都会重新出现在队列头部,把后面的好图永远挤在后面。
|
||||
// 5 次足够覆盖网络抖动与视觉服务重启,又不至于让坏图长期占位。
|
||||
const embeddingMaxAttempts = 5
|
||||
|
||||
// embeddingMaxDim 允许的最大向量维度(FashionSigLIP 是 768,留足余量防误配)。
|
||||
const embeddingMaxDim = 4096
|
||||
|
||||
// WorkerPendingEmbeddings 给回填 worker 派发「缺向量」的图片。
|
||||
//
|
||||
// GET /admin/internal/embeddings/pending?kind=runway&limit=32
|
||||
// → {"items":[{"kind":"runway","image_id":123,"key":"runway/ab.jpg","url":"https://..."}],
|
||||
// "remaining": 812}
|
||||
//
|
||||
// 返回 url 而不只是 key:worker 直接下载,不必知道存储域名怎么拼
|
||||
// (域名换了、换了存储、本地兜底,都在后端一处收敛)。
|
||||
func (h *BackstageHandler) WorkerPendingEmbeddings(c *gin.Context) {
|
||||
if h.emb == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "embedding repository not configured"})
|
||||
return
|
||||
}
|
||||
kind := c.DefaultQuery("kind", "runway")
|
||||
if kind != "runway" && kind != "street" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "kind 只能是 runway / street"})
|
||||
return
|
||||
}
|
||||
limit := 32
|
||||
if v := c.Query("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n <= 0 || n > 200 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "limit 应为 1~200"})
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
|
||||
rows, err := h.emb.ListPendingEmbeddings(c.Request.Context(), kind, limit, embeddingMaxAttempts)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "query pending failed: " + err.Error()})
|
||||
return
|
||||
}
|
||||
items := make([]gin.H, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
items = append(items, gin.H{
|
||||
"kind": r.Kind,
|
||||
"image_id": r.ImageID,
|
||||
"key": r.Key,
|
||||
"url": imgComposer.Compose(r.Key),
|
||||
})
|
||||
}
|
||||
// remaining 用独立 count:worker 靠它判断「还有没有活儿」,不必取到空批次才知道。
|
||||
remaining, err := h.emb.CountPendingEmbeddings(c.Request.Context(), kind, embeddingMaxAttempts)
|
||||
if err != nil {
|
||||
remaining = -1 // -1 = 统计失败,worker 应继续拉取后续批次
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items, "remaining": remaining})
|
||||
}
|
||||
|
||||
// WorkerEmbeddings 接收回填结果:成功的写向量并重算近邻,失败的累加失败计数。
|
||||
//
|
||||
// POST /admin/internal/embeddings
|
||||
// {"items":[{"kind":"runway","image_id":123,"vector":[...768 个数...]}],
|
||||
// "failures":[{"kind":"runway","image_id":124,"error":"下载 404"}]}
|
||||
//
|
||||
// 为什么近邻重算放在这里:image_similar_pairs 是入库时预计算的(页面只读结果)。
|
||||
// 回填补上的向量如果不重算,后台「疑似同款」就永远看不到新入库的图。
|
||||
func (h *BackstageHandler) WorkerEmbeddings(c *gin.Context) {
|
||||
if h.emb == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "embedding repository not configured"})
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Items []struct {
|
||||
Kind string `json:"kind"`
|
||||
ImageID uint32 `json:"image_id"`
|
||||
Vector []float32 `json:"vector"`
|
||||
} `json:"items"`
|
||||
Failures []struct {
|
||||
Kind string `json:"kind"`
|
||||
ImageID uint32 `json:"image_id"`
|
||||
Error string `json:"error"`
|
||||
} `json:"failures"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid json: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if len(body.Items) == 0 && len(body.Failures) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "items 与 failures 不能同时为空"})
|
||||
return
|
||||
}
|
||||
|
||||
// 按 kind 分组批量写:一次往返写一批,避免 32 张图 32 次事务。
|
||||
idsByKind := map[string][]uint32{}
|
||||
vecsByKind := map[string][][]float32{}
|
||||
failed := len(body.Failures)
|
||||
|
||||
for _, it := range body.Items {
|
||||
kind := strings.TrimSpace(it.Kind)
|
||||
if kind != "runway" && kind != "street" {
|
||||
failed++
|
||||
_ = h.emb.RecordEmbeddingFailure(c.Request.Context(), kind, it.ImageID, "kind 非法: "+it.Kind)
|
||||
continue
|
||||
}
|
||||
if n := len(it.Vector); n == 0 || n > embeddingMaxDim {
|
||||
// 维度异常一律拒写:写进去会污染向量列,让近邻检索结果全错。
|
||||
failed++
|
||||
_ = h.emb.RecordEmbeddingFailure(c.Request.Context(), kind, it.ImageID, "向量维度异常: "+strconv.Itoa(n))
|
||||
continue
|
||||
}
|
||||
idsByKind[kind] = append(idsByKind[kind], it.ImageID)
|
||||
vecsByKind[kind] = append(vecsByKind[kind], it.Vector)
|
||||
}
|
||||
|
||||
saved := 0
|
||||
for kind, ids := range idsByKind {
|
||||
if err := h.emb.BulkUpsertEmbedding(c.Request.Context(), kind, ids, vecsByKind[kind]); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "write embedding failed: " + err.Error()})
|
||||
return
|
||||
}
|
||||
saved += len(ids)
|
||||
if h.similar != nil {
|
||||
// 近邻重算**必须异步**:它要对每张图各跑一次向量近邻查询(20KB payload/次),
|
||||
// 一批 100 张就是 100 次往返。同步做会让 worker 的 POST 直接超时
|
||||
// (实测:库在远端时 100 张超过 60s),于是向量其实已经写入、worker 却判失败并重试。
|
||||
//
|
||||
// 用互斥串行而不是每请求一个 goroutine:多个回填 worker 并发时,
|
||||
// 并发跑近邻查询只会互相拖慢,还容易把连接池占满。
|
||||
go precomputePairsAsync(h.similar, kind, ids)
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range body.Failures {
|
||||
_ = h.emb.RecordEmbeddingFailure(c.Request.Context(), f.Kind, f.ImageID, f.Error)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"saved": saved, "failed": failed})
|
||||
}
|
||||
@ -16,6 +16,7 @@ import (
|
||||
"fashionapi/internal/pkg/hashid"
|
||||
"fashionapi/internal/pkg/pagination"
|
||||
"fashionapi/internal/pkg/season"
|
||||
"fashionapi/internal/pkg/storage"
|
||||
"fashionapi/internal/repository"
|
||||
"fashionapi/internal/service"
|
||||
|
||||
@ -45,8 +46,25 @@ type BackstageHandler struct {
|
||||
// manual 后台「手工新建文章」(可为 nil:未装配时新建入口提示不可用)。
|
||||
// 用 SetManual 注入,避免改构造签名(同 SetSimilar 的取舍)。
|
||||
manual *service.ManualArticleService
|
||||
// crawl 品牌抓取任务(crawl_tasks)。用 SetCrawl 注入,避免改构造签名。
|
||||
crawl repository.CrawlRepository
|
||||
// uploader S4 上传器(可为 nil:未配置 S4 时不提供预签名直传)。
|
||||
// 媒体机(spider)用它拿一次性 PUT URL,自己上传图片,不必持有 AK/SK。
|
||||
uploader *storage.S4Uploader
|
||||
// emb 向量仓储(可为 nil:未注入时回填接口返回 503)。
|
||||
// 媒体机直传后后端拿不到图片字节,向量只能靠回填 worker 从存储拉图编码再回传。
|
||||
emb repository.EmbeddingRepository
|
||||
}
|
||||
|
||||
// SetUploader 注入 S4 上传器(装配期调用),用于给媒体机签发直传 URL。
|
||||
func (h *BackstageHandler) SetUploader(up *storage.S4Uploader) { h.uploader = up }
|
||||
|
||||
// SetEmbeddingRepo 注入向量仓储(装配期调用),用于向量回填的取件与回写。
|
||||
func (h *BackstageHandler) SetEmbeddingRepo(repo repository.EmbeddingRepository) { h.emb = repo }
|
||||
|
||||
// SetCrawl 注入抓取任务仓储(装配期调用;未注入时 /admin/crawl 提示不可用)。
|
||||
func (h *BackstageHandler) SetCrawl(repo repository.CrawlRepository) { h.crawl = repo }
|
||||
|
||||
// SetManual 注入手工建稿服务(装配期调用;未注入时新建页提示不可用)。
|
||||
func (h *BackstageHandler) SetManual(svc *service.ManualArticleService) { h.manual = svc }
|
||||
|
||||
@ -145,6 +163,8 @@ func (h *BackstageHandler) Brands(c *gin.Context) {
|
||||
All: c.Query("all") == "1",
|
||||
Page: atoiDefault(c.Query("page"), 1),
|
||||
Locale: "en",
|
||||
// 后台可切换「按最近更新排序」(走秀或抓取任务的最新时间)
|
||||
Sort: c.Query("sort"),
|
||||
}
|
||||
items, total, err := h.brand.AdminList(c.Request.Context(), q)
|
||||
if err != nil {
|
||||
@ -157,13 +177,53 @@ func (h *BackstageHandler) Brands(c *gin.Context) {
|
||||
p.Page = 1
|
||||
}
|
||||
meta := p.NewMeta(total)
|
||||
|
||||
// 当前页品牌的数字 id:下面的抓取状态与数据量统计都要用。
|
||||
ids := make([]uint32, 0, len(items))
|
||||
for _, it := range items {
|
||||
if id, derr := hashid.Decode(it.UID); derr == nil {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
|
||||
// 装饰「抓取状态 / 上次抓取」:从 crawl_tasks 取每个品牌最近一条任务。
|
||||
// 没有任务记录的品牌不在 map 里,模板显示「—」(= 从没加入过抓取队列)。
|
||||
crawlStates := map[string]gin.H{}
|
||||
if h.crawl != nil {
|
||||
if m, lerr := h.crawl.BrandCrawlStates(c.Request.Context(), ids); lerr == nil {
|
||||
for id, st := range m {
|
||||
last := ""
|
||||
if !st.UpdatedAt.IsZero() {
|
||||
last = st.UpdatedAt.Format("2006-01-02 15:04")
|
||||
}
|
||||
crawlStates[hashid.Encode(id)] = gin.H{
|
||||
"Label": crawlStatusLabel(st.Status),
|
||||
"Class": crawlStatusClass(st.Status),
|
||||
"LastAt": last,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 数据量:图集数 / 图片数(只查当前页的 30 个品牌)
|
||||
stats := map[string]gin.H{}
|
||||
if m, serr := h.brand.StatsByIDs(c.Request.Context(), ids); serr == nil {
|
||||
for id, st := range m {
|
||||
stats[hashid.Encode(id)] = gin.H{"Runways": st.Runways, "Images": st.Images}
|
||||
}
|
||||
}
|
||||
|
||||
renderPage(c, http.StatusOK, "brands", gin.H{
|
||||
"Items": items,
|
||||
"Total": total,
|
||||
"Keyword": q.Keyword,
|
||||
"Letter": q.Letter,
|
||||
"All": q.All,
|
||||
"Letters": lettersAtoZ(),
|
||||
"Items": items,
|
||||
"Total": total,
|
||||
"Keyword": q.Keyword,
|
||||
"Letter": q.Letter,
|
||||
"All": q.All,
|
||||
"Sort": q.Sort,
|
||||
"Letters": lettersAtoZ(),
|
||||
"CrawlStates": crawlStates,
|
||||
"Stats": stats,
|
||||
"Added": atoiDefault(c.Query("added"), 0),
|
||||
"Pager": pagination.Pager{
|
||||
PrevURL: pagination.PageLink(c, meta.PrevPage),
|
||||
NextURL: pagination.PageLink(c, meta.NextPage),
|
||||
@ -213,14 +273,40 @@ func (h *BackstageHandler) BrandCreate(c *gin.Context) {
|
||||
c.Redirect(http.StatusFound, "/admin/brands")
|
||||
}
|
||||
|
||||
// BrandEdit 渲染品牌编辑表单(按编码 id 取现有值)。
|
||||
// BrandEdit 渲染品牌编辑表单(按编码 id 取现有值)+ 该品牌的更新记录。
|
||||
//
|
||||
// 更新记录放在这里而不是单独做页面:品牌是这件事的主体,人打开一个品牌
|
||||
// 想看的正是「它最近几次抓取/入库到底做了什么」,跳出去看列表反而多一步。
|
||||
func (h *BackstageHandler) BrandEdit(c *gin.Context) {
|
||||
b, err := h.brand.Get(c.Request.Context(), c.Param("id"))
|
||||
uid := c.Param("id")
|
||||
b, err := h.brand.Get(c.Request.Context(), uid)
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusNotFound, "error", gin.H{"Error": "品牌不存在或已删除: " + c.Param("id")})
|
||||
renderPage(c, http.StatusNotFound, "error", gin.H{"Error": "品牌不存在或已删除: " + uid})
|
||||
return
|
||||
}
|
||||
renderPage(c, http.StatusOK, "brand-form", gin.H{"UID": c.Param("id"), "NameEn": b.NameEn, "NameCn": b.NameCn})
|
||||
data := gin.H{"UID": uid, "NameEn": b.NameEn, "NameCn": b.NameCn}
|
||||
|
||||
if brandID, derr := hashid.Decode(uid); derr == nil && brandID > 0 {
|
||||
// 入库记录:只有状态 + 结果 + 明细,不拉 payload(可能几十 KB)
|
||||
if h.ingest != nil {
|
||||
if recs, rerr := h.ingest.ListBrandRecords(c.Request.Context(), brandID, 30); rerr == nil {
|
||||
data["Records"] = buildBrandRecordViews(recs)
|
||||
}
|
||||
}
|
||||
// 抓取侧:最近一次状态(摘要)+ 抓取历史(brand_crawl_logs,每次一条)
|
||||
if h.crawl != nil {
|
||||
if states, serr := h.crawl.BrandCrawlStates(c.Request.Context(), []uint32{brandID}); serr == nil {
|
||||
if st, ok := states[brandID]; ok {
|
||||
data["CrawlStatus"] = crawlStatusText(st.Status)
|
||||
data["LastCrawl"] = st.UpdatedAt.Format("01-02 15:04")
|
||||
}
|
||||
}
|
||||
if logs, lerr := h.crawl.ListCrawlLogs(c.Request.Context(), brandID, 20); lerr == nil {
|
||||
data["CrawlLogs"] = buildCrawlLogViews(logs)
|
||||
}
|
||||
}
|
||||
}
|
||||
renderPage(c, http.StatusOK, "brand-form", data)
|
||||
}
|
||||
|
||||
// BrandUpdate 处理品牌编辑表单。
|
||||
@ -690,96 +776,6 @@ func (h *BackstageHandler) SuggestTitleCn(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"title_cn": cn})
|
||||
}
|
||||
|
||||
func (h *BackstageHandler) IngestJobs(c *gin.Context) {
|
||||
p := pagination.Parse(c, "page", "pageSize", 1, 50, 200)
|
||||
offset := p.Offset()
|
||||
jobs, total, err := h.ingest.ListJobs(c.Request.Context(), offset, p.PageSize)
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
views := make([]ingestJobView, 0, len(jobs))
|
||||
for _, j := range jobs {
|
||||
ua := ""
|
||||
if j.UpdatedAt > 0 {
|
||||
ua = time.Unix(int64(j.UpdatedAt), 0).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
views = append(views, ingestJobView{
|
||||
ID: j.ID,
|
||||
Status: j.Status,
|
||||
Attempts: j.Attempts,
|
||||
LastError: j.LastError,
|
||||
UpdatedAt: ua,
|
||||
})
|
||||
}
|
||||
// 越界页码(如手动改 URL 成大数)钳制到最后一页展示,与原有行为一致。
|
||||
// meta.Clamp 仅调整展示用的 Page/Prev/Next,不改变上面已算好的查询 offset。
|
||||
meta := p.NewMeta(total)
|
||||
meta.Clamp()
|
||||
renderPage(c, http.StatusOK, "ingest-jobs", gin.H{
|
||||
"Jobs": views,
|
||||
"Page": meta.Page,
|
||||
"PageSize": p.PageSize,
|
||||
"Total": total,
|
||||
"TotalPages": meta.TotalPages,
|
||||
"Pager": pagination.Pager{
|
||||
PrevURL: pagination.PageLinkWith(c, meta.PrevPage, "pageSize", p.PageSize),
|
||||
NextURL: pagination.PageLinkWith(c, meta.NextPage, "pageSize", p.PageSize),
|
||||
HasPrev: meta.HasPrev,
|
||||
HasNext: meta.HasNext,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// IngestJobRetry 把一条 failed 任务重新入队(重置为 pending),worker 会在下一个周期重新处理。
|
||||
func (h *BackstageHandler) IngestJobRetry(c *gin.Context) {
|
||||
id := atoiDefault(c.Param("id"), 0)
|
||||
if id == 0 {
|
||||
renderPage(c, http.StatusNotFound, "error", gin.H{"Error": "任务不存在: " + c.Param("id")})
|
||||
return
|
||||
}
|
||||
if err := h.ingest.RetryJob(c.Request.Context(), uint32(id)); err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.Redirect(http.StatusFound, "/admin/ingest-jobs")
|
||||
}
|
||||
|
||||
// reviewFormFields 从表单收集可编辑字段(runway + street 一并收集,
|
||||
// 最终白名单在仓储层按 kind 强制,非本 kind 的字段会被忽略)。
|
||||
func reviewFormFields(c *gin.Context) map[string]any {
|
||||
f := map[string]any{}
|
||||
if v := c.PostForm("title_en"); v != "" {
|
||||
f["title_en"] = v
|
||||
}
|
||||
if v := c.PostForm("title_cn"); v != "" {
|
||||
f["title_cn"] = v
|
||||
}
|
||||
f["description_en"] = c.PostForm("description_en")
|
||||
f["description_cn"] = c.PostForm("description_cn")
|
||||
if v := c.PostForm("season"); v != "" {
|
||||
f["season"] = v
|
||||
}
|
||||
if v := c.PostForm("collection_type"); v != "" {
|
||||
f["collection_type"] = v
|
||||
}
|
||||
if v := c.PostForm("season_code"); v != "" {
|
||||
f["season_code"] = v
|
||||
}
|
||||
if v := c.PostForm("title"); v != "" {
|
||||
f["title"] = v
|
||||
}
|
||||
if v := c.PostForm("city"); v != "" {
|
||||
f["city"] = v
|
||||
}
|
||||
if v := c.PostForm("year"); v != "" {
|
||||
if y, err := strconv.Atoi(v); err == nil {
|
||||
f["year"] = uint16(y)
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// usernameString 把中间件注入的 username(any)兜底成字符串。
|
||||
func usernameString(v any) string {
|
||||
if s, ok := v.(string); ok {
|
||||
|
||||
@ -30,11 +30,11 @@ func TestBackstageTemplatesParsed(t *testing.T) {
|
||||
"brands",
|
||||
"dashboard",
|
||||
"error",
|
||||
"ingest-jobs",
|
||||
"login",
|
||||
"runway-detail",
|
||||
"runway-edit",
|
||||
"snap-edit",
|
||||
"spider",
|
||||
"street-snap-detail",
|
||||
"users",
|
||||
}
|
||||
@ -73,6 +73,12 @@ func TestBackstageTemplatesRender(t *testing.T) {
|
||||
"Images": []any{},
|
||||
// 列表数据
|
||||
"Items": []any{}, "Users": []any{}, "Kinds": []any{}, "Records": []any{}, "TabCounts": map[string]int64{},
|
||||
// 爬虫页(spider.html):状态条 + 正在抓 + 待抓队列 + 最近入库结果
|
||||
"State": "空闲", "StateClass": "done", "RunningCount": 0, "QueuedCount": 0,
|
||||
"QueuedShown": 0, "QueuedPreview": 50, "PendingIngest": int64(0),
|
||||
"Running": []spiderRunView{}, "Queued": []spiderQueueView{}, "Results": []spiderResultView{},
|
||||
// 品牌页的「抓取状态 / 上次抓取」列(按 hashid 索引)
|
||||
"CrawlStates": map[string]any{}, "Stats": map[string]any{}, "Added": 0,
|
||||
// 详情页实体(.Snap 在两个页面里类型不同,见下面的 perPage 覆盖)
|
||||
"Article": &dto.PublicArticleDetail{UID: "abc123"},
|
||||
"Snap": &dto.PublicStreetSnapDetail{UID: "abc123"},
|
||||
|
||||
312
internal/handler/backstage_spider.go
Normal file
312
internal/handler/backstage_spider.go
Normal file
@ -0,0 +1,312 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"fashionapi/internal/model"
|
||||
"fashionapi/internal/pkg/hashid"
|
||||
)
|
||||
|
||||
// 本文件收口后台「爬虫」页。
|
||||
//
|
||||
// 为什么把两条队列合成一页:
|
||||
//
|
||||
// spider 抓图 → POST /ingest → ingest_jobs → 后端 worker 下载/写库
|
||||
// ↑ 抓取进度写 crawl_tasks ↑ 入库结果写 ingest_jobs.outcome/note
|
||||
//
|
||||
// 这两张表描述的是同一条流水线的上下游。此前分成「抓取任务」「入库任务」两页,
|
||||
// 任何一页都看不全「现在到底在干什么」,排障要在两页之间来回跳。
|
||||
//
|
||||
// 页面是**只读**的:抓取卡死由 300 秒租约自动回收(且有心跳续命),
|
||||
// 入队操作仍在品牌页(勾选 → 加入抓取队列 → 一律全量)。
|
||||
|
||||
// spiderRunView 「正在抓」一行。
|
||||
type spiderRunView struct {
|
||||
ID uint32
|
||||
BrandName string
|
||||
Progress string
|
||||
// Beat 距最近一次心跳多久(见 progressStale 的说明)。
|
||||
Beat string
|
||||
Stale bool
|
||||
Worker string
|
||||
}
|
||||
|
||||
// spiderQueueView 「待抓队列」一行。
|
||||
type spiderQueueView struct {
|
||||
BrandName string
|
||||
Since string
|
||||
Waiting string
|
||||
}
|
||||
|
||||
// spiderResultView 「最近入库结果」一行。
|
||||
type spiderResultView struct {
|
||||
At string
|
||||
BrandName string
|
||||
Title string
|
||||
Result string
|
||||
Note string
|
||||
Bad bool
|
||||
}
|
||||
|
||||
// SpiderBoard 渲染爬虫管理页。
|
||||
func (h *BackstageHandler) SpiderBoard(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": "抓取仓储未装配"})
|
||||
return
|
||||
}
|
||||
ctx := c.Request.Context()
|
||||
now := time.Now().Unix()
|
||||
|
||||
active, err := h.crawl.ListActive(ctx)
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
runs, queues := splitActive(active, now)
|
||||
|
||||
// 入库侧查询失败不该让整页 500:抓取部分仍然有参考价值。
|
||||
results := make([]spiderResultView, 0, 50)
|
||||
if raw, rErr := h.ingest.RecentCrawlResults(ctx, 50); rErr == nil {
|
||||
for _, r := range raw {
|
||||
label, bad := crawlOutcomeLabel(r.Status, r.Outcome)
|
||||
results = append(results, spiderResultView{
|
||||
At: formatUnix(int64(r.UpdatedAt), "01-02 15:04"),
|
||||
BrandName: r.BrandName,
|
||||
Title: r.Title,
|
||||
Result: label,
|
||||
Note: r.Note,
|
||||
Bad: bad,
|
||||
})
|
||||
}
|
||||
}
|
||||
pending, pErr := h.ingest.CountPendingJobs(ctx)
|
||||
if pErr != nil {
|
||||
pending = 0
|
||||
}
|
||||
|
||||
// 队列总数必须走 Counts(SELECT count(*) GROUP BY status),不能用 len(queues):
|
||||
// ListActive 有 LIMIT,队列上千时 len 会永远停在 200 附近,
|
||||
// 看起来像「一直抓不完」,其实是数字被截断了。
|
||||
counts, _ := h.crawl.Counts(ctx)
|
||||
|
||||
// 表格只铺前 50 个,避免一次渲染几百行拖慢页面;真实总数用 count 展示。
|
||||
const queuedPreview = 50
|
||||
shown := queues
|
||||
if len(shown) > queuedPreview {
|
||||
shown = shown[:queuedPreview]
|
||||
}
|
||||
|
||||
state, class := spiderState(active, now)
|
||||
renderPage(c, http.StatusOK, "spider", gin.H{
|
||||
"State": state,
|
||||
"StateClass": class,
|
||||
"RunningCount": int(counts[model.CrawlRunning]),
|
||||
"QueuedCount": int(counts[model.CrawlQueued]),
|
||||
"QueuedShown": len(shown),
|
||||
"QueuedPreview": queuedPreview,
|
||||
"PendingIngest": pending,
|
||||
"Running": runs,
|
||||
"Queued": shown,
|
||||
"Results": results,
|
||||
})
|
||||
}
|
||||
|
||||
// splitActive 把 running / queued 拆成两种视图,并按页面需要格式化时间。
|
||||
func splitActive(rows []model.CrawlTaskRow, now int64) ([]spiderRunView, []spiderQueueView) {
|
||||
runs := make([]spiderRunView, 0, 2)
|
||||
queues := make([]spiderQueueView, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
switch r.Status {
|
||||
case model.CrawlRunning:
|
||||
runs = append(runs, spiderRunView{
|
||||
ID: r.ID,
|
||||
BrandName: r.BrandName,
|
||||
Progress: progressLabel(r.CrawlTask),
|
||||
Beat: humanWait(now - lastBeat(r.CrawlTask)),
|
||||
Stale: progressStale(r.CrawlTask, now),
|
||||
Worker: r.Worker,
|
||||
})
|
||||
case model.CrawlQueued:
|
||||
since := int64(r.CreatedAt)
|
||||
queues = append(queues, spiderQueueView{
|
||||
BrandName: r.BrandName,
|
||||
Since: formatUnix(since, "01-02 15:04"),
|
||||
Waiting: humanWait(now - since),
|
||||
})
|
||||
}
|
||||
}
|
||||
return runs, queues
|
||||
}
|
||||
|
||||
// lastBeat 任务最近一次「有动静」的时间:优先进度心跳,还没有心跳时退化为领取时间。
|
||||
func lastBeat(t model.CrawlTask) int64 {
|
||||
if t.ProgressAt > 0 {
|
||||
return int64(t.ProgressAt)
|
||||
}
|
||||
return int64(t.LockedAt)
|
||||
}
|
||||
|
||||
// spiderState 给出状态条上的一句话和配色。
|
||||
func spiderState(rows []model.CrawlTaskRow, now int64) (string, string) {
|
||||
running := make([]model.CrawlTaskRow, 0, 1)
|
||||
for _, r := range rows {
|
||||
if r.Status == model.CrawlRunning {
|
||||
running = append(running, r)
|
||||
}
|
||||
}
|
||||
if len(running) == 0 {
|
||||
return "空闲", "done"
|
||||
}
|
||||
for _, r := range running {
|
||||
if progressStale(r.CrawlTask, now) {
|
||||
return "可能卡住", "failed"
|
||||
}
|
||||
}
|
||||
return "在线", "done"
|
||||
}
|
||||
|
||||
// SpiderLive 轮询接口:只返回会变化的部分(状态 + 正在抓 + 队列数量)。
|
||||
//
|
||||
// 最近入库结果不参与轮询——那是历史,除了新增行不会变,每 4 秒重传毫无意义。
|
||||
func (h *BackstageHandler) SpiderLive(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "抓取仓储未装配"})
|
||||
return
|
||||
}
|
||||
ctx := c.Request.Context()
|
||||
now := time.Now().Unix()
|
||||
active, err := h.crawl.ListActive(ctx)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "查询失败"})
|
||||
return
|
||||
}
|
||||
runs, _ := splitActive(active, now)
|
||||
state, class := spiderState(active, now)
|
||||
|
||||
items := make([]gin.H, 0, len(runs))
|
||||
for _, r := range runs {
|
||||
items = append(items, gin.H{
|
||||
"id": r.ID,
|
||||
"brand": r.BrandName,
|
||||
"progress": r.Progress,
|
||||
"beat": r.Beat,
|
||||
"stale": r.Stale,
|
||||
"worker": r.Worker,
|
||||
})
|
||||
}
|
||||
pending := int64(0)
|
||||
if n, pErr := h.ingest.CountPendingJobs(ctx); pErr == nil {
|
||||
pending = n
|
||||
}
|
||||
// 同 SpiderBoard:队列数走 count(*),不要用 len(queues)(受 LIMIT 截断)。
|
||||
counts, _ := h.crawl.Counts(ctx)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"state": state,
|
||||
"state_class": class,
|
||||
"running": items,
|
||||
"queued": counts[model.CrawlQueued],
|
||||
"pending_ingest": pending,
|
||||
})
|
||||
}
|
||||
|
||||
// SpiderEnqueue 把单个品牌加入抓取队列(品牌页行内按钮)。一律全量。
|
||||
func (h *BackstageHandler) SpiderEnqueue(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": "抓取仓储未装配"})
|
||||
return
|
||||
}
|
||||
brandID, err := hashid.Decode(c.Param("uid"))
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "品牌 ID 非法"})
|
||||
return
|
||||
}
|
||||
// maxShows 固定 0(全量):小批量模式已下线,抓多少由「已抓全就跳过」决定。
|
||||
if _, err := h.crawl.Enqueue(c.Request.Context(), brandID, 0, 10); err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.Redirect(http.StatusFound, c.PostForm("back"))
|
||||
}
|
||||
|
||||
// SpiderBulk 把「品牌管理页勾选的品牌」批量加入队列。一律全量。
|
||||
func (h *BackstageHandler) SpiderBulk(c *gin.Context) {
|
||||
if h.crawl == nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": "抓取仓储未装配"})
|
||||
return
|
||||
}
|
||||
uids := c.PostFormArray("uid")
|
||||
if len(uids) == 0 {
|
||||
renderPage(c, http.StatusBadRequest, "error", gin.H{"Error": "没有勾选任何品牌"})
|
||||
return
|
||||
}
|
||||
added := 0
|
||||
for _, uid := range uids {
|
||||
id, derr := hashid.Decode(uid)
|
||||
if derr != nil {
|
||||
continue
|
||||
}
|
||||
ok, err := h.crawl.Enqueue(c.Request.Context(), id, 0, 5)
|
||||
if err != nil {
|
||||
renderPage(c, http.StatusInternalServerError, "error", gin.H{"Error": err.Error()})
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
added++
|
||||
}
|
||||
}
|
||||
c.Redirect(http.StatusFound, fmt.Sprintf("/admin/brands?added=%d", added))
|
||||
}
|
||||
|
||||
// crawlOutcomeLabel 把入库结果翻译成中文,并标出是否为失败。
|
||||
func crawlOutcomeLabel(status, outcome string) (string, bool) {
|
||||
if status == model.IngestStatusFailed {
|
||||
return "失败", true
|
||||
}
|
||||
if status == model.IngestStatusPending || status == model.IngestStatusProcessing {
|
||||
return "处理中", false
|
||||
}
|
||||
switch outcome {
|
||||
case model.IngestOutcomeCreated:
|
||||
return "新建", false
|
||||
case model.IngestOutcomeRepaired:
|
||||
return "补齐缺图", false
|
||||
case model.IngestOutcomeReused:
|
||||
return "复用被驳回", false
|
||||
case model.IngestOutcomeSkippedExisting:
|
||||
return "已存在跳过", false
|
||||
case model.IngestOutcomeSkippedDeleted:
|
||||
return "已删除跳过", false
|
||||
case model.IngestOutcomeCleaned:
|
||||
return "清理", false
|
||||
case model.IngestOutcomeNoop:
|
||||
return "无需处理", false
|
||||
}
|
||||
return "完成", false
|
||||
}
|
||||
|
||||
// formatUnix 把 Unix 秒格式化成指定布局;0 返回占位符。
|
||||
func formatUnix(sec int64, layout string) string {
|
||||
if sec <= 0 {
|
||||
return "—"
|
||||
}
|
||||
return time.Unix(sec, 0).Format(layout)
|
||||
}
|
||||
|
||||
// humanWait 把等待秒数说成人话(队列里看「等了多久」比看时间戳直观)。
|
||||
func humanWait(sec int64) string {
|
||||
if sec < 0 {
|
||||
sec = 0
|
||||
}
|
||||
switch {
|
||||
case sec < 60:
|
||||
return fmt.Sprintf("%d 秒", sec)
|
||||
case sec < 3600:
|
||||
return fmt.Sprintf("%d 分钟", sec/60)
|
||||
default:
|
||||
return fmt.Sprintf("%d 小时 %d 分", sec/3600, (sec%3600)/60)
|
||||
}
|
||||
}
|
||||
89
internal/handler/spider_render_test.go
Normal file
89
internal/handler/spider_render_test.go
Normal file
@ -0,0 +1,89 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// TestSpiderPageRenders 守住「爬虫」页的三块内容与轮询锚点。
|
||||
//
|
||||
// 这类模板错误不会编译失败,只会在打开页面时 500 或空白,
|
||||
// 所以用一个渲染测试把「有任务 / 卡住 / 有入库结果」这几种情况固定下来。
|
||||
func TestSpiderPageRenders(t *testing.T) {
|
||||
tpl, ok := pageTemplates["spider"]
|
||||
if !ok {
|
||||
t.Fatal("spider 模板未注册")
|
||||
}
|
||||
|
||||
data := gin.H{
|
||||
"NavKey": "spider",
|
||||
"PendingReviews": 0,
|
||||
"State": "可能卡住",
|
||||
"StateClass": "failed",
|
||||
"RunningCount": 1,
|
||||
"QueuedCount": 2,
|
||||
"QueuedShown": 2,
|
||||
"QueuedPreview": 50,
|
||||
"PendingIngest": 3,
|
||||
"Running": []spiderRunView{{
|
||||
ID: 85,
|
||||
BrandName: "Amiri",
|
||||
Progress: "12/47 场 · Fall 2021 Menswear(24/38 张)",
|
||||
Beat: "11 分钟",
|
||||
Stale: true,
|
||||
Worker: "vm-1-74218-0",
|
||||
}},
|
||||
"Queued": []spiderQueueView{
|
||||
{BrandName: "Celine", Since: "10-25 23:30", Waiting: "8 分钟"},
|
||||
{BrandName: "Gucci", Since: "10-25 23:35", Waiting: "3 分钟"},
|
||||
},
|
||||
"Results": []spiderResultView{
|
||||
{At: "10-25 23:12", BrandName: "Amiri", Title: "Fall 2019 Menswear", Result: "新建", Note: "新建 34 张图"},
|
||||
{At: "10-25 23:08", BrandName: "Gucci", Title: "Fall 2020 RTW", Result: "失败", Note: "图片下载超时", Bad: true},
|
||||
},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := tpl.ExecuteTemplate(&buf, "layout", data); err != nil {
|
||||
t.Fatalf("渲染爬虫页失败: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
|
||||
for _, want := range []string{
|
||||
`id="running-body"`, // 轮询重建的目标容器
|
||||
"/admin/spider/live", // 轮询地址
|
||||
"12/47 场 · Fall 2021 Menswear(24/38 张)", // 进度文案原样输出
|
||||
"Celine", // 待抓队列
|
||||
"新建 34 张图", // 入库结果说明
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("页面缺少 %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
// 卡住要能看出来:进度单元格带 stale 样式。
|
||||
if !strings.Contains(out, "progress stale") {
|
||||
t.Error("疑似卡住的任务没有 stale 样式")
|
||||
}
|
||||
// 失败结果要走 failed 徽标,正常结果走 done 徽标。
|
||||
if !strings.Contains(out, `class="badge failed"`) {
|
||||
t.Error("失败结果没有 failed 徽标")
|
||||
}
|
||||
if !strings.Contains(out, `class="badge done"`) {
|
||||
t.Error("正常结果没有 done 徽标")
|
||||
}
|
||||
|
||||
// 队列被截断时要说清楚:否则「队列 1620 个」配一张 50 行的表会让人以为数据错了。
|
||||
data["QueuedCount"] = 1620
|
||||
data["QueuedShown"] = 50
|
||||
var buf2 bytes.Buffer
|
||||
if err := tpl.ExecuteTemplate(&buf2, "layout", data); err != nil {
|
||||
t.Fatalf("渲染截断提示失败: %v", err)
|
||||
}
|
||||
if !strings.Contains(buf2.String(), "只列前 50 个") {
|
||||
t.Error("队列被截断时缺少提示")
|
||||
}
|
||||
}
|
||||
@ -11,5 +11,47 @@
|
||||
<label>中文名(name_cn)</label><input name="name_cn" value="{{.NameCn}}" placeholder="如 艾克妮">
|
||||
<button type="submit">{{if .UID}}保存{{else}}创建{{end}}</button>
|
||||
</form>
|
||||
|
||||
{{if .UID}}
|
||||
<h2 style="margin-top:32px">抓取历史</h2>
|
||||
<p class="muted">每次抓取一条(每品牌保留最近 30 条)。这一层回答「什么时候爬的、爬了几场、哪次没抓到」。</p>
|
||||
{{if .CrawlLogs}}
|
||||
<table>
|
||||
<thead><tr><th>时间</th><th>结果</th><th>明细</th><th>耗时</th><th>执行者</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .CrawlLogs}}<tr>
|
||||
<td>{{.TimeText}}</td>
|
||||
<td>{{.StatusText}}</td>
|
||||
<td>{{.Detail}}</td>
|
||||
<td>{{if .Elapsed}}{{.Elapsed}}{{else}}-{{end}}</td>
|
||||
<td><code>{{if .Worker}}{{.Worker}}{{else}}-{{end}}</code></td>
|
||||
</tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{else}}
|
||||
<p class="muted">还没有抓取记录。把品牌加入抓取队列后,每次抓取(抓到几场 / 失败原因 / 谁跑的)都会出现在这里。</p>
|
||||
{{end}}
|
||||
|
||||
<h2 style="margin-top:32px">更新记录</h2>
|
||||
<p class="muted">
|
||||
{{if .LastCrawl}}最近抓取:{{.LastCrawl}} · {{.CrawlStatus}}{{else}}还没有抓取记录{{end}}
|
||||
(只显示最近 30 条入库记录;抓取侧每品牌只保留最近一次的状态)
|
||||
</p>
|
||||
{{if .Records}}
|
||||
<table>
|
||||
<thead><tr><th>时间</th><th>场次</th><th>结果</th><th>明细</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .Records}}<tr>
|
||||
<td>{{.TimeText}}</td>
|
||||
<td>{{if .Title}}{{.Title}}{{else}}-{{end}}</td>
|
||||
<td>{{.OutcomeText}}</td>
|
||||
<td>{{.Detail}}</td>
|
||||
</tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{else}}
|
||||
<p class="muted">还没有入库记录。把品牌加入抓取队列后,每一场的处理结果(新建 / 已存在跳过 / 失败原因)都会出现在这里。</p>
|
||||
{{end}}
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
@ -8,21 +8,65 @@
|
||||
<form class="toolbar" method="get" action="/admin/brands">
|
||||
<input name="keyword" placeholder="搜索品牌名…" value="{{.Keyword}}">
|
||||
{{if .All}}<input type="hidden" name="all" value="1">{{end}}
|
||||
{{if .Sort}}<input type="hidden" name="sort" value="{{.Sort}}">{{end}}
|
||||
<button type="submit">搜索</button>
|
||||
{{if .Keyword}}<a href="/admin/brands{{if .All}}?all=1{{end}}">清除</a>{{end}}
|
||||
<span class="spacer"></span>
|
||||
{{if .All}}<a class="view" href="/admin/brands?keyword={{.Keyword}}">只看有图集的品牌</a>{{else}}<a class="view" href="/admin/brands?keyword={{.Keyword}}&all=1">显示全部品牌(含无图集)</a>{{end}}
|
||||
</form>
|
||||
<p class="muted">默认只显示「已有图集」的品牌;品牌目录共 2 万余条,其余请用上面的搜索或开关查看。</p>
|
||||
|
||||
<p class="muted">
|
||||
默认只显示「已有图集」的品牌;品牌目录共 2 万余条,其余请用上面的搜索或开关查看。
|
||||
排序:
|
||||
{{if eq .Sort "updated"}}<strong>最近更新</strong>{{else}}<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}&sort=updated">最近更新</a>{{end}}
|
||||
·
|
||||
{{if eq .Sort "updated"}}<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}">按名字</a>{{else}}<strong>按名字</strong>{{end}}
|
||||
</p>
|
||||
|
||||
{{if .Added}}<p class="muted">✅ 本次已加入抓取队列 <strong>{{.Added}}</strong> 个品牌(已在队列中的不会重复添加)。</p>{{end}}
|
||||
|
||||
<div class="idx">
|
||||
<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}" class="{{if eq .Letter ""}}on{{end}}">全部</a>
|
||||
<a href="/admin/brands?letter=OTHER&keyword={{.Keyword}}{{if .All}}&all=1{{end}}" class="{{if eq .Letter "OTHER"}}on{{end}}">中文</a>
|
||||
{{range $l := .Letters}}<a href="/admin/brands?letter={{$l}}&keyword={{$.Keyword}}{{if $.All}}&all=1{{end}}" class="{{if eq $l $.Letter}}on{{end}}">{{$l}}</a>{{end}}
|
||||
<a href="/admin/brands?keyword={{.Keyword}}{{if .All}}&all=1{{end}}{{if .Sort}}&sort={{.Sort}}{{end}}" class="{{if eq .Letter ""}}on{{end}}">全部</a>
|
||||
<a href="/admin/brands?letter=OTHER&keyword={{.Keyword}}{{if .All}}&all=1{{end}}{{if .Sort}}&sort={{.Sort}}{{end}}" class="{{if eq .Letter "OTHER"}}on{{end}}">中文</a>
|
||||
{{range $l := .Letters}}<a href="/admin/brands?letter={{$l}}&keyword={{$.Keyword}}{{if $.All}}&all=1{{end}}{{if $.Sort}}&sort={{$.Sort}}{{end}}" class="{{if eq $l $.Letter}}on{{end}}">{{$l}}</a>{{end}}
|
||||
</div>
|
||||
|
||||
{{/* 批量操作表单:勾选框用 form="bulk" 关联,避免与行内表单嵌套 */}}
|
||||
<form id="bulk" method="post" action="/admin/spider/bulk"></form>
|
||||
<div class="ops" style="margin:10px 0">
|
||||
<button type="submit" form="bulk" class="op">把勾选的品牌加入抓取队列</button>
|
||||
<span class="muted">
|
||||
(勾选本页任意行,先点复选框列头可全选本页。抓取一律全量:
|
||||
已抓全的场次会被自动跳过,不用担心重复下载。)
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<thead><tr><th>编码 ID</th><th>英文名</th><th>中文名</th><th>操作</th></tr></thead>
|
||||
<thead><tr>
|
||||
<th><input type="checkbox" title="全选本页" onclick="document.querySelectorAll('input[name=uid]').forEach(function(c){c.checked=this.checked}.bind(this))"></th>
|
||||
<th>编码 ID</th><th>英文名</th><th>中文名</th><th>图集</th><th>图片</th><th>抓取状态</th><th>上次抓取</th><th>操作</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{{range .Items}}<tr><td><code>{{.UID}}</code></td><td>{{.NameEn}}</td><td>{{.NameCn}}</td><td><a class="view" href="/admin/brands/{{.UID}}">编辑</a><form class="inline" method="post" action="/admin/brands/{{.UID}}/delete" onsubmit="return confirm('确认删除品牌「{{.NameEn}}」?该品牌下若还有图集会拒绝删除。')"><button type="submit" class="op op-warn">删除</button></form></td></tr>{{end}}
|
||||
{{range .Items}}{{$cs := index $.CrawlStates .UID}}{{$st := index $.Stats .UID}}<tr>
|
||||
<td><input type="checkbox" name="uid" value="{{.UID}}" form="bulk"></td>
|
||||
<td><code>{{.UID}}</code></td>
|
||||
<td>{{.NameEn}}</td>
|
||||
<td>{{.NameCn}}</td>
|
||||
<td>{{if $st}}{{$st.Runways}}{{else}}0{{end}}</td>
|
||||
<td>{{if $st}}{{$st.Images}}{{else}}0{{end}}</td>
|
||||
<td>{{if $cs}}<span class="badge {{$cs.Class}}">{{$cs.Label}}</span>{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td class="muted">{{if $cs}}{{$cs.LastAt}}{{else}}—{{end}}</td>
|
||||
<td>
|
||||
<a class="view" href="/admin/brands/{{.UID}}">编辑</a>
|
||||
<form class="inline" method="post" action="/admin/spider/brands/{{.UID}}" onsubmit="return confirm('把「{{.NameEn}}」加入抓取队列?(全量,已抓全的场次自动跳过)')">
|
||||
<input type="hidden" name="back" value="/admin/brands">
|
||||
<button type="submit" class="op">加入队列</button>
|
||||
</form>
|
||||
<form class="inline" method="post" action="/admin/brands/{{.UID}}/delete" onsubmit="return confirm('确认删除品牌「{{.NameEn}}」?该品牌下若还有图集会拒绝删除。')">
|
||||
<button type="submit" class="op op-warn">删除</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{template "pager" .Pager}}
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
{{define "title"}}入库任务 · 管理后台{{end}}
|
||||
{{define "barLabel"}}管理后台 · 爬虫入库任务{{end}}
|
||||
{{define "barRight"}}<a href="/admin/logout">退出</a>{{end}}
|
||||
{{define "content"}}
|
||||
{{template "bar" .}}
|
||||
<div class="main">
|
||||
<h1>爬虫入库任务</h1>
|
||||
<p class="muted">worker 每 3 秒扫描一次 pending 任务并处理;failed 可一键重新入队。共 {{.Total}} 条,当前第 {{.Page}} / {{if eq .TotalPages 0}}1{{else}}{{.TotalPages}}{{end}} 页(每页 {{.PageSize}} 条)。</p>
|
||||
<table>
|
||||
<thead><tr><th>ID</th><th>状态</th><th>重试</th><th>错误信息</th><th>更新时间</th><th>操作</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .Jobs}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td><span class="badge {{.Status}}">{{.Status}}</span></td>
|
||||
<td>{{.Attempts}}</td>
|
||||
<td class="err break">{{.LastError}}</td>
|
||||
<td class="muted">{{.UpdatedAt}}</td>
|
||||
<td>{{if eq .Status "failed"}}<form class="retry" method="post" action="/admin/ingest-jobs/{{.ID}}/retry"><button class="retry" type="submit">重试</button></form>{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
</tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{template "pager" .Pager}}
|
||||
</div>
|
||||
{{end}}
|
||||
135
internal/handler/templates/pages/spider.html
Normal file
135
internal/handler/templates/pages/spider.html
Normal file
@ -0,0 +1,135 @@
|
||||
{{define "title"}}爬虫 · 管理后台{{end}}
|
||||
{{define "barLabel"}}管理后台 · 爬虫{{end}}
|
||||
{{define "barRight"}}<a href="/admin/brands">去品牌管理加任务</a> · <a href="/admin/logout">退出</a>{{end}}
|
||||
{{define "content"}}
|
||||
{{template "bar" .}}
|
||||
<div class="main">
|
||||
<h1>爬虫</h1>
|
||||
<p class="muted">
|
||||
这里看「爬虫现在在干什么」:<strong>正在抓</strong>是当下,
|
||||
<strong>待抓队列</strong>是排队等着的品牌,
|
||||
<strong>最近入库结果</strong>是后端已经处理完的场次。
|
||||
</p>
|
||||
<p class="muted">
|
||||
入队请去<a href="/admin/brands">品牌管理</a>勾选品牌。抓取一律全量,
|
||||
已经抓全的场次会被自动跳过,不会重复下载。
|
||||
</p>
|
||||
|
||||
<p class="muted" id="spider-status">
|
||||
爬虫 <span class="badge {{.StateClass}}" id="st-state">{{.State}}</span>
|
||||
· 正在抓 <strong id="st-running">{{.RunningCount}}</strong> 个
|
||||
· 队列 <strong id="st-queued">{{.QueuedCount}}</strong> 个
|
||||
· 待入库 <strong id="st-pending">{{.PendingIngest}}</strong> 条
|
||||
</p>
|
||||
|
||||
<h2>正在抓</h2>
|
||||
<table>
|
||||
<thead><tr><th>品牌</th><th>进度</th><th>心跳</th><th>Worker</th></tr></thead>
|
||||
<tbody id="running-body">
|
||||
{{range .Running}}<tr>
|
||||
<td><strong>{{.BrandName}}</strong></td>
|
||||
<td class="progress{{if .Stale}} stale{{end}}">{{if .Progress}}{{.Progress}}{{else}}刚领取,正在拉取秀列表…{{end}}</td>
|
||||
<td class="muted{{if .Stale}} stale{{end}}">{{.Beat}}前</td>
|
||||
<td class="muted">{{.Worker}}</td>
|
||||
</tr>{{end}}
|
||||
{{if not .Running}}<tr id="running-empty"><td colspan="4" class="muted">当前没有正在抓取的任务。</td></tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>待抓队列({{.QueuedCount}})</h2>
|
||||
{{if gt .QueuedCount .QueuedShown}}<p class="muted">队列里共 {{.QueuedCount}} 个,下面只列前 {{.QueuedShown}} 个。</p>{{end}}
|
||||
<table>
|
||||
<thead><tr><th>品牌</th><th>入队时间</th><th>已等待</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .Queued}}<tr>
|
||||
<td>{{.BrandName}}</td>
|
||||
<td class="muted">{{.Since}}</td>
|
||||
<td class="muted">{{.Waiting}}</td>
|
||||
</tr>{{end}}
|
||||
{{if not .Queued}}<tr><td colspan="3" class="muted">队列为空。</td></tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>最近入库结果</h2>
|
||||
<p class="muted">最近 50 条。「已存在跳过」是正常状态——说明这场秀之前抓过,没有重复写库。</p>
|
||||
<table>
|
||||
<thead><tr><th>时间</th><th>品牌</th><th>场次</th><th>结果</th><th>说明</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .Results}}<tr>
|
||||
<td class="muted">{{.At}}</td>
|
||||
<td>{{.BrandName}}</td>
|
||||
<td>{{.Title}}</td>
|
||||
<td>{{if .Bad}}<span class="badge failed">{{.Result}}</span>{{else}}<span class="badge done">{{.Result}}</span>{{end}}</td>
|
||||
<td class="err break">{{.Note}}</td>
|
||||
</tr>{{end}}
|
||||
{{if not .Results}}<tr><td colspan="5" class="muted">还没有入库记录。</td></tr>{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<script>
|
||||
// 抓取进度轮询:只刷新状态条与「正在抓」表格。
|
||||
//
|
||||
// 用 DOM API 建行而不是拼 innerHTML:品牌名来自数据库,直接拼字符串会把
|
||||
// 特殊字符当成标签解析(既会破版,也是一条注入面)。
|
||||
(function () {
|
||||
var body = document.getElementById('running-body');
|
||||
if (!body) return;
|
||||
|
||||
function cell(text, cls) {
|
||||
var td = document.createElement('td');
|
||||
td.textContent = text;
|
||||
if (cls) td.className = cls;
|
||||
return td;
|
||||
}
|
||||
|
||||
function paint(list) {
|
||||
while (body.firstChild) body.removeChild(body.firstChild);
|
||||
if (!list.length) {
|
||||
var tr0 = document.createElement('tr');
|
||||
var td0 = document.createElement('td');
|
||||
td0.colSpan = 4;
|
||||
td0.className = 'muted';
|
||||
td0.textContent = '当前没有正在抓取的任务。';
|
||||
tr0.appendChild(td0);
|
||||
body.appendChild(tr0);
|
||||
return;
|
||||
}
|
||||
list.forEach(function (it) {
|
||||
var tr = document.createElement('tr');
|
||||
|
||||
var tdBrand = document.createElement('td');
|
||||
var b = document.createElement('strong');
|
||||
b.textContent = it.brand || '—';
|
||||
tdBrand.appendChild(b);
|
||||
tr.appendChild(tdBrand);
|
||||
|
||||
tr.appendChild(cell(it.progress || '刚领取,正在拉取秀列表…', it.stale ? 'progress stale' : 'progress'));
|
||||
tr.appendChild(cell((it.beat || '0 秒') + '前', it.stale ? 'muted stale' : 'muted'));
|
||||
tr.appendChild(cell(it.worker || '—', 'muted'));
|
||||
body.appendChild(tr);
|
||||
});
|
||||
}
|
||||
|
||||
function tick() {
|
||||
fetch('/admin/spider/live', { credentials: 'same-origin' })
|
||||
.then(function (r) { return r.ok ? r.json() : null; })
|
||||
.then(function (d) {
|
||||
if (!d) return;
|
||||
var st = document.getElementById('st-state');
|
||||
if (st) { st.textContent = d.state; st.className = 'badge ' + d.state_class; }
|
||||
var rn = document.getElementById('st-running');
|
||||
if (rn) rn.textContent = (d.running || []).length;
|
||||
var qn = document.getElementById('st-queued');
|
||||
if (qn) qn.textContent = d.queued;
|
||||
var pn = document.getElementById('st-pending');
|
||||
if (pn) pn.textContent = d.pending_ingest;
|
||||
paint(d.running || []);
|
||||
})
|
||||
.catch(function () { /* 网络抖动忽略,下一轮再拉 */ });
|
||||
}
|
||||
|
||||
setInterval(tick, 4000);
|
||||
tick();
|
||||
})();
|
||||
</script>
|
||||
{{end}}
|
||||
@ -18,7 +18,7 @@
|
||||
<a href="/admin/" class="{{if eq .NavKey "dashboard"}}on{{end}}">首页</a>
|
||||
<a href="/admin/articles" class="{{if or (eq .NavKey "articles") (eq .NavKey "runway-edit") (eq .NavKey "snap-edit") (eq .NavKey "runway-detail") (eq .NavKey "street-snap-detail")}}on{{end}}">文章{{if gt .PendingReviews 0}}<span class="bar-badge">{{.PendingReviews}}</span>{{end}}</a>
|
||||
<a href="/admin/brands" class="{{if or (eq .NavKey "brands") (eq .NavKey "brand-form")}}on{{end}}">品牌</a>
|
||||
<a href="/admin/ingest-jobs" class="{{if eq .NavKey "ingest-jobs"}}on{{end}}">入库任务</a>
|
||||
<a href="/admin/spider" class="{{if eq .NavKey "spider"}}on{{end}}">爬虫</a>
|
||||
<a href="/admin/users" class="{{if eq .NavKey "users"}}on{{end}}">用户</a>
|
||||
<span class="sep"></span>
|
||||
{{template "barRight" .}}
|
||||
|
||||
30
internal/model/crawl_log.go
Normal file
30
internal/model/crawl_log.go
Normal file
@ -0,0 +1,30 @@
|
||||
package model
|
||||
|
||||
// CrawlLogKeepPerBrand 每个品牌保留的抓取历史条数。
|
||||
//
|
||||
// 抓取历史是「最近发生了什么」的排障材料,不是审计归档;留太多只会让查询变慢。
|
||||
// 30 条足够覆盖「最近一次批量重爬 + 若干次手动重试」。
|
||||
const CrawlLogKeepPerBrand = 30
|
||||
|
||||
// CrawlLog 品牌抓取历史的一行(append-only)。
|
||||
//
|
||||
// 与 ingest_jobs 的分工:
|
||||
// - CrawlLog 记录「这一次抓取」的进程级结果(抓到几场、报了什么错、谁跑的);
|
||||
// - ingest_jobs 记录「每一场秀」的入库结果(新建 / 跳过 / 失败)。
|
||||
// 两者按时间顺序摆在品牌详情页,就是完整的更新链路。
|
||||
type CrawlLog struct {
|
||||
ID uint32 `gorm:"primaryKey;column:id" json:"id"`
|
||||
BrandID uint32 `gorm:"column:brand_id;not null" json:"brand_id"`
|
||||
TaskID uint32 `gorm:"column:task_id;not null;default:0" json:"task_id"`
|
||||
Status string `gorm:"column:status;size:16;not null;default:''" json:"status"` // done / failed
|
||||
Attempt int16 `gorm:"column:attempt;not null;default:1" json:"attempt"`
|
||||
ShowsRequested int16 `gorm:"column:shows_requested;not null;default:0" json:"shows_requested"`
|
||||
ShowsSubmitted int32 `gorm:"column:shows_submitted;not null;default:0" json:"shows_submitted"`
|
||||
Message string `gorm:"column:message;type:text;not null;default:''" json:"message"`
|
||||
Worker string `gorm:"column:worker;size:64;not null;default:''" json:"worker"`
|
||||
StartedAt uint32 `gorm:"column:started_at;not null;default:0" json:"started_at"`
|
||||
FinishedAt uint32 `gorm:"column:finished_at;not null;default:0" json:"finished_at"`
|
||||
}
|
||||
|
||||
// TableName 指定表名。
|
||||
func (CrawlLog) TableName() string { return "brand_crawl_logs" }
|
||||
67
internal/model/crawl_task.go
Normal file
67
internal/model/crawl_task.go
Normal file
@ -0,0 +1,67 @@
|
||||
package model
|
||||
|
||||
// CrawlTask 后台派发给爬虫服务的「品牌抓取任务」。
|
||||
//
|
||||
// 与 ingest_jobs 的分工:
|
||||
// - ingest_jobs:一场走秀的入库任务,由后端 worker 消费(下载图 / 写库 / 编码);
|
||||
// - crawl_tasks:一个品牌的抓取任务,由 spider 服务领取执行(可能在另一台机器)。
|
||||
//
|
||||
// 两层解耦的好处:后台只需表达「我要爬哪些品牌」,不必关心 spider 在哪、跑几个。
|
||||
// spider 拉不到任务就空转,任务积压也不会影响已有数据的读写。
|
||||
const (
|
||||
CrawlQueued = "queued" // 待领取
|
||||
CrawlRunning = "running" // 已被某个 worker 领取
|
||||
CrawlDone = "done" // 执行完成(不代表每场都入库成功,见 last_error)
|
||||
CrawlFailed = "failed" // 执行失败(spider 进程报错 / 品牌在 Vogue 无页面等)
|
||||
CrawlCanceled = "canceled" // 后台手动取消
|
||||
)
|
||||
|
||||
// CrawlStuckTimeoutSec 是 running 任务的「静默时长」阈值:领取后这么久既没回报、
|
||||
// 也没有进度心跳,才视为 worker 已崩溃(断电 / 被杀),任务可被重新领取。
|
||||
//
|
||||
// 300 秒而不是更长:卡住后 5 分钟内自动回收,不必人工干预。
|
||||
// 判活同时看 progress_at(spider 每 60 秒打一次心跳)——只看 locked_at 会把
|
||||
// 跑了十几分钟的正常任务误判成卡死(一次全量抓取可能要几个小时)。
|
||||
const CrawlStuckTimeoutSec = 300
|
||||
|
||||
// CrawlMaxAttempts 自动重试上限;达到后置 failed,需后台手动「重试」。
|
||||
const CrawlMaxAttempts = 3
|
||||
|
||||
// CrawlTask 品牌抓取任务。
|
||||
// 时间字段与项目其它表一致,统一用 Unix 秒(uint32)。
|
||||
type CrawlTask struct {
|
||||
ID uint32 `gorm:"primaryKey;column:id" json:"id"`
|
||||
BrandID uint32 `gorm:"column:brand_id;not null" json:"brand_id"`
|
||||
// Status 见上方常量;默认 queued。
|
||||
Status string `gorm:"column:status;size:16;not null;default:'queued'" json:"status"`
|
||||
// Priority 越大越先被领取;用于「手动加的」优先于「批量加的」。
|
||||
Priority int16 `gorm:"column:priority;not null;default:0" json:"priority"`
|
||||
// MaxShows 单品牌最多抓几场(0=不限)。批量重爬历史品牌时用它控制入库量。
|
||||
MaxShows int16 `gorm:"column:max_shows;not null;default:0" json:"max_shows"`
|
||||
// Worker 领取者标识(主机名 + 进程号),便于排查「谁跑的」。
|
||||
Worker string `gorm:"column:worker;size:64;not null;default:''" json:"worker"`
|
||||
// LockedAt 领取时间,配合 CrawlStuckTimeoutSec 做租约回收。
|
||||
LockedAt uint32 `gorm:"column:locked_at;not null;default:0" json:"locked_at"`
|
||||
// Attempts 已被领取次数;用于自动重试与放弃判定。
|
||||
Attempts int16 `gorm:"column:attempts;not null;default:0" json:"attempts"`
|
||||
// ShowsSubmitted 本任务向 ingest 提交了多少场走秀(spider 回报)。
|
||||
ShowsSubmitted int32 `gorm:"column:shows_submitted;not null;default:0" json:"shows_submitted"`
|
||||
LastError string `gorm:"column:last_error;type:text" json:"last_error"`
|
||||
// 进度:spider 每处理完一场秀上报一次。
|
||||
// 一次抓取可能跑几十分钟,没有这四项后台只能看到「执行中」,分不清慢和卡死。
|
||||
ProgressDone int16 `gorm:"column:progress_done;not null;default:0" json:"progress_done"`
|
||||
ProgressTotal int16 `gorm:"column:progress_total;not null;default:0" json:"progress_total"`
|
||||
ProgressText string `gorm:"column:progress_text;size:160;not null;default:''" json:"progress_text"`
|
||||
ProgressAt uint32 `gorm:"column:progress_at;not null;default:0" json:"progress_at"`
|
||||
CreatedAt uint32 `gorm:"column:created_at;not null;default:0" json:"created_at"`
|
||||
UpdatedAt uint32 `gorm:"column:updated_at;not null;default:0" json:"updated_at"`
|
||||
}
|
||||
|
||||
// TableName 指定表名。
|
||||
func (CrawlTask) TableName() string { return "crawl_tasks" }
|
||||
|
||||
// CrawlTaskRow 是列表/领取接口用的投影行,附带品牌名(列表展示与拼 Vogue URL 都用得上)。
|
||||
type CrawlTaskRow struct {
|
||||
CrawlTask
|
||||
BrandName string `gorm:"column:brand_name"`
|
||||
}
|
||||
@ -19,8 +19,26 @@ type IngestJob struct {
|
||||
LastError string `gorm:"column:last_error;type:text" json:"last_error"`
|
||||
Kind string `gorm:"column:kind;size:16;not null;default:'crawl'" json:"kind"` // crawl|media_cleanup
|
||||
NextAttemptAt uint32 `gorm:"column:next_attempt_at;not null;default:0" json:"next_attempt_at"` // 重试退避:到达该 unix 时间才可被领取
|
||||
// Outcome 结果类型(见 IngestOutcome* 常量),Note 是人可读的细节。
|
||||
// 只有 done 状态才有意义:同为 done,「新建 42 张图」和「实体已存在跳过」是完全不同的事,
|
||||
// 后台的「品牌更新记录」靠这两个字段说清楚每次到底做了什么。
|
||||
Outcome string `gorm:"column:outcome;size:24;not null;default:''" json:"outcome"`
|
||||
Note string `gorm:"column:note;type:text;not null;default:''" json:"note"`
|
||||
// BrandID 提交时就解出的品牌 id(街拍为 0),供品牌详情页直接按品牌查更新记录。
|
||||
BrandID uint32 `gorm:"column:brand_id;not null;default:0" json:"brand_id"`
|
||||
}
|
||||
|
||||
// 入库结果类型(ingest_jobs.outcome)。
|
||||
const (
|
||||
IngestOutcomeCreated = "created" // 新建入站
|
||||
IngestOutcomeReused = "reused_rejected" // 复用被驳回的记录并覆盖
|
||||
IngestOutcomeSkippedExisting = "skipped_existing" // 实体已存在,跳过(不重下图片)
|
||||
IngestOutcomeSkippedDeleted = "skipped_deleted" // 实体已软删,跳过(删除即永久)
|
||||
IngestOutcomeRepaired = "repaired" // 实体已存在,但补齐了缺失的图片(覆盖率不足时重爬)
|
||||
IngestOutcomeCleaned = "cleaned" // 媒体清理任务
|
||||
IngestOutcomeNoop = "noop" // 无需处理(空载荷)
|
||||
)
|
||||
|
||||
// TableName 指定队列表名。
|
||||
func (IngestJob) TableName() string { return "ingest_jobs" }
|
||||
|
||||
|
||||
@ -14,6 +14,11 @@ type BrandRunway struct {
|
||||
UpdatedAt uint32 `gorm:"column:updated_at" json:"updated_at"`
|
||||
IsDeleted uint8 `gorm:"column:is_deleted" json:"is_deleted"`
|
||||
ImageCount uint16 `gorm:"column:image_count" json:"image_count"`
|
||||
// ExpectedImages 该场秀应有的图片总数(spider 上报)。0 = 未知(本字段上线前的历史数据)。
|
||||
//
|
||||
// 用途:重爬前的覆盖率判定——实际张数接近应有张数才算「抓全了」。
|
||||
// 只看「有没有图」会把当初只抓到十几张的场次永久判成已完成,缺的图再也补不上。
|
||||
ExpectedImages int `gorm:"column:expected_images;not null;default:0" json:"expected_images"`
|
||||
BrandID uint32 `gorm:"column:brand_id" json:"brand_id"`
|
||||
Year uint16 `gorm:"column:year" json:"year"`
|
||||
Cover string `gorm:"column:cover" json:"cover"`
|
||||
|
||||
@ -46,6 +46,16 @@ func prefixOf(collectionType, season string) string {
|
||||
// 高定沿用秋冬编码(FW),与排序优先级一致
|
||||
return "FW"
|
||||
case ColMenswear:
|
||||
// 男装必须按真实季节区分:曾经一律返回 FW,导致同年的
|
||||
// 「Spring 2021 Menswear」与「Fall 2021 Menswear」推出同一个 season_code(FW21),
|
||||
// 而实体键是(品牌 + season_code + 系列),于是后到的那场被判「已存在」静默丢弃。
|
||||
// 实测 AMBUSH:Vogue 25 场只入库 21 场,缺的 4 场全是男装。
|
||||
switch season {
|
||||
case SeasonSpring:
|
||||
return "SS"
|
||||
case SeasonFall:
|
||||
return "FW"
|
||||
}
|
||||
return "FW"
|
||||
}
|
||||
switch season {
|
||||
|
||||
@ -3,8 +3,10 @@ package storage
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
@ -67,6 +69,38 @@ func NewS4Uploader(ak, sk, bucket, endpoint, baseURL, region string) *S4Uploader
|
||||
}
|
||||
}
|
||||
|
||||
// PresignPut 为指定 key 生成有时效的 PUT 预签名 URL。
|
||||
//
|
||||
// 用途:让**不持有 S4 密钥**的客户端(例如另部署的 spider 媒体机)直传对象。
|
||||
// 密钥只留在后端,客户端拿到 URL 直接 HTTP PUT 即可 —— 不必在爬虫里引入 S3 SDK,
|
||||
// 也不用把 AK/SK 分发到更多机器。
|
||||
//
|
||||
// contentType 必须与 PUT 请求头一致,否则 S4 会因签名不匹配拒绝(403)。
|
||||
func (u *S4Uploader) PresignPut(ctx context.Context, key, contentType string, ttl time.Duration) (string, error) {
|
||||
if !u.enabled || u.client == nil {
|
||||
return "", errors.New("S4 未启用(缺少 AK/SK 或 bucket)")
|
||||
}
|
||||
if key == "" {
|
||||
return "", errors.New("key 不能为空")
|
||||
}
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = 15 * time.Minute
|
||||
}
|
||||
pc := s3.NewPresignClient(u.client)
|
||||
out, err := pc.PresignPutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(u.bucket),
|
||||
Key: aws.String(key),
|
||||
ContentType: aws.String(contentType),
|
||||
}, s3.WithPresignExpires(ttl))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return out.URL, nil
|
||||
}
|
||||
|
||||
// deriveS4BaseURL 由 endpoint + bucket 推导虚拟主机风格访问域名:
|
||||
// endpoint=https://s3.bitiful.net, bucket=toom → https://toom.s3.bitiful.net
|
||||
func deriveS4BaseURL(endpoint, bucket string) string {
|
||||
|
||||
@ -52,6 +52,16 @@ type BrandRepository interface {
|
||||
// 规则:先按品牌总图片数取前 limit 个热门品牌,再为每个品牌取最新一篇走秀。
|
||||
// 返回的是仓储层裸数据(数字主键),对外编码由 service 层负责。
|
||||
PopularWithCover(ctx context.Context, limit int) ([]PopularBrandRow, error)
|
||||
|
||||
// StatsByIDs 批量统计若干品牌的「图集数 / 图片数」,供后台列表展示数据量。
|
||||
// 只查当前页的 id,避免为了两列统计把整张品牌表都聚合一遍。
|
||||
StatsByIDs(ctx context.Context, ids []uint32) (map[uint32]BrandStats, error)
|
||||
}
|
||||
|
||||
// BrandStats 品牌数据量统计(后台列表用)。
|
||||
type BrandStats struct {
|
||||
Runways int64 // 未删除的图集数
|
||||
Images int64 // 未删除的图片数
|
||||
}
|
||||
|
||||
// PopularBrandRow 热门品牌裸数据:品牌数字主键 + 中英文名称 + 代表走秀封面/中英文标题/文章 ID。
|
||||
@ -174,11 +184,30 @@ func (r *brandRepository) list(ctx context.Context, q dto.BrandQuery, restrictID
|
||||
return []model.Brand{}, 0, nil
|
||||
}
|
||||
|
||||
var items []model.Brand
|
||||
if err := r.db.WithContext(ctx).
|
||||
// 后台可按「最近有动静」排序:走秀图集的 updated_at 与抓取任务的 updated_at 取大。
|
||||
//
|
||||
// 性能注意:不要写成逐行的相关子查询(2.3 万品牌 × 2 个子查询实测 2.7 秒)。
|
||||
// 这里先把两张小表各自按品牌聚合(brand_runways / crawl_tasks 都只有几百行),
|
||||
// 再 LEFT JOIN 到品牌表排序 —— 一次哈希连接 + 一次排序,量级降到百毫秒。
|
||||
// 公开侧不传 Sort,保持原有 name_en ASC,前台顺序不变。
|
||||
itemsQ := r.db.WithContext(ctx).
|
||||
Model(&model.Brand{}).
|
||||
Scopes(scope).
|
||||
Order("name_en ASC").
|
||||
Scopes(scope)
|
||||
if admin && q.Sort == "updated" {
|
||||
itemsQ = itemsQ.
|
||||
Joins(`LEFT JOIN (SELECT brand_id, max(updated_at) AS u
|
||||
FROM brand_runways WHERE is_deleted = 0 GROUP BY brand_id) br
|
||||
ON br.brand_id = brands.id`).
|
||||
Joins(`LEFT JOIN (SELECT brand_id, max(updated_at) AS u
|
||||
FROM crawl_tasks GROUP BY brand_id) ct
|
||||
ON ct.brand_id = brands.id`).
|
||||
Order("GREATEST(coalesce(br.u, 0), coalesce(ct.u, 0)) DESC, name_en ASC")
|
||||
} else {
|
||||
itemsQ = itemsQ.Order("name_en ASC")
|
||||
}
|
||||
|
||||
var items []model.Brand
|
||||
if err := itemsQ.
|
||||
Offset(q.Offset()).
|
||||
Limit(q.Size).
|
||||
Find(&items).Error; err != nil {
|
||||
@ -319,6 +348,36 @@ func (r *brandRepository) CrawlBrands(ctx context.Context, brandID uint32) ([]mo
|
||||
// 3. 结果按热门品牌排名顺序返回
|
||||
//
|
||||
// 注意:created_at/updated_at 是数据导入时间,不能代表走秀新旧,因此必须用 year + season_code。
|
||||
// StatsByIDs 批量统计图集数 / 图片数(只针对传入的 id,通常是一页 30 个)。
|
||||
func (r *brandRepository) StatsByIDs(ctx context.Context, ids []uint32) (map[uint32]BrandStats, error) {
|
||||
out := make(map[uint32]BrandStats, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var rows []struct {
|
||||
BrandID uint32
|
||||
Runways int64
|
||||
Images int64
|
||||
}
|
||||
// 一次查询同时拿两个计数:按品牌聚合图集,再左连图片聚合,避免 N+1。
|
||||
err := r.db.WithContext(ctx).Raw(`
|
||||
SELECT r.brand_id,
|
||||
count(DISTINCT r.id) AS runways,
|
||||
count(i.id) AS images
|
||||
FROM brand_runways r
|
||||
LEFT JOIN brand_runway_images i
|
||||
ON i.runway_id = r.id AND i.is_deleted = 0
|
||||
WHERE r.is_deleted = 0 AND r.brand_id IN ?
|
||||
GROUP BY r.brand_id`, ids).Scan(&rows).Error
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
out[row.BrandID] = BrandStats{Runways: row.Runways, Images: row.Images}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *brandRepository) PopularWithCover(ctx context.Context, limit int) ([]PopularBrandRow, error) {
|
||||
if limit < 1 {
|
||||
limit = 20
|
||||
|
||||
489
internal/repository/crawl_repository.go
Normal file
489
internal/repository/crawl_repository.go
Normal file
@ -0,0 +1,489 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"fashionapi/internal/model"
|
||||
)
|
||||
|
||||
// CrawlRepository 品牌抓取任务(crawl_tasks)的数据访问。
|
||||
//
|
||||
// 任务表同时被两个方向使用:
|
||||
// - 后台(cookie 会话)往里加任务 / 取消 / 重试;
|
||||
// - spider 服务(HMAC 验签)来领取 / 回报结果。
|
||||
//
|
||||
// 领取用 `FOR UPDATE SKIP LOCKED`,多个 spider 实例并发领取不会抢到同一条。
|
||||
type CrawlRepository interface {
|
||||
// Enqueue 把单个品牌加入队列。
|
||||
// 返回 true 表示新入队或把已完成/失败的任务重置为待领取;false 表示该品牌已在队列中/正在跑。
|
||||
Enqueue(ctx context.Context, brandID uint32, maxShows int, priority int) (bool, error)
|
||||
// EnqueueBulk 按「范围 + 前缀」批量入队,返回本次实际新入队(或重置)的数量。
|
||||
EnqueueBulk(ctx context.Context, f BulkFilter) (int, error)
|
||||
// Claim 领取一个待执行任务并置为 running;无任务时返回 nil, nil。
|
||||
Claim(ctx context.Context, worker string) (*model.CrawlTaskRow, error)
|
||||
// Report spider 回报结果。failed 且未达重试上限时自动回到 queued。
|
||||
Report(ctx context.Context, id uint32, status string, shows int, errMsg string) error
|
||||
// List 后台列表(按 running → queued → 其它 排序)。
|
||||
List(ctx context.Context, offset, limit int) ([]model.CrawlTaskRow, int64, error)
|
||||
// SetStatus 后台手动改状态(取消 / 重试)。
|
||||
SetStatus(ctx context.Context, id uint32, status string) error
|
||||
// Delete 删除任务记录(不影响已入库的数据)。
|
||||
Delete(ctx context.Context, id uint32) error
|
||||
// Counts 各状态任务数,供导航角标使用。
|
||||
Counts(ctx context.Context) (map[string]int64, error)
|
||||
// UpdateProgress 记录执行进度(spider 每处理完一场秀上报一次)。
|
||||
// 只对 running 生效:任务已被取消/回收时,迟到的进度不应把状态写回去。
|
||||
UpdateProgress(ctx context.Context, id uint32, done, total int, text string) error
|
||||
// ListActive 取 running + queued 的任务(后台进度轮询用)。
|
||||
ListActive(ctx context.Context) ([]model.CrawlTaskRow, error)
|
||||
// BrandCrawlStates 返回这些品牌在抓取队列里的「最近状态 + 最近时间」。
|
||||
// 品牌列表用它展示「抓取状态 / 上次抓取」;没有任务记录的品牌不会出现在 map 里。
|
||||
BrandCrawlStates(ctx context.Context, brandIDs []uint32) (map[uint32]BrandCrawlState, error)
|
||||
// ListCrawlLogs 取某品牌最近的抓取历史(id 倒序),品牌详情页展示用。
|
||||
//
|
||||
// crawl_tasks 每品牌只有一行且会被覆盖,历史只存在 brand_crawl_logs 里。
|
||||
ListCrawlLogs(ctx context.Context, brandID uint32, limit int) ([]model.CrawlLog, error)
|
||||
// ShowCoverage 返回某品牌「已入库的秀 → 覆盖率」,键是实体键(年份+季节+类型)。
|
||||
//
|
||||
// 用途:spider 抓取前预检,整场跳过已经抓全的秀。
|
||||
// 没有它,重爬会把几百张已经下过的图再下一遍——图片 key 是内容 sha1,
|
||||
// 不下载就无法知道它其实已经存在,带宽全白烧(线上实测 62 分钟 0 新增)。
|
||||
ShowCoverage(ctx context.Context, brandID uint32) (map[ShowKey]ShowCoverageStat, error)
|
||||
}
|
||||
|
||||
// ShowCoverageStat 一场秀的图片覆盖情况。
|
||||
//
|
||||
// Expected 为 0 表示未知(该字段上线前入库的历史数据),调用方应退化为按张数阈值判断。
|
||||
type ShowCoverageStat struct {
|
||||
Images int // 库里实际图片张数
|
||||
Expected int // 应有张数(spider 上报,0=未知)
|
||||
}
|
||||
|
||||
// ShowKey 一场秀的业务主键,与 ingest 的实体键一致(品牌 + 年份 + 季节 + 类型)。
|
||||
type ShowKey struct {
|
||||
Year uint16
|
||||
Season string
|
||||
CollectionType string
|
||||
}
|
||||
|
||||
// BrandCrawlState 品牌最近一次抓取任务的状态快照。
|
||||
type BrandCrawlState struct {
|
||||
Status string // queued / running / done / failed / canceled
|
||||
UpdatedAt time.Time // 该任务最后一次更新时间
|
||||
}
|
||||
|
||||
type crawlRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewCrawlRepository 创建爬取任务仓储。
|
||||
func NewCrawlRepository(db *gorm.DB) CrawlRepository {
|
||||
return &crawlRepository{db: db}
|
||||
}
|
||||
|
||||
const crawlColumns = `t.id, t.brand_id, t.status, t.priority, t.max_shows, t.worker,
|
||||
t.locked_at, t.attempts, t.shows_submitted, t.last_error,
|
||||
t.progress_done, t.progress_total, t.progress_text, t.progress_at,
|
||||
t.created_at, t.updated_at,
|
||||
coalesce(b.name_en, '') AS brand_name`
|
||||
|
||||
func (r *crawlRepository) Enqueue(ctx context.Context, brandID uint32, maxShows int, priority int) (bool, error) {
|
||||
now := uint32(time.Now().Unix())
|
||||
// INSERT ... SELECT 保证品牌不存在(或已软删)时不会凭空建任务;
|
||||
// ON CONFLICT 的 WHERE 让「已在排队/正在跑」的任务不被重置,避免打断正在执行的工作。
|
||||
res := r.db.WithContext(ctx).Exec(`
|
||||
INSERT INTO crawl_tasks (brand_id, status, priority, max_shows, created_at, updated_at)
|
||||
SELECT b.id, ?, ?, ?, ?, ?
|
||||
FROM brands b
|
||||
WHERE b.id = ? AND b.is_deleted = 0
|
||||
ON CONFLICT (brand_id) DO UPDATE
|
||||
SET status = 'queued',
|
||||
priority = EXCLUDED.priority,
|
||||
max_shows = EXCLUDED.max_shows,
|
||||
attempts = 0,
|
||||
locked_at = 0,
|
||||
worker = '',
|
||||
shows_submitted = 0,
|
||||
last_error = '',
|
||||
updated_at = EXCLUDED.updated_at
|
||||
WHERE crawl_tasks.status IN ('done', 'failed', 'canceled')`,
|
||||
model.CrawlQueued, priority, maxShows, now, now, brandID)
|
||||
if res.Error != nil {
|
||||
return false, res.Error
|
||||
}
|
||||
return res.RowsAffected > 0, nil
|
||||
}
|
||||
|
||||
// BulkScope 批量入队的品牌范围。
|
||||
type BulkScope string
|
||||
|
||||
const (
|
||||
// ScopeAll 所有品牌(首次全站导入用,量大)。
|
||||
ScopeAll BulkScope = "all"
|
||||
// ScopeWithData 只挑「已有图集」的品牌 —— 用于增量更新:只补最新场次。
|
||||
ScopeWithData BulkScope = "with-data"
|
||||
// ScopeWithoutData 只挑「还没有任何图集」的品牌 —— 用于首次补全。
|
||||
ScopeWithoutData BulkScope = "without-data"
|
||||
)
|
||||
|
||||
// BulkFilter 批量入队条件。
|
||||
type BulkFilter struct {
|
||||
Prefix string // 品牌英文名前缀,空 = 不限
|
||||
Scope BulkScope // 范围(见上方常量)
|
||||
MaxShows int // 0 = 不限场次(全量),>0 = 只抓最近 N 场
|
||||
Limit int // 本次最多入队多少个品牌(防止一次灌爆)
|
||||
Priority int16 // 越大越先被领取
|
||||
}
|
||||
|
||||
func (r *crawlRepository) EnqueueBulk(ctx context.Context, f BulkFilter) (int, error) {
|
||||
now := uint32(time.Now().Unix())
|
||||
if f.Limit < 1 || f.Limit > 5000 {
|
||||
f.Limit = 500
|
||||
}
|
||||
if f.Scope == "" {
|
||||
f.Scope = ScopeAll
|
||||
}
|
||||
|
||||
where := []string{"b.is_deleted = 0", "b.name_en <> ''"}
|
||||
// 注意:参数顺序必须与 SQL 里 ? 出现的顺序严格一致(INSERT 的 4 个 + 每个 WHERE 条件 + LIMIT)。
|
||||
args := []any{model.CrawlQueued, f.Priority, f.MaxShows, now, now}
|
||||
|
||||
if f.Prefix != "" {
|
||||
where = append(where, `b.name_en ILIKE ? ESCAPE '\'`)
|
||||
args = append(args, escapeLike(f.Prefix)+"%")
|
||||
}
|
||||
switch f.Scope {
|
||||
case ScopeWithData:
|
||||
where = append(where, "EXISTS (SELECT 1 FROM brand_runways r WHERE r.brand_id = b.id AND r.is_deleted = 0)")
|
||||
case ScopeWithoutData:
|
||||
where = append(where, "NOT EXISTS (SELECT 1 FROM brand_runways r WHERE r.brand_id = b.id AND r.is_deleted = 0)")
|
||||
}
|
||||
args = append(args, f.Limit)
|
||||
|
||||
sql := `
|
||||
INSERT INTO crawl_tasks (brand_id, status, priority, max_shows, created_at, updated_at)
|
||||
SELECT b.id, ?, ?, ?, ?, ?
|
||||
FROM brands b
|
||||
WHERE ` + strings.Join(where, " AND ") + `
|
||||
ORDER BY EXISTS (SELECT 1 FROM crawl_tasks t WHERE t.brand_id = b.id) ASC, b.id ASC
|
||||
LIMIT ?
|
||||
ON CONFLICT (brand_id) DO UPDATE
|
||||
SET status = 'queued',
|
||||
priority = EXCLUDED.priority,
|
||||
max_shows = EXCLUDED.max_shows,
|
||||
attempts = 0,
|
||||
locked_at = 0,
|
||||
worker = '',
|
||||
shows_submitted = 0,
|
||||
last_error = '',
|
||||
updated_at = EXCLUDED.updated_at
|
||||
WHERE crawl_tasks.status IN ('done', 'failed', 'canceled')`
|
||||
|
||||
res := r.db.WithContext(ctx).Exec(sql, args...)
|
||||
if res.Error != nil {
|
||||
return 0, res.Error
|
||||
}
|
||||
return int(res.RowsAffected), nil
|
||||
}
|
||||
|
||||
// escapeLike 转义 LIKE 的通配符,让用户输入按字面量匹配。
|
||||
func escapeLike(s string) string {
|
||||
out := make([]rune, 0, len(s))
|
||||
for _, c := range s {
|
||||
switch c {
|
||||
case '%', '_', '\\':
|
||||
out = append(out, '\\')
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func (r *crawlRepository) Claim(ctx context.Context, worker string) (*model.CrawlTaskRow, error) {
|
||||
now := uint32(time.Now().Unix())
|
||||
|
||||
tx := r.db.WithContext(ctx).Begin()
|
||||
if tx.Error != nil {
|
||||
return nil, tx.Error
|
||||
}
|
||||
defer func() {
|
||||
if tx.Error != nil {
|
||||
tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
// 回收「静默」的 running(worker 崩溃 / 断电),否则任务永远卡着。
|
||||
//
|
||||
// 存活判定用**两个信号取较晚者**:locked_at(领取时刻)与 progress_at(进度心跳)。
|
||||
// 只看 locked_at 会误杀长任务——一次全量抓取可能跑几个小时,而 locked_at 一写就不再更新;
|
||||
// 加上心跳后,正常在跑的任务永远不会被回收,只有真的 5 分钟没动静才落回队列。
|
||||
if now > model.CrawlStuckTimeoutSec {
|
||||
cutoff := now - model.CrawlStuckTimeoutSec
|
||||
if err := tx.Exec(`
|
||||
UPDATE crawl_tasks
|
||||
SET status = 'queued', locked_at = 0, worker = '', updated_at = ?
|
||||
WHERE status = 'running' AND locked_at > 0
|
||||
AND locked_at < ? AND coalesce(progress_at, 0) < ?`,
|
||||
now, cutoff, cutoff).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var id uint32
|
||||
if err := tx.Raw(`
|
||||
SELECT id FROM crawl_tasks
|
||||
WHERE status = 'queued'
|
||||
ORDER BY priority DESC, id ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED`).Scan(&id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == 0 {
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := tx.Exec(`
|
||||
UPDATE crawl_tasks
|
||||
SET status = 'running', worker = ?, locked_at = ?, attempts = attempts + 1, updated_at = ?
|
||||
WHERE id = ?`, worker, now, now, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var row model.CrawlTaskRow
|
||||
if err := r.db.WithContext(ctx).Raw(`
|
||||
SELECT `+crawlColumns+`
|
||||
FROM crawl_tasks t LEFT JOIN brands b ON b.id = t.brand_id
|
||||
WHERE t.id = ?`, id).Scan(&row).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &row, nil
|
||||
}
|
||||
|
||||
func (r *crawlRepository) Report(ctx context.Context, id uint32, status string, shows int, errMsg string) error {
|
||||
now := uint32(time.Now().Unix())
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var cur model.CrawlTask
|
||||
if err := tx.Where("id = ?", id).First(&cur).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
next := status
|
||||
// 失败但还有重试额度 → 回到队列,由任意 worker 重试(网络抖动不必人工介入)。
|
||||
if status == model.CrawlFailed && cur.Attempts < model.CrawlMaxAttempts {
|
||||
next = model.CrawlQueued
|
||||
}
|
||||
if err := tx.Model(&model.CrawlTask{}).Where("id = ?", id).
|
||||
Updates(map[string]any{
|
||||
"status": next,
|
||||
"shows_submitted": shows,
|
||||
"last_error": errMsg,
|
||||
"locked_at": 0,
|
||||
"worker": "",
|
||||
"updated_at": now,
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 同事务写一条抓取历史:crawl_tasks 每品牌只有一行会被覆盖,
|
||||
// 不在这里留痕就永远查不到「上次爬是什么时候、抓到几场」。
|
||||
log := model.CrawlLog{
|
||||
BrandID: cur.BrandID,
|
||||
TaskID: cur.ID,
|
||||
Status: status,
|
||||
Attempt: cur.Attempts + 1,
|
||||
ShowsRequested: cur.MaxShows,
|
||||
ShowsSubmitted: int32(shows),
|
||||
Message: errMsg,
|
||||
Worker: cur.Worker,
|
||||
StartedAt: cur.LockedAt,
|
||||
FinishedAt: now,
|
||||
}
|
||||
if err := tx.Create(&log).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 只保留该品牌最近 N 条:历史用于排障,不是归档,避免无限增长。
|
||||
return tx.Exec(
|
||||
"DELETE FROM brand_crawl_logs WHERE brand_id = ? AND id < "+
|
||||
"(SELECT id FROM brand_crawl_logs WHERE brand_id = ? ORDER BY id DESC OFFSET ? LIMIT 1)",
|
||||
cur.BrandID, cur.BrandID, model.CrawlLogKeepPerBrand).Error
|
||||
})
|
||||
}
|
||||
|
||||
func (r *crawlRepository) List(ctx context.Context, offset, limit int) ([]model.CrawlTaskRow, int64, error) {
|
||||
if limit < 1 {
|
||||
limit = 30
|
||||
}
|
||||
var total int64
|
||||
if err := r.db.WithContext(ctx).Table("crawl_tasks").Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var rows []model.CrawlTaskRow
|
||||
err := r.db.WithContext(ctx).Raw(`
|
||||
SELECT `+crawlColumns+`
|
||||
FROM crawl_tasks t LEFT JOIN brands b ON b.id = t.brand_id
|
||||
ORDER BY CASE t.status WHEN 'running' THEN 0 WHEN 'queued' THEN 1 ELSE 2 END,
|
||||
t.priority DESC, t.id DESC
|
||||
LIMIT ? OFFSET ?`, limit, offset).Scan(&rows).Error
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return rows, total, nil
|
||||
}
|
||||
|
||||
// UpdateProgress 写入进度心跳。
|
||||
//
|
||||
// 用 `AND status = 'running'` 而不是无条件更新:
|
||||
// 后台可能刚把任务取消,此时迟到的进度不该把它写活,
|
||||
// 也不该刷新 updated_at 掩盖「已经被取消」的事实。
|
||||
func (r *crawlRepository) UpdateProgress(ctx context.Context, id uint32, done, total int, text string) error {
|
||||
if done < 0 {
|
||||
done = 0
|
||||
}
|
||||
if total < 0 {
|
||||
total = 0
|
||||
}
|
||||
if len(text) > 160 {
|
||||
text = text[:160]
|
||||
}
|
||||
now := uint32(time.Now().Unix())
|
||||
return r.db.WithContext(ctx).Exec(
|
||||
`UPDATE crawl_tasks
|
||||
SET progress_done = ?, progress_total = ?, progress_text = ?,
|
||||
progress_at = ?, updated_at = ?
|
||||
WHERE id = ? AND status = 'running'`,
|
||||
done, total, text, now, now, id).Error
|
||||
}
|
||||
|
||||
// ListActive 取正在跑与排队中的任务,供后台进度轮询(全量返回,不分页)。
|
||||
// 只取 runnning/queued:已完成的任务进度不会再变,没必要每 4 秒重传一遍。
|
||||
func (r *crawlRepository) ListActive(ctx context.Context) ([]model.CrawlTaskRow, error) {
|
||||
var rows []model.CrawlTaskRow
|
||||
err := r.db.WithContext(ctx).Raw(`
|
||||
SELECT `+crawlColumns+`
|
||||
FROM crawl_tasks t LEFT JOIN brands b ON b.id = t.brand_id
|
||||
WHERE t.status IN ('running', 'queued')
|
||||
ORDER BY CASE t.status WHEN 'running' THEN 0 ELSE 1 END, t.id DESC
|
||||
LIMIT 200`).Scan(&rows).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func (r *crawlRepository) SetStatus(ctx context.Context, id uint32, status string) error {
|
||||
return r.db.WithContext(ctx).Model(&model.CrawlTask{}).Where("id = ?", id).
|
||||
Updates(map[string]any{
|
||||
"status": status,
|
||||
"locked_at": 0,
|
||||
"worker": "",
|
||||
"attempts": 0,
|
||||
"last_error": "",
|
||||
"updated_at": uint32(time.Now().Unix()),
|
||||
}).Error
|
||||
}
|
||||
|
||||
func (r *crawlRepository) Delete(ctx context.Context, id uint32) error {
|
||||
return r.db.WithContext(ctx).Exec(`DELETE FROM crawl_tasks WHERE id = ?`, id).Error
|
||||
}
|
||||
|
||||
// ListCrawlLogs 取某品牌最近的抓取历史。
|
||||
func (r *crawlRepository) ListCrawlLogs(ctx context.Context, brandID uint32, limit int) ([]model.CrawlLog, error) {
|
||||
if brandID == 0 || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var rows []model.CrawlLog
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("brand_id = ?", brandID).
|
||||
Order("id DESC").
|
||||
Limit(limit).
|
||||
Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func (r *crawlRepository) BrandCrawlStates(ctx context.Context, brandIDs []uint32) (map[uint32]BrandCrawlState, error) {
|
||||
out := make(map[uint32]BrandCrawlState, len(brandIDs))
|
||||
if len(brandIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var rows []struct {
|
||||
BrandID uint32
|
||||
Status string
|
||||
UpdatedAt int64
|
||||
}
|
||||
// DISTINCT ON 取每个品牌最近一条任务(按 updated_at 倒序),一次查询拿全状态与时间。
|
||||
if err := r.db.WithContext(ctx).Raw(`
|
||||
SELECT DISTINCT ON (brand_id) brand_id, status, updated_at
|
||||
FROM crawl_tasks
|
||||
WHERE brand_id IN ?
|
||||
ORDER BY brand_id, updated_at DESC`, brandIDs).Scan(&rows).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
st := BrandCrawlState{Status: row.Status}
|
||||
if row.UpdatedAt > 0 {
|
||||
st.UpdatedAt = time.Unix(row.UpdatedAt, 0)
|
||||
}
|
||||
out[row.BrandID] = st
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *crawlRepository) Counts(ctx context.Context) (map[string]int64, error) {
|
||||
var rows []struct {
|
||||
Status string
|
||||
N int64
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
`SELECT status, count(*) AS n FROM crawl_tasks GROUP BY status`).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]int64, len(rows))
|
||||
for _, r := range rows {
|
||||
out[r.Status] = r.N
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ShowCoverage 统计每个「(年份, 季节, 类型)」已入库的图片张数。
|
||||
//
|
||||
// 只统计未软删的图:一张被删过的秀如果只剩几张,说明当初没抓全,
|
||||
// 调用方会按阈值判断是否值得重抓,而不是一律跳过。
|
||||
// JOIN 保证 image_count > 0,没有图的秀不会出现在结果里。
|
||||
func (r *crawlRepository) ShowCoverage(ctx context.Context, brandID uint32) (map[ShowKey]ShowCoverageStat, error) {
|
||||
var rows []struct {
|
||||
Year uint16 `gorm:"column:year"`
|
||||
Season string `gorm:"column:season"`
|
||||
CollectionType string `gorm:"column:collection_type"`
|
||||
Images int `gorm:"column:images"`
|
||||
Expected int `gorm:"column:expected"`
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
`SELECT r.year, coalesce(r.season, '') AS season, coalesce(r.collection_type, '') AS collection_type,
|
||||
count(i.id) AS images, max(r.expected_images) AS expected
|
||||
FROM brand_runways r
|
||||
JOIN brand_runway_images i ON i.runway_id = r.id AND i.is_deleted = 0
|
||||
WHERE r.brand_id = ? AND r.is_deleted = 0
|
||||
GROUP BY r.year, r.season, r.collection_type`,
|
||||
brandID).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[ShowKey]ShowCoverageStat, len(rows))
|
||||
for _, row := range rows {
|
||||
out[ShowKey{Year: row.Year, Season: row.Season, CollectionType: row.CollectionType}] =
|
||||
ShowCoverageStat{Images: row.Images, Expected: row.Expected}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@ -58,7 +58,24 @@ type DupStat struct {
|
||||
//
|
||||
// 近邻结果走 image_similar_pairs 预计算表:入库时(或 dbtool similarbackfill 回填时)
|
||||
// 算一次存表,后台页面只读结果。原因见 ReplaceSimilarPairs 注释。
|
||||
// PendingEmbedding 是一张「缺向量」的图片,供回填 worker 拉取。
|
||||
// 带 Key 而不是完整 URL:worker 自己按存储基址拼(后端可能换了域名,不该把地址写死在队列里)。
|
||||
type PendingEmbedding struct {
|
||||
Kind string // runway | street
|
||||
ImageID uint32
|
||||
Key string // 存储 key(S4)或本地相对路径(/uploads/...)
|
||||
}
|
||||
|
||||
type EmbeddingRepository interface {
|
||||
// ListPendingEmbeddings 取一批「还活着、没有向量、失败次数未达上限」的图片,按 id 升序。
|
||||
//
|
||||
// 媒体拆分后这条查询是向量回填的唯一入口:媒体机直传只写 key,后端不碰图片字节,
|
||||
// 向量必须靠外部 worker 回传,而 worker 需要知道「还有哪些图缺向量」。
|
||||
ListPendingEmbeddings(ctx context.Context, kind string, limit, maxAttempts int) ([]PendingEmbedding, error)
|
||||
// CountPendingEmbeddings 统计剩余待回填数量(worker 据此决定是否继续拉)。
|
||||
CountPendingEmbeddings(ctx context.Context, kind string, maxAttempts int) (int, error)
|
||||
// RecordEmbeddingFailure 记一次失败(attempts+1)。达到 maxAttempts 后该图不再进队列。
|
||||
RecordEmbeddingFailure(ctx context.Context, kind string, imageID uint32, reason string) error
|
||||
// UpsertEmbedding 写入或更新一张图的向量((kind, image_id) 唯一)。
|
||||
UpsertEmbedding(ctx context.Context, kind string, imageID uint32, vec []float32) error
|
||||
// BulkUpsertEmbedding 批量写入;vecs[i] 对应 ids[i],vecs[i] 为 nil 则跳过该行。
|
||||
@ -175,6 +192,85 @@ func (r *embeddingRepository) UpsertEmbedding(ctx context.Context, kind string,
|
||||
return r.BulkUpsertEmbedding(ctx, kind, []uint32{imageID}, [][]float32{vec})
|
||||
}
|
||||
|
||||
// ListPendingEmbeddings 取缺失向量的图片。
|
||||
//
|
||||
// 三个 LEFT JOIN 各有明确目的:
|
||||
// - e 判「已有没有向量」;
|
||||
// - f 判「失败次数是否已到上限」——没有它,一张永远编不出来的坏图会让每轮回填都空转;
|
||||
// - i.is_deleted = 0 跳过软删图,避免为已删内容白算向量。
|
||||
//
|
||||
// 表名/列名来自 ownerOf 常量,kind 走参数绑定,无注入面。
|
||||
func (r *embeddingRepository) ListPendingEmbeddings(ctx context.Context, kind string, limit, maxAttempts int) ([]PendingEmbedding, error) {
|
||||
owner, ok := ownerOf(kind)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = 5
|
||||
}
|
||||
var rows []struct {
|
||||
ImageID uint32 `gorm:"column:image_id"`
|
||||
Image string `gorm:"column:image"`
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
"SELECT i.id AS image_id, i.image AS image "+
|
||||
"FROM "+owner.table+" AS i "+
|
||||
"LEFT JOIN image_embeddings AS e ON e.kind = ? AND e.image_id = i.id "+
|
||||
"LEFT JOIN image_embedding_failures AS f ON f.kind = ? AND f.image_id = i.id "+
|
||||
"WHERE i.is_deleted = 0 AND i.image <> '' AND e.image_id IS NULL "+
|
||||
" AND coalesce(f.attempts, 0) < ? "+
|
||||
"ORDER BY i.id LIMIT ?",
|
||||
kind, kind, maxAttempts, limit).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]PendingEmbedding, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, PendingEmbedding{Kind: kind, ImageID: row.ImageID, Key: row.Image})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CountPendingEmbeddings 统计剩余量(与 ListPendingEmbeddings 同口径,避免两边不一致)。
|
||||
func (r *embeddingRepository) CountPendingEmbeddings(ctx context.Context, kind string, maxAttempts int) (int, error) {
|
||||
owner, ok := ownerOf(kind)
|
||||
if !ok {
|
||||
return 0, nil
|
||||
}
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = 5
|
||||
}
|
||||
var n int
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
"SELECT count(*) FROM "+owner.table+" AS i "+
|
||||
"LEFT JOIN image_embeddings AS e ON e.kind = ? AND e.image_id = i.id "+
|
||||
"LEFT JOIN image_embedding_failures AS f ON f.kind = ? AND f.image_id = i.id "+
|
||||
"WHERE i.is_deleted = 0 AND i.image <> '' AND e.image_id IS NULL "+
|
||||
" AND coalesce(f.attempts, 0) < ?",
|
||||
kind, kind, maxAttempts).Scan(&n).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// RecordEmbeddingFailure 累加失败次数。
|
||||
// 用 upsert 而不是先查后写:worker 可能并发回传,两步写法会丢计数。
|
||||
func (r *embeddingRepository) RecordEmbeddingFailure(ctx context.Context, kind string, imageID uint32, reason string) error {
|
||||
if len(reason) > 300 {
|
||||
reason = reason[:300]
|
||||
}
|
||||
return r.db.WithContext(ctx).Exec(`
|
||||
INSERT INTO image_embedding_failures (kind, image_id, attempts, last_error, updated_at)
|
||||
VALUES (?, ?, 1, ?, ?)
|
||||
ON CONFLICT (kind, image_id) DO UPDATE
|
||||
SET attempts = image_embedding_failures.attempts + 1,
|
||||
last_error = EXCLUDED.last_error,
|
||||
updated_at = EXCLUDED.updated_at`,
|
||||
kind, imageID, reason, uint32(time.Now().Unix())).Error
|
||||
}
|
||||
|
||||
// BulkUpsertEmbedding 逐行写入而非 unnest 批量:日入库仅几百张,量级极小,
|
||||
// 换来的是可读性与低出错概率。ON CONFLICT 保证同一图重复入库不会写出多行。
|
||||
func (r *embeddingRepository) BulkUpsertEmbedding(ctx context.Context, kind string, ids []uint32, vecs [][]float32) error {
|
||||
|
||||
@ -28,8 +28,19 @@ type IngestRepository interface {
|
||||
// Claim 原子领取最多 limit 条 pending 任务并置为 processing,返回这些任务。
|
||||
// 用 SKIP LOCKED 保证多 worker 不抢同一条。
|
||||
Claim(ctx context.Context, limit int) ([]model.IngestJob, error)
|
||||
// MarkDone 标记任务成功。
|
||||
MarkDone(ctx context.Context, id uint32) error
|
||||
// MarkDone 标记任务成功,并记录结果类型与人可读细节。
|
||||
//
|
||||
// outcome 取 model.IngestOutcome* 常量,note 例如「新建 42 张图」「已存在(status=published)」。
|
||||
// 二者是后台「品牌更新记录」的全部信息源:只有 status=done 看不出这次到底新建还是跳过。
|
||||
MarkDone(ctx context.Context, id uint32, outcome, note string) error
|
||||
// RunwayTitleExists 按(品牌 + 英文标题)查已有图集,返回其 id 与是否存在。
|
||||
//
|
||||
// 用途:季节码推导规则变更后,同一场秀可能被算出不同的 season_code(如 Spring 男装
|
||||
// 曾是 FWyy、现在是 SSyy)。老实例按旧规则写入会建出「同标题不同季节码」的重复图集。
|
||||
// 建行前用它兜底,把「标题相同」视为同一场秀。
|
||||
RunwayTitleExists(ctx context.Context, brandID uint32, titleEn string) (uint32, bool, error)
|
||||
// ListByBrand 取某品牌的入库记录(按 id 倒序),供品牌详情页展示更新历史。
|
||||
ListByBrand(ctx context.Context, brandID uint32, limit int) ([]BrandIngestRecord, error)
|
||||
// MarkFailed 标记任务失败并记录错误(attempts 自增)。
|
||||
MarkFailed(ctx context.Context, id uint32, errMsg string) error
|
||||
// ReserveNonce 写入一次性随机串;若已存在(重放)返回 ok=false。
|
||||
@ -54,6 +65,15 @@ type IngestRepository interface {
|
||||
BackfillRunwayPublishedAt(ctx context.Context, runwayID uint32, publishedAt uint32, year uint16, targetSeasonCode string) error
|
||||
// RepairRunwayDetailParents 入库后按 sort_order 回填走秀细节图的 parent_image_id(细节图挂到最近主图)。
|
||||
RepairRunwayDetailParents(ctx context.Context, runwayID uint32) error
|
||||
// RunwayImageState 返回某场秀已有图片的 key→id 映射与最大排序号,供「补齐缺图」比对。
|
||||
//
|
||||
// 图片 key 是内容 sha1,spider 直传时会一并上报,所以**不用下载**就能知道哪张缺。
|
||||
RunwayImageState(ctx context.Context, runwayID uint32) (RunwayImageState, error)
|
||||
// CreateRunwayImage 插入一张图片(回填自增 id,供细节图挂 parent_image_id)。
|
||||
CreateRunwayImage(ctx context.Context, img *model.BrandRunwayImage) error
|
||||
// SyncRunwayImageStats 重算 image_count(主图行数),并抬高 expected_images。
|
||||
// expected 只增不减:抓取中途失败导致张数变少时,不应把「应有张数」改小。
|
||||
SyncRunwayImageStats(ctx context.Context, runwayID uint32, expected int) error
|
||||
// StreetSnapEntityState 按实体键(city + year + title)查正式表既有行,返回其状态与是否已被软删。
|
||||
// 与 RunwayEntityState 同源:不再以 is_deleted = 0 过滤,软删实体跳过重爬(删除即永久)。
|
||||
// title 用 COALESCE 与唯一索引 uq_ss_entity 的表达式保持一致。
|
||||
@ -66,6 +86,13 @@ type IngestRepository interface {
|
||||
ListJobs(ctx context.Context, offset, limit int) ([]model.IngestJob, error)
|
||||
// CountJobs 返回 ingest_jobs 总条数(用于分页计算总页数)。
|
||||
CountJobs(ctx context.Context) (int64, error)
|
||||
// CountPendingJobs 返回尚未被 worker 处理的入库任务数(后台状态条)。
|
||||
CountPendingJobs(ctx context.Context) (int64, error)
|
||||
// RecentCrawlResults 取最近若干条爬虫入库结果,带品牌名与场次标题,供「最近入库结果」表格。
|
||||
//
|
||||
// 场次标题从 payload JSON 里取:入库表只存原始载荷,没有 title 列,
|
||||
// 为展示再冗余一列不值得——payload 里本来就有。
|
||||
RecentCrawlResults(ctx context.Context, limit int) ([]CrawlResultRow, error)
|
||||
// RetryJob 把一条 failed 任务重置回 pending,清 last_error/locked_at,等待 worker 重新处理。
|
||||
RetryJob(ctx context.Context, id uint32) error
|
||||
// EnqueueMediaCleanup 写入一条「清理S4孤儿图」任务(payload 为待清理 key 的 JSON)。
|
||||
@ -165,17 +192,65 @@ func (r *ingestRepository) Claim(ctx context.Context, limit int) ([]model.Ingest
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
func (r *ingestRepository) MarkDone(ctx context.Context, id uint32) error {
|
||||
func (r *ingestRepository) MarkDone(ctx context.Context, id uint32, outcome, note string) error {
|
||||
return r.db.WithContext(ctx).
|
||||
Model(&model.IngestJob{}).
|
||||
Where("id = ?", id).
|
||||
Updates(map[string]any{
|
||||
"status": model.IngestStatusDone,
|
||||
"outcome": outcome,
|
||||
"note": note,
|
||||
"last_error": "",
|
||||
"updated_at": uint32(time.Now().Unix()),
|
||||
}).Error
|
||||
}
|
||||
|
||||
// RunwayTitleExists 按(品牌 + 英文标题)查已有图集。
|
||||
func (r *ingestRepository) RunwayTitleExists(ctx context.Context, brandID uint32, titleEn string) (uint32, bool, error) {
|
||||
if brandID == 0 || titleEn == "" {
|
||||
return 0, false, nil
|
||||
}
|
||||
var id uint32
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
"SELECT id FROM brand_runways WHERE brand_id = ? AND title_en = ? AND is_deleted = 0 LIMIT 1",
|
||||
brandID, titleEn).Scan(&id).Error; err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
return id, id > 0, nil
|
||||
}
|
||||
|
||||
// BrandIngestRecord 是品牌详情页需要的「一条更新记录」。
|
||||
//
|
||||
// 只取必要列:payload 可能几十 KB(含全部 look 的 key),一次拉 20 条会传上兆,
|
||||
// 所以标题直接在 SQL 里从 JSON 取出,不回传整段 payload。
|
||||
type BrandIngestRecord struct {
|
||||
ID uint32 `gorm:"column:id"`
|
||||
Title string `gorm:"column:title"`
|
||||
Status string `gorm:"column:status"`
|
||||
Outcome string `gorm:"column:outcome"`
|
||||
Note string `gorm:"column:note"`
|
||||
LastError string `gorm:"column:last_error"`
|
||||
Attempts uint8 `gorm:"column:attempts"`
|
||||
UpdatedAt uint32 `gorm:"column:updated_at"`
|
||||
}
|
||||
|
||||
// ListByBrand 取某品牌的入库记录(按 id 倒序),供品牌详情页展示更新历史。
|
||||
func (r *ingestRepository) ListByBrand(ctx context.Context, brandID uint32, limit int) ([]BrandIngestRecord, error) {
|
||||
if brandID == 0 || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var rows []BrandIngestRecord
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
"SELECT id, status, coalesce(outcome, '') AS outcome, coalesce(note, '') AS note, "+
|
||||
"coalesce(last_error, '') AS last_error, attempts, updated_at, "+
|
||||
"coalesce(payload::json->>'title_en', '') AS title "+
|
||||
"FROM ingest_jobs WHERE brand_id = ? ORDER BY id DESC LIMIT ?",
|
||||
brandID, limit).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func (r *ingestRepository) MarkFailed(ctx context.Context, id uint32, errMsg string) error {
|
||||
return r.db.WithContext(ctx).
|
||||
Model(&model.IngestJob{}).
|
||||
@ -597,3 +672,124 @@ func indexOf(s, sub string) int {
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// CrawlResultRow 一条最近的爬虫入库结果(后台展示用)。
|
||||
type CrawlResultRow struct {
|
||||
UpdatedAt uint32
|
||||
Status string
|
||||
Outcome string
|
||||
Note string
|
||||
BrandName string
|
||||
Title string // 场次英文标题(来自 payload)
|
||||
}
|
||||
|
||||
// CountPendingJobs 统计待处理的入库任务。
|
||||
func (r *ingestRepository) CountPendingJobs(ctx context.Context) (int64, error) {
|
||||
var n int64
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
`SELECT count(*) FROM ingest_jobs WHERE status = ?`, model.IngestStatusPending).Scan(&n).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// RecentCrawlResults 取最近 N 条爬虫入库结果。
|
||||
//
|
||||
// payload 用 `LIKE '{%'` 先挡一道再转 jsonb:正常载荷一定是 JSON 对象,
|
||||
// 但历史脏数据/空串会让 `::jsonb` 抛错并整页 500——展示页不该被一条坏数据拖垮。
|
||||
func (r *ingestRepository) RecentCrawlResults(ctx context.Context, limit int) ([]CrawlResultRow, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
var rows []struct {
|
||||
UpdatedAt uint32 `gorm:"column:updated_at"`
|
||||
Status string `gorm:"column:status"`
|
||||
Outcome string `gorm:"column:outcome"`
|
||||
Note string `gorm:"column:note"`
|
||||
BrandName string `gorm:"column:brand_name"`
|
||||
Title string `gorm:"column:title"`
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Raw(`
|
||||
SELECT j.updated_at, j.status, coalesce(j.outcome, '') AS outcome,
|
||||
coalesce(j.note, '') AS note,
|
||||
coalesce(b.name_en, '') AS brand_name,
|
||||
CASE WHEN j.payload LIKE '{%' THEN coalesce(j.payload::jsonb->>'title_en', '') ELSE '' END AS title
|
||||
FROM ingest_jobs j
|
||||
LEFT JOIN brands b ON b.id = j.brand_id
|
||||
WHERE j.kind = ?
|
||||
ORDER BY j.id DESC
|
||||
LIMIT ?`, model.IngestKindCrawl, limit).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]CrawlResultRow, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, CrawlResultRow{
|
||||
UpdatedAt: row.UpdatedAt, Status: row.Status, Outcome: row.Outcome,
|
||||
Note: row.Note, BrandName: row.BrandName, Title: row.Title,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RunwayImageState 一场秀已有图片的快照:key→id 映射 + 最大排序号。
|
||||
type RunwayImageState struct {
|
||||
// Keys 图片存储 key(如 runway/<sha1>.jpg)→ 图片行 id。
|
||||
Keys map[string]uint32
|
||||
// MaxSort 当前最大 sort_order;补齐的新图从它往后排。
|
||||
MaxSort uint32
|
||||
}
|
||||
|
||||
// RunwayImageState 读取已有图片(只含未删除的行)。
|
||||
//
|
||||
// 用 key 而不是 URL 做比对:库里存的就是 key,spider 直传后上报的也是 key,
|
||||
// 于是「这张图有没有」变成一次纯内存查表,不用下载、不用请求存储。
|
||||
func (r *ingestRepository) RunwayImageState(ctx context.Context, runwayID uint32) (RunwayImageState, error) {
|
||||
var rows []struct {
|
||||
ID uint32 `gorm:"column:id"`
|
||||
Image string `gorm:"column:image"`
|
||||
SortOrder uint32 `gorm:"column:sort_order"`
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Raw(
|
||||
`SELECT id, coalesce(image, '') AS image, coalesce(sort_order, 0) AS sort_order
|
||||
FROM brand_runway_images WHERE runway_id = ? AND is_deleted = 0`,
|
||||
runwayID).Scan(&rows).Error; err != nil {
|
||||
return RunwayImageState{}, err
|
||||
}
|
||||
st := RunwayImageState{Keys: make(map[string]uint32, len(rows))}
|
||||
for _, row := range rows {
|
||||
if row.Image != "" {
|
||||
st.Keys[row.Image] = row.ID
|
||||
}
|
||||
if row.SortOrder > st.MaxSort {
|
||||
st.MaxSort = row.SortOrder
|
||||
}
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// CreateRunwayImage 插入一张图片并回填自增 id。
|
||||
// 细节图需要把父主图的 id 写进 parent_image_id,所以必须拿回 id。
|
||||
func (r *ingestRepository) CreateRunwayImage(ctx context.Context, img *model.BrandRunwayImage) error {
|
||||
now := uint32(time.Now().Unix())
|
||||
img.CreatedAt, img.UpdatedAt = now, now
|
||||
return r.db.WithContext(ctx).Create(img).Error
|
||||
}
|
||||
|
||||
// SyncRunwayImageStats 重算主图张数并抬高应有张数。
|
||||
//
|
||||
// image_count 用子查询从实际行数算,而不是传入累加值:
|
||||
// 并发补图或中途失败时,累加值容易和真实行数漂移,而 image_count 会直接影响
|
||||
// 公开卡片的「N 张」角标与热度排序,必须与真实行一致。
|
||||
func (r *ingestRepository) SyncRunwayImageStats(ctx context.Context, runwayID uint32, expected int) error {
|
||||
if expected < 0 {
|
||||
expected = 0
|
||||
}
|
||||
return r.db.WithContext(ctx).Exec(`
|
||||
UPDATE brand_runways SET
|
||||
image_count = (SELECT count(*) FROM brand_runway_images
|
||||
WHERE runway_id = ? AND is_deleted = 0 AND is_detail = 0),
|
||||
expected_images = greatest(expected_images, ?),
|
||||
updated_at = ?
|
||||
WHERE id = ?`,
|
||||
runwayID, expected, uint32(time.Now().Unix()), runwayID).Error
|
||||
}
|
||||
|
||||
@ -65,6 +65,20 @@ func NewBackstage(opt BackstageOptions) *gin.Engine {
|
||||
internal.Use(middleware.IngestAuth(*opt.IngestAuth))
|
||||
internal.POST("/ingest", opt.Ingest.Submit)
|
||||
internal.GET("/crawl/brands", opt.Ingest.CrawlBrands)
|
||||
// 已入库的秀(按 年份+季节+类型 聚合):spider 抓前预检,整场跳过,避免重复下载。
|
||||
internal.GET("/shows/covered", opt.Backstage.WorkerShowsCovered)
|
||||
// 执行进度:spider 每处理完一场秀上报一次,后台列表页轮询 /admin/crawl/live 读取。
|
||||
internal.POST("/crawl/tasks/:id/progress", opt.Backstage.WorkerProgress)
|
||||
// spider 服务化:领取品牌抓取任务 / 回报结果(与 ingest 共用同一套 HMAC 验签)。
|
||||
internal.GET("/crawl/tasks", opt.Backstage.WorkerTask)
|
||||
internal.POST("/crawl/tasks/:id/result", opt.Backstage.WorkerReport)
|
||||
// 媒体机直传:签发 S4 PUT 预签名 URL(媒体机不持有 AK/SK)。
|
||||
internal.POST("/upload-url", opt.Backstage.WorkerUploadURL)
|
||||
// 向量回填:媒体机直传后后端不碰图片字节,向量由外部 worker 拉图编码后回传。
|
||||
// GET /embeddings/pending → 缺向量的图片(含可直接下载的 url)
|
||||
// POST /embeddings → 回传向量(写库 + 重算近邻)与失败(累加计数)
|
||||
internal.GET("/embeddings/pending", opt.Backstage.WorkerPendingEmbeddings)
|
||||
internal.POST("/embeddings", opt.Backstage.WorkerEmbeddings)
|
||||
} else {
|
||||
r.POST("/admin/internal/ingest", func(c *gin.Context) {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "ingest disabled"})
|
||||
@ -118,9 +132,12 @@ func NewBackstage(opt BackstageOptions) *gin.Engine {
|
||||
admin.POST("/street-snaps/:id/images/:img/delete", opt.Backstage.StreetSnapImageDelete)
|
||||
admin.POST("/street-snaps/:id/cover", opt.Backstage.StreetSnapSetCover)
|
||||
admin.POST("/street-snaps/:id/toggle", opt.Backstage.StreetSnapToggleDeleted)
|
||||
// 爬虫入库任务监控:列出处理进度 + 失败重试
|
||||
admin.GET("/ingest-jobs", opt.Backstage.IngestJobs)
|
||||
admin.POST("/ingest-jobs/:id/retry", opt.Backstage.IngestJobRetry)
|
||||
// 爬虫页:把「抓取」与「入库」两条队列拼成一条流水线(只读)。
|
||||
// 入队仍在品牌页,两个 POST 是被那里的按钮调用的。
|
||||
admin.GET("/spider", opt.Backstage.SpiderBoard)
|
||||
admin.GET("/spider/live", opt.Backstage.SpiderLive)
|
||||
admin.POST("/spider/bulk", opt.Backstage.SpiderBulk)
|
||||
admin.POST("/spider/brands/:uid", opt.Backstage.SpiderEnqueue)
|
||||
// 用户管理:列出用户 + 提级 / 降为 VIP
|
||||
admin.GET("/users", opt.Backstage.Users)
|
||||
admin.POST("/users/:id/tier", opt.Backstage.UserSetTier)
|
||||
|
||||
@ -85,6 +85,11 @@ func (f *fakeBrand) CrawlTasks(ctx context.Context, brandID uint32) ([]dto.Crawl
|
||||
return []dto.CrawlBrand{{BrandUID: "001DESke", Name: "Acne Studios"}}, nil
|
||||
}
|
||||
|
||||
// StatsByIDs 后台品牌列表的数据量统计(测试里返回空表即可)。
|
||||
func (f *fakeBrand) StatsByIDs(ctx context.Context, ids []uint32) (map[uint32]repository.BrandStats, error) {
|
||||
return map[uint32]repository.BrandStats{}, nil
|
||||
}
|
||||
|
||||
// 编译期接口满足性检查
|
||||
var (
|
||||
_ service.ArticleService = (*fakeArticle)(nil)
|
||||
@ -659,7 +664,7 @@ func TestBackstageNavStatsOnEveryPage(t *testing.T) {
|
||||
t.Fatalf("品牌页导航应带待审计数角标(2):\n%s", body)
|
||||
}
|
||||
// 导航已在共享 partial 里,各模块入口都应在
|
||||
for _, label := range []string{"文章", "品牌", "入库任务", "用户"} {
|
||||
for _, label := range []string{"文章", "品牌", "爬虫", "用户"} {
|
||||
if !strings.Contains(body, ">"+label) {
|
||||
t.Fatalf("导航应包含入口 %q", label)
|
||||
}
|
||||
|
||||
@ -20,6 +20,8 @@ type BrandService interface {
|
||||
// AdminList 后台品牌列表:字母索引不要求走秀已发布,
|
||||
// 使只有待审 / 驳回走秀的品牌仍出现在后台(公开 List 读视图,会挡掉它们)。
|
||||
AdminList(ctx context.Context, q dto.BrandQuery) ([]dto.PublicBrand, int64, error)
|
||||
// StatsByIDs 批量统计品牌的图集数 / 图片数(后台列表展示数据量)。
|
||||
StatsByIDs(ctx context.Context, ids []uint32) (map[uint32]repository.BrandStats, error)
|
||||
// Hot 返回热门品牌(按走秀图片数排名前 limit 个)并附带各品牌档案数。
|
||||
// 供 SSG 构建期(RunwayLooks 侧栏 / 品牌弹窗 HOT 标签)一次性取前 N 个热门品牌;
|
||||
// 复用 List 的 featured 收窄逻辑(FeaturedIDs("images", limit)),与首页精选口径一致。
|
||||
@ -201,6 +203,11 @@ func (s *brandService) Delete(ctx context.Context, uid string) error {
|
||||
|
||||
// CrawlTasks 返回爬虫取任务所需的品牌列表:编码 brand_uid + 英文品牌名。
|
||||
// 爬虫直接拿 brand_uid 上送 ingest,无需自己编码;name 用于拼 vogue 抓取 URL。
|
||||
// StatsByIDs 批量统计品牌的图集数 / 图片数(后台列表用)。
|
||||
func (s *brandService) StatsByIDs(ctx context.Context, ids []uint32) (map[uint32]repository.BrandStats, error) {
|
||||
return s.brands.StatsByIDs(ctx, ids)
|
||||
}
|
||||
|
||||
func (s *brandService) CrawlTasks(ctx context.Context, brandID uint32) ([]dto.CrawlBrand, error) {
|
||||
brands, err := s.brands.CrawlBrands(ctx, brandID)
|
||||
if err != nil {
|
||||
|
||||
@ -19,7 +19,9 @@ type captureIngestRepo struct {
|
||||
repository.IngestRepository
|
||||
createdRW *model.BrandRunway
|
||||
createdImages []model.BrandRunwayImage
|
||||
deletedEntity bool // 模拟实体键已软删(删除即永久:应跳过重爬,不重新写入)
|
||||
deletedEntity bool // 模拟实体键已软删(删除即永久:应跳过重爬,不重新写入)
|
||||
doneOutcome string // 最近一次 MarkDone 的结果类型(断言「跳过」是否标清楚)
|
||||
doneNote string
|
||||
}
|
||||
|
||||
func (c *captureIngestRepo) RunwayEntityState(ctx context.Context, brandID uint32, seasonCode, collectionType string) (uint32, string, bool, bool, error) {
|
||||
@ -35,7 +37,23 @@ func (c *captureIngestRepo) CreateRunwayWithImages(ctx context.Context, rw *mode
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (c *captureIngestRepo) MarkDone(ctx context.Context, id uint32) error { return nil }
|
||||
func (c *captureIngestRepo) MarkDone(ctx context.Context, id uint32, outcome, note string) error {
|
||||
// 记下 outcome/note,供「跳过的场次也要标清楚原因」这类断言使用
|
||||
c.doneOutcome = outcome
|
||||
c.doneNote = note
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunwayTitleExists 捕获型仓储没有真实数据,一律视为「不存在」,
|
||||
// 免得测试路径走进「同标题已存在 → 跳过」分支。
|
||||
func (c *captureIngestRepo) RunwayTitleExists(ctx context.Context, brandID uint32, titleEn string) (uint32, bool, error) {
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
// ListByBrand 捕获型仓储不需要品牌维度查询。
|
||||
func (c *captureIngestRepo) ListByBrand(ctx context.Context, brandID uint32, limit int) ([]repository.BrandIngestRecord, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// RepairRunwayDetailParents 在捕获型仓储里无需真正落库回填,直接返回成功(被测路径只关心落库记录)。
|
||||
func (c *captureIngestRepo) RepairRunwayDetailParents(ctx context.Context, runwayID uint32) error {
|
||||
|
||||
83
internal/service/ingest_keypath_test.go
Normal file
83
internal/service/ingest_keypath_test.go
Normal file
@ -0,0 +1,83 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"fashionapi/internal/dto"
|
||||
)
|
||||
|
||||
// TestFetchLookImagesUsesProvidedKeys 验证「媒体机直传」分支:
|
||||
// payload 里带 main_key / detail_keys 时,worker 不应下载、不应上传,
|
||||
// 而是直接把 key 当作要落库的存储地址,且 cover 取首张主图。
|
||||
//
|
||||
// 这条路径是媒体拆分的核心保证:一旦这里退化成下载 URL,
|
||||
// 后端又会重新背上带宽瓶颈,而错误会非常隐蔽(只是变慢,不报错)。
|
||||
func TestFetchLookImagesUsesProvidedKeys(t *testing.T) {
|
||||
s := &IngestService{} // 不装配 uploader/vision:走 key 分支时它们都不该被用到
|
||||
|
||||
looks := []dto.RunwayLook{
|
||||
{
|
||||
MainKey: "runway/aaaa.jpg",
|
||||
MainSize: 100,
|
||||
DetailKeys: []string{"runway/bbbb.jpg"},
|
||||
DetailSizes: []int64{200},
|
||||
},
|
||||
{
|
||||
// 第二张混用旧形态:没有 key,带 URL(这里只验主图,不触发网络)
|
||||
MainKey: "runway/cccc.jpg",
|
||||
MainSize: 300,
|
||||
},
|
||||
}
|
||||
|
||||
cover, rows, vecs, keys, failed, _ := s.fetchLookImages(context.Background(), looks, "runway")
|
||||
|
||||
if failed {
|
||||
t.Fatal("key 分支不应被判定为失败")
|
||||
}
|
||||
if cover != "runway/aaaa.jpg" {
|
||||
t.Fatalf("cover 应取首张主图 key,实际 %q", cover)
|
||||
}
|
||||
if len(rows) != 3 {
|
||||
t.Fatalf("应落 3 行(2 主图 + 1 细节图),实际 %d", len(rows))
|
||||
}
|
||||
if rows[0].Image != "runway/aaaa.jpg" || rows[0].IsDetail != 0 {
|
||||
t.Fatalf("第 1 行应为 aaaa 主图,实际 %+v", rows[0])
|
||||
}
|
||||
if rows[1].Image != "runway/bbbb.jpg" || rows[1].IsDetail != 1 {
|
||||
t.Fatalf("第 2 行应为 bbbb 细节图,实际 %+v", rows[1])
|
||||
}
|
||||
if rows[1].ImageSize != 200 || rows[0].ImageSize != 100 {
|
||||
t.Fatalf("字节数应来自 payload,实际 %d / %d", rows[0].ImageSize, rows[1].ImageSize)
|
||||
}
|
||||
// 媒体机上传的 key 不进回滚清单(内容寻址、可能与他人共享)
|
||||
if len(keys) != 0 {
|
||||
t.Fatalf("key 分支不应产生回滚清单,实际 %v", keys)
|
||||
}
|
||||
if len(vecs) != len(rows) {
|
||||
t.Fatalf("vecs 应与 rows 对齐,实际 %d vs %d", len(vecs), len(rows))
|
||||
}
|
||||
for i, v := range vecs {
|
||||
if v != nil {
|
||||
t.Fatalf("第 %d 张不应带向量(向量由 CLIP 回填),实际 %v", i, v)
|
||||
}
|
||||
}
|
||||
if got := countMainImages(rows); got != 2 {
|
||||
t.Fatalf("主图数应为 2,实际 %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSha1FromKey 锁住「从内容寻址 key 反推 sha1」的行为:批次内去重依赖它,
|
||||
// 若解析退化(例如带上目录前缀),同一张图会被重复落库。
|
||||
func TestSha1FromKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"runway/4080435af23d8f80bbbcf9e6f55a5f1d00c49685.jpg": "4080435af23d8f80bbbcf9e6f55a5f1d00c49685",
|
||||
"street/abc123.png": "abc123",
|
||||
"plain-key": "plain-key",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := sha1FromKey(in); got != want {
|
||||
t.Fatalf("sha1FromKey(%q) = %q,期望 %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@ -116,13 +117,26 @@ func (s *IngestService) Submit(ctx context.Context, payload dto.RunwayIngest) (u
|
||||
if payload.Kind == dto.IngestKindRunway && payload.BrandUID == "" {
|
||||
return 0, NewError(http.StatusBadRequest, "brand_uid required")
|
||||
}
|
||||
job := &model.IngestJob{Payload: string(raw), Kind: model.IngestKindCrawl}
|
||||
// brand_id 提交时就解出来:品牌详情页按它直接查更新记录,
|
||||
// 不必每次把 payload 里的 hashid 解一遍(也避免与查询时的盐不一致)。
|
||||
var brandID uint32
|
||||
if payload.BrandUID != "" {
|
||||
if id, derr := hashid.Decode(payload.BrandUID); derr == nil {
|
||||
brandID = id
|
||||
}
|
||||
}
|
||||
job := &model.IngestJob{Payload: string(raw), Kind: model.IngestKindCrawl, BrandID: brandID}
|
||||
if err := s.repo.Enqueue(ctx, job); err != nil {
|
||||
return 0, internalErr(err.Error())
|
||||
}
|
||||
return job.ID, nil
|
||||
}
|
||||
|
||||
// ListBrandRecords 取某品牌的入库记录(品牌详情页的「更新记录」)。
|
||||
func (s *IngestService) ListBrandRecords(ctx context.Context, brandID uint32, limit int) ([]repository.BrandIngestRecord, error) {
|
||||
return s.repo.ListByBrand(ctx, brandID, limit)
|
||||
}
|
||||
|
||||
// ListJobs 分页列出入库任务(后台监控页用)。返回本页任务、总条数与错误。
|
||||
func (s *IngestService) ListJobs(ctx context.Context, offset, limit int) ([]model.IngestJob, int64, error) {
|
||||
jobs, err := s.repo.ListJobs(ctx, offset, limit)
|
||||
@ -141,6 +155,16 @@ func (s *IngestService) RetryJob(ctx context.Context, id uint32) error {
|
||||
return s.repo.RetryJob(ctx, id)
|
||||
}
|
||||
|
||||
// CountPendingJobs 待处理入库任务数(后台爬虫页状态条)。
|
||||
func (s *IngestService) CountPendingJobs(ctx context.Context) (int64, error) {
|
||||
return s.repo.CountPendingJobs(ctx)
|
||||
}
|
||||
|
||||
// RecentCrawlResults 最近若干条入库结果(后台爬虫页的「最近入库结果」表)。
|
||||
func (s *IngestService) RecentCrawlResults(ctx context.Context, limit int) ([]repository.CrawlResultRow, error) {
|
||||
return s.repo.RecentCrawlResults(ctx, limit)
|
||||
}
|
||||
|
||||
// Run 启动 worker 循环:定时领取 pending 任务并处理,直到 ctx 取消。
|
||||
// 多实例安全:Claim 用 SKIP LOCKED,互不抢同一条。
|
||||
func (s *IngestService) Run(ctx context.Context, batch int, interval time.Duration) {
|
||||
@ -244,11 +268,11 @@ func (s *IngestService) processMediaCleanup(ctx context.Context, job model.Inges
|
||||
return
|
||||
}
|
||||
if len(p.Keys) == 0 {
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeNoop, "媒体清理:无需处理")
|
||||
return
|
||||
}
|
||||
purgeOrphanImages(ctx, s.media, s.del, p.Keys)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeCleaned, "媒体清理完成")
|
||||
}
|
||||
|
||||
// failOrRetry 把临时失败(如网络抖动 / 单图下载失败)按指数退避自动重试:
|
||||
@ -265,13 +289,33 @@ func (s *IngestService) failOrRetry(ctx context.Context, id uint32, errMsg strin
|
||||
// processRunway 走秀入库:品牌校验 → 实体键分三支(新建 / 放弃 / 复用驳回)→ 下载图 → 直写 brand_runways。
|
||||
func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob, p dto.RunwayIngest) {
|
||||
// 1) 品牌必须存在(爬虫负责先建/复用品牌)
|
||||
brandID, err := hashid.Decode(p.BrandUID)
|
||||
if err != nil {
|
||||
_ = s.repo.MarkFailed(ctx, job.ID, "unknown brand_uid")
|
||||
return
|
||||
//
|
||||
// 优先用「提交任务时就解好的 brand_id」:它由接收上报的那个实例解出,盐一定正确。
|
||||
// 只有当 brand_id 缺省(本次改动前的老任务)才回落到解 payload 里的 hashid。
|
||||
//
|
||||
// 为什么必须这样:hashid 的解码依赖 HASHID_SECRET。线上曾出现另一台机器上的
|
||||
// 旧后端连同一套库抢任务,它的盐与线上不同 → 同一个 uid 解出别的 id → 整场被判
|
||||
// brand not found 丢掉(AMBUSH 那批 8 场就是这么没的)。brand_id 落库后,
|
||||
// 即便 worker 与提交方不是同一个实例,也不再有这个分歧。
|
||||
brandID := job.BrandID
|
||||
if brandID == 0 {
|
||||
id, derr := hashid.Decode(p.BrandUID)
|
||||
if derr != nil {
|
||||
_ = s.repo.MarkFailed(ctx, job.ID, "unknown brand_uid")
|
||||
return
|
||||
}
|
||||
brandID = id
|
||||
}
|
||||
if _, err := s.brandRepo.FindByID(ctx, brandID); err != nil {
|
||||
_ = s.repo.MarkFailed(ctx, job.ID, "brand not found")
|
||||
// 必须区分「品牌真不存在」与「查询本身失败」:前者永久失败,后者应退避重试。
|
||||
// 曾把两者一律记为 "brand not found",导致数据库抖动时一批任务被误判为永久失败,
|
||||
// 且真实错误(连接超时 / 池耗尽)在队列里完全看不到,排查时无线索。
|
||||
log.Printf("[ingest][diag] job=%d brand_uid=%q decoded=%d err=%v", job.ID, p.BrandUID, brandID, err)
|
||||
if errors.Is(err, repository.ErrNotFound) {
|
||||
_ = s.repo.MarkFailed(ctx, job.ID, "brand not found")
|
||||
return
|
||||
}
|
||||
s.failOrRetry(ctx, job.ID, "brand lookup: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@ -295,7 +339,7 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
// 删除即永久:已软删的实体键不再因重爬复活(原设计靠部分唯一索引释放键、允许重爬,已改为禁止)。
|
||||
if exist && isDeleted {
|
||||
log.Printf("[ingest] job=%d runway 实体已删除(is_deleted=1),跳过重爬", job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedDeleted, "实体已删除(删除即永久,不重爬)")
|
||||
return
|
||||
}
|
||||
if !exist && p.Year == 0 {
|
||||
@ -310,21 +354,47 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
reuseID = rid
|
||||
if exist && isDeleted {
|
||||
log.Printf("[ingest] job=%d runway 实体已删除(is_deleted=1),跳过重爬", job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedDeleted, "实体已删除(删除即永久,不重爬)")
|
||||
return
|
||||
}
|
||||
}
|
||||
action := decideIngestAction(exist, existStatus)
|
||||
// 季节码漂移保护:实体键按 season_code 去重,但规则改过(Spring 男装由 FWyy 改为 SSyy)。
|
||||
// 若还有实例按旧规则算码,就会把同一场秀当成新场次再建一行。
|
||||
// 这里在建行前用「同品牌同标题」再兜一次底,宁可跳过也不产生重复。
|
||||
if !exist && action != ingestSkip {
|
||||
if dupID, found, terr := s.repo.RunwayTitleExists(ctx, brandID, p.TitleEn); terr == nil && found {
|
||||
log.Printf("[ingest] job=%d runway 已存在同标题图集 id=%d(季节码可能漂移),跳过", job.ID, dupID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedExisting,
|
||||
fmt.Sprintf("已存在同标题图集(id=%d),跳过(季节码漂移保护)", dupID))
|
||||
return
|
||||
}
|
||||
}
|
||||
if action == ingestSkip {
|
||||
log.Printf("[ingest] job=%d runway 实体已存在(status=%s),跳过(回填 published_at)", job.ID, existStatus)
|
||||
// 临时回填:仅更新发布时间与缺失年份,不重下图片(图片已在库)。
|
||||
// 实体已存在 ≠ 抓全了:先按 key 比对把缺的图补上。
|
||||
//
|
||||
// 不补的话,「覆盖率判定说这场不完整 → spider 重爬 → 后端直接跳过」会形成死循环:
|
||||
// 每次重爬都把整场图下一遍,库里却一张不增,缺的图永远补不上。
|
||||
added, mErr := s.mergeMissingImages(ctx, reuseID, brandID, p)
|
||||
if mErr != nil {
|
||||
s.failOrRetry(ctx, job.ID, "merge missing images: "+mErr.Error())
|
||||
return
|
||||
}
|
||||
// 回填发布时间与缺失年份(不重下图片)。
|
||||
if p.PublishedAt != 0 {
|
||||
if err := s.repo.BackfillRunwayPublishedAt(ctx, reuseID, p.PublishedAt, effYear, seasonCode); err != nil {
|
||||
s.failOrRetry(ctx, job.ID, "backfill published_at: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
if added > 0 {
|
||||
log.Printf("[ingest] job=%d runway 实体已存在,补齐 %d 张缺图", job.ID, added)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeRepaired,
|
||||
fmt.Sprintf("已存在(status=%s),补齐 %d 张缺图", existStatus, added))
|
||||
return
|
||||
}
|
||||
log.Printf("[ingest] job=%d runway 实体已存在(status=%s),图片齐全,跳过", job.ID, existStatus)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedExisting, "已存在(status="+existStatus+"),跳过重下")
|
||||
return
|
||||
}
|
||||
|
||||
@ -387,6 +457,10 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
PublishedAt: p.PublishedAt,
|
||||
Cover: cover,
|
||||
ImageCount: imageCount,
|
||||
// 应有张数取「spider 上报值」与「本次实际入库数」的较大者:
|
||||
// 去重会砍掉重复图,payload 里的总数可能偏高;而实际入库数就是本次能拿到的全部。
|
||||
// 取较大者保证覆盖率判定不会因为去重而永远判「不完整」。
|
||||
ExpectedImages: maxInt(p.ExpectedImages, int(imageCount)),
|
||||
Status: model.StatusPending,
|
||||
}
|
||||
|
||||
@ -430,7 +504,11 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
}
|
||||
log.Printf("[ingest] job=%d runway 写正式表 耗时=%v 图片行=%d 复用驳回=%v",
|
||||
job.ID, time.Since(writeStart).Round(time.Millisecond), len(images), action == ingestReuseRejected)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
outcome, note := model.IngestOutcomeCreated, fmt.Sprintf("新建 %d 张图", len(images))
|
||||
if action == ingestReuseRejected {
|
||||
outcome, note = model.IngestOutcomeReused, fmt.Sprintf("复用驳回记录,覆盖 %d 张图", len(images))
|
||||
}
|
||||
_ = s.repo.MarkDone(ctx, job.ID, outcome, note)
|
||||
}
|
||||
|
||||
// fetchLookImages 按 Looks 结构下载主图+细节图并上传到存储,返回可直接落正式图片表的
|
||||
@ -454,22 +532,48 @@ func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayL
|
||||
failed := false
|
||||
|
||||
// 摊平:顺序 = 逐个 look 先主图、再其细节图,与原串行遍历顺序完全一致。
|
||||
// 每张图优先取「媒体机已上传的 key」,没有 key 才回落到「下载原始 URL」。
|
||||
tasks := make([]lookImageTask, 0, len(looks))
|
||||
for li, look := range looks {
|
||||
lookIdx := li + 1
|
||||
if look.Main != "" {
|
||||
if look.MainKey != "" {
|
||||
tasks = append(tasks, lookImageTask{key: look.MainKey, size: look.MainSize, lookIdx: lookIdx})
|
||||
} else if look.Main != "" {
|
||||
tasks = append(tasks, lookImageTask{url: look.Main, lookIdx: lookIdx})
|
||||
}
|
||||
for di, d := range look.Details {
|
||||
if d == "" {
|
||||
// 细节图数量取两个数组的较大者:媒体机直传时可能只给 DetailKeys 而不给 URL,
|
||||
// 只按 Details 遍历会把这些细节图静默丢掉。
|
||||
detailN := len(look.Details)
|
||||
if len(look.DetailKeys) > detailN {
|
||||
detailN = len(look.DetailKeys)
|
||||
}
|
||||
for di := 0; di < detailN; di++ {
|
||||
var d string
|
||||
if di < len(look.Details) {
|
||||
d = look.Details[di]
|
||||
}
|
||||
hasKey := di < len(look.DetailKeys) && look.DetailKeys[di] != ""
|
||||
if d == "" && !hasKey {
|
||||
continue
|
||||
}
|
||||
tasks = append(tasks, lookImageTask{url: d, lookIdx: lookIdx, isDetail: true, detailNo: di + 1})
|
||||
t := lookImageTask{lookIdx: lookIdx, isDetail: true, detailNo: di + 1}
|
||||
if hasKey {
|
||||
t.key = look.DetailKeys[di]
|
||||
if di < len(look.DetailSizes) {
|
||||
t.size = look.DetailSizes[di]
|
||||
}
|
||||
} else {
|
||||
t.url = d
|
||||
}
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
}
|
||||
|
||||
// 并发段:下载 + 上传(耗时大头)。
|
||||
// 并发段:下载 + 上传(耗时大头)。key 路径的图已在存储里,直接构造结果跳过 IO。
|
||||
results := concurrentFetch(ctx, len(tasks), func(c context.Context, i int) downloadResult {
|
||||
if k := tasks[i].key; k != "" {
|
||||
return downloadResult{url: k, key: k, sha1: sha1FromKey(k), size: tasks[i].size}
|
||||
}
|
||||
return s.downloadAndUpload(c, tasks[i].url, prefix)
|
||||
})
|
||||
|
||||
@ -484,7 +588,12 @@ func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayL
|
||||
failed = true
|
||||
continue
|
||||
}
|
||||
keys = append(keys, r.key)
|
||||
// 只把「本次任务自己上传的 key」纳入回滚清单。
|
||||
// 媒体机直传的 key 不进清单:它是内容寻址的,可能被别的图集共享,
|
||||
// 失败时删掉会连带弄坏别人已入库的图;这类孤儿由存储侧清理策略负责。
|
||||
if task.key == "" {
|
||||
keys = append(keys, r.key)
|
||||
}
|
||||
if seen[r.sha1] {
|
||||
continue
|
||||
}
|
||||
@ -513,6 +622,107 @@ func (s *IngestService) fetchLookImages(ctx context.Context, looks []dto.RunwayL
|
||||
return cover, rows, vecs, keys, failed, timing
|
||||
}
|
||||
|
||||
// maxInt 取两数较大者(expected_images 只增不减)。
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// mergeMissingImages 把 payload 里「库里还没有」的图片补进一场已存在的秀,返回补了几张。
|
||||
//
|
||||
// 为什么不用下载:媒体机直传时 payload 带的是 `main_key` / `detail_keys`(内容 sha1),
|
||||
// 库里的 `image` 列存的也是同一个 key,所以「这张图有没有」是一次纯内存查表。
|
||||
// 图片本来就已经在对象存储里了,补的是**数据库行**,不产生任何网络流量。
|
||||
//
|
||||
// 老爬虫只上报 URL(没有 key)时无法比对——那种情况直接返回 0,
|
||||
// 由「覆盖率判定」保证新爬虫一定上报 key,从而让补齐这条路可用。
|
||||
func (s *IngestService) mergeMissingImages(ctx context.Context, runwayID, brandID uint32, p dto.RunwayIngest) (int, error) {
|
||||
if runwayID == 0 || len(p.Looks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
st, err := s.repo.RunwayImageState(ctx, runwayID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
added := 0
|
||||
order := st.MaxSort
|
||||
for li, look := range p.Looks {
|
||||
lookNo := li + 1
|
||||
|
||||
// 主图:已有就复用 id(细节图要挂它),没有就新建一行。
|
||||
var parentID uint32
|
||||
if look.MainKey != "" {
|
||||
if id, ok := st.Keys[look.MainKey]; ok {
|
||||
parentID = id
|
||||
} else {
|
||||
order++
|
||||
img := model.BrandRunwayImage{
|
||||
RunwayID: runwayID,
|
||||
BrandID: brandID,
|
||||
Image: look.MainKey,
|
||||
Name: fmt.Sprintf("Look %d", lookNo),
|
||||
SortOrder: order,
|
||||
IsDetail: 0,
|
||||
ImageSize: look.MainSize,
|
||||
}
|
||||
if err := s.repo.CreateRunwayImage(ctx, &img); err != nil {
|
||||
return added, err
|
||||
}
|
||||
parentID = img.ID
|
||||
st.Keys[look.MainKey] = img.ID
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
// 细节图:必须能挂到父主图,否则宁可跳过(挂不上的细节图在后台无法展开)。
|
||||
if parentID == 0 {
|
||||
continue
|
||||
}
|
||||
for di, dk := range look.DetailKeys {
|
||||
if dk == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := st.Keys[dk]; ok {
|
||||
continue
|
||||
}
|
||||
order++
|
||||
img := model.BrandRunwayImage{
|
||||
RunwayID: runwayID,
|
||||
BrandID: brandID,
|
||||
Image: dk,
|
||||
Name: fmt.Sprintf("Look %d — Detail %d", lookNo, di+1),
|
||||
SortOrder: order,
|
||||
IsDetail: 1,
|
||||
ParentImageID: parentID,
|
||||
ImageSize: sizeAt(look.DetailSizes, di),
|
||||
}
|
||||
if err := s.repo.CreateRunwayImage(ctx, &img); err != nil {
|
||||
return added, err
|
||||
}
|
||||
st.Keys[dk] = img.ID
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
if added > 0 || p.ExpectedImages > 0 {
|
||||
if err := s.repo.SyncRunwayImageStats(ctx, runwayID, p.ExpectedImages); err != nil {
|
||||
return added, err
|
||||
}
|
||||
}
|
||||
return added, nil
|
||||
}
|
||||
|
||||
// sizeAt 越界时返回 0(spider 可能只上报部分 size)。
|
||||
func sizeAt(sizes []int64, i int) int64 {
|
||||
if i >= 0 && i < len(sizes) {
|
||||
return sizes[i]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// countMainImages 数一批待落库图片行中的主图(is_detail = 0)数量,即 image_count 的口径。
|
||||
// 与 SoftDeleteRunwayImage 审核期重算的口径一致(只计主图),保证入库值与后续重算值同源。
|
||||
func countMainImages(images []model.BrandRunwayImage) uint16 {
|
||||
@ -541,13 +751,13 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
// 删除即永久:已软删的实体键不再因重爬复活(与 runway 同源)。
|
||||
if exist && isDeleted {
|
||||
log.Printf("[ingest] job=%d street 实体已删除(is_deleted=1),跳过重爬", job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedDeleted, "街拍实体已删除(删除即永久,不重爬)")
|
||||
return
|
||||
}
|
||||
action := decideIngestAction(exist, existStatus)
|
||||
if action == ingestSkip {
|
||||
log.Printf("[ingest] job=%d street 实体已存在(status=%s),跳过", job.ID, existStatus)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
_ = s.repo.MarkDone(ctx, job.ID, model.IngestOutcomeSkippedExisting, "街拍已存在(status="+existStatus+"),跳过重下")
|
||||
return
|
||||
}
|
||||
|
||||
@ -615,7 +825,11 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
s.precomputeSimilar(ctx, dto.IngestKindStreet, embedIDs)
|
||||
log.Printf("[ingest] job=%d street 写正式表 耗时=%v 图片行=%d 复用驳回=%v",
|
||||
job.ID, time.Since(writeStart).Round(time.Millisecond), len(rows), action == ingestReuseRejected)
|
||||
_ = s.repo.MarkDone(ctx, job.ID)
|
||||
outcome, note := model.IngestOutcomeCreated, fmt.Sprintf("新建街拍 %d 张图", len(rows))
|
||||
if action == ingestReuseRejected {
|
||||
outcome, note = model.IngestOutcomeReused, fmt.Sprintf("复用驳回的街拍,覆盖 %d 张图", len(rows))
|
||||
}
|
||||
_ = s.repo.MarkDone(ctx, job.ID, outcome, note)
|
||||
}
|
||||
|
||||
// fetchImages 下载图片并上传到存储,返回 (cover 地址, 已下载图结构, 已成功上传对象的 key 列表, 是否有任意一张失败)。
|
||||
@ -830,9 +1044,27 @@ func concurrentFetch(ctx context.Context, n int, fn func(context.Context, int) d
|
||||
// 是主图还是第几张细节图,供并发下载完成后重建 Name / 主副图归属 / IsDetail。
|
||||
type lookImageTask struct {
|
||||
url string
|
||||
lookIdx int // 1-based look 序号
|
||||
isDetail bool // true=细节图
|
||||
detailNo int // 细节图序号(1-based),主图为 0
|
||||
key string // 非空 = 媒体机已上传,worker 跳过下载/上传,直接落库
|
||||
size int64 // key 路径下由 spider 上报的字节数(0=未知)
|
||||
lookIdx int // 1-based look 序号
|
||||
isDetail bool // true=细节图
|
||||
detailNo int // 细节图序号(1-based),主图为 0
|
||||
}
|
||||
|
||||
// sha1FromKey 从内容寻址 key(runway/<sha1>.jpg 或 street/<sha1>.png)反推 sha1,
|
||||
// 供批次内去重用;key 不符合该形态时退化为 key 本身(仍能起到去重作用)。
|
||||
func sha1FromKey(key string) string {
|
||||
base := key
|
||||
if i := strings.LastIndex(base, "/"); i >= 0 {
|
||||
base = base[i+1:]
|
||||
}
|
||||
if i := strings.LastIndex(base, "."); i > 0 {
|
||||
base = base[:i]
|
||||
}
|
||||
if base == "" {
|
||||
return key
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
// downloadAndUpload 把单张远程图下载后上传到存储(S4优先,失败兜底本地)。
|
||||
|
||||
@ -17,7 +17,13 @@ func (fakeMediaRepo) CountImageKeyRefs(ctx context.Context, key string) (int, er
|
||||
// fakeIngestRepo 仅实现 processMediaCleanup 需要的 MarkDone/MarkFailed(其余由内嵌 nil 接口兜底,不会被调用)。
|
||||
type fakeIngestRepo struct{ repository.IngestRepository }
|
||||
|
||||
func (fakeIngestRepo) MarkDone(ctx context.Context, id uint32) error { return nil }
|
||||
func (fakeIngestRepo) MarkDone(ctx context.Context, id uint32, outcome, note string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fakeIngestRepo) ListByBrand(ctx context.Context, brandID uint32, limit int) ([]repository.BrandIngestRecord, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (fakeIngestRepo) MarkFailed(ctx context.Context, id uint32, errMsg string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -31,6 +31,20 @@ func (f *fakeEmbeddingRepo) UpsertEmbedding(context.Context, string, uint32, []f
|
||||
return nil
|
||||
}
|
||||
|
||||
// 向量回填相关方法:近邻服务本身用不到,这里只需满足接口。
|
||||
// 回填链路的测试在 handler 层(backstage_embeddings)与本文件的 key 分支单测里覆盖。
|
||||
func (f *fakeEmbeddingRepo) ListPendingEmbeddings(context.Context, string, int, int) ([]repository.PendingEmbedding, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeEmbeddingRepo) CountPendingEmbeddings(context.Context, string, int) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (f *fakeEmbeddingRepo) RecordEmbeddingFailure(context.Context, string, uint32, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeEmbeddingRepo) BulkUpsertEmbedding(context.Context, string, []uint32, [][]float32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -71,7 +71,7 @@ server {
|
||||
|
||||
# 内部:前端构建期取 SSG 数据(只绑宿主回环,前端镜像 build 时用 host 网络访问)
|
||||
server {
|
||||
listen 127.0.0.1:18091;
|
||||
listen 18091; # 容器内必须监听 0.0.0.0,否则 docker 端口发布到容器网卡时连不上;宿主机只发布到 127.0.0.1
|
||||
access_log off;
|
||||
location / {
|
||||
proxy_pass http://$ssg_up;
|
||||
@ -81,7 +81,7 @@ server {
|
||||
|
||||
# 内部:管理后台(只绑宿主回环;远程访问用 SSH 隧道到 18092)
|
||||
server {
|
||||
listen 127.0.0.1:18092;
|
||||
listen 18092; # 同上:容器内回环监听会导致 docker 端口发布不可达(管理后台/爬虫入库走此口)
|
||||
location / {
|
||||
proxy_pass http://$admin_up;
|
||||
proxy_set_header Host $host;
|
||||
|
||||
30
scripts/backup-db.sh
Normal file
30
scripts/backup-db.sh
Normal file
@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# 备份 1Panel 管理的 Postgres(直接 docker exec 进它的容器,不依赖宿主端口)
|
||||
#
|
||||
# ./scripts/backup-db.sh # 输出 backups/db-YYYY-MM-DD-HHMMSS.sql.gz
|
||||
# KEEP_DAYS=7 ./scripts/backup-db.sh # 顺手删掉 7 天前的备份
|
||||
#
|
||||
# cron(每天 4 点):
|
||||
# 0 4 * * * cd /opt/fashion/backend && KEEP_DAYS=7 ./scripts/backup-db.sh >> .deploy/backup.log 2>&1
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
[ -f .env ] || { echo "!! 缺少 .env"; exit 1; }
|
||||
set -a; . ./.env; set +a
|
||||
|
||||
PG_CONTAINER="${PG_CONTAINER:?PG_CONTAINER 未设置:填 .env 里那个 postgres 容器名}"
|
||||
KEEP_DAYS="${KEEP_DAYS:-0}"
|
||||
mkdir -p backups
|
||||
|
||||
OUT="backups/db-$(date +%F-%H%M%S).sql.gz"
|
||||
echo "==> 备份到 $OUT"
|
||||
docker exec -e PGPASSWORD="${DB_PASSWORD:?DB_PASSWORD 未设置}" "$PG_CONTAINER" \
|
||||
pg_dump -U "${DB_USER:-fashion}" "${DB_NAME:-fashion}" | gzip > "$OUT"
|
||||
|
||||
echo " $(du -h "$OUT" | cut -f1)"
|
||||
|
||||
if [ "$KEEP_DAYS" -gt 0 ]; then
|
||||
find backups -name 'db-*.sql.gz' -mtime +"$KEEP_DAYS" -print -delete
|
||||
fi
|
||||
|
||||
echo "==> 完成"
|
||||
94
scripts/check_s4.py
Normal file
94
scripts/check_s4.py
Normal file
@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""在生产机上验证 S4 直传链路:签发预签名 URL → PUT 一个小文件 → 公开读回。
|
||||
|
||||
只输出状态与长度,不输出密钥。
|
||||
用法:python3 check_s4.py
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ENV_PATH = "/opt/fashion/backend_v2/.env"
|
||||
ENDPOINT = "http://127.0.0.1:18092/admin/internal/upload-url"
|
||||
|
||||
|
||||
def load_env(path):
|
||||
env = {}
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
env[key.strip()] = value
|
||||
return env
|
||||
|
||||
|
||||
def main():
|
||||
env = load_env(ENV_PATH)
|
||||
secret = env.get("INGEST_SECRET", "")
|
||||
bucket = env.get("S4_BUCKET", "")
|
||||
if not secret:
|
||||
print("!! .env 里没有 INGEST_SECRET")
|
||||
return 1
|
||||
|
||||
key = f"runway/s4test-{int(time.time())}.jpg"
|
||||
body = json.dumps({"items": [{"key": key, "content_type": "image/jpeg"}]}, separators=(",", ":"))
|
||||
ts = str(int(time.time()))
|
||||
nonce = secrets.token_hex(8)
|
||||
sig = hmac.new(secret.encode(), f"{ts}.{nonce}.{body}".encode(), hashlib.sha256).hexdigest()
|
||||
|
||||
req = urllib.request.Request(
|
||||
ENDPOINT,
|
||||
data=body.encode(),
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Signature": sig,
|
||||
"X-Timestamp": ts,
|
||||
"X-Nonce": nonce,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
print(f"1) 签发预签名: HTTP {resp.status}")
|
||||
data = json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f"1) 签发预签名失败: HTTP {exc.code} {exc.read()[:200]!r}")
|
||||
return 1
|
||||
|
||||
item = data["items"][0]
|
||||
# 预签名 URL 里带签名参数,只显示 host 与 path
|
||||
parsed = urllib.request.urlparse(item["url"])
|
||||
print(f" 签名 URL: {parsed.scheme}://{parsed.netloc}{parsed.path[:40]}... 长度={len(item['url'])}")
|
||||
|
||||
payload = b"\xff\xd8\xff\xe0" + b"fashion-s4-probe" + b"\xff\xd9"
|
||||
put = urllib.request.Request(
|
||||
item["url"], data=payload, method="PUT", headers={"Content-Type": "image/jpeg"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(put, timeout=30) as resp:
|
||||
print(f"2) 上传: HTTP {resp.status}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f"2) 上传失败: HTTP {exc.code} {exc.read()[:300]!r}")
|
||||
return 1
|
||||
|
||||
public_url = f"https://{bucket}.s3.bitiful.net/{key}"
|
||||
try:
|
||||
with urllib.request.urlopen(public_url, timeout=20) as resp:
|
||||
got = resp.read()
|
||||
print(f"3) 公开读取: HTTP {resp.status} 字节={len(got)} 一致={got == payload}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f"3) 公开读取失败: HTTP {exc.code}(桶可能不是公开读)")
|
||||
return 1
|
||||
|
||||
print(f" 测试对象: {key}({len(payload)} 字节,可忽略)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
53
scripts/patch_env.py
Normal file
53
scripts/patch_env.py
Normal file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""把 KEY=VALUE 文件里的键合并进目标 .env(存在则替换,不存在则追加)。
|
||||
|
||||
用途:在不把密钥回显到终端的前提下,更新服务器上的生产配置。
|
||||
用法:python3 patch_env.py <kv文件> <目标.env>
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 3:
|
||||
print("用法: patch_env.py <kv文件> <目标.env>")
|
||||
return 2
|
||||
kv_path, env_path = sys.argv[1], sys.argv[2]
|
||||
|
||||
kv = {}
|
||||
with open(kv_path, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or "=" not in line or line.startswith("#"):
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
kv[key.strip()] = value
|
||||
|
||||
with open(env_path, encoding="utf-8") as fh:
|
||||
lines = fh.read().splitlines()
|
||||
|
||||
seen = set()
|
||||
out = []
|
||||
for line in lines:
|
||||
m = re.match(r"^([A-Z0-9_]+)=", line)
|
||||
if m and m.group(1) in kv:
|
||||
key = m.group(1)
|
||||
out.append(f"{key}={kv[key]}")
|
||||
seen.add(key)
|
||||
else:
|
||||
out.append(line)
|
||||
for key, value in kv.items():
|
||||
if key not in seen:
|
||||
out.append(f"{key}={value}")
|
||||
|
||||
with open(env_path, "w", encoding="utf-8") as fh:
|
||||
fh.write("\n".join(out) + "\n")
|
||||
|
||||
# 只打印键名与长度,绝不打印值
|
||||
for key in sorted(kv):
|
||||
print(f" {key}: {'已更新' if key in seen else '已追加'} 长度={len(kv[key])}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
121
scripts/push-build.sh
Normal file
121
scripts/push-build.sh
Normal file
@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
# 本地构建 + 上传 + 远程发布(前后端都走预构建,服务器不编译)
|
||||
#
|
||||
# SERVER=user@server ./scripts/push-build.sh
|
||||
# SERVER=user@server REMOTE_DIR=/opt/fashion WEB_DIR=../test ./scripts/push-build.sh
|
||||
#
|
||||
# 流程:
|
||||
# 1. 建 SSH 隧道(本地 18091 → 服务器回环 18091):前端构建期靠它读线上 SSG 数据
|
||||
# 2. 本机 npm run build → 静态页内容与线上数据一致,且不占服务器资源
|
||||
# 3. 本机交叉编译 linux/amd64 → 产出 server-linux-amd64
|
||||
# 4. 上传 dist/ 与二进制
|
||||
# 5. 远程 ./deploy.sh all --prebuilt → 服务器只打镜像 + 蓝绿切换
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.." # 后端仓根
|
||||
|
||||
SERVER="${SERVER:-}"
|
||||
if [ -z "$SERVER" ]; then
|
||||
echo "用法: SERVER=user@host ./scripts/push-build.sh"
|
||||
echo ""
|
||||
echo "可选环境变量:"
|
||||
echo " REMOTE_DIR 服务器上的项目根(默认 /opt/fashion)"
|
||||
echo " WEB_DIR 本地前端仓(默认 ../test)"
|
||||
echo " SSH_OPTS 额外 ssh 参数(默认 -o ConnectTimeout=10)"
|
||||
exit 1
|
||||
fi
|
||||
REMOTE_DIR="${REMOTE_DIR:-/opt/fashion}"
|
||||
REMOTE_BACKEND="${REMOTE_BACKEND:-${REMOTE_DIR}/backend_v2}"
|
||||
REMOTE_FRONTEND="${REMOTE_FRONTEND:-${REMOTE_DIR}/frontend_v2}"
|
||||
WEB_DIR="${WEB_DIR:-../frontend_v2}"
|
||||
SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10}"
|
||||
TUNNEL_PORT="${TUNNEL_PORT:-18091}"
|
||||
|
||||
# 本机也放一份 .env(复制服务器那份即可),用于取 SSG_TOKEN 等构建期密钥
|
||||
SSG_TOKEN="${SSG_TOKEN:-}"
|
||||
if [ -f .env ]; then
|
||||
# shellcheck disable=SC1091
|
||||
set -a; . ./.env; set +a
|
||||
fi
|
||||
|
||||
# Git Bash 的 $HOME 常是 /home/<user>,而 Windows 上生成的密钥在 C:\Users\<user>\.ssh 下。
|
||||
# 这里自动兜底,避免在 Git Bash 里 ssh 找不到私钥而报 publickey denied。
|
||||
if [ -z "${SSH_KEY:-}" ]; then
|
||||
# Windows 用户目录优先:Git Bash 的 $HOME 下可能还躺着一把无关的旧钥匙
|
||||
for cand in "/c/Users/${USERNAME:-}/.ssh/id_ed25519" "${USERPROFILE:-}/.ssh/id_ed25519" "$HOME/.ssh/id_ed25519"; do
|
||||
if [ -n "$cand" ] && [ -f "$cand" ]; then
|
||||
SSH_KEY="$cand"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -n "${SSH_KEY:-}" ]; then
|
||||
SSH_OPTS="$SSH_OPTS -i $SSH_KEY"
|
||||
echo " 使用私钥:$SSH_KEY"
|
||||
fi
|
||||
|
||||
TUNNEL_PID=""
|
||||
cleanup() { [ -n "$TUNNEL_PID" ] && kill "$TUNNEL_PID" 2>/dev/null || true; }
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "==> 0/5 检查 SSH 免密登录"
|
||||
# 隧道是后台起的(ssh -N ... &),一旦需要交互输密码就会静默卡住,所以必须免密。
|
||||
if ! ssh $SSH_OPTS -o BatchMode=yes "$SERVER" true 2>/dev/null; then
|
||||
echo "!! 无法免密登录 $SERVER"
|
||||
echo ""
|
||||
echo " Windows(PowerShell):"
|
||||
echo " ssh-keygen -t ed25519 -C deploy@local"
|
||||
echo " type \$env:USERPROFILE\\.ssh\\id_ed25519.pub | ssh $SERVER \"mkdir -p ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys\""
|
||||
echo ""
|
||||
echo " Linux / macOS / WSL:"
|
||||
echo " ssh-keygen -t ed25519 -C deploy@local && ssh-copy-id $SERVER"
|
||||
echo ""
|
||||
echo " 配好后用这条确认:ssh $SERVER true && echo ok"
|
||||
exit 1
|
||||
fi
|
||||
echo " 通过"
|
||||
|
||||
# 服务器若开了 SSG_TOKEN,构建期必须带同一个值。自动从服务器 .env 取,避免本机再维护一份密钥。
|
||||
if [ -z "${SSG_TOKEN:-}" ]; then
|
||||
SSG_TOKEN="$(ssh $SSH_OPTS -o BatchMode=yes "$SERVER" \
|
||||
"grep -m1 '^SSG_TOKEN=' '${REMOTE_BACKEND}/.env' 2>/dev/null | cut -d= -f2-" 2>/dev/null || true)"
|
||||
if [ -n "$SSG_TOKEN" ]; then
|
||||
echo " 已从服务器读取 SSG_TOKEN(${#SSG_TOKEN} 字符)"
|
||||
else
|
||||
echo " 服务器未设置 SSG_TOKEN,构建期不带 token"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "==> 1/5 建立 SSH 隧道(本地 ${TUNNEL_PORT} → ${SERVER}:127.0.0.1:18091)"
|
||||
ssh -N $SSH_OPTS -o ExitOnForwardFailure=yes \
|
||||
-L "${TUNNEL_PORT}:127.0.0.1:18091" "$SERVER" &
|
||||
TUNNEL_PID=$!
|
||||
|
||||
for i in $(seq 1 15); do
|
||||
if (echo > "/dev/tcp/127.0.0.1/${TUNNEL_PORT}") >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 15 ]; then
|
||||
echo "!! 隧道 15 秒内没建起来(检查 SSH 连通性、服务器上 proxy 是否在跑)"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "==> 2/5 构建前端(构建期数据来自线上)"
|
||||
(cd "$WEB_DIR" && BASE_API_SSG="http://127.0.0.1:${TUNNEL_PORT}" SSG_TOKEN="$SSG_TOKEN" npm run build)
|
||||
[ -f "$WEB_DIR/dist/server/entry.mjs" ] || { echo "!! 前端产物缺失:$WEB_DIR/dist/server/entry.mjs"; exit 1; }
|
||||
|
||||
echo "==> 3/5 交叉编译后端(linux/amd64)"
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build -trimpath -ldflags "-s -w" -o server-linux-amd64 ./cmd/server
|
||||
|
||||
echo "==> 4/5 上传产物到 ${SERVER}:${REMOTE_DIR}"
|
||||
ssh $SSH_OPTS "$SERVER" "mkdir -p '${REMOTE_BACKEND}' '${REMOTE_FRONTEND}'"
|
||||
tar -C "$WEB_DIR" -czf - dist | ssh $SSH_OPTS "$SERVER" "tar -xzf - -C '${REMOTE_FRONTEND}'"
|
||||
scp $SSH_OPTS server-linux-amd64 "${SERVER}:${REMOTE_BACKEND}/server-linux-amd64"
|
||||
|
||||
echo "==> 5/5 远程发布(预构建 + 蓝绿切换)"
|
||||
RELEASE="$(date +%Y%m%d-%H%M%S)-$(git rev-parse --short HEAD 2>/dev/null || echo nogit)"
|
||||
ssh $SSH_OPTS "$SERVER" "cd '${REMOTE_BACKEND}' && RELEASE='${RELEASE}' ./deploy.sh all --prebuilt"
|
||||
|
||||
echo "完成:$RELEASE"
|
||||
Reference in New Issue
Block a user