update
This commit is contained in:
toom1996
@ -57,6 +57,7 @@ import (
|
||||
"bufio"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -106,6 +107,10 @@ func main() {
|
||||
runDump(os.Args[2:])
|
||||
case "import":
|
||||
runImport(os.Args[2:])
|
||||
case "fixdup":
|
||||
runFixDup(os.Args[2:])
|
||||
case "purge-rejected":
|
||||
runPurgeRejected(os.Args[2:])
|
||||
default:
|
||||
usage()
|
||||
os.Exit(2)
|
||||
@ -120,6 +125,12 @@ func usage() {
|
||||
导出「结构 + 主键 / 索引 / 视图 + 数据」为单个纯 SQL 文件
|
||||
dbtool import [-config <yml>] [-in <file>] [-clean]
|
||||
把 dump 出来的 SQL 文件灌入目标库
|
||||
dbtool fixdup [-config <yml>]
|
||||
清理「悬空」重复标记（is_duplicate=1 但其源图已删/软删），修编辑页误显「重复」
|
||||
dbtool purge-rejected [-config <yml>]
|
||||
把现存「已驳回（rejected）」且未下架的记录转删除：级联软删其全部图片、记录置
|
||||
is_deleted=1（后台即消失），并把图片 key 入队 media_cleanup，由运行中的 worker
|
||||
按引用计数清理 S4 孤儿文件（与后台点「删除（清空 S4）」等价，但一次性全局处理）
|
||||
|
||||
说明:
|
||||
连接信息读取 configs/config.yml 的 database 段（可用 -config 覆盖）。
|
||||
@ -281,6 +292,164 @@ func runImport(args []string) {
|
||||
log.Printf("✓ 导入完成（%d KB）", len(script)/1024)
|
||||
}
|
||||
|
||||
// runFixDup 清理「悬空」重复标记：is_duplicate=1 但 dup_of 指向的源图行已不存在或已被软删
|
||||
// （is_deleted=1）。这类陈旧标记会让编辑页误显「重复」且源图打不开。仅清悬空项，真实近重复
|
||||
// （源图仍在且存活）不受影响。等价于 DeleteImage 触发 ReresolveRunwayDedup 的重算效果之一，
|
||||
// 但可一次性全局修历史遗留数据，无需逐张删图触发。
|
||||
func runFixDup(args []string) {
|
||||
fs := flag.NewFlagSet("fixdup", flag.ExitOnError)
|
||||
cfgPath := fs.String("config", "", "配置文件路径（默认 configs/config.yml）")
|
||||
_ = fs.Parse(args)
|
||||
|
||||
db, _, err := openDB(*cfgPath)
|
||||
if err != nil {
|
||||
log.Fatalf("✗ %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
for _, t := range []string{"brand_runway_images", "street_snap_images"} {
|
||||
q := fmt.Sprintf(`
|
||||
UPDATE public.%[1]s
|
||||
SET is_duplicate = 0, dup_of = ''
|
||||
WHERE is_duplicate = 1
|
||||
AND (dup_of IS NULL OR dup_of = '' OR dup_of NOT IN (
|
||||
SELECT CAST(id AS text) FROM public.%[1]s WHERE is_deleted = 0));`, qi(t))
|
||||
res, err := db.Exec(q)
|
||||
if err != nil {
|
||||
log.Fatalf("✗ 清理 %s 失败: %v", t, err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
log.Printf("✓ %s: 清理悬空重复标记 %d 行", t, n)
|
||||
}
|
||||
}
|
||||
|
||||
// runPurgeRejected 把现存「已驳回（rejected）」且未下架的记录转删除：
|
||||
// 级联软删其全部图片、记录自身置 is_deleted=1（后台 legWhere 强制 is_deleted=0，记录即从列表消失），
|
||||
// 并把图片 key 作为 media_cleanup 任务入队，由运行中的 worker 按引用计数真删 S4 孤儿文件。
|
||||
//
|
||||
// 与后台点「删除（清空 S4）」完全等价（后者走 articleService/streetSnapService.SetDeleted →
|
||||
// PurgeGallery），但本命令一次性全局处理历史遗留的 rejected 数据，无需逐条手动操作。
|
||||
func runPurgeRejected(args []string) {
|
||||
fs := flag.NewFlagSet("purge-rejected", flag.ExitOnError)
|
||||
cfgPath := fs.String("config", "", "配置文件路径（默认 configs/config.yml）")
|
||||
_ = fs.Parse(args)
|
||||
|
||||
db, _, err := openDB(*cfgPath)
|
||||
if err != nil {
|
||||
log.Fatalf("✗ %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// 两种实体：记录主表 / 图片明细表 / 图片表外键列。
|
||||
kinds := []struct {
|
||||
recTable, imgTable, fk string
|
||||
}{
|
||||
{"brand_runways", "brand_runway_images", "runway_id"},
|
||||
{"street_snaps", "street_snap_images", "snap_id"},
|
||||
}
|
||||
|
||||
now := uint32(time.Now().Unix())
|
||||
totalRecs, totalKeys := 0, 0
|
||||
var allKeys []string
|
||||
|
||||
for _, k := range kinds {
|
||||
ids, err := selectUint32s(db, fmt.Sprintf(
|
||||
"SELECT id FROM public.%s WHERE status = 'rejected' AND is_deleted = 0", qi(k.recTable)))
|
||||
if err != nil {
|
||||
log.Fatalf("✗ 查询 %s 失败: %v", k.recTable, err)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
log.Printf("→ %s: 无 rejected 记录", k.recTable)
|
||||
continue
|
||||
}
|
||||
for _, id := range ids {
|
||||
keys, err := selectStrings(db, fmt.Sprintf(
|
||||
"SELECT image FROM public.%s WHERE %s = $1 AND is_deleted = 0", qi(k.imgTable), k.fk), id)
|
||||
if err != nil {
|
||||
log.Fatalf("✗ 查询 %s 图片失败: %v", k.imgTable, err)
|
||||
}
|
||||
if _, err := db.Exec(fmt.Sprintf(
|
||||
"UPDATE public.%s SET is_deleted = 1, updated_at = $1 WHERE %s = $2 AND is_deleted = 0",
|
||||
qi(k.imgTable), k.fk), now, id); err != nil {
|
||||
log.Fatalf("✗ 软删 %s 图片失败: %v", k.imgTable, err)
|
||||
}
|
||||
if _, err := db.Exec(fmt.Sprintf(
|
||||
"UPDATE public.%s SET is_deleted = 1, updated_at = $1 WHERE id = $2",
|
||||
qi(k.recTable)), now, id); err != nil {
|
||||
log.Fatalf("✗ 置 %s 下架失败: %v", k.recTable, err)
|
||||
}
|
||||
totalRecs++
|
||||
allKeys = append(allKeys, keys...)
|
||||
totalKeys += len(keys)
|
||||
log.Printf("✓ %s #%d: 软删图片 %d 张", k.recTable, id, len(keys))
|
||||
}
|
||||
}
|
||||
|
||||
// 把 key 分批入队 media_cleanup（与 EnqueueMediaCleanup 同格式），worker 按引用计数清 S4。
|
||||
if len(allKeys) > 0 {
|
||||
const batch = 500
|
||||
for start := 0; start < len(allKeys); start += batch {
|
||||
end := start + batch
|
||||
if end > len(allKeys) {
|
||||
end = len(allKeys)
|
||||
}
|
||||
payload, err := json.Marshal(map[string][]string{"keys": allKeys[start:end]})
|
||||
if err != nil {
|
||||
log.Fatalf("✗ 构造 payload 失败: %v", err)
|
||||
}
|
||||
if err := insertMediaCleanup(db, now, string(payload)); err != nil {
|
||||
log.Fatalf("✗ 入队 media_cleanup 失败: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Printf("✓ 完成：转删除记录 %d 条、图片 %d 张，已入队 media_cleanup（运行中 worker 将按引用计数清理 S4）", totalRecs, totalKeys)
|
||||
}
|
||||
|
||||
// selectUint32s 执行返回单列 uint32 的查询。
|
||||
func selectUint32s(db *sql.DB, q string, args ...any) ([]uint32, error) {
|
||||
rows, err := db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []uint32
|
||||
for rows.Next() {
|
||||
var v uint32
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// selectStrings 执行返回单列 string 的查询。
|
||||
func selectStrings(db *sql.DB, q string, args ...any) ([]string, error) {
|
||||
rows, err := db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var v string
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// insertMediaCleanup 写入一条「清理S4孤儿图」任务（与 repository.EnqueueMediaCleanup 同格式）。
|
||||
// payload 为 {"keys":[...]}，由 worker 的 processMediaCleanup 按引用计数判定真孤儿后删除。
|
||||
func insertMediaCleanup(db *sql.DB, now uint32, payload string) error {
|
||||
_, err := db.Exec(
|
||||
"INSERT INTO public.ingest_jobs (kind, payload, status, created_at, updated_at) VALUES ($1, $2, $3, $4, $5)",
|
||||
"media_cleanup", payload, "pending", now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
// dropAllTablesSQL 删除 public 下全部基表（CASCADE 连带索引 / 约束 / 归属该表的序列）。
|
||||
//
|
||||
// 只删表，不动扩展：pgvector 的 vector 类型是 extension 对象，不属于任何表，因此
|
||||
|
||||
363
cmd/diagdup/main.go
Normal file
363
cmd/diagdup/main.go
Normal file
@ -0,0 +1,363 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"fashionapi/internal/pkg/phash"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
const (
|
||||
dsn = "host=127.0.0.1 port=5432 user=fashion password=fashion_dev_2026 dbname=fashion sslmode=disable"
|
||||
base = "https://toomstudio.s3.bitiful.net/"
|
||||
)
|
||||
|
||||
type row struct {
|
||||
ID int64
|
||||
Image string
|
||||
Phash sql.NullString
|
||||
IsDuplicate int16
|
||||
DupOf string
|
||||
RunwayID int64
|
||||
BrandID int64
|
||||
}
|
||||
|
||||
func queryRow(db *sql.DB, like string) *row {
|
||||
r := &row{}
|
||||
q := `SELECT id, image, phash, is_duplicate, dup_of, runway_id, brand_id
|
||||
FROM brand_runway_images WHERE image LIKE $1 ORDER BY id LIMIT 1`
|
||||
err := db.QueryRow(q, like).Scan(&r.ID, &r.Image, &r.Phash, &r.IsDuplicate, &r.DupOf, &r.RunwayID, &r.BrandID)
|
||||
if err != nil {
|
||||
fmt.Printf(" 查询 %s 失败: %v\n", like, err)
|
||||
return nil
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// vectorToUint64 把 "[0,1,...,1]" 转回 uint64（与 phash.ToVectorBits 互逆）。
|
||||
func vectorToUint64(s string) uint64 {
|
||||
s = strings.TrimSpace(s)
|
||||
s = strings.TrimPrefix(s, "[")
|
||||
s = strings.TrimSuffix(s, "]")
|
||||
parts := strings.Split(s, ",")
|
||||
var h uint64
|
||||
for i, p := range parts {
|
||||
if i >= 64 {
|
||||
break
|
||||
}
|
||||
if strings.TrimSpace(p) == "1" {
|
||||
h |= 1 << uint(i)
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func hamming(a, b uint64) int {
|
||||
c := 0
|
||||
x := a ^ b
|
||||
for x != 0 {
|
||||
x &= x - 1
|
||||
c++
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// fetchImage 下载原图，返回解码后的 image 与原始字节。
|
||||
func fetchImage(key string) (image.Image, []byte, bool) {
|
||||
url := base + strings.TrimPrefix(key, "/")
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
fmt.Printf(" 下载 %s 失败: %v\n", url, err)
|
||||
return nil, nil, false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
fmt.Printf(" 下载 %s 状态 %d\n", url, resp.StatusCode)
|
||||
return nil, nil, false
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Printf(" 读 %s 失败: %v\n", url, err)
|
||||
return nil, nil, false
|
||||
}
|
||||
img, _, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
fmt.Printf(" 解码 %s 失败: %v\n", url, err)
|
||||
return nil, nil, false
|
||||
}
|
||||
return img, data, true
|
||||
}
|
||||
|
||||
// aHash 平均哈希：缩到 8x8 灰度，与整图均值比大小，得 64-bit。比 dHash 更关注整体明暗分布。
|
||||
func aHash(img image.Image) uint64 {
|
||||
const n = 8
|
||||
gray := make([][]float64, n)
|
||||
for i := range gray {
|
||||
gray[i] = make([]float64, n)
|
||||
}
|
||||
b := img.Bounds()
|
||||
var total float64
|
||||
for y := 0; y < n; y++ {
|
||||
for x := 0; x < n; x++ {
|
||||
sx := b.Min.X + (x*b.Dx())/n + b.Dx()/(2*n)
|
||||
sy := b.Min.Y + (y*b.Dy())/n + b.Dy()/(2*n)
|
||||
r, g, bl, _ := img.At(sx, sy).RGBA()
|
||||
l := 0.299*float64(r>>8) + 0.587*float64(g>>8) + 0.114*float64(bl>>8)
|
||||
gray[y][x] = l
|
||||
total += l
|
||||
}
|
||||
}
|
||||
mean := total / float64(n*n)
|
||||
var h uint64
|
||||
i := 0
|
||||
for y := 0; y < n; y++ {
|
||||
for x := 0; x < n; x++ {
|
||||
if gray[y][x] > mean {
|
||||
h |= 1 << uint(i)
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// pHash DCT 感知哈希：缩到 32x32 灰度 → 2D DCT → 取左上 8x8 低频系数 → 与中位数比大小得 64-bit。
|
||||
func pHash(img image.Image) uint64 {
|
||||
const N = 32
|
||||
g := [N][N]float64{}
|
||||
b := img.Bounds()
|
||||
for cy := 0; cy < N; cy++ {
|
||||
for cx := 0; cx < N; cx++ {
|
||||
x0 := b.Min.X + cx*b.Dx()/N
|
||||
x1 := b.Min.X + (cx+1)*b.Dx()/N
|
||||
y0 := b.Min.Y + cy*b.Dy()/N
|
||||
y1 := b.Min.Y + (cy+1)*b.Dy()/N
|
||||
if x1 <= x0 {
|
||||
x1 = x0 + 1
|
||||
}
|
||||
if y1 <= y0 {
|
||||
y1 = y0 + 1
|
||||
}
|
||||
var sum float64
|
||||
var n int
|
||||
for y := y0; y < y1 && y < b.Max.Y; y++ {
|
||||
for x := x0; x < x1 && x < b.Max.X; x++ {
|
||||
r, gg, bl, _ := img.At(x, y).RGBA()
|
||||
sum += 0.299*float64(r>>8) + 0.587*float64(gg>>8) + 0.114*float64(bl>>8)
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n > 0 {
|
||||
g[cy][cx] = sum / float64(n)
|
||||
}
|
||||
}
|
||||
}
|
||||
// 行 DCT-II
|
||||
var rows [N][N]float64
|
||||
for y := 0; y < N; y++ {
|
||||
for u := 0; u < N; u++ {
|
||||
var s float64
|
||||
for x := 0; x < N; x++ {
|
||||
s += g[y][x] * math.Cos(math.Pi*float64(u)*(float64(x)+0.5)/float64(N))
|
||||
}
|
||||
rows[y][u] = s
|
||||
}
|
||||
}
|
||||
// 列 DCT-II
|
||||
var dct [N][N]float64
|
||||
for x := 0; x < N; x++ {
|
||||
for v := 0; v < N; v++ {
|
||||
var s float64
|
||||
for y := 0; y < N; y++ {
|
||||
s += rows[y][x] * math.Cos(math.Pi*float64(v)*(float64(y)+0.5)/float64(N))
|
||||
}
|
||||
dct[v][x] = s
|
||||
}
|
||||
}
|
||||
// 取左上 8x8 低频，与中位数比大小
|
||||
var coeffs [64]float64
|
||||
i := 0
|
||||
for v := 0; v < 8; v++ {
|
||||
for u := 0; u < 8; u++ {
|
||||
coeffs[i] = dct[v][u]
|
||||
i++
|
||||
}
|
||||
}
|
||||
sorted := append([]float64{}, coeffs[:]...)
|
||||
for a := 0; a < len(sorted); a++ {
|
||||
for c := a + 1; c < len(sorted); c++ {
|
||||
if sorted[c] < sorted[a] {
|
||||
sorted[a], sorted[c] = sorted[c], sorted[a]
|
||||
}
|
||||
}
|
||||
}
|
||||
median := sorted[len(sorted)/2]
|
||||
var h uint64
|
||||
for k := 0; k < 64; k++ {
|
||||
if coeffs[k] > median {
|
||||
h |= 1 << uint(k)
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// luminanceStats 算亮度均值/标准差，并用 phash.Of 复算指纹。
|
||||
func luminanceStats(img image.Image, data []byte) (mean, std float64, recomputed uint64, ok bool) {
|
||||
recomputed = phash.Of(data)
|
||||
b := img.Bounds()
|
||||
var sum, sum2 float64
|
||||
var n int
|
||||
step := 1
|
||||
if b.Dx() > 300 || b.Dy() > 300 {
|
||||
step = int(math.Max(1, float64(b.Dx())/300))
|
||||
}
|
||||
for y := b.Min.Y; y < b.Max.Y; y += step {
|
||||
for x := b.Min.X; x < b.Max.X; x += step {
|
||||
r, g, bl, _ := img.At(x, y).RGBA()
|
||||
lum := 0.299*float64(r>>8) + 0.587*float64(g>>8) + 0.114*float64(bl>>8)
|
||||
sum += lum
|
||||
sum2 += lum * lum
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n == 0 {
|
||||
return
|
||||
}
|
||||
mean = sum / float64(n)
|
||||
variance := sum2/float64(n) - mean*mean
|
||||
if variance < 0 {
|
||||
variance = 0
|
||||
}
|
||||
std = math.Sqrt(variance)
|
||||
ok = true
|
||||
return
|
||||
}
|
||||
|
||||
func dumpBits(h uint64) string {
|
||||
var sb strings.Builder
|
||||
for i := 0; i < 64; i++ {
|
||||
if (h>>uint(i))&1 == 1 {
|
||||
sb.WriteByte('1')
|
||||
} else {
|
||||
sb.WriteByte('0')
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func main() {
|
||||
db, err := sql.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
fmt.Println("open db:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
k1 := "6837152087e6b9849368860811920d98d6b986dc"
|
||||
k2 := "53f47b14a8799e682864fb7ffc1f942caaeeb56b"
|
||||
if len(os.Args) >= 3 {
|
||||
k1, k2 = os.Args[1], os.Args[2]
|
||||
}
|
||||
|
||||
r1 := queryRow(db, "%"+k1+"%")
|
||||
r2 := queryRow(db, "%"+k2+"%")
|
||||
if r1 == nil || r2 == nil {
|
||||
fmt.Println("未能取到两行，退出")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("==== 行 1 ====")
|
||||
fmt.Printf("id=%d image=%s is_duplicate=%d dup_of=%s runway_id=%d brand_id=%d\n",
|
||||
r1.ID, r1.Image, r1.IsDuplicate, r1.DupOf, r1.RunwayID, r1.BrandID)
|
||||
fmt.Println("==== 行 2 ====")
|
||||
fmt.Printf("id=%d image=%s is_duplicate=%d dup_of=%s runway_id=%d brand_id=%d\n",
|
||||
r2.ID, r2.Image, r2.IsDuplicate, r2.DupOf, r2.RunwayID, r2.BrandID)
|
||||
|
||||
var h1, h2 uint64
|
||||
if r1.Phash.Valid {
|
||||
h1 = vectorToUint64(r1.Phash.String)
|
||||
fmt.Printf("phash1=%s\nbits1=%s\n", r1.Phash.String, dumpBits(h1))
|
||||
} else {
|
||||
fmt.Println("phash1=NULL")
|
||||
}
|
||||
if r2.Phash.Valid {
|
||||
h2 = vectorToUint64(r2.Phash.String)
|
||||
fmt.Printf("phash2=%s\nbits2=%s\n", r2.Phash.String, dumpBits(h2))
|
||||
} else {
|
||||
fmt.Println("phash2=NULL")
|
||||
}
|
||||
|
||||
if r1.Phash.Valid && r2.Phash.Valid {
|
||||
ham := hamming(h1, h2)
|
||||
fmt.Printf("\n>>> 两张图之间的真实汉明距离 = %d (系统阈值 DefaultThreshold=4)\n", ham)
|
||||
if ham <= 4 {
|
||||
fmt.Println(">>> 系统会把这两者判为近重复（≤4）。")
|
||||
} else {
|
||||
fmt.Println(">>> 这两张彼此并不在 ≤4 内；若被标重，必是各自 dup_of 指向了不同的第三者。")
|
||||
}
|
||||
}
|
||||
|
||||
// 亮度标准差 + 指纹复算（验证“平淡图哈希塌缩”假说）+ aHash 区分度
|
||||
fmt.Println("\n==== 像素核验（原图，无 style）====")
|
||||
var imgs [2]image.Image
|
||||
var ahs [2]uint64
|
||||
var phs [2]uint64
|
||||
for i, r := range []*row{r1, r2} {
|
||||
img, data, ok := fetchImage(r.Image)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
imgs[i] = img
|
||||
ahs[i] = aHash(img)
|
||||
phs[i] = pHash(img)
|
||||
mean, std, recomputed, ok2 := luminanceStats(img, data)
|
||||
if !ok2 {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("image=%s\n 亮度均值=%.1f 标准差=%.1f (标准差越低越“平”，哈希越易塌缩)\n",
|
||||
r.Image, mean, std)
|
||||
stored := vectorToUint64(r.Phash.String)
|
||||
fmt.Printf(" 复算phash与库存phash一致=%v 复算bits=%s\n", recomputed == stored, dumpBits(recomputed))
|
||||
fmt.Printf(" aHash bits=%s\n", dumpBits(ahs[i]))
|
||||
fmt.Printf(" pHash bits=%s\n", dumpBits(phs[i]))
|
||||
}
|
||||
if imgs[0] != nil && imgs[1] != nil {
|
||||
dh := 0
|
||||
if r1.Phash.Valid && r2.Phash.Valid {
|
||||
dh = hamming(vectorToUint64(r1.Phash.String), vectorToUint64(r2.Phash.String))
|
||||
}
|
||||
fmt.Printf("\n>>> dHash 汉明距离 = %d（库存判定，≤4 判重）\n", dh)
|
||||
fmt.Printf(">>> aHash 汉明距离 = %d\n", hamming(ahs[0], ahs[1]))
|
||||
fmt.Printf(">>> pHash 汉明距离 = %d (pHash 常规判重阈值约 10~15，远超 dHash 的 4)\n", hamming(phs[0], phs[1]))
|
||||
}
|
||||
|
||||
// 若某行 dup_of 指向第三者，把它也拉出来看
|
||||
for _, r := range []*row{r1, r2} {
|
||||
if r.DupOf != "" && r.DupOf != "0" {
|
||||
tgt := &row{}
|
||||
err := db.QueryRow(`SELECT id, image, phash, runway_id, brand_id FROM brand_runway_images WHERE id=$1`, r.DupOf).
|
||||
Scan(&tgt.ID, &tgt.Image, &tgt.Phash, &tgt.RunwayID, &tgt.BrandID)
|
||||
if err == nil {
|
||||
fmt.Printf("\n>>> 行 %d 的 dup_of=%s 指向:\n id=%d image=%s runway_id=%d brand_id=%d\n",
|
||||
r.ID, r.DupOf, tgt.ID, tgt.Image, tgt.RunwayID, tgt.BrandID)
|
||||
if tgt.Phash.Valid {
|
||||
th := vectorToUint64(tgt.Phash.String)
|
||||
fmt.Printf(" 与该目标汉明距离=%d\n", hamming(vectorToUint64(r.Phash.String), th))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -143,7 +143,7 @@ func main() {
|
||||
}
|
||||
// 爬虫入库服务：注入 mediaRepo（引用计数删孤儿）+ imgDel（S4/本地删除器）。
|
||||
// 必须在 articleSvc / snapSvc 之前创建——图集服务删除时需调用它把「清理存储孤儿图」异步入队。
|
||||
ingestSvc := service.NewIngestService(ingestRepo, brandRepo, mediaRepo, imgUp, imgDel, localUp)
|
||||
ingestSvc := service.NewIngestService(ingestRepo, brandRepo, mediaRepo, imgUp, imgDel, localUp, cfg.Ingest.DownloadProxy)
|
||||
// 图集服务：注入 ingestSvc 作为「清理孤儿图」的入队器，使下架/删除异步触发存储清理。
|
||||
articleSvc = service.NewArticleService(articleRepo, mediaRepo, imgComposer, imgDel, ingestSvc)
|
||||
snapSvc = service.NewStreetSnapService(snapRepo, mediaRepo, imgComposer, imgDel, ingestSvc)
|
||||
|
||||
Reference in New Issue
Block a user