update

2026-08-30 22:50:33 +08:00
parent ec1ee702bd
commit c97a001a76
6 changed files with 217 additions and 0 deletions
--- a/scripts/backfill_city/main.go
+++ b/scripts/backfill_city/main.go
@ -0,0 +1,206 @@
+// Command backfill_city 给 street_snap 加 city 列并从 title（首选）/ source_url（兜底）回填。
+//
+// 项目已移除 AutoMigrate，表结构由外部 SQL 托管（见 scripts/sql/004_add_streetsnap_city.sql）。
+// 本程序是「没有 mysql 客户端时」的执行器：
+//   - ALTER 走 ensureColumn：先查 information_schema，缺才加，可安全重跑（幂等）。
+//   - 回填：优先从 title 第一段（location，\t 分隔）按城市关键词匹配；
+//     命中不到再退到 source_url / cover 的 URL slug 解析。幂等（已填过的不会覆盖）。
+//
+// 存储的 city 值即「规范英文名首字母大写」（Paris / New York …），
+// 与前端街拍列表 City 筛选项 value、header 下拉 ?city= 参数完全一致（精确匹配）。
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+)
+
+const dsn = "root:root@tcp(127.0.0.1:3306)/db_dev?charset=utf8mb4&parseTime=True&loc=Local"
+
+// cityKeywords 城市关键词 → 规范存储值（首字母大写）。
+// 顺序无关；extractCityFromTitle 用「包含」匹配，可容忍 "Paris Fashion Week" / "New York FW25" 等形式。
+var cityKeywords = []struct {
+	kw   string
+	city string
+}{
+	{"paris", "Paris"},
+	{"milan", "Milan"},
+	{"london", "London"},
+	{"new york", "New York"},
+	{"nyfw", "New York"},
+	{"tokyo", "Tokyo"},
+	{"seoul", "Seoul"},
+	{"florence", "Florence"},
+	{"shanghai", "Shanghai"},
+	{"copenhagen", "Copenhagen"},
+	{"berlin", "Berlin"},
+	{"rome", "Rome"},
+	{"madrid", "Madrid"},
+	{"amsterdam", "Amsterdam"},
+	{"antwerp", "Antwerp"},
+	{"los angeles", "Los Angeles"},
+	{"moscow", "Moscow"},
+	{"mumbai", "Mumbai"},
+	{"beijing", "Beijing"},
+}
+
+// citySlug（URL 兜底用）：slug → 规范存储值。多词城市（new-york）必须排在单词组之前被 HasPrefix 命中。
+var citySlug = map[string]string{
+	"paris":       "Paris",
+	"milan":       "Milan",
+	"london":      "London",
+	"new-york":    "New York",
+	"tokyo":       "Tokyo",
+	"seoul":       "Seoul",
+	"copenhagen":  "Copenhagen",
+	"florence":    "Florence",
+	"berlin":      "Berlin",
+	"shanghai":    "Shanghai",
+	"beijing":     "Beijing",
+	"moscow":      "Moscow",
+	"los-angeles": "Los Angeles",
+	"rome":        "Rome",
+	"madrid":      "Madrid",
+	"amsterdam":   "Amsterdam",
+	"antwerp":     "Antwerp",
+	"mumbai":      "Mumbai",
+}
+
+// extractCityFromTitle 从 title 第一段（location，\t 分隔）按关键词匹配规范城市名。
+func extractCityFromTitle(title string) string {
+	seg := title
+	if i := strings.Index(title, "\t"); i >= 0 {
+		seg = title[:i]
+	}
+	segL := strings.ToLower(strings.TrimSpace(seg))
+	if segL == "" {
+		return ""
+	}
+	for _, c := range cityKeywords {
+		if strings.Contains(segL, c.kw) {
+			return c.city
+		}
+	}
+	return ""
+}
+
+// extractCityFromURL 兜底：从 source_url / cover 的 URL 第一段 slug 解析城市。
+func extractCityFromURL(raws ...string) string {
+	for _, raw := range raws {
+		u := strings.TrimSpace(raw)
+		if u == "" {
+			continue
+		}
+		if i := strings.Index(u, "://"); i >= 0 {
+			u = u[i+3:]
+		}
+		if i := strings.Index(u, "/"); i >= 0 {
+			u = u[i+1:]
+		}
+		u = strings.Trim(u, "/")
+		if u == "" {
+			continue
+		}
+		seg := strings.SplitN(u, "/", 2)[0]
+		segL := strings.ToLower(seg)
+		for slug, city := range citySlug {
+			if segL == slug || strings.HasPrefix(segL, slug+"-") {
+				return city
+			}
+		}
+	}
+	return ""
+}
+
+func main() {
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		log.Fatal("open:", err)
+	}
+
+	// 1) 幂等加列（与 004_add_streetsnap_city.sql 对齐）
+	ensureColumn(db, "street_snap", "city", "VARCHAR(128) NOT NULL DEFAULT '' AFTER year")
+
+	// 2) 逐行回填：优先 title 第一段，兜底 URL slug
+	type row struct {
+		ID        uint32
+		Title     string
+		SourceURL string
+		Cover     string
+	}
+	var rows []row
+	if r := db.Raw("SELECT id, title, source_url, cover FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='')").Scan(&rows); r.Error != nil {
+		log.Fatal("select:", r.Error)
+	}
+	var updated int64
+	for _, r := range rows {
+		city := extractCityFromTitle(r.Title)
+		if city == "" {
+			city = extractCityFromURL(r.SourceURL, r.Cover)
+		}
+		if city == "" {
+			continue
+		}
+		if r2 := db.Exec("UPDATE street_snap SET city=? WHERE id=?", city, r.ID); r2.Error != nil {
+			log.Printf("update id=%d failed: %v", r.ID, r2.Error)
+			continue
+		}
+		updated++
+	}
+	fmt.Printf("  updated city for %d rows\n", updated)
+
+	// 3) 校验：输出 distinct city 分布 + 未归类样本
+	printCities(db)
+	printUnclassifiedSamples(db)
+	fmt.Println("backfill city done.")
+}
+
+func ensureColumn(db *gorm.DB, table, col, def string) {
+	var n int64
+	db.Raw("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA='db_dev' AND TABLE_NAME=? AND COLUMN_NAME=?", table, col).Scan(&n)
+	if n > 0 {
+		fmt.Printf("  skip %s.%s (exists)\n", table, col)
+		return
+	}
+	stmt := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, col, def)
+	fmt.Println("  +", stmt)
+	if r := db.Exec(stmt); r.Error != nil {
+		log.Fatalf("alter %s.%s failed: %v", table, col, r.Error)
+	}
+}
+
+func printCities(db *gorm.DB) {
+	type row struct {
+		City string
+		Cnt  int64
+	}
+	var rows []row
+	db.Raw(`SELECT city, COUNT(*) cnt FROM street_snap WHERE is_deleted=0 AND city<>'' GROUP BY city ORDER BY cnt DESC`).Scan(&rows)
+	fmt.Printf("  distinct cities: %d\n", len(rows))
+	for _, r := range rows {
+		fmt.Printf("    %-14q %d\n", r.City, r.Cnt)
+	}
+}
+
+// printUnclassifiedSamples 输出仍未归类行的 title 第一段样本，便于后续补充城市词典。
+func printUnclassifiedSamples(db *gorm.DB) {
+	type row struct {
+		Title string
+	}
+	var rows []row
+	db.Raw(`SELECT title FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='') LIMIT 25`).Scan(&rows)
+	var total int64
+	db.Raw(`SELECT COUNT(*) FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='')`).Scan(&total)
+	fmt.Printf("  unclassified rows: %d (sample locations below)\n", total)
+	for _, r := range rows {
+		loc := r.Title
+		if i := strings.Index(loc, "\t"); i >= 0 {
+			loc = loc[:i]
+		}
+		fmt.Printf("    loc=%-30q title=%-50q\n", loc, r.Title)
+	}
+}