backend_v2/scripts/backfill_city/main.go

// Command backfill_city 给 street_snap 加 city 列并从 title（首选）/ source_url（兜底）回填。
//
// 项目已移除 AutoMigrate，表结构由外部 SQL 托管（见 scripts/sql/004_add_streetsnap_city.sql）。
// 本程序是「没有 mysql 客户端时」的执行器：
//   - ALTER 走 ensureColumn：先查 information_schema，缺才加，可安全重跑（幂等）。
//   - 回填：优先从 title 第一段（location，\t 分隔）按城市关键词匹配；
//     命中不到再退到 source_url / cover 的 URL slug 解析。幂等（已填过的不会覆盖）。
//
// 存储的 city 值即「规范英文名首字母大写」（Paris / New York …），
// 与前端街拍列表 City 筛选项 value、header 下拉 ?city= 参数完全一致（精确匹配）。
package main

import (
	"fmt"
	"log"
	"strings"

	"gorm.io/driver/mysql"
	"gorm.io/gorm"
)

const dsn = "root:root@tcp(127.0.0.1:3306)/db_dev?charset=utf8mb4&parseTime=True&loc=Local"

// cityKeywords 城市关键词 → 规范存储值（首字母大写）。
// 顺序无关；extractCityFromTitle 用「包含」匹配，可容忍 "Paris Fashion Week" / "New York FW25" 等形式。
var cityKeywords = []struct {
	kw   string
	city string
}{
	{"paris", "Paris"},
	{"milan", "Milan"},
	{"london", "London"},
	{"new york", "New York"},
	{"nyfw", "New York"},
	{"tokyo", "Tokyo"},
	{"seoul", "Seoul"},
	{"florence", "Florence"},
	{"shanghai", "Shanghai"},
	{"copenhagen", "Copenhagen"},
	{"berlin", "Berlin"},
	{"rome", "Rome"},
	{"madrid", "Madrid"},
	{"amsterdam", "Amsterdam"},
	{"antwerp", "Antwerp"},
	{"los angeles", "Los Angeles"},
	{"moscow", "Moscow"},
	{"mumbai", "Mumbai"},
	{"beijing", "Beijing"},
}

// citySlug（URL 兜底用）：slug → 规范存储值。多词城市（new-york）必须排在单词组之前被 HasPrefix 命中。
var citySlug = map[string]string{
	"paris":       "Paris",
	"milan":       "Milan",
	"london":      "London",
	"new-york":    "New York",
	"tokyo":       "Tokyo",
	"seoul":       "Seoul",
	"copenhagen":  "Copenhagen",
	"florence":    "Florence",
	"berlin":      "Berlin",
	"shanghai":    "Shanghai",
	"beijing":     "Beijing",
	"moscow":      "Moscow",
	"los-angeles": "Los Angeles",
	"rome":        "Rome",
	"madrid":      "Madrid",
	"amsterdam":   "Amsterdam",
	"antwerp":     "Antwerp",
	"mumbai":      "Mumbai",
}

// extractCityFromTitle 从 title 第一段（location，\t 分隔）按关键词匹配规范城市名。
func extractCityFromTitle(title string) string {
	seg := title
	if i := strings.Index(title, "\t"); i >= 0 {
		seg = title[:i]
	}
	segL := strings.ToLower(strings.TrimSpace(seg))
	if segL == "" {
		return ""
	}
	for _, c := range cityKeywords {
		if strings.Contains(segL, c.kw) {
			return c.city
		}
	}
	return ""
}

// extractCityFromURL 兜底：从 source_url / cover 的 URL 第一段 slug 解析城市。
func extractCityFromURL(raws ...string) string {
	for _, raw := range raws {
		u := strings.TrimSpace(raw)
		if u == "" {
			continue
		}
		if i := strings.Index(u, "://"); i >= 0 {
			u = u[i+3:]
		}
		if i := strings.Index(u, "/"); i >= 0 {
			u = u[i+1:]
		}
		u = strings.Trim(u, "/")
		if u == "" {
			continue
		}
		seg := strings.SplitN(u, "/", 2)[0]
		segL := strings.ToLower(seg)
		for slug, city := range citySlug {
			if segL == slug || strings.HasPrefix(segL, slug+"-") {
				return city
			}
		}
	}
	return ""
}

func main() {
	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
	if err != nil {
		log.Fatal("open:", err)
	}

	// 1) 幂等加列（与 004_add_streetsnap_city.sql 对齐）
	ensureColumn(db, "street_snap", "city", "VARCHAR(128) NOT NULL DEFAULT '' AFTER year")

	// 2) 逐行回填：优先 title 第一段，兜底 URL slug
	type row struct {
		ID        uint32
		Title     string
		SourceURL string
		Cover     string
	}
	var rows []row
	if r := db.Raw("SELECT id, title, source_url, cover FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='')").Scan(&rows); r.Error != nil {
		log.Fatal("select:", r.Error)
	}
	var updated int64
	for _, r := range rows {
		city := extractCityFromTitle(r.Title)
		if city == "" {
			city = extractCityFromURL(r.SourceURL, r.Cover)
		}
		if city == "" {
			continue
		}
		if r2 := db.Exec("UPDATE street_snap SET city=? WHERE id=?", city, r.ID); r2.Error != nil {
			log.Printf("update id=%d failed: %v", r.ID, r2.Error)
			continue
		}
		updated++
	}
	fmt.Printf("  updated city for %d rows\n", updated)

	// 3) 校验：输出 distinct city 分布 + 未归类样本
	printCities(db)
	printUnclassifiedSamples(db)
	fmt.Println("backfill city done.")
}

func ensureColumn(db *gorm.DB, table, col, def string) {
	var n int64
	db.Raw("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA='db_dev' AND TABLE_NAME=? AND COLUMN_NAME=?", table, col).Scan(&n)
	if n > 0 {
		fmt.Printf("  skip %s.%s (exists)\n", table, col)
		return
	}
	stmt := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, col, def)
	fmt.Println("  +", stmt)
	if r := db.Exec(stmt); r.Error != nil {
		log.Fatalf("alter %s.%s failed: %v", table, col, r.Error)
	}
}

func printCities(db *gorm.DB) {
	type row struct {
		City string
		Cnt  int64
	}
	var rows []row
	db.Raw(`SELECT city, COUNT(*) cnt FROM street_snap WHERE is_deleted=0 AND city<>'' GROUP BY city ORDER BY cnt DESC`).Scan(&rows)
	fmt.Printf("  distinct cities: %d\n", len(rows))
	for _, r := range rows {
		fmt.Printf("    %-14q %d\n", r.City, r.Cnt)
	}
}

// printUnclassifiedSamples 输出仍未归类行的 title 第一段样本，便于后续补充城市词典。
func printUnclassifiedSamples(db *gorm.DB) {
	type row struct {
		Title string
	}
	var rows []row
	db.Raw(`SELECT title FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='') LIMIT 25`).Scan(&rows)
	var total int64
	db.Raw(`SELECT COUNT(*) FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='')`).Scan(&total)
	fmt.Printf("  unclassified rows: %d (sample locations below)\n", total)
	for _, r := range rows {
		loc := r.Title
		if i := strings.Index(loc, "\t"); i >= 0 {
			loc = loc[:i]
		}
		fmt.Printf("    loc=%-30q title=%-50q\n", loc, r.Title)
	}
}