// Command backfill_city 给 street_snap 加 city 列并从 title(首选)/ source_url(兜底)回填。 // // 项目已移除 AutoMigrate,表结构由外部 SQL 托管(见 scripts/sql/004_add_streetsnap_city.sql)。 // 本程序是「没有 mysql 客户端时」的执行器: // - ALTER 走 ensureColumn:先查 information_schema,缺才加,可安全重跑(幂等)。 // - 回填:优先从 title 第一段(location,\t 分隔)按城市关键词匹配; // 命中不到再退到 source_url / cover 的 URL slug 解析。幂等(已填过的不会覆盖)。 // // 存储的 city 值即「规范英文名首字母大写」(Paris / New York …), // 与前端街拍列表 City 筛选项 value、header 下拉 ?city= 参数完全一致(精确匹配)。 package main import ( "fmt" "log" "strings" "gorm.io/driver/mysql" "gorm.io/gorm" ) const dsn = "root:root@tcp(127.0.0.1:3306)/db_dev?charset=utf8mb4&parseTime=True&loc=Local" // cityKeywords 城市关键词 → 规范存储值(首字母大写)。 // 顺序无关;extractCityFromTitle 用「包含」匹配,可容忍 "Paris Fashion Week" / "New York FW25" 等形式。 var cityKeywords = []struct { kw string city string }{ {"paris", "Paris"}, {"milan", "Milan"}, {"london", "London"}, {"new york", "New York"}, {"nyfw", "New York"}, {"tokyo", "Tokyo"}, {"seoul", "Seoul"}, {"florence", "Florence"}, {"shanghai", "Shanghai"}, {"copenhagen", "Copenhagen"}, {"berlin", "Berlin"}, {"rome", "Rome"}, {"madrid", "Madrid"}, {"amsterdam", "Amsterdam"}, {"antwerp", "Antwerp"}, {"los angeles", "Los Angeles"}, {"moscow", "Moscow"}, {"mumbai", "Mumbai"}, {"beijing", "Beijing"}, } // citySlug(URL 兜底用):slug → 规范存储值。多词城市(new-york)必须排在单词组之前被 HasPrefix 命中。 var citySlug = map[string]string{ "paris": "Paris", "milan": "Milan", "london": "London", "new-york": "New York", "tokyo": "Tokyo", "seoul": "Seoul", "copenhagen": "Copenhagen", "florence": "Florence", "berlin": "Berlin", "shanghai": "Shanghai", "beijing": "Beijing", "moscow": "Moscow", "los-angeles": "Los Angeles", "rome": "Rome", "madrid": "Madrid", "amsterdam": "Amsterdam", "antwerp": "Antwerp", "mumbai": "Mumbai", } // extractCityFromTitle 从 title 第一段(location,\t 分隔)按关键词匹配规范城市名。 func extractCityFromTitle(title string) string { seg := title if i := strings.Index(title, "\t"); i >= 0 { seg = title[:i] } segL := strings.ToLower(strings.TrimSpace(seg)) if segL == "" { return "" } for _, c := range cityKeywords { if strings.Contains(segL, c.kw) { return c.city } } return "" } // extractCityFromURL 兜底:从 source_url / cover 的 URL 第一段 slug 解析城市。 func extractCityFromURL(raws ...string) string { for _, raw := range raws { u := strings.TrimSpace(raw) if u == "" { continue } if i := strings.Index(u, "://"); i >= 0 { u = u[i+3:] } if i := strings.Index(u, "/"); i >= 0 { u = u[i+1:] } u = strings.Trim(u, "/") if u == "" { continue } seg := strings.SplitN(u, "/", 2)[0] segL := strings.ToLower(seg) for slug, city := range citySlug { if segL == slug || strings.HasPrefix(segL, slug+"-") { return city } } } return "" } func main() { db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) if err != nil { log.Fatal("open:", err) } // 1) 幂等加列(与 004_add_streetsnap_city.sql 对齐) ensureColumn(db, "street_snap", "city", "VARCHAR(128) NOT NULL DEFAULT '' AFTER year") // 2) 逐行回填:优先 title 第一段,兜底 URL slug type row struct { ID uint32 Title string SourceURL string Cover string } var rows []row if r := db.Raw("SELECT id, title, source_url, cover FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='')").Scan(&rows); r.Error != nil { log.Fatal("select:", r.Error) } var updated int64 for _, r := range rows { city := extractCityFromTitle(r.Title) if city == "" { city = extractCityFromURL(r.SourceURL, r.Cover) } if city == "" { continue } if r2 := db.Exec("UPDATE street_snap SET city=? WHERE id=?", city, r.ID); r2.Error != nil { log.Printf("update id=%d failed: %v", r.ID, r2.Error) continue } updated++ } fmt.Printf(" updated city for %d rows\n", updated) // 3) 校验:输出 distinct city 分布 + 未归类样本 printCities(db) printUnclassifiedSamples(db) fmt.Println("backfill city done.") } func ensureColumn(db *gorm.DB, table, col, def string) { var n int64 db.Raw("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA='db_dev' AND TABLE_NAME=? AND COLUMN_NAME=?", table, col).Scan(&n) if n > 0 { fmt.Printf(" skip %s.%s (exists)\n", table, col) return } stmt := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, col, def) fmt.Println(" +", stmt) if r := db.Exec(stmt); r.Error != nil { log.Fatalf("alter %s.%s failed: %v", table, col, r.Error) } } func printCities(db *gorm.DB) { type row struct { City string Cnt int64 } var rows []row db.Raw(`SELECT city, COUNT(*) cnt FROM street_snap WHERE is_deleted=0 AND city<>'' GROUP BY city ORDER BY cnt DESC`).Scan(&rows) fmt.Printf(" distinct cities: %d\n", len(rows)) for _, r := range rows { fmt.Printf(" %-14q %d\n", r.City, r.Cnt) } } // printUnclassifiedSamples 输出仍未归类行的 title 第一段样本,便于后续补充城市词典。 func printUnclassifiedSamples(db *gorm.DB) { type row struct { Title string } var rows []row db.Raw(`SELECT title FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='') LIMIT 25`).Scan(&rows) var total int64 db.Raw(`SELECT COUNT(*) FROM street_snap WHERE is_deleted=0 AND (city IS NULL OR city='')`).Scan(&total) fmt.Printf(" unclassified rows: %d (sample locations below)\n", total) for _, r := range rows { loc := r.Title if i := strings.Index(loc, "\t"); i >= 0 { loc = loc[:i] } fmt.Printf(" loc=%-30q title=%-50q\n", loc, r.Title) } }