update
This commit is contained in:
@ -38,6 +38,7 @@ type IngestService struct {
|
||||
del storage.Deleter // 删除器(S4或本地,media_cleanup 真删用)
|
||||
local *storage.LocalUploader // S4失败时的兜底落地
|
||||
httpClient *http.Client
|
||||
translate *TranslateService // 标题中文翻译(可为 nil:测试/未启用时跳过自动补全)
|
||||
}
|
||||
|
||||
// NewIngestService 创建入库服务。
|
||||
@ -45,7 +46,7 @@ type IngestService struct {
|
||||
// uploader: 主上传器(S4或本地)
|
||||
// del: 删除器(S4或本地),用于 media_cleanup 真删S4/本地孤儿文件
|
||||
// local: 本地兜底上传器(S4上传失败时回退,避免图片完全丢失)
|
||||
func NewIngestService(repo repository.IngestRepository, brandRepo repository.BrandRepository, media repository.MediaRepository, uploader storage.Uploader, del storage.Deleter, local *storage.LocalUploader, downloadProxy string) *IngestService {
|
||||
func NewIngestService(repo repository.IngestRepository, brandRepo repository.BrandRepository, media repository.MediaRepository, uploader storage.Uploader, del storage.Deleter, local *storage.LocalUploader, downloadProxy string, translateSvc *TranslateService) *IngestService {
|
||||
// 下载代理:download_proxy(或 INGEST_DOWNLOAD_PROXY)非空时,下载客户端经该代理访问源站;
|
||||
// 仅作用于图片下载,S4 上传走独立 s3 客户端、pgx 数据库连接不走 http,均不受影响。
|
||||
// 留空回退 http.ProxyFromEnvironment(默认直连,仍可被系统 HTTP(S)_PROXY 影响)。
|
||||
@ -74,6 +75,7 @@ func NewIngestService(repo repository.IngestRepository, brandRepo repository.Bra
|
||||
uploader: uploader,
|
||||
del: del,
|
||||
local: local,
|
||||
translate: translateSvc,
|
||||
// 图片下载是并发执行的(见 imageFetchConcurrency),连接池须匹配并发度:
|
||||
// Go 默认 MaxIdleConnsPerHost=2,多余连接会在每轮下载时反复重建、白付 TLS 握手开销。
|
||||
httpClient: &http.Client{
|
||||
@ -372,6 +374,15 @@ func (s *IngestService) processRunway(ctx context.Context, job model.IngestJob,
|
||||
ImageCount: imageCount,
|
||||
Status: model.StatusPending,
|
||||
}
|
||||
|
||||
// 入库自动补全中文标题:spider 未给 title_cn 时,用翻译服务按「记忆优先、规则兜底」推导;
|
||||
// 已给则保留(绝不覆盖爬虫提供/已有的翻译)。翻译服务未注入(nil)时跳过。
|
||||
if rw.TitleCn == "" && s.translate != nil {
|
||||
if cn, terr := s.translate.SuggestTitleCn(ctx, rw.TitleEn); terr == nil {
|
||||
rw.TitleCn = cn
|
||||
}
|
||||
}
|
||||
|
||||
for i := range images {
|
||||
images[i].BrandID = brandID
|
||||
}
|
||||
@ -531,9 +542,19 @@ func (s *IngestService) processStreet(ctx context.Context, job model.IngestJob,
|
||||
|
||||
// 3) 直写正式表(status=pending)
|
||||
writeStart := time.Now()
|
||||
|
||||
// 街拍入库自动补全中文标题(同 runway:spider 未给才补,且翻译服务需注入)。
|
||||
titleCn := p.TitleCn
|
||||
if titleCn == "" && s.translate != nil {
|
||||
if cn, terr := s.translate.SuggestTitleCn(ctx, title); terr == nil {
|
||||
titleCn = cn
|
||||
}
|
||||
}
|
||||
|
||||
snap := &model.StreetSnap{
|
||||
JobID: job.ID,
|
||||
Title: title, // 街拍单标题,爬虫优先填 title_en(已归一化,与查重口径一致)
|
||||
TitleCn: titleCn,
|
||||
Year: p.Year,
|
||||
City: p.City,
|
||||
Cover: cover,
|
||||
|
||||
Reference in New Issue
Block a user