143 lines
6.7 KiB
Python
143 lines
6.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""在【项目根目录】运行: 把本次更新需要的 config.json 配置键就地改好。
|
|
只动下面列出的键, 绝不碰你的密钥/COS路径/其它已调过的值。幂等, 可重复运行。
|
|
|
|
用法: python3 tools/patch_config.py
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
P = "config/config.json"
|
|
if not os.path.exists(P):
|
|
print("✗ 找不到 config/config.json, 请 cd 到项目根目录再运行")
|
|
sys.exit(1)
|
|
|
|
d = json.load(open(P, encoding="utf-8"))
|
|
sf = d.setdefault("splitframe", {})
|
|
sub = d.setdefault("subtitle", {})
|
|
ocr = d.setdefault("tencentcloud", {}).setdefault("ocr", {})
|
|
dp = d.setdefault("dispatch", {})
|
|
|
|
changes = []
|
|
|
|
# -1) COS 任务分发(常驻 worker 认领任务 + 每日 数据.jsonl 备份)。缺失才加, 不动已有值。
|
|
# jobs_root=取任务地址; daily_backup_path=数据.jsonl 备份位置; machine_id 留空=用主机名/环境变量。
|
|
_bkt = (d.get("tencentcloud", {}).get("cos", {}) or {}).get("bucket", "asr-dig-data-1253181912")
|
|
_dispatch_defaults = [
|
|
("enabled", True),
|
|
("source", "cos"), # cos=从COS认领 / local=只读本地 ./text2(启动脚本据此选模式)
|
|
("run_long_video", False), # true=本机专跑超长队列(long_list_backup_path)
|
|
("auto_fallback_long_video", True), # false=主队列空闲时也不兼跑长视频队列,只等主队列
|
|
("global_dedup_on_startup", True), # false=启动时不拉全队去重清单,只按本机 数据.jsonl 去重
|
|
("machine_id", ""),
|
|
("jobs_root", "cos://%s/_jobs" % _bkt),
|
|
("shard_size", 2000),
|
|
("keep_alive", True),
|
|
("idle_poll_seconds", 60),
|
|
("claim_shuffle", True),
|
|
("claim_match", ""),
|
|
("heartbeat_seconds", 300),
|
|
("daily_backup_path", "cos://%s/_jobs/daily" % _bkt),
|
|
("long_list_backup_path", "cos://%s/_jobs/long_video" % _bkt),
|
|
("monitor_root", "cos://%s/_jobs/monitor" % _bkt),
|
|
("long_list_clear_after_upload", True),
|
|
("long_list_min_batch", 1000), # 领取长视频队列时凑够这么多条任务才开始处理一批
|
|
("daily_backup_hour", 10),
|
|
("stop_flag_key", "_jobs/STOP"),
|
|
]
|
|
for k, v in _dispatch_defaults:
|
|
if k not in dp:
|
|
dp[k] = v
|
|
changes.append("dispatch.%s -> %s (新增, COS任务分发)" % (k, v))
|
|
|
|
# 0) OCR 重试封顶 + 单次超时 (防"端点全超时时一张图卡~1小时"拖死worker)
|
|
if "max_attempts" not in ocr:
|
|
ocr["max_attempts"] = 4
|
|
changes.append("ocr.max_attempts -> 4 (新增, 重试封顶)")
|
|
# 单次OCR超时: 调到30s(大图负载下最慢~12s, 12s砍太狠; 有max_attempts封顶不会卡死)
|
|
if ocr.get("timeout", 0) < 30:
|
|
_old = ocr.get("timeout")
|
|
ocr["timeout"] = 30
|
|
changes.append("ocr.timeout %s -> 30 (大图留足余量)" % _old)
|
|
if "dispatch_log_every" not in ocr:
|
|
ocr["dispatch_log_every"] = 1000
|
|
changes.append("ocr.dispatch_log_every -> 1000 (新增, 每1000次派发落盘计数, 0=关)")
|
|
|
|
# 0b) 处理完删源视频(本来就在COS上, 本地是冗余副本)。防 ./video/ 堆积撑爆磁盘。
|
|
if sf.get("delete_video") is not True:
|
|
sf["delete_video"] = True
|
|
changes.append("delete_video -> True (处理完删源视频, 防磁盘爆)")
|
|
|
|
# 1) 抽帧 ffmpeg 线程数: 统一收敛到 "auto"(按 CPU核数÷并发进程数 自动摊分, GPU/CPU两条
|
|
# 抽帧路径现在都吃这个值了, 比固定数字更适合核数不一的机队)。已经是 "auto" 的跳过；
|
|
# 只升级明显偏低的字面值(<5)或完全没配的情况, 已手动调更高的字面值保留不动。
|
|
old = sf.get("ffmpeg_threads", 2)
|
|
_old_is_auto = isinstance(old, str) and old.strip().lower() == "auto"
|
|
if not _old_is_auto and (not isinstance(old, (int, float)) or old < 5):
|
|
sf["ffmpeg_threads"] = "auto"
|
|
changes.append(f"ffmpeg_threads {old!r} -> auto")
|
|
|
|
# 2) 超长视频补跑列表 (缺失才加)
|
|
if "long_video_list" not in sub:
|
|
sub["long_video_list"] = "./超长待重跑.txt"
|
|
changes.append("long_video_list -> ./超长待重跑.txt (新增)")
|
|
|
|
# 2b) 降低 OCR 拼接帧数 (大图OCR>12s会超时; 只往下调, 已更小的保留)
|
|
_mt = sub.get("max_tiles_per_composite", 40)
|
|
if _mt > 24:
|
|
sub["max_tiles_per_composite"] = 24
|
|
changes.append("max_tiles_per_composite %d -> 24 (拼接图减小, 防OCR超时)" % _mt)
|
|
|
|
# 2b2) 降全局OCR并发, 削掉过载突发尾延迟(端点平均快但突发会排队爆30s); 只往下调
|
|
_mg = sub.get("max_global_ocr", 80)
|
|
if _mg > 50:
|
|
sub["max_global_ocr"] = 50
|
|
changes.append("max_global_ocr %d -> 50 (降负载削尾延迟)" % _mg)
|
|
|
|
# 2b3) OCR图片像素上限: 超大合成图(区域过高)缩小再OCR, 防单张>30s超时
|
|
if "ocr_max_image_side" not in sub:
|
|
sub["ocr_max_image_side"] = 4000
|
|
changes.append("ocr_max_image_side -> 4000 (新增, 超大图缩小防超时)")
|
|
|
|
# 2c) 不丢帧: OCR多轮重试, 仍失败则整段视频判失败去重跑(不产出带洞SRT)
|
|
for k, v in [("ocr_no_drop", True), ("ocr_max_passes", 5),
|
|
("ocr_retry_sleep", 2), ("ocr_fail_list", "./OCR失败待重跑.txt")]:
|
|
if k not in sub:
|
|
sub[k] = v
|
|
changes.append("%s -> %s (新增, 不丢帧)" % (k, v))
|
|
|
|
# 2d) 低置信占比过滤: 幻灯片/编程视频OCR出垃圾(低置信50%+)且吃3-5倍OCR, 整段判非字幕跳过
|
|
for k, v in [("filter_lowconf", True), ("lowconf_thr", 80),
|
|
("lowconf_filter_frac", 0.6), ("lowconf_min_det", 200)]:
|
|
if k not in sub:
|
|
sub[k] = v
|
|
changes.append("%s -> %s (新增, 过滤幻灯片/代码类非字幕视频)" % (k, v))
|
|
|
|
# 2e) 状态归类清单: 这些原因是"无可用字幕的正常过滤", 应记 no_subtitle 而非 error。
|
|
# 旧逻辑把字幕太少/冗余/置信度/非字幕等错记成 error(线上 error 中~79%其实是它们)。缺失才加。
|
|
if "no_subtitle_reasons" not in sub:
|
|
sub["no_subtitle_reasons"] = [
|
|
"字幕太少", "冗余文字过多", "冗余文字太多", "无字幕区域",
|
|
"非字幕内容", "置信度太低", "未检测", "特殊字体",
|
|
"视频过短", "视频过长",
|
|
]
|
|
changes.append("no_subtitle_reasons -> [10项] (新增, 过滤型结论归 no_subtitle 不计 error)")
|
|
|
|
# 3) 抽帧自动选空闲GPU (避开被OCR服务占用的卡, 缺失才加)
|
|
if "gpu_auto_select" not in sf:
|
|
sf["gpu_auto_select"] = True
|
|
changes.append("gpu_auto_select -> True (新增, 抽帧自动避开被占用的GPU)")
|
|
if "gpu_busy_mem_mb" not in sf:
|
|
sf["gpu_busy_mem_mb"] = 4096
|
|
changes.append("gpu_busy_mem_mb -> 4096 (新增, 显存占用超此判为忙)")
|
|
sf.setdefault("gpu_skip_indices", []) # 手动跳过列表(默认空, 交给自动探测)
|
|
|
|
if changes:
|
|
json.dump(d, open(P, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
|
print("✓ config 已更新:")
|
|
for c in changes:
|
|
print(" -", c)
|
|
else:
|
|
print("✓ config 无需改动(已是最新)")
|