#!/usr/bin/env bash # 一键更新脚本(在线上【项目根目录】运行)。 # 作用: 用 _new/ 里的新代码覆盖改过的文件, 并把新配置项【合并】进现有 config.json # (只加缺失的键, 绝不动你已有的密钥/路径/已调过的值), 然后自检。 # 用法: 解压更新包到项目根目录后, 执行 bash apply_update.sh set -u ROOT="$(pwd)" TS="$(date +%Y%m%d_%H%M%S)" BAK="_backup_${TS}" NEW="_new" echo "==> 项目目录: $ROOT" if [ ! -d "tool" ] || [ ! -f "config/config.json" ]; then echo "✗ 当前目录不像项目根(缺 tool/ 或 config/config.json)。请 cd 到项目根再运行。" exit 1 fi if [ ! -d "$NEW" ]; then echo "✗ 找不到 $NEW/ 目录。请先把更新包解压到项目根目录。" exit 1 fi # 0. 更新前先把本机 数据.jsonl + 超长待重跑列表 备份到 COS(不等每天10点的自动备份)。 # 更新代码本身不会碰 数据.jsonl, 这一步纯粹是"顺手让全局数据保持最新"(比如全局去重要用), # COS 访问失败不阻断更新, 只提示。 echo "==> 更新前备份 数据.jsonl / 超长待重跑列表 到 COS" python3 tools/upload_daily.py || echo " ⚠ 备份失败(不影响本次更新继续), 稍后可手动重跑: python3 tools/upload_daily.py" FILES="main.py run_worker.py tool/process.py tool/merge.py tool/getsrt.py tool/whisper_manager.py tool/VideoProcessing/multithread_algo.py tool/Ocr/tencentcloud_ocr.py tool/job_cos.py config/config_utils.py" # 1. 备份 echo "==> 备份到 $BAK/" mkdir -p "$BAK/tool/VideoProcessing" "$BAK/tool/Ocr" "$BAK/config" for f in $FILES; do [ -f "$f" ] && cp -p "$f" "$BAK/$f" done cp -p config/config.json "$BAK/config/config.json" # 2. 覆盖代码文件 echo "==> 更新代码文件" for f in $FILES; do if [ ! -f "$NEW/$f" ]; then echo "✗ 更新包缺少 $NEW/$f, 中止(已备份, 未改动)"; exit 1 fi cp -p "$NEW/$f" "$f" echo " ✓ $f" done # 2b. 附带的助手脚本(有就拷, 没有不报错) for f in tools/拆分超长.py tools/seed_jobs.py tools/upload_daily.py tools/release_machine.py tools/patch_config.py; do if [ -f "$NEW/$f" ]; then mkdir -p "$(dirname "$f")" cp -p "$NEW/$f" "$f" && echo " ✓ $f" fi done # 3. 合并配置(只加缺失键, 不覆盖已有值; 报告与推荐值不同的键) echo "==> 合并 config.json 新配置项(保留你已有的密钥/路径/设置)" python3 - "$ROOT/config/config.json" <<'PYEOF' import json, sys p = sys.argv[1] d = json.load(open(p, encoding="utf-8")) au = d.setdefault("audio", {}) sf = d.setdefault("splitframe", {}) sub = d.setdefault("subtitle", {}) ocr = d.setdefault("tencentcloud", {}).setdefault("ocr", {}) dp = d.setdefault("dispatch", {}) _bkt = (d.get("tencentcloud", {}).get("cos", {}) or {}).get("bucket", "asr-dig-data-1253181912") NEW_KEYS = [ (ocr, "max_attempts", 4), (ocr, "timeout", 30), (ocr, "dispatch_log_every", 1000), (au, "cpu_threads", 2), (sf, "crop_bottom_ratio", None), (sub, "min_duration_seconds", 300), (sub, "no_composite", False), (sub, "merge_workers", 8), (sub, "max_tiles_per_composite", 24), (sub, "ocr_max_image_side", 4000), (sub, "cv_detect", True), (sub, "cv_use_region", False), (sub, "filter_static", True), (sub, "static_frac", 0.25), (sub, "filter_lowconf", True), (sub, "lowconf_thr", 80), (sub, "lowconf_filter_frac", 0.6), (sub, "lowconf_min_det", 200), (sub, "strip_static_prefix", False), (sub, "filter_watermark_pos", True), (sub, "wm_dominant_frac", 0.25), (sub, "wm_min_count", 20), (sub, "split_far_boxes", True), (sub, "max_duration_seconds", 5400), (sub, "long_video_list", "./超长待重跑.txt"), (sub, "ocr_no_drop", True), (sub, "ocr_max_passes", 5), (sub, "ocr_retry_sleep", 2), (sub, "ocr_fail_list", "./OCR失败待重跑.txt"), (sub, "no_subtitle_reasons", ["字幕太少","冗余文字过多","冗余文字太多","无字幕区域","非字幕内容","置信度太低","未检测","特殊字体","视频过短","视频过长"]), (sf, "ffmpeg_timeout", 3600), (sf, "ffmpeg_threads", "auto"), (sf, "gpu_skip_indices", []), (sf, "gpu_auto_select", True), (sf, "gpu_busy_mem_mb", 4096), (dp, "enabled", True), (dp, "source", "cos"), (dp, "run_long_video", False), (dp, "auto_fallback_long_video", True), (dp, "global_dedup_on_startup", True), (dp, "machine_id", ""), (dp, "jobs_root", "cos://%s/_jobs" % _bkt), (dp, "shard_size", 2000), (dp, "keep_alive", True), (dp, "idle_poll_seconds", 60), (dp, "claim_shuffle", True), (dp, "claim_match", ""), (dp, "heartbeat_seconds", 300), (dp, "daily_backup_path", "cos://%s/_jobs/daily" % _bkt), (dp, "long_list_backup_path", "cos://%s/_jobs/long_video" % _bkt), (dp, "monitor_root", "cos://%s/_jobs/monitor" % _bkt), (dp, "long_list_clear_after_upload", True), (dp, "long_list_min_batch", 1000), (dp, "daily_backup_hour", 10), (dp, "stop_flag_key", "_jobs/STOP"), ] added, differ = [], [] for sec, k, v in NEW_KEYS: if k not in sec: sec[k] = v; added.append((k, v)) elif sec[k] != v: differ.append((k, sec[k], v)) json.dump(d, open(p, "w", encoding="utf-8"), ensure_ascii=False, indent=2) print(" 新增键:", ", ".join(f"{k}={v}" for k, v in added) or "(无, 已都存在)") if differ: print(" ⚠ 以下键你线上已有值, 未改动(如想用推荐值请手动改):") for k, cur, rec in differ: print(f" {k}: 当前={cur} 推荐={rec}") PYEOF if [ $? -ne 0 ]; then echo "✗ 配置合并失败, 正在回滚 config.json" cp -p "$BAK/config/config.json" config/config.json exit 1 fi # 4. 自检 echo "==> 自检 import" if python3 -c "import main, run_worker, tool.job_cos, tool.merge, tool.process, tool.getsrt, tool.whisper_manager, tool.VideoProcessing.multithread_algo; print(' ✓ import OK')"; then : else echo "✗ import 失败! 正在回滚所有改动" for f in $FILES; do cp -p "$BAK/$f" "$f"; done cp -p "$BAK/config/config.json" config/config.json echo "已回滚到更新前。备份在 $BAK/" exit 1 fi echo "" echo "=============================================" echo "✓ 更新完成。备份在: $BAK/" echo " 回退: 把 $BAK/ 里的文件拷回原位即可。" echo "" echo "下一步: 先小批量验证, 没问题再上量。" echo " python main.py --source-list ./text2 --max-workers 32 --executor process" echo "============================================="