168 lines
6.9 KiB
Bash
168 lines
6.9 KiB
Bash
#!/usr/bin/env bash
|
|
# 一键更新脚本(在线上【项目根目录】运行)。
|
|
# 作用: 用 _new/ 里的新代码覆盖改过的文件, 并把新配置项【合并】进现有 config.json
|
|
# (只加缺失的键, 绝不动你已有的密钥/路径/已调过的值), 然后自检。
|
|
# 用法: 解压更新包到项目根目录后, 执行 bash apply_update.sh
|
|
set -u
|
|
|
|
ROOT="$(pwd)"
|
|
TS="$(date +%Y%m%d_%H%M%S)"
|
|
BAK="_backup_${TS}"
|
|
NEW="_new"
|
|
|
|
echo "==> 项目目录: $ROOT"
|
|
|
|
if [ ! -d "tool" ] || [ ! -f "config/config.json" ]; then
|
|
echo "✗ 当前目录不像项目根(缺 tool/ 或 config/config.json)。请 cd 到项目根再运行。"
|
|
exit 1
|
|
fi
|
|
if [ ! -d "$NEW" ]; then
|
|
echo "✗ 找不到 $NEW/ 目录。请先把更新包解压到项目根目录。"
|
|
exit 1
|
|
fi
|
|
|
|
# 0. 更新前先把本机 数据.jsonl + 超长待重跑列表 备份到 COS(不等每天10点的自动备份)。
|
|
# 更新代码本身不会碰 数据.jsonl, 这一步纯粹是"顺手让全局数据保持最新"(比如全局去重要用),
|
|
# COS 访问失败不阻断更新, 只提示。
|
|
echo "==> 更新前备份 数据.jsonl / 超长待重跑列表 到 COS"
|
|
python3 tools/upload_daily.py || echo " ⚠ 备份失败(不影响本次更新继续), 稍后可手动重跑: python3 tools/upload_daily.py"
|
|
|
|
FILES="main.py run_worker.py tool/process.py tool/merge.py tool/getsrt.py tool/whisper_manager.py tool/VideoProcessing/multithread_algo.py tool/Ocr/tencentcloud_ocr.py tool/job_cos.py config/config_utils.py"
|
|
|
|
# 1. 备份
|
|
echo "==> 备份到 $BAK/"
|
|
mkdir -p "$BAK/tool/VideoProcessing" "$BAK/tool/Ocr" "$BAK/config"
|
|
for f in $FILES; do
|
|
[ -f "$f" ] && cp -p "$f" "$BAK/$f"
|
|
done
|
|
cp -p config/config.json "$BAK/config/config.json"
|
|
|
|
# 2. 覆盖代码文件
|
|
echo "==> 更新代码文件"
|
|
for f in $FILES; do
|
|
if [ ! -f "$NEW/$f" ]; then
|
|
echo "✗ 更新包缺少 $NEW/$f, 中止(已备份, 未改动)"; exit 1
|
|
fi
|
|
cp -p "$NEW/$f" "$f"
|
|
echo " ✓ $f"
|
|
done
|
|
|
|
# 2b. 附带的助手脚本(有就拷, 没有不报错)
|
|
for f in tools/拆分超长.py tools/seed_jobs.py tools/upload_daily.py tools/release_machine.py tools/patch_config.py; do
|
|
if [ -f "$NEW/$f" ]; then
|
|
mkdir -p "$(dirname "$f")"
|
|
cp -p "$NEW/$f" "$f" && echo " ✓ $f"
|
|
fi
|
|
done
|
|
|
|
# 3. 合并配置(只加缺失键, 不覆盖已有值; 报告与推荐值不同的键)
|
|
echo "==> 合并 config.json 新配置项(保留你已有的密钥/路径/设置)"
|
|
python3 - "$ROOT/config/config.json" <<'PYEOF'
|
|
import json, sys
|
|
p = sys.argv[1]
|
|
d = json.load(open(p, encoding="utf-8"))
|
|
au = d.setdefault("audio", {})
|
|
sf = d.setdefault("splitframe", {})
|
|
sub = d.setdefault("subtitle", {})
|
|
ocr = d.setdefault("tencentcloud", {}).setdefault("ocr", {})
|
|
dp = d.setdefault("dispatch", {})
|
|
_bkt = (d.get("tencentcloud", {}).get("cos", {}) or {}).get("bucket", "asr-dig-data-1253181912")
|
|
|
|
NEW_KEYS = [
|
|
(ocr, "max_attempts", 4),
|
|
(ocr, "timeout", 30),
|
|
(ocr, "dispatch_log_every", 1000),
|
|
(au, "cpu_threads", 2),
|
|
(sf, "crop_bottom_ratio", None),
|
|
(sub, "min_duration_seconds", 300),
|
|
(sub, "no_composite", False),
|
|
(sub, "merge_workers", 8),
|
|
(sub, "max_tiles_per_composite", 24),
|
|
(sub, "ocr_max_image_side", 4000),
|
|
(sub, "cv_detect", True),
|
|
(sub, "cv_use_region", False),
|
|
(sub, "filter_static", True),
|
|
(sub, "static_frac", 0.25),
|
|
(sub, "filter_lowconf", True),
|
|
(sub, "lowconf_thr", 80),
|
|
(sub, "lowconf_filter_frac", 0.6),
|
|
(sub, "lowconf_min_det", 200),
|
|
(sub, "strip_static_prefix", False),
|
|
(sub, "filter_watermark_pos", True),
|
|
(sub, "wm_dominant_frac", 0.25),
|
|
(sub, "wm_min_count", 20),
|
|
(sub, "split_far_boxes", True),
|
|
(sub, "max_duration_seconds", 5400),
|
|
(sub, "long_video_list", "./超长待重跑.txt"),
|
|
(sub, "ocr_no_drop", True),
|
|
(sub, "ocr_max_passes", 5),
|
|
(sub, "ocr_retry_sleep", 2),
|
|
(sub, "ocr_fail_list", "./OCR失败待重跑.txt"),
|
|
(sub, "no_subtitle_reasons", ["字幕太少","冗余文字过多","冗余文字太多","无字幕区域","非字幕内容","置信度太低","未检测","特殊字体","视频过短","视频过长"]),
|
|
(sf, "ffmpeg_timeout", 3600),
|
|
(sf, "ffmpeg_threads", "auto"),
|
|
(sf, "gpu_skip_indices", []),
|
|
(sf, "gpu_auto_select", True),
|
|
(sf, "gpu_busy_mem_mb", 4096),
|
|
(dp, "enabled", True),
|
|
(dp, "source", "cos"),
|
|
(dp, "run_long_video", False),
|
|
(dp, "auto_fallback_long_video", True),
|
|
(dp, "global_dedup_on_startup", True),
|
|
(dp, "machine_id", ""),
|
|
(dp, "jobs_root", "cos://%s/_jobs" % _bkt),
|
|
(dp, "shard_size", 2000),
|
|
(dp, "keep_alive", True),
|
|
(dp, "idle_poll_seconds", 60),
|
|
(dp, "claim_shuffle", True),
|
|
(dp, "claim_match", ""),
|
|
(dp, "heartbeat_seconds", 300),
|
|
(dp, "daily_backup_path", "cos://%s/_jobs/daily" % _bkt),
|
|
(dp, "long_list_backup_path", "cos://%s/_jobs/long_video" % _bkt),
|
|
(dp, "monitor_root", "cos://%s/_jobs/monitor" % _bkt),
|
|
(dp, "long_list_clear_after_upload", True),
|
|
(dp, "long_list_min_batch", 1000),
|
|
(dp, "daily_backup_hour", 10),
|
|
(dp, "stop_flag_key", "_jobs/STOP"),
|
|
]
|
|
added, differ = [], []
|
|
for sec, k, v in NEW_KEYS:
|
|
if k not in sec:
|
|
sec[k] = v; added.append((k, v))
|
|
elif sec[k] != v:
|
|
differ.append((k, sec[k], v))
|
|
|
|
json.dump(d, open(p, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
|
print(" 新增键:", ", ".join(f"{k}={v}" for k, v in added) or "(无, 已都存在)")
|
|
if differ:
|
|
print(" ⚠ 以下键你线上已有值, 未改动(如想用推荐值请手动改):")
|
|
for k, cur, rec in differ:
|
|
print(f" {k}: 当前={cur} 推荐={rec}")
|
|
PYEOF
|
|
if [ $? -ne 0 ]; then
|
|
echo "✗ 配置合并失败, 正在回滚 config.json"
|
|
cp -p "$BAK/config/config.json" config/config.json
|
|
exit 1
|
|
fi
|
|
|
|
# 4. 自检
|
|
echo "==> 自检 import"
|
|
if python3 -c "import main, run_worker, tool.job_cos, tool.merge, tool.process, tool.getsrt, tool.whisper_manager, tool.VideoProcessing.multithread_algo; print(' ✓ import OK')"; then
|
|
:
|
|
else
|
|
echo "✗ import 失败! 正在回滚所有改动"
|
|
for f in $FILES; do cp -p "$BAK/$f" "$f"; done
|
|
cp -p "$BAK/config/config.json" config/config.json
|
|
echo "已回滚到更新前。备份在 $BAK/"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "============================================="
|
|
echo "✓ 更新完成。备份在: $BAK/"
|
|
echo " 回退: 把 $BAK/ 里的文件拷回原位即可。"
|
|
echo ""
|
|
echo "下一步: 先小批量验证, 没问题再上量。"
|
|
echo " python main.py --source-list ./text2 --max-workers 32 --executor process"
|
|
echo "============================================="
|