首次提交
This commit is contained in:
152
scripts/diag.sh
Normal file
152
scripts/diag.sh
Normal file
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env bash
|
||||
# 一键诊断: 收集机器/进程/内存/OOM/日志/数据 全部信息到一个文件, 发回给我分析。
|
||||
# 用法(在项目根目录): bash scripts/diag.sh
|
||||
# 产物: diag_YYYYmmdd_HHMMSS.txt
|
||||
|
||||
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" || exit 1
|
||||
OUT="diag_$(date +%Y%m%d_%H%M%S).txt"
|
||||
PY="$(command -v python || command -v python3)"
|
||||
|
||||
# 所有输出同时进文件
|
||||
exec > >(tee "$OUT") 2>&1
|
||||
|
||||
echo "================ 诊断报告 $(date) ================"
|
||||
echo "主机: $(hostname) 目录: $(pwd)"
|
||||
echo ""
|
||||
|
||||
echo "========== 1. 机器资源 =========="
|
||||
echo "[内存 free -g]"; free -g
|
||||
echo ""
|
||||
echo "[CPU核数] $(nproc)"
|
||||
echo "[负载 uptime] $(uptime)"
|
||||
echo ""
|
||||
|
||||
echo "========== 2. OOM 检查(夜里有没有被系统杀) =========="
|
||||
echo "[dmesg killed process]"
|
||||
( dmesg -T 2>/dev/null || dmesg 2>/dev/null ) | grep -iE "killed process|out of memory|oom-kill" | tail -20 || echo " (无 / dmesg无权限)"
|
||||
echo "[/var/log/messages oom]"
|
||||
grep -iE "killed process|out of memory|oom" /var/log/messages 2>/dev/null | tail -10 || echo " (无 / 无权限)"
|
||||
echo ""
|
||||
|
||||
echo "========== 3. 进程状态 =========="
|
||||
echo "[main.py 进程数] $(ps -eo args 2>/dev/null | grep -c '[m]ain.py')"
|
||||
echo "[ffmpeg 进程数] $(ps -eo comm 2>/dev/null | grep -c '[f]fmpeg')"
|
||||
echo "[forkserver/worker] $(ps -eo args 2>/dev/null | grep -c '[m]ultiprocessing\|[f]orkserver')"
|
||||
echo ""
|
||||
echo "[吃内存最多的10个进程 RSS]"
|
||||
ps -eo pid,rss,etime,args --sort=-rss 2>/dev/null | head -11
|
||||
echo ""
|
||||
|
||||
echo "========== 4. Python / 环境 =========="
|
||||
echo "[python] $($PY --version 2>&1) 路径: $PY"
|
||||
echo "[which python] $(command -v python 2>/dev/null)"
|
||||
echo "[自包含python在不在] $(ls python/bin/python3 2>/dev/null && echo yes || echo no)"
|
||||
echo ""
|
||||
|
||||
echo "========== 5. config 关键值 =========="
|
||||
if [ -f config/config.json ]; then
|
||||
$PY - <<'PYEOF'
|
||||
import json
|
||||
try:
|
||||
d=json.load(open("config/config.json",encoding="utf-8"))
|
||||
s=d.get("subtitle",{}); sf=d.get("splitframe",{}); au=d.get("audio",{})
|
||||
print(" ocr_workers=%s no_composite=%s merge_workers=%s max_tiles=%s"%(
|
||||
s.get("ocr_workers"),s.get("no_composite"),s.get("merge_workers"),s.get("max_tiles_per_composite")))
|
||||
print(" cv_detect=%s filter_static=%s filter_watermark_pos=%s split_far_boxes=%s"%(
|
||||
s.get("cv_detect"),s.get("filter_static"),s.get("filter_watermark_pos"),s.get("split_far_boxes")))
|
||||
print(" min_duration=%s crop_bottom_ratio=%s ffmpeg_threads=%s use_gpu=%s"%(
|
||||
s.get("min_duration_seconds"),sf.get("crop_bottom_ratio"),sf.get("ffmpeg_threads"),sf.get("use_gpu")))
|
||||
print(" audio.model_size=%s cpu_threads=%s endpoints数=%s"%(
|
||||
au.get("model_size"),au.get("cpu_threads"),len(d.get("tencentcloud",{}).get("ocr",{}).get("endpoints",[]))))
|
||||
except Exception as e:
|
||||
print(" 读config失败:",e)
|
||||
PYEOF
|
||||
else
|
||||
echo " config/config.json 不存在"
|
||||
fi
|
||||
echo "[启动命令] $(ps -eo args 2>/dev/null | grep '[m]ain.py' | head -1)"
|
||||
echo ""
|
||||
|
||||
echo "========== 6. app.log 分析 =========="
|
||||
if [ -f app.log ]; then
|
||||
echo "[总行数] $(wc -l < app.log)"
|
||||
echo "[关键词计数]"
|
||||
for kw in 成功 无字幕 失败 暂停启动 抽帧完成 拼接完成 字幕完成 内存不足 BrokenProcess 崩溃 worker崩 Traceback Error 超时 timeout 冷却 端点; do
|
||||
c=$(grep -c "$kw" app.log 2>/dev/null)
|
||||
[ "${c:-0}" -gt 0 ] && printf " %-14s %s\n" "$kw" "$c"
|
||||
done
|
||||
echo ""
|
||||
echo "[最近 40 行]"
|
||||
tail -40 app.log
|
||||
else
|
||||
echo " app.log 不存在"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "========== 7. 数据.jsonl 分析 =========="
|
||||
if [ -f 数据.jsonl ]; then
|
||||
$PY - <<'PYEOF'
|
||||
import json,statistics
|
||||
from collections import Counter,defaultdict
|
||||
from datetime import datetime,timedelta
|
||||
latest={}
|
||||
for line in open("数据.jsonl",encoding="utf-8"):
|
||||
line=line.strip()
|
||||
if not line: continue
|
||||
try: r=json.loads(line)
|
||||
except: continue
|
||||
v=r.get("video_cos_path")
|
||||
if v: latest[v]=r
|
||||
n=len(latest)
|
||||
st=Counter(r.get("status") for r in latest.values())
|
||||
print(" 去重视频 %d | 状态 %s"%(n,dict(st)))
|
||||
if n: print(" 成功率 %.1f%%"%(st.get("success",0)/n*100))
|
||||
# 阶段耗时(成功)
|
||||
tot=defaultdict(float); te=0
|
||||
for r in latest.values():
|
||||
if r.get("status")!="success": continue
|
||||
sd=r.get("stage_durations") or {}; te+=r.get("elapsed_seconds") or 0
|
||||
for k in ["split_frame","merge","getsrt"]:
|
||||
if k in sd: tot[k]+=sd[k]
|
||||
if te:
|
||||
print(" 阶段占比: 抽帧%.0f%% 拼接%.0f%% OCR+字幕%.0f%%"%(tot["split_frame"]/te*100,tot["merge"]/te*100,tot["getsrt"]/te*100))
|
||||
# OCR/帧
|
||||
rs=[ic.get("total_ocr_count")/ic.get("frame_count") for r in latest.values() if (ic:=r.get("image_counts") or {}).get("frame_count") and ic.get("total_ocr_count")]
|
||||
if rs: print(" OCR/帧 中位 %.3f"%statistics.median(rs))
|
||||
# 时长
|
||||
durs=sorted((r.get("duration") or 0) for r in latest.values() if r.get("duration"))
|
||||
if durs: print(" 视频时长 中位%.0fs max%.0fs | >20分钟 %d个"%(statistics.median(durs),max(durs),sum(1 for d in durs if d>1200)))
|
||||
# 单视频平均耗时
|
||||
els=[r.get("elapsed_seconds") or 0 for r in latest.values() if r.get("status")=="success"]
|
||||
if els: print(" 成功视频平均耗时 %.0fs 中位%.0fs"%(sum(els)/len(els),statistics.median(els)))
|
||||
# 每小时吞吐 + 有效并发(重叠法)
|
||||
busy=defaultdict(float); cnt=defaultdict(int)
|
||||
for r in latest.values():
|
||||
try: s=datetime.fromisoformat(r["process_start"]); e=datetime.fromisoformat(r["process_end"])
|
||||
except: continue
|
||||
if e<=s: continue
|
||||
cnt[s.strftime("%m-%d %H")]+=1
|
||||
cur=s
|
||||
while cur<e:
|
||||
hk=cur.strftime("%m-%d %H"); nx=cur.replace(minute=0,second=0,microsecond=0)+timedelta(hours=1)
|
||||
seg=min(e,nx); busy[hk]+=(seg-cur).total_seconds(); cur=seg
|
||||
print(" 每小时(完成数 / 有效并发):")
|
||||
for hk in sorted(busy)[-8:]:
|
||||
print(" %s : %d条 / 并发%.1f"%(hk,cnt[hk],busy[hk]/3600))
|
||||
PYEOF
|
||||
else
|
||||
echo " 数据.jsonl 不存在"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "========== 8. 磁盘 / GPU =========="
|
||||
echo "[磁盘 df -h .]"; df -h . 2>/dev/null
|
||||
echo "[split/ 大小] $(du -sh split 2>/dev/null || echo '无')"
|
||||
echo "[正在处理的视频数 split/] $(ls split 2>/dev/null | wc -l)"
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
echo "[GPU]"; nvidia-smi --query-gpu=index,memory.used,memory.total,utilization.gpu --format=csv,noheader 2>/dev/null
|
||||
fi
|
||||
echo ""
|
||||
echo "================ 诊断结束 ================"
|
||||
echo ""
|
||||
echo ">>> 报告已保存到: $OUT (把这个文件发回来)"
|
||||
Reference in New Issue
Block a user