首次提交

tools/compare_jsonl.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+"""
+并排对比两个(或多个)机器的 数据.jsonl，定位谁慢、慢在哪个阶段。
+
+用法:
+  python temp/compare_jsonl.py 2号机器数据.jsonl 3号机器数据.jsonl
+  python temp/compare_jsonl.py a.jsonl b.jsonl c.jsonl ...
+"""
+import json
+import sys
+from datetime import datetime
+
+
+def parse_dt(s):
+    try:
+        return datetime.fromisoformat(s)
+    except (ValueError, TypeError):
+        return None
+
+
+def analyze(path):
+    """读一个 jsonl，返回汇总指标。"""
+    s = {
+        "n": 0, "ok": 0, "err": 0, "no_sub": 0,
+        "dur": 0.0, "elapsed": 0.0,
+        "stage": {"split_frame": 0.0, "merge": 0.0, "getsrt": 0.0},
+        "ocr": 0, "n_ocr": 0,
+        "starts": [], "ends": [],
+    }
+    try:
+        f = open(path, encoding="utf-8")
+    except FileNotFoundError:
+        return None
+    with f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                r = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            s["n"] += 1
+            st = r.get("status")
+            if st == "success":
+                s["ok"] += 1
+            elif st == "no_subtitle":
+                s["no_sub"] += 1
+            else:
+                s["err"] += 1
+            try:
+                s["dur"] += float(r.get("duration") or 0)
+            except (TypeError, ValueError):
+                pass
+            el = r.get("elapsed_seconds")
+            if el:
+                try:
+                    s["elapsed"] += float(el)
+                except (TypeError, ValueError):
+                    pass
+            sd = r.get("stage_durations") or {}
+            for k in s["stage"]:
+                try:
+                    s["stage"][k] += float(sd.get(k) or 0)
+                except (TypeError, ValueError):
+                    pass
+            ic = r.get("image_counts") or {}
+            toc = ic.get("total_ocr_count")
+            if toc:
+                try:
+                    s["ocr"] += int(toc)
+                    s["n_ocr"] += 1
+                except (TypeError, ValueError):
+                    pass
+            ds = parse_dt(r.get("process_start"))
+            de = parse_dt(r.get("process_end"))
+            if ds:
+                s["starts"].append(ds)
+            if de:
+                s["ends"].append(de)
+    return s
+
+
+def derive(s):
+    """从汇总算出对比指标。"""
+    d = {}
+    n = max(1, s["n"])
+    d["记录数"] = s["n"]
+    d["成功率%"] = round(s["ok"] / n * 100, 1)
+    d["视频总时长h"] = round(s["dur"] / 3600, 1)
+    # 墙钟跨度 + 实时倍率
+    span = 0.0
+    if s["starts"] and s["ends"]:
+        span = (max(s["ends"]) - min(s["starts"])).total_seconds()
+    d["墙钟跨度h"] = round(span / 3600, 2)
+    d["实时倍率x"] = round(s["dur"] / span, 2) if span > 0 else 0
+    d["视频/天"] = round(s["n"] / span * 86400) if span > 0 else 0
+    d["有效并发"] = round(s["elapsed"] / span, 1) if span > 0 else 0
+    # 平均每视频各阶段秒数
+    d["抽帧s/视频"] = round(s["stage"]["split_frame"] / n, 1)
+    d["拼接s/视频"] = round(s["stage"]["merge"] / n, 1)
+    d["getsrt s/视频"] = round(s["stage"]["getsrt"] / n, 1)
+    total_stage = sum(s["stage"].values()) or 1
+    d["抽帧占比%"] = round(s["stage"]["split_frame"] / total_stage * 100)
+    d["拼接占比%"] = round(s["stage"]["merge"] / total_stage * 100)
+    d["getsrt占比%"] = round(s["stage"]["getsrt"] / total_stage * 100)
+    d["平均OCR次/视频"] = round(s["ocr"] / s["n_ocr"]) if s["n_ocr"] else 0
+    d["平均耗时s/视频"] = round(s["elapsed"] / n, 1)
+    return d
+
+
+def main():
+    paths = sys.argv[1:]
+    if len(paths) < 2:
+        print("用法: python temp/compare_jsonl.py a.jsonl b.jsonl [c.jsonl ...]")
+        return
+
+    cols, names = [], []
+    for p in paths:
+        s = analyze(p)
+        if s is None:
+            print(f"找不到文件: {p}")
+            return
+        cols.append(derive(s))
+        names.append(p.replace("数据.jsonl", "").replace(".jsonl", "") or p)
+
+    keys = list(cols[0].keys())
+    w = max(14, max(len(n) for n in names) + 2)
+    print("=" * (18 + w * len(cols)))
+    print(f"{'指标':<16}" + "".join(f"{n:>{w}}" for n in names))
+    print("-" * (18 + w * len(cols)))
+    for k in keys:
+        print(f"{k:<16}" + "".join(f"{str(c[k]):>{w}}" for c in cols))
+    print("=" * (18 + w * len(cols)))
+
+    # 自动点评(两台时)
+    if len(cols) == 2:
+        a, b = cols
+        slow_i = 0 if a["实时倍率x"] < b["实时倍率x"] else 1
+        fast_i = 1 - slow_i
+        print(f"\n【分析】{names[slow_i]} 更慢(实时倍率 "
+              f"{cols[slow_i]['实时倍率x']}x vs {cols[fast_i]['实时倍率x']}x)")
+        # 找慢在哪个阶段
+        for stage, key in [("抽帧", "抽帧s/视频"), ("拼接", "拼接s/视频"), ("getsrt(OCR)", "getsrt s/视频")]:
+            sv, fv = cols[slow_i][key], cols[fast_i][key]
+            if fv and sv > fv * 1.3:
+                print(f"  ▸ {stage} 慢: {sv}s vs {fv}s (慢 {sv/fv:.1f}x)")
+        if cols[slow_i]["有效并发"] < cols[fast_i]["有效并发"] * 0.8:
+            print(f"  ▸ 并发低: {cols[slow_i]['有效并发']} vs {cols[fast_i]['有效并发']}"
+                  f"(CPU/内存/OCR 排队?)")
+        if cols[slow_i]["平均OCR次/视频"] > cols[fast_i]["平均OCR次/视频"] * 1.3:
+            print(f"  ▸ OCR 次数多: {cols[slow_i]['平均OCR次/视频']} vs "
+                  f"{cols[fast_i]['平均OCR次/视频']}(视频更长/composite更多)")
+
+
+if __name__ == "__main__":
+    main()