首次提交
This commit is contained in:
159
tools/throughput_jsonl.py
Normal file
159
tools/throughput_jsonl.py
Normal file
@@ -0,0 +1,159 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
分析 数据.jsonl 的处理吞吐：单台机器一天能处理多少小时的视频数据。
|
||||
|
||||
两种口径：
|
||||
A) 墙钟吞吐（真实）：用 process_start/process_end 求整批的实际墙钟跨度，
|
||||
总视频时长 ÷ 墙钟跨度 = 机器真实吞吐（含当时的并发/排队效果）。
|
||||
B) 单视频平均（理想并发）：按每个视频 duration/elapsed 的实时倍率，
|
||||
再乘并发数估算（假设无 OCR/CPU 排队，即"OCR 速度够"的上限）。
|
||||
|
||||
用法: python throughput_jsonl.py [数据.jsonl路径] [并发数]
|
||||
默认: python throughput_jsonl.py 数据.jsonl 1
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
JSONL_PATH = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||||
WORKERS = int(sys.argv[2]) if len(sys.argv) > 2 else 1
|
||||
OCR_QPS_PER_MACHINE = float(sys.argv[3]) if len(sys.argv) > 3 else 14.0 # 单台 OCR 的 QPS
|
||||
|
||||
|
||||
def parse_dt(s):
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(s)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def fmt_h(hours):
|
||||
return f"{hours:.1f} 小时"
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
f = open(JSONL_PATH, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
n = 0
|
||||
total_dur = 0.0 # 全部视频时长(秒)
|
||||
total_elapsed = 0.0 # 全部单视频处理耗时(秒)，含并发
|
||||
starts, ends = [], []
|
||||
per_video_ratio = [] # 每视频 duration/elapsed
|
||||
stage_sum = {} # 各阶段耗时累计
|
||||
with_timing = 0
|
||||
total_ocr = 0 # 全部 OCR 请求次数累计
|
||||
with_ocr = 0
|
||||
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
n += 1
|
||||
dur = rec.get("duration") or 0
|
||||
try:
|
||||
dur = float(dur)
|
||||
except (TypeError, ValueError):
|
||||
dur = 0.0
|
||||
total_dur += dur
|
||||
|
||||
el = rec.get("elapsed_seconds")
|
||||
if el:
|
||||
total_elapsed += float(el)
|
||||
if dur > 0:
|
||||
per_video_ratio.append(dur / float(el))
|
||||
|
||||
st = parse_dt(rec.get("process_start"))
|
||||
en = parse_dt(rec.get("process_end"))
|
||||
if st:
|
||||
starts.append(st)
|
||||
if en:
|
||||
ends.append(en)
|
||||
|
||||
sd = rec.get("stage_durations") or {}
|
||||
if sd:
|
||||
with_timing += 1
|
||||
for k, v in sd.items():
|
||||
try:
|
||||
stage_sum[k] = stage_sum.get(k, 0.0) + float(v)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
ic = rec.get("image_counts") or {}
|
||||
toc = ic.get("total_ocr_count")
|
||||
if toc:
|
||||
try:
|
||||
total_ocr += int(toc)
|
||||
with_ocr += 1
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
print(f"文件: {JSONL_PATH}")
|
||||
print("=" * 52)
|
||||
print(f"记录数: {n}")
|
||||
print(f"视频总时长: {total_dur/3600:.1f} 小时 ({total_dur:.0f} 秒)")
|
||||
print(f"单视频耗时累加: {total_elapsed/3600:.1f} 小时 ({total_elapsed:.0f} 秒)")
|
||||
if with_ocr:
|
||||
print(f"OCR 请求总数: {total_ocr} (平均 {total_ocr/with_ocr:.0f} 次/视频, {with_ocr} 条有统计)")
|
||||
|
||||
# ---- 口径A：真实墙钟吞吐 ----
|
||||
print("\n[A] 真实墙钟吞吐（按 process_start/end 的整批实际跨度）")
|
||||
if starts and ends:
|
||||
span = (max(ends) - min(starts)).total_seconds()
|
||||
if span > 0:
|
||||
tp = total_dur / span # 视频秒/墙钟秒
|
||||
print(f" 整批墙钟跨度: {span/3600:.2f} 小时")
|
||||
print(f" 实时倍率: {tp:.2f}x （每 1 秒墙钟处理 {tp:.2f} 秒视频）")
|
||||
print(f" → 一天可处理: {fmt_h(tp*24)} 的视频")
|
||||
vids_per_day = n / span * 86400 if span else 0
|
||||
print(f" → 一天可处理: {vids_per_day:.0f} 个视频")
|
||||
|
||||
# ---- OCR 需求：这台机器实际产生多少 OCR/秒、能喂几台 OCR ----
|
||||
if total_ocr:
|
||||
ocr_per_sec = total_ocr / span
|
||||
machines_fed = ocr_per_sec / OCR_QPS_PER_MACHINE
|
||||
print(f"\n [OCR 配比]（单台 OCR 按 {OCR_QPS_PER_MACHINE:.0f} QPS 计）")
|
||||
print(f" 本机 OCR 速率: {ocr_per_sec:.2f} OCR/秒")
|
||||
print(f" → 本机能喂满: {machines_fed:.2f} 台 OCR")
|
||||
if machines_fed > 0:
|
||||
print(f" → 一台 OCR 可喂: {1/machines_fed:.1f} 台这样的处理机")
|
||||
else:
|
||||
print(" 墙钟跨度为0，无法计算")
|
||||
else:
|
||||
print(" 缺少 process_start/process_end 字段，跳过")
|
||||
|
||||
# ---- 口径B：按单视频平均实时倍率 × 并发 ----
|
||||
print(f"\n[B] 理想吞吐（单视频平均倍率 × 并发{WORKERS}，假设无排队=OCR够快）")
|
||||
if per_video_ratio:
|
||||
avg_ratio = sum(per_video_ratio) / len(per_video_ratio)
|
||||
# 也给一个总量口径：总视频时长/总耗时（更稳，受长短视频加权）
|
||||
weighted = total_dur / total_elapsed if total_elapsed else 0
|
||||
print(f" 单视频平均倍率: {avg_ratio:.2f}x")
|
||||
print(f" 加权倍率(总/总):{weighted:.2f}x")
|
||||
print(f" → 单并发一天: {fmt_h(weighted*24)} 的视频")
|
||||
print(f" → {WORKERS}并发一天: {fmt_h(weighted*24*WORKERS)} 的视频（理想线性，实际受CPU限制）")
|
||||
else:
|
||||
print(" 缺少 elapsed_seconds 字段，跳过")
|
||||
|
||||
# ---- 各阶段占比 ----
|
||||
if stage_sum:
|
||||
print(f"\n[阶段耗时占比]（{with_timing} 条有计时）")
|
||||
main_t = stage_sum.get("main", sum(v for k, v in stage_sum.items() if k != "main"))
|
||||
for k in ["split_frame", "merge", "getsrt"]:
|
||||
if k in stage_sum:
|
||||
pct = stage_sum[k] / main_t * 100 if main_t else 0
|
||||
print(f" {k:12s}: {stage_sum[k]/3600:6.1f} 小时 ({pct:.0f}%)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user