160 lines
6.2 KiB
Python
160 lines
6.2 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
分析 数据.jsonl 的处理吞吐：单台机器一天能处理多少小时的视频数据。
|
||
|
||
两种口径：
|
||
A) 墙钟吞吐（真实）：用 process_start/process_end 求整批的实际墙钟跨度，
|
||
总视频时长 ÷ 墙钟跨度 = 机器真实吞吐（含当时的并发/排队效果）。
|
||
B) 单视频平均（理想并发）：按每个视频 duration/elapsed 的实时倍率，
|
||
再乘并发数估算（假设无 OCR/CPU 排队，即"OCR 速度够"的上限）。
|
||
|
||
用法: python throughput_jsonl.py [数据.jsonl路径] [并发数]
|
||
默认: python throughput_jsonl.py 数据.jsonl 1
|
||
"""
|
||
import json
|
||
import sys
|
||
from datetime import datetime
|
||
|
||
JSONL_PATH = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||
WORKERS = int(sys.argv[2]) if len(sys.argv) > 2 else 1
|
||
OCR_QPS_PER_MACHINE = float(sys.argv[3]) if len(sys.argv) > 3 else 14.0 # 单台 OCR 的 QPS
|
||
|
||
|
||
def parse_dt(s):
|
||
if not s:
|
||
return None
|
||
try:
|
||
return datetime.fromisoformat(s)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def fmt_h(hours):
|
||
return f"{hours:.1f} 小时"
|
||
|
||
|
||
def main():
|
||
try:
|
||
f = open(JSONL_PATH, encoding="utf-8")
|
||
except FileNotFoundError:
|
||
print(f"找不到文件: {JSONL_PATH}")
|
||
return
|
||
|
||
n = 0
|
||
total_dur = 0.0 # 全部视频时长(秒)
|
||
total_elapsed = 0.0 # 全部单视频处理耗时(秒)，含并发
|
||
starts, ends = [], []
|
||
per_video_ratio = [] # 每视频 duration/elapsed
|
||
stage_sum = {} # 各阶段耗时累计
|
||
with_timing = 0
|
||
total_ocr = 0 # 全部 OCR 请求次数累计
|
||
with_ocr = 0
|
||
|
||
with f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
rec = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
n += 1
|
||
dur = rec.get("duration") or 0
|
||
try:
|
||
dur = float(dur)
|
||
except (TypeError, ValueError):
|
||
dur = 0.0
|
||
total_dur += dur
|
||
|
||
el = rec.get("elapsed_seconds")
|
||
if el:
|
||
total_elapsed += float(el)
|
||
if dur > 0:
|
||
per_video_ratio.append(dur / float(el))
|
||
|
||
st = parse_dt(rec.get("process_start"))
|
||
en = parse_dt(rec.get("process_end"))
|
||
if st:
|
||
starts.append(st)
|
||
if en:
|
||
ends.append(en)
|
||
|
||
sd = rec.get("stage_durations") or {}
|
||
if sd:
|
||
with_timing += 1
|
||
for k, v in sd.items():
|
||
try:
|
||
stage_sum[k] = stage_sum.get(k, 0.0) + float(v)
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
ic = rec.get("image_counts") or {}
|
||
toc = ic.get("total_ocr_count")
|
||
if toc:
|
||
try:
|
||
total_ocr += int(toc)
|
||
with_ocr += 1
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
print(f"文件: {JSONL_PATH}")
|
||
print("=" * 52)
|
||
print(f"记录数: {n}")
|
||
print(f"视频总时长: {total_dur/3600:.1f} 小时 ({total_dur:.0f} 秒)")
|
||
print(f"单视频耗时累加: {total_elapsed/3600:.1f} 小时 ({total_elapsed:.0f} 秒)")
|
||
if with_ocr:
|
||
print(f"OCR 请求总数: {total_ocr} (平均 {total_ocr/with_ocr:.0f} 次/视频, {with_ocr} 条有统计)")
|
||
|
||
# ---- 口径A：真实墙钟吞吐 ----
|
||
print("\n[A] 真实墙钟吞吐（按 process_start/end 的整批实际跨度）")
|
||
if starts and ends:
|
||
span = (max(ends) - min(starts)).total_seconds()
|
||
if span > 0:
|
||
tp = total_dur / span # 视频秒/墙钟秒
|
||
print(f" 整批墙钟跨度: {span/3600:.2f} 小时")
|
||
print(f" 实时倍率: {tp:.2f}x （每 1 秒墙钟处理 {tp:.2f} 秒视频）")
|
||
print(f" → 一天可处理: {fmt_h(tp*24)} 的视频")
|
||
vids_per_day = n / span * 86400 if span else 0
|
||
print(f" → 一天可处理: {vids_per_day:.0f} 个视频")
|
||
|
||
# ---- OCR 需求：这台机器实际产生多少 OCR/秒、能喂几台 OCR ----
|
||
if total_ocr:
|
||
ocr_per_sec = total_ocr / span
|
||
machines_fed = ocr_per_sec / OCR_QPS_PER_MACHINE
|
||
print(f"\n [OCR 配比]（单台 OCR 按 {OCR_QPS_PER_MACHINE:.0f} QPS 计）")
|
||
print(f" 本机 OCR 速率: {ocr_per_sec:.2f} OCR/秒")
|
||
print(f" → 本机能喂满: {machines_fed:.2f} 台 OCR")
|
||
if machines_fed > 0:
|
||
print(f" → 一台 OCR 可喂: {1/machines_fed:.1f} 台这样的处理机")
|
||
else:
|
||
print(" 墙钟跨度为0，无法计算")
|
||
else:
|
||
print(" 缺少 process_start/process_end 字段，跳过")
|
||
|
||
# ---- 口径B：按单视频平均实时倍率 × 并发 ----
|
||
print(f"\n[B] 理想吞吐（单视频平均倍率 × 并发{WORKERS}，假设无排队=OCR够快）")
|
||
if per_video_ratio:
|
||
avg_ratio = sum(per_video_ratio) / len(per_video_ratio)
|
||
# 也给一个总量口径：总视频时长/总耗时（更稳，受长短视频加权）
|
||
weighted = total_dur / total_elapsed if total_elapsed else 0
|
||
print(f" 单视频平均倍率: {avg_ratio:.2f}x")
|
||
print(f" 加权倍率(总/总):{weighted:.2f}x")
|
||
print(f" → 单并发一天: {fmt_h(weighted*24)} 的视频")
|
||
print(f" → {WORKERS}并发一天: {fmt_h(weighted*24*WORKERS)} 的视频（理想线性，实际受CPU限制）")
|
||
else:
|
||
print(" 缺少 elapsed_seconds 字段，跳过")
|
||
|
||
# ---- 各阶段占比 ----
|
||
if stage_sum:
|
||
print(f"\n[阶段耗时占比]（{with_timing} 条有计时）")
|
||
main_t = stage_sum.get("main", sum(v for k, v in stage_sum.items() if k != "main"))
|
||
for k in ["split_frame", "merge", "getsrt"]:
|
||
if k in stage_sum:
|
||
pct = stage_sum[k] / main_t * 100 if main_t else 0
|
||
print(f" {k:12s}: {stage_sum[k]/3600:6.1f} 小时 ({pct:.0f}%)")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|