首次提交
This commit is contained in:
174
tools/hourly_stats.py
Normal file
174
tools/hourly_stats.py
Normal file
@@ -0,0 +1,174 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
按【小时】统计 数据.jsonl：每小时处理多少条视频、视频总时长、成功情况、处理倍速。
|
||||
小时取 process_start（没有则回退 timestamp）的 YYYY-MM-DD HH。
|
||||
|
||||
"处理倍速" = 该小时内处理掉的【视频内容总时长】 / 1 墙钟小时。
|
||||
例: 一个墙钟小时里处理了 40 小时的视频内容 → 倍速 40x → 等效有效并发≈40。
|
||||
这个数最能反映真实吞吐/并发利用率(对照 max-workers 看是否吃满)。
|
||||
|
||||
用法(项目根目录运行):
|
||||
python temp/hourly_stats.py [数据.jsonl路径] [--last N]
|
||||
--last N : 只看最近 N 个有数据的小时(默认全部)
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import unicodedata
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def _parse(ts):
|
||||
try:
|
||||
return datetime.fromisoformat(ts)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _dw(s):
|
||||
"""字符串显示宽度(中文/全角算2)。"""
|
||||
return sum(2 if unicodedata.east_asian_width(c) in ("W", "F") else 1 for c in str(s))
|
||||
|
||||
|
||||
def _padr(s, w):
|
||||
"""按显示宽度右对齐。"""
|
||||
s = str(s)
|
||||
return " " * max(0, w - _dw(s)) + s
|
||||
|
||||
|
||||
def _padl(s, w):
|
||||
"""按显示宽度左对齐。"""
|
||||
s = str(s)
|
||||
return s + " " * max(0, w - _dw(s))
|
||||
|
||||
|
||||
# 列宽(显示宽度)
|
||||
COLW = {"小时": 16, "视频数": 8, "成功": 6, "无字幕": 8, "错误": 6,
|
||||
"总时长h": 10, "成功h": 9, "处理倍速": 10, "有效并发": 10}
|
||||
|
||||
|
||||
def _row(hour, count, success, no_sub, error, dur_h, ok_h, speed, eff):
|
||||
return (_padl(hour, COLW["小时"]) + _padr(count, COLW["视频数"]) +
|
||||
_padr(success, COLW["成功"]) + _padr(no_sub, COLW["无字幕"]) +
|
||||
_padr(error, COLW["错误"]) + _padr(dur_h, COLW["总时长h"]) +
|
||||
_padr(ok_h, COLW["成功h"]) + _padr(speed, COLW["处理倍速"]) +
|
||||
_padr(eff, COLW["有效并发"]))
|
||||
|
||||
args = [a for a in sys.argv[1:]]
|
||||
last_n = None
|
||||
if "--last" in args:
|
||||
i = args.index("--last")
|
||||
try:
|
||||
last_n = int(args[i + 1])
|
||||
del args[i:i + 2]
|
||||
except (IndexError, ValueError):
|
||||
last_n = None
|
||||
JSONL_PATH = args[0] if args else "数据.jsonl"
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
f = open(JSONL_PATH, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
hours = defaultdict(lambda: {"count": 0, "success": 0, "no_subtitle": 0,
|
||||
"error": 0, "dur": 0.0, "dur_ok": 0.0, "elapsed": 0.0})
|
||||
busy = defaultdict(float) # 每个墙钟小时内, 所有视频实际占用的总秒数(精确并发用)
|
||||
no_date = 0
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
ts = rec.get("process_start") or rec.get("timestamp") or ""
|
||||
hour = ts[:13] if len(ts) >= 13 else None # YYYY-MM-DD HH(按开始时间归桶)
|
||||
if not hour:
|
||||
no_date += 1
|
||||
continue
|
||||
|
||||
h = hours[hour]
|
||||
h["count"] += 1
|
||||
status = rec.get("status")
|
||||
if status == "success":
|
||||
h["success"] += 1
|
||||
elif status == "no_subtitle":
|
||||
h["no_subtitle"] += 1
|
||||
else:
|
||||
h["error"] += 1
|
||||
|
||||
try:
|
||||
dur = float(rec.get("duration") or 0)
|
||||
h["dur"] += dur
|
||||
if status == "success":
|
||||
h["dur_ok"] += dur
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
try:
|
||||
h["elapsed"] += float(rec.get("elapsed_seconds") or 0)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# 精确并发: 把 [process_start, process_end] 的时长按实际跨越的每个小时切开累加,
|
||||
# 避免"长视频整段算进开始那个小时"导致失真。
|
||||
st = _parse(rec.get("process_start"))
|
||||
en = _parse(rec.get("process_end"))
|
||||
if st and en and en > st:
|
||||
cur = st
|
||||
while cur < en:
|
||||
hk = cur.strftime("%Y-%m-%dT%H")
|
||||
nxt = cur.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
|
||||
seg_end = min(en, nxt)
|
||||
busy[hk] += (seg_end - cur).total_seconds()
|
||||
cur = seg_end
|
||||
|
||||
if not hours:
|
||||
print("没有可统计的数据")
|
||||
return
|
||||
|
||||
keys = sorted(hours)
|
||||
if last_n:
|
||||
keys = keys[-last_n:]
|
||||
|
||||
width = sum(COLW.values())
|
||||
print(f"文件: {JSONL_PATH}")
|
||||
print("=" * width)
|
||||
print(_row("小时", "视频数", "成功", "无字幕", "错误", "总时长h", "成功h", "处理倍速", "有效并发"))
|
||||
print("-" * width)
|
||||
|
||||
tot = {"count": 0, "success": 0, "no_subtitle": 0, "error": 0,
|
||||
"dur": 0.0, "dur_ok": 0.0, "elapsed": 0.0}
|
||||
busy_sum = 0.0
|
||||
for hour in keys:
|
||||
h = hours[hour]
|
||||
speed = h["dur"] / 3600 # 处理倍速: 视频内容小时数 / 1墙钟小时
|
||||
eff = busy[hour] / 3600 # 有效并发: 该小时所有视频实际占用秒数 / 3600(=平均并发)
|
||||
busy_sum += busy[hour]
|
||||
print(_row(hour, h["count"], h["success"], h["no_subtitle"], h["error"],
|
||||
f"{h['dur']/3600:.1f}", f"{h['dur_ok']/3600:.1f}",
|
||||
f"{speed:.1f}x", f"{eff:.1f}"))
|
||||
for k in tot:
|
||||
tot[k] += h[k]
|
||||
|
||||
print("-" * width)
|
||||
nh = len(keys)
|
||||
print(_row("合计/均值", tot["count"], tot["success"], tot["no_subtitle"], tot["error"],
|
||||
f"{tot['dur']/3600:.1f}", f"{tot['dur_ok']/3600:.1f}",
|
||||
f"{tot['dur']/3600/nh:.1f}x", f"{busy_sum/3600/nh:.1f}"))
|
||||
if no_date:
|
||||
print(f"\n（{no_date} 条无时间字段，已跳过）")
|
||||
print(f"\n共 {nh} 个小时 | 时均处理 {tot['count']/nh:.0f} 条 | "
|
||||
f"时均成功 {tot['success']/nh:.0f} 条")
|
||||
if tot["count"]:
|
||||
print(f"成功率 {tot['success']/tot['count']*100:.1f}%")
|
||||
print("提示: '有效并发'明显小于你设的 max-workers → 有串行瓶颈(GIL/锁/等待),不是并发不够。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user