首次提交
This commit is contained in:
36
tools/_dl.py
Normal file
36
tools/_dl.py
Normal file
@@ -0,0 +1,36 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import json, os, sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from config.config_utils import config
|
||||
|
||||
cos = config.get_cos_config()
|
||||
client = CosS3Client(CosConfig(Region=cos["region"], SecretId=cos["secret_id"],
|
||||
SecretKey=cos["secret_key"]))
|
||||
bucket = cos["bucket"]
|
||||
|
||||
rows = []
|
||||
for line in open("数据.jsonl", encoding="utf-8"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if r.get("status") == "success" and r.get("srt_cos_path"):
|
||||
rows.append(r)
|
||||
recent = rows[-15:]
|
||||
|
||||
os.makedirs("srt_check", exist_ok=True)
|
||||
ok = 0
|
||||
for r in recent:
|
||||
p = r["srt_cos_path"]
|
||||
key = p.split("://", 1)[1].split("/", 1)[1] # 去掉 cos://bucket/
|
||||
name = os.path.basename(key)
|
||||
try:
|
||||
client.download_file(Bucket=bucket, Key=key, DestFilePath=os.path.join("srt_check", name))
|
||||
ok += 1
|
||||
except Exception as e:
|
||||
print("下载失败", name, e)
|
||||
print(f"下载成功 {ok}/{len(recent)} → srt_check/")
|
||||
100
tools/analyze_ocr_result.py
Normal file
100
tools/analyze_ocr_result.py
Normal file
@@ -0,0 +1,100 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
分析一个 ocr_result.jsonl，判断"冗余文字过多"是真冗余还是误杀。
|
||||
复现过滤逻辑：看文字的垂直位置分布(distinct_bands)、每帧文字框数、有没有稳定字幕行。
|
||||
|
||||
用法: python temp/analyze_ocr_result.py "ocr_result (1).jsonl"
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
PATH = sys.argv[1] if len(sys.argv) > 1 else "ocr_result.jsonl"
|
||||
|
||||
|
||||
def parse(path):
|
||||
"""解析 'pic_path: X' + json 交替格式，返回 [(frame, detections), ...]。"""
|
||||
pairs = []
|
||||
cur_frame = None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.rstrip("\n")
|
||||
if line.startswith("pic_path:"):
|
||||
cur_frame = line.split("pic_path:", 1)[1].strip()
|
||||
elif line.strip():
|
||||
try:
|
||||
data = json.loads(line)
|
||||
pairs.append((cur_frame, data.get("TextDetections", [])))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return pairs
|
||||
|
||||
|
||||
def center_y(det):
|
||||
poly = det.get("Polygon", [])
|
||||
ys = [p.get("Y", 0) for p in poly]
|
||||
return sum(ys) / len(ys) if ys else 0
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
pairs = parse(PATH)
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {PATH}")
|
||||
return
|
||||
if not pairs:
|
||||
print("没解析到数据(格式不对?)")
|
||||
return
|
||||
|
||||
n_frames = len(pairs)
|
||||
boxes_per_frame = [len(d) for _, d in pairs]
|
||||
total_boxes = sum(boxes_per_frame)
|
||||
frames_with_text = sum(1 for b in boxes_per_frame if b)
|
||||
|
||||
# 所有文字框的中心 Y
|
||||
all_cy = []
|
||||
for _, dets in pairs:
|
||||
for det in dets:
|
||||
all_cy.append(center_y(det))
|
||||
|
||||
print(f"文件: {PATH}")
|
||||
print("=" * 60)
|
||||
print(f"帧/图数: {n_frames}")
|
||||
print(f"有文字的: {frames_with_text}")
|
||||
print(f"文字框总数: {total_boxes}")
|
||||
print(f"平均每帧框数: {total_boxes / n_frames:.1f}")
|
||||
print(f"最多一帧框数: {max(boxes_per_frame) if boxes_per_frame else 0}")
|
||||
|
||||
# distinct vertical bands(复现 merge.py 的冗余判定：中心Y间隔>80算一个新垂直簇)
|
||||
cys = sorted(all_cy)
|
||||
bands = 1 if cys else 0
|
||||
for i in range(1, len(cys)):
|
||||
if cys[i] - cys[i - 1] > 80:
|
||||
bands += 1
|
||||
print(f"不同垂直位置簇: {bands} (>=8 会被判冗余)")
|
||||
|
||||
# Y 分布直方图(每 100px 一档)，看文字集中在哪还是散布全屏
|
||||
hist = defaultdict(int)
|
||||
for y in all_cy:
|
||||
hist[int(y // 100) * 100] += 1
|
||||
print("\n文字垂直分布(Y每100px一档，看文字集中度):")
|
||||
if hist:
|
||||
mx = max(hist.values())
|
||||
for y in sorted(hist):
|
||||
bar = "#" * int(hist[y] / mx * 40)
|
||||
print(f" y={y:>5}-{y+99:<5} {hist[y]:>6} {bar}")
|
||||
|
||||
# 判定
|
||||
print("\n" + "=" * 60)
|
||||
if bands >= 8:
|
||||
print(f"❌ 真冗余：文字散布在 {bands} 个不同垂直位置(满屏文字/弹幕/文档类)。")
|
||||
elif total_boxes / n_frames > 10:
|
||||
print(f"⚠️ 偏冗余：平均每帧 {total_boxes/n_frames:.0f} 个文字框，画面文字很多。")
|
||||
else:
|
||||
topy = max(hist, key=hist.get) if hist else 0
|
||||
print(f"✅ 可能误杀：文字主要集中在 y={topy}~{topy+99}，像是有固定字幕行的。")
|
||||
print(" 若确实有正常字幕，可放宽 merge.py 的 distinct_bands 阈值。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
463
tools/bench_ocr_endpoints.py
Normal file
463
tools/bench_ocr_endpoints.py
Normal file
@@ -0,0 +1,463 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
压测每个 OCR 端点(IP)的实际 QPS，并把结果写成权重文件 config/ocr_weights.json，
|
||||
供运行时按 QPS 加权分发请求(见 tool/Ocr/tencentcloud_ocr.py 的 OcrDispatcher)。
|
||||
|
||||
做法：对每个端点用固定并发、固定请求数发同一张样例图，
|
||||
QPS = 成功请求数 / 墙钟耗时；同时报告平均时延和错误率。
|
||||
|
||||
本脚本完全自包含：单文件，不读 config、不 import 项目任何模块，拷过去就能跑。
|
||||
|
||||
用法:
|
||||
python bench_ocr_endpoints.py [样例图片] [--requests N] [--concurrency C] [--no-write]
|
||||
python bench_ocr_endpoints.py [样例图片] --url http://IP:PORT [--url http://IP2:PORT ...]
|
||||
|
||||
样例图片不给时，自动在 ./composite、./split 下找一张 .jpg。
|
||||
端点：默认内置 Polaris 地址；要测别的 IP 用 --url(可多次)。
|
||||
--requests 每个端点发的总请求数(默认 60)
|
||||
--concurrency 每个端点的并发线程数(默认 10)
|
||||
--no-write 只打印，不写 ocr_weights.json
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import glob
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import requests
|
||||
|
||||
OCR_PATH = "/youtu/ocrapi/hpgeneralocr_v3"
|
||||
OCR_PORT = 18081
|
||||
|
||||
# 内置的多台 OCR 服务 IP（自动负载均衡测试用）
|
||||
DEFAULT_IPS = [
|
||||
"30.154.78.83"
|
||||
]
|
||||
DEFAULT_ENDPOINTS = [f"http://{ip}:{OCR_PORT}" for ip in DEFAULT_IPS]
|
||||
WEIGHTS_FILE = "./ocr_weights.json"
|
||||
|
||||
|
||||
def get_endpoints(cli_urls):
|
||||
"""命令行 --url 优先，否则用内置默认端点。"""
|
||||
if cli_urls:
|
||||
return [u.rstrip("/") for u in cli_urls]
|
||||
return list(DEFAULT_ENDPOINTS)
|
||||
|
||||
|
||||
def find_sample_image(given):
|
||||
if given:
|
||||
if os.path.exists(given):
|
||||
return given
|
||||
print(f"指定的图片不存在: {given}")
|
||||
return None
|
||||
for root in ("./composite", "./split"):
|
||||
hits = glob.glob(os.path.join(root, "**", "*.jpg"), recursive=True)
|
||||
if hits:
|
||||
return hits[0]
|
||||
return None
|
||||
|
||||
|
||||
def img2base64(img_path):
|
||||
with open(img_path, "rb") as f:
|
||||
return base64.b64encode(f.read()).decode()
|
||||
|
||||
|
||||
def ocr_request_once(base_url, image_b64, timeout=60):
|
||||
"""向指定端点发一次 OCR 请求，返回原始响应文本；失败抛异常。"""
|
||||
url = base_url.rstrip("/") + OCR_PATH
|
||||
app_id = f"{int(time.time() * 1000)}{random.randint(100000, 999999)}"
|
||||
payload = {
|
||||
"app_id": app_id,
|
||||
"image": image_b64,
|
||||
"ocr_template": "OCR",
|
||||
"options": {"pdf_page_index": 0, "detect_text": True},
|
||||
}
|
||||
resp = requests.post(url, headers={"Content-Type": "application/json"},
|
||||
json=payload, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
|
||||
def _ok(raw):
|
||||
"""判断一次响应是否算成功：有内容且不是 error。"""
|
||||
if not raw:
|
||||
return False
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return False
|
||||
if "TextDetections" in data:
|
||||
return True
|
||||
return data.get("error_code", 0) == 0
|
||||
|
||||
|
||||
def bench_endpoint(base_url, image_b64, requests_n, concurrency):
|
||||
"""对单个端点压测，返回统计 dict（含一个失败原因采样 sample_err）。"""
|
||||
latencies = []
|
||||
ok = 0
|
||||
err = 0
|
||||
sample_err = [None] # 采样第一个失败原因
|
||||
|
||||
def one_call():
|
||||
t0 = time.time()
|
||||
try:
|
||||
raw = ocr_request_once(base_url, image_b64, timeout=60)
|
||||
dt = time.time() - t0
|
||||
if _ok(raw):
|
||||
return (True, dt)
|
||||
# HTTP 200 但内容是错误：采样响应体片段
|
||||
if sample_err[0] is None:
|
||||
sample_err[0] = f"响应非成功: {raw[:200]}"
|
||||
return (False, dt)
|
||||
except Exception as e:
|
||||
if sample_err[0] is None:
|
||||
sample_err[0] = f"{type(e).__name__}: {e}"
|
||||
return (False, time.time() - t0)
|
||||
|
||||
t_start = time.time()
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
||||
futures = [ex.submit(one_call) for _ in range(requests_n)]
|
||||
for fut in as_completed(futures):
|
||||
success, dt = fut.result()
|
||||
latencies.append(dt)
|
||||
if success:
|
||||
ok += 1
|
||||
else:
|
||||
err += 1
|
||||
wall = time.time() - t_start
|
||||
|
||||
return {
|
||||
"qps": ok / wall if wall > 0 else 0.0,
|
||||
"ok": ok,
|
||||
"err": err,
|
||||
"wall": wall,
|
||||
"avg_latency": sum(latencies) / len(latencies) if latencies else 0.0,
|
||||
"err_rate": err / requests_n if requests_n else 0.0,
|
||||
"sample_err": sample_err[0],
|
||||
}
|
||||
|
||||
|
||||
def soak_endpoint(base_url, image_pool, concurrency, report_interval=5):
|
||||
"""对单个端点持续满速压测：固定 concurrency 个线程不停发请求，
|
||||
每次从 image_pool 随机抽一张图(避免被服务端缓存)，每 report_interval 秒打印滚动 QPS，
|
||||
直到 Ctrl+C。image_pool 可以是单张或几千张。"""
|
||||
stop = threading.Event()
|
||||
lock = threading.Lock()
|
||||
st = {"ok": 0, "err": 0, "lat": 0.0, "n": 0}
|
||||
sample_err = [None]
|
||||
|
||||
def worker():
|
||||
while not stop.is_set():
|
||||
img = random.choice(image_pool) # 每次随机抽一张
|
||||
t0 = time.time()
|
||||
try:
|
||||
raw = ocr_request_once(base_url, img, timeout=60)
|
||||
success = _ok(raw)
|
||||
if not success and sample_err[0] is None:
|
||||
sample_err[0] = f"响应非成功: {raw[:160]}"
|
||||
except Exception as e:
|
||||
success = False
|
||||
if sample_err[0] is None:
|
||||
sample_err[0] = f"{type(e).__name__}: {e}"
|
||||
dt = time.time() - t0
|
||||
with lock:
|
||||
st["n"] += 1
|
||||
st["lat"] += dt
|
||||
if success:
|
||||
st["ok"] += 1
|
||||
else:
|
||||
st["err"] += 1
|
||||
|
||||
threads = [threading.Thread(target=worker, daemon=True) for _ in range(concurrency)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
print(f"持续满速压测 {base_url} 并发={concurrency} 每{report_interval}s 报一次 (Ctrl+C 停止)")
|
||||
print(f"{'用时s':>6} {'累计OK':>9} {'瞬时QPS':>9} {'平均QPS':>9} {'时延ms':>8} {'错误率':>7}")
|
||||
print("-" * 56)
|
||||
t_start = time.time()
|
||||
last_ok, last_t = 0, t_start
|
||||
try:
|
||||
while True:
|
||||
time.sleep(report_interval)
|
||||
now = time.time()
|
||||
with lock:
|
||||
ok, err, n, lat = st["ok"], st["err"], st["n"], st["lat"]
|
||||
inst = (ok - last_ok) / (now - last_t) if now > last_t else 0
|
||||
avg = ok / (now - t_start) if now > t_start else 0
|
||||
avg_lat = (lat / n * 1000) if n else 0
|
||||
er = err / n * 100 if n else 0
|
||||
print(f"{now-t_start:6.0f} {ok:>9} {inst:>9.1f} {avg:>9.1f} {avg_lat:>8.0f} {er:>6.0f}%", flush=True)
|
||||
last_ok, last_t = ok, now
|
||||
except KeyboardInterrupt:
|
||||
stop.set()
|
||||
el = time.time() - t_start
|
||||
with lock:
|
||||
ok, n = st["ok"], st["n"]
|
||||
print("-" * 56)
|
||||
print(f"停止。{el:.0f}s 内成功 {ok} / 共发 {n}，平均 {ok/el if el else 0:.1f} QPS")
|
||||
if sample_err[0]:
|
||||
print(f"失败采样: {sample_err[0]}")
|
||||
|
||||
|
||||
def health_check(endpoints, image_b64):
|
||||
"""给每个端点发一次请求，返回 (存活列表, 死亡列表)。"""
|
||||
alive, dead = [], []
|
||||
|
||||
def _check(ep):
|
||||
try:
|
||||
return ep, _ok(ocr_request_once(ep, image_b64, timeout=15))
|
||||
except Exception:
|
||||
return ep, False
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(32, len(endpoints))) as ex:
|
||||
for ep, ok in ex.map(_check, endpoints):
|
||||
(alive if ok else dead).append(ep)
|
||||
return alive, dead
|
||||
|
||||
|
||||
def bench_aggregate(endpoints, image_b64, requests_n, concurrency):
|
||||
"""把请求轮询分发到所有端点（模拟运行时自动负载均衡），测合并 QPS。"""
|
||||
rr = itertools.cycle(endpoints)
|
||||
rr_lock = threading.Lock()
|
||||
latencies = []
|
||||
ok = 0
|
||||
err = 0
|
||||
stat_lock = threading.Lock()
|
||||
|
||||
def next_ep():
|
||||
with rr_lock:
|
||||
return next(rr)
|
||||
|
||||
def one_call(_):
|
||||
ep = next_ep()
|
||||
t0 = time.time()
|
||||
try:
|
||||
raw = ocr_request_once(ep, image_b64, timeout=60)
|
||||
success = _ok(raw)
|
||||
except Exception:
|
||||
success = False
|
||||
dt = time.time() - t0
|
||||
with stat_lock:
|
||||
latencies.append(dt)
|
||||
return success
|
||||
|
||||
t_start = time.time()
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
||||
for success in ex.map(one_call, range(requests_n)):
|
||||
if success:
|
||||
ok += 1
|
||||
else:
|
||||
err += 1
|
||||
wall = time.time() - t_start
|
||||
|
||||
return {
|
||||
"qps": ok / wall if wall > 0 else 0.0,
|
||||
"ok": ok,
|
||||
"err": err,
|
||||
"wall": wall,
|
||||
"avg_latency": sum(latencies) / len(latencies) if latencies else 0.0,
|
||||
"err_rate": err / requests_n if requests_n else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def run_aggregate(endpoints, image_b64, requests_n, concurrency):
|
||||
"""先健康检查筛掉死端点，再在存活端点上做聚合负载测试。"""
|
||||
print(f"健康检查 {len(endpoints)} 个端点 ...", flush=True)
|
||||
alive, dead = health_check(endpoints, image_b64)
|
||||
print(f" 存活: {len(alive)} 死亡: {len(dead)}")
|
||||
if dead:
|
||||
for ep in dead:
|
||||
print(f" ✗ {ep}")
|
||||
if not alive:
|
||||
print("没有存活端点，无法测试。")
|
||||
return
|
||||
print("=" * 64)
|
||||
print(f"聚合负载测试: {len(alive)} 个存活端点轮询分发，"
|
||||
f"总请求 {requests_n}，总并发 {concurrency}")
|
||||
s = bench_aggregate(alive, image_b64, requests_n, concurrency)
|
||||
print("-" * 64)
|
||||
print(f" 合并 QPS = {s['qps']:.2f} 成功={s['ok']} 失败={s['err']}"
|
||||
f" 平均时延={s['avg_latency']*1000:.0f}ms 错误率={s['err_rate']*100:.0f}%")
|
||||
if alive:
|
||||
print(f" 单端点平均 ≈ {s['qps']/len(alive):.2f} QPS"
|
||||
f" （{len(alive)} 台合计 {s['qps']:.0f} QPS）")
|
||||
print("=" * 64)
|
||||
|
||||
|
||||
def sweep_endpoint(base_url, image_b64, start, max_c, per_worker):
|
||||
"""自动加压找极限 QPS：并发从 start 逐级翻倍到 max_c，
|
||||
QPS 不再随并发上升(或错误率升高)即判定到极限，停止。"""
|
||||
print(f"自动加压 {base_url} (并发 {start}→{max_c} 逐级翻倍)")
|
||||
print(f"{'并发':>6} {'QPS':>9} {'时延ms':>9} {'错误率':>8}")
|
||||
print("-" * 36)
|
||||
best_qps, best_c = 0.0, 0
|
||||
prev_qps = 0.0
|
||||
c = start
|
||||
while c <= max_c:
|
||||
reqs = max(c * per_worker, 30) # 每个并发至少发 per_worker 次，样本够稳
|
||||
s = bench_endpoint(base_url, image_b64, reqs, c)
|
||||
print(f"{c:>6} {s['qps']:>9.2f} {s['avg_latency']*1000:>9.0f} {s['err_rate']*100:>7.0f}%")
|
||||
if s["qps"] > best_qps:
|
||||
best_qps, best_c = s["qps"], c
|
||||
if s["err_rate"] > 0.1:
|
||||
print(" → 错误率升高，已超过服务承载，停止。")
|
||||
break
|
||||
if prev_qps > 0 and s["qps"] < prev_qps * 1.08:
|
||||
print(" → QPS 不再随并发上升，已到极限，停止。")
|
||||
break
|
||||
prev_qps = s["qps"]
|
||||
c *= 2
|
||||
print("-" * 36)
|
||||
print(f"极限 QPS ≈ {best_qps:.2f} (最佳并发 {best_c})")
|
||||
return best_qps, best_c
|
||||
|
||||
|
||||
def gen_random_images(n, size=(1280, 200)):
|
||||
"""生成 n 张带随机文字的图片(让 OCR 真干活)，返回 base64 列表。需要 Pillow。"""
|
||||
try:
|
||||
import io
|
||||
import string
|
||||
from PIL import Image, ImageDraw
|
||||
except ImportError:
|
||||
print("生成随机图需要 Pillow(pip install pillow)；改用 --pool 指向真实图目录")
|
||||
raise
|
||||
chars = string.ascii_letters + string.digits + " ,.!?-"
|
||||
pool = []
|
||||
print(f"生成 {n} 张随机文字图...", flush=True)
|
||||
for i in range(n):
|
||||
bg = random.randint(200, 255)
|
||||
img = Image.new("RGB", size, (bg, bg, bg))
|
||||
d = ImageDraw.Draw(img)
|
||||
for row in range(random.randint(1, 3)):
|
||||
text = "".join(random.choice(chars) for _ in range(random.randint(10, 40)))
|
||||
d.text((20, 20 + row * 50), text, fill=(0, 0, 0))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=85)
|
||||
pool.append(base64.b64encode(buf.getvalue()).decode())
|
||||
if (i + 1) % 1000 == 0:
|
||||
print(f" 已生成 {i+1}/{n}", flush=True)
|
||||
return pool
|
||||
|
||||
|
||||
def build_image_pool(args):
|
||||
"""构建测试图池(base64 列表)：
|
||||
--pool DIR 用真实图目录(最贴近实际负载) > --gen N 随机生成 > 单张样例图。"""
|
||||
if getattr(args, "pool", None):
|
||||
files = (glob.glob(os.path.join(args.pool, "**", "*.jpg"), recursive=True) +
|
||||
glob.glob(os.path.join(args.pool, "**", "*.png"), recursive=True))
|
||||
if not files:
|
||||
print(f"目录 {args.pool} 下没有图片")
|
||||
return []
|
||||
random.shuffle(files)
|
||||
cap = args.gen if args.gen else 5000
|
||||
files = files[:cap]
|
||||
print(f"从 {args.pool} 取 {len(files)} 张真实图作为图池")
|
||||
return [img2base64(f) for f in files]
|
||||
if getattr(args, "gen", 0):
|
||||
return gen_random_images(args.gen)
|
||||
sample = find_sample_image(args.image)
|
||||
if not sample:
|
||||
return []
|
||||
print(f"单张样例图: {sample}")
|
||||
return [img2base64(sample)]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("image", nargs="?", default=None)
|
||||
ap.add_argument("--url", action="append", default=None, help="OCR 端点(可多次)")
|
||||
ap.add_argument("--requests", type=int, default=60)
|
||||
ap.add_argument("--concurrency", type=int, default=10)
|
||||
ap.add_argument("--aggregate", action="store_true",
|
||||
help="聚合负载测试:请求轮询分发到所有存活端点，测合并总 QPS")
|
||||
ap.add_argument("--sweep", action="store_true",
|
||||
help="自动加压找极限 QPS(并发逐级翻倍直到 QPS 不再上升)")
|
||||
ap.add_argument("--soak", action="store_true",
|
||||
help="单端点持续满速压测(固定并发不停发，实时滚动QPS，Ctrl+C 停)")
|
||||
ap.add_argument("--interval", type=int, default=5, help="soak 模式打印间隔秒")
|
||||
ap.add_argument("--gen", type=int, default=0,
|
||||
help="随机生成 N 张带文字的图作为图池(每次请求随机抽，避免被缓存)")
|
||||
ap.add_argument("--pool", default=None,
|
||||
help="用该目录下的真实图作为图池(最贴近实际负载，如 ./split/某视频)")
|
||||
ap.add_argument("--start-concurrency", type=int, default=10, help="sweep 起始并发")
|
||||
ap.add_argument("--max-concurrency", type=int, default=320, help="sweep 最大并发")
|
||||
ap.add_argument("--per-worker", type=int, default=6, help="sweep 每档每并发发多少次请求")
|
||||
ap.add_argument("--no-write", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
endpoints = get_endpoints(args.url)
|
||||
if not endpoints:
|
||||
print("没有可用的 OCR 端点")
|
||||
return
|
||||
|
||||
# 构建图池：--pool 真实图目录 / --gen 随机生成 / 默认单张样例
|
||||
image_pool = build_image_pool(args)
|
||||
if not image_pool:
|
||||
print("没有可用图片：给一张 jpg 路径、或用 --gen N / --pool 目录。")
|
||||
return
|
||||
print(f"图池大小: {len(image_pool)} 张 | 端点数: {len(endpoints)} 并发: {args.concurrency}")
|
||||
print("=" * 64)
|
||||
|
||||
image_b64 = image_pool[0] # 非 soak 模式用单张(测纯吞吐)
|
||||
|
||||
# ---- soak 模式：单端点持续满速压测，每次随机抽图，直到 Ctrl+C ----
|
||||
if args.soak:
|
||||
soak_endpoint(endpoints[0], image_pool, args.concurrency, args.interval)
|
||||
return
|
||||
|
||||
# ---- aggregate 模式：所有端点轮询分发，测合并总 QPS ----
|
||||
if args.aggregate:
|
||||
run_aggregate(endpoints, image_b64, args.requests, args.concurrency)
|
||||
return
|
||||
|
||||
# ---- sweep 模式：自动加压找极限，不写权重文件 ----
|
||||
if args.sweep:
|
||||
for i, ep in enumerate(endpoints, 1):
|
||||
print(f"\n[{i}/{len(endpoints)}]", end=" ")
|
||||
sweep_endpoint(ep, image_b64, args.start_concurrency,
|
||||
args.max_concurrency, args.per_worker)
|
||||
return
|
||||
|
||||
results = {}
|
||||
for i, ep in enumerate(endpoints, 1):
|
||||
print(f"[{i}/{len(endpoints)}] 压测 {ep} ...", flush=True)
|
||||
stat = bench_endpoint(ep, image_b64, args.requests, args.concurrency)
|
||||
results[ep] = stat
|
||||
print(f" QPS={stat['qps']:.2f} 成功={stat['ok']} 失败={stat['err']}"
|
||||
f" 平均时延={stat['avg_latency']*1000:.0f}ms"
|
||||
f" 错误率={stat['err_rate']*100:.0f}%")
|
||||
if stat["err"] and stat.get("sample_err"):
|
||||
print(f" 失败原因(采样): {stat['sample_err']}")
|
||||
|
||||
print("=" * 64)
|
||||
print("汇总(按 QPS 降序):")
|
||||
for ep, s in sorted(results.items(), key=lambda kv: kv[1]["qps"], reverse=True):
|
||||
print(f" {ep:48s} QPS={s['qps']:6.2f} 时延={s['avg_latency']*1000:5.0f}ms"
|
||||
f" 错误率={s['err_rate']*100:.0f}%")
|
||||
|
||||
total_qps = sum(s["qps"] for s in results.values())
|
||||
print(f"\n合计可用 QPS ≈ {total_qps:.2f}")
|
||||
|
||||
# 写权重文件：QPS 即权重；错误率过高(>50%)的端点权重清零(不参与分发)
|
||||
weights = {ep: (round(s["qps"], 3) if s["err_rate"] <= 0.5 else 0.0)
|
||||
for ep, s in results.items()}
|
||||
|
||||
if args.no_write:
|
||||
print("\n--no-write：不写权重文件。")
|
||||
return
|
||||
|
||||
os.makedirs(os.path.dirname(WEIGHTS_FILE), exist_ok=True)
|
||||
with open(WEIGHTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(weights, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n已写入权重文件: {WEIGHTS_FILE}")
|
||||
print("运行时会按此权重把 OCR 请求加权分发到各端点（重启进程或新批次生效）。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
117
tools/check_srt.py
Normal file
117
tools/check_srt.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
使用 cos-python-sdk-v5 全量扫描 0004/data/ 下的文件，
|
||||
找出已有 SRT 文件的目录，将对应的 MP4 COS 路径写入 txt 文件。
|
||||
|
||||
安装依赖：pip install cos-python-sdk-v5
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from config.config_utils import config as app_config
|
||||
|
||||
SCAN_PREFIX = "0004/data/"
|
||||
OUTPUT_TXT = "./text/0004_has_srt.txt"
|
||||
DIR_RE = re.compile(r'.+/data/[^/]+/[^/]+/$')
|
||||
|
||||
|
||||
def build_client() -> tuple[CosS3Client, str]:
|
||||
cos_cfg = app_config.get_cos_config()
|
||||
secret_id = cos_cfg.get("secret_id")
|
||||
secret_key = cos_cfg.get("secret_key")
|
||||
region = cos_cfg.get("region", "ap-chongqing")
|
||||
bucket = cos_cfg.get("bucket", "asr-dig-data-1253181912")
|
||||
|
||||
if not secret_id or not secret_key or "填入" in secret_id:
|
||||
raise ValueError("请先在 config/config.json 的 tencentcloud.cos 中填入 secret_id 和 secret_key")
|
||||
|
||||
cfg = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key)
|
||||
client = CosS3Client(cfg)
|
||||
return client, bucket
|
||||
|
||||
|
||||
def list_all_objects(client: CosS3Client, bucket: str, prefix: str) -> list[str]:
|
||||
"""分页获取 prefix 下全部 Key"""
|
||||
keys = []
|
||||
marker = ""
|
||||
page = 0
|
||||
|
||||
while True:
|
||||
page += 1
|
||||
kwargs = dict(Bucket=bucket, Prefix=prefix, MaxKeys=1000)
|
||||
if marker:
|
||||
kwargs["Marker"] = marker
|
||||
|
||||
resp = client.list_objects(**kwargs)
|
||||
contents = resp.get("Contents", [])
|
||||
keys.extend(item["Key"] for item in contents)
|
||||
|
||||
print(f"\r 第 {page} 页，已获取 {len(keys)} 个文件...", end="", flush=True)
|
||||
|
||||
if resp.get("IsTruncated") == "true":
|
||||
marker = resp.get("NextMarker", "")
|
||||
else:
|
||||
break
|
||||
|
||||
print()
|
||||
return keys
|
||||
|
||||
|
||||
def main():
|
||||
client, bucket = build_client()
|
||||
bucket_prefix = f"cos://{bucket}"
|
||||
|
||||
print(f"正在全量扫描 {bucket_prefix}/{SCAN_PREFIX} ...")
|
||||
keys = list_all_objects(client, bucket, SCAN_PREFIX)
|
||||
print(f"共找到 {len(keys)} 个文件")
|
||||
|
||||
if not keys:
|
||||
print("未找到任何文件，请检查密钥和路径是否正确")
|
||||
return
|
||||
|
||||
# 按父目录分组
|
||||
dir_files: dict[str, list[str]] = defaultdict(list)
|
||||
for key in keys:
|
||||
idx = key.rfind("/")
|
||||
dir_path = key[:idx + 1] # e.g. "0004/data/BV1xx/BV1xx/"
|
||||
filename = key[idx + 1:]
|
||||
if filename:
|
||||
dir_files[dir_path].append(filename)
|
||||
|
||||
# 找有 SRT 的目录，收集 MP4 路径
|
||||
mp4_paths = []
|
||||
has_srt_cnt = 0
|
||||
no_srt_cnt = 0
|
||||
|
||||
for dir_path, filenames in sorted(dir_files.items()):
|
||||
# 只处理 data/{id}/{id}/ 结构
|
||||
if not DIR_RE.match(dir_path):
|
||||
continue
|
||||
|
||||
has_srt = any(f.lower().endswith(".srt") for f in filenames)
|
||||
mp4_files = [f for f in filenames if f.lower().endswith(".mp4")]
|
||||
|
||||
if has_srt:
|
||||
has_srt_cnt += 1
|
||||
for mp4 in mp4_files:
|
||||
mp4_paths.append(f"{bucket_prefix}/{dir_path}{mp4}")
|
||||
else:
|
||||
no_srt_cnt += 1
|
||||
|
||||
print(f"有 SRT 的目录: {has_srt_cnt} 个 → {len(mp4_paths)} 个 MP4")
|
||||
print(f"无 SRT 的目录: {no_srt_cnt} 个")
|
||||
|
||||
if not mp4_paths:
|
||||
print("没有找到含 SRT 的目录")
|
||||
return
|
||||
|
||||
os.makedirs("./text", exist_ok=True)
|
||||
with open(OUTPUT_TXT, "w", encoding="utf-8") as f:
|
||||
for path in mp4_paths:
|
||||
f.write(path + "\n")
|
||||
|
||||
print(f"已写入: {OUTPUT_TXT} ({len(mp4_paths)} 条)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
157
tools/compare_jsonl.py
Normal file
157
tools/compare_jsonl.py
Normal file
@@ -0,0 +1,157 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
并排对比两个(或多个)机器的 数据.jsonl，定位谁慢、慢在哪个阶段。
|
||||
|
||||
用法:
|
||||
python temp/compare_jsonl.py 2号机器数据.jsonl 3号机器数据.jsonl
|
||||
python temp/compare_jsonl.py a.jsonl b.jsonl c.jsonl ...
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def parse_dt(s):
|
||||
try:
|
||||
return datetime.fromisoformat(s)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def analyze(path):
|
||||
"""读一个 jsonl，返回汇总指标。"""
|
||||
s = {
|
||||
"n": 0, "ok": 0, "err": 0, "no_sub": 0,
|
||||
"dur": 0.0, "elapsed": 0.0,
|
||||
"stage": {"split_frame": 0.0, "merge": 0.0, "getsrt": 0.0},
|
||||
"ocr": 0, "n_ocr": 0,
|
||||
"starts": [], "ends": [],
|
||||
}
|
||||
try:
|
||||
f = open(path, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
s["n"] += 1
|
||||
st = r.get("status")
|
||||
if st == "success":
|
||||
s["ok"] += 1
|
||||
elif st == "no_subtitle":
|
||||
s["no_sub"] += 1
|
||||
else:
|
||||
s["err"] += 1
|
||||
try:
|
||||
s["dur"] += float(r.get("duration") or 0)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
el = r.get("elapsed_seconds")
|
||||
if el:
|
||||
try:
|
||||
s["elapsed"] += float(el)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
sd = r.get("stage_durations") or {}
|
||||
for k in s["stage"]:
|
||||
try:
|
||||
s["stage"][k] += float(sd.get(k) or 0)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
ic = r.get("image_counts") or {}
|
||||
toc = ic.get("total_ocr_count")
|
||||
if toc:
|
||||
try:
|
||||
s["ocr"] += int(toc)
|
||||
s["n_ocr"] += 1
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
ds = parse_dt(r.get("process_start"))
|
||||
de = parse_dt(r.get("process_end"))
|
||||
if ds:
|
||||
s["starts"].append(ds)
|
||||
if de:
|
||||
s["ends"].append(de)
|
||||
return s
|
||||
|
||||
|
||||
def derive(s):
|
||||
"""从汇总算出对比指标。"""
|
||||
d = {}
|
||||
n = max(1, s["n"])
|
||||
d["记录数"] = s["n"]
|
||||
d["成功率%"] = round(s["ok"] / n * 100, 1)
|
||||
d["视频总时长h"] = round(s["dur"] / 3600, 1)
|
||||
# 墙钟跨度 + 实时倍率
|
||||
span = 0.0
|
||||
if s["starts"] and s["ends"]:
|
||||
span = (max(s["ends"]) - min(s["starts"])).total_seconds()
|
||||
d["墙钟跨度h"] = round(span / 3600, 2)
|
||||
d["实时倍率x"] = round(s["dur"] / span, 2) if span > 0 else 0
|
||||
d["视频/天"] = round(s["n"] / span * 86400) if span > 0 else 0
|
||||
d["有效并发"] = round(s["elapsed"] / span, 1) if span > 0 else 0
|
||||
# 平均每视频各阶段秒数
|
||||
d["抽帧s/视频"] = round(s["stage"]["split_frame"] / n, 1)
|
||||
d["拼接s/视频"] = round(s["stage"]["merge"] / n, 1)
|
||||
d["getsrt s/视频"] = round(s["stage"]["getsrt"] / n, 1)
|
||||
total_stage = sum(s["stage"].values()) or 1
|
||||
d["抽帧占比%"] = round(s["stage"]["split_frame"] / total_stage * 100)
|
||||
d["拼接占比%"] = round(s["stage"]["merge"] / total_stage * 100)
|
||||
d["getsrt占比%"] = round(s["stage"]["getsrt"] / total_stage * 100)
|
||||
d["平均OCR次/视频"] = round(s["ocr"] / s["n_ocr"]) if s["n_ocr"] else 0
|
||||
d["平均耗时s/视频"] = round(s["elapsed"] / n, 1)
|
||||
return d
|
||||
|
||||
|
||||
def main():
|
||||
paths = sys.argv[1:]
|
||||
if len(paths) < 2:
|
||||
print("用法: python temp/compare_jsonl.py a.jsonl b.jsonl [c.jsonl ...]")
|
||||
return
|
||||
|
||||
cols, names = [], []
|
||||
for p in paths:
|
||||
s = analyze(p)
|
||||
if s is None:
|
||||
print(f"找不到文件: {p}")
|
||||
return
|
||||
cols.append(derive(s))
|
||||
names.append(p.replace("数据.jsonl", "").replace(".jsonl", "") or p)
|
||||
|
||||
keys = list(cols[0].keys())
|
||||
w = max(14, max(len(n) for n in names) + 2)
|
||||
print("=" * (18 + w * len(cols)))
|
||||
print(f"{'指标':<16}" + "".join(f"{n:>{w}}" for n in names))
|
||||
print("-" * (18 + w * len(cols)))
|
||||
for k in keys:
|
||||
print(f"{k:<16}" + "".join(f"{str(c[k]):>{w}}" for c in cols))
|
||||
print("=" * (18 + w * len(cols)))
|
||||
|
||||
# 自动点评(两台时)
|
||||
if len(cols) == 2:
|
||||
a, b = cols
|
||||
slow_i = 0 if a["实时倍率x"] < b["实时倍率x"] else 1
|
||||
fast_i = 1 - slow_i
|
||||
print(f"\n【分析】{names[slow_i]} 更慢(实时倍率 "
|
||||
f"{cols[slow_i]['实时倍率x']}x vs {cols[fast_i]['实时倍率x']}x)")
|
||||
# 找慢在哪个阶段
|
||||
for stage, key in [("抽帧", "抽帧s/视频"), ("拼接", "拼接s/视频"), ("getsrt(OCR)", "getsrt s/视频")]:
|
||||
sv, fv = cols[slow_i][key], cols[fast_i][key]
|
||||
if fv and sv > fv * 1.3:
|
||||
print(f" ▸ {stage} 慢: {sv}s vs {fv}s (慢 {sv/fv:.1f}x)")
|
||||
if cols[slow_i]["有效并发"] < cols[fast_i]["有效并发"] * 0.8:
|
||||
print(f" ▸ 并发低: {cols[slow_i]['有效并发']} vs {cols[fast_i]['有效并发']}"
|
||||
f"(CPU/内存/OCR 排队?)")
|
||||
if cols[slow_i]["平均OCR次/视频"] > cols[fast_i]["平均OCR次/视频"] * 1.3:
|
||||
print(f" ▸ OCR 次数多: {cols[slow_i]['平均OCR次/视频']} vs "
|
||||
f"{cols[fast_i]['平均OCR次/视频']}(视频更长/composite更多)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
99
tools/daily_stats.py
Normal file
99
tools/daily_stats.py
Normal file
@@ -0,0 +1,99 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
按天统计 数据.jsonl：每天处理了多少条视频、处理的视频总时长是多少。
|
||||
日期取 process_start（没有则回退 timestamp）的年月日。
|
||||
|
||||
用法（在项目根目录运行）:
|
||||
python temp/daily_stats.py [数据.jsonl路径]
|
||||
默认: python temp/daily_stats.py 数据.jsonl
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
JSONL_PATH = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
f = open(JSONL_PATH, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
# 每天: count/success/no_subtitle/error 计数；dur=全部视频时长；dur_ok=成功视频时长
|
||||
days = defaultdict(lambda: {"count": 0, "success": 0, "no_subtitle": 0,
|
||||
"error": 0, "dur": 0.0, "dur_ok": 0.0})
|
||||
no_date = 0
|
||||
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
ts = rec.get("process_start") or rec.get("timestamp") or ""
|
||||
day = ts[:10] if len(ts) >= 10 else None # YYYY-MM-DD
|
||||
if not day:
|
||||
no_date += 1
|
||||
continue
|
||||
|
||||
d = days[day]
|
||||
d["count"] += 1
|
||||
|
||||
status = rec.get("status")
|
||||
if status == "success":
|
||||
d["success"] += 1
|
||||
elif status == "no_subtitle":
|
||||
d["no_subtitle"] += 1
|
||||
else:
|
||||
d["error"] += 1
|
||||
|
||||
dur = rec.get("duration") or 0
|
||||
try:
|
||||
dur = float(dur)
|
||||
d["dur"] += dur
|
||||
if status == "success":
|
||||
d["dur_ok"] += dur # 只累计成功出字幕的视频时长
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if not days:
|
||||
print("没有可统计的数据")
|
||||
return
|
||||
|
||||
print(f"文件: {JSONL_PATH}")
|
||||
print("=" * 84)
|
||||
print(f"{'日期':<12}{'视频数':>7}{'成功':>7}{'无字幕':>7}{'错误':>7}"
|
||||
f"{'总时长(h)':>12}{'成功时长(h)':>14}")
|
||||
print("-" * 84)
|
||||
|
||||
tot = {"count": 0, "success": 0, "no_subtitle": 0, "error": 0, "dur": 0.0, "dur_ok": 0.0}
|
||||
for day in sorted(days):
|
||||
d = days[day]
|
||||
print(f"{day:<12}{d['count']:>7}{d['success']:>7}{d['no_subtitle']:>7}"
|
||||
f"{d['error']:>7}{d['dur']/3600:>12.1f}{d['dur_ok']/3600:>14.1f}")
|
||||
for k in tot:
|
||||
tot[k] += d[k]
|
||||
|
||||
print("-" * 84)
|
||||
print(f"{'合计':<12}{tot['count']:>7}{tot['success']:>7}{tot['no_subtitle']:>7}"
|
||||
f"{tot['error']:>7}{tot['dur']/3600:>12.1f}{tot['dur_ok']/3600:>14.1f}")
|
||||
if no_date:
|
||||
print(f"\n（{no_date} 条无日期字段，已跳过）")
|
||||
|
||||
n_days = len(days)
|
||||
if n_days:
|
||||
print(f"\n共 {n_days} 天 | 日均处理 {tot['count']/n_days:.0f} 条 / "
|
||||
f"{tot['dur']/3600/n_days:.1f} h | 日均成功 {tot['success']/n_days:.0f} 条 / "
|
||||
f"{tot['dur_ok']/3600/n_days:.1f} h")
|
||||
print(f"成功率 {tot['success']/tot['count']*100:.1f}% | "
|
||||
f"成功时长占比 {tot['dur_ok']/tot['dur']*100:.1f}%" if tot['dur'] else "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
96
tools/download_srt.py
Normal file
96
tools/download_srt.py
Normal file
@@ -0,0 +1,96 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从 数据.jsonl 中读取成功记录，将 MP4 和 SRT 文件下载到本地 ./成功/ 目录。
|
||||
用法：python download_srt.py [数据.jsonl路径] [本地输出目录]
|
||||
默认：python download_srt.py 数据.jsonl ./成功
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from config.config_utils import config
|
||||
|
||||
JSONL_PATH = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||||
OUTPUT_DIR = sys.argv[2] if len(sys.argv) > 2 else "./成功"
|
||||
|
||||
|
||||
def download(coscli_path, cos_config_path, cos_src, local_path):
|
||||
"""下载单个文件，返回是否成功"""
|
||||
if os.path.exists(local_path):
|
||||
return "skip"
|
||||
cmd = [coscli_path, "-c", cos_config_path, "cp", cos_src, local_path]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
return "ok" if result.returncode == 0 else result.stderr.strip()
|
||||
|
||||
|
||||
def main():
|
||||
if not os.path.exists(JSONL_PATH):
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
cfg = config.get_splitframe_config()
|
||||
coscli_path = cfg.get('coscli_path', './coscli-linux-amd64')
|
||||
cos_config_path = cfg.get('cosconfig_path', '.cos.yaml')
|
||||
|
||||
# 读取所有成功记录
|
||||
records = []
|
||||
with open(JSONL_PATH, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
if rec.get('status') == 'success' and rec.get('srt_cos_path'):
|
||||
records.append(rec)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
records = records[:50]
|
||||
total = len(records)
|
||||
print(f"共找到 {total} 条成功记录，开始下载...")
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
|
||||
success = skip = failed = 0
|
||||
|
||||
for i, rec in enumerate(records, 1):
|
||||
video_cos = rec.get('video_cos_path').replace(".mp4", ".mp3")
|
||||
srt_cos = rec.get('srt_cos_path')
|
||||
|
||||
video_name = os.path.splitext(os.path.basename(video_cos))[0] if video_cos else None
|
||||
|
||||
print(f"[{i}/{total}] {video_name}")
|
||||
|
||||
# 下载 MP4
|
||||
if video_cos:
|
||||
local_mp4 = os.path.join(OUTPUT_DIR, os.path.basename(video_cos))
|
||||
ret = download(coscli_path, cos_config_path, video_cos, local_mp4)
|
||||
if ret == "ok":
|
||||
success += 1
|
||||
print(f" ✓ 视频: {os.path.basename(video_cos)}")
|
||||
elif ret == "skip":
|
||||
skip += 1
|
||||
print(f" - 视频已存在，跳过")
|
||||
else:
|
||||
failed += 1
|
||||
print(f" ✗ 视频下载失败: {ret}")
|
||||
|
||||
# 下载 SRT
|
||||
if srt_cos:
|
||||
local_srt = os.path.join(OUTPUT_DIR, os.path.basename(srt_cos))
|
||||
ret = download(coscli_path, cos_config_path, srt_cos, local_srt)
|
||||
if ret == "ok":
|
||||
success += 1
|
||||
print(f" ✓ 字幕: {os.path.basename(srt_cos)}")
|
||||
elif ret == "skip":
|
||||
skip += 1
|
||||
print(f" - 字幕已存在，跳过")
|
||||
else:
|
||||
failed += 1
|
||||
print(f" ✗ 字幕下载失败: {ret}")
|
||||
|
||||
print(f"\n完成 — 下载: {success} 失败: {failed} 跳过(已存在): {skip}")
|
||||
print(f"文件保存在: {os.path.abspath(OUTPUT_DIR)}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
69
tools/extract_errors.py
Normal file
69
tools/extract_errors.py
Normal file
@@ -0,0 +1,69 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从 数据.jsonl 提取【最终状态是错误/无字幕】的视频 COS 路径，写到 txt(可当任务列表重跑测试)。
|
||||
按每个视频的【最后一条记录】判定最终状态(忽略中间重试)。
|
||||
|
||||
用法:
|
||||
python temp/extract_errors.py 2号机器数据.jsonl error_videos.txt
|
||||
python temp/extract_errors.py 2号机器数据.jsonl out.txt error,no_subtitle # 也要无字幕的
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
JSONL = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||||
OUT = sys.argv[2] if len(sys.argv) > 2 else "error_videos.txt"
|
||||
STATUSES = sys.argv[3].split(",") if len(sys.argv) > 3 else ["error"]
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
f = open(JSONL, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL}")
|
||||
return
|
||||
|
||||
latest = {} # video_cos_path -> (status, error_reason)
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
vid = r.get("video_cos_path")
|
||||
if not vid:
|
||||
continue
|
||||
latest[vid] = (r.get("status"), r.get("error_reason", ""))
|
||||
|
||||
# 最终状态计数
|
||||
status_count = Counter(s for s, _ in latest.values())
|
||||
# 选中的(最终状态在 STATUSES 里)
|
||||
selected = [(v, er) for v, (s, er) in latest.items() if s in STATUSES]
|
||||
|
||||
# 写 txt(每行一个 COS 路径)
|
||||
with open(OUT, "w", encoding="utf-8") as w:
|
||||
for v, _ in selected:
|
||||
w.write(v + "\n")
|
||||
|
||||
print(f"文件: {JSONL}")
|
||||
print(f"去重后视频数: {len(latest)}")
|
||||
print(f"最终状态分布: {dict(status_count)}")
|
||||
print(f"提取状态 {STATUSES}: {len(selected)} 个 → 写入 {OUT}")
|
||||
|
||||
# 错误原因 Top（帮你看主要错在哪）
|
||||
reasons = Counter()
|
||||
for _, er in selected:
|
||||
if er:
|
||||
key = er.split(":")[0][:60] if ":" in er else er[:60]
|
||||
reasons[key] += 1
|
||||
if reasons:
|
||||
print("\n错误原因 Top:")
|
||||
for k, c in reasons.most_common(10):
|
||||
print(f" {c:>5} {k}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
316
tools/fleet_monitor.py
Normal file
316
tools/fleet_monitor.py
Normal file
@@ -0,0 +1,316 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""全队健康度监控: 汇总所有机器最新一小时的处理摘要, 跑异常检测, 产出报告。
|
||||
配合 run_worker.py 里的 job_cos.run_hourly_digest_loop() 后台线程(每台机器每小时自动上传
|
||||
一份小摘要到 COS)使用 —— 本脚本负责把这些小摘要拉回来汇总判断。
|
||||
|
||||
用法(项目根目录运行, 建议 crontab 每小时跑一次，比如 5分/10分 触发一次都行，脚本自己会
|
||||
算准要看哪个整点):
|
||||
python tools/fleet_monitor.py # 单小时报告 + 异常检测(原有行为)
|
||||
python tools/fleet_monitor.py --history 24 # 全队历史趋势(最近24个小时, 不跑异常检测)
|
||||
python tools/fleet_monitor.py --history # 不带数字 = 默认 24 小时
|
||||
|
||||
设计文档: docs/superpowers/specs/2026-07-08-fleet-monitoring-design.md
|
||||
不接外部通知渠道(企业微信/钉钉/飞书webhook 是后续可插拔扩展点, 本次只落地本地+COS报告文件)。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unicodedata
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from tool import job_cos # noqa: E402
|
||||
|
||||
# ---- 异常检测阈值(先写死成常量, 要调直接改这里) ----
|
||||
OFFLINE_HOURS = 2 # 机器掉线: 最新摘要距目标小时 >= 这么多小时
|
||||
SUCCESS_RATE_MIN = 0.5 # 单小时成功率骤降阈值
|
||||
SUCCESS_RATE_MIN_SAMPLE = 10 # 样本太少(成功+错误 < 这个数)不判断成功率, 避免噪音
|
||||
MACHINE_DROP_RATIO = 0.3 # 单机吞吐骤降: 本小时产出 < 自己24h均值的这个比例
|
||||
FLEET_DROP_RATIO = 0.5 # 全队吞吐骤降: 全队本小时产出 < 全队24h均值的这个比例
|
||||
BASELINE_HOURS = 24 # 基线取多少小时的历史均值(不含目标小时本身)
|
||||
KNOWN_MACHINES_LOOKBACK = 48 # "这台机器算不算存在"往回看多少小时
|
||||
|
||||
REPORT_DIR = "./monitor_reports"
|
||||
|
||||
DEFAULT_HISTORY_HOURS = 24 # --history 不带数字时默认看多少小时
|
||||
|
||||
|
||||
def _dw(s):
|
||||
"""字符串显示宽度(中文/全角算2), 用于历史趋势表格对齐。"""
|
||||
return sum(2 if unicodedata.east_asian_width(c) in ("W", "F") else 1 for c in str(s))
|
||||
|
||||
|
||||
def _padl(s, w):
|
||||
"""按显示宽度左对齐(文字在前, 空格补在后面) —— 跟 hourly_stats.py 的同名函数保持一致约定。"""
|
||||
s = str(s)
|
||||
return s + " " * max(0, w - _dw(s))
|
||||
|
||||
|
||||
def _padr(s, w):
|
||||
"""按显示宽度右对齐(空格补在前面, 文字贴右边)。"""
|
||||
return " " * max(0, w - _dw(s)) + str(s)
|
||||
|
||||
|
||||
def target_hour(now=None) -> datetime:
|
||||
"""确定要汇总的目标整点小时: 最新一个已经过完的整点; 如果当前分钟数<5(刚跨过整点没多久,
|
||||
部分机器可能还没上传完), 再往回退一个小时留缓冲。"""
|
||||
now = now or datetime.now()
|
||||
completed = now.replace(minute=0, second=0, microsecond=0) - timedelta(hours=1)
|
||||
if now.minute < 5:
|
||||
completed -= timedelta(hours=1)
|
||||
return completed
|
||||
|
||||
|
||||
def fetch_range_digests(end_hour_dt: datetime, hours_back: int) -> dict:
|
||||
"""拉取 [end_hour_dt - hours_back, end_hour_dt] 闭区间(按小时)所有机器的摘要,
|
||||
返回 {hour_key(YYYYMMDDHH): {machine_id: digest_dict} | None}。
|
||||
区分两种"没数据": 该小时【列COS目录本身失败】(比如COS暂时不可达)记为 None——这是需要
|
||||
main() 判断是否整体报错退出的信号; 该小时【列成功但目录是空的】(单纯没人上报)记为 {}——
|
||||
这是正常情况, 不代表故障。单台机器的摘要【下载】失败只跳过该机器、打印警告, 不影响这个
|
||||
小时其它机器的数据, 也不算列目录失败。"""
|
||||
result = {}
|
||||
for i in range(hours_back + 1):
|
||||
hdt = end_hour_dt - timedelta(hours=i)
|
||||
hkey = hdt.strftime("%Y%m%d%H")
|
||||
try:
|
||||
files = job_cos.list_hourly_digest_files_for_hour(hdt)
|
||||
except Exception as e:
|
||||
print(f"[WARN] 列出 {hkey} 的摘要文件失败: {e}")
|
||||
result[hkey] = None
|
||||
continue
|
||||
machines = {}
|
||||
for machine_id, key in files:
|
||||
try:
|
||||
machines[machine_id] = job_cos.download_hourly_digest(key)
|
||||
except Exception as e:
|
||||
print(f"[WARN] 下载 {machine_id} 在 {hkey} 的摘要失败, 跳过: {e}")
|
||||
result[hkey] = machines
|
||||
return result
|
||||
|
||||
|
||||
def compute_baseline(range_digests: dict, target_hdt: datetime) -> dict:
|
||||
"""算每台机器过去 BASELINE_HOURS 小时(不含目标小时本身)的 success_hours 均值。
|
||||
只用真实拉到的小时数据平均(缺的小时/列目录失败的小时都不计入, 不当0算, 避免拖低均值),
|
||||
没有任何数据的机器不出现在结果里。"""
|
||||
sums = {}
|
||||
counts = {}
|
||||
for i in range(1, BASELINE_HOURS + 1):
|
||||
hkey = (target_hdt - timedelta(hours=i)).strftime("%Y%m%d%H")
|
||||
for machine_id, digest in (range_digests.get(hkey) or {}).items():
|
||||
sums[machine_id] = sums.get(machine_id, 0.0) + digest.get("success_hours", 0.0)
|
||||
counts[machine_id] = counts.get(machine_id, 0) + 1
|
||||
return {m: sums[m] / counts[m] for m in sums}
|
||||
|
||||
|
||||
def known_machines_and_latest_hour(range_digests: dict) -> dict:
|
||||
"""扫全部拉到的小时, 返回 {machine_id: 该机器出现过的最新 hour_key}。列目录失败(None)的
|
||||
小时直接跳过, 不当"没有任何机器"处理(避免把偶发的单小时COS故障误判成全队一起掉线)。"""
|
||||
latest = {}
|
||||
for hkey, machines in range_digests.items():
|
||||
if machines is None:
|
||||
continue
|
||||
for machine_id in machines:
|
||||
if machine_id not in latest or hkey > latest[machine_id]:
|
||||
latest[machine_id] = hkey
|
||||
return latest
|
||||
|
||||
|
||||
def aggregate_history(range_digests: dict, hours_back: int, end_hdt: datetime) -> list:
|
||||
"""把 fetch_range_digests() 拉到的多小时摘要,按小时汇总成全队合计一行,按时间正序(从早到晚)
|
||||
返回 [(hour_key, machine_count, count, success, no_subtitle, error, success_hours, ok), ...]。
|
||||
ok=False 表示这个小时列COS目录本身失败(range_digests[hkey] is None), 其余字段填0/0.0;
|
||||
ok=True 且 machine_count=0 是"列目录成功但没人上报"的正常空窗口(比如还没到那个整点),
|
||||
跟 ok=False(拉取本身失败)是两码事,不能混在一起看成同一种"没数据"。"""
|
||||
rows = []
|
||||
for i in range(hours_back, -1, -1):
|
||||
hdt = end_hdt - timedelta(hours=i)
|
||||
hkey = hdt.strftime("%Y%m%d%H")
|
||||
machines = range_digests.get(hkey)
|
||||
if machines is None:
|
||||
rows.append((hkey, 0, 0, 0, 0, 0, 0.0, False))
|
||||
continue
|
||||
count = sum(d.get("count", 0) for d in machines.values())
|
||||
success = sum(d.get("success", 0) for d in machines.values())
|
||||
no_sub = sum(d.get("no_subtitle", 0) for d in machines.values())
|
||||
error = sum(d.get("error", 0) for d in machines.values())
|
||||
succ_hours = sum(d.get("success_hours", 0.0) for d in machines.values())
|
||||
rows.append((hkey, len(machines), count, success, no_sub, error, succ_hours, True))
|
||||
return rows
|
||||
|
||||
|
||||
def print_history(hours_back: int):
|
||||
"""打印全队历史多个小时的汇总趋势(拉COS各机器已上传的小时摘要, 不依赖本机 数据.jsonl,
|
||||
只要各机器在跑带小时摘要上传功能的新版 run_worker.py 就能看到)。跟 tools/hourly_stats.py
|
||||
的单机版本是同一个"按process_start所在小时分桶计数"的逻辑, 只是数据源从本机 数据.jsonl
|
||||
换成了COS上各机器上传的小时摘要, 且是全队加总, 不是单机。"""
|
||||
thdt = target_hour()
|
||||
print(f"目标小时: {thdt.strftime('%Y-%m-%d %H:00')} (往前拉 {hours_back} 小时, 共 {hours_back+1} 个小时)")
|
||||
range_digests = fetch_range_digests(thdt, hours_back)
|
||||
rows = aggregate_history(range_digests, hours_back, thdt)
|
||||
|
||||
# 每列宽度都比内容本身多留至少2个显示宽度的间隔, 避免"小时"这种定长字符串(如
|
||||
# "2026-07-09 13:00"共16字符)紧贴着下一列数字、看起来像连在一起。
|
||||
W = {"小时": 18, "机器数": 8, "视频数": 8, "成功": 6, "无字幕": 8, "错误": 6, "产出小时数": 12}
|
||||
header = (_padl("小时", W["小时"]) + _padr("机器数", W["机器数"]) + _padr("视频数", W["视频数"]) +
|
||||
_padr("成功", W["成功"]) + _padr("无字幕", W["无字幕"]) + _padr("错误", W["错误"]) +
|
||||
_padr("产出小时数", W["产出小时数"]))
|
||||
width = sum(W.values())
|
||||
print("=" * width)
|
||||
print(header)
|
||||
print("-" * width)
|
||||
|
||||
tot = {"count": 0, "success": 0, "no_subtitle": 0, "error": 0, "succ_hours": 0.0}
|
||||
fail_hours = []
|
||||
for hkey, mcount, count, success, no_sub, error, succ_hours, ok in rows:
|
||||
disp_hour = f"{hkey[:4]}-{hkey[4:6]}-{hkey[6:8]} {hkey[8:10]}:00"
|
||||
if not ok:
|
||||
print(_padl(disp_hour, W["小时"]) + _padr("(拉取失败)", width - W["小时"]))
|
||||
fail_hours.append(hkey)
|
||||
continue
|
||||
print(_padl(disp_hour, W["小时"]) + _padr(mcount, W["机器数"]) + _padr(count, W["视频数"]) +
|
||||
_padr(success, W["成功"]) + _padr(no_sub, W["无字幕"]) + _padr(error, W["错误"]) +
|
||||
_padr(f"{succ_hours:.1f}", W["产出小时数"]))
|
||||
tot["count"] += count
|
||||
tot["success"] += success
|
||||
tot["no_subtitle"] += no_sub
|
||||
tot["error"] += error
|
||||
tot["succ_hours"] += succ_hours
|
||||
|
||||
print("-" * width)
|
||||
nh = len(rows) - len(fail_hours)
|
||||
if nh > 0:
|
||||
print(_padl("合计/时均", W["小时"]) + _padr(tot["count"], W["视频数"] + W["机器数"]) +
|
||||
_padr(tot["success"], W["成功"]) + _padr(tot["no_subtitle"], W["无字幕"]) +
|
||||
_padr(tot["error"], W["错误"]) + _padr(f"{tot['succ_hours']:.1f}", W["产出小时数"]))
|
||||
print(f"\n共 {nh} 个有效小时 | 时均处理 {tot['count']/nh:.0f} 条 | "
|
||||
f"时均产出 {tot['succ_hours']/nh:.1f} 小时视频内容")
|
||||
if fail_hours:
|
||||
print(f"\n({len(fail_hours)} 个小时COS拉取失败, 已跳过: {', '.join(fail_hours)})")
|
||||
|
||||
|
||||
def detect_anomalies(target_hdt: datetime, target_digests: dict,
|
||||
baseline: dict, latest_hour: dict) -> list:
|
||||
"""跑4条异常检测规则, 返回 [(level, message), ...]，level ∈ {"严重","警告"}。"""
|
||||
anomalies = []
|
||||
|
||||
# ① 机器掉线: 最新一次上报距目标小时 >= OFFLINE_HOURS
|
||||
for machine_id, last_hkey in latest_hour.items():
|
||||
last_hdt = datetime.strptime(last_hkey, "%Y%m%d%H")
|
||||
gap_hours = (target_hdt - last_hdt).total_seconds() / 3600
|
||||
if gap_hours >= OFFLINE_HOURS:
|
||||
anomalies.append(("严重", f"机器 {machine_id} 已连续{gap_hours:.0f}小时无上报数据，可能停机"))
|
||||
|
||||
# ② 单小时成功率骤降
|
||||
for machine_id, digest in target_digests.items():
|
||||
succ, err = digest.get("success", 0), digest.get("error", 0)
|
||||
sample = succ + err
|
||||
if sample >= SUCCESS_RATE_MIN_SAMPLE:
|
||||
rate = succ / sample
|
||||
if rate < SUCCESS_RATE_MIN:
|
||||
anomalies.append(("警告", f"机器 {machine_id} 本小时成功率仅{rate*100:.0f}%({succ}/{sample})"))
|
||||
|
||||
# ③ 单机吞吐骤降
|
||||
for machine_id, digest in target_digests.items():
|
||||
base = baseline.get(machine_id)
|
||||
if base and base > 0:
|
||||
cur = digest.get("success_hours", 0.0)
|
||||
if cur < base * MACHINE_DROP_RATIO:
|
||||
anomalies.append(("警告", f"机器 {machine_id} 本小时产出{cur:.1f}小时，远低于24h均值{base:.1f}小时"))
|
||||
|
||||
# ④ 全队吞吐骤降
|
||||
fleet_cur = sum(d.get("success_hours", 0.0) for d in target_digests.values())
|
||||
fleet_base = sum(baseline.values())
|
||||
if fleet_base > 0 and fleet_cur < fleet_base * FLEET_DROP_RATIO:
|
||||
anomalies.append(("严重", f"全队本小时总产出{fleet_cur:.1f}小时，远低于24h均值{fleet_base:.1f}小时，可能是全局性问题(比如OCR端点故障)"))
|
||||
|
||||
return anomalies
|
||||
|
||||
|
||||
def render_report(target_hdt: datetime, target_digests: dict,
|
||||
baseline: dict, anomalies: list) -> str:
|
||||
"""生成 Markdown 报告文本。"""
|
||||
lines = [f"# 全队监控报告 {target_hdt.strftime('%Y-%m-%d %H:00')}", ""]
|
||||
lines.append("## 异常告警")
|
||||
if anomalies:
|
||||
for level, msg in anomalies:
|
||||
icon = "🔴" if level == "严重" else "🟡"
|
||||
lines.append(f"- {icon} [{level}] {msg}")
|
||||
else:
|
||||
lines.append("✅ 本小时一切正常")
|
||||
lines.append("")
|
||||
lines.append("## 本小时明细")
|
||||
lines.append("| 机器 | 视频数 | 成功 | 无字幕 | 错误 | 产出小时数 |")
|
||||
lines.append("|---|---|---|---|---|---|")
|
||||
for machine_id in sorted(target_digests):
|
||||
d = target_digests[machine_id]
|
||||
lines.append(f"| {machine_id} | {d.get('count',0)} | {d.get('success',0)} | "
|
||||
f"{d.get('no_subtitle',0)} | {d.get('error',0)} | {d.get('success_hours',0):.1f} |")
|
||||
lines.append("")
|
||||
lines.append("## 过去24小时全队趋势")
|
||||
fleet_cur = sum(d.get("success_hours", 0.0) for d in target_digests.values())
|
||||
fleet_base = sum(baseline.values())
|
||||
lines.append(f"全队本小时产出小时数总和：{fleet_cur:.1f}，24小时均值：{fleet_base:.1f}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _parse_history_arg(argv):
|
||||
"""解析 --history [N] —— 带上就返回要看多少小时(不带数字用默认值), 没带这个参数返回 None
|
||||
(跑原来的单小时报告模式)。"""
|
||||
if "--history" not in argv:
|
||||
return None
|
||||
i = argv.index("--history")
|
||||
if i + 1 < len(argv):
|
||||
try:
|
||||
return int(argv[i + 1])
|
||||
except ValueError:
|
||||
pass
|
||||
return DEFAULT_HISTORY_HOURS
|
||||
|
||||
|
||||
def main():
|
||||
hours_back = _parse_history_arg(sys.argv[1:])
|
||||
if hours_back is not None:
|
||||
print_history(hours_back)
|
||||
return
|
||||
|
||||
thdt = target_hour()
|
||||
print(f"目标小时: {thdt.strftime('%Y-%m-%d %H:00')}")
|
||||
|
||||
range_digests = fetch_range_digests(thdt, KNOWN_MACHINES_LOOKBACK)
|
||||
target_hkey = thdt.strftime("%Y%m%d%H")
|
||||
target_slot = range_digests.get(target_hkey)
|
||||
if target_slot is None:
|
||||
# 目标小时本身列目录都失败了(COS大概率整体不可达) -> 按设计文档"错误处理"要求直接报错退出,
|
||||
# 不生成一份看起来正常、实际数据全空的假报告掩盖问题。
|
||||
print(f"[FAIL] 目标小时 {target_hkey} 的摘要目录列取失败(COS可能不可达), 中止, 不生成报告")
|
||||
sys.exit(1)
|
||||
target_digests = target_slot
|
||||
if not target_digests:
|
||||
print(f"[WARN] 目标小时 {target_hkey} 没有拉到任何机器的摘要, 可能还没到点(正常空目录, 不是故障)")
|
||||
|
||||
baseline = compute_baseline(range_digests, thdt)
|
||||
latest_hour = known_machines_and_latest_hour(range_digests)
|
||||
anomalies = detect_anomalies(thdt, target_digests, baseline, latest_hour)
|
||||
|
||||
report = render_report(thdt, target_digests, baseline, anomalies)
|
||||
|
||||
os.makedirs(REPORT_DIR, exist_ok=True)
|
||||
local_path = os.path.join(REPORT_DIR, f"{thdt.strftime('%Y%m%d_%H')}.md")
|
||||
with open(local_path, "w", encoding="utf-8") as f:
|
||||
f.write(report)
|
||||
print(f"[OK] 报告已写入 {local_path}")
|
||||
|
||||
try:
|
||||
dst = job_cos.upload_monitor_report(local_path, thdt)
|
||||
if dst:
|
||||
print(f"[OK] 报告已上传 -> {dst}")
|
||||
except Exception as e:
|
||||
print(f"[WARN] 报告上传COS失败(本地文件已保留): {e}")
|
||||
|
||||
if anomalies:
|
||||
print(f"\n共检测到 {len(anomalies)} 条异常，详见报告。")
|
||||
else:
|
||||
print("\n本小时一切正常。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
47
tools/gen_test_srt.py
Normal file
47
tools/gen_test_srt.py
Normal file
@@ -0,0 +1,47 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""用真实OCR数据生成一份【已过滤水印】的测试SRT。跳过读图(monkeypatch)。
|
||||
用法: python temp/gen_test_srt.py <composite目录> <输出srt>
|
||||
"""
|
||||
import sys, os, json
|
||||
sys.path.insert(0, os.path.abspath("."))
|
||||
import tool.getsrt as G
|
||||
from tool.log.log_manager import UnifiedLogger
|
||||
|
||||
G.find_best_ocr_match = lambda *a, **k: (None, 0) # 跳过首尾音频匹配的读图
|
||||
|
||||
D = sys.argv[1] if len(sys.argv) > 1 else "test_srt/BV1BamdYLExJ"
|
||||
OUT = sys.argv[2] if len(sys.argv) > 2 else "test_srt/BV1BamdYLExJ.filtered.srt"
|
||||
META = os.path.join(D, "metadata.json")
|
||||
logger = UnifiedLogger(); logger.reset(); logger.init_logging("rerun_log.json"); logger.start_program("rerun")
|
||||
|
||||
pairs = []
|
||||
lines = open(os.path.join(D, "ocr_result.jsonl"), encoding="utf-8").readlines()
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
if i + 1 < len(lines) and lines[i].startswith("pic_path: "):
|
||||
fn = os.path.basename(lines[i].strip()[len("pic_path: "):])
|
||||
try:
|
||||
pairs.append([os.path.join(D, fn), json.loads(lines[i + 1])]); i += 2; continue
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
i += 1
|
||||
|
||||
# 坐标水印过滤(就是接进 getsrt 的那个)
|
||||
G._filter_watermark_by_position(pairs, logger)
|
||||
|
||||
best = G.analyze_subtitle_blocks(pairs, META, logger)
|
||||
h = G.analyze_h(pairs, META)
|
||||
results = []
|
||||
for idx in range(len(pairs)):
|
||||
try:
|
||||
text, conf, flag, sim = G.extract_highest_subtitle(idx, pairs, best, META, h, logger)
|
||||
except Exception:
|
||||
text = ""
|
||||
if text and not G.check_subtitle_quality_enhanced(text) and conf <= 90:
|
||||
text = ""
|
||||
results.append((idx, text or ""))
|
||||
# 静态文字过滤(频率, 兜底)
|
||||
results = G._filter_static_text(results, logger)
|
||||
G.generate_srt_from_results(results, OUT, interval_seconds=0.33)
|
||||
n = sum(1 for l in open(OUT, encoding="utf-8") if "-->" in l)
|
||||
print("生成: %s 共 %d 条字幕" % (OUT, n))
|
||||
174
tools/hourly_stats.py
Normal file
174
tools/hourly_stats.py
Normal file
@@ -0,0 +1,174 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
按【小时】统计 数据.jsonl：每小时处理多少条视频、视频总时长、成功情况、处理倍速。
|
||||
小时取 process_start（没有则回退 timestamp）的 YYYY-MM-DD HH。
|
||||
|
||||
"处理倍速" = 该小时内处理掉的【视频内容总时长】 / 1 墙钟小时。
|
||||
例: 一个墙钟小时里处理了 40 小时的视频内容 → 倍速 40x → 等效有效并发≈40。
|
||||
这个数最能反映真实吞吐/并发利用率(对照 max-workers 看是否吃满)。
|
||||
|
||||
用法(项目根目录运行):
|
||||
python temp/hourly_stats.py [数据.jsonl路径] [--last N]
|
||||
--last N : 只看最近 N 个有数据的小时(默认全部)
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import unicodedata
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def _parse(ts):
|
||||
try:
|
||||
return datetime.fromisoformat(ts)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _dw(s):
|
||||
"""字符串显示宽度(中文/全角算2)。"""
|
||||
return sum(2 if unicodedata.east_asian_width(c) in ("W", "F") else 1 for c in str(s))
|
||||
|
||||
|
||||
def _padr(s, w):
|
||||
"""按显示宽度右对齐。"""
|
||||
s = str(s)
|
||||
return " " * max(0, w - _dw(s)) + s
|
||||
|
||||
|
||||
def _padl(s, w):
|
||||
"""按显示宽度左对齐。"""
|
||||
s = str(s)
|
||||
return s + " " * max(0, w - _dw(s))
|
||||
|
||||
|
||||
# 列宽(显示宽度)
|
||||
COLW = {"小时": 16, "视频数": 8, "成功": 6, "无字幕": 8, "错误": 6,
|
||||
"总时长h": 10, "成功h": 9, "处理倍速": 10, "有效并发": 10}
|
||||
|
||||
|
||||
def _row(hour, count, success, no_sub, error, dur_h, ok_h, speed, eff):
|
||||
return (_padl(hour, COLW["小时"]) + _padr(count, COLW["视频数"]) +
|
||||
_padr(success, COLW["成功"]) + _padr(no_sub, COLW["无字幕"]) +
|
||||
_padr(error, COLW["错误"]) + _padr(dur_h, COLW["总时长h"]) +
|
||||
_padr(ok_h, COLW["成功h"]) + _padr(speed, COLW["处理倍速"]) +
|
||||
_padr(eff, COLW["有效并发"]))
|
||||
|
||||
args = [a for a in sys.argv[1:]]
|
||||
last_n = None
|
||||
if "--last" in args:
|
||||
i = args.index("--last")
|
||||
try:
|
||||
last_n = int(args[i + 1])
|
||||
del args[i:i + 2]
|
||||
except (IndexError, ValueError):
|
||||
last_n = None
|
||||
JSONL_PATH = args[0] if args else "数据.jsonl"
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
f = open(JSONL_PATH, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
hours = defaultdict(lambda: {"count": 0, "success": 0, "no_subtitle": 0,
|
||||
"error": 0, "dur": 0.0, "dur_ok": 0.0, "elapsed": 0.0})
|
||||
busy = defaultdict(float) # 每个墙钟小时内, 所有视频实际占用的总秒数(精确并发用)
|
||||
no_date = 0
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
ts = rec.get("process_start") or rec.get("timestamp") or ""
|
||||
hour = ts[:13] if len(ts) >= 13 else None # YYYY-MM-DD HH(按开始时间归桶)
|
||||
if not hour:
|
||||
no_date += 1
|
||||
continue
|
||||
|
||||
h = hours[hour]
|
||||
h["count"] += 1
|
||||
status = rec.get("status")
|
||||
if status == "success":
|
||||
h["success"] += 1
|
||||
elif status == "no_subtitle":
|
||||
h["no_subtitle"] += 1
|
||||
else:
|
||||
h["error"] += 1
|
||||
|
||||
try:
|
||||
dur = float(rec.get("duration") or 0)
|
||||
h["dur"] += dur
|
||||
if status == "success":
|
||||
h["dur_ok"] += dur
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
try:
|
||||
h["elapsed"] += float(rec.get("elapsed_seconds") or 0)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# 精确并发: 把 [process_start, process_end] 的时长按实际跨越的每个小时切开累加,
|
||||
# 避免"长视频整段算进开始那个小时"导致失真。
|
||||
st = _parse(rec.get("process_start"))
|
||||
en = _parse(rec.get("process_end"))
|
||||
if st and en and en > st:
|
||||
cur = st
|
||||
while cur < en:
|
||||
hk = cur.strftime("%Y-%m-%dT%H")
|
||||
nxt = cur.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
|
||||
seg_end = min(en, nxt)
|
||||
busy[hk] += (seg_end - cur).total_seconds()
|
||||
cur = seg_end
|
||||
|
||||
if not hours:
|
||||
print("没有可统计的数据")
|
||||
return
|
||||
|
||||
keys = sorted(hours)
|
||||
if last_n:
|
||||
keys = keys[-last_n:]
|
||||
|
||||
width = sum(COLW.values())
|
||||
print(f"文件: {JSONL_PATH}")
|
||||
print("=" * width)
|
||||
print(_row("小时", "视频数", "成功", "无字幕", "错误", "总时长h", "成功h", "处理倍速", "有效并发"))
|
||||
print("-" * width)
|
||||
|
||||
tot = {"count": 0, "success": 0, "no_subtitle": 0, "error": 0,
|
||||
"dur": 0.0, "dur_ok": 0.0, "elapsed": 0.0}
|
||||
busy_sum = 0.0
|
||||
for hour in keys:
|
||||
h = hours[hour]
|
||||
speed = h["dur"] / 3600 # 处理倍速: 视频内容小时数 / 1墙钟小时
|
||||
eff = busy[hour] / 3600 # 有效并发: 该小时所有视频实际占用秒数 / 3600(=平均并发)
|
||||
busy_sum += busy[hour]
|
||||
print(_row(hour, h["count"], h["success"], h["no_subtitle"], h["error"],
|
||||
f"{h['dur']/3600:.1f}", f"{h['dur_ok']/3600:.1f}",
|
||||
f"{speed:.1f}x", f"{eff:.1f}"))
|
||||
for k in tot:
|
||||
tot[k] += h[k]
|
||||
|
||||
print("-" * width)
|
||||
nh = len(keys)
|
||||
print(_row("合计/均值", tot["count"], tot["success"], tot["no_subtitle"], tot["error"],
|
||||
f"{tot['dur']/3600:.1f}", f"{tot['dur_ok']/3600:.1f}",
|
||||
f"{tot['dur']/3600/nh:.1f}x", f"{busy_sum/3600/nh:.1f}"))
|
||||
if no_date:
|
||||
print(f"\n（{no_date} 条无时间字段，已跳过）")
|
||||
print(f"\n共 {nh} 个小时 | 时均处理 {tot['count']/nh:.0f} 条 | "
|
||||
f"时均成功 {tot['success']/nh:.0f} 条")
|
||||
if tot["count"]:
|
||||
print(f"成功率 {tot['success']/tot['count']*100:.1f}%")
|
||||
print("提示: '有效并发'明显小于你设的 max-workers → 有串行瓶颈(GIL/锁/等待),不是并发不够。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
142
tools/patch_config.py
Normal file
142
tools/patch_config.py
Normal file
@@ -0,0 +1,142 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""在【项目根目录】运行: 把本次更新需要的 config.json 配置键就地改好。
|
||||
只动下面列出的键, 绝不碰你的密钥/COS路径/其它已调过的值。幂等, 可重复运行。
|
||||
|
||||
用法: python3 tools/patch_config.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
P = "config/config.json"
|
||||
if not os.path.exists(P):
|
||||
print("✗ 找不到 config/config.json, 请 cd 到项目根目录再运行")
|
||||
sys.exit(1)
|
||||
|
||||
d = json.load(open(P, encoding="utf-8"))
|
||||
sf = d.setdefault("splitframe", {})
|
||||
sub = d.setdefault("subtitle", {})
|
||||
ocr = d.setdefault("tencentcloud", {}).setdefault("ocr", {})
|
||||
dp = d.setdefault("dispatch", {})
|
||||
|
||||
changes = []
|
||||
|
||||
# -1) COS 任务分发(常驻 worker 认领任务 + 每日 数据.jsonl 备份)。缺失才加, 不动已有值。
|
||||
# jobs_root=取任务地址; daily_backup_path=数据.jsonl 备份位置; machine_id 留空=用主机名/环境变量。
|
||||
_bkt = (d.get("tencentcloud", {}).get("cos", {}) or {}).get("bucket", "asr-dig-data-1253181912")
|
||||
_dispatch_defaults = [
|
||||
("enabled", True),
|
||||
("source", "cos"), # cos=从COS认领 / local=只读本地 ./text2(启动脚本据此选模式)
|
||||
("run_long_video", False), # true=本机专跑超长队列(long_list_backup_path)
|
||||
("auto_fallback_long_video", True), # false=主队列空闲时也不兼跑长视频队列,只等主队列
|
||||
("global_dedup_on_startup", True), # false=启动时不拉全队去重清单,只按本机 数据.jsonl 去重
|
||||
("machine_id", ""),
|
||||
("jobs_root", "cos://%s/_jobs" % _bkt),
|
||||
("shard_size", 2000),
|
||||
("keep_alive", True),
|
||||
("idle_poll_seconds", 60),
|
||||
("claim_shuffle", True),
|
||||
("claim_match", ""),
|
||||
("heartbeat_seconds", 300),
|
||||
("daily_backup_path", "cos://%s/_jobs/daily" % _bkt),
|
||||
("long_list_backup_path", "cos://%s/_jobs/long_video" % _bkt),
|
||||
("monitor_root", "cos://%s/_jobs/monitor" % _bkt),
|
||||
("long_list_clear_after_upload", True),
|
||||
("long_list_min_batch", 1000), # 领取长视频队列时凑够这么多条任务才开始处理一批
|
||||
("daily_backup_hour", 10),
|
||||
("stop_flag_key", "_jobs/STOP"),
|
||||
]
|
||||
for k, v in _dispatch_defaults:
|
||||
if k not in dp:
|
||||
dp[k] = v
|
||||
changes.append("dispatch.%s -> %s (新增, COS任务分发)" % (k, v))
|
||||
|
||||
# 0) OCR 重试封顶 + 单次超时 (防"端点全超时时一张图卡~1小时"拖死worker)
|
||||
if "max_attempts" not in ocr:
|
||||
ocr["max_attempts"] = 4
|
||||
changes.append("ocr.max_attempts -> 4 (新增, 重试封顶)")
|
||||
# 单次OCR超时: 调到30s(大图负载下最慢~12s, 12s砍太狠; 有max_attempts封顶不会卡死)
|
||||
if ocr.get("timeout", 0) < 30:
|
||||
_old = ocr.get("timeout")
|
||||
ocr["timeout"] = 30
|
||||
changes.append("ocr.timeout %s -> 30 (大图留足余量)" % _old)
|
||||
if "dispatch_log_every" not in ocr:
|
||||
ocr["dispatch_log_every"] = 1000
|
||||
changes.append("ocr.dispatch_log_every -> 1000 (新增, 每1000次派发落盘计数, 0=关)")
|
||||
|
||||
# 0b) 处理完删源视频(本来就在COS上, 本地是冗余副本)。防 ./video/ 堆积撑爆磁盘。
|
||||
if sf.get("delete_video") is not True:
|
||||
sf["delete_video"] = True
|
||||
changes.append("delete_video -> True (处理完删源视频, 防磁盘爆)")
|
||||
|
||||
# 1) 抽帧 ffmpeg 线程数: 统一收敛到 "auto"(按 CPU核数÷并发进程数 自动摊分, GPU/CPU两条
|
||||
# 抽帧路径现在都吃这个值了, 比固定数字更适合核数不一的机队)。已经是 "auto" 的跳过；
|
||||
# 只升级明显偏低的字面值(<5)或完全没配的情况, 已手动调更高的字面值保留不动。
|
||||
old = sf.get("ffmpeg_threads", 2)
|
||||
_old_is_auto = isinstance(old, str) and old.strip().lower() == "auto"
|
||||
if not _old_is_auto and (not isinstance(old, (int, float)) or old < 5):
|
||||
sf["ffmpeg_threads"] = "auto"
|
||||
changes.append(f"ffmpeg_threads {old!r} -> auto")
|
||||
|
||||
# 2) 超长视频补跑列表 (缺失才加)
|
||||
if "long_video_list" not in sub:
|
||||
sub["long_video_list"] = "./超长待重跑.txt"
|
||||
changes.append("long_video_list -> ./超长待重跑.txt (新增)")
|
||||
|
||||
# 2b) 降低 OCR 拼接帧数 (大图OCR>12s会超时; 只往下调, 已更小的保留)
|
||||
_mt = sub.get("max_tiles_per_composite", 40)
|
||||
if _mt > 24:
|
||||
sub["max_tiles_per_composite"] = 24
|
||||
changes.append("max_tiles_per_composite %d -> 24 (拼接图减小, 防OCR超时)" % _mt)
|
||||
|
||||
# 2b2) 降全局OCR并发, 削掉过载突发尾延迟(端点平均快但突发会排队爆30s); 只往下调
|
||||
_mg = sub.get("max_global_ocr", 80)
|
||||
if _mg > 50:
|
||||
sub["max_global_ocr"] = 50
|
||||
changes.append("max_global_ocr %d -> 50 (降负载削尾延迟)" % _mg)
|
||||
|
||||
# 2b3) OCR图片像素上限: 超大合成图(区域过高)缩小再OCR, 防单张>30s超时
|
||||
if "ocr_max_image_side" not in sub:
|
||||
sub["ocr_max_image_side"] = 4000
|
||||
changes.append("ocr_max_image_side -> 4000 (新增, 超大图缩小防超时)")
|
||||
|
||||
# 2c) 不丢帧: OCR多轮重试, 仍失败则整段视频判失败去重跑(不产出带洞SRT)
|
||||
for k, v in [("ocr_no_drop", True), ("ocr_max_passes", 5),
|
||||
("ocr_retry_sleep", 2), ("ocr_fail_list", "./OCR失败待重跑.txt")]:
|
||||
if k not in sub:
|
||||
sub[k] = v
|
||||
changes.append("%s -> %s (新增, 不丢帧)" % (k, v))
|
||||
|
||||
# 2d) 低置信占比过滤: 幻灯片/编程视频OCR出垃圾(低置信50%+)且吃3-5倍OCR, 整段判非字幕跳过
|
||||
for k, v in [("filter_lowconf", True), ("lowconf_thr", 80),
|
||||
("lowconf_filter_frac", 0.6), ("lowconf_min_det", 200)]:
|
||||
if k not in sub:
|
||||
sub[k] = v
|
||||
changes.append("%s -> %s (新增, 过滤幻灯片/代码类非字幕视频)" % (k, v))
|
||||
|
||||
# 2e) 状态归类清单: 这些原因是"无可用字幕的正常过滤", 应记 no_subtitle 而非 error。
|
||||
# 旧逻辑把字幕太少/冗余/置信度/非字幕等错记成 error(线上 error 中~79%其实是它们)。缺失才加。
|
||||
if "no_subtitle_reasons" not in sub:
|
||||
sub["no_subtitle_reasons"] = [
|
||||
"字幕太少", "冗余文字过多", "冗余文字太多", "无字幕区域",
|
||||
"非字幕内容", "置信度太低", "未检测", "特殊字体",
|
||||
"视频过短", "视频过长",
|
||||
]
|
||||
changes.append("no_subtitle_reasons -> [10项] (新增, 过滤型结论归 no_subtitle 不计 error)")
|
||||
|
||||
# 3) 抽帧自动选空闲GPU (避开被OCR服务占用的卡, 缺失才加)
|
||||
if "gpu_auto_select" not in sf:
|
||||
sf["gpu_auto_select"] = True
|
||||
changes.append("gpu_auto_select -> True (新增, 抽帧自动避开被占用的GPU)")
|
||||
if "gpu_busy_mem_mb" not in sf:
|
||||
sf["gpu_busy_mem_mb"] = 4096
|
||||
changes.append("gpu_busy_mem_mb -> 4096 (新增, 显存占用超此判为忙)")
|
||||
sf.setdefault("gpu_skip_indices", []) # 手动跳过列表(默认空, 交给自动探测)
|
||||
|
||||
if changes:
|
||||
json.dump(d, open(P, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||||
print("✓ config 已更新:")
|
||||
for c in changes:
|
||||
print(" -", c)
|
||||
else:
|
||||
print("✓ config 无需改动(已是最新)")
|
||||
41
tools/release_machine.py
Normal file
41
tools/release_machine.py
Normal file
@@ -0,0 +1,41 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""人工救援: 当某台机器【永久报废】(不会再重启续做)时, 把它名下所有未完成分片的锁删掉,
|
||||
任务退回 todo 让别的机器重新认领。
|
||||
|
||||
⚠️ 只在确认机器真报废后用。机器只是重启的话, 它自己会续做, 不要跑这个。
|
||||
|
||||
用法:
|
||||
python tools/release_machine.py <机器ID> # 实际释放
|
||||
python tools/release_machine.py <机器ID> --dry-run # 只列出, 不删
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from tool import job_cos # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("用法: python tools/release_machine.py <机器ID> [--dry-run]")
|
||||
sys.exit(1)
|
||||
target = sys.argv[1]
|
||||
dry = "--dry-run" in sys.argv[2:]
|
||||
|
||||
shards = job_cos.my_open_claims(target)
|
||||
if not shards:
|
||||
print(f"机器 {target} 名下没有未完成分片。")
|
||||
return
|
||||
print(f"机器 {target} 名下未完成分片 {len(shards)} 个:")
|
||||
for s in shards:
|
||||
print(f" - {s}")
|
||||
if dry:
|
||||
print("(--dry-run, 未删除)")
|
||||
return
|
||||
for s in shards:
|
||||
job_cos._delete(job_cos._key("claimed", s)) # 只删锁; todo 仍在 -> 别人会重领
|
||||
print(f"✓ 已释放 {len(shards)} 个分片的锁, 它们会被其他机器重新认领。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
99
tools/rerun_srt.py
Normal file
99
tools/rerun_srt.py
Normal file
@@ -0,0 +1,99 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从已有 OCR 数据重新生成 SRT，不重新调用 OCR API。
|
||||
用法: python rerun_srt.py
|
||||
"""
|
||||
import sys, os, json
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from difflib import SequenceMatcher
|
||||
from tool.log.log_manager import UnifiedLogger
|
||||
from tool.getsrt import (
|
||||
analyze_subtitle_blocks, analyze_h,
|
||||
extract_highest_subtitle, check_subtitle_quality_enhanced,
|
||||
evaluate_text_quality_rules, generate_srt_from_results
|
||||
)
|
||||
|
||||
COMPOSITE_DIR = sys.argv[1] if len(sys.argv) > 1 else "./composite/BV1aL4y1p7Bi_p11"
|
||||
JSONL_PATH = os.path.join(COMPOSITE_DIR, "ocr_result.jsonl")
|
||||
META_PATH = os.path.join(COMPOSITE_DIR, "metadata.json")
|
||||
OUTPUT_SRT = os.path.join(COMPOSITE_DIR, "BV1aL4y1p7Bi_p11.new.srt")
|
||||
INTERVAL = 0.33
|
||||
|
||||
logger = UnifiedLogger()
|
||||
logger.reset()
|
||||
logger.init_logging("rerun_log.json")
|
||||
logger.start_program("rerun")
|
||||
|
||||
# 读取已有 OCR 数据，将旧路径修正为当前实际路径
|
||||
all_pic_ocr_pair = []
|
||||
with open(JSONL_PATH, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
if i + 1 < len(lines) and lines[i].startswith("pic_path: "):
|
||||
raw_path = lines[i].strip()[len("pic_path: "):]
|
||||
filename = os.path.basename(raw_path)
|
||||
actual_path = os.path.join(COMPOSITE_DIR, filename)
|
||||
try:
|
||||
ocr_json = json.loads(lines[i + 1])
|
||||
all_pic_ocr_pair.append([actual_path, ocr_json])
|
||||
i += 2
|
||||
continue
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
i += 1
|
||||
|
||||
print(f"读取 {len(all_pic_ocr_pair)} 帧的OCR数据")
|
||||
|
||||
# 分析字幕网格位置
|
||||
best_grids = analyze_subtitle_blocks(all_pic_ocr_pair, META_PATH, logger)
|
||||
h = analyze_h(all_pic_ocr_pair, META_PATH)
|
||||
|
||||
if not best_grids.get("composite"):
|
||||
print("ERROR: 未找到合适的字幕区域")
|
||||
sys.exit(1)
|
||||
|
||||
# 逐帧提取字幕
|
||||
results = []
|
||||
DEBUG_KEYWORD = "曹小月从角落里拿出雨伞"
|
||||
for idx in range(len(all_pic_ocr_pair)):
|
||||
print(f"\r处理 {idx + 1}/{len(all_pic_ocr_pair)}", end="", flush=True)
|
||||
text, conf, flag, sim = extract_highest_subtitle(
|
||||
idx, all_pic_ocr_pair, best_grids, META_PATH, h, logger
|
||||
)
|
||||
if text and not check_subtitle_quality_enhanced(text) and conf <= 90:
|
||||
text = ""
|
||||
# 打印包含关键词的帧的原始OCR和提取结果
|
||||
if DEBUG_KEYWORD in (text or ""):
|
||||
pic_path, ocr_json = all_pic_ocr_pair[idx]
|
||||
print(f"\n[DEBUG 帧{idx}] 提取: {repr(text)}")
|
||||
for td in ocr_json['TextDetections']:
|
||||
print(f" box: {repr(td['DetectedText'])} conf={td['Confidence']} watermark={td.get('is_watermark','?')}")
|
||||
results.append((idx, text or ""))
|
||||
|
||||
non_empty = sum(1 for _, t in results if t)
|
||||
print(f"\n共 {non_empty}/{len(results)} 帧有字幕")
|
||||
|
||||
# 生成 SRT
|
||||
generate_srt_from_results(results, OUTPUT_SRT, interval_seconds=INTERVAL)
|
||||
print(f"SRT 已输出: {OUTPUT_SRT}")
|
||||
|
||||
# 展示问题字幕附近的结果
|
||||
with open(OUTPUT_SRT, 'r', encoding='utf-8') as f:
|
||||
srt_content = f.readlines()
|
||||
|
||||
print("\n=== 修复验证（'曹小月从角落里拿出雨伞' 附近） ===")
|
||||
found = False
|
||||
for j, line in enumerate(srt_content):
|
||||
if "曹小月从角落里拿出雨伞" in line:
|
||||
start = max(0, j - 3)
|
||||
end = min(len(srt_content), j + 10)
|
||||
print("".join(srt_content[start:end]))
|
||||
print("---")
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
print("未在输出 SRT 中找到目标字幕，请检查提取结果。")
|
||||
89
tools/seed_jobs.py
Normal file
89
tools/seed_jobs.py
Normal file
@@ -0,0 +1,89 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""把一份/多份视频清单切成分片, 上传到 COS 的 todo/, 供常驻 worker 认领。
|
||||
|
||||
用法:
|
||||
python tools/seed_jobs.py 清单1.txt [清单2.txt ...]
|
||||
python tools/seed_jobs.py ./text2 # 目录: 取其中所有 *.txt
|
||||
python tools/seed_jobs.py ./text2 --skip-processed # 灌之前先过滤掉全局已处理过的视频
|
||||
分片大小取 config.dispatch.shard_size(默认2000)。分片名: shard_<起始序号>。
|
||||
幂等性: 同名分片已存在(todo 里有)则跳过, 不覆盖。
|
||||
|
||||
--skip-processed: 聚合 30 台机器各自最新一天的 数据.jsonl 快照(dispatch.daily_backup_path),
|
||||
合并成全局已处理视频集合再过滤清单 —— 避免同一视频被不同机器重复处理(本机 数据.jsonl 只能
|
||||
挡住"这台机器"重复跑, 挡不住跨机器重复)。有最长约1天(daily_backup_hour)的数据延迟。
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from config.config_utils import config # noqa: E402
|
||||
from tool import job_cos # noqa: E402
|
||||
|
||||
|
||||
def _collect_lines(paths):
|
||||
lines = []
|
||||
for p in paths:
|
||||
files = glob.glob(os.path.join(p, "*.txt")) if os.path.isdir(p) else [p]
|
||||
for fp in files:
|
||||
with open(fp, encoding="utf-8") as f:
|
||||
lines += [ln.strip() for ln in f if ln.strip()]
|
||||
# 去重并保序
|
||||
seen, out = set(), []
|
||||
for ln in lines:
|
||||
if ln not in seen:
|
||||
seen.add(ln)
|
||||
out.append(ln)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="切片灌任务到 COS todo/")
|
||||
ap.add_argument("paths", nargs="+", help="清单.txt 或目录(取其中所有 *.txt)")
|
||||
ap.add_argument("--skip-processed", action="store_true",
|
||||
help="灌之前先过滤掉全局(所有机器)已处理过的视频，见文件头说明")
|
||||
ap.add_argument("--all-dates", action="store_true",
|
||||
help="配合 --skip-processed: 不只取每台机器最新一天,把所有历史日期的已处理清单都拉下来合并判断"
|
||||
"(机群已停机、想更保险地核对时用;正常灌任务不需要,只会多下载很多文件)")
|
||||
args = ap.parse_args()
|
||||
|
||||
shard_size = int((config.get_dispatch_config() or {}).get("shard_size", 2000))
|
||||
lines = _collect_lines(args.paths)
|
||||
if not lines:
|
||||
print("✗ 没有读到任何视频路径")
|
||||
sys.exit(1)
|
||||
|
||||
if args.skip_processed:
|
||||
if args.all_dates:
|
||||
print("正在聚合所有机器【全部历史日期】的已处理清单(daily_backup_path)...")
|
||||
else:
|
||||
print("正在聚合各机器最新一天的已处理清单(daily_backup_path)...")
|
||||
processed = job_cos.global_processed_set(verbose=True, all_dates=args.all_dates)
|
||||
before = len(lines)
|
||||
lines = [ln for ln in lines if ln not in processed]
|
||||
print(f"全局已处理 {len(processed)} 条 -> 过滤掉 {before - len(lines)} 条，剩余 {len(lines)} 条待灌。")
|
||||
if not lines:
|
||||
print("✓ 全部已处理过，无需灌任务。")
|
||||
return
|
||||
|
||||
existing = set(job_cos.list_todo())
|
||||
n_shards = (len(lines) + shard_size - 1) // shard_size
|
||||
print(f"共 {len(lines)} 条 -> {n_shards} 个分片(每片≤{shard_size})。已存在 {len(existing)} 个分片。")
|
||||
|
||||
added = skipped = 0
|
||||
for i in range(0, len(lines), shard_size):
|
||||
shard = f"shard_{i:08d}"
|
||||
if shard in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
body = "\n".join(lines[i:i + shard_size]).encode("utf-8")
|
||||
job_cos._put(job_cos._key("todo", shard), body) # 普通写(任务清单, 不需禁止覆盖)
|
||||
added += 1
|
||||
if added % 50 == 0:
|
||||
print(f" 已上传 {added} 个分片...")
|
||||
print(f"✓ 完成: 新增 {added} 个分片, 跳过已存在 {skipped} 个。worker 会自动认领。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
30
tools/split_txt.py
Normal file
30
tools/split_txt.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""
|
||||
将 txt 文件按指定行数拆分为多个子文件。
|
||||
用法：python split_txt.py [输入文件] [每份行数]
|
||||
默认：python split_txt.py text/0004_has_srt.txt 100
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
def split_txt(input_path: str, chunk_size: int = 100):
|
||||
with open(input_path, 'r', encoding='utf-8') as f:
|
||||
lines = [line.strip() for line in f if line.strip()]
|
||||
|
||||
total = len(lines)
|
||||
num_files = (total + chunk_size - 1) // chunk_size
|
||||
out_dir = os.path.dirname(input_path)
|
||||
base_name = os.path.splitext(os.path.basename(input_path))[0]
|
||||
|
||||
for i in range(num_files):
|
||||
chunk = lines[i * chunk_size:(i + 1) * chunk_size]
|
||||
out_path = os.path.join(out_dir, f"{base_name}_{i + 1:04d}.txt")
|
||||
with open(out_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(chunk) + '\n')
|
||||
print(f" {out_path} ({len(chunk)} 条)")
|
||||
|
||||
print(f"\n共 {total} 条，拆分为 {num_files} 个文件（每份 {chunk_size} 条）")
|
||||
|
||||
if __name__ == '__main__':
|
||||
input_file = sys.argv[1] if len(sys.argv) > 1 else 'text/0004_has_srt.txt'
|
||||
chunk_size = int(sys.argv[2]) if len(sys.argv) > 2 else 100
|
||||
split_txt(input_file, chunk_size)
|
||||
73
tools/stats_jsonl.py
Normal file
73
tools/stats_jsonl.py
Normal file
@@ -0,0 +1,73 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
统计 数据.jsonl：成功条数、总条数、成功数据总秒数、全部数据总秒数。
|
||||
用法: python stats_jsonl.py [数据.jsonl路径]
|
||||
默认: python stats_jsonl.py 数据.jsonl
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
JSONL_PATH = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||||
|
||||
|
||||
def fmt(seconds):
|
||||
"""秒 → H时M分S秒，便于阅读"""
|
||||
seconds = int(round(seconds))
|
||||
h, rem = divmod(seconds, 3600)
|
||||
m, s = divmod(rem, 60)
|
||||
return f"{h}时{m}分{s}秒"
|
||||
|
||||
|
||||
def main():
|
||||
total = 0
|
||||
success = 0
|
||||
total_sec = 0.0
|
||||
success_sec = 0.0
|
||||
status_counter = Counter()
|
||||
|
||||
try:
|
||||
f = open(JSONL_PATH, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
total += 1
|
||||
status = rec.get("status")
|
||||
status_counter[status] += 1
|
||||
|
||||
dur = rec.get("duration") or 0
|
||||
try:
|
||||
dur = float(dur)
|
||||
except (TypeError, ValueError):
|
||||
dur = 0.0
|
||||
total_sec += dur
|
||||
|
||||
if status == "success":
|
||||
success += 1
|
||||
success_sec += dur
|
||||
|
||||
rate = (success / total * 100) if total else 0.0
|
||||
|
||||
print(f"文件: {JSONL_PATH}")
|
||||
print("-" * 40)
|
||||
print(f"总条数: {total}")
|
||||
print(f"成功条数: {success} ({rate:.1f}%)")
|
||||
print(f"各状态明细: {dict(status_counter)}")
|
||||
print("-" * 40)
|
||||
print(f"成功数据秒数: {success_sec:.2f} 秒 ({fmt(success_sec)})")
|
||||
print(f"全部数据秒数: {total_sec:.2f} 秒 ({fmt(total_sec)})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
117
tools/test_ocr.py
Normal file
117
tools/test_ocr.py
Normal file
@@ -0,0 +1,117 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
向 OCR 服务发一次请求做 OCR，验证端点可用、看识别结果。
|
||||
|
||||
用法:
|
||||
python test_ocr.py <图片路径>
|
||||
python test_ocr.py <图片路径> --url http://其它IP:18081
|
||||
python test_ocr.py <图片路径> --raw # 额外打印原始 JSON 响应
|
||||
|
||||
不给图片时，自动在 ./composite、./split 下找一张 .jpg 试。
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
DEFAULT_URL = "http://asr.ocr.general.smart.l20.production.polaris:18081"
|
||||
OCR_PATH = "/youtu/ocrapi/hpgeneralocr_v3"
|
||||
|
||||
|
||||
def find_sample_image(given):
|
||||
if given:
|
||||
return given if os.path.exists(given) else None
|
||||
for root in ("./composite", "./split"):
|
||||
hits = glob.glob(os.path.join(root, "**", "*.jpg"), recursive=True)
|
||||
if hits:
|
||||
return hits[0]
|
||||
return None
|
||||
|
||||
|
||||
def img2base64(img_path):
|
||||
with open(img_path, "rb") as f:
|
||||
return base64.b64encode(f.read()).decode()
|
||||
|
||||
|
||||
def ocr_request(base_url, image_b64, timeout=30):
|
||||
"""向端点发一次 OCR 请求，返回 (原始响应文本, 耗时秒)。"""
|
||||
url = base_url.rstrip("/") + OCR_PATH
|
||||
app_id = f"{int(time.time() * 1000)}{random.randint(100000, 999999)}"
|
||||
payload = {
|
||||
"app_id": app_id,
|
||||
"image": image_b64,
|
||||
"ocr_template": "OCR",
|
||||
"options": {"pdf_page_index": 0, "detect_text": True},
|
||||
}
|
||||
t0 = time.time()
|
||||
resp = requests.post(url, headers={"Content-Type": "application/json"},
|
||||
json=payload, timeout=timeout)
|
||||
dt = time.time() - t0
|
||||
resp.raise_for_status()
|
||||
return resp.text, dt
|
||||
|
||||
|
||||
def extract_texts(raw):
|
||||
"""从 Youtu OCR 响应里抽出每行识别文字 (content, conf)。"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
if data.get("error_code", 0) != 0:
|
||||
return f"OCR 返回错误: error_code={data.get('error_code')}, msg={data.get('error_msg')}"
|
||||
lines = []
|
||||
for rec in data.get("recognize_list", []):
|
||||
for item in rec.get("item_content", {}).get("item_list", []):
|
||||
lines.append((item.get("content", ""), item.get("conf", 0)))
|
||||
return lines
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("image", nargs="?", default=None, help="图片路径")
|
||||
ap.add_argument("--url", default=DEFAULT_URL, help="OCR 端点 (默认 Polaris 地址)")
|
||||
ap.add_argument("--raw", action="store_true", help="打印原始 JSON 响应")
|
||||
args = ap.parse_args()
|
||||
|
||||
img = find_sample_image(args.image)
|
||||
if not img:
|
||||
print("找不到图片，请把一张 jpg 路径作为参数传入。")
|
||||
return
|
||||
|
||||
print(f"端点: {args.url}{OCR_PATH}")
|
||||
print(f"图片: {img}")
|
||||
print("-" * 60)
|
||||
|
||||
try:
|
||||
raw, dt = ocr_request(args.url, img2base64(img))
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"✗ 请求失败: {e}")
|
||||
return
|
||||
|
||||
print(f"✓ 请求成功，耗时 {dt * 1000:.0f}ms")
|
||||
|
||||
if args.raw:
|
||||
print("\n原始响应:")
|
||||
print(raw)
|
||||
print("-" * 60)
|
||||
|
||||
result = extract_texts(raw)
|
||||
if result is None:
|
||||
print("响应不是合法 JSON")
|
||||
elif isinstance(result, str):
|
||||
print(result)
|
||||
elif not result:
|
||||
print("（未识别到文字）")
|
||||
else:
|
||||
print(f"\n识别到 {len(result)} 行文字:")
|
||||
for content, conf in result:
|
||||
print(f" [{conf:.2f}] {content}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
159
tools/throughput_jsonl.py
Normal file
159
tools/throughput_jsonl.py
Normal file
@@ -0,0 +1,159 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
分析 数据.jsonl 的处理吞吐：单台机器一天能处理多少小时的视频数据。
|
||||
|
||||
两种口径：
|
||||
A) 墙钟吞吐（真实）：用 process_start/process_end 求整批的实际墙钟跨度，
|
||||
总视频时长 ÷ 墙钟跨度 = 机器真实吞吐（含当时的并发/排队效果）。
|
||||
B) 单视频平均（理想并发）：按每个视频 duration/elapsed 的实时倍率，
|
||||
再乘并发数估算（假设无 OCR/CPU 排队，即"OCR 速度够"的上限）。
|
||||
|
||||
用法: python throughput_jsonl.py [数据.jsonl路径] [并发数]
|
||||
默认: python throughput_jsonl.py 数据.jsonl 1
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
JSONL_PATH = sys.argv[1] if len(sys.argv) > 1 else "数据.jsonl"
|
||||
WORKERS = int(sys.argv[2]) if len(sys.argv) > 2 else 1
|
||||
OCR_QPS_PER_MACHINE = float(sys.argv[3]) if len(sys.argv) > 3 else 14.0 # 单台 OCR 的 QPS
|
||||
|
||||
|
||||
def parse_dt(s):
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(s)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def fmt_h(hours):
|
||||
return f"{hours:.1f} 小时"
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
f = open(JSONL_PATH, encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
print(f"找不到文件: {JSONL_PATH}")
|
||||
return
|
||||
|
||||
n = 0
|
||||
total_dur = 0.0 # 全部视频时长(秒)
|
||||
total_elapsed = 0.0 # 全部单视频处理耗时(秒)，含并发
|
||||
starts, ends = [], []
|
||||
per_video_ratio = [] # 每视频 duration/elapsed
|
||||
stage_sum = {} # 各阶段耗时累计
|
||||
with_timing = 0
|
||||
total_ocr = 0 # 全部 OCR 请求次数累计
|
||||
with_ocr = 0
|
||||
|
||||
with f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
n += 1
|
||||
dur = rec.get("duration") or 0
|
||||
try:
|
||||
dur = float(dur)
|
||||
except (TypeError, ValueError):
|
||||
dur = 0.0
|
||||
total_dur += dur
|
||||
|
||||
el = rec.get("elapsed_seconds")
|
||||
if el:
|
||||
total_elapsed += float(el)
|
||||
if dur > 0:
|
||||
per_video_ratio.append(dur / float(el))
|
||||
|
||||
st = parse_dt(rec.get("process_start"))
|
||||
en = parse_dt(rec.get("process_end"))
|
||||
if st:
|
||||
starts.append(st)
|
||||
if en:
|
||||
ends.append(en)
|
||||
|
||||
sd = rec.get("stage_durations") or {}
|
||||
if sd:
|
||||
with_timing += 1
|
||||
for k, v in sd.items():
|
||||
try:
|
||||
stage_sum[k] = stage_sum.get(k, 0.0) + float(v)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
ic = rec.get("image_counts") or {}
|
||||
toc = ic.get("total_ocr_count")
|
||||
if toc:
|
||||
try:
|
||||
total_ocr += int(toc)
|
||||
with_ocr += 1
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
print(f"文件: {JSONL_PATH}")
|
||||
print("=" * 52)
|
||||
print(f"记录数: {n}")
|
||||
print(f"视频总时长: {total_dur/3600:.1f} 小时 ({total_dur:.0f} 秒)")
|
||||
print(f"单视频耗时累加: {total_elapsed/3600:.1f} 小时 ({total_elapsed:.0f} 秒)")
|
||||
if with_ocr:
|
||||
print(f"OCR 请求总数: {total_ocr} (平均 {total_ocr/with_ocr:.0f} 次/视频, {with_ocr} 条有统计)")
|
||||
|
||||
# ---- 口径A：真实墙钟吞吐 ----
|
||||
print("\n[A] 真实墙钟吞吐（按 process_start/end 的整批实际跨度）")
|
||||
if starts and ends:
|
||||
span = (max(ends) - min(starts)).total_seconds()
|
||||
if span > 0:
|
||||
tp = total_dur / span # 视频秒/墙钟秒
|
||||
print(f" 整批墙钟跨度: {span/3600:.2f} 小时")
|
||||
print(f" 实时倍率: {tp:.2f}x （每 1 秒墙钟处理 {tp:.2f} 秒视频）")
|
||||
print(f" → 一天可处理: {fmt_h(tp*24)} 的视频")
|
||||
vids_per_day = n / span * 86400 if span else 0
|
||||
print(f" → 一天可处理: {vids_per_day:.0f} 个视频")
|
||||
|
||||
# ---- OCR 需求：这台机器实际产生多少 OCR/秒、能喂几台 OCR ----
|
||||
if total_ocr:
|
||||
ocr_per_sec = total_ocr / span
|
||||
machines_fed = ocr_per_sec / OCR_QPS_PER_MACHINE
|
||||
print(f"\n [OCR 配比]（单台 OCR 按 {OCR_QPS_PER_MACHINE:.0f} QPS 计）")
|
||||
print(f" 本机 OCR 速率: {ocr_per_sec:.2f} OCR/秒")
|
||||
print(f" → 本机能喂满: {machines_fed:.2f} 台 OCR")
|
||||
if machines_fed > 0:
|
||||
print(f" → 一台 OCR 可喂: {1/machines_fed:.1f} 台这样的处理机")
|
||||
else:
|
||||
print(" 墙钟跨度为0，无法计算")
|
||||
else:
|
||||
print(" 缺少 process_start/process_end 字段，跳过")
|
||||
|
||||
# ---- 口径B：按单视频平均实时倍率 × 并发 ----
|
||||
print(f"\n[B] 理想吞吐（单视频平均倍率 × 并发{WORKERS}，假设无排队=OCR够快）")
|
||||
if per_video_ratio:
|
||||
avg_ratio = sum(per_video_ratio) / len(per_video_ratio)
|
||||
# 也给一个总量口径：总视频时长/总耗时（更稳，受长短视频加权）
|
||||
weighted = total_dur / total_elapsed if total_elapsed else 0
|
||||
print(f" 单视频平均倍率: {avg_ratio:.2f}x")
|
||||
print(f" 加权倍率(总/总):{weighted:.2f}x")
|
||||
print(f" → 单并发一天: {fmt_h(weighted*24)} 的视频")
|
||||
print(f" → {WORKERS}并发一天: {fmt_h(weighted*24*WORKERS)} 的视频（理想线性，实际受CPU限制）")
|
||||
else:
|
||||
print(" 缺少 elapsed_seconds 字段，跳过")
|
||||
|
||||
# ---- 各阶段占比 ----
|
||||
if stage_sum:
|
||||
print(f"\n[阶段耗时占比]（{with_timing} 条有计时）")
|
||||
main_t = stage_sum.get("main", sum(v for k, v in stage_sum.items() if k != "main"))
|
||||
for k in ["split_frame", "merge", "getsrt"]:
|
||||
if k in stage_sum:
|
||||
pct = stage_sum[k] / main_t * 100 if main_t else 0
|
||||
print(f" {k:12s}: {stage_sum[k]/3600:6.1f} 小时 ({pct:.0f}%)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
91
tools/upload_composites.py
Normal file
91
tools/upload_composites.py
Normal file
@@ -0,0 +1,91 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
扫描本地 ./composite/ 下的视频目录，逐个上传到其在 COS 上对应视频目录的 composite/ 子目录，
|
||||
上传成功后删除本地。视频的 COS 路径从 数据.jsonl 的 video_cos_path 反查。
|
||||
|
||||
用法: python upload_composites.py [composite根目录] [数据.jsonl路径]
|
||||
默认: python upload_composites.py ./composite 数据.jsonl
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from config.config_utils import config
|
||||
|
||||
COMPOSITE_ROOT = sys.argv[1] if len(sys.argv) > 1 else "./composite"
|
||||
JSONL_PATH = sys.argv[2] if len(sys.argv) > 2 else "数据.jsonl"
|
||||
|
||||
|
||||
def build_cosdir_map(jsonl_path):
|
||||
"""从 数据.jsonl 建立 video_name -> cos_dir 映射。"""
|
||||
mapping = {}
|
||||
if not os.path.exists(jsonl_path):
|
||||
print(f"警告: 找不到 {jsonl_path}，无法反查 COS 路径")
|
||||
return mapping
|
||||
with open(jsonl_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
vid = rec.get("video_cos_path")
|
||||
if not vid:
|
||||
continue
|
||||
cos_dir = vid[:vid.rfind("/") + 1] # 含末尾 /
|
||||
name = os.path.splitext(os.path.basename(vid))[0]
|
||||
mapping[name] = cos_dir
|
||||
return mapping
|
||||
|
||||
|
||||
def upload_dir(local_dir, cos_dest_dir, coscli_path, cos_config_path):
|
||||
if not cos_dest_dir.endswith("/"):
|
||||
cos_dest_dir += "/"
|
||||
src = local_dir if local_dir.endswith(os.sep) else local_dir + os.sep
|
||||
cmd = [coscli_path, "-c", cos_config_path, "cp", "-r", src, cos_dest_dir]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
return (cos_dest_dir if result.returncode == 0 else None), result.stderr
|
||||
|
||||
|
||||
def main():
|
||||
if not os.path.isdir(COMPOSITE_ROOT):
|
||||
print(f"目录不存在: {COMPOSITE_ROOT}")
|
||||
return
|
||||
|
||||
cfg = config.get_splitframe_config()
|
||||
coscli_path = cfg.get("coscli_path", "./coscli-linux-amd64")
|
||||
cos_config_path = cfg.get("cosconfig_path", ".cos.yaml")
|
||||
|
||||
cosdir_map = build_cosdir_map(JSONL_PATH)
|
||||
|
||||
dirs = [d for d in os.listdir(COMPOSITE_ROOT)
|
||||
if os.path.isdir(os.path.join(COMPOSITE_ROOT, d))]
|
||||
print(f"发现 {len(dirs)} 个本地 composite 目录，开始上传...")
|
||||
|
||||
ok = fail = skip = 0
|
||||
for i, name in enumerate(dirs, 1):
|
||||
local_dir = os.path.join(COMPOSITE_ROOT, name)
|
||||
cos_dir = cosdir_map.get(name)
|
||||
if not cos_dir:
|
||||
skip += 1
|
||||
print(f"[{i}/{len(dirs)}] {name} ✗ 在 数据.jsonl 中找不到 COS 路径，跳过")
|
||||
continue
|
||||
cos_dest = cos_dir + "composite/"
|
||||
uploaded, err = upload_dir(local_dir, cos_dest, coscli_path, cos_config_path)
|
||||
if uploaded:
|
||||
shutil.rmtree(local_dir, ignore_errors=True)
|
||||
ok += 1
|
||||
print(f"[{i}/{len(dirs)}] {name} ✓ → {cos_dest}（已删本地）")
|
||||
else:
|
||||
fail += 1
|
||||
print(f"[{i}/{len(dirs)}] {name} ✗ 上传失败，保留本地: {err.strip()[:120]}")
|
||||
|
||||
print(f"\n完成 — 成功: {ok} 失败: {fail} 跳过(无COS路径): {skip}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
35
tools/upload_daily.py
Normal file
35
tools/upload_daily.py
Normal file
@@ -0,0 +1,35 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""手动/定时把本地 数据.jsonl 备份到 COS(文件名带机器号+日期)。
|
||||
|
||||
常驻 worker(run_worker.py)内部已有每天10点自动备份线程; 本脚本用于:
|
||||
- 不跑常驻 worker、想用系统 crontab 定时备份的场景;
|
||||
- 想立刻手动备份一次。
|
||||
|
||||
crontab 例(每天10点):
|
||||
0 10 * * * cd /项目根 && python3 tools/upload_daily.py >> log/upload_daily.log 2>&1
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from tool import job_cos # noqa: E402
|
||||
|
||||
JSONL = "数据.jsonl"
|
||||
|
||||
|
||||
def main():
|
||||
mid = job_cos.get_machine_id()
|
||||
dst = job_cos.upload_daily_backup(JSONL, mid)
|
||||
if dst:
|
||||
print(f"✓ 数据.jsonl 已上传 -> {dst}")
|
||||
else:
|
||||
print(f"✗ 本地无 {JSONL}, 未上传")
|
||||
ll = job_cos.upload_longlist_backup(mid)
|
||||
if ll:
|
||||
print(f"✓ 超长待重跑列表已上传 -> {ll}")
|
||||
else:
|
||||
print("✗ 本地无超长待重跑列表, 未上传")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
207
tools/upload_data_standalone.py
Normal file
207
tools/upload_data_standalone.py
Normal file
@@ -0,0 +1,207 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""独立备份脚本: 把本地 数据.jsonl + 超长待重跑列表 上传到 COS。
|
||||
|
||||
不 import 项目里任何其他 .py(不依赖 tool/、main.py 等), 只用标准库 + 第三方 qcloud_cos SDK,
|
||||
直接读 config/config.json —— 哪怕代码刚更新完、其他模块有问题, 这个脚本也能独立跑, 适合放在
|
||||
重新部署/更新代码前后单独执行一次, 确保数据先落到 COS。
|
||||
|
||||
用法(项目根目录下运行):
|
||||
python tools/upload_data_standalone.py
|
||||
|
||||
行为跟 tools/upload_daily.py + tool/job_cos.py 里的 upload_daily_backup/upload_longlist_backup
|
||||
一致(上传目标、命名、覆盖安全性都对齐):
|
||||
数据.jsonl -> dispatch.daily_backup_path/<日期>/数据_<机器号>.jsonl (全量快照,覆盖安全)
|
||||
已处理清单(轻量) -> dispatch.daily_backup_path/<日期>/已处理_<机器号>.txt
|
||||
(从jsonl提取的纯 video_cos_path 列表, 一行一个; 全局去重 global_processed_set()
|
||||
只拉这个小文件, 不用为了去重下载/解析整份带状态耗时字段的jsonl)
|
||||
超长待重跑列表 -> dispatch.long_list_backup_path/todo/<原文件名>_<机器号>_<日期>_<时分秒>.txt
|
||||
(增量式:传完清空本地已传部分; key精确到秒,同一天多次调用不会互相覆盖)
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
CONFIG_PATH = os.path.join(ROOT, "config", "config.json")
|
||||
JSONL_PATH = os.path.join(ROOT, "数据.jsonl")
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def parse_cos_url(url: str, default_bucket: str):
|
||||
"""cos://bucket/a/b -> (bucket, 'a/b')。无 cos:// 前缀时按传入的默认桶补全。"""
|
||||
url = url.strip()
|
||||
if url.startswith("cos://"):
|
||||
rest = url[len("cos://"):]
|
||||
bucket, _, key = rest.partition("/")
|
||||
return bucket, key.strip("/")
|
||||
return default_bucket, url.strip("/")
|
||||
|
||||
|
||||
def detect_ip() -> str:
|
||||
"""取本机对外默认路由的网卡IP(UDP connect触发内核选路由,不实际发包,天然跳过docker网桥/回环);
|
||||
取不到退化到 `hostname -I` 第一个。跟 tool/job_cos.py 的 _detect_ip() 逻辑一致。"""
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
try:
|
||||
s.connect(("8.8.8.8", 80))
|
||||
ip = s.getsockname()[0]
|
||||
finally:
|
||||
s.close()
|
||||
if ip and not ip.startswith("127."):
|
||||
return ip
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
r = subprocess.run(["hostname", "-I"], capture_output=True, text=True, timeout=5)
|
||||
ips = (r.stdout or "").split()
|
||||
return ips[0] if ips else ""
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return ""
|
||||
|
||||
|
||||
def get_machine_id(dcfg: dict) -> str:
|
||||
"""优先级: config.dispatch.machine_id > 环境变量 MACHINE_ID > 本机固定IP > 主机名。
|
||||
跟 tool/job_cos.py 的 get_machine_id() 保持一致, 这样跟常驻 worker 认领的是同一个身份。"""
|
||||
mid = (dcfg.get("machine_id") or "").strip()
|
||||
if mid:
|
||||
return mid
|
||||
mid = (os.environ.get("MACHINE_ID") or "").strip()
|
||||
if mid:
|
||||
return mid
|
||||
ip = detect_ip()
|
||||
if ip:
|
||||
return ip
|
||||
return socket.gethostname()
|
||||
|
||||
|
||||
# 数据.jsonl 是全量快照, 产能高的机器上百 MB/上 GB。put_object 单请求默认超时 30 秒, 大到
|
||||
# 传不完就一直 "The write operation timed out"。超过这个大小改用 coscli 分块上传(几 GB 也行),
|
||||
# coscli 失败自动回退 put_object。跟 tool/job_cos.py 的 _upload_dated_file 是同一套策略。
|
||||
_COSCLI_UPLOAD_THRESHOLD = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
|
||||
def _coscli_put(sf_cfg: dict, local_path: str, cos_url: str) -> bool:
|
||||
"""用 coscli 上传大文件。成功 True 失败 False。sf_cfg = config 的 splitframe 段。"""
|
||||
try:
|
||||
coscli = sf_cfg.get("coscli_path", "./coscli-linux-amd64")
|
||||
coscfg = sf_cfg.get("cosconfig_path", ".cos.yaml")
|
||||
r = subprocess.run([coscli, "-c", coscfg, "cp", local_path, cos_url],
|
||||
capture_output=True, text=True, timeout=1800)
|
||||
if r.returncode == 0:
|
||||
return True
|
||||
print(f" coscli 上传失败(回退 put_object): {(r.stderr or '').strip()[:200]}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" coscli 上传异常(回退 put_object): {type(e).__name__}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def upload_jsonl(client, default_bucket: str, dcfg: dict, sf_cfg: dict, machine_id: str):
|
||||
daily_root = dcfg.get("daily_backup_path")
|
||||
if not daily_root:
|
||||
print("✗ 未配置 dispatch.daily_backup_path, 跳过 数据.jsonl")
|
||||
return
|
||||
if not os.path.exists(JSONL_PATH):
|
||||
print(f"✗ 本地无 {JSONL_PATH}, 跳过")
|
||||
return
|
||||
bkt, base = parse_cos_url(daily_root, default_bucket)
|
||||
date = datetime.now().strftime("%Y%m%d")
|
||||
key = f"{base}/{date}/数据_{machine_id}.jsonl"
|
||||
cos_url = f"cos://{bkt}/{key}"
|
||||
with open(JSONL_PATH, "rb") as f:
|
||||
data = f.read()
|
||||
|
||||
size = len(data)
|
||||
uploaded = False
|
||||
if size > _COSCLI_UPLOAD_THRESHOLD:
|
||||
print(f" 数据.jsonl 有 {size / 1024 / 1024:.0f}MB, 用 coscli 分块上传")
|
||||
uploaded = _coscli_put(sf_cfg, JSONL_PATH, cos_url)
|
||||
if not uploaded:
|
||||
client.put_object(Bucket=bkt, Body=data, Key=key)
|
||||
print(f"✓ 数据.jsonl 已上传 -> {cos_url}")
|
||||
|
||||
# 顺带生成一份轻量"已处理清单"(只有 video_cos_path, 一行一个), 供 global_processed_set()
|
||||
# 只拉这个小文件去重用, 不用为了去重下载/解析整份带状态/耗时字段的 jsonl。
|
||||
paths = []
|
||||
for line in data.decode("utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
vid = json.loads(line).get("video_cos_path")
|
||||
if vid:
|
||||
paths.append(vid)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if paths:
|
||||
list_key = f"{base}/{date}/已处理_{machine_id}.txt"
|
||||
client.put_object(Bucket=bkt, Body=("\n".join(paths) + "\n").encode("utf-8"), Key=list_key)
|
||||
print(f"✓ 已处理清单已上传 -> cos://{bkt}/{list_key} ({len(paths)} 条)")
|
||||
|
||||
|
||||
def upload_long_list(client, default_bucket: str, dcfg: dict, sub_cfg: dict, machine_id: str):
|
||||
if dcfg.get("run_long_video"):
|
||||
print("本机 run_long_video=true(专职长视频机), 不回传超长列表, 跳过")
|
||||
return
|
||||
long_root = dcfg.get("long_list_backup_path")
|
||||
if not long_root:
|
||||
print("✗ 未配置 dispatch.long_list_backup_path, 跳过超长待重跑列表")
|
||||
return
|
||||
local_long = sub_cfg.get("long_video_list", "./超长待重跑.txt")
|
||||
if not os.path.isabs(local_long):
|
||||
local_long = os.path.join(ROOT, local_long)
|
||||
if not os.path.exists(local_long):
|
||||
print(f"✗ 本地无 {local_long}, 跳过")
|
||||
return
|
||||
with open(local_long, "r", encoding="utf-8") as f:
|
||||
snapshot = f.readlines()
|
||||
if not any(ln.strip() for ln in snapshot):
|
||||
print("✗ 超长待重跑列表为空, 跳过")
|
||||
return
|
||||
|
||||
bkt, base = parse_cos_url(long_root, default_bucket)
|
||||
stem = os.path.splitext(os.path.basename(local_long))[0]
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
key = f"{base}/todo/{stem}_{machine_id}_{ts}.txt"
|
||||
client.put_object(Bucket=bkt, Body="".join(snapshot).encode("utf-8"), Key=key)
|
||||
print(f"✓ 超长待重跑列表已上传 -> cos://{bkt}/{key}")
|
||||
|
||||
if dcfg.get("long_list_clear_after_upload", True):
|
||||
try:
|
||||
with open(local_long, "r", encoding="utf-8") as f:
|
||||
cur = f.readlines()
|
||||
with open(local_long, "w", encoding="utf-8") as f:
|
||||
f.writelines(cur[len(snapshot):])
|
||||
print(f" 本地已清空已传的 {len(snapshot)} 行, 保留上传期间新增的 {len(cur) - len(snapshot)} 行")
|
||||
except OSError as e:
|
||||
print(f" ⚠ 清空本地文件失败(不影响已上传的部分): {e}")
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load_config()
|
||||
cos_cfg = cfg.get("tencentcloud", {}).get("cos", {})
|
||||
dcfg = cfg.get("dispatch", {}) or {}
|
||||
sub_cfg = cfg.get("subtitle", {}) or {}
|
||||
sf_cfg = cfg.get("splitframe", {}) or {} # coscli 路径/配置在这段
|
||||
|
||||
client = CosS3Client(CosConfig(
|
||||
Region=cos_cfg["region"], SecretId=cos_cfg["secret_id"], SecretKey=cos_cfg["secret_key"],
|
||||
))
|
||||
default_bucket = cos_cfg["bucket"]
|
||||
machine_id = get_machine_id(dcfg)
|
||||
print(f"机器ID: {machine_id}")
|
||||
|
||||
upload_jsonl(client, default_bucket, dcfg, sf_cfg, machine_id)
|
||||
upload_long_list(client, default_bucket, dcfg, sub_cfg, machine_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
150
tools/video_bench.py
Normal file
150
tools/video_bench.py
Normal file
@@ -0,0 +1,150 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
下载一个视频 → 抽帧 → 用这些【真实帧】对 OCR 端点做压测。
|
||||
比随机生成的图更贴近线上负载（真实字幕内容，OCR 干的活和实际一致）。
|
||||
|
||||
用法（项目根目录运行）:
|
||||
python temp/video_bench.py <视频...> --url http://IP:18081 [选项]
|
||||
|
||||
<视频...> 一个或多个，cos://bucket/path/x.mp4 或本地 ./video/x.mp4
|
||||
多个视频时各抽一部分帧合并成图池(max-frames 在视频间均分)
|
||||
也可用 --list 文件 从文件读视频列表(每行一个)
|
||||
|
||||
选项:
|
||||
--url URL OCR 端点(可多次，soak 只用第一个)
|
||||
--concurrency N 并发(默认 100；单台拐点约 80)
|
||||
--mode soak|sweep soak=持续满速(默认)；sweep=自动加压找极限
|
||||
--fps F 抽帧频率(默认 1，压测用不需要 3fps，省时省内存)
|
||||
--max-frames N 最多用多少帧做图池(默认 2000)
|
||||
--interval N soak 打印间隔秒(默认 5)
|
||||
--ffmpeg PATH ffmpeg 路径(默认 ./bin/ffmpeg)
|
||||
--coscli PATH coscli 路径(默认 ./coscli-linux-amd64)
|
||||
--cos-config PATH coscli 配置(默认 .cos.yaml)
|
||||
--keep 跑完不删临时视频/帧
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
# 复用同目录 bench 脚本的压测逻辑
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import bench_ocr_endpoints as bench # noqa: E402
|
||||
|
||||
|
||||
def download_video(src, coscli, cos_config, dst):
|
||||
"""cos:// 路径用 coscli 下载到 dst；本地路径直接返回。"""
|
||||
if not src.startswith("cos://"):
|
||||
if not os.path.exists(src):
|
||||
print(f"本地视频不存在: {src}")
|
||||
return None
|
||||
return src
|
||||
print(f"下载视频: {src}")
|
||||
r = subprocess.run([coscli, "-c", cos_config, "cp", src, dst],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0 or not os.path.exists(dst):
|
||||
print(f"下载失败: {r.stderr.strip()[:300]}")
|
||||
return None
|
||||
print(f" → {dst} ({os.path.getsize(dst)/1024/1024:.0f} MB)")
|
||||
return dst
|
||||
|
||||
|
||||
def extract_frames(ffmpeg, video, frame_dir, fps):
|
||||
"""用 ffmpeg 抽帧到 frame_dir，返回帧文件列表。"""
|
||||
os.makedirs(frame_dir, exist_ok=True)
|
||||
print(f"抽帧 (fps={fps}) ...")
|
||||
cmd = [ffmpeg, "-i", video, "-vf", f"fps={fps}", "-q:v", "2",
|
||||
os.path.join(frame_dir, "f_%06d.jpg"), "-loglevel", "error", "-y"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
print(f"抽帧失败: {r.stderr.strip()[:400]}")
|
||||
return []
|
||||
frames = sorted(glob.glob(os.path.join(frame_dir, "*.jpg")))
|
||||
print(f" 抽出 {len(frames)} 帧")
|
||||
return frames
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("videos", nargs="*", default=[], help="一个或多个视频(cos://... 或本地路径)")
|
||||
ap.add_argument("--list", default=None, help="从文件读视频列表(每行一个，可代替直接传)")
|
||||
ap.add_argument("--url", action="append", default=None, help="OCR 端点(可多次)")
|
||||
ap.add_argument("--concurrency", type=int, default=100)
|
||||
ap.add_argument("--mode", choices=["soak", "sweep"], default="soak")
|
||||
ap.add_argument("--fps", type=float, default=1.0)
|
||||
ap.add_argument("--max-frames", type=int, default=2000)
|
||||
ap.add_argument("--interval", type=int, default=5)
|
||||
ap.add_argument("--ffmpeg", default="./bin/ffmpeg")
|
||||
ap.add_argument("--coscli", default="./coscli-linux-amd64")
|
||||
ap.add_argument("--cos-config", default=".cos.yaml")
|
||||
ap.add_argument("--keep", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
endpoints = bench.get_endpoints(args.url)
|
||||
if not endpoints:
|
||||
print("没有 OCR 端点，用 --url 指定")
|
||||
return
|
||||
|
||||
# 收集视频列表：命令行参数 + --list 文件
|
||||
videos = list(args.videos)
|
||||
if args.list and os.path.exists(args.list):
|
||||
with open(args.list, encoding="utf-8") as f:
|
||||
videos += [ln.strip() for ln in f if ln.strip()]
|
||||
if not videos:
|
||||
print("没有视频，传一个或多个视频路径，或用 --list 文件")
|
||||
return
|
||||
# 每个视频均分帧额度，避免一个长视频占满图池
|
||||
per_video_cap = max(1, args.max_frames // len(videos))
|
||||
|
||||
work = tempfile.mkdtemp(prefix="video_bench_")
|
||||
try:
|
||||
all_frames = []
|
||||
for i, src in enumerate(videos):
|
||||
print(f"\n[{i+1}/{len(videos)}] {src}")
|
||||
vlocal = os.path.join(work, f"video_{i}.mp4")
|
||||
fdir = os.path.join(work, f"frames_{i}")
|
||||
v = download_video(src, args.coscli, args.cos_config, vlocal)
|
||||
if not v:
|
||||
continue
|
||||
fr = extract_frames(args.ffmpeg, v, fdir, args.fps)
|
||||
if len(fr) > per_video_cap:
|
||||
fr = fr[:per_video_cap]
|
||||
print(f" 取前 {per_video_cap} 帧")
|
||||
all_frames += fr
|
||||
# 处理完即删该视频，省空间
|
||||
if v == vlocal:
|
||||
try:
|
||||
os.remove(vlocal)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if not all_frames:
|
||||
print("没抽到任何帧")
|
||||
return
|
||||
|
||||
# 帧 → base64 图池
|
||||
print(f"\n加载 {len(all_frames)} 帧为图池 ...")
|
||||
pool = [bench.img2base64(f) for f in all_frames]
|
||||
print("=" * 60)
|
||||
print(f"图池: {len(pool)} 张真实帧(来自 {len(videos)} 个视频) | "
|
||||
f"端点: {endpoints[0]} | 并发: {args.concurrency}")
|
||||
print("=" * 60)
|
||||
|
||||
# 4) 压测
|
||||
if args.mode == "sweep":
|
||||
bench.sweep_endpoint(endpoints[0], pool[0], 10, 320, 6)
|
||||
else:
|
||||
bench.soak_endpoint(endpoints[0], pool, args.concurrency, args.interval)
|
||||
|
||||
finally:
|
||||
if args.keep:
|
||||
print(f"\n临时文件保留在: {work}")
|
||||
else:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
128
tools/回溯过滤.py
Normal file
128
tools/回溯过滤.py
Normal file
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
回溯过滤: 对【已跑出来】的成功视频, 用低置信占比信号找出"非字幕内容"(幻灯片/代码/满屏文字),
|
||||
和新加的实时过滤同一套标准。默认只列出(dry-run); --delete 才删 COS 上的垃圾 SRT。
|
||||
|
||||
用法:
|
||||
python3 tools/回溯过滤.py # 扫全部成功视频, 只列出
|
||||
python3 tools/回溯过滤.py --recent 200 # 只扫最近200个(快速试)
|
||||
python3 tools/回溯过滤.py --frac 0.35 --thr 80 # 阈值(同config默认)
|
||||
python3 tools/回溯过滤.py --delete # ★删掉判定为非字幕的 COS SRT★(不可逆!)
|
||||
产物: 非字幕清单.txt (video_cos_path, 低置信占比, srt路径)
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from config.config_utils import config
|
||||
|
||||
cos = config.get_cos_config()
|
||||
B = cos["bucket"]
|
||||
_client = CosS3Client(CosConfig(Region=cos["region"], SecretId=cos["secret_id"],
|
||||
SecretKey=cos["secret_key"]))
|
||||
|
||||
|
||||
def key_of(cos_url):
|
||||
return cos_url.split("://", 1)[1].split("/", 1)[1]
|
||||
|
||||
|
||||
def lowconf_ratio(composite_cos_path, thr):
|
||||
"""下载该视频的 ocr_result.jsonl, 返回 (低置信占比, 检测数); 失败返回 (None, 0)。"""
|
||||
key = key_of(composite_cos_path).rstrip("/") + "/ocr_result.jsonl"
|
||||
try:
|
||||
body = _client.get_object(Bucket=B, Key=key)["Body"].get_raw_stream().read()
|
||||
confs = []
|
||||
for line in body.decode("utf-8", "ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
for det in d.get("TextDetections", []):
|
||||
if det.get("DetectedText"):
|
||||
confs.append(det.get("Confidence", 0))
|
||||
if not confs:
|
||||
return None, 0
|
||||
return sum(1 for c in confs if c < thr) / len(confs), len(confs)
|
||||
except Exception:
|
||||
return None, 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--jsonl", default="数据.jsonl")
|
||||
ap.add_argument("--frac", type=float, default=0.6)
|
||||
ap.add_argument("--thr", type=int, default=80)
|
||||
ap.add_argument("--min-det", type=int, default=200)
|
||||
ap.add_argument("--recent", type=int, default=0, help="只扫最近N个(0=全部)")
|
||||
ap.add_argument("--workers", type=int, default=16)
|
||||
ap.add_argument("--delete", action="store_true", help="删掉非字幕视频的 COS SRT(不可逆!)")
|
||||
ap.add_argument("--out", default="非字幕清单.txt")
|
||||
args = ap.parse_args()
|
||||
|
||||
latest = {}
|
||||
for line in open(args.jsonl, encoding="utf-8"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
v = r.get("video_cos_path")
|
||||
if v:
|
||||
latest[v] = r
|
||||
vids = [r for r in latest.values()
|
||||
if r.get("status") == "success" and r.get("composite_cos_path")]
|
||||
if args.recent:
|
||||
vids = vids[-args.recent:]
|
||||
print(f"待扫描成功视频: {len(vids)} | 阈值: 低置信<{args.thr} 占比>{args.frac*100:.0f}% | "
|
||||
f"{'★删除模式★' if args.delete else 'dry-run(只列出)'}")
|
||||
|
||||
bad, done, errored = [], 0, 0
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
fut = {ex.submit(lowconf_ratio, r["composite_cos_path"], args.thr): r for r in vids}
|
||||
for f in as_completed(fut):
|
||||
r = fut[f]
|
||||
ratio, ndet = f.result()
|
||||
done += 1
|
||||
if ratio is None:
|
||||
errored += 1
|
||||
elif ndet >= args.min_det and ratio > args.frac:
|
||||
bad.append((r["video_cos_path"], ratio, ndet, r.get("srt_cos_path")))
|
||||
if done % 100 == 0:
|
||||
print(f" ...{done}/{len(vids)} 已扫, 命中非字幕 {len(bad)}")
|
||||
|
||||
bad.sort(key=lambda x: -x[1])
|
||||
with open(args.out, "w", encoding="utf-8") as f:
|
||||
for v, ratio, ndet, srt in bad:
|
||||
f.write(f"{v}\t低置信{ratio*100:.0f}%\t{ndet}检测\t{srt}\n")
|
||||
print(f"\n扫描完成: {done} 个 | 非字幕(应过滤) {len(bad)} 个 | 读取失败 {errored} 个")
|
||||
print(f"清单 → {args.out}")
|
||||
print("命中样例(低置信占比最高的几个):")
|
||||
for v, ratio, ndet, srt in bad[:8]:
|
||||
print(f" {ratio*100:3.0f}% {ndet:5d}检测 {os.path.basename(v)}")
|
||||
|
||||
if args.delete and bad:
|
||||
print(f"\n★删除模式: 删除 {len(bad)} 个非字幕视频的 COS SRT...")
|
||||
dn = 0
|
||||
for v, ratio, ndet, srt in bad:
|
||||
if srt:
|
||||
try:
|
||||
_client.delete_object(Bucket=B, Key=key_of(srt))
|
||||
dn += 1
|
||||
except Exception as e:
|
||||
print(" 删除失败", srt, e)
|
||||
print(f"已删除 {dn} 个 COS SRT。(清单已存 {args.out}, 需要可重跑这些视频)")
|
||||
elif bad:
|
||||
print(f"\n这是 dry-run。确认清单后, 加 --delete 删除这些垃圾SRT。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
74
tools/导出成功xlsx.py
Normal file
74
tools/导出成功xlsx.py
Normal file
@@ -0,0 +1,74 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
把 数据/ 下所有 jsonl 的【成功】记录去重汇总成一个 xlsx。
|
||||
列: 视频地址 | 字幕地址 | 时长(秒) | 时长(分:秒)
|
||||
|
||||
用法: python3 tools/导出成功xlsx.py [--dir 数据] [--out 成功字幕汇总.xlsx]
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
from openpyxl import Workbook
|
||||
|
||||
|
||||
def fmt_dur(sec):
|
||||
try:
|
||||
s = int(round(float(sec)))
|
||||
except (TypeError, ValueError):
|
||||
return ""
|
||||
if s <= 0:
|
||||
return ""
|
||||
h, rem = divmod(s, 3600)
|
||||
m, ss = divmod(rem, 60)
|
||||
return f"{h}:{m:02d}:{ss:02d}" if h else f"{m}:{ss:02d}"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dir", default="数据")
|
||||
ap.add_argument("--out", default="成功字幕汇总.xlsx")
|
||||
args = ap.parse_args()
|
||||
|
||||
files = glob.glob(os.path.join(args.dir, "*.jsonl"))
|
||||
print(f"读取 {len(files)} 个 jsonl ...")
|
||||
latest = {}
|
||||
for fp in files:
|
||||
for line in open(fp, encoding="utf-8"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
v = r.get("video_cos_path")
|
||||
if v:
|
||||
latest[v] = r
|
||||
|
||||
rows = [r for r in latest.values() if r.get("status") == "success"]
|
||||
# 按视频地址排序, 输出稳定
|
||||
rows.sort(key=lambda r: r.get("video_cos_path", ""))
|
||||
print(f"去重视频 {len(latest)} | 成功 {len(rows)} → 写 xlsx")
|
||||
|
||||
wb = Workbook(write_only=True) # 流式写, 省内存(几十万行也扛得住)
|
||||
ws = wb.create_sheet("成功字幕")
|
||||
ws.append(["视频地址", "字幕地址", "时长(秒)", "时长(分:秒)"])
|
||||
n = 0
|
||||
for r in rows:
|
||||
dur = r.get("duration")
|
||||
try:
|
||||
dsec = round(float(dur), 1) if dur is not None else ""
|
||||
except (TypeError, ValueError):
|
||||
dsec = ""
|
||||
ws.append([r.get("video_cos_path", ""),
|
||||
r.get("srt_cos_path", ""),
|
||||
dsec,
|
||||
fmt_dur(dur)])
|
||||
n += 1
|
||||
wb.save(args.out)
|
||||
print(f"完成: {n} 行 → {args.out} ({os.path.getsize(args.out)//1024//1024} MB)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
92
tools/拆分超长.py
Normal file
92
tools/拆分超长.py
Normal file
@@ -0,0 +1,92 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从已有 数据.jsonl 里把【超长视频】的 COS 路径提取到一个 txt，供以后补跑。
|
||||
|
||||
用途：本次改动之前，被"视频过长"过滤掉的视频没有单独留列表，源 txt 又已删除。
|
||||
但 数据.jsonl 每条都存了 duration，可据此把它们捞回来。
|
||||
|
||||
判定为"待补跑的超长视频"：
|
||||
- 该视频最新一条记录 reason == "视频过长"（新版记录有此字段）；或
|
||||
- duration > 阈值 且 status != success（旧记录没有 reason，靠时长兜底）
|
||||
成功处理过的(status=success)不导出。
|
||||
|
||||
用法:
|
||||
python tools/拆分超长.py # 读 ./数据.jsonl，阈值取 config 的 max_duration_seconds
|
||||
python tools/拆分超长.py 数据.jsonl --min 2400 # 指定文件和阈值(秒)
|
||||
python tools/拆分超长.py a.jsonl b.jsonl -o 超长待重跑.txt
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
def _default_threshold():
|
||||
try:
|
||||
from config.config_utils import config
|
||||
return float(config.get_subtitle_config().get('max_duration_seconds', 5400) or 5400)
|
||||
except Exception:
|
||||
return 5400.0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('jsonl', nargs='*', default=['数据.jsonl'],
|
||||
help='一个或多个 数据.jsonl(默认 ./数据.jsonl)')
|
||||
ap.add_argument('--min', type=float, default=None,
|
||||
help='超长阈值(秒)，默认取 config.max_duration_seconds')
|
||||
ap.add_argument('-o', '--out', default='超长待重跑.txt', help='输出 txt')
|
||||
args = ap.parse_args()
|
||||
|
||||
thr = args.min if args.min is not None else _default_threshold()
|
||||
files = args.jsonl or ['数据.jsonl']
|
||||
|
||||
# 多文件合并，按 video_cos_path 取最新一条
|
||||
latest = {}
|
||||
for path in files:
|
||||
if not os.path.exists(path):
|
||||
print(f"[跳过] 不存在: {path}")
|
||||
continue
|
||||
with open(path, encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
v = r.get('video_cos_path')
|
||||
if v:
|
||||
latest[v] = r
|
||||
|
||||
out_paths, seen = [], set()
|
||||
n_reason, n_dur = 0, 0
|
||||
for v, r in latest.items():
|
||||
if r.get('status') == 'success':
|
||||
continue
|
||||
reason = r.get('reason')
|
||||
dur = r.get('duration')
|
||||
hit = False
|
||||
if reason == '视频过长':
|
||||
hit = True; n_reason += 1
|
||||
elif dur is not None and dur > thr:
|
||||
hit = True; n_dur += 1
|
||||
if hit and v not in seen:
|
||||
seen.add(v); out_paths.append(v)
|
||||
|
||||
with open(args.out, 'w', encoding='utf-8') as f:
|
||||
for p in out_paths:
|
||||
f.write(p + '\n')
|
||||
|
||||
print(f"阈值 {thr:.0f}s | 扫描 {len(latest)} 个唯一视频")
|
||||
print(f"导出 {len(out_paths)} 个超长视频 → {args.out} (reason命中 {n_reason}, 时长兜底 {n_dur})")
|
||||
print("\n补跑提示: 放进 text2/ 并【换一个 数据.jsonl】(否则会被当已处理跳过):")
|
||||
print(f" mv {args.out} text2/long_rerun.txt")
|
||||
print(" # 建议在 GPU 机器上跑(长视频GPU硬解快); 放大 config.max_duration_seconds 让它们不再被过滤")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
93
tools/查合成图尺寸.py
Normal file
93
tools/查合成图尺寸.py
Normal file
@@ -0,0 +1,93 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
统计合成图的【像素尺寸】, 看有多少张超大(最长边>阈值) → 判断是不是OCR超时的根因。
|
||||
纯标准库, 不依赖 PIL / cv2 / 项目代码。只读JPEG文件头, 不解码, 很快。
|
||||
|
||||
用法:
|
||||
python3 tools/查合成图尺寸.py # 默认扫 ./split/**/composite/, 阈值4000
|
||||
python3 tools/查合成图尺寸.py --dir ./split --max 4000
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
|
||||
|
||||
def jpeg_size(path):
|
||||
"""只读 JPEG 文件头拿 (宽,高), 不解码。失败返回 None。"""
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
if f.read(2) != b'\xff\xd8':
|
||||
return None
|
||||
while True:
|
||||
b = f.read(1)
|
||||
while b and b != b'\xff':
|
||||
b = f.read(1)
|
||||
marker = f.read(1)
|
||||
while marker == b'\xff':
|
||||
marker = f.read(1)
|
||||
if not marker:
|
||||
return None
|
||||
m = marker[0]
|
||||
if 0xC0 <= m <= 0xCF and m not in (0xC4, 0xC8, 0xCC):
|
||||
f.read(3)
|
||||
h = int.from_bytes(f.read(2), 'big')
|
||||
w = int.from_bytes(f.read(2), 'big')
|
||||
return w, h
|
||||
seg = int.from_bytes(f.read(2), 'big')
|
||||
if seg < 2:
|
||||
return None
|
||||
f.seek(seg - 2, 1)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--dir', default='./split')
|
||||
ap.add_argument('--max', type=int, default=4000, help='超大阈值(最长边像素)')
|
||||
args = ap.parse_args()
|
||||
|
||||
pats = [os.path.join(args.dir, '**', 'composite', '*.jpg'),
|
||||
os.path.join(args.dir, '**', '*.jpg')]
|
||||
files = []
|
||||
for p in pats:
|
||||
files = glob.glob(p, recursive=True)
|
||||
if files:
|
||||
break
|
||||
files = [f for f in files if 'composite' in f]
|
||||
if not files:
|
||||
print(f"在 {args.dir} 下没找到 composite/*.jpg")
|
||||
return
|
||||
|
||||
tot = over = mx = 0
|
||||
sizes = []
|
||||
for f in files:
|
||||
sz = jpeg_size(f)
|
||||
if sz:
|
||||
tot += 1
|
||||
m = max(sz)
|
||||
mx = max(mx, m)
|
||||
sizes.append((os.path.getsize(f), sz, f))
|
||||
if m > args.max:
|
||||
over += 1
|
||||
|
||||
print("=" * 56)
|
||||
print(f"合成图 {tot} 张 | 最长边>{args.max} 的 {over} 张 "
|
||||
f"({over/tot*100:.0f}%) | 最大最长边 {mx}px")
|
||||
print("=" * 56)
|
||||
print("最大的 8 张:")
|
||||
for fsize, (w, h), f in sorted(sizes, reverse=True)[:8]:
|
||||
flag = ' <== 超大' if max(w, h) > args.max else ''
|
||||
print(f" {fsize//1024:5d}KB {w}x{h} 最长边={max(w,h)}{flag}")
|
||||
print()
|
||||
if over > tot * 0.3:
|
||||
print(f"⚠ {over}/{tot} 张超大 → 实锤: 超大合成图就是OCR超时根因。")
|
||||
print(f" 部署 ocr_max_image_side={args.max} 后, 这些缩到 {args.max}px, OCR 从30s降到几秒。")
|
||||
elif over == 0:
|
||||
print("✓ 没有超大图(尺寸都正常) → 超时不是图大, 是端点问题(共享/波动)。")
|
||||
else:
|
||||
print(f"部分超大({over}张), 部署尺寸上限能缓解这部分。")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
147
tools/查端点存活.py
Normal file
147
tools/查端点存活.py
Normal file
@@ -0,0 +1,147 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
逐个探测 config 里的 OCR 端点, 看有多少台【真的在服务】。
|
||||
用来确认"120台只活了20台"这类问题——死端点越多, 活端点被压得越狠 → 超时。
|
||||
|
||||
用法:
|
||||
python3 tools/查端点存活.py # 自动找一张真实帧图做OCR探测
|
||||
python3 tools/查端点存活.py --tcp # 只测TCP端口连通(更快, 不发OCR)
|
||||
python3 tools/查端点存活.py --timeout 5
|
||||
python3 tools/查端点存活.py --list-dead # 额外列出死端点IP
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import glob
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import requests
|
||||
from config.config_utils import config
|
||||
|
||||
OCR_PATH = "/youtu/ocrapi/hpgeneralocr_v3"
|
||||
DEFAULT_PORT = 18081
|
||||
|
||||
|
||||
def norm(e):
|
||||
e = str(e).strip().rstrip("/")
|
||||
if not e:
|
||||
return None
|
||||
if not e.startswith("http"):
|
||||
e = "http://" + e
|
||||
host = e.split("://", 1)[1]
|
||||
if ":" not in host:
|
||||
e = e + ":" + str(DEFAULT_PORT)
|
||||
return e
|
||||
|
||||
|
||||
def host_port(ep):
|
||||
h = ep.split("://", 1)[1]
|
||||
host, _, port = h.partition(":")
|
||||
return host, int(port or DEFAULT_PORT)
|
||||
|
||||
|
||||
def find_image_b64():
|
||||
"""找一张真实帧图转 base64; 找不到返回 None(改用TCP探测)。"""
|
||||
for pat in ("./split/**/*.jpg", "./composite/**/*.jpg", "./split/*/*.jpg"):
|
||||
for p in glob.glob(pat, recursive=True):
|
||||
try:
|
||||
with open(p, "rb") as f:
|
||||
return base64.b64encode(f.read()).decode()
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def probe_ocr(ep, img_b64, timeout):
|
||||
url = ep + OCR_PATH
|
||||
payload = {"app_id": str(int(time.time() * 1000)), "image": img_b64,
|
||||
"ocr_template": "OCR", "options": {"pdf_page_index": 0, "detect_text": True}}
|
||||
t0 = time.time()
|
||||
try:
|
||||
r = requests.post(url, json=payload, timeout=timeout,
|
||||
headers={"Content-Type": "application/json"})
|
||||
dt = time.time() - t0
|
||||
return (ep, "alive", dt, "") if r.status_code == 200 else (ep, "http_err", dt, f"HTTP {r.status_code}")
|
||||
except requests.exceptions.ConnectTimeout:
|
||||
return (ep, "dead", time.time() - t0, "连接超时")
|
||||
except requests.exceptions.ReadTimeout:
|
||||
return (ep, "slow", time.time() - t0, "读超时(连得上不回)")
|
||||
except requests.exceptions.ConnectionError:
|
||||
return (ep, "dead", time.time() - t0, "拒绝/不可达")
|
||||
except Exception as e:
|
||||
return (ep, "dead", time.time() - t0, str(e)[:40])
|
||||
|
||||
|
||||
def probe_tcp(ep, timeout):
|
||||
host, port = host_port(ep)
|
||||
t0 = time.time()
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return (ep, "alive", time.time() - t0, "TCP通")
|
||||
except socket.timeout:
|
||||
return (ep, "dead", time.time() - t0, "TCP超时")
|
||||
except Exception as e:
|
||||
return (ep, "dead", time.time() - t0, str(e)[:40])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--timeout", type=float, default=5.0)
|
||||
ap.add_argument("--tcp", action="store_true", help="只测TCP端口连通(不发OCR)")
|
||||
ap.add_argument("--list-dead", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
eps = config.get_tencent_ocr_config().get("endpoints") or []
|
||||
eps = [u for u in (norm(e) for e in eps) if u]
|
||||
|
||||
img = None
|
||||
if not args.tcp:
|
||||
img = find_image_b64()
|
||||
if img is None:
|
||||
print("没找到帧图, 改用 TCP 连通探测(--tcp)。")
|
||||
mode = "TCP连通" if (args.tcp or img is None) else "真实OCR请求"
|
||||
print(f"探测 {len(eps)} 个端点 (方式: {mode}, 超时 {args.timeout}s)...")
|
||||
|
||||
def work(e):
|
||||
if args.tcp or img is None:
|
||||
return probe_tcp(e, args.timeout)
|
||||
return probe_ocr(e, img, args.timeout)
|
||||
|
||||
res = []
|
||||
with ThreadPoolExecutor(max_workers=64) as ex:
|
||||
for f in as_completed([ex.submit(work, e) for e in eps]):
|
||||
res.append(f.result())
|
||||
|
||||
c = Counter(r[1] for r in res)
|
||||
alive = [r for r in res if r[1] in ("alive", "http_err")]
|
||||
lats = sorted(r[2] for r in alive)
|
||||
total = len(eps)
|
||||
good = c["alive"] + c["http_err"]
|
||||
print("=" * 52)
|
||||
print(f" 可用(alive/http_err): {good}")
|
||||
print(f" 慢(读超时): {c['slow']}")
|
||||
print(f" 死(连不上/拒绝): {c['dead']}")
|
||||
if lats:
|
||||
print(f" 可用端点延迟: 中位 {lats[len(lats)//2]*1000:.0f}ms 最慢 {lats[-1]*1000:.0f}ms")
|
||||
print("=" * 52)
|
||||
print(f"结论: {total} 个端点里 {good} 个可用 ({good/total*100:.0f}%)")
|
||||
if good < total * 0.8:
|
||||
print(f"⚠️ {total-good} 个端点不可用! 负载全压在 {good} 个上 → 这极可能就是OCR超时根因。")
|
||||
print(f" 把死端点服务拉起 / 或从config删掉死的, OCR容量最高可提升 {total/max(good,1):.1f} 倍。")
|
||||
else:
|
||||
print("端点基本都活着 → OCR超时不是'端点少', 是'整体容量/CPU抢占'问题。")
|
||||
|
||||
if args.list_dead:
|
||||
print("\n死/慢端点:")
|
||||
for ep, st, dt, msg in sorted(res, key=lambda x: x[1]):
|
||||
if st in ("dead", "slow"):
|
||||
print(f" {st:6s} {ep:32s} {msg}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
111
tools/检查srt质量.py
Normal file
111
tools/检查srt质量.py
Normal file
@@ -0,0 +1,111 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
扫描 SRT 文件, 报告字幕质量指标 + 抽样, 帮你快速判断OCR字幕好不好。
|
||||
纯标准库。
|
||||
|
||||
用法:
|
||||
python3 tools/检查srt质量.py ./srt # 扫目录下所有 .srt
|
||||
python3 tools/检查srt质量.py a.srt b.srt # 扫指定文件
|
||||
python3 tools/检查srt质量.py ./srt --sample 8 # 多抽几条看
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
|
||||
CJK = re.compile(r'[一-鿿]')
|
||||
|
||||
|
||||
def parse_srt(path):
|
||||
"""返回 [(idx, start, end, text), ...]"""
|
||||
try:
|
||||
raw = open(path, encoding='utf-8', errors='ignore').read()
|
||||
except OSError:
|
||||
return []
|
||||
blocks = re.split(r'\n\s*\n', raw.strip())
|
||||
out = []
|
||||
for b in blocks:
|
||||
lines = [x for x in b.splitlines() if x.strip()]
|
||||
if len(lines) < 2:
|
||||
continue
|
||||
m = re.search(r'(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)', b)
|
||||
if not m:
|
||||
continue
|
||||
g = list(map(int, m.groups()))
|
||||
st = g[0]*3600 + g[1]*60 + g[2] + g[3]/1000.0
|
||||
en = g[4]*3600 + g[5]*60 + g[6] + g[7]/1000.0
|
||||
ti = next((i for i, l in enumerate(lines) if '-->' in l), 0)
|
||||
text = ' '.join(lines[ti+1:]).strip()
|
||||
out.append((st, en, text))
|
||||
return out
|
||||
|
||||
|
||||
def has_self_repeat(s, minlen=4):
|
||||
"""检测整条里前半段==后半段这种自我重复(如 AABB → AB 重复)。"""
|
||||
n = len(s)
|
||||
if n < minlen * 2:
|
||||
return False
|
||||
h = n // 2
|
||||
return s[:h] == s[h:2*h] and len(s[:h]) >= minlen
|
||||
|
||||
|
||||
def scan(paths, sample):
|
||||
files = []
|
||||
for p in paths:
|
||||
if os.path.isdir(p):
|
||||
files += glob.glob(os.path.join(p, '**', '*.srt'), recursive=True)
|
||||
elif p.endswith('.srt'):
|
||||
files.append(p)
|
||||
if not files:
|
||||
print("没找到 .srt")
|
||||
return
|
||||
|
||||
tot_e = empty = dup = selfrep = tooshort = lowcjk = badtime = 0
|
||||
chars = 0
|
||||
samples = []
|
||||
for f in files:
|
||||
subs = parse_srt(f)
|
||||
prev = None
|
||||
for st, en, text in subs:
|
||||
tot_e += 1
|
||||
chars += len(text)
|
||||
if not text:
|
||||
empty += 1
|
||||
if prev is not None and text and text == prev:
|
||||
dup += 1
|
||||
if has_self_repeat(text):
|
||||
selfrep += 1
|
||||
if text and len(text) <= 1:
|
||||
tooshort += 1
|
||||
if text and len(CJK.findall(text)) < len(text) * 0.3 and len(text) > 4:
|
||||
lowcjk += 1 # 中文占比过低 → 可能乱码/纯符号
|
||||
if en <= st or (en - st) > 30:
|
||||
badtime += 1
|
||||
prev = text
|
||||
if subs and len(samples) < sample:
|
||||
mid = len(subs) // 2
|
||||
for s in subs[mid:mid+2]:
|
||||
samples.append((os.path.basename(f), s))
|
||||
|
||||
print("=" * 58)
|
||||
print(f"SRT 文件 {len(files)} 个 | 字幕条目 {tot_e} 条 | 平均 {chars/max(tot_e,1):.1f} 字/条")
|
||||
print("-" * 58)
|
||||
def pct(x): return f"{x} ({x/max(tot_e,1)*100:.1f}%)"
|
||||
print(f" 空条目: {pct(empty)} <- 应≈0")
|
||||
print(f" 连续重复(同上条): {pct(dup)} <- 高=去重没做好")
|
||||
print(f" 自我重复(AABB): {pct(selfrep)} <- 高=自重复bug")
|
||||
print(f" 单字条目: {pct(tooshort)}")
|
||||
print(f" 低中文占比(疑乱码):{pct(lowcjk)} <- 高=OCR乱/纯符号")
|
||||
print(f" 时间轴异常: {pct(badtime)} <- 重叠/0时长/超30s")
|
||||
print("=" * 58)
|
||||
print("抽样字幕:")
|
||||
for fn, (st, en, text) in samples[:sample]:
|
||||
print(f" [{st:6.1f}-{en:6.1f}] {text[:50]} ({fn})")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('paths', nargs='+')
|
||||
ap.add_argument('--sample', type=int, default=6)
|
||||
a = ap.parse_args()
|
||||
scan(a.paths, a.sample)
|
||||
92
tools/看OCR分布.py
Normal file
92
tools/看OCR分布.py
Normal file
@@ -0,0 +1,92 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
聚合 ./ocr_dispatch/<pid>.json (各 worker 进程写的派发计数), 看 119 个 OCR 端点各分到多少,
|
||||
直接验证 P2C 负载均衡是否均匀。
|
||||
|
||||
用法:
|
||||
python3 tools/看OCR分布.py # 默认只统计最近5分钟内更新过的进程(活进程)
|
||||
python3 tools/看OCR分布.py --all # 包含所有(含已退出的旧进程文件)
|
||||
python3 tools/看OCR分布.py --fresh 120 # 只算最近120秒更新的
|
||||
python3 tools/看OCR分布.py --top 20 # 只看派发最多/最少各20个
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import glob
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dir", default="./ocr_dispatch", help="计数文件目录")
|
||||
ap.add_argument("--fresh", type=int, default=300, help="只统计最近N秒更新过的进程文件(默认300)")
|
||||
ap.add_argument("--all", action="store_true", help="统计所有文件(忽略新鲜度)")
|
||||
ap.add_argument("--top", type=int, default=0, help="只显示派发最多/最少各N个(0=全部)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not os.path.isdir(args.dir):
|
||||
print(f"没有目录 {args.dir} —— 还没产生派发计数。")
|
||||
print("确认 config.tencentcloud.ocr.dispatch_log_every > 0, 且程序已跑一会儿(每1000次派发才落盘)。")
|
||||
raise SystemExit(0)
|
||||
|
||||
now = time.time()
|
||||
files = glob.glob(os.path.join(args.dir, "*.json"))
|
||||
counts = {} # endpoint -> 总派发数
|
||||
lat_sum = {} # endpoint -> 延迟加权和(按该进程total加权)
|
||||
lat_w = {}
|
||||
nproc = 0
|
||||
grand = 0
|
||||
for fp in files:
|
||||
try:
|
||||
if not args.all and (now - os.path.getmtime(fp)) > args.fresh:
|
||||
continue
|
||||
d = json.load(open(fp, encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
nproc += 1
|
||||
for ep, c in (d.get("counts") or {}).items():
|
||||
counts[ep] = counts.get(ep, 0) + c
|
||||
grand += c
|
||||
for ep, l in (d.get("lat_ewma") or {}).items():
|
||||
tot = d.get("total", 0) or 0
|
||||
lat_sum[ep] = lat_sum.get(ep, 0.0) + l * tot
|
||||
lat_w[ep] = lat_w.get(ep, 0) + tot
|
||||
|
||||
if not counts:
|
||||
print(f"没有可统计的计数(活进程文件0个)。用 --all 看历史, 或确认程序在跑。")
|
||||
raise SystemExit(0)
|
||||
|
||||
n = len(counts)
|
||||
avg = grand / n if n else 0
|
||||
rows = []
|
||||
for ep, c in counts.items():
|
||||
lat = (lat_sum.get(ep, 0) / lat_w[ep]) if lat_w.get(ep) else 0.0
|
||||
rows.append((c, ep, lat))
|
||||
rows.sort(reverse=True)
|
||||
|
||||
mx = rows[0][0] if rows else 0
|
||||
mn = rows[-1][0] if rows else 0
|
||||
import statistics
|
||||
cs = [r[0] for r in rows]
|
||||
std = statistics.pstdev(cs) if len(cs) > 1 else 0
|
||||
|
||||
print("=" * 60)
|
||||
print(f"统计进程数: {nproc} 端点数: {n} 总派发: {grand}")
|
||||
print(f"每端点平均: {avg:.0f} 最多: {mx} 最少: {mn} max/min: {(mx/mn) if mn else float('inf'):.2f}")
|
||||
print(f"标准差: {std:.0f} 变异系数(越小越均匀): {(std/avg*100) if avg else 0:.1f}%")
|
||||
print("=" * 60)
|
||||
|
||||
show = rows
|
||||
note = ""
|
||||
if args.top and len(rows) > 2 * args.top:
|
||||
show = rows[:args.top] + [(-1, "...", -1)] + rows[-args.top:]
|
||||
note = f"(只显示最多/最少各 {args.top} 个)"
|
||||
print(f"端点派发明细 {note}:")
|
||||
print(f" {'端点':<24}{'派发数':>8}{'占比':>8}{'延迟ms':>8}")
|
||||
for c, ep, lat in show:
|
||||
if c < 0:
|
||||
print(" ...")
|
||||
continue
|
||||
print(f" {ep:<24}{c:>8}{c/grand*100:>7.1f}%{lat*1000:>7.0f}")
|
||||
|
||||
print()
|
||||
print("判读: 变异系数 <15% 算很均匀; max/min 接近1最好。")
|
||||
print(" 某端点'派发少+延迟高'=它慢被P2C自动避开(正常); '派发少+延迟正常'=可能刚冷却过。")
|
||||
Reference in New Issue
Block a user