首次提交

2026-09-28 16:09:31 +08:00
commit 4a6839319a
71 changed files with 12144 additions and 0 deletions
--- a/tools/回溯过滤.py
+++ b/tools/回溯过滤.py
@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+"""
+回溯过滤: 对【已跑出来】的成功视频, 用低置信占比信号找出"非字幕内容"(幻灯片/代码/满屏文字),
+和新加的实时过滤同一套标准。默认只列出(dry-run); --delete 才删 COS 上的垃圾 SRT。
+
+用法:
+  python3 tools/回溯过滤.py                       # 扫全部成功视频, 只列出
+  python3 tools/回溯过滤.py --recent 200           # 只扫最近200个(快速试)
+  python3 tools/回溯过滤.py --frac 0.35 --thr 80   # 阈值(同config默认)
+  python3 tools/回溯过滤.py --delete               # ★删掉判定为非字幕的 COS SRT★(不可逆!)
+产物: 非字幕清单.txt (video_cos_path, 低置信占比, srt路径)
+"""
+import argparse
+import json
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from qcloud_cos import CosConfig, CosS3Client
+from config.config_utils import config
+
+cos = config.get_cos_config()
+B = cos["bucket"]
+_client = CosS3Client(CosConfig(Region=cos["region"], SecretId=cos["secret_id"],
+                                SecretKey=cos["secret_key"]))
+
+
+def key_of(cos_url):
+    return cos_url.split("://", 1)[1].split("/", 1)[1]
+
+
+def lowconf_ratio(composite_cos_path, thr):
+    """下载该视频的 ocr_result.jsonl, 返回 (低置信占比, 检测数); 失败返回 (None, 0)。"""
+    key = key_of(composite_cos_path).rstrip("/") + "/ocr_result.jsonl"
+    try:
+        body = _client.get_object(Bucket=B, Key=key)["Body"].get_raw_stream().read()
+        confs = []
+        for line in body.decode("utf-8", "ignore").splitlines():
+            line = line.strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                d = json.loads(line)
+            except Exception:
+                continue
+            for det in d.get("TextDetections", []):
+                if det.get("DetectedText"):
+                    confs.append(det.get("Confidence", 0))
+        if not confs:
+            return None, 0
+        return sum(1 for c in confs if c < thr) / len(confs), len(confs)
+    except Exception:
+        return None, 0
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--jsonl", default="数据.jsonl")
+    ap.add_argument("--frac", type=float, default=0.6)
+    ap.add_argument("--thr", type=int, default=80)
+    ap.add_argument("--min-det", type=int, default=200)
+    ap.add_argument("--recent", type=int, default=0, help="只扫最近N个(0=全部)")
+    ap.add_argument("--workers", type=int, default=16)
+    ap.add_argument("--delete", action="store_true", help="删掉非字幕视频的 COS SRT(不可逆!)")
+    ap.add_argument("--out", default="非字幕清单.txt")
+    args = ap.parse_args()
+
+    latest = {}
+    for line in open(args.jsonl, encoding="utf-8"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            r = json.loads(line)
+        except Exception:
+            continue
+        v = r.get("video_cos_path")
+        if v:
+            latest[v] = r
+    vids = [r for r in latest.values()
+            if r.get("status") == "success" and r.get("composite_cos_path")]
+    if args.recent:
+        vids = vids[-args.recent:]
+    print(f"待扫描成功视频: {len(vids)} | 阈值: 低置信<{args.thr} 占比>{args.frac*100:.0f}% | "
+          f"{'★删除模式★' if args.delete else 'dry-run(只列出)'}")
+
+    bad, done, errored = [], 0, 0
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        fut = {ex.submit(lowconf_ratio, r["composite_cos_path"], args.thr): r for r in vids}
+        for f in as_completed(fut):
+            r = fut[f]
+            ratio, ndet = f.result()
+            done += 1
+            if ratio is None:
+                errored += 1
+            elif ndet >= args.min_det and ratio > args.frac:
+                bad.append((r["video_cos_path"], ratio, ndet, r.get("srt_cos_path")))
+            if done % 100 == 0:
+                print(f"  ...{done}/{len(vids)} 已扫, 命中非字幕 {len(bad)}")
+
+    bad.sort(key=lambda x: -x[1])
+    with open(args.out, "w", encoding="utf-8") as f:
+        for v, ratio, ndet, srt in bad:
+            f.write(f"{v}\t低置信{ratio*100:.0f}%\t{ndet}检测\t{srt}\n")
+    print(f"\n扫描完成: {done} 个 | 非字幕(应过滤) {len(bad)} 个 | 读取失败 {errored} 个")
+    print(f"清单 → {args.out}")
+    print("命中样例(低置信占比最高的几个):")
+    for v, ratio, ndet, srt in bad[:8]:
+        print(f"  {ratio*100:3.0f}%  {ndet:5d}检测  {os.path.basename(v)}")
+
+    if args.delete and bad:
+        print(f"\n★删除模式: 删除 {len(bad)} 个非字幕视频的 COS SRT...")
+        dn = 0
+        for v, ratio, ndet, srt in bad:
+            if srt:
+                try:
+                    _client.delete_object(Bucket=B, Key=key_of(srt))
+                    dn += 1
+                except Exception as e:
+                    print("  删除失败", srt, e)
+        print(f"已删除 {dn} 个 COS SRT。(清单已存 {args.out}, 需要可重跑这些视频)")
+    elif bad:
+        print(f"\n这是 dry-run。确认清单后, 加 --delete 删除这些垃圾SRT。")
+
+
+if __name__ == "__main__":
+    main()