93 lines
3.4 KiB
Python
93 lines
3.4 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
从已有 数据.jsonl 里把【超长视频】的 COS 路径提取到一个 txt，供以后补跑。
|
||
|
||
用途：本次改动之前，被"视频过长"过滤掉的视频没有单独留列表，源 txt 又已删除。
|
||
但 数据.jsonl 每条都存了 duration，可据此把它们捞回来。
|
||
|
||
判定为"待补跑的超长视频"：
|
||
- 该视频最新一条记录 reason == "视频过长"（新版记录有此字段）；或
|
||
- duration > 阈值 且 status != success（旧记录没有 reason，靠时长兜底）
|
||
成功处理过的(status=success)不导出。
|
||
|
||
用法:
|
||
python tools/拆分超长.py # 读 ./数据.jsonl，阈值取 config 的 max_duration_seconds
|
||
python tools/拆分超长.py 数据.jsonl --min 2400 # 指定文件和阈值(秒)
|
||
python tools/拆分超长.py a.jsonl b.jsonl -o 超长待重跑.txt
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import sys
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
|
||
def _default_threshold():
|
||
try:
|
||
from config.config_utils import config
|
||
return float(config.get_subtitle_config().get('max_duration_seconds', 5400) or 5400)
|
||
except Exception:
|
||
return 5400.0
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument('jsonl', nargs='*', default=['数据.jsonl'],
|
||
help='一个或多个 数据.jsonl(默认 ./数据.jsonl)')
|
||
ap.add_argument('--min', type=float, default=None,
|
||
help='超长阈值(秒)，默认取 config.max_duration_seconds')
|
||
ap.add_argument('-o', '--out', default='超长待重跑.txt', help='输出 txt')
|
||
args = ap.parse_args()
|
||
|
||
thr = args.min if args.min is not None else _default_threshold()
|
||
files = args.jsonl or ['数据.jsonl']
|
||
|
||
# 多文件合并，按 video_cos_path 取最新一条
|
||
latest = {}
|
||
for path in files:
|
||
if not os.path.exists(path):
|
||
print(f"[跳过] 不存在: {path}")
|
||
continue
|
||
with open(path, encoding='utf-8') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
r = json.loads(line)
|
||
except Exception:
|
||
continue
|
||
v = r.get('video_cos_path')
|
||
if v:
|
||
latest[v] = r
|
||
|
||
out_paths, seen = [], set()
|
||
n_reason, n_dur = 0, 0
|
||
for v, r in latest.items():
|
||
if r.get('status') == 'success':
|
||
continue
|
||
reason = r.get('reason')
|
||
dur = r.get('duration')
|
||
hit = False
|
||
if reason == '视频过长':
|
||
hit = True; n_reason += 1
|
||
elif dur is not None and dur > thr:
|
||
hit = True; n_dur += 1
|
||
if hit and v not in seen:
|
||
seen.add(v); out_paths.append(v)
|
||
|
||
with open(args.out, 'w', encoding='utf-8') as f:
|
||
for p in out_paths:
|
||
f.write(p + '\n')
|
||
|
||
print(f"阈值 {thr:.0f}s | 扫描 {len(latest)} 个唯一视频")
|
||
print(f"导出 {len(out_paths)} 个超长视频 → {args.out} (reason命中 {n_reason}, 时长兜底 {n_dur})")
|
||
print("\n补跑提示: 放进 text2/ 并【换一个 数据.jsonl】(否则会被当已处理跳过):")
|
||
print(f" mv {args.out} text2/long_rerun.txt")
|
||
print(" # 建议在 GPU 机器上跑(长视频GPU硬解快); 放大 config.max_duration_seconds 让它们不再被过滤")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|