629 lines
29 KiB
Python
629 lines
29 KiB
Python
import argparse
|
||
import glob
|
||
import json
|
||
import shutil
|
||
import subprocess
|
||
import logging
|
||
from logging.handlers import RotatingFileHandler
|
||
import os
|
||
import stat
|
||
import threading
|
||
import time
|
||
import cv2
|
||
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
|
||
from datetime import datetime
|
||
from config.config_utils import config
|
||
from tool.process import process_single_video, derive_video_name
|
||
from tool.log.log_manager import logger as steplog
|
||
from tool import job_cos
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s [%(levelname)s] %(message)s',
|
||
handlers=[
|
||
# 常驻worker一跑就是数天/数周, 普通FileHandler会无限增长; 100M/份, 最多留5份(共500M封顶)
|
||
RotatingFileHandler('batch_processor.log', maxBytes=100 * 1024 * 1024, backupCount=5, encoding='utf-8'),
|
||
logging.StreamHandler()
|
||
]
|
||
)
|
||
|
||
JSONL_PATH = "数据.jsonl"
|
||
|
||
|
||
# ==================== 工具函数 ====================
|
||
|
||
def parse_cos_path(cos_path: str):
|
||
"""cos://bucket/path/video.mp4 → ('cos://bucket/path/', 'video.mp4')"""
|
||
cos_path = cos_path.strip()
|
||
idx = cos_path.rfind('/')
|
||
return cos_path[:idx + 1], cos_path[idx + 1:]
|
||
|
||
|
||
def get_video_duration(local_path: str) -> float:
|
||
if not os.path.exists(local_path):
|
||
return 0.0
|
||
cap = cv2.VideoCapture(local_path)
|
||
if not cap.isOpened():
|
||
return 0.0
|
||
frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)
|
||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||
cap.release()
|
||
return round(frames / fps, 2) if fps > 0 else 0.0
|
||
|
||
|
||
def _read_log_stats(video_name: str):
|
||
"""从 log/<video>.json 读取图片数量统计与各阶段耗时。
|
||
返回 (image_counts: dict|None, stage_durations: dict|None, video_duration: float|None)，读取失败返回全 None。"""
|
||
log_path = os.path.join("log", f"{video_name}.json")
|
||
if not os.path.exists(log_path):
|
||
return None, None, None
|
||
try:
|
||
with open(log_path, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
meta = data.get("metadata", {})
|
||
custom = meta.get("custom_stats", {}) or {}
|
||
durations = meta.get("program_durations", {}) or {}
|
||
image_counts = {
|
||
"frame_count": custom.get("frame_count"),
|
||
"composite_ocr_count": custom.get("composite_ocr_count"),
|
||
"single_frame_ocr_count": custom.get("single_frame_ocr_count"),
|
||
"total_ocr_count": custom.get("total_ocr_count"),
|
||
}
|
||
# 各阶段耗时（秒），四舍五入到2位
|
||
stage_durations = {k: round(v, 2) for k, v in durations.items()}
|
||
video_duration = custom.get("video_duration") # process.py 在删视频前用 ffprobe 存的(AV1也准)
|
||
return image_counts, stage_durations, video_duration
|
||
except (OSError, IOError, json.JSONDecodeError) as e:
|
||
logging.warning(f"读取日志统计失败 {log_path}: {e}")
|
||
return None, None, None
|
||
|
||
|
||
def upload_srt(srt_local: str, cos_dir: str, coscli_path: str, cos_config_path: str):
|
||
"""上传 SRT 文件到视频对应的 COS 目录，返回 COS 路径；失败返回 None"""
|
||
if not os.path.exists(srt_local):
|
||
return None
|
||
cos_dest = cos_dir + os.path.basename(srt_local)
|
||
cmd = [coscli_path, "-c", cos_config_path, "cp", srt_local, cos_dest]
|
||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||
if result.returncode == 0:
|
||
return cos_dest
|
||
logging.error(f"SRT 上传失败: {result.stderr}")
|
||
return None
|
||
|
||
|
||
def get_available_mem_gb():
|
||
"""读取系统可用内存(GB)。Linux 读 /proc/meminfo 的 MemAvailable；
|
||
非 Linux 或读取失败返回 None（此时内存保护自动失效）。"""
|
||
try:
|
||
with open('/proc/meminfo', 'r') as f:
|
||
for line in f:
|
||
if line.startswith('MemAvailable:'):
|
||
return float(line.split()[1]) / 1024 / 1024 # kB → GB
|
||
except (OSError, ValueError, IndexError):
|
||
return None
|
||
return None
|
||
|
||
|
||
def get_available_disk_gb(path: str = "."):
|
||
"""读取 path 所在磁盘的可用空间(GB)。用 shutil.disk_usage，跨平台(Windows/Linux都行，
|
||
不像内存检查那样只能读 /proc/meminfo)。读取失败(路径不存在等)返回 None，此时磁盘保护自动失效。"""
|
||
try:
|
||
return shutil.disk_usage(path).free / 1024 / 1024 / 1024
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
def get_load_avg():
|
||
"""返回 (1, 5, 15) 分钟负载；非 Unix 返回 None。"""
|
||
try:
|
||
return os.getloadavg()
|
||
except (OSError, AttributeError):
|
||
return None
|
||
|
||
|
||
def load_processed(jsonl_path: str) -> set:
|
||
"""从数据.jsonl 读取已处理的 COS 路径集合，用于跨批次断点续传"""
|
||
processed = set()
|
||
if not os.path.exists(jsonl_path):
|
||
return processed
|
||
with open(jsonl_path, 'r', encoding='utf-8') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
rec = json.loads(line)
|
||
vid = rec.get('video_cos_path')
|
||
if vid:
|
||
processed.add(vid)
|
||
except json.JSONDecodeError:
|
||
pass
|
||
return processed
|
||
|
||
|
||
def force_rmtree(path):
|
||
"""强制删除目录（兼容 Windows 和 Linux）"""
|
||
if not os.path.exists(path):
|
||
return
|
||
for attempt in range(3):
|
||
try:
|
||
shutil.rmtree(path)
|
||
return
|
||
except PermissionError:
|
||
if attempt < 2:
|
||
time.sleep(1)
|
||
for root, dirs, files in os.walk(path):
|
||
for name in files:
|
||
try:
|
||
os.chmod(os.path.join(root, name), stat.S_IWUSR | stat.S_IRUSR)
|
||
except Exception:
|
||
pass
|
||
else:
|
||
if os.name == 'nt':
|
||
os.system(f'rmdir /s /q "{path}"')
|
||
else:
|
||
os.system(f'rm -rf "{path}"')
|
||
|
||
|
||
# ==================== 统计 ====================
|
||
|
||
class Stats:
|
||
def __init__(self, total: int):
|
||
self._lock = threading.Lock()
|
||
self.success = 0
|
||
self.no_subtitle = 0
|
||
self.error = 0
|
||
self.done = 0
|
||
self.total = total
|
||
|
||
def update(self, status: str):
|
||
with self._lock:
|
||
self.done += 1
|
||
if status == 'success':
|
||
self.success += 1
|
||
elif status == 'no_subtitle':
|
||
self.no_subtitle += 1
|
||
else:
|
||
self.error += 1
|
||
print(
|
||
f"[进度 {self.done}/{self.total}] "
|
||
f"成功: {self.success} 无字幕: {self.no_subtitle} 错误: {self.error} "
|
||
f"剩余: {self.total - self.done}",
|
||
flush=True
|
||
)
|
||
|
||
|
||
# ==================== 处理器 ====================
|
||
|
||
def _wait_for_memory(video_name: str, mem_guard_gb: float):
|
||
"""内存可用量低于阈值时阻塞，等已在处理的视频释放内存后再启动新视频，避免 OOM。
|
||
模块级函数：进程池下每个 worker 进程独立调用。读不到内存时直接放行。"""
|
||
if mem_guard_gb <= 0:
|
||
return
|
||
waited = 0
|
||
while True:
|
||
avail = get_available_mem_gb()
|
||
if avail is None or avail >= mem_guard_gb:
|
||
return
|
||
if waited % 15 == 0:
|
||
logging.warning(
|
||
f"内存不足(可用 {avail:.1f}G < 阈值 {mem_guard_gb}G)，"
|
||
f"暂停启动 {video_name}，等待回收... 已等 {waited}s"
|
||
)
|
||
time.sleep(3)
|
||
waited += 3
|
||
|
||
|
||
def _wait_for_disk(video_name: str, disk_guard_gb: float):
|
||
"""可用磁盘空间低于阈值时阻塞，等已在处理的视频清理释放空间后再启动新视频。
|
||
防的是磁盘写满级联失败(下载/抽帧写一半失败、GPU硬解报错误判成"GPU失败"其实是写不进磁盘、
|
||
info_error都写不进去导致视频异常处理链路整个乱掉——实际发生过的连锁故障)。
|
||
模块级函数：进程池下每个 worker 进程独立调用。读不到磁盘空间时直接放行。"""
|
||
if disk_guard_gb <= 0:
|
||
return
|
||
waited = 0
|
||
while True:
|
||
avail = get_available_disk_gb(".")
|
||
if avail is None or avail >= disk_guard_gb:
|
||
return
|
||
if waited % 15 == 0:
|
||
logging.warning(
|
||
f"磁盘空间不足(可用 {avail:.1f}G < 阈值 {disk_guard_gb}G)，"
|
||
f"暂停启动 {video_name}，等待清理释放空间... 已等 {waited}s"
|
||
)
|
||
time.sleep(3)
|
||
waited += 3
|
||
|
||
|
||
def _short_reason(text: str) -> str:
|
||
"""把 info_error 里的长错误串归一成一个短分类标签，便于 数据.jsonl 里按原因统计/筛选。"""
|
||
if not text:
|
||
return None
|
||
for kw in ("视频过长", "视频过短", "字幕太少", "冗余文字过多",
|
||
"下载失败", "抽帧", "拼接失败", "字幕生成失败", "OCR"):
|
||
if kw in text:
|
||
return kw
|
||
if "置信度" in text or "confidence" in text.lower():
|
||
return "置信度太低"
|
||
if "未找到合适的字幕区域" in text or "未检测" in text:
|
||
return "无字幕区域"
|
||
# 兜底截断: 之前只截40字符——异常串常见格式是"RuntimeError: 视频<video_name>真正原因",
|
||
# video_name(文件名+MD5后缀)本身就有二三十字符，40字符的窗口经常在真正原因(冒号后面
|
||
# 那段)出现之前就被切没了，数据.jsonl里全变成"RuntimeError: 视频<name>"这种没有信息量
|
||
# 的记录（实测发生过：一小时63条error全部是这个模式）。加大到150，给真实原因留出空间；
|
||
# 完整未截断的原始异常串仍然保留在 record["error_reason"] 里，这里只是短标签。
|
||
return text.strip()[:150]
|
||
|
||
|
||
def _read_error_reason(video_name: str) -> str:
|
||
"""从 ./info_error/<video>.json 读出过滤/错误原因(归类后的短标签)。"""
|
||
err_json = f"./info_error/{video_name}.json"
|
||
if not os.path.exists(err_json):
|
||
return None
|
||
try:
|
||
with open(err_json, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
return _short_reason((data.get("main_process") or {}).get("error"))
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
# 这些原因是"正常过滤(视频本就没有可用字幕)", 不是程序硬故障 → 应记 no_subtitle 而非 error。
|
||
# 旧逻辑用 '"被过滤" in json文本' 猜测, 但过滤原因只存在 main_process.error 里(不含"被过滤"),
|
||
# 于是字幕太少/冗余/置信度/非字幕等全被错记成 error。改为按原因关键词匹配, 清单可在
|
||
# config.subtitle.no_subtitle_reasons 覆盖。
|
||
_NO_SUB_REASONS_DEFAULT = [
|
||
"字幕太少", "冗余文字过多", "冗余文字太多", "无字幕区域",
|
||
"非字幕内容", "置信度太低", "未检测", "特殊字体",
|
||
"视频过短", "视频过长",
|
||
]
|
||
|
||
|
||
def _is_no_subtitle_reason(reason: str) -> bool:
|
||
"""原因是否属于'无可用字幕的正常过滤'(→ no_subtitle), 而非硬故障(→ error)。"""
|
||
if not reason:
|
||
return False
|
||
try:
|
||
keys = config.get_subtitle_config().get('no_subtitle_reasons') or _NO_SUB_REASONS_DEFAULT
|
||
except Exception:
|
||
keys = _NO_SUB_REASONS_DEFAULT
|
||
return any(k in reason for k in keys)
|
||
|
||
|
||
def process_video_task(cos_path: str, coscli_path: str, cos_config_path: str,
|
||
split_cfg: dict, mem_guard_gb: float, disk_guard_gb: float = 0) -> dict:
|
||
"""处理单个视频，返回完整 record（不触碰共享状态，可被进程池 pickle 调用）。
|
||
进程池模式下：本函数整体在独立子进程里跑 → 各自的 Python 解释器/GIL/whisper 模型 → 真多核。"""
|
||
cos_dir, filename = parse_cos_path(cos_path)
|
||
video_name = derive_video_name(cos_path) # 含路径MD5后缀, 避免不同父目录同名视频本地/COS撞车
|
||
local_video = os.path.join("./video", video_name + os.path.splitext(filename)[1])
|
||
|
||
# 内存/磁盘保护（放在计时前，等待时间不计入 elapsed_seconds）
|
||
_wait_for_memory(video_name, mem_guard_gb)
|
||
_wait_for_disk(video_name, disk_guard_gb)
|
||
|
||
status = "error"
|
||
start_dt = datetime.now()
|
||
start_perf = time.time()
|
||
record = {
|
||
"timestamp": start_dt.isoformat(),
|
||
"video_cos_path": cos_path,
|
||
"status": None,
|
||
"reason": None,
|
||
"srt_cos_path": None,
|
||
"duration": None,
|
||
"process_start": start_dt.isoformat(),
|
||
"process_end": None,
|
||
"elapsed_seconds": None,
|
||
"image_counts": None,
|
||
"stage_durations": None,
|
||
}
|
||
|
||
try:
|
||
success = process_single_video(cos_path, cos_dir)
|
||
record["duration"] = get_video_duration(local_video)
|
||
|
||
if success:
|
||
srt_local = f"./split/{video_name}/composite/{video_name}.ocr.srt"
|
||
srt_cos = upload_srt(srt_local, cos_dir, coscli_path, cos_config_path)
|
||
status = "success"
|
||
record["srt_cos_path"] = srt_cos
|
||
if os.path.exists(srt_local):
|
||
os.makedirs("./srt", exist_ok=True)
|
||
shutil.move(srt_local, os.path.join("./srt", f"{video_name}.srt"))
|
||
steplog.step(video_name, "上传完成", "srt已传" if srt_cos else "srt上传失败")
|
||
logging.info(f"成功 {filename} → {srt_cos}")
|
||
else:
|
||
# 按原因关键词归类(config 可调): 无可用字幕的正常过滤 → no_subtitle, 其余 → error。
|
||
reason = _read_error_reason(video_name)
|
||
record["reason"] = reason
|
||
is_filtered = _is_no_subtitle_reason(reason)
|
||
status = "no_subtitle" if is_filtered else "error"
|
||
logging.info(f"{'无字幕' if is_filtered else '失败'} {filename}")
|
||
|
||
except Exception as e:
|
||
reason = _short_reason(f"{type(e).__name__}: {e}")
|
||
record["error_reason"] = f"{type(e).__name__}: {e}"
|
||
record["reason"] = reason
|
||
# 过滤型异常(如 raise '未检测到字幕区域'/'视频过短') 也应归 no_subtitle, 不是硬故障。
|
||
status = "no_subtitle" if _is_no_subtitle_reason(reason) else "error"
|
||
logging.error(f"异常 {filename}: {e}")
|
||
|
||
finally:
|
||
# composite（含拼接大图/OCR中间结果）不上传 COS，处理完直接随 split_dir 一起删除本地。
|
||
split_dir = f"./split/{video_name}"
|
||
if os.path.exists(split_dir):
|
||
force_rmtree(split_dir)
|
||
if split_cfg.get('delete_video') and os.path.exists(local_video):
|
||
try:
|
||
os.remove(local_video)
|
||
except OSError:
|
||
pass
|
||
|
||
end_dt = datetime.now()
|
||
record["status"] = status
|
||
record["process_end"] = end_dt.isoformat()
|
||
record["elapsed_seconds"] = round(time.time() - start_perf, 2)
|
||
image_counts, stage_durations, video_duration = _read_log_stats(video_name)
|
||
record["image_counts"] = image_counts
|
||
record["stage_durations"] = stage_durations
|
||
# 时长优先用 process.py 删视频前 ffprobe 存的(cv2读不了AV1、且delete_video后源视频已删)
|
||
if (not record.get("duration")) and video_duration:
|
||
record["duration"] = video_duration
|
||
|
||
# 该读的(日志统计/错误原因)都读完了，本视频的临时小文件立即删，不等整批/整片跑完再统一清
|
||
# (之前是攒到 main.py 主循环/run_worker._process_shard 末尾才批量删，一片上千个视频攒一路)。
|
||
for f in (f"./log/{video_name}.json", f"./info/{video_name}.json", f"./info_error/{video_name}.json"):
|
||
try:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
except OSError:
|
||
pass
|
||
return record
|
||
|
||
|
||
def _ocr_worker_init(sem):
|
||
"""进程池 worker 启动时调用：注入【全局 OCR 并发信号量】,使所有进程共享一个硬性并发上限。"""
|
||
try:
|
||
from tool.Ocr import tencentcloud_ocr
|
||
tencentcloud_ocr.set_ocr_semaphore(sem)
|
||
except Exception as e:
|
||
logging.warning(f"注入全局OCR信号量失败: {e}")
|
||
|
||
|
||
class VideoProcessor:
|
||
def __init__(self, max_workers: int = 10, mem_guard_gb: float = 2.0, executor: str = "process",
|
||
max_tasks_per_child: int = 30, disk_guard_gb: float = 0):
|
||
self.max_workers = max_workers
|
||
self.mem_guard_gb = mem_guard_gb # 可用内存低于此值(GB)时暂停启动新视频，<=0 关闭
|
||
self.disk_guard_gb = disk_guard_gb # 本地磁盘可用空间低于此值(GB)时暂停启动新视频，<=0 关闭
|
||
self.executor = executor # "process"=进程池(绕开GIL,真多核) / "thread"=线程池
|
||
self.max_tasks_per_child = max_tasks_per_child # 进程池: worker处理这么多视频后重启释放内存
|
||
self._jsonl_lock = threading.Lock()
|
||
|
||
def _append_jsonl(self, record: dict):
|
||
with self._jsonl_lock:
|
||
with open(JSONL_PATH, 'a', encoding='utf-8') as f:
|
||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
|
||
def process_batch(self, tasks: list, coscli_path: str,
|
||
cos_config_path: str, split_cfg: dict) -> Stats:
|
||
stats = Stats(total=len(tasks))
|
||
mode = "进程池(多核,绕GIL)" if self.executor == "process" else "线程池"
|
||
logging.info(f"开始处理 {len(tasks)} 个视频 ({mode} 并发={self.max_workers})")
|
||
|
||
# 后台定时打印进度（每30秒一次，处理完后停止）
|
||
stop_event = threading.Event()
|
||
def _heartbeat():
|
||
while not stop_event.wait(30):
|
||
with stats._lock:
|
||
d, t = stats.done, stats.total
|
||
s, n, e = stats.success, stats.no_subtitle, stats.error
|
||
print(
|
||
f"[进行中 {d}/{t}] 成功: {s} 无字幕: {n} 错误: {e} 剩余: {t - d}",
|
||
flush=True
|
||
)
|
||
threading.Thread(target=_heartbeat, daemon=True).start()
|
||
|
||
# 进程池: 每个视频在独立子进程跑 → 各自 GIL/whisper → 真多核(getsrt 的 jieba/拼音不再串行)
|
||
# 线程池: 退化兜底(--executor thread)
|
||
# 用 forkserver: 从干净的服务进程 fork worker,避免"主进程多线程+fork"继承到被持有的锁而死锁;
|
||
# 重模块(cv2/whisper)在 forkserver 里只导入一次, worker 共享(COW),启动快。
|
||
if self.executor == "process":
|
||
import multiprocessing
|
||
try:
|
||
_ctx = multiprocessing.get_context("forkserver")
|
||
except ValueError:
|
||
_ctx = multiprocessing.get_context() # 非 POSIX 等回退默认
|
||
# 全局OCR并发上限: 父进程建跨进程信号量, 经 initializer 注入每个 worker →
|
||
# 不管 max-workers × ocr_workers 多大, 全局同时在飞的OCR请求被硬性卡在此值, 防打爆端点。
|
||
_max_global_ocr = int(config.get_subtitle_config().get('max_global_ocr', 0) or 0)
|
||
_init, _initargs = None, ()
|
||
if _max_global_ocr > 0:
|
||
_init = _ocr_worker_init
|
||
_initargs = (_ctx.Semaphore(_max_global_ocr),)
|
||
logging.info(f"全局OCR并发上限: {_max_global_ocr}")
|
||
# max_tasks_per_child: worker 处理这么多视频后【自动重启】释放内存(需 Python≥3.11)。
|
||
# ⚠️ Python 3.11 该机制有"补不满"的坑: 频繁回收会让 worker 数随时间从 N 缩到个位数甚至0
|
||
# → as_completed 永久挂起 → 进度冻住。内存充裕时建议【关闭】(设 0/None), 靠整体重启释放内存。
|
||
_mtpc = self.max_tasks_per_child if (self.max_tasks_per_child and self.max_tasks_per_child > 0) else None
|
||
if _mtpc is None:
|
||
logging.info("max_tasks_per_child=关闭(worker不自动回收, 避免3.11回收缩水; 内存靠整体重启/mem-guard控制)")
|
||
try:
|
||
pool = ProcessPoolExecutor(max_workers=self.max_workers, mp_context=_ctx,
|
||
max_tasks_per_child=_mtpc,
|
||
initializer=_init, initargs=_initargs)
|
||
except TypeError:
|
||
logging.warning("当前Python不支持 max_tasks_per_child(需≥3.11), 进程不自动回收, 注意内存")
|
||
pool = ProcessPoolExecutor(max_workers=self.max_workers, mp_context=_ctx,
|
||
initializer=_init, initargs=_initargs)
|
||
else:
|
||
# 线程池模式: 同进程内, 用 threading 信号量直接限流(可选)
|
||
_max_global_ocr = int(config.get_subtitle_config().get('max_global_ocr', 0) or 0)
|
||
if _max_global_ocr > 0:
|
||
from tool.Ocr import tencentcloud_ocr
|
||
tencentcloud_ocr.set_ocr_semaphore(threading.Semaphore(_max_global_ocr))
|
||
logging.info(f"全局OCR并发上限: {_max_global_ocr}")
|
||
pool = ThreadPoolExecutor(max_workers=self.max_workers)
|
||
with pool as executor:
|
||
futures = {
|
||
executor.submit(
|
||
process_video_task, cos_path,
|
||
coscli_path, cos_config_path, split_cfg, self.mem_guard_gb, self.disk_guard_gb
|
||
): cos_path
|
||
for cos_path in tasks
|
||
}
|
||
# 父进程单线程收集结果 → 统一写 jsonl(单写者,无需跨进程锁) + 更新统计
|
||
for future in as_completed(futures):
|
||
cos_path = futures[future]
|
||
try:
|
||
record = future.result()
|
||
except Exception as e:
|
||
# 子进程崩溃(段错误/被OOM杀)等 → 仍记一条 error
|
||
logging.error(f"任务异常 {cos_path}: {e}")
|
||
record = {"video_cos_path": cos_path, "status": "error",
|
||
"error_reason": f"worker崩溃: {type(e).__name__}: {e}",
|
||
"process_end": datetime.now().isoformat()}
|
||
self._append_jsonl(record)
|
||
stats.update(record.get("status", "error"))
|
||
|
||
stop_event.set() # 停止心跳线程
|
||
logging.info(
|
||
f"批次完成 — 成功: {stats.success} | "
|
||
f"无字幕: {stats.no_subtitle} | 错误: {stats.error}"
|
||
)
|
||
return stats
|
||
|
||
|
||
# ==================== 入口 ====================
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='视频字幕批量提取（txt 内填 COS 完整路径）')
|
||
parser.add_argument('--source-list', default='./text2',
|
||
help='包含 COS 路径列表的 txt 文件目录')
|
||
parser.add_argument('--max-workers', type=int, default=None,
|
||
help='最大并发线程数 (不填/0 = 自动按本机CPU核数选)')
|
||
parser.add_argument('--mem-guard-gb', type=float, default=2.0,
|
||
help='可用内存低于此值(GB)时暂停启动新视频，防 OOM；设 0 关闭 (默认: 2.0)')
|
||
parser.add_argument('--disk-guard-gb', type=float, default=10.0,
|
||
help='本地磁盘可用空间低于此值(GB)时暂停启动新视频，给下载/抽帧/合成图预留空间，'
|
||
'防止磁盘写满导致级联失败；设 0 关闭 (默认: 10.0)')
|
||
parser.add_argument('--max-tasks-per-child', type=int, default=30,
|
||
help='进程池模式: 每个worker处理这么多视频后重启释放内存(默认30, 需Python≥3.11)')
|
||
parser.add_argument('--executor', choices=['process', 'thread'], default='process',
|
||
help='并发方式: process=进程池(默认,绕开GIL真多核) / thread=线程池(旧行为,CPU重计算会被GIL串行)')
|
||
args = parser.parse_args()
|
||
|
||
# ---- 启动时打印机器资源与并发设置，便于判断参数是否合理 ----
|
||
cpu = os.cpu_count() or 0
|
||
if not args.max_workers:
|
||
args.max_workers = job_cos.auto_max_workers(cpu, disk_guard_gb=args.disk_guard_gb)
|
||
logging.info(f"--max-workers 未指定, 自动选为 {args.max_workers}(本机CPU核数={cpu or '未知'}; "
|
||
f"若本机还跑了OCR服务但不在 tencentcloud.ocr.endpoints 里, 自动探测不到, 请手动传小一点的值; "
|
||
f"同时按可用磁盘 ÷ 每视频预估占用(dispatch.est_disk_gb_per_video)做了收紧)")
|
||
# 供子进程(forkserver fork出来的worker)读取: splitframe.ffmpeg_threads=auto 时按此摊分CPU
|
||
os.environ['OCR_MAX_WORKERS'] = str(args.max_workers)
|
||
mem = get_available_mem_gb()
|
||
disk = get_available_disk_gb(".")
|
||
load = get_load_avg()
|
||
mem_str = f"{mem:.1f}G" if mem is not None else "未知(非Linux?)"
|
||
disk_str = f"{disk:.1f}G" if disk is not None else "未知"
|
||
load_str = (f"{load[0]:.2f}/{load[1]:.2f}/{load[2]:.2f}" if load else "未知")
|
||
logging.info("=" * 56)
|
||
logging.info(f"机器资源: CPU {cpu} 核 | 可用内存 {mem_str} | 可用磁盘 {disk_str} | 负载(1/5/15) {load_str}")
|
||
_exec_str = "进程池(多核,绕GIL)" if args.executor == "process" else "线程池(GIL会串行CPU计算)"
|
||
logging.info(f"并发设置: {_exec_str} | max-workers={args.max_workers} | "
|
||
f"内存保护阈值={args.mem_guard_gb}G | 磁盘保护阈值={args.disk_guard_gb}G")
|
||
if args.executor == "process" and cpu:
|
||
logging.info(f"提示: 进程池模式建议 max-workers ≈ CPU核数({cpu})上下，每个视频独立进程真多核；"
|
||
f"内存够的话(每进程约+0.5~1G含whisper模型)可略超")
|
||
if mem is not None and args.mem_guard_gb >= mem:
|
||
logging.warning(f"⚠️ 内存保护阈值({args.mem_guard_gb}G) ≥ 当前可用内存({mem:.1f}G)，"
|
||
f"几乎所有视频都会被卡住等待，请调小 --mem-guard-gb 或释放内存")
|
||
if disk is not None and args.disk_guard_gb >= disk:
|
||
logging.warning(f"⚠️ 磁盘保护阈值({args.disk_guard_gb}G) ≥ 当前可用磁盘({disk:.1f}G)，"
|
||
f"几乎所有视频都会被卡住等待，请调小 --disk-guard-gb 或清理磁盘")
|
||
if cpu and args.max_workers > cpu * 3:
|
||
logging.warning(f"⚠️ max-workers({args.max_workers}) > CPU核数({cpu})×3，"
|
||
f"可能超额订阅反而变慢")
|
||
if cpu and args.max_workers < cpu:
|
||
logging.info(f"提示: max-workers({args.max_workers}) < CPU核数({cpu})，"
|
||
f"CPU 可能吃不满，可适当调大")
|
||
logging.info("=" * 56)
|
||
|
||
split_cfg = config.get_splitframe_config()
|
||
coscli_path = split_cfg.get('coscli_path', './coscli-linux-amd64')
|
||
cos_config_path = split_cfg.get('cosconfig_path', '.cos.yaml')
|
||
|
||
processor = VideoProcessor(max_workers=args.max_workers, mem_guard_gb=args.mem_guard_gb,
|
||
executor=args.executor, max_tasks_per_child=args.max_tasks_per_child,
|
||
disk_guard_gb=args.disk_guard_gb)
|
||
|
||
# 从 数据.jsonl 读取历史，支持跨批次断点续传
|
||
processed = load_processed(JSONL_PATH)
|
||
if processed:
|
||
logging.info(f"已读取 {len(processed)} 条历史记录，将跳过已处理视频")
|
||
|
||
os.makedirs("./video", exist_ok=True)
|
||
|
||
# 每日备份后台线程: 本地模式也自动上传 数据.jsonl + 超长列表到 COS(与 COS 模式一致)。
|
||
# 仅在 dispatch 已配置备份地址时启用; 上传函数内部对缺失/失败都做了容错。
|
||
_backup_stop = threading.Event()
|
||
_dcfg = config.get_dispatch_config() or {}
|
||
if _dcfg.get("enabled", True) and (_dcfg.get("daily_backup_path") or _dcfg.get("jobs_root")):
|
||
_mid = job_cos.get_machine_id()
|
||
_hour = int(_dcfg.get("daily_backup_hour", 10))
|
||
threading.Thread(target=job_cos.run_daily_backup_loop,
|
||
args=(_mid, _hour, _backup_stop, JSONL_PATH),
|
||
daemon=True).start()
|
||
logging.info(f"[每日备份] 已启用: 每天{_hour}点上传 数据.jsonl + 超长列表 (机器号={_mid})")
|
||
|
||
while True:
|
||
txt_files = glob.glob(os.path.join(args.source_list, "*.txt"))
|
||
if not txt_files:
|
||
break
|
||
|
||
file_path = txt_files[0]
|
||
logging.info(f"开始处理任务文件: {file_path}")
|
||
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
all_tasks = [line.strip() for line in f if line.strip()]
|
||
|
||
# 跳过已处理
|
||
tasks = [t for t in all_tasks if t not in processed]
|
||
skipped = len(all_tasks) - len(tasks)
|
||
if skipped:
|
||
logging.info(f"跳过 {skipped} 条已处理记录")
|
||
|
||
if tasks:
|
||
processor.process_batch(tasks, coscli_path, cos_config_path, split_cfg)
|
||
processed.update(tasks)
|
||
|
||
# 清理共享临时目录
|
||
for d in ["./info", "./info_error", "./log", "./video"]:
|
||
if os.path.exists(d):
|
||
force_rmtree(d)
|
||
|
||
os.makedirs("./video", exist_ok=True)
|
||
os.remove(file_path)
|
||
logging.info(f"已完成并删除任务文件: {file_path}")
|
||
|
||
_backup_stop.set()
|
||
# 收尾: 本地批次跑完即上传一次(不必等到点), 保证哪怕在 backup_hour 之前结束也传到 COS。
|
||
if _dcfg.get("enabled", True) and (_dcfg.get("daily_backup_path") or _dcfg.get("jobs_root")):
|
||
try:
|
||
_mid = job_cos.get_machine_id()
|
||
_d = job_cos.upload_daily_backup(JSONL_PATH, _mid)
|
||
if _d:
|
||
logging.info(f"[收尾备份] 数据.jsonl 已上传 -> {_d}")
|
||
_l = job_cos.upload_longlist_backup(_mid)
|
||
if _l:
|
||
logging.info(f"[收尾备份] 超长待重跑列表已上传 -> {_l}")
|
||
except Exception as e:
|
||
logging.error(f"[收尾备份] 上传失败: {e}")
|
||
logging.info("所有任务处理完成")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|