import argparse
import glob
import json
import shutil
import subprocess
import logging
from logging.handlers import RotatingFileHandler
import os
import stat
import threading
import time
import cv2
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
from datetime import datetime
from config.config_utils import config
from tool.process import process_single_video, derive_video_name
from tool.log.log_manager import logger as steplog
from tool import job_cos

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    handlers=[
        # 常驻worker一跑就是数天/数周, 普通FileHandler会无限增长; 100M/份, 最多留5份(共500M封顶)
        RotatingFileHandler('batch_processor.log', maxBytes=100 * 1024 * 1024, backupCount=5, encoding='utf-8'),
        logging.StreamHandler()
    ]
)

JSONL_PATH = "数据.jsonl"


# ==================== 工具函数 ====================

def parse_cos_path(cos_path: str):
    """cos://bucket/path/video.mp4 → ('cos://bucket/path/', 'video.mp4')"""
    cos_path = cos_path.strip()
    idx = cos_path.rfind('/')
    return cos_path[:idx + 1], cos_path[idx + 1:]


def get_video_duration(local_path: str) -> float:
    if not os.path.exists(local_path):
        return 0.0
    cap = cv2.VideoCapture(local_path)
    if not cap.isOpened():
        return 0.0
    frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)
    fps = cap.get(cv2.CAP_PROP_FPS)
    cap.release()
    return round(frames / fps, 2) if fps > 0 else 0.0


def _read_log_stats(video_name: str):
    """从 log/<video>.json 读取图片数量统计与各阶段耗时。
    返回 (image_counts: dict|None, stage_durations: dict|None, video_duration: float|None)，读取失败返回全 None。"""
    log_path = os.path.join("log", f"{video_name}.json")
    if not os.path.exists(log_path):
        return None, None, None
    try:
        with open(log_path, 'r', encoding='utf-8') as f:
            data = json.load(f)
        meta = data.get("metadata", {})
        custom = meta.get("custom_stats", {}) or {}
        durations = meta.get("program_durations", {}) or {}
        image_counts = {
            "frame_count": custom.get("frame_count"),
            "composite_ocr_count": custom.get("composite_ocr_count"),
            "single_frame_ocr_count": custom.get("single_frame_ocr_count"),
            "total_ocr_count": custom.get("total_ocr_count"),
        }
        # 各阶段耗时（秒），四舍五入到2位
        stage_durations = {k: round(v, 2) for k, v in durations.items()}
        video_duration = custom.get("video_duration")   # process.py 在删视频前用 ffprobe 存的(AV1也准)
        return image_counts, stage_durations, video_duration
    except (OSError, IOError, json.JSONDecodeError) as e:
        logging.warning(f"读取日志统计失败 {log_path}: {e}")
        return None, None, None


def upload_srt(srt_local: str, cos_dir: str, coscli_path: str, cos_config_path: str):
    """上传 SRT 文件到视频对应的 COS 目录，返回 COS 路径；失败返回 None"""
    if not os.path.exists(srt_local):
        return None
    cos_dest = cos_dir + os.path.basename(srt_local)
    cmd = [coscli_path, "-c", cos_config_path, "cp", srt_local, cos_dest]
    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
    if result.returncode == 0:
        return cos_dest
    logging.error(f"SRT 上传失败: {result.stderr}")
    return None


def get_available_mem_gb():
    """读取系统可用内存(GB)。Linux 读 /proc/meminfo 的 MemAvailable；
    非 Linux 或读取失败返回 None（此时内存保护自动失效）。"""
    try:
        with open('/proc/meminfo', 'r') as f:
            for line in f:
                if line.startswith('MemAvailable:'):
                    return float(line.split()[1]) / 1024 / 1024   # kB → GB
    except (OSError, ValueError, IndexError):
        return None
    return None


def get_available_disk_gb(path: str = "."):
    """读取 path 所在磁盘的可用空间(GB)。用 shutil.disk_usage，跨平台(Windows/Linux都行，
    不像内存检查那样只能读 /proc/meminfo)。读取失败(路径不存在等)返回 None，此时磁盘保护自动失效。"""
    try:
        return shutil.disk_usage(path).free / 1024 / 1024 / 1024
    except OSError:
        return None


def get_load_avg():
    """返回 (1, 5, 15) 分钟负载；非 Unix 返回 None。"""
    try:
        return os.getloadavg()
    except (OSError, AttributeError):
        return None


def load_processed(jsonl_path: str) -> set:
    """从数据.jsonl 读取已处理的 COS 路径集合，用于跨批次断点续传"""
    processed = set()
    if not os.path.exists(jsonl_path):
        return processed
    with open(jsonl_path, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                rec = json.loads(line)
                vid = rec.get('video_cos_path')
                if vid:
                    processed.add(vid)
            except json.JSONDecodeError:
                pass
    return processed


def force_rmtree(path):
    """强制删除目录（兼容 Windows 和 Linux）"""
    if not os.path.exists(path):
        return
    for attempt in range(3):
        try:
            shutil.rmtree(path)
            return
        except PermissionError:
            if attempt < 2:
                time.sleep(1)
                for root, dirs, files in os.walk(path):
                    for name in files:
                        try:
                            os.chmod(os.path.join(root, name), stat.S_IWUSR | stat.S_IRUSR)
                        except Exception:
                            pass
            else:
                if os.name == 'nt':
                    os.system(f'rmdir /s /q "{path}"')
                else:
                    os.system(f'rm -rf "{path}"')


# ==================== 统计 ====================

class Stats:
    def __init__(self, total: int):
        self._lock = threading.Lock()
        self.success = 0
        self.no_subtitle = 0
        self.error = 0
        self.done = 0
        self.total = total

    def update(self, status: str):
        with self._lock:
            self.done += 1
            if status == 'success':
                self.success += 1
            elif status == 'no_subtitle':
                self.no_subtitle += 1
            else:
                self.error += 1
            print(
                f"[进度 {self.done}/{self.total}]  "
                f"成功: {self.success}  无字幕: {self.no_subtitle}  错误: {self.error}  "
                f"剩余: {self.total - self.done}",
                flush=True
            )


# ==================== 处理器 ====================

def _wait_for_memory(video_name: str, mem_guard_gb: float):
    """内存可用量低于阈值时阻塞，等已在处理的视频释放内存后再启动新视频，避免 OOM。
    模块级函数：进程池下每个 worker 进程独立调用。读不到内存时直接放行。"""
    if mem_guard_gb <= 0:
        return
    waited = 0
    while True:
        avail = get_available_mem_gb()
        if avail is None or avail >= mem_guard_gb:
            return
        if waited % 15 == 0:
            logging.warning(
                f"内存不足(可用 {avail:.1f}G < 阈值 {mem_guard_gb}G)，"
                f"暂停启动 {video_name}，等待回收... 已等 {waited}s"
            )
        time.sleep(3)
        waited += 3


def _wait_for_disk(video_name: str, disk_guard_gb: float):
    """可用磁盘空间低于阈值时阻塞，等已在处理的视频清理释放空间后再启动新视频。
    防的是磁盘写满级联失败(下载/抽帧写一半失败、GPU硬解报错误判成"GPU失败"其实是写不进磁盘、
    info_error都写不进去导致视频异常处理链路整个乱掉——实际发生过的连锁故障)。
    模块级函数：进程池下每个 worker 进程独立调用。读不到磁盘空间时直接放行。"""
    if disk_guard_gb <= 0:
        return
    waited = 0
    while True:
        avail = get_available_disk_gb(".")
        if avail is None or avail >= disk_guard_gb:
            return
        if waited % 15 == 0:
            logging.warning(
                f"磁盘空间不足(可用 {avail:.1f}G < 阈值 {disk_guard_gb}G)，"
                f"暂停启动 {video_name}，等待清理释放空间... 已等 {waited}s"
            )
        time.sleep(3)
        waited += 3


def _short_reason(text: str) -> str:
    """把 info_error 里的长错误串归一成一个短分类标签，便于 数据.jsonl 里按原因统计/筛选。"""
    if not text:
        return None
    for kw in ("视频过长", "视频过短", "字幕太少", "冗余文字过多",
               "下载失败", "抽帧", "拼接失败", "字幕生成失败", "OCR"):
        if kw in text:
            return kw
    if "置信度" in text or "confidence" in text.lower():
        return "置信度太低"
    if "未找到合适的字幕区域" in text or "未检测" in text:
        return "无字幕区域"
    # 兜底截断: 之前只截40字符——异常串常见格式是"RuntimeError: 视频<video_name>真正原因",
    # video_name(文件名+MD5后缀)本身就有二三十字符，40字符的窗口经常在真正原因(冒号后面
    # 那段)出现之前就被切没了，数据.jsonl里全变成"RuntimeError: 视频<name>"这种没有信息量
    # 的记录（实测发生过：一小时63条error全部是这个模式）。加大到150，给真实原因留出空间；
    # 完整未截断的原始异常串仍然保留在 record["error_reason"] 里，这里只是短标签。
    return text.strip()[:150]


def _read_error_reason(video_name: str) -> str:
    """从 ./info_error/<video>.json 读出过滤/错误原因(归类后的短标签)。"""
    err_json = f"./info_error/{video_name}.json"
    if not os.path.exists(err_json):
        return None
    try:
        with open(err_json, 'r', encoding='utf-8') as f:
            data = json.load(f)
        return _short_reason((data.get("main_process") or {}).get("error"))
    except Exception:
        return None


# 这些原因是"正常过滤(视频本就没有可用字幕)", 不是程序硬故障 → 应记 no_subtitle 而非 error。
# 旧逻辑用 '"被过滤" in json文本' 猜测, 但过滤原因只存在 main_process.error 里(不含"被过滤"),
# 于是字幕太少/冗余/置信度/非字幕等全被错记成 error。改为按原因关键词匹配, 清单可在
# config.subtitle.no_subtitle_reasons 覆盖。
_NO_SUB_REASONS_DEFAULT = [
    "字幕太少", "冗余文字过多", "冗余文字太多", "无字幕区域",
    "非字幕内容", "置信度太低", "未检测", "特殊字体",
    "视频过短", "视频过长",
]


def _is_no_subtitle_reason(reason: str) -> bool:
    """原因是否属于'无可用字幕的正常过滤'(→ no_subtitle), 而非硬故障(→ error)。"""
    if not reason:
        return False
    try:
        keys = config.get_subtitle_config().get('no_subtitle_reasons') or _NO_SUB_REASONS_DEFAULT
    except Exception:
        keys = _NO_SUB_REASONS_DEFAULT
    return any(k in reason for k in keys)


def process_video_task(cos_path: str, coscli_path: str, cos_config_path: str,
                       split_cfg: dict, mem_guard_gb: float, disk_guard_gb: float = 0) -> dict:
    """处理单个视频，返回完整 record（不触碰共享状态，可被进程池 pickle 调用）。
    进程池模式下：本函数整体在独立子进程里跑 → 各自的 Python 解释器/GIL/whisper 模型 → 真多核。"""
    cos_dir, filename = parse_cos_path(cos_path)
    video_name = derive_video_name(cos_path)   # 含路径MD5后缀, 避免不同父目录同名视频本地/COS撞车
    local_video = os.path.join("./video", video_name + os.path.splitext(filename)[1])

    # 内存/磁盘保护（放在计时前，等待时间不计入 elapsed_seconds）
    _wait_for_memory(video_name, mem_guard_gb)
    _wait_for_disk(video_name, disk_guard_gb)

    status = "error"
    start_dt = datetime.now()
    start_perf = time.time()
    record = {
        "timestamp": start_dt.isoformat(),
        "video_cos_path": cos_path,
        "status": None,
        "reason": None,
        "srt_cos_path": None,
        "duration": None,
        "process_start": start_dt.isoformat(),
        "process_end": None,
        "elapsed_seconds": None,
        "image_counts": None,
        "stage_durations": None,
    }

    try:
        success = process_single_video(cos_path, cos_dir)
        record["duration"] = get_video_duration(local_video)

        if success:
            srt_local = f"./split/{video_name}/composite/{video_name}.ocr.srt"
            srt_cos = upload_srt(srt_local, cos_dir, coscli_path, cos_config_path)
            status = "success"
            record["srt_cos_path"] = srt_cos
            if os.path.exists(srt_local):
                os.makedirs("./srt", exist_ok=True)
                shutil.move(srt_local, os.path.join("./srt", f"{video_name}.srt"))
            steplog.step(video_name, "上传完成", "srt已传" if srt_cos else "srt上传失败")
            logging.info(f"成功 {filename} → {srt_cos}")
        else:
            # 按原因关键词归类(config 可调): 无可用字幕的正常过滤 → no_subtitle, 其余 → error。
            reason = _read_error_reason(video_name)
            record["reason"] = reason
            is_filtered = _is_no_subtitle_reason(reason)
            status = "no_subtitle" if is_filtered else "error"
            logging.info(f"{'无字幕' if is_filtered else '失败'} {filename}")

    except Exception as e:
        reason = _short_reason(f"{type(e).__name__}: {e}")
        record["error_reason"] = f"{type(e).__name__}: {e}"
        record["reason"] = reason
        # 过滤型异常(如 raise '未检测到字幕区域'/'视频过短') 也应归 no_subtitle, 不是硬故障。
        status = "no_subtitle" if _is_no_subtitle_reason(reason) else "error"
        logging.error(f"异常 {filename}: {e}")

    finally:
        # composite（含拼接大图/OCR中间结果）不上传 COS，处理完直接随 split_dir 一起删除本地。
        split_dir = f"./split/{video_name}"
        if os.path.exists(split_dir):
            force_rmtree(split_dir)
        if split_cfg.get('delete_video') and os.path.exists(local_video):
            try:
                os.remove(local_video)
            except OSError:
                pass

    end_dt = datetime.now()
    record["status"] = status
    record["process_end"] = end_dt.isoformat()
    record["elapsed_seconds"] = round(time.time() - start_perf, 2)
    image_counts, stage_durations, video_duration = _read_log_stats(video_name)
    record["image_counts"] = image_counts
    record["stage_durations"] = stage_durations
    # 时长优先用 process.py 删视频前 ffprobe 存的(cv2读不了AV1、且delete_video后源视频已删)
    if (not record.get("duration")) and video_duration:
        record["duration"] = video_duration

    # 该读的(日志统计/错误原因)都读完了，本视频的临时小文件立即删，不等整批/整片跑完再统一清
    # (之前是攒到 main.py 主循环/run_worker._process_shard 末尾才批量删，一片上千个视频攒一路)。
    for f in (f"./log/{video_name}.json", f"./info/{video_name}.json", f"./info_error/{video_name}.json"):
        try:
            if os.path.exists(f):
                os.remove(f)
        except OSError:
            pass
    return record


def _ocr_worker_init(sem):
    """进程池 worker 启动时调用：注入【全局 OCR 并发信号量】,使所有进程共享一个硬性并发上限。"""
    try:
        from tool.Ocr import tencentcloud_ocr
        tencentcloud_ocr.set_ocr_semaphore(sem)
    except Exception as e:
        logging.warning(f"注入全局OCR信号量失败: {e}")


class VideoProcessor:
    def __init__(self, max_workers: int = 10, mem_guard_gb: float = 2.0, executor: str = "process",
                 max_tasks_per_child: int = 30, disk_guard_gb: float = 0):
        self.max_workers = max_workers
        self.mem_guard_gb = mem_guard_gb     # 可用内存低于此值(GB)时暂停启动新视频，<=0 关闭
        self.disk_guard_gb = disk_guard_gb   # 本地磁盘可用空间低于此值(GB)时暂停启动新视频，<=0 关闭
        self.executor = executor             # "process"=进程池(绕开GIL,真多核) / "thread"=线程池
        self.max_tasks_per_child = max_tasks_per_child  # 进程池: worker处理这么多视频后重启释放内存
        self._jsonl_lock = threading.Lock()

    def _append_jsonl(self, record: dict):
        with self._jsonl_lock:
            with open(JSONL_PATH, 'a', encoding='utf-8') as f:
                f.write(json.dumps(record, ensure_ascii=False) + "\n")

    def process_batch(self, tasks: list, coscli_path: str,
                      cos_config_path: str, split_cfg: dict) -> Stats:
        stats = Stats(total=len(tasks))
        mode = "进程池(多核,绕GIL)" if self.executor == "process" else "线程池"
        logging.info(f"开始处理 {len(tasks)} 个视频 ({mode} 并发={self.max_workers})")

        # 后台定时打印进度（每30秒一次，处理完后停止）
        stop_event = threading.Event()
        def _heartbeat():
            while not stop_event.wait(30):
                with stats._lock:
                    d, t = stats.done, stats.total
                    s, n, e = stats.success, stats.no_subtitle, stats.error
                print(
                    f"[进行中 {d}/{t}]  成功: {s}  无字幕: {n}  错误: {e}  剩余: {t - d}",
                    flush=True
                )
        threading.Thread(target=_heartbeat, daemon=True).start()

        # 进程池: 每个视频在独立子进程跑 → 各自 GIL/whisper → 真多核(getsrt 的 jieba/拼音不再串行)
        # 线程池: 退化兜底(--executor thread)
        # 用 forkserver: 从干净的服务进程 fork worker,避免"主进程多线程+fork"继承到被持有的锁而死锁;
        # 重模块(cv2/whisper)在 forkserver 里只导入一次, worker 共享(COW),启动快。
        if self.executor == "process":
            import multiprocessing
            try:
                _ctx = multiprocessing.get_context("forkserver")
            except ValueError:
                _ctx = multiprocessing.get_context()   # 非 POSIX 等回退默认
            # 全局OCR并发上限: 父进程建跨进程信号量, 经 initializer 注入每个 worker →
            # 不管 max-workers × ocr_workers 多大, 全局同时在飞的OCR请求被硬性卡在此值, 防打爆端点。
            _max_global_ocr = int(config.get_subtitle_config().get('max_global_ocr', 0) or 0)
            _init, _initargs = None, ()
            if _max_global_ocr > 0:
                _init = _ocr_worker_init
                _initargs = (_ctx.Semaphore(_max_global_ocr),)
                logging.info(f"全局OCR并发上限: {_max_global_ocr}")
            # max_tasks_per_child: worker 处理这么多视频后【自动重启】释放内存(需 Python≥3.11)。
            # ⚠️ Python 3.11 该机制有"补不满"的坑: 频繁回收会让 worker 数随时间从 N 缩到个位数甚至0
            #    → as_completed 永久挂起 → 进度冻住。内存充裕时建议【关闭】(设 0/None), 靠整体重启释放内存。
            _mtpc = self.max_tasks_per_child if (self.max_tasks_per_child and self.max_tasks_per_child > 0) else None
            if _mtpc is None:
                logging.info("max_tasks_per_child=关闭(worker不自动回收, 避免3.11回收缩水; 内存靠整体重启/mem-guard控制)")
            try:
                pool = ProcessPoolExecutor(max_workers=self.max_workers, mp_context=_ctx,
                                           max_tasks_per_child=_mtpc,
                                           initializer=_init, initargs=_initargs)
            except TypeError:
                logging.warning("当前Python不支持 max_tasks_per_child(需≥3.11), 进程不自动回收, 注意内存")
                pool = ProcessPoolExecutor(max_workers=self.max_workers, mp_context=_ctx,
                                           initializer=_init, initargs=_initargs)
        else:
            # 线程池模式: 同进程内, 用 threading 信号量直接限流(可选)
            _max_global_ocr = int(config.get_subtitle_config().get('max_global_ocr', 0) or 0)
            if _max_global_ocr > 0:
                from tool.Ocr import tencentcloud_ocr
                tencentcloud_ocr.set_ocr_semaphore(threading.Semaphore(_max_global_ocr))
                logging.info(f"全局OCR并发上限: {_max_global_ocr}")
            pool = ThreadPoolExecutor(max_workers=self.max_workers)
        with pool as executor:
            futures = {
                executor.submit(
                    process_video_task, cos_path,
                    coscli_path, cos_config_path, split_cfg, self.mem_guard_gb, self.disk_guard_gb
                ): cos_path
                for cos_path in tasks
            }
            # 父进程单线程收集结果 → 统一写 jsonl(单写者,无需跨进程锁) + 更新统计
            for future in as_completed(futures):
                cos_path = futures[future]
                try:
                    record = future.result()
                except Exception as e:
                    # 子进程崩溃(段错误/被OOM杀)等 → 仍记一条 error
                    logging.error(f"任务异常 {cos_path}: {e}")
                    record = {"video_cos_path": cos_path, "status": "error",
                              "error_reason": f"worker崩溃: {type(e).__name__}: {e}",
                              "process_end": datetime.now().isoformat()}
                self._append_jsonl(record)
                stats.update(record.get("status", "error"))

        stop_event.set()  # 停止心跳线程
        logging.info(
            f"批次完成 — 成功: {stats.success} | "
            f"无字幕: {stats.no_subtitle} | 错误: {stats.error}"
        )
        return stats


# ==================== 入口 ====================

def main():
    parser = argparse.ArgumentParser(description='视频字幕批量提取（txt 内填 COS 完整路径）')
    parser.add_argument('--source-list', default='./text2',
                        help='包含 COS 路径列表的 txt 文件目录')
    parser.add_argument('--max-workers', type=int, default=None,
                        help='最大并发线程数 (不填/0 = 自动按本机CPU核数选)')
    parser.add_argument('--mem-guard-gb', type=float, default=2.0,
                        help='可用内存低于此值(GB)时暂停启动新视频，防 OOM；设 0 关闭 (默认: 2.0)')
    parser.add_argument('--disk-guard-gb', type=float, default=10.0,
                        help='本地磁盘可用空间低于此值(GB)时暂停启动新视频，给下载/抽帧/合成图预留空间，'
                             '防止磁盘写满导致级联失败；设 0 关闭 (默认: 10.0)')
    parser.add_argument('--max-tasks-per-child', type=int, default=30,
                        help='进程池模式: 每个worker处理这么多视频后重启释放内存(默认30, 需Python≥3.11)')
    parser.add_argument('--executor', choices=['process', 'thread'], default='process',
                        help='并发方式: process=进程池(默认,绕开GIL真多核) / thread=线程池(旧行为,CPU重计算会被GIL串行)')
    args = parser.parse_args()

    # ---- 启动时打印机器资源与并发设置，便于判断参数是否合理 ----
    cpu = os.cpu_count() or 0
    if not args.max_workers:
        args.max_workers = job_cos.auto_max_workers(cpu, disk_guard_gb=args.disk_guard_gb)
        logging.info(f"--max-workers 未指定, 自动选为 {args.max_workers}(本机CPU核数={cpu or '未知'}; "
                     f"若本机还跑了OCR服务但不在 tencentcloud.ocr.endpoints 里, 自动探测不到, 请手动传小一点的值; "
                     f"同时按可用磁盘 ÷ 每视频预估占用(dispatch.est_disk_gb_per_video)做了收紧)")
    # 供子进程(forkserver fork出来的worker)读取: splitframe.ffmpeg_threads=auto 时按此摊分CPU
    os.environ['OCR_MAX_WORKERS'] = str(args.max_workers)
    mem = get_available_mem_gb()
    disk = get_available_disk_gb(".")
    load = get_load_avg()
    mem_str = f"{mem:.1f}G" if mem is not None else "未知(非Linux?)"
    disk_str = f"{disk:.1f}G" if disk is not None else "未知"
    load_str = (f"{load[0]:.2f}/{load[1]:.2f}/{load[2]:.2f}" if load else "未知")
    logging.info("=" * 56)
    logging.info(f"机器资源:  CPU {cpu} 核  |  可用内存 {mem_str}  |  可用磁盘 {disk_str}  |  负载(1/5/15) {load_str}")
    _exec_str = "进程池(多核,绕GIL)" if args.executor == "process" else "线程池(GIL会串行CPU计算)"
    logging.info(f"并发设置:  {_exec_str}  |  max-workers={args.max_workers}  |  "
                 f"内存保护阈值={args.mem_guard_gb}G  |  磁盘保护阈值={args.disk_guard_gb}G")
    if args.executor == "process" and cpu:
        logging.info(f"提示: 进程池模式建议 max-workers ≈ CPU核数({cpu})上下，每个视频独立进程真多核；"
                     f"内存够的话(每进程约+0.5~1G含whisper模型)可略超")
    if mem is not None and args.mem_guard_gb >= mem:
        logging.warning(f"⚠️ 内存保护阈值({args.mem_guard_gb}G) ≥ 当前可用内存({mem:.1f}G)，"
                        f"几乎所有视频都会被卡住等待，请调小 --mem-guard-gb 或释放内存")
    if disk is not None and args.disk_guard_gb >= disk:
        logging.warning(f"⚠️ 磁盘保护阈值({args.disk_guard_gb}G) ≥ 当前可用磁盘({disk:.1f}G)，"
                        f"几乎所有视频都会被卡住等待，请调小 --disk-guard-gb 或清理磁盘")
    if cpu and args.max_workers > cpu * 3:
        logging.warning(f"⚠️ max-workers({args.max_workers}) > CPU核数({cpu})×3，"
                        f"可能超额订阅反而变慢")
    if cpu and args.max_workers < cpu:
        logging.info(f"提示: max-workers({args.max_workers}) < CPU核数({cpu})，"
                     f"CPU 可能吃不满，可适当调大")
    logging.info("=" * 56)

    split_cfg = config.get_splitframe_config()
    coscli_path = split_cfg.get('coscli_path', './coscli-linux-amd64')
    cos_config_path = split_cfg.get('cosconfig_path', '.cos.yaml')

    processor = VideoProcessor(max_workers=args.max_workers, mem_guard_gb=args.mem_guard_gb,
                               executor=args.executor, max_tasks_per_child=args.max_tasks_per_child,
                               disk_guard_gb=args.disk_guard_gb)

    # 从 数据.jsonl 读取历史，支持跨批次断点续传
    processed = load_processed(JSONL_PATH)
    if processed:
        logging.info(f"已读取 {len(processed)} 条历史记录，将跳过已处理视频")

    os.makedirs("./video", exist_ok=True)

    # 每日备份后台线程: 本地模式也自动上传 数据.jsonl + 超长列表到 COS(与 COS 模式一致)。
    # 仅在 dispatch 已配置备份地址时启用; 上传函数内部对缺失/失败都做了容错。
    _backup_stop = threading.Event()
    _dcfg = config.get_dispatch_config() or {}
    if _dcfg.get("enabled", True) and (_dcfg.get("daily_backup_path") or _dcfg.get("jobs_root")):
        _mid = job_cos.get_machine_id()
        _hour = int(_dcfg.get("daily_backup_hour", 10))
        threading.Thread(target=job_cos.run_daily_backup_loop,
                         args=(_mid, _hour, _backup_stop, JSONL_PATH),
                         daemon=True).start()
        logging.info(f"[每日备份] 已启用: 每天{_hour}点上传 数据.jsonl + 超长列表 (机器号={_mid})")

    while True:
        txt_files = glob.glob(os.path.join(args.source_list, "*.txt"))
        if not txt_files:
            break

        file_path = txt_files[0]
        logging.info(f"开始处理任务文件: {file_path}")

        with open(file_path, 'r', encoding='utf-8') as f:
            all_tasks = [line.strip() for line in f if line.strip()]

        # 跳过已处理
        tasks = [t for t in all_tasks if t not in processed]
        skipped = len(all_tasks) - len(tasks)
        if skipped:
            logging.info(f"跳过 {skipped} 条已处理记录")

        if tasks:
            processor.process_batch(tasks, coscli_path, cos_config_path, split_cfg)
            processed.update(tasks)

        # 清理共享临时目录
        for d in ["./info", "./info_error", "./log", "./video"]:
            if os.path.exists(d):
                force_rmtree(d)

        os.makedirs("./video", exist_ok=True)
        os.remove(file_path)
        logging.info(f"已完成并删除任务文件: {file_path}")

    _backup_stop.set()
    # 收尾: 本地批次跑完即上传一次(不必等到点), 保证哪怕在 backup_hour 之前结束也传到 COS。
    if _dcfg.get("enabled", True) and (_dcfg.get("daily_backup_path") or _dcfg.get("jobs_root")):
        try:
            _mid = job_cos.get_machine_id()
            _d = job_cos.upload_daily_backup(JSONL_PATH, _mid)
            if _d:
                logging.info(f"[收尾备份] 数据.jsonl 已上传 -> {_d}")
            _l = job_cos.upload_longlist_backup(_mid)
            if _l:
                logging.info(f"[收尾备份] 超长待重跑列表已上传 -> {_l}")
        except Exception as e:
            logging.error(f"[收尾备份] 上传失败: {e}")
    logging.info("所有任务处理完成")


if __name__ == '__main__':
    main()