首次提交
This commit is contained in:
394
run_worker.py
Normal file
394
run_worker.py
Normal file
@@ -0,0 +1,394 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""常驻 worker: 从 COS 自动认领任务分片、处理、回写完成标记, 没任务就空闲轮询;
|
||||
每天指定时刻把本地 数据.jsonl 备份到 COS。
|
||||
|
||||
用法:
|
||||
python run_worker.py --max-workers 32 --executor process
|
||||
|
||||
机器ID、取任务地址(jobs_root)、每日备份位置(daily_backup_path)等全部读 config.dispatch。
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
from config.config_utils import config
|
||||
from tool import job_cos
|
||||
# 复用 main.py 既有的处理流水线与清理逻辑
|
||||
from main import (VideoProcessor, load_processed,
|
||||
force_rmtree, JSONL_PATH, get_available_disk_gb)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] %(message)s',
|
||||
handlers=[
|
||||
# 常驻worker一跑就是数天/数周, 普通FileHandler会无限增长; 100M/份, 最多留5份(共500M封顶)
|
||||
RotatingFileHandler('batch_processor.log', maxBytes=100 * 1024 * 1024, backupCount=5, encoding='utf-8'),
|
||||
logging.StreamHandler(),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------ 取下一片 ----
|
||||
|
||||
def _next_shard(machine_id: str, shuffle: bool, claim_match=None):
|
||||
"""返回下一片要做的 shard 名; 没有可领返回 None。
|
||||
优先级: ①续做我名下未完成的(重启续跑, 不受 claim_match 限制) -> ②原子认领一片新的(受 claim_match 过滤)。"""
|
||||
# ① 粘性续做: 我上次没跑完的分片(别的机器不会碰; 已经是我的, 不再过滤)
|
||||
for shard in job_cos.my_open_claims(machine_id):
|
||||
logging.info(f"[续做] 接回我名下未完成分片: {shard}")
|
||||
return shard
|
||||
# ② 认领新分片
|
||||
todo = job_cos.list_todo()
|
||||
# 排除已完成 / 已被锁的
|
||||
todo = [s for s in todo if not job_cos.is_done(s)]
|
||||
# 只认领名称匹配的(claim_match 留空=全部)
|
||||
if claim_match:
|
||||
todo = [s for s in todo if job_cos.shard_matches(s, claim_match)]
|
||||
if shuffle:
|
||||
random.shuffle(todo)
|
||||
for shard in todo:
|
||||
if job_cos.read_lock(shard) is not None:
|
||||
continue # 已被人锁
|
||||
if job_cos.claim(shard, machine_id):
|
||||
logging.info(f"[认领] 抢到分片: {shard}")
|
||||
return shard
|
||||
# 409: 刚好被别人抢走, 试下一片
|
||||
return None
|
||||
|
||||
|
||||
def _claim_long_batch(machine_id: str, shuffle: bool, min_batch: int):
|
||||
"""长视频队列专用: 一次性多领几片, 累计任务数够 min_batch(或没片可领了)才停手 ——
|
||||
长视频队列每天上传的还是各机器各自攒的小分片(可能就几十条), 这里只是在【领取】这一侧
|
||||
把多个小分片攒成一大批一起处理, 不改上传那边的行为。
|
||||
先照 _next_shard() 同样的优先级续做粘性未完成分片(不受 min_batch 限制, 必须先了结),
|
||||
没有才开始新领; min_batch<=0 视为1(至少领一片，不做批量)。
|
||||
返回 [(shard, tasks), ...]；某一片没片可领时提前结束(不会因为凑不够 min_batch 空等)。"""
|
||||
min_batch = max(1, int(min_batch or 1))
|
||||
batch = []
|
||||
total = 0
|
||||
|
||||
# ① 粘性续做: 我名下所有未完成分片先原样接回来, 不受 min_batch 限制(必须先了结)
|
||||
open_claims = job_cos.my_open_claims(machine_id)
|
||||
for shard in open_claims:
|
||||
tasks = job_cos.download_shard(shard)
|
||||
batch.append((shard, tasks))
|
||||
total += len(tasks)
|
||||
if batch:
|
||||
logging.info(f"[长视频批量] 接回 {len(batch)} 片未完成分片, 共{total}条, 直接处理(不再新领)")
|
||||
return batch
|
||||
|
||||
# ② 新领: 不断认领新分片直到攒够 min_batch 条(或没片可领了)
|
||||
todo = job_cos.list_todo()
|
||||
todo = [s for s in todo if not job_cos.is_done(s)]
|
||||
if shuffle:
|
||||
random.shuffle(todo)
|
||||
for shard in todo:
|
||||
if total >= min_batch:
|
||||
break
|
||||
if job_cos.read_lock(shard) is not None:
|
||||
continue # 已被人锁
|
||||
if not job_cos.claim(shard, machine_id):
|
||||
continue # 409: 刚好被别人抢走
|
||||
tasks = job_cos.download_shard(shard)
|
||||
batch.append((shard, tasks))
|
||||
total += len(tasks)
|
||||
logging.info(f"[长视频批量] 认领分片: {shard} ({len(tasks)}条, 累计{total}/{min_batch})")
|
||||
return batch
|
||||
|
||||
|
||||
def _process_shard_batch(batch, processor, machine_id, split_cfg,
|
||||
coscli_path, cos_config_path, hb_interval, global_processed=None):
|
||||
"""处理一批长视频分片(多片任务清单先合并成一个大列表再统一跑一次 process_batch,
|
||||
而不是一片一片分别跑) —— 逻辑和 _process_shard 一致, 只是输入是 [(shard, tasks), ...]。
|
||||
完成后把批次里每一片都各自 mark_done + release(全部标同一份聚合统计, 只是省事,
|
||||
不是精确到每片的独立统计, 不影响正确性)。"""
|
||||
shards = [s for s, _ in batch]
|
||||
tasks = [t for _, ts in batch for t in ts]
|
||||
processed = load_processed(JSONL_PATH)
|
||||
if global_processed:
|
||||
processed = processed | global_processed
|
||||
todo_tasks = [t for t in tasks if t not in processed]
|
||||
skipped = len(tasks) - len(todo_tasks)
|
||||
logging.info(f"长视频批次({len(shards)}片): 共 {len(tasks)} 条, 跳过已处理 {skipped}, "
|
||||
f"待处理 {len(todo_tasks)}")
|
||||
|
||||
hb_stop = threading.Event()
|
||||
def _hb():
|
||||
while not hb_stop.wait(hb_interval):
|
||||
for shard in shards:
|
||||
try:
|
||||
job_cos.heartbeat(shard, machine_id)
|
||||
except Exception as e:
|
||||
logging.warning(f"心跳刷新失败 {shard}: {e}")
|
||||
threading.Thread(target=_hb, daemon=True).start()
|
||||
|
||||
stats_dict = {"total": len(todo_tasks), "success": 0, "no_subtitle": 0, "error": 0}
|
||||
try:
|
||||
if todo_tasks:
|
||||
stats = processor.process_batch(todo_tasks, coscli_path, cos_config_path, split_cfg)
|
||||
stats_dict.update({"success": stats.success,
|
||||
"no_subtitle": stats.no_subtitle, "error": stats.error})
|
||||
finally:
|
||||
hb_stop.set()
|
||||
|
||||
for shard in shards:
|
||||
job_cos.mark_done(shard, machine_id, stats_dict)
|
||||
job_cos.release(shard)
|
||||
logging.info(f"长视频批次({len(shards)}片)完成并释放: {stats_dict}")
|
||||
|
||||
for d in ["./info", "./info_error", "./log", "./video"]:
|
||||
if os.path.exists(d):
|
||||
force_rmtree(d)
|
||||
os.makedirs("./video", exist_ok=True)
|
||||
|
||||
|
||||
def _fetch_global_processed(attempts: int = 3, backoff: float = 10.0) -> set:
|
||||
"""启动时拉全局已处理集合, 失败重试(COS 抖动/网络问题很常见, 直接放弃太可惜)。
|
||||
每次失败等 backoff 秒(线性,不是指数,重试次数少没必要搞复杂)再试; 全部失败返回空集合,
|
||||
调用方会据此退化为只按本机 数据.jsonl 去重, 不阻塞 worker 启动。"""
|
||||
last_err = None
|
||||
for i in range(1, attempts + 1):
|
||||
try:
|
||||
return job_cos.global_processed_set()
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
logging.warning(f"[全局去重] 第{i}/{attempts}次拉取失败: {e}")
|
||||
if i < attempts:
|
||||
time.sleep(backoff)
|
||||
logging.warning(f"[全局去重] 重试{attempts}次仍失败, 本次运行退化为只按本机 数据.jsonl 去重: {last_err}")
|
||||
return set()
|
||||
|
||||
|
||||
def _process_shard(shard, processor, machine_id, split_cfg,
|
||||
coscli_path, cos_config_path, hb_interval, global_processed=None):
|
||||
"""处理一个分片: 下载清单 -> 跳过(本机已处理 ∪ 全局已处理) -> 跑流水线 -> 写 done -> 释放。
|
||||
global_processed: 启动时拉取的全局已处理集合(见 main() 里的 job_cos.global_processed_set()),
|
||||
用于跨机器去重 —— 本机 数据.jsonl 只挡得住"这台机器"重复跑同一个视频, 挡不住这个视频被
|
||||
错误地灌进了另一台机器认领的分片; 传 None/空集合时退化为只按本机去重(旧行为)。"""
|
||||
tasks = job_cos.download_shard(shard)
|
||||
processed = load_processed(JSONL_PATH) # 本机已处理(续跑去重)
|
||||
if global_processed:
|
||||
processed = processed | global_processed # 并上全局已处理(跨机器去重)
|
||||
todo_tasks = [t for t in tasks if t not in processed]
|
||||
skipped = len(tasks) - len(todo_tasks)
|
||||
logging.info(f"分片 {shard}: 共 {len(tasks)} 条, 跳过已处理 {skipped}, 待处理 {len(todo_tasks)}")
|
||||
|
||||
# 心跳线程: 处理期间周期刷新锁的 hb_ts(留作监控/未来回收用)
|
||||
hb_stop = threading.Event()
|
||||
def _hb():
|
||||
while not hb_stop.wait(hb_interval):
|
||||
try:
|
||||
job_cos.heartbeat(shard, machine_id)
|
||||
except Exception as e:
|
||||
logging.warning(f"心跳刷新失败 {shard}: {e}")
|
||||
threading.Thread(target=_hb, daemon=True).start()
|
||||
|
||||
stats_dict = {"total": len(todo_tasks), "success": 0, "no_subtitle": 0, "error": 0}
|
||||
try:
|
||||
if todo_tasks:
|
||||
stats = processor.process_batch(todo_tasks, coscli_path, cos_config_path, split_cfg)
|
||||
stats_dict.update({"success": stats.success,
|
||||
"no_subtitle": stats.no_subtitle, "error": stats.error})
|
||||
finally:
|
||||
hb_stop.set()
|
||||
|
||||
# 完成: 写 done -> 删 todo + lock
|
||||
job_cos.mark_done(shard, machine_id, stats_dict)
|
||||
job_cos.release(shard)
|
||||
logging.info(f"分片 {shard} 完成并释放: {stats_dict}")
|
||||
|
||||
# 清理共享临时目录(沿用 main 的批次后清理)
|
||||
for d in ["./info", "./info_error", "./log", "./video"]:
|
||||
if os.path.exists(d):
|
||||
force_rmtree(d)
|
||||
os.makedirs("./video", exist_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="常驻 worker: 从 COS 认领任务并处理")
|
||||
ap.add_argument('--max-workers', type=int, default=None,
|
||||
help='不填/0 = 自动按本机CPU核数选')
|
||||
ap.add_argument('--mem-guard-gb', type=float, default=2.0)
|
||||
ap.add_argument('--disk-guard-gb', type=float, default=10.0,
|
||||
help='本地磁盘可用空间低于此值(GB)时暂停启动新视频，给数据处理预留空间；设 0 关闭')
|
||||
ap.add_argument('--max-tasks-per-child', type=int, default=30)
|
||||
ap.add_argument('--executor', choices=['process', 'thread'], default='process')
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.max_workers:
|
||||
cpu = os.cpu_count() or 0
|
||||
args.max_workers = job_cos.auto_max_workers(cpu, disk_guard_gb=args.disk_guard_gb)
|
||||
logging.info(f"--max-workers 未指定, 自动选为 {args.max_workers}(本机CPU核数={cpu or '未知'}; "
|
||||
f"若本机还跑了OCR服务但不在 tencentcloud.ocr.endpoints 里, 自动探测不到, 请手动传小一点的值; "
|
||||
f"同时按可用磁盘 ÷ 每视频预估占用(dispatch.est_disk_gb_per_video)做了收紧)")
|
||||
# 供子进程(forkserver fork出来的worker)读取: splitframe.ffmpeg_threads=auto 时按此摊分CPU
|
||||
os.environ['OCR_MAX_WORKERS'] = str(args.max_workers)
|
||||
|
||||
dcfg = config.get_dispatch_config() or {}
|
||||
if not dcfg.get("enabled", True):
|
||||
logging.error("config.dispatch.enabled=false, 退出。如需启用请改 config。")
|
||||
return
|
||||
|
||||
machine_id = job_cos.get_machine_id()
|
||||
keep_alive = bool(dcfg.get("keep_alive", True))
|
||||
idle_poll = int(dcfg.get("idle_poll_seconds", 60))
|
||||
shuffle = bool(dcfg.get("claim_shuffle", True))
|
||||
hb_interval = int(dcfg.get("heartbeat_seconds", 300))
|
||||
backup_hour = int(dcfg.get("daily_backup_hour", 10))
|
||||
claim_match = dcfg.get("claim_match") or None # 只领匹配的分片(留空=全部)
|
||||
|
||||
run_long = bool(dcfg.get("run_long_video"))
|
||||
long_root = dcfg.get("long_list_backup_path") or None
|
||||
# 只有"本来跑主队列"的机器(非专职长视频机)、且配了长视频队列地址、且没有被显式关掉时,
|
||||
# 才会在主队列空闲时顺手兼跑长视频队列 —— 专职长视频机本来就一直在长视频队列里, 不涉及
|
||||
# 这个开关。auto_fallback_long_video 默认 true(维持原行为); 想让某台机器只专心跑主队列、
|
||||
# 空闲就等着(不去碰长视频)的话, 在这台机器的 config.json 里配 false 即可, 不影响它自己
|
||||
# 超长待重跑清单的每日备份(那个走 job_cos.upload_longlist_backup 各自独立)。
|
||||
fallback_enabled = dcfg.get("auto_fallback_long_video", True)
|
||||
auto_fallback_long = (not run_long) and bool(long_root) and bool(fallback_enabled)
|
||||
active_root = job_cos._active_jobs_root()
|
||||
logging.info("=" * 56)
|
||||
logging.info(f"机器ID: {machine_id} | {'【专跑长视频】队列' if run_long else '主队列'}: {active_root}")
|
||||
if auto_fallback_long:
|
||||
logging.info(f"主队列空闲时自动兼跑长视频队列: {long_root}")
|
||||
if claim_match:
|
||||
logging.info(f"只认领匹配分片: {claim_match}")
|
||||
logging.info(f"常驻: {keep_alive} 空闲轮询: {idle_poll}s | 每日备份: 每天{backup_hour}点")
|
||||
disk = get_available_disk_gb(".")
|
||||
disk_str = f"{disk:.1f}G" if disk is not None else "未知"
|
||||
logging.info(f"磁盘保护阈值: {args.disk_guard_gb}G | 当前可用磁盘: {disk_str}")
|
||||
if disk is not None and args.disk_guard_gb >= disk:
|
||||
logging.warning(f"⚠️ 磁盘保护阈值({args.disk_guard_gb}G) ≥ 当前可用磁盘({disk:.1f}G)，"
|
||||
f"几乎所有视频都会被卡住等待，请调小 --disk-guard-gb 或清理磁盘")
|
||||
logging.info("=" * 56)
|
||||
|
||||
# 启动时拉一次全局已处理集合(各机器最新一天的已处理清单, 见 job_cos.global_processed_set),
|
||||
# 跟本机 数据.jsonl 一起用于分片去重 —— 挡住"这个视频被错误地灌进了另一台机器分片"的情况。
|
||||
# 只在启动时拉一次、缓存到本次运行结束, 不是每领一片分片都重新拉(清单本身也就每天更新一次,
|
||||
# 没必要频繁重拉); 失败会重试(见 _fetch_global_processed), 重试仍失败才退化为只按本机去重。
|
||||
# global_dedup_on_startup=false 时直接跳过这一步、只按本机 数据.jsonl 去重 —— 每台机器的
|
||||
# 已处理清单可能有几MB, 20多台机器挨个拉下来这一步可能耗时较久(且下载期间没有中间日志输出,
|
||||
# 看起来像卡住了), 网络不好/机器多的场合可以关掉。
|
||||
global_processed = set()
|
||||
if dcfg.get("daily_backup_path") and dcfg.get("global_dedup_on_startup", True):
|
||||
global_processed = _fetch_global_processed()
|
||||
if global_processed:
|
||||
logging.info(f"[全局去重] 启动时拉取全局已处理集合: {len(global_processed)} 条")
|
||||
else:
|
||||
logging.info("[全局去重] 已关闭(global_dedup_on_startup=false), 只按本机 数据.jsonl 去重")
|
||||
|
||||
split_cfg = config.get_splitframe_config()
|
||||
coscli_path = split_cfg.get('coscli_path', './coscli-linux-amd64')
|
||||
cos_config_path = split_cfg.get('cosconfig_path', '.cos.yaml')
|
||||
processor = VideoProcessor(max_workers=args.max_workers, mem_guard_gb=args.mem_guard_gb,
|
||||
executor=args.executor, max_tasks_per_child=args.max_tasks_per_child,
|
||||
disk_guard_gb=args.disk_guard_gb)
|
||||
|
||||
os.makedirs("./video", exist_ok=True)
|
||||
|
||||
# 每日备份后台线程(数据.jsonl + 超长列表)
|
||||
stop = threading.Event()
|
||||
threading.Thread(target=job_cos.run_daily_backup_loop,
|
||||
args=(machine_id, backup_hour, stop, JSONL_PATH),
|
||||
daemon=True).start()
|
||||
|
||||
# 每小时监控摘要上传线程(常态化监控用, 见 docs/superpowers/specs/2026-07-08-fleet-monitoring-design.md)
|
||||
threading.Thread(target=job_cos.run_hourly_digest_loop,
|
||||
args=(machine_id, stop, JSONL_PATH),
|
||||
daemon=True).start()
|
||||
|
||||
# 长视频队列每次【领取】攒够这么多条任务才开始处理(领的时候多片一起领, 上传那边还是各机器
|
||||
# 各自每天传小分片不受影响) —— 见 _claim_long_batch。<=0 视为1(不做批量, 领一片处理一片)。
|
||||
min_long_batch = int(dcfg.get("long_list_min_batch", 1000) or 1)
|
||||
|
||||
try:
|
||||
while True:
|
||||
if job_cos.stop_requested():
|
||||
logging.info("检测到 COS STOP 标记, 优雅停机。")
|
||||
break
|
||||
|
||||
# 专职长视频机: 全程都在长视频队列, 直接批量领+批量处理, 不走主队列那套单片逻辑
|
||||
if run_long:
|
||||
try:
|
||||
batch = _claim_long_batch(machine_id, shuffle, min_long_batch)
|
||||
except Exception as e:
|
||||
logging.error(f"✗ 访问长视频队列失败(稍后重试): {e}")
|
||||
time.sleep(min(idle_poll, 30))
|
||||
continue
|
||||
if not batch:
|
||||
if keep_alive:
|
||||
logging.info(f"长视频队列无可领任务, 空闲等待 {idle_poll}s...")
|
||||
time.sleep(idle_poll)
|
||||
continue
|
||||
else:
|
||||
logging.info("长视频队列无可领任务, keep_alive=false, 退出。")
|
||||
break
|
||||
try:
|
||||
_process_shard_batch(batch, processor, machine_id, split_cfg,
|
||||
coscli_path, cos_config_path, hb_interval, global_processed)
|
||||
except Exception as e:
|
||||
logging.error(f"✗ 长视频批次处理出错(锁保留, 重启可续做): {e}")
|
||||
time.sleep(5)
|
||||
continue
|
||||
|
||||
# ① 先看主队列(单片领取, 行为不变)
|
||||
try:
|
||||
shard = _next_shard(machine_id, shuffle, claim_match)
|
||||
except Exception as e:
|
||||
logging.error(f"✗ 访问 COS 队列失败(稍后重试): {e}")
|
||||
time.sleep(min(idle_poll, 30))
|
||||
continue
|
||||
|
||||
# ② 主队列没活 -> 顺手看看长视频队列有没有能领的(批量领, 不是单片)
|
||||
used_long = False
|
||||
long_batch = None
|
||||
if shard is None and auto_fallback_long:
|
||||
job_cos.set_active_root_override(long_root)
|
||||
try:
|
||||
long_batch = _claim_long_batch(machine_id, shuffle, min_long_batch)
|
||||
except Exception as e:
|
||||
logging.error(f"✗ 访问长视频队列失败(稍后重试): {e}")
|
||||
job_cos.set_active_root_override(None)
|
||||
time.sleep(min(idle_poll, 30))
|
||||
continue
|
||||
if not long_batch:
|
||||
job_cos.set_active_root_override(None) # 长视频队列也没活, 恢复主队列上下文
|
||||
else:
|
||||
used_long = True
|
||||
logging.info(f"主队列空闲, 转去处理长视频队列批次: {len(long_batch)}片")
|
||||
|
||||
if shard is None and not long_batch:
|
||||
# 两个队列都真·没活了
|
||||
if keep_alive:
|
||||
logging.info(f"无可领任务, 空闲等待 {idle_poll}s(加新任务到 todo/ 会自动接上)...")
|
||||
time.sleep(idle_poll)
|
||||
continue
|
||||
else:
|
||||
logging.info("无可领任务, keep_alive=false, 退出。")
|
||||
break
|
||||
|
||||
try:
|
||||
if long_batch:
|
||||
_process_shard_batch(long_batch, processor, machine_id, split_cfg,
|
||||
coscli_path, cos_config_path, hb_interval, global_processed)
|
||||
else:
|
||||
_process_shard(shard, processor, machine_id, split_cfg,
|
||||
coscli_path, cos_config_path, hb_interval, global_processed)
|
||||
except Exception as e:
|
||||
logging.error(f"✗ 分片处理出错(锁保留, 重启可续做): {e}")
|
||||
time.sleep(5)
|
||||
finally:
|
||||
if used_long:
|
||||
job_cos.set_active_root_override(None) # 这批长视频处理完/出错, 下一轮先回去看主队列
|
||||
finally:
|
||||
stop.set()
|
||||
logging.info("worker 退出。")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user