whale-fall-code
2bb83f3685
App 端免签名接口（设备注册 + detail + CDN 直链），一体化流水线 下载→抽音→传COS→记录→删本地，AI 作品四档判定，四份文档： README（使用指南）、PROGRESS（批次进度）、RE（逆向全过程）、AI（判定规则）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
534 lines
22 KiB
Python
534 lines
22 KiB
Python
#!/usr/bin/env python3
|
||
"""抖音采集一体化流水线：下载 -> 抽 mp3 -> 传 COS -> 记元信息 -> 删本地。
|
||
|
||
跟原来分阶段跑（全下完再全抽完再全传完）最大的区别是磁盘占用：
|
||
分阶段必须把所有 mp4 摊在盘上（上一批是 53 GB），这里每条走完就删，
|
||
峰值只有 workers x (mp4+mp3)，约 100 MB，且与总量无关。
|
||
|
||
每条视频端到端走 9 步：
|
||
|
||
1. fetch_detail 拿元信息 + raw
|
||
2. 下载 mp4
|
||
3. ffmpeg 抽 mp3 [信号量限流，见下]
|
||
4. ffprobe 探两个的时长
|
||
5. 校验 mp3 时长 ~= mp4 时长 <- 抽取闸门
|
||
6. 上传 mp4（覆盖同名）
|
||
7. 上传 mp3（覆盖同名）
|
||
8. head_object 核对大小 <- 删除闸门（在 CosUploader.put 里）
|
||
9. 写 record，删本地两个文件
|
||
|
||
任何一步失败：保留本地文件不删，写进 failed 文件，继续下一条。
|
||
|
||
并发：下载和上传是网络密集，按 --workers 并发；ffmpeg 是 CPU 密集，
|
||
单独用信号量限到 CPU 核数，避免 8 个 worker 同时撞进第 3 步把 CPU 打满。
|
||
|
||
用法:
|
||
$env:COS_SECRET_ID="xxx"; $env:COS_SECRET_KEY="yyy"
|
||
python pipeline.py --from-file links.txt --region ap-chongqing --limit 20 --keep-local
|
||
python pipeline.py --from-file links.txt --region ap-chongqing --workers 8
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import List, Optional
|
||
|
||
import requests
|
||
|
||
from douyin import (
|
||
AppClient,
|
||
DeviceRegisterError,
|
||
DouyinError,
|
||
NotAVideo,
|
||
ParseError,
|
||
download,
|
||
extract_aweme_id,
|
||
)
|
||
from douyin.audio import build_cmd, ensure_ffmpeg, probe
|
||
from douyin.batch import DevicePool
|
||
from douyin.cos import CosError, CosUploader, DEFAULT_BUCKET, DEFAULT_PREFIX
|
||
from douyin.meta import RecordLog, build_record
|
||
|
||
|
||
class StageError(RuntimeError):
|
||
"""带阶段名的失败，便于 failed 文件里归因。"""
|
||
|
||
def __init__(self, stage: str, msg: str) -> None:
|
||
super().__init__(msg)
|
||
self.stage = stage
|
||
|
||
|
||
@dataclass
|
||
class Stats:
|
||
total: int = 0
|
||
done: int = 0
|
||
skipped: int = 0
|
||
failed: int = 0
|
||
bytes_up: int = 0
|
||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||
|
||
|
||
_tls = threading.local()
|
||
|
||
|
||
def session() -> requests.Session:
|
||
"""每个线程一个下载用 session。CDN 直链不需要设备头，裸 session 即可。"""
|
||
s = getattr(_tls, "dl", None)
|
||
if s is None:
|
||
s = requests.Session()
|
||
s.headers.update({"User-Agent": "okhttp/3.12.13"})
|
||
_tls.dl = s
|
||
return s
|
||
|
||
|
||
def read_targets(args) -> List[str]:
|
||
raw: List[str] = list(args.target or [])
|
||
if args.from_file:
|
||
raw += [ln.strip() for ln in
|
||
Path(args.from_file).read_text(encoding="utf-8").splitlines()
|
||
if ln.strip() and not ln.strip().startswith("#")]
|
||
return raw
|
||
|
||
|
||
def next_abort_wait(streak: int, base: int, cap: int) -> int:
|
||
"""撞 IP 级注册限流后等多久再续跑。streak = 连着撞了几次（不含这次）。
|
||
|
||
抖音封的是 IP，不是设备指纹——DevicePool 换 UA 没用，只能等。实测一轮
|
||
能跑 5~6 小时、44 次设备轮换才触顶，冷却大概几小时。连撞就翻倍，说明
|
||
还没解封；有一段跑成了就把 streak 清零、等待打回底价。
|
||
"""
|
||
if base <= 0:
|
||
return 0
|
||
return min(base * (2 ** streak), cap)
|
||
|
||
|
||
def plan_batch(pending, max_items: int):
|
||
"""把待处理队列切一段出来跑，返回 (这一段, 还有没有剩下的)。
|
||
|
||
为什么要切段：单个 python 进程跑到 1.5 万条上下，Windows 会话的桌面堆
|
||
就被几万个 ffmpeg/ffprobe 子进程耗光，之后 CreateProcess 一律失败
|
||
（rc=3221225794 / STATUS_DLL_INIT_FAILED）。表现很好认：完成数冻结、
|
||
失败数猛涨、上传量不动、work/ 暴涨（下载成功但抽音失败的 mp4 按设计
|
||
留着）。实测撞过两次 —— 三进程并跑时一次，单进程跑到 16399 条又一次，
|
||
所以是**累积量**的问题，不是并发数。
|
||
|
||
段跑完 os.execv 重启自己，桌面堆跟着重置。断点续跑本来就靠
|
||
records.jsonl，重启不会重下。max_items=0 表示不切。
|
||
"""
|
||
if max_items and len(pending) > max_items:
|
||
return pending[:max_items], True
|
||
return pending, False
|
||
|
||
|
||
def sweep_work_dir(work: Path, records: Path) -> int:
|
||
"""清掉 work/ 里已经没用的残留，返回删了几个文件。
|
||
|
||
删两类：
|
||
* aweme_id 已经在 records.jsonl 里的 —— 早传上 COS 了，留着纯属占地方；
|
||
* `.part` 半截下载 —— 断点续传的 Range 起点，进程被杀后就是垃圾。
|
||
|
||
**不碰失败项的文件**：上传失败、抽音截断这些是故意留在本地的，重跑时
|
||
省一次下载，见 test_upload_failure_keeps_local_and_writes_no_record。
|
||
|
||
为什么要有这个函数：正常路径删文件（见下面 unlink 那两行），失败路径
|
||
故意不删，可失败项的文件从此再没人回收 —— 66411 条那批跑到 2.7% 就在
|
||
work/ 里攒了 11 GB，其中 414 个的 aweme_id 已经在 records.jsonl 里。
|
||
（那 414 个的 mp4 修改时间比记录时间还晚 17~166 秒，说明是记录写完之后
|
||
才落地的，具体怎么产生的没查清；这个清扫无论如何都能兜住。）
|
||
"""
|
||
if not work.is_dir():
|
||
return 0
|
||
done = set()
|
||
if records.exists():
|
||
# aweme_id 是 RecordLog 写出的第一个字段，按行前缀扫，不解析整行 JSON
|
||
# （records.jsonl 带 raw，1.8 GB，逐行 json.loads 要几分钟）。
|
||
# 容忍冒号后的空格：紧凑写法没有，但别让格式差异把清扫悄悄变成空操作。
|
||
pat = re.compile(rb'^\{\s*"aweme_id"\s*:\s*"(\d+)"')
|
||
with open(records, "rb") as f:
|
||
for line in f:
|
||
m = pat.match(line)
|
||
if m:
|
||
done.add(m.group(1).decode())
|
||
freed = 0
|
||
for p in work.iterdir():
|
||
if not p.is_file():
|
||
continue
|
||
if p.name.endswith(".part") or p.stem in done:
|
||
try:
|
||
p.unlink()
|
||
freed += 1
|
||
except OSError:
|
||
pass # 还被占着，下一轮再说
|
||
return freed
|
||
|
||
|
||
def extract_mp3(ffmpeg: str, src: Path, dst: Path, bitrate: str,
|
||
timeout: float = 600.0) -> None:
|
||
cmd = build_cmd(ffmpeg, src, dst, "mp3", bitrate)
|
||
try:
|
||
p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||
except subprocess.TimeoutExpired:
|
||
raise StageError("audio", f"ffmpeg 超时 >{timeout}s")
|
||
if p.returncode != 0 or not dst.exists() or dst.stat().st_size == 0:
|
||
err = (p.stderr or "").strip().splitlines()
|
||
raise StageError("audio", err[-1][:150] if err else f"rc={p.returncode}")
|
||
|
||
|
||
def decoded_seconds(path: Path, ffprobe: str, timeout: float = 300.0) -> Optional[float]:
|
||
"""实际解码算出的音频时长。
|
||
|
||
不能用 `format=duration`：LAME 写的 Xing 头声明了原始总帧数，ffprobe
|
||
直接信它而不解码。实测把 mp3 截断到 20%，format=duration 报的还是
|
||
10.0078s，和完整文件一模一样——拿它做截断校验等于没做。
|
||
|
||
`-count_frames` 会真解码。代价很小：120s 的 mp3 是 0.109s 对 0.053s。
|
||
"""
|
||
cmd = [ffprobe, "-v", "error", "-select_streams", "a:0", "-count_frames",
|
||
"-show_entries", "stream=nb_read_frames,sample_rate",
|
||
"-of", "default=noprint_wrappers=1"] # 保留 key= 前缀
|
||
try:
|
||
p = subprocess.run(cmd + [str(path)], capture_output=True, text=True,
|
||
timeout=timeout)
|
||
except subprocess.TimeoutExpired:
|
||
return None
|
||
|
||
# 按 key 解析，不靠位置：ffprobe 是按流内字段顺序输出的，不是按请求顺序
|
||
kv = {}
|
||
for line in (p.stdout or "").splitlines():
|
||
if "=" in line:
|
||
k, _, v = line.partition("=")
|
||
kv[k.strip()] = v.strip()
|
||
try:
|
||
frames = int(kv["nb_read_frames"])
|
||
rate = int(kv["sample_rate"])
|
||
except (KeyError, ValueError):
|
||
return None
|
||
if frames <= 0 or rate <= 0:
|
||
return None
|
||
# MPEG1 Layer III 每帧 1152 采样；MPEG2/2.5（<32kHz）是 576
|
||
per_frame = 1152 if rate >= 32000 else 576
|
||
return frames * per_frame / rate
|
||
|
||
|
||
def check_duration(mp4: Path, mp3: Path, ffprobe: str) -> float:
|
||
"""返回 mp4 的精确时长；顺便校验 mp3 没被截断。
|
||
|
||
ffmpeg 返回 0 但产出截断文件是见过的，光看 returncode 不够。mp4 传上去
|
||
之后本地就删了，所以这道闸门必须在删除之前把话说死。
|
||
"""
|
||
d4 = probe(mp4, ffprobe)
|
||
if not d4:
|
||
raise StageError("probe", "ffprobe 读不出 mp4 时长")
|
||
d3 = decoded_seconds(mp3, ffprobe)
|
||
if not d3:
|
||
raise StageError("probe", "mp3 解不出有效音频帧")
|
||
|
||
tol = max(1.0, d4 * 0.02)
|
||
if abs(d3 - d4) > tol:
|
||
raise StageError("probe",
|
||
f"mp3 实解 {d3:.3f}s 与 mp4 {d4:.3f}s 相差超过 {tol:.3f}s，"
|
||
"疑似抽取被截断")
|
||
return d4
|
||
|
||
|
||
def run_one(aweme_id: str, source_url: str, *, pool: DevicePool, up: CosUploader,
|
||
log: RecordLog, args, ffmpeg: str, ff_sem: threading.Semaphore,
|
||
stats: Stats, fail, abort: threading.Event) -> None:
|
||
# 注册配额是 IP 级的，耗尽后换设备也没用。这时候必须整批停下来等，
|
||
# 而不是让剩下几千条一条条去撞同一堵墙、全标成失败——上一批就是这么
|
||
# 烧掉 678 条的。abort 一旦拉起，后续任务直接返回，不计失败。
|
||
if abort.is_set():
|
||
with stats.lock:
|
||
stats.skipped += 1
|
||
return
|
||
|
||
work = Path(args.work_dir)
|
||
mp4 = work / f"{aweme_id}.mp4"
|
||
mp3 = work / f"{aweme_id}.mp3"
|
||
|
||
try:
|
||
# 1. 元信息
|
||
try:
|
||
device, gen = pool.current()
|
||
except DeviceRegisterError as exc:
|
||
abort.set()
|
||
with stats.lock:
|
||
stats.failed += 1
|
||
fail(aweme_id, "abort", f"IP 级注册配额耗尽，已中止整批：{exc}")
|
||
return
|
||
try:
|
||
aweme = AppClient(device).fetch_detail(aweme_id)
|
||
except NotAVideo as exc:
|
||
raise StageError("detail", str(exc))
|
||
except DouyinError as exc:
|
||
# 空 body 多半是设备配额到了，换设备重来一次
|
||
if "空 body" in str(exc):
|
||
pool.report_dead(gen)
|
||
device, gen = pool.current()
|
||
aweme = AppClient(device).fetch_detail(aweme_id)
|
||
else:
|
||
raise StageError("detail", str(exc))
|
||
|
||
if not aweme.video_urls and not aweme.uri:
|
||
raise StageError("detail", "没有可用的视频直链")
|
||
|
||
# 2. 下载
|
||
try:
|
||
res = download(session(), aweme.video_urls or [aweme.play_url()],
|
||
mp4, progress=False)
|
||
except Exception as exc:
|
||
raise StageError("download", f"{type(exc).__name__}: {exc}")
|
||
if res.size <= 0:
|
||
raise StageError("download", "下载产物是空文件")
|
||
|
||
# 3. 抽 mp3（CPU 密集，限流）
|
||
with ff_sem:
|
||
extract_mp3(ffmpeg, mp4, mp3, args.bitrate)
|
||
|
||
# 4~5. 探时长 + 校验没被截断
|
||
duration = check_duration(mp4, mp3, args.ffprobe)
|
||
|
||
# 6~8. 上传，put() 内部做 head_object 大小校验
|
||
try:
|
||
url4, size4 = up.put(mp4, up.key_for(mp4.name))
|
||
url3, size3 = up.put(mp3, up.key_for(mp3.name))
|
||
except CosError as exc:
|
||
raise StageError("upload", str(exc))
|
||
|
||
# 9. 记录，然后才敢删
|
||
log.write(build_record(aweme, duration_sec=duration,
|
||
cos_mp4=url4, cos_mp3=url3,
|
||
size_mp4=size4, size_mp3=size3,
|
||
source_url=source_url))
|
||
|
||
if not args.keep_local:
|
||
mp4.unlink(missing_ok=True)
|
||
mp3.unlink(missing_ok=True)
|
||
|
||
with stats.lock:
|
||
stats.done += 1
|
||
stats.bytes_up += size4 + size3
|
||
|
||
except DeviceRegisterError as exc:
|
||
abort.set()
|
||
with stats.lock:
|
||
stats.failed += 1
|
||
fail(aweme_id, "abort", f"IP 级注册配额耗尽，已中止整批：{exc}")
|
||
except StageError as exc:
|
||
with stats.lock:
|
||
stats.failed += 1
|
||
fail(aweme_id, exc.stage, str(exc))
|
||
except Exception as exc: # 兜底，别让一条炸掉整批
|
||
with stats.lock:
|
||
stats.failed += 1
|
||
fail(aweme_id, "unknown", f"{type(exc).__name__}: {exc}")
|
||
|
||
|
||
def main(argv: Optional[List[str]] = None) -> int:
|
||
ap = argparse.ArgumentParser(description="抖音采集一体化流水线")
|
||
ap.add_argument("target", nargs="*", help="链接或 aweme_id")
|
||
ap.add_argument("--from-file", default="", help="从文件读链接，一行一个")
|
||
ap.add_argument("--records", default="records/records.jsonl", help="元信息落点")
|
||
ap.add_argument("--failed", default="records/pipeline_failed.txt")
|
||
ap.add_argument("--work-dir", default="work", help="mp4/mp3 临时目录，传完即删")
|
||
ap.add_argument("--region", required=True)
|
||
ap.add_argument("--bucket", default=DEFAULT_BUCKET)
|
||
ap.add_argument("--prefix", default=DEFAULT_PREFIX)
|
||
ap.add_argument("--url-style", choices=["cos", "https", "path"], default="cos")
|
||
ap.add_argument("--workers", type=int, default=8)
|
||
ap.add_argument("--bitrate", default="192k")
|
||
ap.add_argument("--ffmpeg", default="ffmpeg")
|
||
ap.add_argument("--ffprobe", default="ffprobe")
|
||
ap.add_argument("--ffmpeg-workers", type=int, default=0,
|
||
help="并发 ffmpeg 数，0=CPU 核数")
|
||
ap.add_argument("--device-quota", type=int, default=550,
|
||
help="单设备 detail 调用上限。实测约 575 次后被限流，"
|
||
"留一点余量；调高可减少注册次数")
|
||
ap.add_argument("--limit", type=int, default=0, help="只跑前 N 条，用于试跑")
|
||
ap.add_argument("--max-items", type=int, default=12000,
|
||
help="单个进程最多跑多少条，跑满就重启自己接着跑。"
|
||
"防 Windows 桌面堆耗尽，见 plan_batch。0=不切段")
|
||
ap.add_argument("--abort-wait", type=int, default=1800,
|
||
help="撞上 IP 级注册限流后等多少秒再自动续跑。"
|
||
"连撞会翻倍，见 next_abort_wait。0=不等，直接退出")
|
||
ap.add_argument("--abort-wait-cap", type=int, default=14400,
|
||
help="上面那个等待的上限秒数，默认 4 小时")
|
||
ap.add_argument("--keep-local", action="store_true",
|
||
help="上传后不删本地文件（试跑时用）")
|
||
args = ap.parse_args(argv)
|
||
|
||
targets = read_targets(args)
|
||
if not targets:
|
||
ap.error("没有目标。给链接，或用 --from-file。")
|
||
|
||
try:
|
||
ffmpeg = ensure_ffmpeg(args.ffmpeg)
|
||
except Exception as exc:
|
||
print(f"[错误] {exc}", file=sys.stderr)
|
||
return 2
|
||
|
||
try:
|
||
up = CosUploader(region=args.region, bucket=args.bucket, prefix=args.prefix,
|
||
url_style=args.url_style)
|
||
except CosError as exc:
|
||
print(f"[错误] {exc}", file=sys.stderr)
|
||
return 2
|
||
|
||
log = RecordLog(Path(args.records))
|
||
print(f"已完成 {len(log)} 条（从 {args.records} 读出），本次输入 {len(targets)} 条")
|
||
|
||
# 解析 id，顺便去重和跳过已完成的
|
||
pending = []
|
||
seen = set()
|
||
bad = 0
|
||
for raw in targets:
|
||
try:
|
||
aid = extract_aweme_id(raw, session())
|
||
except ParseError:
|
||
bad += 1
|
||
continue
|
||
if aid in seen:
|
||
continue
|
||
seen.add(aid)
|
||
if aid in log:
|
||
continue
|
||
pending.append((aid, raw))
|
||
if args.limit:
|
||
pending = pending[: args.limit]
|
||
|
||
skipped = len(seen) - len(pending) if not args.limit else 0
|
||
print(f"待处理 {len(pending)} 条"
|
||
+ (f"，跳过已完成 {skipped} 条" if skipped else "")
|
||
+ (f"，无法解析 {bad} 条" if bad else ""))
|
||
if not pending:
|
||
return 0
|
||
|
||
pending, more = plan_batch(pending, args.max_items)
|
||
if more:
|
||
print(f"本段只跑 {len(pending)} 条，跑完自动重启进程接着来"
|
||
f"（见 plan_batch 的注释：桌面堆）")
|
||
|
||
Path(args.work_dir).mkdir(parents=True, exist_ok=True)
|
||
if not args.keep_local:
|
||
n = sweep_work_dir(Path(args.work_dir), Path(args.records))
|
||
if n:
|
||
print(f"清掉 work/ 里 {n} 个已入库或半截的残留文件")
|
||
stats = Stats(total=len(pending))
|
||
fail_lock = threading.Lock()
|
||
fail_path = Path(args.failed)
|
||
|
||
def fail(aweme_id: str, stage: str, msg: str) -> None:
|
||
with fail_lock:
|
||
with open(fail_path, "a", encoding="utf-8") as f:
|
||
f.write(f"{aweme_id}\t{stage}\t{msg}\n")
|
||
|
||
pool = DevicePool(max_calls=args.device_quota)
|
||
# DevicePool 是惰性注册的。先主动要一次，注册不了就当场退出，
|
||
# 而不是让每个 worker 各自撞一遍、把整批标成失败。
|
||
try:
|
||
pool.current()
|
||
except DeviceRegisterError as exc:
|
||
# 开跑就注册不上 = IP 还在冷却期。这条路以前直接退出，等于
|
||
# "撞限流后自动续跑"在冷却没结束时会白白放弃，所以这里也要等。
|
||
streak = int(os.environ.get("_DY_ABORT_STREAK", "0"))
|
||
wait = next_abort_wait(streak, args.abort_wait, args.abort_wait_cap)
|
||
if wait:
|
||
os.environ["_DY_ABORT_STREAK"] = str(streak + 1)
|
||
until = time.strftime("%H:%M:%S", time.localtime(time.time() + wait))
|
||
print(f"[限流] 设备注册失败：{exc}", file=sys.stderr)
|
||
print(f"IP 还在冷却，等 {wait // 60} 分钟到 {until} 再试…", flush=True)
|
||
sys.stdout.flush()
|
||
sys.stderr.flush()
|
||
time.sleep(wait)
|
||
os.execv(sys.executable, [sys.executable] + sys.argv)
|
||
print(f"[错误] 设备注册失败：{exc}", file=sys.stderr)
|
||
return 3
|
||
|
||
n_ff = args.ffmpeg_workers or (os.cpu_count() or 4)
|
||
ff_sem = threading.Semaphore(n_ff)
|
||
print(f"并发 {args.workers}，其中 ffmpeg 限 {n_ff}"
|
||
+ ("，本地文件保留不删" if args.keep_local else ""))
|
||
|
||
t0 = time.time()
|
||
stop = threading.Event()
|
||
abort = threading.Event()
|
||
|
||
def report() -> None:
|
||
ticks = 0
|
||
while not stop.wait(15):
|
||
ticks += 1
|
||
with stats.lock:
|
||
d, f_, tot = stats.done, stats.failed, stats.total
|
||
gb = stats.bytes_up / 1024 ** 3
|
||
el = time.time() - t0
|
||
rate = d / el * 60 if el else 0
|
||
left = (tot - d - f_) / rate if rate else 0
|
||
print(f" [{d + f_}/{tot}] 完成{d} 失败{f_} | {gb:.2f}GB 上传 | "
|
||
f"{rate:.0f}条/分 | 剩约 {left:.0f}分", flush=True)
|
||
# 每 5 分钟回收一次：光靠开跑前扫一次，长批跑到一半磁盘就满了
|
||
if ticks % 20 == 0 and not args.keep_local:
|
||
n = sweep_work_dir(Path(args.work_dir), Path(args.records))
|
||
if n:
|
||
print(f" [清扫] 回收 {n} 个残留文件", flush=True)
|
||
|
||
threading.Thread(target=report, daemon=True).start()
|
||
try:
|
||
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||
for aid, raw in pending:
|
||
ex.submit(run_one, aid, raw, pool=pool, up=up, log=log, args=args,
|
||
ffmpeg=ffmpeg, ff_sem=ff_sem, stats=stats, fail=fail,
|
||
abort=abort)
|
||
finally:
|
||
stop.set()
|
||
|
||
el = time.time() - t0
|
||
print(f"\n完成 {stats.done}，失败 {stats.failed}，"
|
||
f"上传 {stats.bytes_up / 1024 ** 3:.2f} GB，耗时 {el / 60:.1f} 分")
|
||
print(f"元信息 -> {args.records}（累计 {len(log)} 条）")
|
||
if stats.failed:
|
||
print(f"失败明细 -> {fail_path}（本地文件已保留，可直接重跑本脚本续传）")
|
||
|
||
def _restart(why: str, wait: int = 0) -> None:
|
||
"""重启自己接着跑。断点续跑靠 records.jsonl，重启不会重下。"""
|
||
if wait:
|
||
until = time.strftime("%H:%M:%S", time.localtime(time.time() + wait))
|
||
print(f"{why}，等 {wait // 60} 分钟到 {until} 再续…", flush=True)
|
||
else:
|
||
print(f"{why}，重启进程接着跑…", flush=True)
|
||
sys.stdout.flush()
|
||
sys.stderr.flush()
|
||
if wait:
|
||
time.sleep(wait)
|
||
os.execv(sys.executable, [sys.executable] + sys.argv)
|
||
|
||
# 连撞计数走环境变量，execv 换掉进程映像后靠它记住上一轮的状态
|
||
streak = int(os.environ.get("_DY_ABORT_STREAK", "0"))
|
||
if abort.is_set():
|
||
# IP 级注册配额耗尽。抖音封的是 IP，换设备没用，只能等冷却。
|
||
wait = next_abort_wait(streak, args.abort_wait, args.abort_wait_cap)
|
||
if wait:
|
||
# 这一轮但凡出了活就说明限流是跑着跑着才触发的，下次从底价重等
|
||
os.environ["_DY_ABORT_STREAK"] = "0" if stats.done else str(streak + 1)
|
||
_restart("撞上 IP 级注册限流", wait)
|
||
print("撞上 IP 级注册限流，已停下。等几小时 IP 冷却后重跑本命令即可续传。")
|
||
return 5
|
||
|
||
os.environ["_DY_ABORT_STREAK"] = "0"
|
||
if more:
|
||
_restart("本段跑完")
|
||
|
||
return 0 if stats.failed == 0 else 4
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|