初始提交：抖音视频批量采集流水线

App 端免签名接口（设备注册 + detail + CDN 直链），一体化流水线下载→抽音→传COS→记录→删本地，AI 作品四档判定，四份文档： README（使用指南）、PROGRESS（批次进度）、RE（逆向全过程）、AI（判定规则）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-09-29 16:31:10 +08:00
commit 2bb83f3685
39 changed files with 298874 additions and 0 deletions
--- a/douyin/batch.py
+++ b/douyin/batch.py
@@ -0,0 +1,197 @@
+"""批量下载：并发、可断点续跑、单条失败不影响整批。
+
+设计要点：
+  - 设备身份只注册一次，各 worker 共用；但每个 worker 有独立 Session
+    （requests.Session 并发下不保证安全）
+  - 落盘文件名就是 aweme_id，所以「是否已下载」直接看文件在不在，
+    不依赖去重记录 —— 记录丢了也能无损续跑
+  - 失败条目写 failed.txt，跑完可直接拿它当输入重试
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional, Sequence
+
+import requests
+
+from .app import AppClient, Device, NotAVideo
+from .client import DouyinError
+from .download import Registry, download
+
+
+class DevicePool:
+    """设备身份池。
+
+    实测单个 device_id 在约 575 次 detail 调用后被限流，表现是接口一直
+    回 HTTP 200 空 body；同一 IP 下换个新注册的设备立刻恢复 —— 说明配额
+    是设备级而非 IP 级。注册本身免签名、成本极低，所以按量轮换即可。
+
+    用 generation 标记当前设备：worker 拿到的 gen 和池里的对得上才允许
+    触发轮换，避免多个 worker 同时发现失败、把设备连换好几次。
+    """
+
+    def __init__(self, max_calls: int = 400, initial: Optional[Device] = None) -> None:
+        self._lock = threading.Lock()
+        self._device = initial
+        self._gen = 0
+        self._calls = 0
+        self.max_calls = max_calls
+        self.rotations = 0
+
+    def current(self) -> tuple[Device, int]:
+        with self._lock:
+            if self._device is None or self._calls >= self.max_calls:
+                self._rotate_locked("配额用尽")
+            self._calls += 1
+            return self._device, self._gen
+
+    def report_dead(self, gen: int) -> None:
+        """worker 发现当前设备被限流时调用。"""
+        with self._lock:
+            if gen == self._gen:
+                self._rotate_locked("被限流")
+
+    def _rotate_locked(self, why: str) -> None:
+        self._device = Device.register()
+        self._gen += 1
+        self._calls = 0
+        self.rotations += 1
+        print(f"  [设备轮换#{self.rotations}] {why} -> "
+              f"device_id={self._device.device_id}", flush=True)
+
+
+@dataclass
+class BatchStats:
+    total: int = 0
+    done: int = 0
+    skipped: int = 0
+    not_video: int = 0
+    failed: int = 0
+    bytes_: int = 0
+    lock: threading.Lock = field(default_factory=threading.Lock)
+
+    @property
+    def finished(self) -> int:
+        return self.done + self.skipped + self.not_video + self.failed
+
+
+def run_batch(aweme_ids: Sequence[str], out_dir: Path, device: Optional[Device] = None,
+              workers: int = 5, delay: float = 0.3,
+              registry: Optional[Registry] = None,
+              log_every: int = 25, max_calls_per_device: int = 400) -> BatchStats:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    registry = registry or Registry(out_dir / ".downloaded.json")
+    pool = DevicePool(max_calls=max_calls_per_device, initial=device)
+
+    stats = BatchStats(total=len(aweme_ids))
+    failed_path = out_dir / "failed.txt"
+    # 每轮重来，否则上一轮已修复的失败会一直留在文件里造成误导
+    failed_path.write_text("", encoding="utf-8")
+    failed_lock = threading.Lock()
+    local = threading.local()
+    t0 = time.time()
+
+    def client_for(device_: Device, gen: int) -> AppClient:
+        """每线程每代设备一个 Session（Session 并发不安全）。"""
+        if getattr(local, "gen", None) != gen:
+            local.client = AppClient(device=device_)
+            local.gen = gen
+        return local.client
+
+    def dl_session() -> requests.Session:
+        """下载用的 Session：CDN 免鉴权，不该占设备配额，也不随设备轮换重建。"""
+        if not hasattr(local, "dl"):
+            local.dl = requests.Session()
+            local.dl.headers["User-Agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+            )
+        return local.dl
+
+    def record_failure(aweme_id: str, why: str) -> None:
+        with failed_lock:
+            with open(failed_path, "a", encoding="utf-8") as f:
+                f.write(f"{aweme_id}\t{why}\n")
+
+    def fetch_with_rotation(aweme_id: str):
+        """限流表现为空 body。碰到就换设备再试一次，而不是直接判失败。"""
+        last: Optional[Exception] = None
+        for _ in range(2):
+            dev, gen = pool.current()
+            client = client_for(dev, gen)
+            if delay:
+                time.sleep(delay)
+            try:
+                return client.fetch_detail(aweme_id, retries=2)
+            except NotAVideo:
+                raise
+            except DouyinError as exc:
+                last = exc
+                if "空 body" in str(exc):
+                    pool.report_dead(gen)
+                    continue
+                raise
+        raise last  # type: ignore[misc]
+
+    def handle(aweme_id: str) -> None:
+        dest = out_dir / f"{aweme_id}.mp4"
+        # 文件在就算下过，不必碰网络也不依赖记录
+        if dest.exists() and dest.stat().st_size > 0:
+            with stats.lock:
+                stats.skipped += 1
+            return
+
+        try:
+            aweme = fetch_with_rotation(aweme_id)
+        except NotAVideo as exc:
+            with stats.lock:
+                stats.not_video += 1
+            record_failure(aweme_id, f"not_video: {exc}")
+            return
+        except Exception as exc:
+            with stats.lock:
+                stats.failed += 1
+            record_failure(aweme_id, f"detail: {type(exc).__name__}: {exc}")
+            return
+
+        if not aweme.video_urls:
+            with stats.lock:
+                stats.not_video += 1
+            record_failure(aweme_id, "no_video_url")
+            return
+
+        try:
+            result = download(dl_session(), aweme.video_urls, dest, progress=False)
+        except Exception as exc:
+            with stats.lock:
+                stats.failed += 1
+            record_failure(aweme_id, f"download: {type(exc).__name__}: {exc}")
+            return
+
+        registry.add(aweme_id, result.path, result.size,
+                     desc=aweme.desc, author=aweme.author, uri=aweme.uri)
+        with stats.lock:
+            stats.done += 1
+            stats.bytes_ += result.size
+
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = [executor.submit(handle, a) for a in aweme_ids]
+        for n, _ in enumerate(as_completed(futures), 1):
+            if n % log_every == 0 or n == len(futures):
+                el = time.time() - t0
+                rate = n / el if el else 0
+                eta = (len(futures) - n) / rate if rate else 0
+                print(f"[{n}/{len(futures)}] 成功{stats.done} 跳过{stats.skipped} "
+                      f"图集{stats.not_video} 失败{stats.failed} "
+                      f"| {stats.bytes_ / 1024 ** 3:.2f}GB "
+                      f"| {rate * 60:.0f}条/分 | 换设备{pool.rotations}次 "
+                      f"| 剩约 {eta / 60:.0f}分",
+                      flush=True)
+
+    registry.flush()
+    return stats