WhaleFall/douyin_video_download
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始提交：抖音视频批量采集流水线

Browse Source 
App 端免签名接口（设备注册 + detail + CDN 直链），一体化流水线
下载→抽音→传COS→记录→删本地，AI 作品四档判定，四份文档：
README（使用指南）、PROGRESS（批次进度）、RE（逆向全过程）、AI（判定规则）。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
whale-fall-code
2026-09-29 16:31:10 +08:00
commit 2bb83f3685
39 changed files with 298874 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
douyin/__init__.py Normal file
View File

@@ -0,0 +1,37 @@
"""抖音单视频无水印下载。"""
from .app import AppClient, Device, DeviceRegisterError, NotAVideo
from .batch import BatchStats, run_batch
from .client import (
Aweme,
DouyinClient,
DouyinError,
LoginRequired,
decode_abort,
play_url_from_uri,
)
from .cos import CosError, CosUploader, SizeMismatch
from .download import Registry, download, safe_filename
from .meta import RecordLog, build_record, iter_records, split_desc
from .signer import (
ABogusConfig,
ABogusSigner,
NodeSigner,
SignError,
Signer,
SignerNotConfigured,
)
from .urls import ParseError, extract_aweme_id
__all__ = [
"AppClient", "Device", "DeviceRegisterError", "NotAVideo",
"BatchStats", "run_batch",
"Aweme", "DouyinClient", "DouyinError", "LoginRequired", "decode_abort",
"play_url_from_uri",
"Registry", "download", "safe_filename",
"CosError", "CosUploader", "SizeMismatch",
"RecordLog", "build_record", "iter_records", "split_desc",
"ABogusConfig", "ABogusSigner", "NodeSigner", "SignError", "Signer",
"SignerNotConfigured",
"ParseError", "extract_aweme_id",
]

258
douyin/aiclass.py Normal file
View File

@@ -0,0 +1,258 @@
"""判定一条抖音作品是不是 AI 相关，以及是哪一类相关。
分四档，档位之间不重叠，取最强的那一档：
platform 抖音自己打了 AIGC 标（ai.is_aigc）。最可信，而且是唯一能抓到
"作者什么都没标、但确实是 AI 生成"的信号 —— 实测 1744 条属于
这种，文案标签里一个 ai 字都没有（脑筋急转弯、历史解说、原创动画）。
strong 文案/标签里有明确的"AI 生成作品"信号：AIGC、AI 生成类工具名、
ai+创作类词、平台的 AI 赛事标签。
topic 只提到 AI 本身（ai 教程、人工智能、大模型、ChatGPT 之类），
片子讲的是 AI，但不一定是 AI 做的。
author 作品本身没信号，只有作者昵称里带 AI/AIGC。最弱，会误收
（实测 @牛牛Ai 发的普法解说、@你好，我是大福 的真人游泳视频）。
坑（都是从这批 40087 条真实数据里挖出来的，别顺手删）：
1. "Al"（大写 A + 小写 L）被当成 AI 用，标签 `抖音al` 出现 129 次，
文案里也有 `#Al创作`。必须归一化，否则整片漏掉。
2. 反过来，`ai` 是英文单词的常见片段（email / detail / again / hair），
所以只认前后不接字母数字的独立 ai，不能用 in 判断。
3. `剪映`/`关键帧`/`蒙版`/`转场`/`剪辑教程` 是剪辑术语，不是 AI。
它们在这批数据里量很大（剪映 375、关键帧 314、蒙版 203），
早期版本把"动画/绘画/分身"这类词单独当信号，误收了几千条。
所以中文泛词一律要求跟 AI 词同现，不单独成立。
4. 工具名不含 ai 字样的一大票：即梦 / seedance / libtv / flovaai /
小云雀 / 随变 / runninghub / oiioii / shotlab，只能靠名单。
5. 音乐标题默认是 `@<作者昵称>创作的原声`，40087 条里有 32555 条是这个壳。
昵称带 AI 就会连"创作"一起命中，等于让作者昵称从 music 这条路
偷偷升到 strong —— 实测 771 条这样混进来（@冠宇Aigc 的猴子炒菜、
@牛牛Ai 的大案纪实）。所以先剥壳，剥完只当弱信号：全库剥完只剩
16 条音乐标题还有命中，这条路本来就没什么价值。
注意壳有两种写法：jsonl 里是 `@昵称创作的原声`，csv 导出的是
`@昵称创作的原声 - 昵称`，后缀那半截也要一起剥。
6. 一批工具名本身就是常用词，单独出现几乎全是误判，必须要求跟 ai/aigc
同现（见 _AMBIG_*）。实测：豆包 224 条误判（狗名、豆沙包、"反强包
之神"）对 145 条真命中；海螺 13 条是三峰的"海螺"帐篷；元宝 11 条是
元宝勺辅食餐具；万相 17 条是《龙神万相》和万相集；星流 1 条来自
"明星流量"（子串）；清影 1 条来自"起舞弄清影"。而真命中的那些**无一
例外**都同时带 AI 字样，所以加这道门槛不掉召回。
7. 别把"数字人""换脸""分身"放开成单独信号：数字人民币含"数字人"、
微信分身/五五分身材含"分身"、穿搭视频也说"这套穿搭能换脸"。
实测放开只进误判，一条真的都捞不到。
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Tuple
# 独立的 ai —— 前后不能接字母数字，挡掉 email、detail、hair。
# 第二个字符收了三种冒充写法：i、小写 L、罗马数字 Ⅰ/ⅰ（U+2160/U+2170）。
# 实测标签 `抖音al` 129 次、`#AⅠ漫剧` 1 条。别改成简单的 in 判断。
_I = r'[ilⅠⅰ]'
_AI = re.compile(r'(?<![a-z0-9])a' + _I + r'(?![a-z0-9])', re.I)
_AIGC = re.compile(r'(?<![a-z0-9])a' + _I + r'gc(?![a-z0-9])', re.I)
# 生成类工具 / 平台。命中即 strong —— 用了这些东西，片子就是 AI 做的。
_GEN_TOOLS = [
"即梦", "dreamina", "jimeng", "seedance", "seedream", "可灵", "kling",
"hailuo", "vidu", "pixverse", "pika", "luma", "runway", "sora",
"viggle", "heygen", "midjourney", "comfyui", "runninghub", "flux",
"nanobanana", "nano banana", "stable diffusion", "liblib",
"krea", "ideogram", "recraft", "leonardo",
"higgsfield", "suno", "udio", "libtv", "flovaai", "海艺", "小云雀",
"shotlab", "sekoai", "movieflow", "updream", "cineart", "oiioii",
"minimax", "妙鸭", "一帧秒创", "触手ai", "拍我ai", "美图ai",
"文生视频", "图生视频", "文生图", "图生图", "生成式",
# --- 下面这批是 2026-08 补的。全部在 12.7 万条真实数据上验过：
# 要么零命中（纯属给以后的文件备着），要么命中的全是真 AI，
# 没有一个引入新误判。判断标准是"这串字在中文短视频文案里
# 不可能有别的意思"，所以像 tripo（三脚架 tripod）、seaweed
# （海藻）、lora（撞 valorant）、captions、playground 这些
# 一律没收 —— 实测 lora 命中 3 条全是无畏契约电竞。
"veo3", "wan2.1", "wanx", "通义万相", "hunyuanvideo", "混元视频",
"cogvideo", "open-sora", "animatediff", "controlnet", "智谱清影",
"step-video", "hedra", "kaiber", "haiper", "moonvalley", "genmo",
"domoai", "synthesia", "akool", "deepbrain", "opusclip", "argil",
"kreadoai", "腾讯智影", "讯飞智作", "硅基智能", "魔音工坊", "元气口播",
"万兴播爆", "闪剪", "声音克隆", "语音克隆", "克隆音色", "音色克隆",
]
# AI 漫剧圈的专有名词：**单独出现即 strong**，不要求跟 ai 同现。
# 抖音的 AI 漫剧已经形成固定圈子，这些词在 18.4 万条真实数据里几乎只出现在
# AI 作品上（括号=全库命中/其中现在还漏掉的）。选词标准是"这串字在中文短视频
# 文案里不可能有别的意思"：
# 漫剧(1568/64) 治愈系漫画(80/0) 渡灵书(62/0) 超级裂变(46/0) 漫剧推荐(38/2)
# 二次元动漫(33/0) 月落小溪(25/4) 七罪创世(21/0) 晚夏小梦(67/4)
# 灵绘漫客(12/0) neowow(12/0) 造梦国度(2/0) 猫箱跨次元相遇(114/12)
# 没收的（实测污染严重，它们是题材不是 AI 标记）：
# 西游记(+26 真·玄奘故里游记) 孙悟空(+16) 宫崎骏(+10 龙猫童年) 民间故事(+7《狼来了》)
# 睡前故事(+5 爷俩演西游) 神话故事(+2) 国漫推荐(+4《剑来》真人制作)
# 原创动漫(+18 手绘二次元) 创作者激励计划(+12 原神/卡西尼号) 未来导演扶持计划(+4)
# 开放赛道(+2) —— 后两个是抖音通用活动，不专属 AI
_MANJU = [
"漫剧", "治愈系漫画", "渡灵书", "超级裂变", "二次元动漫", "月落小溪",
"七罪创世", "晚夏小梦", "灵绘漫客", "neowow", "造梦国度", "猫箱跨次元相遇",
# 红果（字节的免费短剧 App）上的 AI 漫剧片名。来源是新闻报道的热榜，
# **不是红果的完整目录** —— 红果不公开片库，AI 漫剧实际有成百上千部，
# 这里只有见诸报道的头部作品，以后遇到新的往这加。
# 在 18.4 万条上验过：16/18 零命中（纯属备着），万妖图录传 1 条且正是
# 现在漏掉的真 AI 漫剧，零污染。
"万妖图录传", "发配边关", "罪妻开荒养出战神", "儿媳的重生棋局",
"斩仙台", "菩提临世", "聚宝仙盆", "杂灵根", "渔乡守真心", "金钱照冷暖",
"杀猪辣媳", "夫人一个过肩摔", "瘫子总裁站起来", "凡人百世书",
"都重生了谁还装富二代",
# **"红果"/"红果短剧" 不能进这个表**：红果主体是真人短剧，AI 漫剧只是
# 其中一个赛道。实测 3 条命中里 2 条是真人短剧（《家里家外2》《冷御失守》），
# 1 条是短剧演员面试。要抓红果的 AI 漫剧只能靠具体片名。
]
# 名字本身是常用词的生成类工具：只有跟 ai/aigc 同现才算 strong。
# 见模块开头第 6 条 —— 单独出现基本全是误判，真命中都带 AI 字样。
_AMBIG_GEN = [
"海螺", "白日梦", "万相", "可图", "清影", "星流", "秒画",
"造梦日记", "随变",
]
# 对话/通用大模型。这些更像是"在聊 AI"，归 topic。
_CHAT_TOOLS = [
"chatgpt", "deepseek", "kimi", "通义", "千问", "qwen", "文心",
"ernie", "智谱", "glm", "copilot", "gemini", "grok",
"人工智能", "大模型", "神经网络", "机器学习", "深度学习",
]
# 同上，但归 topic 档的那批常用词模型名
_AMBIG_CHAT = ["豆包", "元宝", "混元"]
# 单独成词才算，避免 gpt 撞上别的串
_CHAT_RE = re.compile(r'(?<![a-z0-9])(?:gpt|claude|llm|agi|ai\s*agent)(?![a-z0-9])')
# 跟 ai 同现才成立的"创作/生成"语境词。单独出现一律不算。
_GEN_CTX = [
"短剧", "漫剧", "动画", "动漫", "视频", "短片", "电影", "影视", "绘画",
"绘图", "创作", "制作", "生成", "分身", "剧情", "作画", "配音", "换脸",
"特效", "导演", "演员", "数字人", "虚拟人", "成片", "新影像", "影像",
"浪潮", "大赛", "赛道", "大赏", "工作台", "写真", "画师", "作品", "片子",
"微短剧", "番剧", "剪辑", "转绘", "扩图", "修图", "提示词",
]
_GEN_CTX_RE = re.compile("|".join(map(re.escape, _GEN_CTX)))
def _alt(words: List[str]) -> re.Pattern:
"""把词表编成一个正则。纯 ASCII 的词加字母边界：前后不能紧接字母，
但**允许接数字和中文** —— 数据里满地都是 `Seedance2.0`、`sora2`、
`runninghub0.21`、`libtv创作者`、`flovaai超创`，一刀切加 \\b 会全丢。
只挡真正的词中词：`udio` 藏在 `theonestudio`/`Grass Studio` 里 17 次，
`flux` 藏在 `Fluxxwave` 曲名里 1 次，两个都是从 music 混进 author 档的。
"""
parts = []
for w in words:
e = re.escape(w)
if re.fullmatch(r"[a-z0-9 .\-]+", w):
e = r"(?<![a-z])" + e + r"(?![a-z])"
parts.append(e)
return re.compile("|".join(parts))
_TOOL_RE = _alt(_GEN_TOOLS)
_CHATW_RE = _alt(_CHAT_TOOLS)
_MANJU_RE = _alt(_MANJU)
_AMBIG_GEN_RE = _alt(_AMBIG_GEN)
_AMBIG_CHAT_RE = _alt(_AMBIG_CHAT)
# 抖音默认原声的标题壳，里面就是作者昵称，不含额外信息。
# jsonl 是 `@昵称创作的原声`，csv 导出多一截 ` - 昵称`，见模块开头第 5 条。
_SOUND_WRAP = re.compile(r'^@.*?(?:创作的原声|的原声)(?:\s*-\s*.*)?$')
TIERS = ("platform", "strong", "topic", "author", "none")
_RANK = {t: i for i, t in enumerate(TIERS)}
def _fields(rec: Dict[str, Any]) -> Tuple[str, str, str]:
"""(作品文本, 作者昵称, 音乐标题)。作品文本 = desc + 所有标签名。"""
tags = [t.get("name", "") if isinstance(t, dict) else str(t)
for t in (rec.get("hashtags") or [])]
body = " ".join([rec.get("desc") or ""] + tags)
author = (rec.get("author") or {}).get("nickname") or ""
music = ((rec.get("music") or {}).get("title") or "").strip()
if _SOUND_WRAP.match(music):
music = "" # 只是作者昵称的复读，见模块开头第 5 条
return body, author, music
def _scan(text: str, where: str) -> Tuple[str, List[str]]:
"""扫一段文本，返回 (档位, 命中的信号)。档位只可能是 strong/topic/none。"""
low = text.lower()
sig: List[str] = []
tier = "none"
has_aigc = bool(_AIGC.search(low))
has_ai = bool(_AI.search(low))
# 歧义词表的开关：文本里得先有 ai/aigc 字样，那些常用词才作数
ai_present = has_ai or has_aigc
if has_aigc:
sig.append(f"{where}:aigc")
tier = "strong"
for m in sorted(set(_TOOL_RE.findall(low))):
sig.append(f"{where}:工具:{m}")
tier = "strong"
# 漫剧专名单独成立，不要求 ai 同现
for m in sorted(set(_MANJU_RE.findall(low))):
sig.append(f"{where}:漫剧:{m}")
tier = "strong"
if ai_present:
for m in sorted(set(_AMBIG_GEN_RE.findall(low))):
sig.append(f"{where}:工具:{m}")
tier = "strong"
if has_ai:
ctx = sorted(set(_GEN_CTX_RE.findall(text)))
if ctx:
sig.append(f"{where}:ai+{'/'.join(ctx[:4])}")
tier = "strong"
else:
sig.append(f"{where}:ai")
if tier == "none":
tier = "topic"
chat = set(_CHATW_RE.findall(low)) | set(_CHAT_RE.findall(low))
if ai_present:
chat |= set(_AMBIG_CHAT_RE.findall(low))
chat = sorted(chat)
if chat:
sig.append(f"{where}:模型:{'/'.join(chat[:4])}")
if tier == "none":
tier = "topic"
return tier, sig
def classify(rec: Dict[str, Any]) -> Dict[str, Any]:
"""判一条记录。返回 {tier, signals, is_aigc}。"""
body, author, music = _fields(rec)
signals: List[str] = []
tier = "none"
is_aigc = bool((rec.get("ai") or {}).get("is_aigc"))
label = ((rec.get("ai") or {}).get("label") or "").strip()
if is_aigc:
tier = "platform"
signals.append(f"platform:{label or 'is_aigc'}")
t, s = _scan(body, "desc")
signals.extend(s)
if _RANK[t] < _RANK[tier]:
tier = t
# 作者昵称和音乐标题都只是"谁发的"，不是"作品是什么"。
# 只在作品自己毫无信号时兜底成 author 档，绝不往上顶。
for where, text in (("author", author), ("music", music)):
wt, wsig = _scan(text, where)
if wt != "none":
signals.extend(wsig)
if tier == "none":
tier = "author"
return {"tier": tier, "signals": signals, "is_aigc": is_aigc}

343
douyin/app.py Normal file
View File

@@ -0,0 +1,343 @@
"""App 端接口客户端 —— 无签名、无 Cookie、无登录。
实测链路（每一步都验过）：
1. POST /service/2/device_register/ 引导调用，不需要任何签名
-> device_id / install_id
2. GET /aweme/v1/aweme/detail/ 带上真实 device_id/iid 即可
-> 完整 aweme_detail，含 video.play_addr.uri
3. GET /aweme/v1/play/?video_id=URI 免鉴权，302 到 douyinvod CDN
-> mp4
对比 web 端：web 的 /aweme/v1/web/aweme/detail/ 对匿名用户返回
{"name":"强制登录"}，且要算 a_bogus。App 端两样都不需要。
早先测 App 端拿到空 body，是因为 device_id/iid 是伪造的；换成注册得来的
真实凭据就正常了 —— 空 body 不代表要签名。
"""
from __future__ import annotations
import json
import random
import time
import uuid
from pathlib import Path
from typing import Any, Dict, Optional
from urllib.parse import urlencode
import requests
from .client import Aweme, DouyinClient, DouyinError
REGISTER_HOSTS = (
"https://log.snssdk.com/service/2/device_register/",
"https://api.amemv.com/service/2/device_register/",
"https://log-api.amemv.com/service/2/device_register/",
"https://toblog.ctobsnssdk.com/service/2/device_register/",
)
API_HOSTS = (
"https://aweme.snssdk.com",
"https://api.amemv.com",
)
APP_VERSION = "9.9.9"
VERSION_CODE = "990"
UPDATE_VERSION_CODE = "99013"
APP_ID = "1128"
CHANNEL = "douyin_official"
# 机型池。openudid/cdid 本来就每次随机，但 UA、机型、分辨率这些以前是写死的，
# 意味着我们所有注册请求在这些维度上完全一致。
#
# 注册受限时到底是按 IP 还是按"UA+机型"这类特征分桶，我们没验证过——本机没有
# 第二个出口 IP，做不了对照实验。把机型随机化是成本极低的两用手段：
# 如果是按机型特征分桶，它能缓解；如果是纯 IP，它无害。
#
# 每项是 (device_type, device_brand, os_version, os_api, resolution, dpi)。
DEVICE_POOL = (
("SM-G9750", "samsung", "12", "31", "1080*2340", "420"),
("SM-S9080", "samsung", "13", "33", "1440*3088", "560"),
("PDRM00", "OPPO", "12", "31", "1080*2400", "480"),
("V2118A", "vivo", "12", "31", "1080*2376", "480"),
("M2012K11AC", "Xiaomi", "13", "33", "1080*2400", "440"),
("ELS-AN00", "HUAWEI", "12", "31", "1176*2400", "480"),
("21091116AC", "Redmi", "12", "31", "1080*2400", "440"),
("NOH-AN01", "HONOR", "12", "31", "1344*2772", "480"),
)
# 兼容旧引用：默认取池子里第一项
DEVICE_TYPE, DEVICE_BRAND, OS_VERSION, OS_API = DEVICE_POOL[0][:4]
class DeviceRegisterError(DouyinError):
pass
class NotAVideo(DouyinError):
"""作品存在，但不是视频（图文/图集）。"""
def _hex(n: int) -> str:
return "".join(random.choice("0123456789abcdef") for _ in range(n))
class Device:
"""一台虚拟设备的身份。注册一次可长期复用，所以落盘缓存。"""
def __init__(self, device_id: str, install_id: str, openudid: str,
cdid: str, clientudid: str,
profile: Optional[tuple] = None) -> None:
self.device_id = device_id
self.install_id = install_id
self.openudid = openudid
self.cdid = cdid
self.clientudid = clientudid
# 机型档案必须跟着设备走：注册时报的是小米、后续 detail 请求的 UA
# 却是三星，这种自相矛盾本身就是个可疑特征。
self.profile = tuple(profile) if profile else DEVICE_POOL[0]
@property
def device_type(self) -> str:
return self.profile[0]
@property
def device_brand(self) -> str:
return self.profile[1]
@property
def os_version(self) -> str:
return self.profile[2]
@property
def os_api(self) -> str:
return self.profile[3]
@property
def resolution(self) -> str:
return self.profile[4]
@property
def dpi(self) -> str:
return self.profile[5]
@property
def user_agent(self) -> str:
return (f"com.ss.android.ugc.aweme/{UPDATE_VERSION_CODE} "
f"(Linux; U; Android {self.os_version}; zh_CN; {self.device_type}; "
f"Build/QP1A.190711.020; Cronet/TTNetVersion:6c7b9b1a 2022-11-29)")
def to_dict(self) -> Dict[str, Any]:
return {
"device_id": self.device_id, "install_id": self.install_id,
"openudid": self.openudid, "cdid": self.cdid,
"clientudid": self.clientudid, "profile": list(self.profile),
}
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> "Device":
# profile 是后加的，旧缓存没有这个字段，退回池子第一项
return cls(d["device_id"], d["install_id"], d["openudid"],
d["cdid"], d["clientudid"], d.get("profile"))
# ------------------------------------------------------------------ #
@classmethod
def register(cls, session: Optional[requests.Session] = None,
timeout: float = 20.0, retries: int = 5,
backoff: float = 30.0) -> "Device":
"""向 ByteDance 注册一台新设备。这一步不需要签名。
短时间内注册太多次，所有 host 都会返回 device_id='0'。实测：
指纹（openudid/cdid）每次都是新的照样被拒，4 个 host 同时被拒，
约 10 分钟后自行恢复 —— 所以是临时限流，且不是按设备指纹或按
单个 host 分桶的。
至于分桶的键到底是 IP 还是 UA/机型这类特征，我们没验证过：本机
没有第二个出口 IP，做不了对照实验。DEVICE_POOL 的随机化对两种
情况都无害，权当两用。
这里做指数退避重试而不是直接抛错，否则调用方会把后续所有条目
瞬间判为失败。
"""
last: Optional[Exception] = None
for attempt in range(retries):
try:
return cls._register_once(session, timeout)
except DeviceRegisterError as exc:
last = exc
if attempt < retries - 1:
wait = backoff * (2 ** attempt)
print(f" [注册受限] 等待 {wait:.0f}s 后重试 "
f"({attempt + 1}/{retries})", flush=True)
time.sleep(wait)
raise last # type: ignore[misc]
@classmethod
def _register_once(cls, session: Optional[requests.Session] = None,
timeout: float = 20.0) -> "Device":
session = session or requests.Session()
openudid = _hex(16)
cdid = str(uuid.UUID(int=random.getrandbits(128)))
clientudid = str(uuid.UUID(int=random.getrandbits(128)))
profile = random.choice(DEVICE_POOL)
dev_type, dev_brand, os_ver, os_api, resolution, dpi = profile
ua = (f"com.ss.android.ugc.aweme/{UPDATE_VERSION_CODE} "
f"(Linux; U; Android {os_ver}; zh_CN; {dev_type}; "
f"Build/QP1A.190711.020; Cronet/TTNetVersion:6c7b9b1a 2022-11-29)")
query = {
"ac": "wifi", "channel": CHANNEL, "aid": APP_ID, "app_name": "aweme",
"version_code": VERSION_CODE, "version_name": APP_VERSION,
"device_platform": "android", "os": "android", "ssmix": "a",
"device_type": dev_type, "device_brand": dev_brand,
"language": "zh", "os_api": os_api, "os_version": os_ver,
"manifest_version_code": VERSION_CODE, "resolution": resolution,
"dpi": dpi, "update_version_code": UPDATE_VERSION_CODE,
"_rticket": str(int(time.time() * 1000)),
"cdid": cdid, "openudid": openudid,
}
payload = {
"magic_tag": "ss_app_log",
"header": {
"display_name": "抖音",
"update_version_code": int(UPDATE_VERSION_CODE),
"manifest_version_code": int(VERSION_CODE), "aid": int(APP_ID),
"channel": CHANNEL, "package": "com.ss.android.ugc.aweme",
"app_version": APP_VERSION, "version_code": int(VERSION_CODE),
"sdk_version": "2.13.5", "os": "Android",
"os_version": os_ver, "os_api": int(os_api),
"device_model": dev_type, "device_brand": dev_brand,
"device_manufacturer": dev_brand, "cpu_abi": "arm64-v8a",
"density_dpi": int(dpi), "display_density": "mdpi",
"resolution": resolution.replace("*", "x"),
"language": "zh", "timezone": 8,
"access": "wifi", "not_request_sender": 0, "region": "CN",
"tz_name": "Asia/Shanghai", "tz_offset": 28800,
"sim_region": "cn", "openudid": openudid,
"clientudid": clientudid, "cdid": cdid,
"rom": "QP1A.190711.020", "rom_version": f"{dev_brand}/{os_ver}",
"device_id": 0, "install_id": 0,
},
"_gen_time": int(time.time() * 1000),
}
headers = {
"User-Agent": ua,
"Content-Type": "application/json; charset=utf-8",
"Accept-Encoding": "gzip",
"X-SS-REQ-TICKET": str(int(time.time() * 1000)),
}
errors = []
for host in REGISTER_HOSTS:
try:
r = session.post(host + "?" + urlencode(query),
data=json.dumps(payload), headers=headers,
timeout=timeout)
data = r.json()
except (requests.RequestException, ValueError) as exc:
errors.append(f"{host}: {type(exc).__name__}")
continue
did = str(data.get("device_id_str") or data.get("device_id") or "")
iid = str(data.get("install_id_str") or data.get("install_id") or "")
if did and did != "0":
return cls(did, iid, openudid, cdid, clientudid, profile)
errors.append(f"{host}: device_id={did!r}")
raise DeviceRegisterError("设备注册全部失败: " + "; ".join(errors))
@classmethod
def load_or_register(cls, cache: str | Path,
session: Optional[requests.Session] = None) -> "Device":
cache = Path(cache)
if cache.exists():
try:
return cls.from_dict(json.loads(cache.read_text(encoding="utf-8")))
except (ValueError, KeyError, OSError):
pass
dev = cls.register(session)
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text(json.dumps(dev.to_dict(), indent=2), encoding="utf-8")
return dev
class AppClient:
"""按 aweme_id 取详情。不需要签名、Cookie 或登录态。"""
def __init__(self, device: Optional[Device] = None,
cache: str | Path = ".device.json",
timeout: float = 25.0) -> None:
self.session = requests.Session()
self.device = device or Device.load_or_register(cache, self.session)
self.timeout = timeout
self.session.headers.update({
"User-Agent": self.device.user_agent,
"Accept-Encoding": "gzip",
"Connection": "keep-alive",
"sdk-version": "2",
})
def _params(self) -> Dict[str, str]:
return {
"device_platform": "android", "os": "android", "ssmix": "a",
"_rticket": str(int(time.time() * 1000)),
"cdid": self.device.cdid, "channel": CHANNEL, "aid": APP_ID,
"app_name": "aweme", "version_code": VERSION_CODE,
"version_name": APP_VERSION, "manifest_version_code": VERSION_CODE,
"update_version_code": UPDATE_VERSION_CODE,
"resolution": self.device.resolution, "dpi": self.device.dpi,
"device_type": self.device.device_type,
"device_brand": self.device.device_brand,
"language": "zh", "os_api": self.device.os_api,
"os_version": self.device.os_version,
"openudid": self.device.openudid,
"device_id": self.device.device_id, "iid": self.device.install_id,
"ac": "wifi", "ts": str(int(time.time())),
}
def fetch_detail(self, aweme_id: str, retries: int = 3) -> Aweme:
last = ""
for attempt in range(retries):
host = API_HOSTS[attempt % len(API_HOSTS)]
params = self._params()
params["aweme_id"] = aweme_id
url = f"{host}/aweme/v1/aweme/detail/?" + urlencode(params)
try:
r = self.session.get(url, timeout=self.timeout)
except requests.RequestException as exc:
last = f"{type(exc).__name__}: {exc}"
continue
if not r.content:
last = f"HTTP {r.status_code} 空 body"
time.sleep(1 + attempt)
continue
try:
data: Dict[str, Any] = r.json()
except ValueError:
last = f"非 JSON: {r.text[:120]!r}"
continue
detail = data.get("aweme_detail")
if not detail:
# 服务端会在 filter_detail 里说明为什么不给正片，
# 例如图文作品 filter_reason=images_base
f = data.get("filter_detail") or {}
reason = f.get("filter_reason") or ""
if reason == "images_base":
raise NotAVideo(f"{aweme_id} 是图文作品（图集），不是视频")
if reason:
raise DouyinError(
f"{aweme_id} 被服务端过滤：{reason}"
+ (f"（{f.get('notice')}）" if f.get("notice") else "")
)
raise DouyinError(
f"无 aweme_detail，status_code={data.get('status_code')}，"
"作品可能已删除或为私密"
)
return DouyinClient.parse(detail)
raise DouyinError(f"App detail 连续 {retries} 次失败（{last}）")

71
douyin/audio.py Normal file
View File

@@ -0,0 +1,71 @@
"""ffmpeg / ffprobe 的薄封装：找可执行文件、拼抽音命令、探时长。
批量抽取的调度不在这里 —— `pipeline.py` 自己按条编排（下载→抽音→上传→删），
所以这个模块只提供单次调用要用的零件。
从 mp4 里提取音轨。
源音频是 AAC 44.1kHz 立体声 128kbps，据此有两种模式：
mp3 转码成 MP3。有损转有损，必然掉质量。码率给太高只是浪费空间
（源就 128k），默认 192k 是留余量避免二次编码劣化，不是"更清晰"。
copy 直接把 AAC 流拷进 .m4a 容器。无损、秒完、体积更小。
只有在必须是 .mp3 后缀时才用 mp3 模式。
依赖 ffmpeg，需在 PATH 里。
"""
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
from typing import List, Optional
class FFmpegMissing(RuntimeError):
pass
def ensure_ffmpeg(exe: str = "ffmpeg") -> str:
path = shutil.which(exe)
if not path:
raise FFmpegMissing(
f"找不到 {exe}。装一个并加进 PATH，或用 --ffmpeg 指定完整路径。")
return path
def build_cmd(ffmpeg: str, src: Path, dst: Path, mode: str, bitrate: str) -> List[str]:
cmd = [ffmpeg, "-hide_banner", "-loglevel", "error", "-y", "-i", str(src), "-vn"]
if mode == "copy":
# 不重新编码，直接搬运 AAC 流
cmd += ["-acodec", "copy"]
else:
cmd += ["-acodec", "libmp3lame", "-b:a", bitrate]
cmd.append(str(dst))
return cmd
def probe(path, ffprobe: str = "ffprobe", timeout: float = 60.0):
"""探一个媒体文件的容器时长，探不出来返回 None。
时长取自 mp4（源），不取 mp3 —— mp3 是转码产物，虽然实测两者一致，
但源文件才是权威。
注意这里读的是容器声明的 format=duration，对 mp4 没问题，
但**不能拿去判断 mp3 有没有被截断**：LAME 写的 Xing 头声明的是原始
总帧数，ffprobe 直接信它而不解码，实测把 mp3 截到 20% 它报的还是原时长。
那种场景要用 pipeline.decoded_seconds()（-count_frames 真解码）。
"""
cmd = [ffprobe, "-v", "error", "-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1", str(path)]
try:
p_ = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
except subprocess.TimeoutExpired:
return None
try:
d = float((p_.stdout or "").strip())
except ValueError:
return None
return d if d > 0 else None

197
douyin/batch.py Normal file
View File

@@ -0,0 +1,197 @@
"""批量下载：并发、可断点续跑、单条失败不影响整批。
设计要点：
- 设备身份只注册一次，各 worker 共用；但每个 worker 有独立 Session
（requests.Session 并发下不保证安全）
- 落盘文件名就是 aweme_id，所以「是否已下载」直接看文件在不在，
不依赖去重记录 —— 记录丢了也能无损续跑
- 失败条目写 failed.txt，跑完可直接拿它当输入重试
"""
from __future__ import annotations
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Sequence
import requests
from .app import AppClient, Device, NotAVideo
from .client import DouyinError
from .download import Registry, download
class DevicePool:
"""设备身份池。
实测单个 device_id 在约 575 次 detail 调用后被限流，表现是接口一直
回 HTTP 200 空 body；同一 IP 下换个新注册的设备立刻恢复 —— 说明配额
是设备级而非 IP 级。注册本身免签名、成本极低，所以按量轮换即可。
用 generation 标记当前设备：worker 拿到的 gen 和池里的对得上才允许
触发轮换，避免多个 worker 同时发现失败、把设备连换好几次。
"""
def __init__(self, max_calls: int = 400, initial: Optional[Device] = None) -> None:
self._lock = threading.Lock()
self._device = initial
self._gen = 0
self._calls = 0
self.max_calls = max_calls
self.rotations = 0
def current(self) -> tuple[Device, int]:
with self._lock:
if self._device is None or self._calls >= self.max_calls:
self._rotate_locked("配额用尽")
self._calls += 1
return self._device, self._gen
def report_dead(self, gen: int) -> None:
"""worker 发现当前设备被限流时调用。"""
with self._lock:
if gen == self._gen:
self._rotate_locked("被限流")
def _rotate_locked(self, why: str) -> None:
self._device = Device.register()
self._gen += 1
self._calls = 0
self.rotations += 1
print(f" [设备轮换#{self.rotations}] {why} -> "
f"device_id={self._device.device_id}", flush=True)
@dataclass
class BatchStats:
total: int = 0
done: int = 0
skipped: int = 0
not_video: int = 0
failed: int = 0
bytes_: int = 0
lock: threading.Lock = field(default_factory=threading.Lock)
@property
def finished(self) -> int:
return self.done + self.skipped + self.not_video + self.failed
def run_batch(aweme_ids: Sequence[str], out_dir: Path, device: Optional[Device] = None,
workers: int = 5, delay: float = 0.3,
registry: Optional[Registry] = None,
log_every: int = 25, max_calls_per_device: int = 400) -> BatchStats:
out_dir.mkdir(parents=True, exist_ok=True)
registry = registry or Registry(out_dir / ".downloaded.json")
pool = DevicePool(max_calls=max_calls_per_device, initial=device)
stats = BatchStats(total=len(aweme_ids))
failed_path = out_dir / "failed.txt"
# 每轮重来，否则上一轮已修复的失败会一直留在文件里造成误导
failed_path.write_text("", encoding="utf-8")
failed_lock = threading.Lock()
local = threading.local()
t0 = time.time()
def client_for(device_: Device, gen: int) -> AppClient:
"""每线程每代设备一个 Session（Session 并发不安全）。"""
if getattr(local, "gen", None) != gen:
local.client = AppClient(device=device_)
local.gen = gen
return local.client
def dl_session() -> requests.Session:
"""下载用的 Session：CDN 免鉴权，不该占设备配额，也不随设备轮换重建。"""
if not hasattr(local, "dl"):
local.dl = requests.Session()
local.dl.headers["User-Agent"] = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
)
return local.dl
def record_failure(aweme_id: str, why: str) -> None:
with failed_lock:
with open(failed_path, "a", encoding="utf-8") as f:
f.write(f"{aweme_id}\t{why}\n")
def fetch_with_rotation(aweme_id: str):
"""限流表现为空 body。碰到就换设备再试一次，而不是直接判失败。"""
last: Optional[Exception] = None
for _ in range(2):
dev, gen = pool.current()
client = client_for(dev, gen)
if delay:
time.sleep(delay)
try:
return client.fetch_detail(aweme_id, retries=2)
except NotAVideo:
raise
except DouyinError as exc:
last = exc
if "空 body" in str(exc):
pool.report_dead(gen)
continue
raise
raise last # type: ignore[misc]
def handle(aweme_id: str) -> None:
dest = out_dir / f"{aweme_id}.mp4"
# 文件在就算下过，不必碰网络也不依赖记录
if dest.exists() and dest.stat().st_size > 0:
with stats.lock:
stats.skipped += 1
return
try:
aweme = fetch_with_rotation(aweme_id)
except NotAVideo as exc:
with stats.lock:
stats.not_video += 1
record_failure(aweme_id, f"not_video: {exc}")
return
except Exception as exc:
with stats.lock:
stats.failed += 1
record_failure(aweme_id, f"detail: {type(exc).__name__}: {exc}")
return
if not aweme.video_urls:
with stats.lock:
stats.not_video += 1
record_failure(aweme_id, "no_video_url")
return
try:
result = download(dl_session(), aweme.video_urls, dest, progress=False)
except Exception as exc:
with stats.lock:
stats.failed += 1
record_failure(aweme_id, f"download: {type(exc).__name__}: {exc}")
return
registry.add(aweme_id, result.path, result.size,
desc=aweme.desc, author=aweme.author, uri=aweme.uri)
with stats.lock:
stats.done += 1
stats.bytes_ += result.size
with ThreadPoolExecutor(max_workers=workers) as executor:
futures = [executor.submit(handle, a) for a in aweme_ids]
for n, _ in enumerate(as_completed(futures), 1):
if n % log_every == 0 or n == len(futures):
el = time.time() - t0
rate = n / el if el else 0
eta = (len(futures) - n) / rate if rate else 0
print(f"[{n}/{len(futures)}] 成功{stats.done} 跳过{stats.skipped} "
f"图集{stats.not_video} 失败{stats.failed} "
f"| {stats.bytes_ / 1024 ** 3:.2f}GB "
f"| {rate * 60:.0f}条/分 | 换设备{pool.rotations}次 "
f"| 剩约 {eta / 60:.0f}分",
flush=True)
registry.flush()
return stats

289
douyin/client.py Normal file
View File

@@ -0,0 +1,289 @@
"""抖音 Web 接口客户端。
实测要点（都是打真实接口验出来的）：
1. 空 body 有两种成因，必须分开看：
- 新 session 的第一个请求必然空，第二个才通 —— cookie 预热，见 _warm_up()
- 服务端主动拦截 —— 响应头 X-Whale-Throughput-Abort-Data 里有 base64 原因
2. /aweme/detail/ 对匿名用户返回 abort={"name":"强制登录"}，必须带登录 Cookie。
3. /aweme/related/ 匿名可用，返回完整 play_addr，但不含目标视频自身。
4. 请求头 UA 必须与签名时的 UA 完全一致，params 里的 browser_version 也要跟着对齐。
"""
from __future__ import annotations
import base64
import json
import random
import string
import time
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from urllib.parse import urlencode
import requests
from .signer import Signer
DETAIL_URL = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
RELATED_URL = "https://www.douyin.com/aweme/v1/web/aweme/related/"
TTWID_URL = "https://ttwid.bytedance.com/ttwid/union/register/"
BASE_PARAMS: Dict[str, str] = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"pc_client_type": "1",
"version_code": "190500",
"version_name": "19.5.0",
"cookie_enabled": "true",
"screen_width": "1920",
"screen_height": "1080",
"browser_language": "zh-CN",
"browser_platform": "Win32",
"browser_name": "Chrome",
"browser_version": "126.0.0.0",
"browser_online": "true",
"engine_name": "Blink",
"engine_version": "126.0.0.0",
"os_name": "Windows",
"os_version": "10",
"cpu_core_num": "8",
"device_memory": "8",
"platform": "PC",
"downlink": "10",
"effective_type": "4g",
"round_trip_time": "50",
}
PLAY_URL = "https://www.douyin.com/aweme/v1/play/"
def play_url_from_uri(uri: str, ratio: str = "1080p", watermark: bool = False) -> str:
"""由 video URI 拼出播放直链。
实测这个接口完全免鉴权：无 a_bogus、无 Cookie、无登录，裸 requests 即可，
302 跳到 douyinvod.com 的 CDN 地址。所以只要拿到了 URI，后续下载不再需要
任何身份——URI 值得缓存复用。
"""
path = "playwm" if watermark else "play"
return f"https://www.douyin.com/aweme/v1/{path}/?video_id={uri}&ratio={ratio}&line=0"
class DouyinError(RuntimeError):
pass
class LoginRequired(DouyinError):
"""服务端明确要求登录。"""
@dataclass
class Aweme:
aweme_id: str
desc: str
author: str
create_time: int
video_urls: List[str]
cover_urls: List[str]
music_url: Optional[str]
uri: str
raw: Dict[str, Any]
@property
def is_video(self) -> bool:
return bool(self.video_urls or self.uri)
def play_url(self, ratio: str = "1080p") -> str:
"""免鉴权直链。play_addr 里的地址带时效，URI 换出来的是现算的。"""
if not self.uri:
raise DouyinError("该作品没有 video URI")
return play_url_from_uri(self.uri, ratio)
def _rand(n: int, pool: str) -> str:
return "".join(random.choice(pool) for _ in range(n))
def gen_ms_token(length: int = 107) -> str:
return _rand(length, string.ascii_letters + string.digits + "=-_")
def gen_s_v_web_id() -> str:
body = "".join(
"_" if i in (8, 13, 18, 23) else random.choice("0123456789abcdef")
for i in range(32)
)
return f"verify_{format(int(time.time()), 'x')}_{body}"
def decode_abort(resp: requests.Response) -> str:
"""解出服务端拦截原因；没有则返回空串。"""
raw = resp.headers.get("X-Whale-Throughput-Abort-Data")
if not raw:
return ""
try:
info = json.loads(base64.b64decode(raw))
return info.get("name") or info.get("content") or ""
except (ValueError, TypeError):
return "unknown"
class DouyinClient:
def __init__(self, signer: Signer, cookie: str = "", timeout: float = 25.0,
warm_up: bool = True) -> None:
self.signer = signer
self.timeout = timeout
self.user_agent = signer.user_agent
self.fp = gen_s_v_web_id()
# params 里的浏览器版本必须跟签名用的 UA 对上，否则服务端会判不一致
self.params = dict(BASE_PARAMS)
if "Chrome/" in self.user_agent:
ver = self.user_agent.split("Chrome/")[1].split(" ")[0]
self.params["browser_version"] = ver
self.params["engine_version"] = ver
self.session = requests.Session()
self.session.headers.update({
"User-Agent": self.user_agent,
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9",
"Referer": "https://www.douyin.com/",
})
self._logged_in = bool(cookie)
if cookie:
self.session.headers["Cookie"] = cookie
else:
self._bootstrap_cookies()
self._warmed = False
if warm_up:
self._warm_up()
# ------------------------------------------------------------------ #
def _bootstrap_cookies(self) -> None:
try:
self.session.post(TTWID_URL, timeout=self.timeout, json={
"region": "cn", "aid": 1768, "needFid": False,
"service": "www.ixigua.com",
"migrate_info": {"ticket": "", "source": "node"},
"cbUrlProtocol": "https", "union": True,
})
except requests.RequestException:
pass
self.session.cookies.set("s_v_web_id", self.fp, domain=".douyin.com")
self.session.cookies.set("odin_tt", _rand(160, "0123456789abcdef"),
domain=".douyin.com")
def _warm_up(self) -> None:
"""新 session 的第一个接口请求必然返回空，先打一发把它消耗掉。"""
if self._warmed:
return
try:
self._get(RELATED_URL, {"aweme_id": "1", "count": "1", "filterGids": "1"})
except (DouyinError, requests.RequestException):
pass
self._warmed = True
def _build_url(self, base: str, extra: Dict[str, str]) -> str:
p = dict(self.params)
p.update(extra)
p["msToken"] = gen_ms_token()
return f"{base}?{urlencode(p)}"
def _get(self, base: str, extra: Dict[str, str],
retries: int = 3) -> Dict[str, Any]:
"""签名 + 请求 + 解析。
「空 body 且无拦截原因」是可重试的瞬时态（session 还没热起来），
「空 body 且原因含登录」是硬门槛，立即抛错不重试。
"""
last = ""
for attempt in range(1, retries + 1):
url = self.signer.sign_url(self._build_url(base, extra), self.fp)
resp = self.session.get(url, timeout=self.timeout)
if resp.content:
try:
return resp.json()
except ValueError as exc:
raise DouyinError(f"响应不是 JSON: {resp.text[:200]!r}") from exc
reason = decode_abort(resp)
if "登录" in reason:
raise LoginRequired(
f"接口要求登录（服务端返回 {reason!r}）。"
"请用 --cookie 传入已登录的抖音 Cookie。"
)
last = (f"HTTP {resp.status_code}"
+ (f"，拦截原因 {reason!r}" if reason else "，无拦截原因"))
if attempt < retries:
time.sleep(2 * attempt)
raise DouyinError(f"接口连续 {retries} 次返回空 body（{last}）")
# ------------------------------------------------------------------ #
def fetch_detail(self, aweme_id: str) -> Aweme:
"""按 aweme_id 取作品详情。匿名调用会抛 LoginRequired。"""
self.session.headers["Referer"] = f"https://www.douyin.com/video/{aweme_id}"
data = self._get(DETAIL_URL, {"aweme_id": aweme_id})
detail = data.get("aweme_detail")
if not detail:
raise DouyinError(
f"响应无 aweme_detail，status_code={data.get('status_code')}，"
"作品可能已删除或为私密"
)
return self.parse(detail)
def fetch_related(self, aweme_id: str, count: int = 10) -> List[Aweme]:
"""相关推荐。匿名可用，但不含 aweme_id 自身。"""
data = self._get(RELATED_URL, {
"aweme_id": aweme_id, "count": str(count), "filterGids": aweme_id,
})
return [self.parse(a) for a in (data.get("aweme_list") or [])]
# ------------------------------------------------------------------ #
@staticmethod
def parse(detail: Dict[str, Any]) -> Aweme:
video = detail.get("video") or {}
# play_addr 带水印，playwm -> play 即无水印源。
# bit_rate 里有多档清晰度，优先挑码率最高的。
urls: List[str] = []
gears = sorted(
(b for b in (video.get("bit_rate") or []) if b.get("play_addr")),
key=lambda b: b.get("bit_rate") or 0, reverse=True,
)
for gear in gears:
urls.extend((gear["play_addr"].get("url_list") or []))
urls.extend((video.get("play_addr") or {}).get("url_list") or [])
seen: set[str] = set()
video_urls = []
for u in urls:
u = u.replace("playwm", "play")
if u not in seen:
seen.add(u)
video_urls.append(u)
cover = video.get("origin_cover") or video.get("cover") or {}
music_list = ((detail.get("music") or {}).get("play_url") or {}).get("url_list") or []
return Aweme(
aweme_id=str(detail.get("aweme_id", "")),
desc=(detail.get("desc") or "").strip(),
author=((detail.get("author") or {}).get("nickname") or "").strip(),
create_time=int(detail.get("create_time") or 0),
video_urls=video_urls,
cover_urls=list(cover.get("url_list") or []),
music_url=music_list[0] if music_list else None,
uri=(video.get("play_addr") or {}).get("uri") or "",
raw=detail,
)

94
douyin/cos.py Normal file
View File

@@ -0,0 +1,94 @@
"""腾讯云 COS 上传，带上传后校验。
流水线传完就删本地文件，所以「传成功了吗」这个判断必须硬。
`upload_file` 不抛异常只说明请求没报错，不说明对象真的落到桶里、
大小也对。所以 put() 一律在上传后 head_object 取回 Content-Length
跟本地字节数比对，对不上就当失败——删除闸门卡在这个返回值上。
"""
from __future__ import annotations
import os
import threading
from pathlib import Path
from typing import Optional
DEFAULT_BUCKET = "asr-dig-data-1253181912"
DEFAULT_PREFIX = "0005/zh/"
class CosError(RuntimeError):
pass
class SizeMismatch(CosError):
"""对象传上去了，但远端大小和本地对不上。"""
def object_url(key: str, bucket: str, region: str, style: str = "cos") -> str:
if style == "cos":
return f"cos://{bucket}/{key}"
if style == "https":
return f"https://{bucket}.cos.{region}.myqcloud.com/{key}"
return f"/{bucket}/{key}"
class CosUploader:
"""线程安全的上传器。同名对象一律覆盖，不做存在性检查。
重复劳动由上层的 records.jsonl 断点续跑拦掉（已完成的 id 整条跳过，
连下载都不会发生），所以这里不需要再靠 object_exists 去省事——
少一个分支，行为也更好预测。
"""
def __init__(self, region: str, bucket: str = DEFAULT_BUCKET,
prefix: str = DEFAULT_PREFIX,
secret_id: str = "", secret_key: str = "",
url_style: str = "cos", scheme: str = "https") -> None:
secret_id = secret_id or os.environ.get("COS_SECRET_ID", "")
secret_key = secret_key or os.environ.get("COS_SECRET_KEY", "")
if not secret_id or not secret_key:
raise CosError(
"缺 COS 凭证。设环境变量 COS_SECRET_ID / COS_SECRET_KEY，"
"别写在命令行里（会留在 shell history）。")
from qcloud_cos import CosConfig, CosS3Client
self.bucket = bucket
self.region = region
self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else ""
self.url_style = url_style
self._client = CosS3Client(CosConfig(
Region=region, SecretId=secret_id, SecretKey=secret_key, Scheme=scheme))
self._lock = threading.Lock()
def key_for(self, name: str) -> str:
return f"{self.prefix}{name}"
def url_for(self, key: str) -> str:
return object_url(key, self.bucket, self.region, self.url_style)
def remote_size(self, key: str) -> Optional[int]:
try:
return int(self._client.head_object(Bucket=self.bucket, Key=key)["Content-Length"])
except Exception:
return None
def put(self, local: Path, key: str) -> tuple[str, int]:
"""上传并校验。返回 (url, 字节数)。校验不过抛 SizeMismatch。"""
local = Path(local)
size = local.stat().st_size
if size == 0:
raise CosError(f"{local.name} 是空文件，拒绝上传")
# EnableMD5 关掉是为了速度；正确性靠上传后的 head_object 校验兜底
self._client.upload_file(
Bucket=self.bucket, Key=key, LocalFilePath=str(local),
EnableMD5=False, PartSize=10, MAXThread=2,
)
remote = self.remote_size(key)
if remote is None:
raise SizeMismatch(f"{key} 上传后 head_object 拿不到，判定失败")
if remote != size:
raise SizeMismatch(f"{key} 大小不符：COS={remote} 本地={size}")
return self.url_for(key), size

179
douyin/crypto.py Normal file
View File

@@ -0,0 +1,179 @@
"""a_bogus 依赖的底层密码学原语：SM3、RC4、自定义 Base64。
这三样都是公开算法，与抖音无关，可以独立验证正确性：
- SM3 见 GB/T 32905-2016，本文件底部带官方测试向量自检
- RC4 标准流密码
- 自定义 Base64 只是把标准码表换成抖音自己那套字符顺序
"""
from __future__ import annotations
from typing import Iterable, List, Sequence
# --------------------------------------------------------------------------- #
# SM3
# --------------------------------------------------------------------------- #
_IV = [
0x7380166F, 0x4914B2B9, 0x172442D7, 0xDA8A0600,
0xA96F30BC, 0x163138AA, 0xE38DEE4D, 0xB0FB0E4E,
]
_MASK = 0xFFFFFFFF
def _rotl(x: int, n: int) -> int:
n &= 31
return ((x << n) | (x >> (32 - n))) & _MASK
def _ff(j: int, x: int, y: int, z: int) -> int:
return (x ^ y ^ z) if j < 16 else ((x & y) | (x & z) | (y & z))
def _gg(j: int, x: int, y: int, z: int) -> int:
return (x ^ y ^ z) if j < 16 else ((x & y) | ((~x & _MASK) & z))
def _p0(x: int) -> int:
return x ^ _rotl(x, 9) ^ _rotl(x, 17)
def _p1(x: int) -> int:
return x ^ _rotl(x, 15) ^ _rotl(x, 23)
def _cf(v: Sequence[int], block: bytes) -> List[int]:
"""压缩函数 CF(V, B)。"""
w = [int.from_bytes(block[i * 4:i * 4 + 4], "big") for i in range(16)]
for j in range(16, 68):
w.append(
_p1(w[j - 16] ^ w[j - 9] ^ _rotl(w[j - 3], 15))
^ _rotl(w[j - 13], 7)
^ w[j - 6]
)
w1 = [w[j] ^ w[j + 4] for j in range(64)]
a, b, c, d, e, f, g, h = v
for j in range(64):
t = 0x79CC4519 if j < 16 else 0x7A879D8A
ss1 = _rotl((_rotl(a, 12) + e + _rotl(t, j)) & _MASK, 7)
ss2 = ss1 ^ _rotl(a, 12)
tt1 = (_ff(j, a, b, c) + d + ss2 + w1[j]) & _MASK
tt2 = (_gg(j, e, f, g) + h + ss1 + w[j]) & _MASK
d = c
c = _rotl(b, 9)
b = a
a = tt1
h = g
g = _rotl(f, 19)
f = e
e = _p0(tt2)
return [x ^ y for x, y in zip(v, (a, b, c, d, e, f, g, h))]
def sm3(data: bytes) -> bytes:
"""SM3 摘要，返回 32 字节。"""
bit_len = len(data) * 8
padded = data + b"\x80"
padded += b"\x00" * ((56 - len(padded) % 64) % 64)
padded += bit_len.to_bytes(8, "big")
v = list(_IV)
for i in range(0, len(padded), 64):
v = _cf(v, padded[i:i + 64])
return b"".join(x.to_bytes(4, "big") for x in v)
def sm3_to_list(data: bytes | Iterable[int]) -> List[int]:
"""SM3 摘要，返回 32 个 int（a_bogus 内部按字节数组传递）。"""
if not isinstance(data, (bytes, bytearray)):
data = bytes(bytearray(data))
return list(sm3(bytes(data)))
# --------------------------------------------------------------------------- #
# RC4
# --------------------------------------------------------------------------- #
def rc4(key: bytes, data: bytes) -> bytes:
"""标准 RC4。加解密同一函数。"""
s = list(range(256))
j = 0
klen = len(key)
for i in range(256):
j = (j + s[i] + key[i % klen]) & 0xFF
s[i], s[j] = s[j], s[i]
out = bytearray(len(data))
i = j = 0
for n, ch in enumerate(data):
i = (i + 1) & 0xFF
j = (j + s[i]) & 0xFF
s[i], s[j] = s[j], s[i]
out[n] = ch ^ s[(s[i] + s[j]) & 0xFF]
return bytes(out)
# --------------------------------------------------------------------------- #
# 自定义码表 Base64
# --------------------------------------------------------------------------- #
STD_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
def b64_encode(data: bytes, alphabet: str, pad: bool = True) -> str:
"""按给定 64(+1) 字符码表做 Base64 编码。
alphabet 长度为 64 时不补位；为 65 时最后一位作为填充字符。
"""
if len(alphabet) not in (64, 65):
raise ValueError(f"码表长度必须是 64 或 65，收到 {len(alphabet)}")
fill = alphabet[64] if len(alphabet) == 65 else ""
out: List[str] = []
for i in range(0, len(data), 3):
chunk = data[i:i + 3]
n = int.from_bytes(chunk + b"\x00" * (3 - len(chunk)), "big")
quad = [alphabet[(n >> shift) & 0x3F] for shift in (18, 12, 6, 0)]
keep = len(chunk) + 1 # 1字节->2字符, 2字节->3字符, 3字节->4字符
out.extend(quad[:keep])
if pad and fill:
out.extend(fill * (4 - keep))
return "".join(out)
# --------------------------------------------------------------------------- #
# 自检
# --------------------------------------------------------------------------- #
# GB/T 32905-2016 附录 A 给出的两组标准测试向量
_SM3_VECTORS = [
(b"abc",
"66c7f0f462eeedd9d1f2d46bdc10e4e24167c4875cf2f7a2297da02b8f4ba8e0"),
(b"abcd" * 16,
"debe9ff92275b8a138604889c18e5a4d6fdb70e5387e5765293dcba39c0c5732"),
]
def self_test() -> None:
"""跑通即说明 SM3/RC4/Base64 实现无误。"""
for msg, want in _SM3_VECTORS:
got = sm3(msg).hex()
assert got == want, f"SM3({msg[:8]!r}...) = {got}, 期望 {want}"
# RC4 官方测试向量：key="Key", plaintext="Plaintext" -> BBF316E8D940AF0AD3
assert rc4(b"Key", b"Plaintext").hex().upper() == "BBF316E8D940AF0AD3"
# 用标准码表时，自定义 Base64 应与 base64 模块完全一致
import base64
import os
for n in range(1, 40):
raw = os.urandom(n)
assert b64_encode(raw, STD_ALPHABET) == base64.b64encode(raw).decode()
print("crypto self-test OK: SM3 / RC4 / custom-base64 均通过")
if __name__ == "__main__":
self_test()

174
douyin/download.py Normal file
View File

@@ -0,0 +1,174 @@
"""断点续传下载 + 已下载去重。"""
from __future__ import annotations
import json
import os
import re
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, Optional
import requests
CHUNK = 256 * 1024
# Windows 文件名非法字符，外加控制字符
_ILLEGAL = re.compile(r'[\\/:*?"<>|\x00-\x1f]')
def safe_filename(name: str, max_len: int = 80) -> str:
name = _ILLEGAL.sub("_", name).strip(" .")
name = re.sub(r"\s+", " ", name)
if len(name) > max_len:
name = name[:max_len].rstrip()
return name or "untitled"
class Registry:
"""记录已下载的 aweme_id，避免重复下载。
批量场景下会被多线程写，所以加锁；并且不再每条都落盘（5000 条时
每次全量 dump 是 O(n^2) 的 IO），改成攒够 save_every 条再写，
结束时用 flush() 收尾。文件名本身就是 aweme_id，记录丢了也能靠
目录重建，所以这个折中是安全的。
"""
def __init__(self, path: Path, save_every: int = 1) -> None:
self.path = path
self.save_every = save_every
self._lock = threading.Lock()
self._pending = 0
self._data: Dict[str, dict] = {}
if path.exists():
try:
self._data = json.loads(path.read_text(encoding="utf-8"))
except (ValueError, OSError):
self._data = {}
def __contains__(self, aweme_id: str) -> bool:
entry = self._data.get(aweme_id)
if not entry:
return False
# 记录在但文件被手动删了，就当没下过
return Path(entry.get("file", "")).exists()
def get(self, aweme_id: str) -> Optional[dict]:
return self._data.get(aweme_id)
def add(self, aweme_id: str, file: Path, size: int, desc: str = "",
author: str = "", uri: str = "") -> None:
"""记一条。文件名只是 aweme_id，作者/标题这些就全靠这里留存。"""
with self._lock:
self._data[aweme_id] = {
"file": str(file),
"size": size,
"author": author,
"desc": desc,
"uri": uri,
"time": int(time.time()),
}
self._pending += 1
if self._pending >= self.save_every:
self._save_locked()
def flush(self) -> None:
"""强制落盘。批量结束时调用。"""
with self._lock:
if self._pending:
self._save_locked()
def save(self) -> None:
with self._lock:
self._save_locked()
def _save_locked(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
tmp = self.path.with_suffix(self.path.suffix + ".tmp")
tmp.write_text(json.dumps(self._data, ensure_ascii=False, indent=2),
encoding="utf-8")
tmp.replace(self.path)
self._pending = 0
@dataclass
class DownloadResult:
path: Path
size: int
resumed: bool
skipped: bool = False
def _human(n: float) -> str:
for unit in ("B", "KB", "MB", "GB"):
if n < 1024:
return f"{n:.1f}{unit}"
n /= 1024
return f"{n:.1f}TB"
def download(session: requests.Session, urls: Iterable[str], dest: Path,
timeout: float = 30.0, retries: int = 4,
progress: bool = True) -> DownloadResult:
"""下载到 dest，支持 Range 续传；urls 为多个备用直链，逐个尝试。
抖音的视频直链会 302 到 CDN，且校验 Referer/UA，所以必须带上原 session 的头。
"""
dest.parent.mkdir(parents=True, exist_ok=True)
part = dest.with_suffix(dest.suffix + ".part")
if dest.exists() and dest.stat().st_size > 0:
return DownloadResult(dest, dest.stat().st_size, resumed=False, skipped=True)
url_list = [u for u in urls if u]
if not url_list:
raise ValueError("没有可用的下载直链")
last_err: Optional[Exception] = None
for url in url_list:
for attempt in range(1, retries + 1):
done = part.stat().st_size if part.exists() else 0
headers = {"Range": f"bytes={done}-"} if done else {}
try:
with session.get(url, headers=headers, stream=True,
timeout=timeout, allow_redirects=True) as r:
if done and r.status_code == 200:
# 服务端不认 Range，只能从头来
done = 0
part.unlink(missing_ok=True)
elif done and r.status_code != 206:
r.raise_for_status()
else:
r.raise_for_status()
total = int(r.headers.get("Content-Length") or 0) + done
mode = "ab" if done else "wb"
with open(part, mode) as f:
for chunk in r.iter_content(CHUNK):
if not chunk:
continue
f.write(chunk)
done += len(chunk)
if progress:
pct = f"{done / total * 100:5.1f}%" if total else " ? "
print(f"\r {pct} {_human(done)}"
+ (f" / {_human(total)}" if total else ""),
end="", flush=True)
if progress:
print()
if total and done < total:
raise IOError(f"下载不完整: {done}/{total}")
part.replace(dest)
return DownloadResult(dest, done, resumed=bool(headers))
except (requests.RequestException, IOError) as exc:
last_err = exc
if progress:
print(f"\r 第 {attempt}/{retries} 次失败: {exc}")
time.sleep(min(2 ** attempt, 8))
raise RuntimeError(f"所有直链均下载失败，最后一个错误: {last_err}")

252
douyin/meta.py Normal file
View File

@@ -0,0 +1,252 @@
"""元信息：标题/标签拆分，以及 records.jsonl 的追加写。
records.jsonl 是流水线唯一的落点，一条视频一行，raw 也内嵌在里面，
所以单行可能几十 KB、整个文件几百 MB。由此有两条硬性约定：
1. 读取一律逐行流式，任何地方都不许把整个文件读进内存。
2. `aweme_id` 必须是每行的第一个字段。续跑扫描靠 _ID_RE 从行首直接
抠 id，不解析整行 JSON —— 400 MB 的差别是 ~1 秒 对 十几秒。
重排字段不会报错，只会让启动悄悄变慢，所以这是契约不是排版。
"""
from __future__ import annotations
import json
import re
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Tuple
# 行首抠 id，不解析整行
_ID_RE = re.compile(r'^\{"aweme_id":\s*"(\d+)"')
# 一个标签：# 后面跟到下一个空白或下一个 # 为止。
# 抖音有 "#四川话#方言" 这种连写，所以 # 也要作为终止符。
_TAG_RE = re.compile(r"#([^\s#]+)")
# 标签两端常粘着括号、引号、逗号之类的标点，剥掉
_EDGE_PUNCT = "（）()[]【】{}「」<>《》\"'“”‘’,，.。!！?？:：;；、~～-—_/\\|"
def _clean_tag(name: str) -> str:
"""归一化一个标签候选，判定不是标签就返回空串。"""
name = (name or "").strip().strip(_EDGE_PUNCT).strip()
# 带 @ 的是提及，不是标签。实测 desc 里有作者手打的
# "#(提示词出自@青木"，抖音自己也没把它收进 text_extra。
if not name or "@" in name:
return ""
return name
def _dedupe(items) -> List[str]:
"""按大小写不敏感去重，保留首次出现的写法。
抖音把 text_extra 里的标签统一转成了小写，desc 里却是作者的原始大小写。
实测同一条视频拿到 ['一站式AI创作工作台', ..., '一站式ai创作工作台', ...]，
同一个标签存了两份。desc 排在前面，所以保留的是原始大小写。
"""
seen, out = set(), []
for x in items:
x = _clean_tag(x)
if not x:
continue
k = x.casefold()
if k not in seen:
seen.add(k)
out.append(x)
return out
def split_desc(desc: Optional[str],
text_extra: Optional[Sequence[Dict[str, Any]]] = None,
) -> Tuple[str, List[str]]:
"""把 desc 拆成 (纯标题, 标签列表)。
标签以 desc 为准（正则取全集），text_extra 只用来补充。实测 12 条样本
里有 2 条的标签在 desc 里有、text_extra 里没有，以 text_extra 为准会
静默丢标签。
剥标题时优先用 text_extra 给的 start/end 偏移——它能认出 "C# 教程"
里的 # 不是标签。但偏移不一定可信（越界、错位都见过），所以每个都先
验证 desc[start:end] 确实是 "#" + hashtag_name，验不过就退回正则。
兜底删的是 "#" + 已知标签名 这个精确串，不是任意 #，所以 "C# 教程"
是安全的。唯一的有损场景：标签名以 "#名字" 的形式也出现在正文里，
那两处会一起被删掉（见 test_split_repeated_tagname_in_body_is_lossy）。
"""
desc = (desc or "").strip()
entries = list(text_extra or [])
# 两件事要分开：
# raw_hits —— desc 里 # 后面的原文，用来剥标题（必须能字面匹配上）
# tags —— 清洗+去重后的结果，是对外的字段
# 混用会导致 "#（括号标签）" 剥不掉：清洗后的名字没有那对括号，
# replace 匹配不上，标签就留在标题里了。
raw_hits = _TAG_RE.findall(desc)
from_te = [e.get("hashtag_name") for e in entries if e.get("hashtag_name")]
tags = _dedupe(raw_hits + from_te)
if not desc:
return "", tags
# 能用偏移剥掉的标签，逐个记下要挖掉的区间
spans: List[Tuple[int, int]] = []
covered = set()
for e in entries:
name = e.get("hashtag_name")
start, end = e.get("start"), e.get("end")
if not name or not isinstance(start, int) or not isinstance(end, int):
continue
if not (0 <= start < end <= len(desc)):
continue
if desc[start:end] != "#" + name:
continue
spans.append((start, end))
covered.add(name)
title = desc
for start, end in sorted(spans, reverse=True): # 从后往前挖，偏移不会串位
title = title[:start] + title[end:]
# 偏移没盖到的，用 desc 里的原文删（不是清洗过的 tags）
for hit in raw_hits:
if hit not in covered:
title = title.replace("#" + hit, "")
return re.sub(r"\s+", " ", title).strip(), tags
def build_record(aweme, *, duration_sec: float, cos_mp4: str, cos_mp3: str,
size_mp4: int, size_mp3: int,
source_url: str = "") -> Dict[str, Any]:
"""组装一条 record。字段顺序是有意的，aweme_id 必须在最前。"""
raw = aweme.raw or {}
title, tags = split_desc(aweme.desc, raw.get("text_extra"))
author = raw.get("author") or {}
stat = raw.get("statistics") or {}
music = raw.get("music") or {}
video = raw.get("video") or {}
ct = int(aweme.create_time or 0)
return {
"aweme_id": str(aweme.aweme_id),
"title": title,
"desc": aweme.desc,
"tags": tags,
"author": {
"nickname": aweme.author,
"uid": str(author.get("uid") or ""),
"sec_uid": str(author.get("sec_uid") or ""),
},
"create_time": ct,
"create_time_str": (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ct))
if ct else ""),
"duration_sec": duration_sec,
# 服务端声明值，与 ffprobe 解出来的不等（实测 41750 vs 41703 ms）。
# 两个都留着，切分要用 duration_sec。
"duration_ms_api": video.get("duration") or raw.get("duration") or 0,
"stats": {
"digg": stat.get("digg_count", 0),
"comment": stat.get("comment_count", 0),
"collect": stat.get("collect_count", 0),
"share": stat.get("share_count", 0),
"play": stat.get("play_count", 0),
},
"music": {
"title": (music.get("title") or "").strip(),
"author": (music.get("author") or "").strip(),
"url": aweme.music_url or "",
},
"cover_url": aweme.cover_urls[0] if aweme.cover_urls else "",
"video": {
"uri": aweme.uri,
"width": video.get("width") or 0,
"height": video.get("height") or 0,
},
"cos_mp4": cos_mp4,
"cos_mp3": cos_mp3,
"size_mp4": size_mp4,
"size_mp3": size_mp3,
"source_url": source_url or f"https://www.douyin.com/video/{aweme.aweme_id}",
"fetched_at": int(time.time()),
"raw": raw,
}
class RecordLog:
"""records.jsonl 的追加写 + 已完成 id 集合。多线程安全。"""
def __init__(self, path: Path) -> None:
self.path = Path(path)
self._lock = threading.Lock()
self._ids = self._scan()
def _scan(self) -> set:
"""流式扫已有 id。只看行首，不解析整行。"""
ids = set()
if not self.path.exists():
return ids
with open(self.path, "r", encoding="utf-8", errors="replace") as f:
for line in f:
m = _ID_RE.match(line)
if m:
ids.add(m.group(1))
return ids
def __contains__(self, aweme_id: str) -> bool:
return str(aweme_id) in self._ids
def __len__(self) -> int:
return len(self._ids)
@property
def ids(self) -> set:
return set(self._ids)
def _needs_newline(self) -> bool:
"""上一次可能被 Ctrl-C 打断，留下没换行的半行。"""
try:
if self.path.stat().st_size == 0:
return False
except OSError:
return False
with open(self.path, "rb") as f:
f.seek(-1, 2)
return f.read(1) != b"\n"
def write(self, record: Dict[str, Any]) -> bool:
"""追加一条。已存在同 id 则跳过，返回 False。"""
aweme_id = str(record.get("aweme_id") or "")
if not aweme_id:
raise ValueError("record 缺 aweme_id")
with self._lock:
if aweme_id in self._ids:
return False
self.path.parent.mkdir(parents=True, exist_ok=True)
prefix = "\n" if self._needs_newline() else ""
line = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
if not line.startswith('{"aweme_id":'):
raise ValueError("aweme_id 必须是第一个字段，见本模块开头的约定")
with open(self.path, "a", encoding="utf-8") as f:
f.write(prefix + line + "\n")
self._ids.add(aweme_id)
return True
def iter_records(path: Path):
"""逐行读 records.jsonl，跳过坏行（可能有被打断留下的半行）。"""
path = Path(path)
if not path.exists():
return
with open(path, "r", encoding="utf-8", errors="replace") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
yield json.loads(line)
except ValueError:
continue

156
douyin/signer.py Normal file
View File

@@ -0,0 +1,156 @@
"""a_bogus 签名器。
两种后端，接口相同、可互换：
NodeSigner 调本地 node_signer（真实 webmssdk 跑在伪造浏览器环境里）。
当前唯一可用的实现，已实测能被抖音接受。
ABogusSigner 纯 Python 实现。底层 SM3/RC4/自定义 Base64 已在 crypto.py
验证通过，剩字节拼装顺序待从 ab.js 逆出来。
签名器同时决定 User-Agent：a_bogus 把 UA 编进签名，服务端会交叉校验，
所以请求头的 UA 必须与签名时用的完全一致。
"""
from __future__ import annotations
import json
import subprocess
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Protocol, Sequence
class SignError(RuntimeError):
pass
class SignerNotConfigured(SignError):
pass
class Signer(Protocol):
"""client 只依赖这两个成员。"""
@property
def user_agent(self) -> str:
"""签名时使用的 UA，请求头必须用同一个。"""
...
def sign_url(self, url: str, fp: str = "") -> str:
"""给一条裸 URL，返回带 a_bogus 的完整 URL。"""
...
# --------------------------------------------------------------------------- #
# Node 后端
# --------------------------------------------------------------------------- #
class NodeSigner:
"""通过 node_signer/sign.js 签名。
node 启动 + 加载 347KB 的 SDK 大约 0.7s，所以支持 batch 一次签多条，
把这份固定开销摊掉。
"""
def __init__(self, signer_dir: str | Path, node: str = "node",
timeout: float = 120.0) -> None:
self.dir = Path(signer_dir)
self.node = node
self.timeout = timeout
self._ua: str | None = None
script = self.dir / "sign.js"
if not script.exists():
raise SignerNotConfigured(f"找不到签名脚本: {script}")
def _run(self, args: Sequence[str], stdin: str | None = None) -> dict:
try:
proc = subprocess.run(
[self.node, "sign.js", *args], cwd=self.dir, input=stdin,
capture_output=True, text=True, encoding="utf-8",
timeout=self.timeout,
)
except FileNotFoundError as exc:
raise SignerNotConfigured(f"找不到 node 可执行文件: {self.node}") from exc
except subprocess.TimeoutExpired as exc:
raise SignError(f"签名超时（>{self.timeout}s）") from exc
out = (proc.stdout or "").strip()
if not out:
raise SignError(f"签名器无输出，rc={proc.returncode}，stderr={proc.stderr[:400]}")
try:
# sign.js 只在最后一行吐 JSON，前面可能有 SDK 自己的噪声
return json.loads(out.splitlines()[-1])
except ValueError as exc:
raise SignError(f"签名器输出不是 JSON: {out[:300]!r}") from exc
@property
def user_agent(self) -> str:
if self._ua is None:
probe = self._run(["https://www.douyin.com/aweme/v1/web/aweme/detail/?aweme_id=1", ""])
self._ua = probe.get("userAgent") or ""
if not self._ua:
raise SignError("签名器没有返回 userAgent")
return self._ua
def sign_url(self, url: str, fp: str = "") -> str:
res = self._run([url, fp])
if not res.get("ok"):
raise SignError(f"签名失败: {res.get('error')}")
self._ua = res.get("userAgent") or self._ua
return res["signedUrl"]
def sign_many(self, urls: List[str], fp: str = "") -> List[str]:
"""一次 node 启动签完多条。"""
if not urls:
return []
res = self._run(["--batch", fp], stdin=json.dumps(urls))
if not res.get("ok"):
raise SignError(f"批量签名失败: {res.get('error')}")
self._ua = res.get("userAgent") or self._ua
signed = []
for item in res["results"]:
if not item.get("ok"):
raise SignError(f"某条签名失败: {item.get('error')} url={item.get('rawUrl')}")
signed.append(item["signedUrl"])
return signed
# --------------------------------------------------------------------------- #
# 纯 Python 后端（待补）
# --------------------------------------------------------------------------- #
@dataclass
class ABogusConfig:
"""a_bogus 算法常量。必须与线上 webmssdk.js 当前版本一致。"""
alphabets: Dict[str, str] = field(default_factory=dict)
ua_key: bytes = b""
arguments: List[int] = field(default_factory=list)
browser_fp: str = ""
end_string: str = ""
user_agent: str = ""
def validate(self) -> None:
missing = [n for n in ("alphabets", "ua_key", "arguments",
"browser_fp", "end_string", "user_agent")
if not getattr(self, n)]
if missing:
raise SignerNotConfigured("a_bogus 常量未配置，缺少: " + ", ".join(missing))
class ABogusSigner:
"""纯 Python a_bogus。crypto.py 里的 SM3/RC4/Base64 已验证可用。"""
def __init__(self, config: ABogusConfig) -> None:
config.validate()
self.config = config
@property
def user_agent(self) -> str:
return self.config.user_agent
def sign_url(self, url: str, fp: str = "") -> str:
raise NotImplementedError(
"a_bogus 字节拼装顺序待从 ab.js 逆出。目前请用 NodeSigner。"
)

85
douyin/urls.py Normal file
View File

@@ -0,0 +1,85 @@
"""链接解析：把用户随手粘的东西还原成 aweme_id。
支持的输入形态：
7xxxxxxxxxxxxxxxxxx 裸 id
https://v.douyin.com/xxxxxxx/ 分享短链（需要一次网络跳转）
4.53 复制打开抖音... https://v.douyin.com/xxx/ ... 整段分享口令
https://www.douyin.com/video/7xxxxxx
https://www.douyin.com/note/7xxxxxx 图文
https://www.douyin.com/?modal_id=7xxxxxx
https://www.iesdouyin.com/share/video/7xxxxxx/
"""
from __future__ import annotations
import re
from typing import Optional
import requests
# 从一段中文分享口令里把 url 抠出来
URL_RE = re.compile(r"https?://[-\w./?=&%#:]+")
SHORT_HOSTS = ("v.douyin.com", "v.iesdouyin.com")
# 按优先级尝试的 id 提取规则
ID_PATTERNS = (
re.compile(r"/(?:video|note|slides|share/video|share/note)/(\d{15,25})"),
re.compile(r"[?&](?:modal_id|aweme_id|item_ids|vid)=(\d{15,25})"),
)
BARE_ID_RE = re.compile(r"^\d{15,25}$")
class ParseError(ValueError):
"""输入里找不到可用的 aweme_id。"""
def _first_url(text: str) -> Optional[str]:
m = URL_RE.search(text)
return m.group(0) if m else None
def _extract_id(url: str) -> Optional[str]:
for pat in ID_PATTERNS:
m = pat.search(url)
if m:
return m.group(1)
return None
def resolve_short_link(url: str, session: requests.Session, timeout: float = 15.0) -> str:
"""跟随 v.douyin.com 短链跳转，返回落地长链。"""
resp = session.get(url, timeout=timeout, allow_redirects=True)
return resp.url
def extract_aweme_id(raw: str, session: Optional[requests.Session] = None) -> str:
"""从任意输入中解析出 aweme_id，必要时联网还原短链。"""
raw = raw.strip()
if BARE_ID_RE.match(raw):
return raw
url = _first_url(raw)
if not url:
raise ParseError(f"输入里没找到链接也不是纯数字 id: {raw[:80]!r}")
aid = _extract_id(url)
if aid:
return aid
if any(h in url for h in SHORT_HOSTS):
if session is None:
session = requests.Session()
session.headers["User-Agent"] = (
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 "
"Mobile/15E148 Safari/604.1"
)
final = resolve_short_link(url, session)
aid = _extract_id(final)
if aid:
return aid
raise ParseError(f"短链 {url} 跳转到 {final}，但里面没有 aweme_id")
raise ParseError(f"无法从 {url} 中解析出 aweme_id")