初始提交:抖音视频批量采集流水线

App 端免签名接口(设备注册 + detail + CDN 直链),一体化流水线
下载→抽音→传COS→记录→删本地,AI 作品四档判定,四份文档:
README(使用指南)、PROGRESS(批次进度)、RE(逆向全过程)、AI(判定规则)。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
2026-09-29 16:31:10 +08:00
commit 2bb83f3685
39 changed files with 298874 additions and 0 deletions

77
tests/test_smoke.py Normal file
View File

@@ -0,0 +1,77 @@
"""不联网的冒烟测试:验证 crypto 正确性、链接解析、文件名清洗、去重记录。"""
from __future__ import annotations
# 移到 scripts/ tests/ 后,直接运行时脚本目录成了 sys.path[0],
# 把项目根加回来,import douyin / pipeline 才找得到。
import sys as _sys
from pathlib import Path as _Path
_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
import tempfile
from pathlib import Path
from douyin import Registry, extract_aweme_id, safe_filename
from douyin.crypto import self_test
from douyin.urls import ParseError
def test_crypto():
self_test()
def test_extract_id():
cases = {
"7123456789012345678": "7123456789012345678",
"https://www.douyin.com/video/7123456789012345678": "7123456789012345678",
"https://www.douyin.com/note/7123456789012345678": "7123456789012345678",
"https://www.douyin.com/?modal_id=7123456789012345678": "7123456789012345678",
"https://www.iesdouyin.com/share/video/7123456789012345678/": "7123456789012345678",
"看看这个 https://www.douyin.com/video/7123456789012345678?a=1 好玩":
"7123456789012345678",
}
for raw, want in cases.items():
got = extract_aweme_id(raw)
assert got == want, f"{raw!r} -> {got}, 期望 {want}"
for bad in ["随便一段没有链接的文字", "https://example.com/foo"]:
try:
extract_aweme_id(bad)
except ParseError:
pass
else:
raise AssertionError(f"{bad!r} 本应报 ParseError")
print("extract_aweme_id OK")
def test_safe_filename():
assert safe_filename('a/b\\c:d*e?f"g<h>i|j') == "a_b_c_d_e_f_g_h_i_j"
assert safe_filename(" ") == "untitled"
assert len(safe_filename("x" * 200)) == 80
assert safe_filename("多个 空格") == "多个 空格"
print("safe_filename OK")
def test_registry():
with tempfile.TemporaryDirectory() as d:
root = Path(d)
reg = Registry(root / ".downloaded.json")
f = root / "v.mp4"
f.write_bytes(b"x" * 10)
assert "111" not in reg
reg.add("111", f, 10, "标题")
assert "111" in reg
# 重新加载后仍在
assert "111" in Registry(root / ".downloaded.json")
# 文件被删则视为未下载
f.unlink()
assert "111" not in Registry(root / ".downloaded.json")
print("Registry OK")
if __name__ == "__main__":
test_crypto()
test_extract_id()
test_safe_filename()
test_registry()
print("\n全部冒烟测试通过")