初始提交：抖音视频批量采集流水线

App 端免签名接口（设备注册 + detail + CDN 直链），一体化流水线
下载→抽音→传COS→记录→删本地，AI 作品四档判定，四份文档：
README（使用指南）、PROGRESS（批次进度）、RE（逆向全过程）、AI（判定规则）。

Co-Authored-By: Claude Code <noreply@anthropic.com>

tests/test_aiclass.py

@@ -0,0 +1,276 @@
+"""douyin.aiclass 的离线测试。用例全部取自 40087 条真实数据里的真实文案。"""
+
+from __future__ import annotations
+
+# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
+# 把项目根加回来，import douyin / pipeline 才找得到。
+import sys as _sys
+from pathlib import Path as _Path
+_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
+
+from douyin.aiclass import classify
+
+
+def rec(desc="", tags=(), author="", music="", is_aigc=False, label=""):
+    return {
+        "desc": desc,
+        "hashtags": [{"name": t} for t in tags],
+        "author": {"nickname": author},
+        "music": {"title": music},
+        "ai": {"is_aigc": is_aigc, "label": label},
+    }
+
+
+def tier(**kw):
+    return classify(rec(**kw))["tier"]
+
+
+# ---------- platform：唯一能抓到"作者没标但确实是 AI"的信号 ----------
+
+def test_platform_label_wins_even_with_no_ai_text():
+    """实测 1744 条：平台标了 AIGC，文案标签里一个 ai 字都没有。"""
+    assert tier(desc="#趣味脑筋急转弯 #创作者中心 #创作灵感",
+                tags=["趣味脑筋急转弯", "创作者中心", "创作灵感"],
+                author="申哥爱剪辑", is_aigc=True,
+                label="作者声明：内容由 AI 生成") == "platform"
+    assert tier(desc="澳洲的野狗 #中华田园犬 #历史", tags=["中华田园犬", "历史"],
+                author="胖嘟嘟历史", is_aigc=True) == "platform"
+
+
+# ---------- strong：作品自己声明是 AI 做的 ----------
+
+def test_strong_ai_compound():
+    assert tier(desc="AI花絮：我很快就会驯服AI演员啦 #AI短剧#AI电影",
+                tags=["AI短剧", "AI电影"]) == "strong"
+
+
+def test_strong_tool_name_without_ai_letters():
+    """即梦 / seedance / libtv 这些名字里没有 ai，只能靠名单。"""
+    assert tier(desc="干啥呀即梦，这是掀桌子不玩了吗？", tags=["即梦"]) == "strong"
+    assert tier(desc="", tags=["seedance2.0"]) == "strong"
+    assert tier(desc="本视频由libtv制作生成", tags=["Libtv"]) == "strong"
+    assert tier(desc="", tags=["runninghub"]) == "strong"
+
+
+def test_strong_aigc_token():
+    assert tier(desc="自制AIGC《银骸》第0集！") == "strong"
+
+
+def test_strong_handles_capital_A_lowercase_L():
+    """`抖音al` 在这批数据里出现 129 次，`#Al创作` 也有。不归一化就整片漏掉。"""
+    assert tier(desc="他种下几千棵橡树 #Al创作 #爱情", tags=["Al创作", "爱情"]) == "strong"
+    assert tier(desc="", tags=["抖音al"]) == "topic"      # 没有创作语境词，降到 topic
+
+
+# ---------- topic：在讲 AI，不一定是 AI 做的 ----------
+
+def test_topic_pure_ai_mention():
+    assert tier(desc="布布讲ai之如何写提示词 #ai", tags=["ai"]) == "strong"  # 提示词是创作语境
+    assert tier(desc="今天聊聊 AI 会不会取代人类 #ai", tags=["ai"]) == "topic"
+
+
+def test_topic_chat_models():
+    assert tier(desc="deepseek 又更新了") == "topic"
+    assert tier(desc="用 ChatGPT 写周报") == "topic"
+    assert tier(desc="#人工智能 的一点思考", tags=["人工智能"]) == "topic"
+
+
+# ---------- author：最弱，作品本身没信号 ----------
+
+def test_author_only():
+    r = classify(rec(desc="研究生偷吃外卖被毒S #湘潭大学 #普法教育",
+                     tags=["湘潭大学", "普法教育"], author="牛牛Ai"))
+    assert r["tier"] == "author", r
+
+
+def test_author_never_upgrades_a_scored_item():
+    """作者昵称只能给 none 兜底，不能把 topic 顶成 strong。"""
+    r = classify(rec(desc="聊聊 AI 的伦理问题", author="星幕AIGC"))
+    assert r["tier"] == "topic", r
+
+
+def test_default_original_sound_is_not_a_work_signal():
+    """音乐标题默认是 `@<昵称>创作的原声`，40087 条里有 32555 条是这个壳。
+    昵称带 AI 会连"创作"一起命中，曾让 771 条从 music 这条路混进 strong
+    （@冠宇Aigc 的猴子炒菜、@牛牛Ai 的大案纪实）。只能落到 author 档。"""
+    r = classify(rec(desc="#猴子炒菜#猴子成精#原创视频",
+                     tags=["猴子炒菜", "猴子成精"],
+                     author="冠宇Aigc(AI剧场)",
+                     music="@冠宇Aigc(AI剧场)创作的原声"))
+    assert r["tier"] == "author", r
+    # 作者昵称也不带 AI 时，默认原声不能凭空造出信号
+    assert tier(desc="今天天气不错", author="老王",
+                music="@老王创作的原声") == "none"
+
+
+def test_real_track_name_still_counts_but_only_weakly():
+    assert tier(desc="随便发发", music="即梦AI生成的配乐") == "author"
+
+
+# ---------- 误报：这些绝不能收 ----------
+
+def test_english_words_containing_ai_are_not_signals():
+    for d in ["email me for detail", "again and again", "hair salon",
+              "wait a minute", "chairman", "trained model"]:
+        assert tier(desc=d) == "none", d
+
+
+def test_editing_jargon_is_not_ai():
+    """剪映 375 / 关键帧 314 / 蒙版 203 —— 剪辑术语，不是 AI。"""
+    assert tier(desc="导入一张照片即可制作 #剪映", tags=["剪映"]) == "none"
+    assert tier(desc="#关键帧 #蒙版 #转场 #剪辑教程",
+                tags=["关键帧", "蒙版", "转场", "剪辑教程"]) == "none"
+
+
+def test_generic_chinese_words_alone_are_not_ai():
+    """动画/绘画/分身/动漫 单独出现不算 —— 否则 830 条 #原创动画 全被误收。"""
+    assert tier(desc="第33 #原创动画", tags=["原创动画"]) == "none"
+    assert tier(desc="跟小同桌回老家 #二次元 #原创动画 #恋爱番",
+                tags=["二次元", "原创动画", "恋爱番"]) == "none"
+    assert tier(desc="他的分身术真牛") == "none"
+
+
+def test_ordinary_content_is_none():
+    assert tier(desc="户外露营过夜用的这款帐篷也太实用啦！ #露营装备",
+                tags=["露营装备"], author="蚁巢户外直营店") == "none"
+    assert tier(desc="4月16日，重庆，张雪陪同合作伙伴参观工厂 #张雪机车",
+                tags=["张雪机车"], author="第一现场") == "none"
+
+
+def test_signals_are_recorded_for_audit():
+    r = classify(rec(desc="#即梦AI #AI短剧", tags=["即梦ai", "ai短剧"]))
+    assert r["tier"] == "strong"
+    assert any("工具:即梦" in s for s in r["signals"]), r["signals"]
+
+
+def test_empty_record():
+    assert classify(rec())["tier"] == "none"
+
+
+# ---------- 常用词工具名：必须跟 ai 同现，见 aiclass 模块开头第 6 条 ----------
+
+def test_ambiguous_tool_alone_is_not_ai():
+    """87110 条里实测的误判，全部取自真实文案。"""
+    # 三峰的"海螺"帐篷，撞上 MiniMax 的海螺 AI
+    assert tier(desc="三峰出-海螺。重量超级轻的一款帐篷，有tpu，有烟囱口。",
+                tags=["户外装备", "帐篷", "露营"]) == "none"
+    # 元宝勺，撞上腾讯元宝
+    assert tier(desc="边缘光滑才是好勺子！316不锈钢元宝勺。",
+                tags=["辅食勺", "勺子", "元宝勺"]) == "none"
+    # 豆包是只狗 / 一种包子，不是那个大模型
+    assert tier(desc="豆包监督学生自习这一块 豆包：交给我你就闹心吧！",
+                tags=["豆包", "搞笑", "晚自习"]) == "none"
+    # 《龙神万相》，撞上阿里万相
+    assert tier(desc="龙神万相之龙魂定档 迫不及待想沉浸式感受龙族的热血与传奇了！",
+                tags=["龙神万相之龙魂", "西行纪"]) == "none"
+    # "明星流量"里含"星流"；"起舞弄清影"里含"清影"
+    assert tier(desc="商家别乱蹭明星流量！4 名被告擅自使用张雨绮肖像及姓名商用",
+                tags=["张雨绮", "侵权"]) == "none"
+    assert tier(desc="柳下闻瑶琴起舞弄清影 ～好久没跳了") == "none"
+    assert tier(desc="做白日梦从拥有弹丝云床开始",
+                tags=["户外露营", "自动充气床"]) == "none"
+
+
+def test_ambiguous_tool_counts_when_ai_present():
+    """真命中的无一例外都同时带 AI 字样，所以这道门槛不掉召回。"""
+    r = classify(rec(desc="海螺AI生成的短片 #ai视频", tags=["ai视频"]))
+    assert r["tier"] == "strong"
+    assert any("工具:海螺" in s for s in r["signals"]), r["signals"]
+    assert tier(desc="用豆包写的脚本，AI 生成 #ai", tags=["ai"]) == "strong"
+
+
+def test_digital_rmb_is_not_a_digital_human():
+    """数字人民币含"数字人"，微信分身含"分身"，穿搭也说"换脸"。
+    实测放开这三个词只进误判，一条真的都捞不到。"""
+    assert tier(desc="一地宣布：5月起，公务员工资用数字人民币发放",
+                tags=["公务员", "工资", "数字人民币"]) == "none"
+    assert tier(desc="抖音双开 小米华为苹果手机 微信怎么多开",
+                tags=["微信分身双开", "抖音多开分身"]) == "none"
+    assert tier(desc="救命！这套穿搭真的能换脸！",
+                tags=["今天穿什么", "冬季穿搭"]) == "none"
+
+
+def test_sound_wrapper_csv_form_is_stripped():
+    """csv 导出的原声标题多一截 ` - 昵称`，也得剥掉，
+    否则作者昵称又会从 music 这条路偷偷升档。"""
+    assert tier(desc="今天去钓鱼", music="@牛牛Ai创作的原声 - 牛牛Ai") == "none"
+    assert tier(desc="今天去钓鱼", music="@牛牛Ai创作的原声") == "none"
+
+
+def test_minimax_is_a_tool():
+    assert tier(desc="minimax 生成的片子", tags=["minimax"]) == "strong"
+
+
+def test_roman_numeral_impersonates_ai():
+    """`AⅠ漫剧` 用罗马数字 Ⅰ(U+2160) 冒充字母 I，跟 `抖音al` 同一类花招。"""
+    r = classify(rec(desc="AⅠ漫剧你就这样崛起吧！", tags=["AⅠ漫剧", "漫剧推荐"]))
+    assert r["tier"] == "strong", r
+
+
+def test_voice_cloning_is_generative():
+    assert tier(desc="克隆音色怎么弄 声音可以克隆吗",
+                tags=["克隆音色", "声音克隆", "剪映教程"]) == "strong"
+
+
+def test_veo3_beats_a_weak_author_signal():
+    """原先这批只靠作者昵称算 author 档，认出 veo3 后该升到 strong。"""
+    assert tier(desc="#猴子成精 #猴子炒菜 #veo3", tags=["猴子炒菜", "veo3"],
+                author="冠宇Aigc") == "strong"
+
+
+def test_rejected_candidates_stay_none():
+    """这些词看着像 AI，实测全是子串陷阱，加进词表会污染结果。"""
+    assert tier(desc="全国生猪价格已连续16周下跌，行业亏损幅度加大") == "none"   # 度加
+    assert tier(desc="女生相亲要求要30万彩礼，听到要求男士都拒绝") == "none"     # 万彩
+    assert tier(desc="中国男篮硬刚美国梦八队，诞生数副名场面！") == "none"       # 生数
+    assert tier(desc="决定全年经济走势关键节点上，国务院召开重要会议") == "none"  # 节点
+    assert tier(desc="卡池都快结束了，肝了个大保底怎么还不出货",
+                tags=["抽卡日常", "崩坏星穹铁道"]) == "none"                  # 抽卡
+    assert tier(desc="圈外人不语，只是一味求捏脸", tags=["燕云十六声捏脸"]) == "none"  # 捏脸
+    assert tier(desc="杀神降临！82-36！全力帕到底有多恐怖？",
+                tags=["无畏契约", "valorant"]) == "none"                     # lora
+    assert tier(desc="剑来!!!", tags=["国产动漫", "国漫超燃", "剑来动画"]) == "none"  # 国漫
+
+
+def test_manju_names_stand_alone():
+    """漫剧专名单独成立，不要求跟 ai 同现（用户要求：出现关键词就算）。"""
+    r = classify(rec(desc="究竟是什么样的小说仅是漫改，就能瞬间火爆全网破30亿的热度",
+                     tags=["万妖图录传", "二次元", "充能计划"]))
+    assert r["tier"] == "strong", r
+    assert any("漫剧:万妖图录传" in x for x in r["signals"]), r["signals"]
+    assert tier(desc="晚夏小梦1-27：128分钟大合集看爽",
+                tags=["二次元", "原创动画", "晚夏小梦"]) == "strong"
+    assert tier(desc="小溪废片合集~", tags=["二次元", "月落小溪"]) == "strong"
+
+
+def test_hongguo_platform_name_is_not_an_ai_signal():
+    """红果主体是真人短剧，AI 漫剧只是其中一个赛道，平台名不能当信号。"""
+    assert tier(desc="把生活的爽感拉满！看《家里家外2》要备好纸巾",
+                tags=["红果短剧", "短剧推荐", "家里家外2", "治愈系短剧"]) == "none"
+    assert tier(desc="《冷御失守》后续", tags=["红果短剧"]) == "none"
+
+
+def test_manju_topic_words_are_not_ai_markers():
+    """漫剧圈里 100% 是 AI 的题材词，放全库就炸 —— 它们是题材不是 AI 标记。"""
+    assert tier(desc="玄奘故里，也就是西游记里唐僧的出生地，位于偃师缑氏镇") == "none"
+    assert tier(desc="宫崎骏的夏天大概就是这个样子吧！", tags=["宫崎骏的夏天", "龙猫"]) == "none"
+    assert tier(desc="《狼来了》第一集：还记得小时候听的《狼来了》吗？",
+                tags=["狼来了", "民间故事", "原创"]) == "none"
+
+
+def main() -> int:
+    tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
+    failed = 0
+    for t in tests:
+        try:
+            t()
+            print(f"  ok   {t.__name__}")
+        except AssertionError as exc:
+            failed += 1
+            print(f"  FAIL {t.__name__}: {exc}")
+    print(f"\n{len(tests) - failed}/{len(tests)} 通过")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_meta.py

@@ -0,0 +1,217 @@
+"""douyin.meta 的离线测试：标题/标签拆分、记录追加与幂等。不联网。"""
+
+from __future__ import annotations
+
+# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
+# 把项目根加回来，import douyin / pipeline 才找得到。
+import sys as _sys
+from pathlib import Path as _Path
+_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
+
+import json
+import tempfile
+from pathlib import Path
+
+from douyin.meta import RecordLog, split_desc
+
+
+def te(desc, *names):
+    """按真实位置构造 text_extra —— 偏移量算出来，别手数。"""
+    out = []
+    for n in names:
+        i = desc.index("#" + n)
+        out.append({"type": 1, "hashtag_name": n, "hashtag_id": "1",
+                    "start": i, "end": i + len(n) + 1})
+    return out
+
+
+def te_broken(*names):
+    """偏移量是垃圾（越界）的 text_extra。"""
+    return [{"type": 1, "hashtag_name": n, "hashtag_id": "1",
+             "start": 999, "end": 1200} for n in names]
+
+
+def test_split_basic():
+    desc = "是谁教他这样平账的？？？ #沙雕动画 #原创动画"
+    title, tags = split_desc(desc, te(desc, "沙雕动画", "原创动画"))
+    assert title == "是谁教他这样平账的？？？", repr(title)
+    assert tags == ["沙雕动画", "原创动画"], tags
+
+
+def test_split_adjacent_no_space():
+    # 实测抖音有 "#四川话#方言" 这种连着写的
+    title, tags = split_desc("老板说话#四川话#方言", None)
+    assert title == "老板说话", repr(title)
+    assert tags == ["四川话", "方言"], tags
+
+
+def test_split_tag_missing_from_text_extra():
+    """text_extra 会漏标签（实测 12 条里有 2 条），tags 必须以 desc 为准。"""
+    desc = "太搞笑了 #搞笑配音 #专治不开心 #非洲搞笑视频"
+    title, tags = split_desc(desc, te(desc, "搞笑配音", "专治不开心"))
+    assert tags == ["搞笑配音", "专治不开心", "非洲搞笑视频"], tags
+    assert title == "太搞笑了", repr(title)
+
+
+def test_split_tag_only_in_text_extra():
+    """反过来，desc 里没写但 text_extra 有的也不能丢。"""
+    _, tags = split_desc("光秃秃的标题", te_broken("隐藏标签"))
+    assert tags == ["隐藏标签"], tags
+
+
+def test_split_tag_in_middle():
+    title, tags = split_desc("前面 #中间 后面", None)
+    assert title == "前面 后面", repr(title)
+    assert tags == ["中间"], tags
+
+
+def test_split_no_tags():
+    title, tags = split_desc("  纯标题没有标签  ", None)
+    assert title == "纯标题没有标签", repr(title)
+    assert tags == [], tags
+
+
+def test_split_empty():
+    assert split_desc("", None) == ("", [])
+    assert split_desc(None, None) == ("", [])
+
+
+def test_split_dedupes_tags():
+    _, tags = split_desc("标题 #方言 #方言 #四川话", None)
+    assert tags == ["方言", "四川话"], tags
+
+
+def test_split_bad_offsets_dont_corrupt():
+    """偏移量对不上时必须退回正则，绝不能把标题切坏。"""
+    title, tags = split_desc("标题正文 #标签", te_broken("标签"))
+    assert title == "标题正文", repr(title)
+    assert tags == ["标签"], tags
+
+
+def test_split_keeps_hash_inside_word():
+    """偏移量可信时不该误伤正文里的 #（如 C# 教程）——这正是用偏移的理由。"""
+    desc = "C# 入门教程 #编程"
+    title, tags = split_desc(desc, te(desc, "编程"))
+    assert title == "C# 入门教程", repr(title)
+    assert tags == ["编程"], tags
+
+
+def test_split_lone_hash_is_not_a_tag():
+    """# 后面必须跟非空白才算标签，所以 "C# 教程" 里的 # 不会被当标签。"""
+    title, tags = split_desc("C# 入门教程 #编程", None)
+    assert tags == ["编程"], tags
+    assert title == "C# 入门教程", repr(title)
+
+
+def test_split_repeated_tagname_in_body_is_lossy():
+    """没有可信偏移、且标签名以 #名字 的形式也出现在正文里时，兜底会连正文
+    一起删。这是正则兜底的已知代价，钉住它免得以后误以为无损。"""
+    title, tags = split_desc("聊聊#教程 这件事 #教程", None)
+    assert tags == ["教程"], tags
+    assert title == "聊聊 这件事", repr(title)   # 正文里那个 #教程 也没了
+
+    # 有可信偏移时就不会误伤——只挖服务端标注的那一处
+    desc = "聊聊#教程 这件事 #教程"
+    i = desc.rindex("#教程")
+    ok = [{"type": 1, "hashtag_name": "教程", "start": i, "end": i + 3}]
+    title2, _ = split_desc(desc, ok)
+    assert title2 == "聊聊#教程 这件事", repr(title2)
+
+
+def test_split_case_insensitive_dedupe():
+    """抖音把 text_extra 里的标签归一化成小写，desc 里却是原始大小写。
+    并集必须按大小写不敏感去重，否则同一个标签会存两份（实测踩到）。"""
+    desc = "教学 #一站式AI创作工作台 #国产AI"
+    te = [{"type": 1, "hashtag_name": "一站式ai创作工作台", "start": -1, "end": -1},
+          {"type": 1, "hashtag_name": "国产ai", "start": -1, "end": -1}]
+    title, tags = split_desc(desc, te)
+    # 保留 desc 里的原始大小写，不是 text_extra 的小写版
+    assert tags == ["一站式AI创作工作台", "国产AI"], tags
+    assert title == "教学", repr(title)
+
+
+def test_split_drops_at_mention_pseudo_tag():
+    """正文里手打的 # 会被正则捕到，但带 @ 的明显是提及不是标签。
+    实测：desc 里的 "#(提示词出自@青木"，抖音自己也没把它当标签。"""
+    desc = "示例#正经标签 #(提示词出自@青木 文本#另一个"
+    title, tags = split_desc(desc, None)
+    assert tags == ["正经标签", "另一个"], tags
+    # 它不进 tags，但仍然从标题里剥掉——作者本来就是当标签打的
+    assert title == "示例 文本", repr(title)
+
+
+def test_split_strips_edge_punctuation():
+    title, tags = split_desc("标题 #（括号标签） #正常", None)
+    assert tags == ["括号标签", "正常"], tags
+    # 标题里那对括号也得跟着标签一起剥掉——剥标题用的是 desc 原文，
+    # 不是清洗过的标签名，否则会匹配不上而留在标题里
+    assert title == "标题", repr(title)
+
+
+def test_recordlog_roundtrip_and_resume():
+    with tempfile.TemporaryDirectory() as d:
+        p = Path(d) / "records.jsonl"
+
+        log = RecordLog(p)
+        assert len(log) == 0
+        assert log.write({"aweme_id": "111", "title": "甲", "raw": {"x": 1}}) is True
+        assert log.write({"aweme_id": "222", "title": "乙", "raw": {"x": 2}}) is True
+        assert "111" in log and "222" in log and "333" not in log
+
+        # 重开：应该扫出已有的两条
+        log2 = RecordLog(p)
+        assert len(log2) == 2, len(log2)
+        assert "111" in log2 and "222" in log2
+
+        # 重复写入被幂等挡掉
+        assert log2.write({"aweme_id": "111", "title": "改过的"}) is False
+        assert len(p.read_text(encoding="utf-8").strip().splitlines()) == 2
+
+        # aweme_id 必须是每行第一个字段（续跑扫描的契约）
+        for line in p.read_text(encoding="utf-8").splitlines():
+            assert line.startswith('{"aweme_id":'), line[:40]
+
+        rows = [json.loads(x) for x in p.read_text(encoding="utf-8").splitlines()]
+        assert rows[0]["title"] == "甲" and rows[0]["raw"] == {"x": 1}
+
+
+def test_recordlog_survives_broken_tail():
+    """进程被 Ctrl-C 打断可能留下半行，扫描不能因此崩掉。"""
+    with tempfile.TemporaryDirectory() as d:
+        p = Path(d) / "records.jsonl"
+        p.write_text('{"aweme_id":"111","title":"完整"}\n{"aweme_id":"222","tit',
+                     encoding="utf-8")
+        log = RecordLog(p)
+        assert "111" in log and "222" in log   # 半行的 id 也认，照样跳过
+
+        log.write({"aweme_id": "333", "title": "新的"})
+        lines = p.read_text(encoding="utf-8").splitlines()
+        # 补写前必须先断行，不能跟半行黏成一行
+        assert lines[-1].startswith('{"aweme_id":"333"'), lines[-1][:40]
+        assert json.loads(lines[-1])["title"] == "新的"
+
+
+def test_recordlog_no_ascii_escaping():
+    """中文必须原样存，不能转成 \\uXXXX——否则文件体积翻几倍且不可读。"""
+    with tempfile.TemporaryDirectory() as d:
+        p = Path(d) / "r.jsonl"
+        RecordLog(p).write({"aweme_id": "1", "title": "四川方言"})
+        assert "四川方言" in p.read_text(encoding="utf-8")
+
+
+def main() -> int:
+    tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
+    failed = 0
+    for t in tests:
+        try:
+            t()
+            print(f"  ok   {t.__name__}")
+        except AssertionError as exc:
+            failed += 1
+            print(f"  FAIL {t.__name__}: {exc}")
+    print(f"\n{len(tests) - failed}/{len(tests)} 通过")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_pipeline.py

@@ -0,0 +1,335 @@
+"""pipeline.run_one 的端到端测试。不联网、不碰 COS。
+
+网络（fetch_detail / download）和 COS（CosUploader）都打桩，但 ffmpeg 抽取、
+ffprobe 时长闸门、记录写入、本地删除全都真跑——这几步才是会悄悄丢数据的地方。
+测试用的 mp4 由 ffmpeg 现造。
+"""
+
+from __future__ import annotations
+
+# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
+# 把项目根加回来，import douyin / pipeline 才找得到。
+import sys as _sys
+from pathlib import Path as _Path
+_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
+
+import argparse
+import json
+import shutil
+import subprocess
+import tempfile
+import threading
+from pathlib import Path
+
+import pipeline
+from douyin.client import Aweme
+from douyin.app import DeviceRegisterError
+from douyin.cos import SizeMismatch
+from douyin.download import DownloadResult
+from douyin.meta import RecordLog
+
+DUR = 3.0
+
+
+def make_mp4(path: Path, seconds: float = DUR) -> None:
+    subprocess.run(
+        ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
+         "-f", "lavfi", "-i", f"testsrc=duration={seconds}:size=320x240:rate=10",
+         "-f", "lavfi", "-i", f"sine=frequency=440:duration={seconds}",
+         "-c:v", "libx264", "-c:a", "aac", "-shortest", str(path)],
+        check=True, capture_output=True)
+
+
+def fake_aweme(aweme_id="7000000000000000001"):
+    raw = {
+        "aweme_id": aweme_id,
+        "desc": "测试标题 #方言 #四川话",
+        "text_extra": [{"type": 1, "hashtag_name": "方言", "hashtag_id": "9",
+                        "start": 4, "end": 7}],
+        "author": {"nickname": "某作者", "uid": "123", "sec_uid": "MS4w"},
+        "statistics": {"digg_count": 10, "comment_count": 2, "collect_count": 1,
+                       "share_count": 3, "play_count": 999},
+        "music": {"title": "原声", "author": "某作者"},
+        "video": {"duration": 3050, "width": 320, "height": 240},
+    }
+    return Aweme(aweme_id=aweme_id, desc=raw["desc"], author="某作者",
+                 create_time=1582635538, video_urls=["http://x/v.mp4"],
+                 cover_urls=["http://x/c.jpg"], music_url="http://x/m.mp3",
+                 uri="v0200fg10000test", raw=raw)
+
+
+class StubPool:
+    def current(self):
+        return object(), 0
+
+    def report_dead(self, gen):
+        pass
+
+
+class StubUploader:
+    """记下传了什么，不真传。"""
+
+    def __init__(self, fail_on=None):
+        self.put_calls = []
+        self.fail_on = fail_on
+
+    def key_for(self, name):
+        return f"0005/zh/{name}"
+
+    def put(self, local, key):
+        if self.fail_on and self.fail_on in key:
+            raise SizeMismatch(f"{key} 大小不符：COS=1 本地=2")
+        size = Path(local).stat().st_size
+        self.put_calls.append((key, size))
+        return f"cos://bucket/{key}", size
+
+
+def make_args(work, keep_local=False):
+    return argparse.Namespace(work_dir=str(work), bitrate="96k", ffprobe="ffprobe",
+                              keep_local=keep_local)
+
+
+def run(tmp, *, src_mp4, keep_local=False, uploader=None, aweme=None,
+        patch_download=None):
+    """跑一次 run_one，返回 (records, failures, uploader, work_dir)。"""
+    work = Path(tmp) / "work"
+    work.mkdir(exist_ok=True)
+    aweme = aweme or fake_aweme()
+    up = uploader or StubUploader()
+    log = RecordLog(Path(tmp) / "records.jsonl")
+    fails = []
+
+    def dl(session, urls, dest, **kw):
+        shutil.copy(src_mp4, dest)
+        return DownloadResult(dest, dest.stat().st_size, resumed=False)
+
+    orig_dl, orig_fetch = pipeline.download, pipeline.AppClient
+    pipeline.download = patch_download or dl
+    pipeline.AppClient = lambda dev: type("C", (), {"fetch_detail": lambda s, i: aweme})()
+    try:
+        pipeline.run_one(
+            aweme.aweme_id, "http://src", pool=StubPool(), up=up, log=log,
+            args=make_args(work, keep_local), ffmpeg="ffmpeg",
+            ff_sem=threading.Semaphore(2), stats=pipeline.Stats(),
+            fail=lambda i, s, m: fails.append((i, s, m)),
+            abort=threading.Event())
+    finally:
+        pipeline.download, pipeline.AppClient = orig_dl, orig_fetch
+
+    recs = [json.loads(l) for l in
+            (Path(tmp) / "records.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()] \
+        if (Path(tmp) / "records.jsonl").exists() else []
+    return recs, fails, up, work
+
+
+def test_happy_path_writes_record_and_deletes_local():
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "src.mp4"
+        make_mp4(src)
+        recs, fails, up, work = run(tmp, src_mp4=src)
+
+        assert not fails, fails
+        assert len(recs) == 1, recs
+        r = recs[0]
+        assert r["aweme_id"] == "7000000000000000001"
+        assert r["title"] == "测试标题", repr(r["title"])
+        assert r["tags"] == ["方言", "四川话"], r["tags"]
+        assert r["author"]["uid"] == "123"
+        assert r["stats"]["play"] == 999
+        assert abs(r["duration_sec"] - DUR) < 0.5, r["duration_sec"]
+        assert r["duration_ms_api"] == 3050          # API 声明值原样留着
+        assert r["cos_mp4"].endswith(".mp4") and r["cos_mp3"].endswith(".mp3")
+        assert r["raw"]["author"]["sec_uid"] == "MS4w"
+
+        # 两个都传了
+        assert len(up.put_calls) == 2, up.put_calls
+        # 本地已删干净
+        assert list(work.iterdir()) == [], list(work.iterdir())
+
+
+def test_keep_local_keeps_files():
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "src.mp4"
+        make_mp4(src)
+        recs, fails, up, work = run(tmp, src_mp4=src, keep_local=True)
+        assert not fails, fails
+        names = sorted(p.suffix for p in work.iterdir())
+        assert names == [".mp3", ".mp4"], names
+
+
+def test_upload_failure_keeps_local_and_writes_no_record():
+    """上传失败必须保留本地文件——不然重跑就得重新下载。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "src.mp4"
+        make_mp4(src)
+        recs, fails, up, work = run(tmp, src_mp4=src,
+                                    uploader=StubUploader(fail_on=".mp3"))
+        assert recs == [], recs
+        assert len(fails) == 1 and fails[0][1] == "upload", fails
+        kept = sorted(p.suffix for p in work.iterdir())
+        assert kept == [".mp3", ".mp4"], kept
+
+
+def test_truncated_mp3_is_caught_before_delete():
+    """时长闸门：mp3 被截断时必须拦下，且不删本地、不写记录。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "src.mp4"
+        make_mp4(src, seconds=10.0)
+
+        orig = pipeline.extract_mp3
+
+        def truncating(ffmpeg, s, dst, bitrate, timeout=600.0):
+            orig(ffmpeg, s, dst, bitrate, timeout)
+            data = dst.read_bytes()
+            dst.write_bytes(data[: len(data) // 5])      # 砍掉 80%
+
+        pipeline.extract_mp3 = truncating
+        try:
+            recs, fails, up, work = run(tmp, src_mp4=src)
+        finally:
+            pipeline.extract_mp3 = orig
+
+        assert recs == [], recs
+        assert len(fails) == 1 and fails[0][1] == "probe", fails
+        assert "截断" in fails[0][2], fails[0][2]
+        assert up.put_calls == [], "截断的文件绝不该被上传"
+        assert len(list(work.iterdir())) == 2          # 本地保留待查
+
+
+def test_register_exhausted_aborts_batch():
+    """IP 级注册配额耗尽时必须中止整批，不能把剩余任务逐条烧成失败。
+    上一批就是这么烧掉 678 条的。"""
+    class DeadPool:
+        def current(self):
+            raise DeviceRegisterError("所有 host 都返回 device_id=0")
+
+        def report_dead(self, gen):
+            pass
+
+    with tempfile.TemporaryDirectory() as tmp:
+        work = Path(tmp) / "work"
+        work.mkdir()
+        log = RecordLog(Path(tmp) / "r.jsonl")
+        fails = []
+        abort = threading.Event()
+
+        for i in range(3):
+            pipeline.run_one(f"700000000000000000{i}", "u", pool=DeadPool(),
+                             up=StubUploader(), log=log, args=make_args(work),
+                             ffmpeg="ffmpeg", ff_sem=threading.Semaphore(2),
+                             stats=pipeline.Stats(), fail=lambda a, b, c: fails.append((a, b, c)),
+                             abort=abort)
+
+        assert abort.is_set(), "第一条就该拉起中止信号"
+        # 只有触发中止的那条记一次失败，后两条直接返回，不重复撞注册
+        assert len(fails) == 1, fails
+        assert fails[0][1] == "abort", fails
+
+
+def test_download_failure_is_reported():
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "src.mp4"
+        make_mp4(src)
+
+        def boom(session, urls, dest, **kw):
+            raise OSError("连接被重置")
+
+        recs, fails, up, work = run(tmp, src_mp4=src, patch_download=boom)
+        assert recs == [] and len(fails) == 1
+        assert fails[0][1] == "download" and "连接被重置" in fails[0][2], fails
+
+
+def test_resume_skips_completed():
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "src.mp4"
+        make_mp4(src)
+        run(tmp, src_mp4=src)
+        recs2, fails2, up2, _ = run(tmp, src_mp4=src)      # 再跑一次同一条
+        assert len(recs2) == 1, "重复跑不该写出第二行"
+        assert not fails2, fails2
+
+
+def test_sweep_removes_files_already_in_records():
+    """work/ 里凡是已经在 records.jsonl 里的文件都是废物：早就传上去了，
+    留着只会撑爆磁盘。实测一次 66411 条的批跑到 2.7% 就攒了 11 GB。
+    但**失败项的文件必须留下**——那是重跑续传的本钱，见
+    test_upload_failure_keeps_local_and_writes_no_record。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        work = Path(tmp) / "work"
+        work.mkdir()
+        recs = Path(tmp) / "records.jsonl"
+        recs.write_text(
+            json.dumps({"aweme_id": "111", "cos_mp4": "cos://x/111.mp4"}) + chr(10)
+            + json.dumps({"aweme_id": "222", "cos_mp4": "cos://x/222.mp4"}) + chr(10),
+            encoding="utf-8")
+        for name in ("111.mp4", "111.mp3", "222.mp4", "333.mp4", "333.mp3"):
+            (work / name).write_bytes(b"x" * 10)
+        (work / "444.mp4.part").write_bytes(b"x" * 10)
+
+        freed = pipeline.sweep_work_dir(work, recs)
+
+        left = sorted(p.name for p in work.iterdir())
+        assert left == ["333.mp3", "333.mp4"], left      # 333 没入库，留着重跑
+        # 111.mp4 + 111.mp3 + 222.mp4 已入库，加上 444.mp4.part
+        assert freed == 4, freed
+
+
+def test_sweep_tolerates_missing_paths():
+    with tempfile.TemporaryDirectory() as tmp:
+        work = Path(tmp) / "work"
+        assert pipeline.sweep_work_dir(work, Path(tmp) / "nope.jsonl") == 0
+        work.mkdir()
+        assert pipeline.sweep_work_dir(work, Path(tmp) / "nope.jsonl") == 0
+
+
+def test_plan_batch_splits_long_runs():
+    """长跑必须切段：单个进程跑到 1.5 万条上下，Windows 会话的桌面堆就被
+    几万个 ffmpeg/ffprobe 子进程耗光，之后 CreateProcess 一律失败
+    （rc=3221225794 / STATUS_DLL_INIT_FAILED）。表现是完成数冻结、失败数
+    猛涨、work/ 暴涨。实测两次：三进程并跑时撞过一次，单进程跑到 16399 条
+    又撞一次 —— 所以是累积量的问题，不是并发数。切段后每段跑完重启进程，
+    桌面堆跟着重置。"""
+    pending = [(str(i), f"u{i}") for i in range(100)]
+    batch, more = pipeline.plan_batch(pending, 30)
+    assert len(batch) == 30 and more is True
+    batch, more = pipeline.plan_batch(pending, 100)
+    assert len(batch) == 100 and more is False, "刚好跑完不该再重启"
+    batch, more = pipeline.plan_batch(pending, 500)
+    assert len(batch) == 100 and more is False
+    batch, more = pipeline.plan_batch(pending, 0)
+    assert len(batch) == 100 and more is False, "0 = 不切段"
+
+
+def test_next_abort_wait_backs_off():
+    """撞 IP 级注册限流后要等冷却再续跑。连着撞说明还没解封，等得越来越久；
+    一旦有一段跑成了就把等待打回底价。上限别太长，不然白白空转。"""
+    assert pipeline.next_abort_wait(0, base=1800, cap=14400) == 1800
+    assert pipeline.next_abort_wait(1, base=1800, cap=14400) == 3600
+    assert pipeline.next_abort_wait(2, base=1800, cap=14400) == 7200
+    assert pipeline.next_abort_wait(3, base=1800, cap=14400) == 14400
+    assert pipeline.next_abort_wait(9, base=1800, cap=14400) == 14400, "要封顶"
+    assert pipeline.next_abort_wait(0, base=0, cap=14400) == 0, "0=不等，直接退出"
+
+
+def main() -> int:
+    if not shutil.which("ffmpeg") or not shutil.which("ffprobe"):
+        print("跳过：需要 ffmpeg/ffprobe")
+        return 0
+    tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
+    failed = 0
+    for t in tests:
+        try:
+            t()
+            print(f"  ok   {t.__name__}")
+        except AssertionError as exc:
+            failed += 1
+            print(f"  FAIL {t.__name__}: {exc}")
+        except Exception as exc:
+            failed += 1
+            print(f"  ERR  {t.__name__}: {type(exc).__name__}: {exc}")
+    print(f"\n{len(tests) - failed}/{len(tests)} 通过")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_smoke.py

@@ -0,0 +1,77 @@
+"""不联网的冒烟测试：验证 crypto 正确性、链接解析、文件名清洗、去重记录。"""
+
+from __future__ import annotations
+
+# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
+# 把项目根加回来，import douyin / pipeline 才找得到。
+import sys as _sys
+from pathlib import Path as _Path
+_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
+
+import tempfile
+from pathlib import Path
+
+from douyin import Registry, extract_aweme_id, safe_filename
+from douyin.crypto import self_test
+from douyin.urls import ParseError
+
+
+def test_crypto():
+    self_test()
+
+
+def test_extract_id():
+    cases = {
+        "7123456789012345678": "7123456789012345678",
+        "https://www.douyin.com/video/7123456789012345678": "7123456789012345678",
+        "https://www.douyin.com/note/7123456789012345678": "7123456789012345678",
+        "https://www.douyin.com/?modal_id=7123456789012345678": "7123456789012345678",
+        "https://www.iesdouyin.com/share/video/7123456789012345678/": "7123456789012345678",
+        "看看这个 https://www.douyin.com/video/7123456789012345678?a=1 好玩":
+            "7123456789012345678",
+    }
+    for raw, want in cases.items():
+        got = extract_aweme_id(raw)
+        assert got == want, f"{raw!r} -> {got}, 期望 {want}"
+
+    for bad in ["随便一段没有链接的文字", "https://example.com/foo"]:
+        try:
+            extract_aweme_id(bad)
+        except ParseError:
+            pass
+        else:
+            raise AssertionError(f"{bad!r} 本应报 ParseError")
+    print("extract_aweme_id OK")
+
+
+def test_safe_filename():
+    assert safe_filename('a/b\\c:d*e?f"g<h>i|j') == "a_b_c_d_e_f_g_h_i_j"
+    assert safe_filename("   ") == "untitled"
+    assert len(safe_filename("x" * 200)) == 80
+    assert safe_filename("多个   空格") == "多个 空格"
+    print("safe_filename OK")
+
+
+def test_registry():
+    with tempfile.TemporaryDirectory() as d:
+        root = Path(d)
+        reg = Registry(root / ".downloaded.json")
+        f = root / "v.mp4"
+        f.write_bytes(b"x" * 10)
+        assert "111" not in reg
+        reg.add("111", f, 10, "标题")
+        assert "111" in reg
+        # 重新加载后仍在
+        assert "111" in Registry(root / ".downloaded.json")
+        # 文件被删则视为未下载
+        f.unlink()
+        assert "111" not in Registry(root / ".downloaded.json")
+    print("Registry OK")
+
+
+if __name__ == "__main__":
+    test_crypto()
+    test_extract_id()
+    test_safe_filename()
+    test_registry()
+    print("\n全部冒烟测试通过")