初始提交：抖音视频批量采集流水线
App 端免签名接口（设备注册 + detail + CDN 直链），一体化流水线 下载→抽音→传COS→记录→删本地，AI 作品四档判定，四份文档： README（使用指南）、PROGRESS（批次进度）、RE（逆向全过程）、AI（判定规则）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
174
scripts/clean_ai.py
Normal file
174
scripts/clean_ai.py
Normal file
@@ -0,0 +1,174 @@
|
||||
"""从抖音链接 jsonl 里筛出 AI 相关的作品。
|
||||
|
||||
判定逻辑在 douyin/aiclass.py，分 platform / strong / topic / author 四档，
|
||||
这里只负责流式读写、统计和落盘。
|
||||
|
||||
输出：
|
||||
*.txt 纯链接一行一个，直接喂给 pipeline.py --from-file
|
||||
*.jsonl 原记录 + ai_tier + ai_signals，字段全留着，方便复核和再切。
|
||||
这份体积接近源文件（实测 13967 条就有 169MB），而且随时能重跑
|
||||
生成，所以默认**不写**，要复核再加 --with-jsonl。
|
||||
|
||||
用法:
|
||||
python clean_ai.py -i 抖音链接20260827_40087.jsonl --stats
|
||||
python clean_ai.py -i 抖音链接20260827_40087.jsonl --tier platform,strong -o ai
|
||||
python clean_ai.py -i 抖音链接20260827_40087.jsonl --tier platform,strong -o ai --with-jsonl
|
||||
python clean_ai.py -i xxx.jsonl --tier platform,strong --exclude-forbidden -o ai
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
|
||||
# 把项目根加回来，import douyin / pipeline 才找得到。
|
||||
import sys as _sys
|
||||
from pathlib import Path as _Path
|
||||
_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from douyin.aiclass import TIERS, classify
|
||||
|
||||
|
||||
def iter_jsonl(path: Path):
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||||
for n, line in enumerate(f, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
yield n, json.loads(line)
|
||||
except ValueError:
|
||||
print(f"[跳过] 第 {n} 行不是合法 JSON", file=sys.stderr)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="筛出 AI 相关作品")
|
||||
ap.add_argument("-i", "--input", required=True)
|
||||
ap.add_argument("-o", "--out", default="", help="输出前缀，生成 <前缀>.jsonl 和 <前缀>.txt")
|
||||
ap.add_argument("--tier", default="platform,strong",
|
||||
help=f"保留哪些档，逗号分隔。可选 {'/'.join(TIERS)}")
|
||||
ap.add_argument("--exclude-forbidden", action="store_true",
|
||||
help="剔掉 allow_download=false 的（抖音对 AIGC 常给 aigc_forbidden）")
|
||||
ap.add_argument("--with-jsonl", action="store_true",
|
||||
help="连带写出 <前缀>.jsonl（体积接近源文件，默认不写）")
|
||||
ap.add_argument("--platform-needs-text", action="store_true",
|
||||
help="platform 档要求文案/标签也有 AI 信号才算。抖音给 AI 配音、"
|
||||
"AI 封面的普通带货视频也打 AIGC 标，露营那批 87110 条里"
|
||||
"有 1288 条是这种，占 platform 档的一半")
|
||||
ap.add_argument("--stats", action="store_true", help="只统计不落盘")
|
||||
ap.add_argument("--samples", type=int, default=0, help="每档打印几条样例到 stderr")
|
||||
args = ap.parse_args()
|
||||
|
||||
keep = {t.strip() for t in args.tier.split(",") if t.strip()}
|
||||
bad = keep - set(TIERS)
|
||||
if bad:
|
||||
ap.error(f"未知档位 {sorted(bad)}，可选 {TIERS}")
|
||||
|
||||
src = Path(args.input)
|
||||
if not src.exists():
|
||||
ap.error(f"找不到 {src}")
|
||||
|
||||
tiers = Counter()
|
||||
reasons = Counter()
|
||||
sigs = Counter()
|
||||
dl = Counter()
|
||||
dur_ms = 0
|
||||
kept = 0
|
||||
seen = set()
|
||||
dup = 0
|
||||
samples = {t: [] for t in TIERS}
|
||||
|
||||
out_j = out_t = None
|
||||
if not args.stats:
|
||||
if not args.out:
|
||||
ap.error("要落盘就给 -o 前缀，或者加 --stats 只看统计")
|
||||
out_t = open(f"{args.out}.txt", "w", encoding="utf-8")
|
||||
out_j = open(f"{args.out}.jsonl", "w", encoding="utf-8") if args.with_jsonl else None
|
||||
|
||||
total = 0
|
||||
for _, r in iter_jsonl(src):
|
||||
total += 1
|
||||
aid = str(r.get("aweme_id") or "")
|
||||
if aid in seen:
|
||||
dup += 1
|
||||
continue
|
||||
seen.add(aid)
|
||||
|
||||
v = classify(r)
|
||||
t = v["tier"]
|
||||
if t == "platform" and args.platform_needs_text:
|
||||
# 平台标了 AIGC，但作品自己一个 AI 信号都没有 -> 多半是 AI 配音/封面。
|
||||
# 作者昵称带 AI（@可灵AI、@松饼大人Ai剧）也算佐证 —— 是本人在做 AI，
|
||||
# 不像 music 那条路会被 `@X创作的原声` 和 studio/Fluxxwave 污染。
|
||||
if not any(s.startswith(("desc:", "author:")) for s in v["signals"]):
|
||||
t = "none"
|
||||
reasons["platform 但无文案/作者佐证，被 --platform-needs-text 剔除"] += 1
|
||||
tiers[t] += 1
|
||||
if args.samples and len(samples[t]) < args.samples:
|
||||
samples[t].append((aid, (r.get("desc") or "").replace("\n", " ")[:90],
|
||||
"|".join(v["signals"][:5])))
|
||||
if t == "none":
|
||||
continue
|
||||
for s in v["signals"]:
|
||||
sigs[s.split(":", 1)[0] + ":" + s.split(":")[1] if ":" in s else s] += 1
|
||||
|
||||
if t not in keep:
|
||||
continue
|
||||
|
||||
ai = r.get("ai") or {}
|
||||
if not ai.get("allow_download", True):
|
||||
dl[ai.get("download_fail_reason") or "unknown"] += 1
|
||||
if args.exclude_forbidden:
|
||||
reasons["被 allow_download=false 剔除"] += 1
|
||||
continue
|
||||
|
||||
kept += 1
|
||||
dur_ms += int(r.get("duration_ms") or 0)
|
||||
if out_t:
|
||||
out_t.write((r.get("url") or f"https://www.douyin.com/video/{aid}") + "\n")
|
||||
if out_j:
|
||||
rec = {"aweme_id": aid, "ai_tier": t, "ai_signals": v["signals"]}
|
||||
rec.update({k: val for k, val in r.items() if k != "aweme_id"})
|
||||
out_j.write(json.dumps(rec, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
|
||||
if out_t:
|
||||
out_t.close()
|
||||
if out_j:
|
||||
out_j.close()
|
||||
|
||||
w = sys.stderr
|
||||
print(f"读入 {total} 条，去重后 {len(seen)} 条（重复 {dup}）", file=w)
|
||||
print("\n档位分布:", file=w)
|
||||
for t in TIERS:
|
||||
mark = " <- 保留" if t in keep else ""
|
||||
print(f" {t:9s} {tiers[t]:7d}{mark}", file=w)
|
||||
ai_any = sum(tiers[t] for t in TIERS if t != "none")
|
||||
print(f" {'AI 相关合计':9s} {ai_any:7d} ({ai_any / max(total,1):.1%})", file=w)
|
||||
|
||||
if dl:
|
||||
print("\n保留项里 allow_download=false 的原因:", file=w)
|
||||
for k, v2 in dl.most_common():
|
||||
print(f" {k:24s} {v2:7d}", file=w)
|
||||
for k, v2 in reasons.items():
|
||||
print(f" {k}: {v2}", file=w)
|
||||
|
||||
print(f"\n最终保留 {kept} 条，总时长 {dur_ms/3600000:.2f} 小时", file=w)
|
||||
if out_t:
|
||||
print(f"写出 {args.out}.txt" + (f" 和 {args.out}.jsonl" if out_j else ""), file=w)
|
||||
|
||||
if args.samples:
|
||||
for t in TIERS:
|
||||
if not samples[t]:
|
||||
continue
|
||||
print(f"\n----- {t} 样例 -----", file=w)
|
||||
for aid, d, s in samples[t]:
|
||||
print(f" {aid} {d}\n 信号: {s}", file=w)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
228
scripts/export_sample_xlsx.py
Normal file
228
scripts/export_sample_xlsx.py
Normal file
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python
|
||||
"""把 records.jsonl 导成交付用的样本 xlsx。
|
||||
|
||||
列是甲方给的字段表（序号跟着他们的编号走，中间 4/5/9 他们没要）：
|
||||
|
||||
1 样本 ID 必填 aweme_id
|
||||
2 视频标题 必填 title，为空时退回 desc 首行
|
||||
3 平台 必填 固定「抖音」
|
||||
6 生成方式 AI 生成 / AI 辅助 / 疑似 AI / 未标注
|
||||
7 生成工具 / 模型 从文案标签里认出来的工具名，逗号分隔
|
||||
8 题材 / 内容类别 按标签归类
|
||||
10 分辨率 / 清晰度 必填 1024x576（540p）
|
||||
11 发布时间 必填 2026-08-06 14:12:47
|
||||
12 是否有 AI 标识/水印 必填 抖音 risk_infos 打的标，原文照抄
|
||||
|
||||
用法:
|
||||
python export_sample_xlsx.py # 全量
|
||||
python export_sample_xlsx.py --ai-only # 只要判定为 AI 的
|
||||
python export_sample_xlsx.py --limit 500 -o 样本.xlsx
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
|
||||
# 把项目根加回来，import douyin / pipeline 才找得到。
|
||||
import sys as _sys
|
||||
from pathlib import Path as _Path
|
||||
_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from douyin.aiclass import _fields, _scan, classify
|
||||
|
||||
# risk_infos.type 里属于 AIGC 的那几个码，别的都是跟 AI 无关的免责声明
|
||||
# （45=虚构演绎、336/351=个人观点、54=来源于网络…）。见 README §4.5。
|
||||
_AIGC_TYPES = {44, 47, 76, 328}
|
||||
|
||||
# 生成方式：抖音标签的措辞本身就分了轻重，直接用它，别自己发明
|
||||
_HOW_BY_TYPE = {
|
||||
44: "AI 生成（作者声明）",
|
||||
76: "AI 生成（平台判定）",
|
||||
328: "AI 生成（平台判定）",
|
||||
47: "疑似 AI 生成（平台检测）",
|
||||
}
|
||||
|
||||
# 题材归类。**顺序即优先级**，第一个命中的赢 —— 一条片子既是「漫剧」又带
|
||||
# 「剧情」很常见，先匹配到的那个更具体。
|
||||
#
|
||||
# 词要挑「只在这个题材里出现」的。踩过的坑：「古风/国风/画质」放进
|
||||
# 写真/摄影，结果把一整个仙侠短片系列（169/300 条）全吸过去了 ——
|
||||
# 那些是叙事片，只是标签带古风。宽词一律不要，宁可落进「其他」。
|
||||
_GENRES = [
|
||||
("漫剧 / 动画", "漫剧|动漫|动画|二次元|国漫|番剧|漫画"),
|
||||
("短剧 / 剧情", "短剧|剧情|微短剧|电影|影视|短片|反转|结局|"
|
||||
"第[一二三四五六七八九十\\d]+集|合集"),
|
||||
("教程 / 科普", "教程|教学|技巧|入门|保姆级|科普|干货|工作流|提示词"),
|
||||
("资讯 / 新闻", "新闻|资讯|报道|快讯|媒体精选|发布会|财经|股市"),
|
||||
("带货 / 电商", "好物|同款|下单|优惠|源头工厂|旗舰店|带货|种草|清仓"),
|
||||
("户外 / 露营", "露营|帐篷|户外|野营|钓鱼|自驾|徒步|登山"),
|
||||
("音乐 / 舞蹈", "音乐|歌曲|翻唱|舞蹈|跳舞"),
|
||||
("写真 / 摄影", "写真|摄影|人像|壁纸|头像"),
|
||||
("游戏", "手游|端游|原神|王者荣耀|电竞|无畏契约|崩坏"),
|
||||
("生活 / 日常", "日常|vlog|美食|旅行|萌宠|搞笑"),
|
||||
]
|
||||
_GENRE_RE = [(name, re.compile(pat, re.I)) for name, pat in _GENRES]
|
||||
|
||||
|
||||
def _risk(rec: dict) -> tuple:
|
||||
"""(type, content)。抖音把 AI 标识放在 raw.risk_infos 里。"""
|
||||
ri = (rec.get("raw") or {}).get("risk_infos") or {}
|
||||
if not isinstance(ri, dict):
|
||||
return None, ""
|
||||
return ri.get("type"), (ri.get("content") or "").strip()
|
||||
|
||||
|
||||
def how_generated(rec: dict, verdict: dict) -> str:
|
||||
t, _ = _risk(rec)
|
||||
if t in _HOW_BY_TYPE:
|
||||
return _HOW_BY_TYPE[t]
|
||||
if verdict["tier"] == "strong":
|
||||
return "AI 生成（作品自述）"
|
||||
if verdict["tier"] in ("topic", "author"):
|
||||
return "疑似 AI 相关"
|
||||
return "未标注"
|
||||
|
||||
|
||||
def tools_used(verdict: dict) -> str:
|
||||
"""从判定信号里把工具名摘出来。信号形如 `desc:工具:即梦`。"""
|
||||
names = []
|
||||
for s in verdict["signals"]:
|
||||
parts = s.split(":")
|
||||
if len(parts) >= 3 and parts[1] in ("工具", "漫剧"):
|
||||
n = parts[2]
|
||||
if n not in names:
|
||||
names.append(n)
|
||||
return "、".join(names)
|
||||
|
||||
|
||||
def genre(rec: dict) -> str:
|
||||
text = " ".join([rec.get("title") or "", rec.get("desc") or ""]
|
||||
+ list(rec.get("tags") or []))
|
||||
for name, rx in _GENRE_RE:
|
||||
if rx.search(text):
|
||||
return name
|
||||
return "其他"
|
||||
|
||||
|
||||
def resolution(rec: dict) -> str:
|
||||
v = rec.get("video") or {}
|
||||
w, h = v.get("width"), v.get("height")
|
||||
ratio = ((rec.get("raw") or {}).get("video") or {}).get("ratio") or ""
|
||||
if w and h:
|
||||
return f"{w}x{h}" + (f"（{ratio}）" if ratio else "")
|
||||
return ratio or ""
|
||||
|
||||
|
||||
def ai_mark(rec: dict) -> str:
|
||||
"""是否有 AI 标识/水印 —— 抖音的原文照抄，没有就写「无」。
|
||||
只认 _AIGC_TYPES；45/336/351 那些是虚构演绎、个人观点，跟 AI 无关。"""
|
||||
t, content = _risk(rec)
|
||||
if t in _AIGC_TYPES and content:
|
||||
return f"有：{content}"
|
||||
return "无"
|
||||
|
||||
|
||||
def title_of(rec: dict) -> str:
|
||||
"""标题 + 标签拼一格。
|
||||
|
||||
records.jsonl 里的 title 是 split_desc 把标签摘掉之后的正文，标签单独放
|
||||
在 tags 里。甲方只要一列，所以拼回去 —— 顺序按原文案的习惯：正文在前，
|
||||
`#标签` 跟在后面。
|
||||
"""
|
||||
t = (rec.get("title") or "").strip()
|
||||
if not t:
|
||||
d = (rec.get("desc") or "").strip().splitlines()
|
||||
t = d[0][:120] if d else ""
|
||||
tags = " ".join(f"#{x}" for x in (rec.get("tags") or []) if x)
|
||||
return (t + " " + tags).strip() if tags else t
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="records.jsonl -> 交付样本 xlsx")
|
||||
ap.add_argument("-i", "--input", default="records/records.jsonl")
|
||||
ap.add_argument("-o", "--out", default="exports/AI视频样本.xlsx")
|
||||
ap.add_argument("--ai-only", action="store_true",
|
||||
help="只导出判定为 AI 的（platform/strong 档）")
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
try:
|
||||
from openpyxl import Workbook
|
||||
except ImportError:
|
||||
print("需要 openpyxl：pip install openpyxl", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
src = Path(args.input)
|
||||
if not src.exists():
|
||||
ap.error(f"找不到 {src}")
|
||||
|
||||
# write_only：11 万行常规模式会把内存吃光
|
||||
wb = Workbook(write_only=True)
|
||||
ws = wb.create_sheet("样本")
|
||||
# 前 9 列是甲方字段表要的，后面几列是核对/取数用的：题材判得对不对，
|
||||
# 光看标题看不出来，得有标签和链接才能点开验；COS 路径是拿原始文件的地址。
|
||||
# 作者插在「平台」后面 —— 甲方编号 1/2/3 的三列保持原位原序，
|
||||
# 作者正好落进他们跳过的 4/5 那两个空号里。
|
||||
ws.append(["样本 ID", "视频标题", "平台", "作者",
|
||||
"生成方式", "生成工具 / 模型",
|
||||
"题材 / 内容类别", "分辨率 / 清晰度", "发布时间",
|
||||
"是否有 AI 标识 / 水印",
|
||||
"时长(秒)", "视频链接",
|
||||
"COS 视频路径", "COS 音频路径"])
|
||||
|
||||
n = written = bad = 0
|
||||
with open(src, "r", encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
n += 1
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except ValueError:
|
||||
bad += 1
|
||||
continue
|
||||
v = classify(rec)
|
||||
if args.ai_only and v["tier"] not in ("platform", "strong"):
|
||||
continue
|
||||
# 每格都过一遍 str()：openpyxl 会把 None 原样写成 "None"
|
||||
ws.append([
|
||||
str(rec.get("aweme_id") or ""),
|
||||
title_of(rec),
|
||||
"抖音",
|
||||
(rec.get("author") or {}).get("nickname") or "",
|
||||
# 生成方式 / 生成工具：留空，人工填。
|
||||
# 自动推的那版（how_generated / tools_used）只能看抖音的标和
|
||||
# 文案里写没写工具名，作者不写就抓不到 —— 实测 44.8% 的工具
|
||||
# 列是空的，而"生成方式"里 81.8% 全都压在"作者声明"一个值上，
|
||||
# 区分度太低，不如空着让人填。两个函数留着，要用加回来即可。
|
||||
"",
|
||||
"",
|
||||
genre(rec),
|
||||
resolution(rec),
|
||||
rec.get("create_time_str") or "",
|
||||
ai_mark(rec),
|
||||
round(float(rec.get("duration_sec") or 0), 1),
|
||||
rec.get("source_url")
|
||||
or f"https://www.douyin.com/video/{rec.get('aweme_id')}",
|
||||
rec.get("cos_mp4") or "",
|
||||
rec.get("cos_mp3") or "",
|
||||
])
|
||||
written += 1
|
||||
if args.limit and written >= args.limit:
|
||||
break
|
||||
if written % 20000 == 0:
|
||||
print(f" 已写 {written} 行…", flush=True)
|
||||
|
||||
wb.save(args.out)
|
||||
print(f"读入 {n} 行（坏行 {bad}），写出 {written} 行 -> {args.out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
222
scripts/jsonl_to_xlsx.py
Normal file
222
scripts/jsonl_to_xlsx.py
Normal file
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""把 jsonl 清洗并导出为 xlsx。
|
||||
|
||||
同时认两种输入：
|
||||
|
||||
records.jsonl 流水线产物，字段全（标题/作者/标签/时长/COS 地址/raw）
|
||||
manifest.jsonl 老格式，只有 cos_mp4 / cos_mp3 / duration
|
||||
|
||||
records.jsonl 内嵌了 raw，单行可能几十 KB、整个文件几百 MB，所以这里
|
||||
一律逐行流式读，任何时候都不把整个文件读进内存。选中的列也只留需要的
|
||||
那几个字段，raw 不会进内存驻留。
|
||||
|
||||
清洗动作（每一项都会报告剔除了多少）：
|
||||
1. 丢弃无法解析的行（被 Ctrl-C 打断可能留下半行）
|
||||
2. 丢弃缺字段 / 字段类型不对的行
|
||||
3. 丢弃时长非正数的行
|
||||
4. 丢弃 URL 前缀不符合预期的行
|
||||
5. 按 mp4 地址去重，保留首次出现
|
||||
|
||||
用法:
|
||||
python jsonl_to_xlsx.py
|
||||
python jsonl_to_xlsx.py -i records.jsonl -o "抖音-四川话.xlsx"
|
||||
python jsonl_to_xlsx.py -i records.jsonl --columns cos_mp4,cos_mp3,duration_sec
|
||||
python jsonl_to_xlsx.py --list-columns
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# 移到 scripts/ tests/ 后，直接运行时脚本目录成了 sys.path[0]，
|
||||
# 把项目根加回来，import douyin / pipeline 才找得到。
|
||||
import sys as _sys
|
||||
from pathlib import Path as _Path
|
||||
_sys.path.insert(0, str(_Path(__file__).resolve().parent.parent))
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List
|
||||
|
||||
EXPECTED_PREFIX = "cos://asr-dig-data-1253181912/0005/zh/"
|
||||
|
||||
# 列名 -> (表头, 取值函数)。取值函数只碰需要的字段，raw 不会被带进来。
|
||||
COLUMNS: Dict[str, tuple] = {
|
||||
"aweme_id": ("视频ID", lambda r: str(r.get("aweme_id") or "")),
|
||||
"title": ("标题", lambda r: r.get("title") or ""),
|
||||
"desc": ("原始描述", lambda r: r.get("desc") or ""),
|
||||
"tags": ("标签", lambda r: " ".join(r.get("tags") or [])),
|
||||
"author": ("作者", lambda r: (r.get("author") or {}).get("nickname", "")),
|
||||
"author_uid": ("作者UID", lambda r: (r.get("author") or {}).get("uid", "")),
|
||||
"create_time": ("发布时间", lambda r: r.get("create_time_str") or ""),
|
||||
"duration_sec": ("时长(秒)", _dur := (lambda r: r.get("duration_sec",
|
||||
r.get("duration")))),
|
||||
"duration": ("时长(秒)", _dur),
|
||||
"cos_mp4": ("cosmp4地址", lambda r: r.get("cos_mp4") or ""),
|
||||
"cos_mp3": ("cosmp3地址", lambda r: r.get("cos_mp3") or ""),
|
||||
"size_mp4": ("mp4字节数", lambda r: r.get("size_mp4") or 0),
|
||||
"size_mp3": ("mp3字节数", lambda r: r.get("size_mp3") or 0),
|
||||
"play": ("播放数", lambda r: (r.get("stats") or {}).get("play", 0)),
|
||||
"digg": ("点赞数", lambda r: (r.get("stats") or {}).get("digg", 0)),
|
||||
"comment": ("评论数", lambda r: (r.get("stats") or {}).get("comment", 0)),
|
||||
"source_url": ("原始链接", lambda r: r.get("source_url") or ""),
|
||||
}
|
||||
|
||||
DEFAULT_COLUMNS = ["cos_mp4", "cos_mp3", "duration_sec"]
|
||||
FULL_COLUMNS = ["aweme_id", "title", "author", "create_time", "tags",
|
||||
"duration_sec", "cos_mp4", "cos_mp3"]
|
||||
|
||||
NUMERIC = {"duration_sec", "duration", "size_mp4", "size_mp3",
|
||||
"play", "digg", "comment"}
|
||||
|
||||
|
||||
def iter_lines(path: Path) -> Iterator[str]:
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
yield line
|
||||
|
||||
|
||||
def clean(path: Path, prefix: str, columns: List[str]):
|
||||
"""流式清洗，只保留选中的列。"""
|
||||
rows: List[List[Any]] = []
|
||||
seen: set = set()
|
||||
dropped = {"解析失败": 0, "字段缺失": 0, "时长非法": 0, "地址异常": 0, "重复": 0}
|
||||
total = 0
|
||||
dur_idx = next((i for i, c in enumerate(columns)
|
||||
if c in ("duration_sec", "duration")), None)
|
||||
|
||||
for line in iter_lines(path):
|
||||
total += 1
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except ValueError:
|
||||
dropped["解析失败"] += 1
|
||||
continue
|
||||
|
||||
mp4, mp3 = r.get("cos_mp4"), r.get("cos_mp3")
|
||||
dur = r.get("duration_sec", r.get("duration"))
|
||||
if not isinstance(mp4, str) or not isinstance(mp3, str) or dur is None:
|
||||
dropped["字段缺失"] += 1
|
||||
continue
|
||||
if not isinstance(dur, (int, float)) or isinstance(dur, bool) or dur <= 0:
|
||||
dropped["时长非法"] += 1
|
||||
continue
|
||||
if not mp4.startswith(prefix) or not mp3.startswith(prefix):
|
||||
dropped["地址异常"] += 1
|
||||
continue
|
||||
if mp4 in seen:
|
||||
dropped["重复"] += 1
|
||||
continue
|
||||
|
||||
seen.add(mp4)
|
||||
row = [COLUMNS[c][1](r) for c in columns]
|
||||
if dur_idx is not None:
|
||||
row[dur_idx] = float(row[dur_idx])
|
||||
rows.append(row)
|
||||
|
||||
return rows, dropped, total, dur_idx
|
||||
|
||||
|
||||
def write_xlsx(rows, headers, out: Path, sheet: str, dur_idx) -> None:
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = sheet
|
||||
|
||||
ws.append(headers)
|
||||
head_font = Font(bold=True, color="FFFFFF")
|
||||
head_fill = PatternFill("solid", fgColor="4472C4")
|
||||
for c in range(1, len(headers) + 1):
|
||||
cell = ws.cell(row=1, column=c)
|
||||
cell.font = head_font
|
||||
cell.fill = head_fill
|
||||
cell.alignment = Alignment(horizontal="center", vertical="center")
|
||||
|
||||
for r in rows:
|
||||
ws.append(r)
|
||||
|
||||
# 时长保留 3 位小数，与 ffprobe 输出精度一致
|
||||
if dur_idx is not None:
|
||||
for row in ws.iter_rows(min_row=2, min_col=dur_idx + 1, max_col=dur_idx + 1):
|
||||
for cell in row:
|
||||
cell.number_format = "0.000"
|
||||
|
||||
for i, head in enumerate(headers):
|
||||
widest = max((len(str(r[i])) for r in rows), default=10)
|
||||
width = min(max(len(head) + 4, widest + 2), 60)
|
||||
ws.column_dimensions[get_column_letter(i + 1)].width = width
|
||||
|
||||
ws.freeze_panes = "A2" # 表头固定
|
||||
ws.auto_filter.ref = ws.dimensions # 开筛选
|
||||
wb.save(out)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="jsonl -> xlsx")
|
||||
ap.add_argument("-i", "--input", default="records/records.jsonl")
|
||||
ap.add_argument("-o", "--output", default="exports/抖音-四川话.xlsx")
|
||||
ap.add_argument("--sheet", default="抖音-四川话")
|
||||
ap.add_argument("--prefix", default=EXPECTED_PREFIX)
|
||||
ap.add_argument("--columns", default="",
|
||||
help=f"逗号分隔的列，默认 {','.join(DEFAULT_COLUMNS)}；"
|
||||
"用 full 展开为常用全字段")
|
||||
ap.add_argument("--list-columns", action="store_true", help="列出可用列名后退出")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.list_columns:
|
||||
print("可用列：")
|
||||
for k, (head, _) in COLUMNS.items():
|
||||
print(f" {k:<14} {head}")
|
||||
return 0
|
||||
|
||||
if not args.columns:
|
||||
columns = list(DEFAULT_COLUMNS)
|
||||
elif args.columns.strip() == "full":
|
||||
columns = list(FULL_COLUMNS)
|
||||
else:
|
||||
columns = [c.strip() for c in args.columns.split(",") if c.strip()]
|
||||
bad = [c for c in columns if c not in COLUMNS]
|
||||
if bad:
|
||||
print(f"[错误] 未知列名 {bad}，用 --list-columns 看可选项", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
src = Path(args.input)
|
||||
if not src.exists():
|
||||
print(f"[错误] 找不到 {src}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
rows, dropped, total, dur_idx = clean(src, args.prefix, columns)
|
||||
|
||||
print(f"读入 {total} 行")
|
||||
if sum(dropped.values()):
|
||||
for k, v in dropped.items():
|
||||
if v:
|
||||
print(f" 剔除 {k}: {v}")
|
||||
else:
|
||||
print(" 清洗检查全部通过，无需剔除")
|
||||
print(f"保留 {len(rows)} 行")
|
||||
|
||||
if not rows:
|
||||
print("[错误] 清洗后没有数据", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
headers = [COLUMNS[c][0] for c in columns]
|
||||
out = Path(args.output)
|
||||
write_xlsx(rows, headers, out, args.sheet, dur_idx)
|
||||
|
||||
print(f"\n已写入 {out}")
|
||||
print(f" {len(rows)} 行 × {len(headers)} 列：{' | '.join(headers)}")
|
||||
if dur_idx is not None:
|
||||
ds = [r[dur_idx] for r in rows]
|
||||
print(f" 总时长 {sum(ds) / 3600:.2f} 小时，平均 {sum(ds) / len(ds):.1f} 秒，"
|
||||
f"最短 {min(ds):.3f}，最长 {max(ds):.3f}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user