初始提交:抖音视频批量采集流水线
App 端免签名接口(设备注册 + detail + CDN 直链),一体化流水线 下载→抽音→传COS→记录→删本地,AI 作品四档判定,四份文档: README(使用指南)、PROGRESS(批次进度)、RE(逆向全过程)、AI(判定规则)。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
94
douyin/cos.py
Normal file
94
douyin/cos.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""腾讯云 COS 上传,带上传后校验。
|
||||
|
||||
流水线传完就删本地文件,所以「传成功了吗」这个判断必须硬。
|
||||
`upload_file` 不抛异常只说明请求没报错,不说明对象真的落到桶里、
|
||||
大小也对。所以 put() 一律在上传后 head_object 取回 Content-Length
|
||||
跟本地字节数比对,对不上就当失败——删除闸门卡在这个返回值上。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
DEFAULT_BUCKET = "asr-dig-data-1253181912"
|
||||
DEFAULT_PREFIX = "0005/zh/"
|
||||
|
||||
|
||||
class CosError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class SizeMismatch(CosError):
|
||||
"""对象传上去了,但远端大小和本地对不上。"""
|
||||
|
||||
|
||||
def object_url(key: str, bucket: str, region: str, style: str = "cos") -> str:
|
||||
if style == "cos":
|
||||
return f"cos://{bucket}/{key}"
|
||||
if style == "https":
|
||||
return f"https://{bucket}.cos.{region}.myqcloud.com/{key}"
|
||||
return f"/{bucket}/{key}"
|
||||
|
||||
|
||||
class CosUploader:
|
||||
"""线程安全的上传器。同名对象一律覆盖,不做存在性检查。
|
||||
|
||||
重复劳动由上层的 records.jsonl 断点续跑拦掉(已完成的 id 整条跳过,
|
||||
连下载都不会发生),所以这里不需要再靠 object_exists 去省事——
|
||||
少一个分支,行为也更好预测。
|
||||
"""
|
||||
|
||||
def __init__(self, region: str, bucket: str = DEFAULT_BUCKET,
|
||||
prefix: str = DEFAULT_PREFIX,
|
||||
secret_id: str = "", secret_key: str = "",
|
||||
url_style: str = "cos", scheme: str = "https") -> None:
|
||||
secret_id = secret_id or os.environ.get("COS_SECRET_ID", "")
|
||||
secret_key = secret_key or os.environ.get("COS_SECRET_KEY", "")
|
||||
if not secret_id or not secret_key:
|
||||
raise CosError(
|
||||
"缺 COS 凭证。设环境变量 COS_SECRET_ID / COS_SECRET_KEY,"
|
||||
"别写在命令行里(会留在 shell history)。")
|
||||
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
self.bucket = bucket
|
||||
self.region = region
|
||||
self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else ""
|
||||
self.url_style = url_style
|
||||
self._client = CosS3Client(CosConfig(
|
||||
Region=region, SecretId=secret_id, SecretKey=secret_key, Scheme=scheme))
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def key_for(self, name: str) -> str:
|
||||
return f"{self.prefix}{name}"
|
||||
|
||||
def url_for(self, key: str) -> str:
|
||||
return object_url(key, self.bucket, self.region, self.url_style)
|
||||
|
||||
def remote_size(self, key: str) -> Optional[int]:
|
||||
try:
|
||||
return int(self._client.head_object(Bucket=self.bucket, Key=key)["Content-Length"])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def put(self, local: Path, key: str) -> tuple[str, int]:
|
||||
"""上传并校验。返回 (url, 字节数)。校验不过抛 SizeMismatch。"""
|
||||
local = Path(local)
|
||||
size = local.stat().st_size
|
||||
if size == 0:
|
||||
raise CosError(f"{local.name} 是空文件,拒绝上传")
|
||||
|
||||
# EnableMD5 关掉是为了速度;正确性靠上传后的 head_object 校验兜底
|
||||
self._client.upload_file(
|
||||
Bucket=self.bucket, Key=key, LocalFilePath=str(local),
|
||||
EnableMD5=False, PartSize=10, MAXThread=2,
|
||||
)
|
||||
|
||||
remote = self.remote_size(key)
|
||||
if remote is None:
|
||||
raise SizeMismatch(f"{key} 上传后 head_object 拿不到,判定失败")
|
||||
if remote != size:
|
||||
raise SizeMismatch(f"{key} 大小不符:COS={remote} 本地={size}")
|
||||
return self.url_for(key), size
|
||||
Reference in New Issue
Block a user