"""腾讯云 COS 上传,带上传后校验。 流水线传完就删本地文件,所以「传成功了吗」这个判断必须硬。 `upload_file` 不抛异常只说明请求没报错,不说明对象真的落到桶里、 大小也对。所以 put() 一律在上传后 head_object 取回 Content-Length 跟本地字节数比对,对不上就当失败——删除闸门卡在这个返回值上。 """ from __future__ import annotations import os import threading from pathlib import Path from typing import Optional DEFAULT_BUCKET = "asr-dig-data-1253181912" DEFAULT_PREFIX = "0005/zh/" class CosError(RuntimeError): pass class SizeMismatch(CosError): """对象传上去了,但远端大小和本地对不上。""" def object_url(key: str, bucket: str, region: str, style: str = "cos") -> str: if style == "cos": return f"cos://{bucket}/{key}" if style == "https": return f"https://{bucket}.cos.{region}.myqcloud.com/{key}" return f"/{bucket}/{key}" class CosUploader: """线程安全的上传器。同名对象一律覆盖,不做存在性检查。 重复劳动由上层的 records.jsonl 断点续跑拦掉(已完成的 id 整条跳过, 连下载都不会发生),所以这里不需要再靠 object_exists 去省事—— 少一个分支,行为也更好预测。 """ def __init__(self, region: str, bucket: str = DEFAULT_BUCKET, prefix: str = DEFAULT_PREFIX, secret_id: str = "", secret_key: str = "", url_style: str = "cos", scheme: str = "https") -> None: secret_id = secret_id or os.environ.get("COS_SECRET_ID", "") secret_key = secret_key or os.environ.get("COS_SECRET_KEY", "") if not secret_id or not secret_key: raise CosError( "缺 COS 凭证。设环境变量 COS_SECRET_ID / COS_SECRET_KEY," "别写在命令行里(会留在 shell history)。") from qcloud_cos import CosConfig, CosS3Client self.bucket = bucket self.region = region self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else "" self.url_style = url_style self._client = CosS3Client(CosConfig( Region=region, SecretId=secret_id, SecretKey=secret_key, Scheme=scheme)) self._lock = threading.Lock() def key_for(self, name: str) -> str: return f"{self.prefix}{name}" def url_for(self, key: str) -> str: return object_url(key, self.bucket, self.region, self.url_style) def remote_size(self, key: str) -> Optional[int]: try: return int(self._client.head_object(Bucket=self.bucket, Key=key)["Content-Length"]) except Exception: return None def put(self, local: Path, key: str) -> tuple[str, int]: """上传并校验。返回 (url, 字节数)。校验不过抛 SizeMismatch。""" local = Path(local) size = local.stat().st_size if size == 0: raise CosError(f"{local.name} 是空文件,拒绝上传") # EnableMD5 关掉是为了速度;正确性靠上传后的 head_object 校验兜底 self._client.upload_file( Bucket=self.bucket, Key=key, LocalFilePath=str(local), EnableMD5=False, PartSize=10, MAXThread=2, ) remote = self.remote_size(key) if remote is None: raise SizeMismatch(f"{key} 上传后 head_object 拿不到,判定失败") if remote != size: raise SizeMismatch(f"{key} 大小不符:COS={remote} 本地={size}") return self.url_for(key), size