首次提交
This commit is contained in:
0
tool/log/__init__.py
Normal file
0
tool/log/__init__.py
Normal file
218
tool/log/log_manager.py
Normal file
218
tool/log/log_manager.py
Normal file
@@ -0,0 +1,218 @@
|
||||
# log_manager.py
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
from collections import deque
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
|
||||
class ProgramFilter(logging.Filter):
|
||||
"""自定义日志过滤器，为日志记录添加程序名称属性"""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not hasattr(record, 'program'):
|
||||
record.program = "UNKNOWN"
|
||||
return True
|
||||
|
||||
|
||||
class UnifiedLogger:
|
||||
"""统一日志管理器（线程安全单例），支持多程序日志收集"""
|
||||
|
||||
_instance: Optional['UnifiedLogger'] = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __new__(cls) -> 'UnifiedLogger':
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def __init__(self) -> None:
|
||||
# 避免重复初始化
|
||||
if hasattr(self, '_initialized') and self._initialized:
|
||||
return
|
||||
|
||||
# 每视频状态使用线程本地存储：批处理多线程并发时，每个视频在各自线程内
|
||||
# 独立记录日志/计时/统计，互不串扰。（嵌套子线程如OCR并发worker不应直接
|
||||
# 调用本logger的状态方法——它们会落到worker线程的独立状态而丢失。）
|
||||
self._local = threading.local()
|
||||
|
||||
# 是否打印详细日志到控制台。默认 False：控制台只显示"视频→步骤"里程碑，
|
||||
# 详细日志仍完整存进每视频 JSON(log/<video>.json)。设 LOG_VERBOSE=1 恢复全部。
|
||||
self.verbose = os.environ.get("LOG_VERBOSE", "0") == "1"
|
||||
|
||||
# 自定义 logger（避免污染根日志器）
|
||||
self.logger = logging.getLogger("UnifiedLogger")
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
self.logger.propagate = False # 不传播到 root，避免每条日志重复打印
|
||||
self.logger.addFilter(ProgramFilter())
|
||||
|
||||
# 控制台 Handler：非 verbose 时只放行 WARNING+，把 INFO 细节挡在控制台外(仍进 JSON)
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.DEBUG if self.verbose else logging.WARNING)
|
||||
console_formatter = logging.Formatter(
|
||||
'%(asctime)s [%(levelname)s] [%(program)s] %(message)s'
|
||||
)
|
||||
console_handler.setFormatter(console_formatter)
|
||||
self.logger.addHandler(console_handler)
|
||||
|
||||
# 第三方库降噪：jieba 的 DEBUG("Building prefix dict...")太吵
|
||||
logging.getLogger("jieba").setLevel(logging.WARNING)
|
||||
# qcloud_cos SDK 噪音大: 每次 get/put/head object 都打一条 INFO(带完整url/headers),
|
||||
# 而且它把"对象不存在"(HEAD 404, 本项目里大量用来判断分片是否已完成/锁是否存在,
|
||||
# 属于正常控制流)也当 WARNING 打一条 —— 两种都没信息量, 真正的失败走异常
|
||||
# (CosServiceError), 本项目自己的 try/except 会记录, 不受这里调级影响。
|
||||
logging.getLogger("qcloud_cos").setLevel(logging.ERROR)
|
||||
|
||||
self._initialized = True
|
||||
|
||||
def step(self, video: str, step: str, extra: str = "") -> None:
|
||||
"""打印一条简洁的"视频 → 步骤"里程碑（始终显示，不受 verbose 影响）。"""
|
||||
line = f"[{datetime.now().strftime('%H:%M:%S')}] {video} → {step}"
|
||||
if extra:
|
||||
line += f" ({extra})"
|
||||
print(line, flush=True)
|
||||
try:
|
||||
self._state()["log_buffer"].append({
|
||||
"timestamp": datetime.now().isoformat(), "level": "STEP",
|
||||
"program": "step", "message": f"{step} {extra}".strip(),
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _state(self) -> dict:
|
||||
"""获取当前线程的日志状态，惰性初始化。"""
|
||||
st = getattr(self._local, 'state', None)
|
||||
if st is None:
|
||||
st = {
|
||||
"log_buffer": deque(maxlen=10000),
|
||||
"log_file": "unified_log.json",
|
||||
"program_sequence": [],
|
||||
"program_start_times": {},
|
||||
"program_end_times": {},
|
||||
"custom_stats": {},
|
||||
}
|
||||
self._local.state = st
|
||||
return st
|
||||
|
||||
def init_logging(self, log_file: str = "unified_log.json") -> None:
|
||||
"""初始化日志系统（可重新指定日志文件路径）"""
|
||||
self._state()["log_file"] = log_file
|
||||
log_dir = os.path.dirname(log_file)
|
||||
if log_dir:
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
def reset(self) -> None:
|
||||
"""重置当前线程的日志状态（用于多线程场景下每个视频独立记录）"""
|
||||
st = self._state()
|
||||
st["log_buffer"].clear()
|
||||
st["program_sequence"].clear()
|
||||
st["program_start_times"].clear()
|
||||
st["program_end_times"].clear()
|
||||
st["custom_stats"].clear()
|
||||
|
||||
def set_stat(self, key: str, value: object) -> None:
|
||||
"""记录一项自定义统计，会写入日志 metadata.custom_stats"""
|
||||
self._state()["custom_stats"][key] = value
|
||||
|
||||
def _get_level_int(self, level: Union[str, int]) -> int:
|
||||
"""将日志级别转换为 logging 模块的整数常量"""
|
||||
if isinstance(level, int):
|
||||
return level
|
||||
level_map = {
|
||||
"DEBUG": logging.DEBUG,
|
||||
"INFO": logging.INFO,
|
||||
"WARNING": logging.WARNING,
|
||||
"ERROR": logging.ERROR,
|
||||
"CRITICAL": logging.CRITICAL
|
||||
}
|
||||
return level_map.get(level.upper(), logging.INFO)
|
||||
|
||||
def log(self, message: str, level: Union[str, int] = "info", program: str = "A") -> None:
|
||||
"""记录一条日志"""
|
||||
timestamp = datetime.now().isoformat()
|
||||
level_str = level if isinstance(level, str) else logging.getLevelName(level)
|
||||
log_entry = {
|
||||
"timestamp": timestamp,
|
||||
"level": level_str,
|
||||
"program": program,
|
||||
"message": message
|
||||
}
|
||||
|
||||
self._state()["log_buffer"].append(log_entry)
|
||||
|
||||
# 输出到自定义 logger
|
||||
level_int = self._get_level_int(level)
|
||||
self.logger.log(level_int, message, extra={"program": program})
|
||||
|
||||
def start_program(self, program_name: str) -> None:
|
||||
"""记录程序开始执行"""
|
||||
st = self._state()
|
||||
if program_name in st["program_start_times"]:
|
||||
self.logger.warning(f"程序 {program_name} 已开始，忽略重复 start 调用", extra={"program": "SYSTEM"})
|
||||
return
|
||||
|
||||
st["program_start_times"][program_name] = time.time()
|
||||
st["program_sequence"].append(program_name)
|
||||
|
||||
self.log(f"程序 {program_name} 开始执行", program=program_name)
|
||||
|
||||
def end_program(self, program_name: str) -> None:
|
||||
"""记录程序执行结束"""
|
||||
st = self._state()
|
||||
if program_name not in st["program_start_times"]:
|
||||
self.logger.error(f"程序 {program_name} 未调用 start_program，无法记录结束", extra={"program": "SYSTEM"})
|
||||
return
|
||||
|
||||
end_time = time.time()
|
||||
st["program_end_times"][program_name] = end_time
|
||||
duration = end_time - st["program_start_times"][program_name]
|
||||
|
||||
self.log(f"程序 {program_name} 执行完成 (耗时: {duration:.2f}s)", program=program_name)
|
||||
|
||||
def save_logs(self) -> None:
|
||||
"""将当前线程的所有日志（含元数据）保存到 JSON 文件"""
|
||||
st = self._state()
|
||||
logs_copy = list(st["log_buffer"])
|
||||
start_times_copy = dict(st["program_start_times"])
|
||||
end_times_copy = dict(st["program_end_times"])
|
||||
sequence_copy = list(st["program_sequence"])
|
||||
stats_copy = dict(st["custom_stats"])
|
||||
log_file = st["log_file"]
|
||||
|
||||
# 计算程序执行耗时
|
||||
durations = {}
|
||||
for prog, start in start_times_copy.items():
|
||||
end = end_times_copy.get(prog, time.time())
|
||||
durations[prog] = end - start
|
||||
|
||||
metadata = {
|
||||
"log_created": datetime.now().isoformat(),
|
||||
"program_sequence": sequence_copy,
|
||||
"total_entries": len(logs_copy),
|
||||
"program_durations": durations,
|
||||
"custom_stats": stats_copy
|
||||
}
|
||||
|
||||
full_log = {
|
||||
"metadata": metadata,
|
||||
"logs": logs_copy
|
||||
}
|
||||
|
||||
try:
|
||||
with open(log_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(full_log, f, ensure_ascii=False, indent=2)
|
||||
self.logger.info(f"所有日志已保存到: {log_file}", extra={"program": "SYSTEM"})
|
||||
except (OSError, IOError) as e:
|
||||
self.logger.error(f"保存日志失败: {e}", extra={"program": "SYSTEM"})
|
||||
|
||||
def get_logs(self) -> List[dict]:
|
||||
"""获取当前线程缓冲区中的所有日志副本"""
|
||||
return list(self._state()["log_buffer"])
|
||||
|
||||
|
||||
# 全局单例实例
|
||||
logger = UnifiedLogger()
|
||||
Reference in New Issue
Block a user