165 lines
7.0 KiB
Bash
165 lines
7.0 KiB
Bash
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# 一键启动(防 SIGHUP/外部kill 脱离终端 + 调好参数 + 启动前自检 + 防重复启动)
|
|
#
|
|
# 两种模式(MODE 环境变量切换):
|
|
# MODE=worker (默认) —— 常驻 worker, 从 COS 自动认领任务(run_worker.py)。不需要本地 text2。
|
|
# MODE=local —— 旧模式, 处理本地 ./text2 下的任务txt(main.py)。
|
|
#
|
|
# 用法:
|
|
# bash scripts/启动.sh # 默认 worker 模式启动
|
|
# bash scripts/启动.sh 32 # 临时把 max-workers 改成 32
|
|
# MODE=local bash scripts/启动.sh # 用旧的本地 text2 模式
|
|
# MAX_WORKERS=20 MEM_GUARD=40 bash scripts/启动.sh
|
|
#
|
|
# 停止: bash scripts/stop.sh -y
|
|
# =============================================================================
|
|
set -u
|
|
cd "$(cd "$(dirname "$0")/.." && pwd)"
|
|
ROOT="$(pwd)"
|
|
|
|
# ---- 可调参数(命令行第1个参数 / 环境变量 均可覆盖) ----
|
|
# MODE 未显式指定时, 读 config.dispatch.source 决定: local -> 本地text2, 其余 -> COS常驻
|
|
if [ -z "${MODE:-}" ]; then
|
|
_src="$(python -c "from config.config_utils import config; print((config.get_dispatch_config() or {}).get('source','cos'))" 2>/dev/null || echo cos)"
|
|
[ "$_src" = "local" ] && MODE="local" || MODE="worker"
|
|
fi
|
|
MODE="${MODE:-worker}" # worker=COS认领常驻 / local=本地text2
|
|
MAX_WORKERS="${1:-${MAX_WORKERS:-25}}" # 0=不传给python,由main.py/run_worker.py自动按本机CPU核数选
|
|
# 共享机/跟OCR服务同机部署的, 建议显式传小一点的值(如24), 别用自动
|
|
MAX_TASKS="${MAX_TASKS:-0}" # 0=不回收worker(避免3.11回收缩水把并发缩到0)
|
|
MEM_GUARD="${MEM_GUARD:-30}" # 可用内存低于此(G)暂停启动新视频
|
|
SOURCE_LIST="${SOURCE_LIST:-./text2}" # 仅 local 模式: 任务txt目录
|
|
EXECUTOR="${EXECUTOR:-process}" # process=进程池(绕GIL) / thread
|
|
LOG="${LOG:-app.log}"
|
|
NICE="${NICE:-15}" # 处理进程CPU优先级(0~19,越大越低)
|
|
LOG_MAX_MB="${LOG_MAX_MB:-200}" # $LOG 超过这个大小(MB)就在启动前轮转一次, 0=不轮转
|
|
|
|
# ---- 启动前轮转一次日志(常驻进程一跑数天/数周, >>追加写不会自己变小) ----
|
|
_rotate_log() {
|
|
local f="$1" maxmb="$2"
|
|
[ "$maxmb" -le 0 ] && return 0
|
|
[ -f "$f" ] || return 0
|
|
local sz_mb=$(( $(stat -c%s "$f" 2>/dev/null || echo 0) / 1024 / 1024 ))
|
|
if [ "$sz_mb" -ge "$maxmb" ]; then
|
|
local ts; ts="$(date +%Y%m%d_%H%M%S)"
|
|
mv "$f" "${f}.${ts}" 2>/dev/null && echo "[日志轮转] $f (${sz_mb}M) → ${f}.${ts}, 重新开始写"
|
|
ls -t "${f}".[0-9]* 2>/dev/null | tail -n +6 | xargs -r rm -f # 只留最近5份轮转备份
|
|
fi
|
|
}
|
|
_rotate_log "$LOG" "$LOG_MAX_MB"
|
|
|
|
# 按模式确定入口与进程匹配模式
|
|
if [ "$MODE" = "worker" ]; then
|
|
ENTRY="run_worker.py"
|
|
PROC_PAT='python .*run_worker\.py'
|
|
else
|
|
ENTRY="main.py"
|
|
PROC_PAT='python .*main\.py'
|
|
fi
|
|
|
|
MAX_WORKERS_DISP="$MAX_WORKERS"
|
|
[ "$MAX_WORKERS" = "0" ] && MAX_WORKERS_DISP="0(自动按CPU核数选, 实际值见下面python启动日志)"
|
|
|
|
echo "=================================================="
|
|
echo " 启动 $ENTRY (MODE=$MODE)"
|
|
echo " max-workers=$MAX_WORKERS_DISP max-tasks-per-child=$MAX_TASKS mem-guard=${MEM_GUARD}G"
|
|
echo " executor=$EXECUTOR nice=$NICE(让OCR服务优先) 日志→$LOG"
|
|
[ "$MODE" = "local" ] && echo " source-list=$SOURCE_LIST"
|
|
echo "=================================================="
|
|
|
|
# ---- 1. 防重复启动 ----
|
|
if pgrep -af "$PROC_PAT" 2>/dev/null | grep -v 'multiprocessing' | grep -v 'grep' >/dev/null; then
|
|
echo "✗ 已经有 $ENTRY 在运行了, 不重复启动。"
|
|
echo " 查看: pgrep -af '$PROC_PAT' | grep -v multiprocessing"
|
|
echo " 如要重启: 先 bash scripts/stop.sh -y"
|
|
exit 1
|
|
fi
|
|
|
|
# ---- 2. 启动前自检 ----
|
|
fail=0
|
|
need() { command -v "$1" >/dev/null 2>&1 && echo " ✓ $1" || { echo " ✗ 缺少 $1 (需在 PATH 或项目目录)"; fail=1; }; }
|
|
echo "[依赖检查]"
|
|
need python
|
|
need ffmpeg
|
|
need ffprobe
|
|
if command -v coscli >/dev/null 2>&1; then echo " ✓ coscli"
|
|
elif [ -x "./coscli-linux-amd64" ]; then echo " ✓ ./coscli-linux-amd64"
|
|
else echo " ✗ 找不到 coscli / ./coscli-linux-amd64"; fail=1; fi
|
|
|
|
echo "[配置/任务检查]"
|
|
[ -f config/config.json ] && echo " ✓ config/config.json" || { echo " ✗ 缺 config/config.json"; fail=1; }
|
|
|
|
if [ "$MODE" = "worker" ]; then
|
|
# worker 模式: 任务来自 COS, 不需要本地 text2; 检查 dispatch 配置与机器ID + COS 队列可达
|
|
chk=$(python - <<'PY' 2>&1
|
|
import sys
|
|
try:
|
|
from config.config_utils import config
|
|
from tool import job_cos
|
|
dp = config.get_dispatch_config() or {}
|
|
if not dp.get("jobs_root"):
|
|
print("NO_JOBS_ROOT"); sys.exit(0)
|
|
mid = job_cos.get_machine_id()
|
|
todo = job_cos.list_todo() # 顺带验证 COS 队列可达
|
|
print("OK\t%s\t%d" % (mid, len(todo)))
|
|
except Exception as e:
|
|
print("ERR\t%s" % e)
|
|
PY
|
|
)
|
|
case "$chk" in
|
|
OK*) mid=$(echo "$chk" | cut -f2); n=$(echo "$chk" | cut -f3)
|
|
echo " ✓ dispatch 配置OK 机器ID=$mid todo剩余分片=$n"
|
|
[ "$n" = "0" ] && echo " ⚠ todo 队列为空(worker会空闲等待; 先 python tools/seed_jobs.py 灌任务)";;
|
|
NO_JOBS_ROOT) echo " ✗ config.dispatch.jobs_root 未配置(取任务地址)"; fail=1;;
|
|
*) echo " ✗ 访问 COS 任务队列失败: ${chk#ERR }"; fail=1;;
|
|
esac
|
|
else
|
|
if ls "$SOURCE_LIST"/*.txt >/dev/null 2>&1; then
|
|
n=$(ls "$SOURCE_LIST"/*.txt 2>/dev/null | wc -l)
|
|
echo " ✓ $SOURCE_LIST 有 $n 个任务txt"
|
|
else
|
|
echo " ✗ $SOURCE_LIST 下没有 .txt 任务文件"; fail=1
|
|
fi
|
|
fi
|
|
|
|
if [ "$fail" -ne 0 ]; then
|
|
echo ""
|
|
echo "✗ 自检未通过, 已中止(没启动)。修好上面 ✗ 的项再跑。"
|
|
exit 1
|
|
fi
|
|
|
|
# ---- 3. 启动(setsid+nohup+disown 三保险, SSH断开也杀不到) ----
|
|
echo ""
|
|
echo "[启动中...]"
|
|
if [ "$MODE" = "worker" ]; then
|
|
setsid nohup nice -n "$NICE" python run_worker.py \
|
|
--max-workers "$MAX_WORKERS" \
|
|
--max-tasks-per-child "$MAX_TASKS" \
|
|
--mem-guard-gb "$MEM_GUARD" \
|
|
--executor "$EXECUTOR" >> "$LOG" 2>&1 &
|
|
else
|
|
setsid nohup nice -n "$NICE" python main.py \
|
|
--source-list "$SOURCE_LIST" \
|
|
--max-workers "$MAX_WORKERS" \
|
|
--max-tasks-per-child "$MAX_TASKS" \
|
|
--mem-guard-gb "$MEM_GUARD" \
|
|
--executor "$EXECUTOR" >> "$LOG" 2>&1 &
|
|
fi
|
|
disown 2>/dev/null || true
|
|
|
|
# ---- 4. 等几秒确认起来了 ----
|
|
sleep 6
|
|
if pgrep -af "$PROC_PAT" 2>/dev/null | grep -v 'multiprocessing' | grep -v 'grep' >/dev/null; then
|
|
echo "✓ 已启动。日志末尾:"
|
|
tail -n 5 "$LOG" 2>/dev/null | sed 's/^/ /'
|
|
echo ""
|
|
echo " 实时看日志: tail -f $LOG"
|
|
echo " 数worker: pgrep -af multiprocessing | wc -l (稍等会接近 $MAX_WORKERS_DISP)"
|
|
echo " 停止: bash scripts/stop.sh -y"
|
|
else
|
|
echo "✗ 启动后没检测到进程, 查日志看原因:"
|
|
tail -n 20 "$LOG" 2>/dev/null | sed 's/^/ /'
|
|
exit 1
|
|
fi
|