Files
lc-mcp/lc_mcp/tasks.py
T

243 lines
10 KiB
Python

# -*- coding: utf-8 -*-
"""长任务管理: 同一时刻最多一个 AALC 任务子进程。"""
import glob
import json
import os
import re
import signal
import subprocess
import sys
import threading
import time
from collections import deque
from .aalc_task import RESTORE_ENV
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
IS_WIN = sys.platform.startswith('win')
RESTORE_SUFFIX = '.restore.json'
def default_argv(task, params):
return [sys.executable, '-m', 'lc_mcp.aalc_task', task, json.dumps(params)]
def default_restore_argv(path):
return [sys.executable, '-m', 'lc_mcp.aalc_task', '--restore', path]
def tail(path, n):
try:
with open(path, 'rb') as f:
f.seek(0, os.SEEK_END)
f.seek(max(0, f.tell() - 64 * 1024))
lines = f.read().decode('utf-8', 'replace').splitlines()
return lines[-n:]
except OSError:
return []
ERROR_MARKERS = ('[ERROR]', 'Traceback (most recent call last)')
ERROR_SCAN_LINES = 400
def log_errors(path, limit=3):
"""流式扫描整份任务日志, 只保留最后 limit 条报错。
AALC 的每日流程在单次战斗失败后会继续跑完并正常退出(exit 0), 只看退出码
会把"超出最大尝试次数,未能进入战斗"这类失败当成成功, 所以状态里额外附上
日志里的报错行。
"""
hits = deque(maxlen=limit)
try:
with open(path, encoding='utf-8', errors='replace') as f:
for line in f:
if any(marker in line for marker in ERROR_MARKERS):
hits.append(line.rstrip('\r\n'))
except OSError:
pass
return list(hits)
# AALC 卡死时不报错、进程健在, 只是反复输出同一类日志。
# "配队人数OCR得到" 只在编队人数检查失败的重试里打印, 健康运行中一次都不会出现;
# 2026-10-05 的三次空队伍卡死里它分别刷了 242/306/24 次。
STALL_MARKERS = {'empty_team': '配队人数OCR得到'}
STALL_THRESHOLD = 10
# 计数法抓不到"界面误判"型卡死: AALC 把别的界面认成目标界面, 对着误匹配的
# 模板反复盲点, 日志照样疯涨, 只是再没有任何流程进展。2026-10-13 实测在镜牢
# 入口把 home/luxcavation_assets.png 匹配到 0.836, 连点 18 分钟不停。
# 所以再加一条与内容无关的判据: 里程碑(开始执行/结束执行)多久没前进了。
STALL_NO_PROGRESS_S = int(os.environ.get('LCP_STALL_NO_PROGRESS_S', 10 * 60))
# 里程碑可能离文件尾很远(卡死期间日志涨得很快), 回溯窗口给大一点
PROGRESS_SCAN_BYTES = 8 * 1024 * 1024
TIMESTAMP_RE = re.compile(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})')
PROGRESS_RE = re.compile(r'(开始执行|结束执行)\s')
def last_progress_epoch(path):
"""日志里最后一条里程碑的时间戳(epoch 秒); 找不到返回 None。"""
try:
with open(path, 'rb') as f:
f.seek(0, os.SEEK_END)
f.seek(max(0, f.tell() - PROGRESS_SCAN_BYTES))
blob = f.read().decode('utf-8', 'replace')
except OSError:
return None
last = None
for line in blob.splitlines():
if not PROGRESS_RE.search(line):
continue
m = TIMESTAMP_RE.search(line)
if not m:
continue
try:
last = time.mktime(time.strptime(m.group(1), '%Y-%m-%d %H:%M:%S'))
except ValueError:
continue
return last
def log_stalls(path, now=None):
"""统计日志尾部的卡死信号, 达到阈值的返回 {名称: 次数}。"""
lines = tail(path, ERROR_SCAN_LINES)
counts = {name: sum(marker in line for line in lines) for name, marker in STALL_MARKERS.items()}
stalls = {name: n for name, n in counts.items() if n >= STALL_THRESHOLD}
# 必须随真实时间推进: 进程卡住不再写日志时, mtime 也会冻结。
# 离线复核历史日志时可显式传入当时的 now。
progress = last_progress_epoch(path)
if progress is not None:
idle = (time.time() if now is None else now) - progress
if idle >= STALL_NO_PROGRESS_S:
stalls['no_progress'] = int(idle)
return stalls
class TaskRunner:
def __init__(self, run_dir, argv_factory=default_argv, keep_logs=20, restore_argv=default_restore_argv):
self.run_dir = os.path.abspath(run_dir)
os.makedirs(self.run_dir, exist_ok=True)
self.argv_factory = argv_factory
self.restore_argv = restore_argv
self.keep_logs = keep_logs
self._lock = threading.Lock()
self.proc = None
self.info = None
def running(self):
return self.proc is not None and self.proc.poll() is None
def _env(self):
return dict(os.environ, PYTHONPATH=REPO + os.pathsep + os.environ.get('PYTHONPATH', ''),
PYTHONIOENCODING='utf-8', PYTHONUNBUFFERED='1')
def _pending_restores(self):
"""被强杀/崩溃的任务留下的原值文件(不含正在运行的任务自己的)。新的在前:
有多份时最后写回的是最早一份, 也就是真正的原值。"""
current = self.info.get('restore') if self.info and self.running() else None
return [p for p in sorted(glob.glob(os.path.join(self.run_dir, '*' + RESTORE_SUFFIX)), reverse=True)
if p != current]
def _restore_leftovers(self):
"""把遗留的原值写回 config.yaml。成功返回 None, 否则返回原因。"""
for path in self._pending_restores():
name = os.path.basename(path)
try:
r = subprocess.run(self.restore_argv(path), cwd=REPO, env=self._env(),
stdin=subprocess.DEVNULL, capture_output=True, timeout=120)
except (OSError, subprocess.TimeoutExpired) as e:
return 'config restore from %s failed: %s' % (name, e)
if r.returncode != 0 or os.path.exists(path):
out = (r.stdout + r.stderr).decode('utf-8', 'replace').strip().splitlines()[-3:]
return 'config restore from %s failed (exit %s): %s' % (name, r.returncode, ' | '.join(out))
return None
def start(self, task, params):
with self._lock:
if self.running():
return {'ok': False, 'error': '已有任务在跑: %s (先 stop_task)' % self.info['task']}
err = self._restore_leftovers()
if err:
# 否则新任务会把残留的覆盖值当成原值记下, 真正的原值就丢了
return {'ok': False, 'error': err + '; 写回 config.yaml 之前不启动新任务'}
stamp = time.strftime('%Y%m%d_%H%M%S')
base = os.path.join(self.run_dir, 'task_%s_%s' % (stamp, task))
env = self._env()
env[RESTORE_ENV] = base + RESTORE_SUFFIX
kw = {'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WIN else {'start_new_session': True}
with open(base + '.log', 'wb') as logf:
self.proc = subprocess.Popen(self.argv_factory(task, params), cwd=REPO, env=env,
stdin=subprocess.DEVNULL, stdout=logf, stderr=subprocess.STDOUT, **kw)
self.info = {'task': task, 'params': params, 'log': (base + '.log').replace('\\', '/'),
'restore': env[RESTORE_ENV], 'started': time.time(), 'stopped_by_user': False}
self._prune()
return {'ok': True, 'task': task, 'pid': self.proc.pid, 'log': self.info['log']}
def status(self, lines=30):
with self._lock:
if self.info is None:
r = {'ok': True, 'state': 'idle'}
else:
code = self.proc.poll()
if code is None:
state = 'running'
else:
self.info.setdefault('ended', time.time())
state = 'stopped' if self.info['stopped_by_user'] else 'succeeded' if code == 0 else 'failed'
r = {'ok': True, 'state': state, 'task': self.info['task'], 'params': self.info['params'],
'elapsed_s': int((self.info.get('ended') or time.time()) - self.info['started']),
'log': self.info['log'], 'log_tail': tail(self.info['log'], max(1, min(int(lines), 200)))}
if code is None:
stalls = log_stalls(self.info['log'])
if stalls:
r['stall'] = stalls
else:
r['exit_code'] = code
# 子进程退出后日志已写完; 结果只计算一次, 避免轮询反复扫描大文件。
if 'errors' not in self.info:
self.info['errors'] = log_errors(self.info['log'])
errors = self.info['errors']
if errors:
r['errors'] = errors
pending = self._pending_restores()
if pending:
r['pending_restore'] = [os.path.basename(p) for p in pending]
return r
def stop(self):
with self._lock:
if self.running():
self.info['stopped_by_user'] = True
pid = self.proc.pid
if IS_WIN:
subprocess.run(['taskkill', '/PID', str(pid), '/T', '/F'], capture_output=True)
else:
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
try:
self.proc.wait(timeout=10)
except subprocess.TimeoutExpired:
return {'ok': False, 'error': '进程 %d 10 秒内没退出' % pid}
self.info['ended'] = time.time()
r = {'ok': True, 'state': 'stopped', 'task': self.info['task']}
else:
r = {'ok': True, 'state': 'idle', 'note': '没有在跑的任务'}
# 强杀时任务的 finally 不会执行, 由这里把临时覆盖的次数写回
err = self._restore_leftovers()
if err:
r.update(ok=False, error=err)
return r
def _prune(self):
logs = sorted(glob.glob(os.path.join(self.run_dir, 'task_*.log')), key=os.path.getmtime)
for p in logs[:-self.keep_logs]:
try:
os.remove(p)
except OSError:
pass