# -*- coding: utf-8 -*- """长任务管理: 同一时刻最多一个 AALC 任务子进程。""" import glob import json import os import re import signal import subprocess import sys import threading import time from collections import deque from .aalc_task import RESTORE_ENV from .decisions import DECISIONS_ENV, SUFFIX as DECISIONS_SUFFIX, summarize as summarize_decisions from .killswitch import EXIT_CODE as USER_ABORT_EXIT REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) IS_WIN = sys.platform.startswith('win') RESTORE_SUFFIX = '.restore.json' def default_argv(task, params): return [sys.executable, '-m', 'lc_mcp.aalc_task', task, json.dumps(params)] def default_restore_argv(path): return [sys.executable, '-m', 'lc_mcp.aalc_task', '--restore', path] def tail(path, n): try: with open(path, 'rb') as f: f.seek(0, os.SEEK_END) f.seek(max(0, f.tell() - 64 * 1024)) lines = f.read().decode('utf-8', 'replace').splitlines() return lines[-n:] except OSError: return [] ERROR_MARKERS = ('[ERROR]', 'Traceback (most recent call last)') ERROR_SCAN_LINES = 400 def log_errors(path, limit=3): """流式扫描整份任务日志, 只保留最后 limit 条报错。 AALC 的每日流程在单次战斗失败后会继续跑完并正常退出(exit 0), 只看退出码 会把"超出最大尝试次数,未能进入战斗"这类失败当成成功, 所以状态里额外附上 日志里的报错行。 """ hits = deque(maxlen=limit) try: with open(path, encoding='utf-8', errors='replace') as f: for line in f: if any(marker in line for marker in ERROR_MARKERS): hits.append(line.rstrip('\r\n')) except OSError: pass return list(hits) # AALC 卡死时不报错、进程健在, 只是反复输出同一类日志。 # "配队人数OCR得到" 只在编队人数检查失败的重试里打印, 健康运行中一次都不会出现; # 2026-10-05 的三次空队伍卡死里它分别刷了 242/306/24 次。 STALL_MARKERS = {'empty_team': '配队人数OCR得到'} STALL_THRESHOLD = 10 # 计数法抓不到"界面误判"型卡死: AALC 把别的界面认成目标界面, 对着误匹配的 # 模板反复盲点, 日志照样疯涨, 只是再没有任何流程进展。2026-10-13 实测在镜牢 # 入口把 home/luxcavation_assets.png 匹配到 0.836, 连点 18 分钟不停。 # 所以再加一条与内容无关的判据: 里程碑(开始执行/结束执行)多久没前进了。 STALL_NO_PROGRESS_S = int(os.environ.get('LCP_STALL_NO_PROGRESS_S', 10 * 60)) # 里程碑可能离文件尾很远(卡死期间日志涨得很快), 回溯窗口给大一点 PROGRESS_SCAN_BYTES = 8 * 1024 * 1024 TIMESTAMP_RE = re.compile(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})') PROGRESS_RE = re.compile(r'(开始执行|结束执行)\s') def last_progress_epoch(path): """日志里最后一条里程碑的时间戳(epoch 秒); 找不到返回 None。""" try: with open(path, 'rb') as f: f.seek(0, os.SEEK_END) f.seek(max(0, f.tell() - PROGRESS_SCAN_BYTES)) blob = f.read().decode('utf-8', 'replace') except OSError: return None last = None for line in blob.splitlines(): if not PROGRESS_RE.search(line): continue m = TIMESTAMP_RE.search(line) if not m: continue try: last = time.mktime(time.strptime(m.group(1), '%Y-%m-%d %H:%M:%S')) except ValueError: continue return last def log_stalls(path, now=None): """统计日志尾部的卡死信号, 达到阈值的返回 {名称: 次数}。""" lines = tail(path, ERROR_SCAN_LINES) counts = {name: sum(marker in line for line in lines) for name, marker in STALL_MARKERS.items()} stalls = {name: n for name, n in counts.items() if n >= STALL_THRESHOLD} # 必须随真实时间推进: 进程卡住不再写日志时, mtime 也会冻结。 # 离线复核历史日志时可显式传入当时的 now。 progress = last_progress_epoch(path) if progress is not None: idle = (time.time() if now is None else now) - progress if idle >= STALL_NO_PROGRESS_S: stalls['no_progress'] = int(idle) return stalls class TaskRunner: def __init__(self, run_dir, argv_factory=default_argv, keep_logs=20, restore_argv=default_restore_argv): self.run_dir = os.path.abspath(run_dir) os.makedirs(self.run_dir, exist_ok=True) self.argv_factory = argv_factory self.restore_argv = restore_argv self.keep_logs = keep_logs self._lock = threading.Lock() self.proc = None self.info = None def running(self): return self.proc is not None and self.proc.poll() is None def _env(self): return dict(os.environ, PYTHONPATH=REPO + os.pathsep + os.environ.get('PYTHONPATH', ''), PYTHONIOENCODING='utf-8', PYTHONUNBUFFERED='1') def _pending_restores(self): """被强杀/崩溃的任务留下的原值文件(不含正在运行的任务自己的)。新的在前: 有多份时最后写回的是最早一份, 也就是真正的原值。""" current = self.info.get('restore') if self.info and self.running() else None return [p for p in sorted(glob.glob(os.path.join(self.run_dir, '*' + RESTORE_SUFFIX)), reverse=True) if p != current] def _restore_leftovers(self): """把遗留的原值写回 config.yaml。成功返回 None, 否则返回原因。""" for path in self._pending_restores(): name = os.path.basename(path) try: r = subprocess.run(self.restore_argv(path), cwd=REPO, env=self._env(), stdin=subprocess.DEVNULL, capture_output=True, timeout=120) except (OSError, subprocess.TimeoutExpired) as e: return 'config restore from %s failed: %s' % (name, e) if r.returncode != 0 or os.path.exists(path): out = (r.stdout + r.stderr).decode('utf-8', 'replace').strip().splitlines()[-3:] return 'config restore from %s failed (exit %s): %s' % (name, r.returncode, ' | '.join(out)) return None def start(self, task, params): with self._lock: if self.running(): return {'ok': False, 'error': '已有任务在跑: %s (先 stop_task)' % self.info['task']} err = self._restore_leftovers() if err: # 否则新任务会把残留的覆盖值当成原值记下, 真正的原值就丢了 return {'ok': False, 'error': err + '; 写回 config.yaml 之前不启动新任务'} stamp = time.strftime('%Y%m%d_%H%M%S') base = os.path.join(self.run_dir, 'task_%s_%s' % (stamp, task)) env = self._env() env[RESTORE_ENV] = base + RESTORE_SUFFIX env[DECISIONS_ENV] = base + DECISIONS_SUFFIX kw = {'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WIN else {'start_new_session': True} with open(base + '.log', 'wb') as logf: self.proc = subprocess.Popen(self.argv_factory(task, params), cwd=REPO, env=env, stdin=subprocess.DEVNULL, stdout=logf, stderr=subprocess.STDOUT, **kw) self.info = {'task': task, 'params': params, 'log': (base + '.log').replace('\\', '/'), 'decisions': (base + DECISIONS_SUFFIX).replace('\\', '/'), 'restore': env[RESTORE_ENV], 'started': time.time(), 'stopped_by_user': False} self._prune() return {'ok': True, 'task': task, 'pid': self.proc.pid, 'log': self.info['log']} def status(self, lines=30): with self._lock: if self.info is None: r = {'ok': True, 'state': 'idle'} else: code = self.proc.poll() if code is None: state = 'running' else: self.info.setdefault('ended', time.time()) state = ('stopped' if self.info['stopped_by_user'] or code == USER_ABORT_EXIT else 'succeeded' if code == 0 else 'failed') r = {'ok': True, 'state': state, 'task': self.info['task'], 'params': self.info['params'], 'elapsed_s': int((self.info.get('ended') or time.time()) - self.info['started']), 'log': self.info['log'], 'log_tail': tail(self.info['log'], max(1, min(int(lines), 200)))} counts = summarize_decisions(self.info['decisions']) if counts: r['decisions'] = {'path': self.info['decisions'], 'counts': counts} if code is None: stalls = log_stalls(self.info['log']) if stalls: r['stall'] = stalls else: r['exit_code'] = code # 子进程退出后日志已写完; 结果只计算一次, 避免轮询反复扫描大文件。 if 'errors' not in self.info: self.info['errors'] = log_errors(self.info['log']) errors = self.info['errors'] if errors: r['errors'] = errors pending = self._pending_restores() if pending: r['pending_restore'] = [os.path.basename(p) for p in pending] return r def stop(self): with self._lock: if self.running(): self.info['stopped_by_user'] = True pid = self.proc.pid if IS_WIN: subprocess.run(['taskkill', '/PID', str(pid), '/T', '/F'], capture_output=True) else: try: os.killpg(pid, signal.SIGKILL) except ProcessLookupError: pass try: self.proc.wait(timeout=10) except subprocess.TimeoutExpired: return {'ok': False, 'error': '进程 %d 10 秒内没退出' % pid} self.info['ended'] = time.time() r = {'ok': True, 'state': 'stopped', 'task': self.info['task']} else: r = {'ok': True, 'state': 'idle', 'note': '没有在跑的任务'} # 强杀时任务的 finally 不会执行, 由这里把临时覆盖的次数写回 err = self._restore_leftovers() if err: r.update(ok=False, error=err) return r def _prune(self): logs = sorted(glob.glob(os.path.join(self.run_dir, 'task_*.log')), key=os.path.getmtime) for p in logs[:-self.keep_logs]: for path in (p, p[:-len('.log')] + DECISIONS_SUFFIX): try: os.remove(path) except OSError: pass