# -*- coding: utf-8 -*- """长任务管理: 同一时刻最多一个 AALC 任务子进程。""" import glob import json import os import signal import subprocess import sys import threading import time from .aalc_task import RESTORE_ENV REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) IS_WIN = sys.platform.startswith('win') RESTORE_SUFFIX = '.restore.json' def default_argv(task, params): return [sys.executable, '-m', 'lc_mcp.aalc_task', task, json.dumps(params)] def default_restore_argv(path): return [sys.executable, '-m', 'lc_mcp.aalc_task', '--restore', path] def tail(path, n): try: with open(path, 'rb') as f: f.seek(0, os.SEEK_END) f.seek(max(0, f.tell() - 64 * 1024)) lines = f.read().decode('utf-8', 'replace').splitlines() return lines[-n:] except OSError: return [] ERROR_MARKERS = ('[ERROR]', 'Traceback (most recent call last)') ERROR_SCAN_LINES = 400 def log_errors(path, limit=3): """挑出日志尾部的报错行。 AALC 的每日流程在单次战斗失败后会继续跑完并正常退出(exit 0), 只看退出码 会把"超出最大尝试次数,未能进入战斗"这类失败当成成功, 所以状态里额外附上 日志里的报错行。 """ hits = [line for line in tail(path, ERROR_SCAN_LINES) if any(marker in line for marker in ERROR_MARKERS)] return hits[-limit:] class TaskRunner: def __init__(self, run_dir, argv_factory=default_argv, keep_logs=20, restore_argv=default_restore_argv): self.run_dir = os.path.abspath(run_dir) os.makedirs(self.run_dir, exist_ok=True) self.argv_factory = argv_factory self.restore_argv = restore_argv self.keep_logs = keep_logs self._lock = threading.Lock() self.proc = None self.info = None def running(self): return self.proc is not None and self.proc.poll() is None def _env(self): return dict(os.environ, PYTHONPATH=REPO + os.pathsep + os.environ.get('PYTHONPATH', ''), PYTHONIOENCODING='utf-8', PYTHONUNBUFFERED='1') def _pending_restores(self): """被强杀/崩溃的任务留下的原值文件(不含正在运行的任务自己的)。新的在前: 有多份时最后写回的是最早一份, 也就是真正的原值。""" current = self.info.get('restore') if self.info and self.running() else None return [p for p in sorted(glob.glob(os.path.join(self.run_dir, '*' + RESTORE_SUFFIX)), reverse=True) if p != current] def _restore_leftovers(self): """把遗留的原值写回 config.yaml。成功返回 None, 否则返回原因。""" for path in self._pending_restores(): name = os.path.basename(path) try: r = subprocess.run(self.restore_argv(path), cwd=REPO, env=self._env(), stdin=subprocess.DEVNULL, capture_output=True, timeout=120) except (OSError, subprocess.TimeoutExpired) as e: return 'config restore from %s failed: %s' % (name, e) if r.returncode != 0 or os.path.exists(path): out = (r.stdout + r.stderr).decode('utf-8', 'replace').strip().splitlines()[-3:] return 'config restore from %s failed (exit %s): %s' % (name, r.returncode, ' | '.join(out)) return None def start(self, task, params): with self._lock: if self.running(): return {'ok': False, 'error': '已有任务在跑: %s (先 stop_task)' % self.info['task']} err = self._restore_leftovers() if err: # 否则新任务会把残留的覆盖值当成原值记下, 真正的原值就丢了 return {'ok': False, 'error': err + '; 写回 config.yaml 之前不启动新任务'} stamp = time.strftime('%Y%m%d_%H%M%S') base = os.path.join(self.run_dir, 'task_%s_%s' % (stamp, task)) env = self._env() env[RESTORE_ENV] = base + RESTORE_SUFFIX kw = {'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WIN else {'start_new_session': True} with open(base + '.log', 'wb') as logf: self.proc = subprocess.Popen(self.argv_factory(task, params), cwd=REPO, env=env, stdin=subprocess.DEVNULL, stdout=logf, stderr=subprocess.STDOUT, **kw) self.info = {'task': task, 'params': params, 'log': (base + '.log').replace('\\', '/'), 'restore': env[RESTORE_ENV], 'started': time.time(), 'stopped_by_user': False} self._prune() return {'ok': True, 'task': task, 'pid': self.proc.pid, 'log': self.info['log']} def status(self, lines=30): with self._lock: if self.info is None: r = {'ok': True, 'state': 'idle'} else: code = self.proc.poll() if code is None: state = 'running' else: self.info.setdefault('ended', time.time()) state = 'stopped' if self.info['stopped_by_user'] else 'succeeded' if code == 0 else 'failed' r = {'ok': True, 'state': state, 'task': self.info['task'], 'params': self.info['params'], 'elapsed_s': int((self.info.get('ended') or time.time()) - self.info['started']), 'log': self.info['log'], 'log_tail': tail(self.info['log'], max(1, min(int(lines), 200)))} if code is not None: r['exit_code'] = code errors = log_errors(self.info['log']) if errors: r['errors'] = errors pending = self._pending_restores() if pending: r['pending_restore'] = [os.path.basename(p) for p in pending] return r def stop(self): with self._lock: if self.running(): self.info['stopped_by_user'] = True pid = self.proc.pid if IS_WIN: subprocess.run(['taskkill', '/PID', str(pid), '/T', '/F'], capture_output=True) else: try: os.killpg(pid, signal.SIGKILL) except ProcessLookupError: pass try: self.proc.wait(timeout=10) except subprocess.TimeoutExpired: return {'ok': False, 'error': '进程 %d 10 秒内没退出' % pid} self.info['ended'] = time.time() r = {'ok': True, 'state': 'stopped', 'task': self.info['task']} else: r = {'ok': True, 'state': 'idle', 'note': '没有在跑的任务'} # 强杀时任务的 finally 不会执行, 由这里把临时覆盖的次数写回 err = self._restore_leftovers() if err: r.update(ok=False, error=err) return r def _prune(self): logs = sorted(glob.glob(os.path.join(self.run_dir, 'task_*.log')), key=os.path.getmtime) for p in logs[:-self.keep_logs]: try: os.remove(p) except OSError: pass