115 lines
4.9 KiB
Python
115 lines
4.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""卡死检测: 无进展判据。
|
|
|
|
背景 —— 2026-10-13 实机测试撞到的第二种卡死:
|
|
AALC 把镜牢入口误认成经验本界面, 把 home/luxcavation_assets.png 匹配到 0.836,
|
|
然后对着误匹配的位置连点了 18 分钟。日志一直在涨, 所以原有的计数法
|
|
(STALL_MARKERS 只认 "配队人数OCR得到") 完全抓不到, 外层只能干等 45 分钟超时。
|
|
|
|
修法是加一条与日志内容无关的判据: 里程碑(开始执行/结束执行)多久没前进。
|
|
"""
|
|
import os
|
|
import gzip
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from lc_mcp import tasks
|
|
|
|
|
|
def _write(lines):
|
|
fd, path = tempfile.mkstemp(suffix='.log')
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(lines) + '\n')
|
|
return path
|
|
|
|
|
|
def _stamp(offset_s=0):
|
|
return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() - offset_s))
|
|
|
|
|
|
class NoProgressStall(unittest.TestCase):
|
|
|
|
def tearDown(self):
|
|
for p in getattr(self, '_paths', []):
|
|
try:
|
|
os.remove(p)
|
|
except OSError:
|
|
pass
|
|
|
|
def _make(self, lines):
|
|
p = _write(lines)
|
|
self._paths = getattr(self, '_paths', []) + [p]
|
|
return p
|
|
|
|
def test_stale_milestone_is_reported(self):
|
|
"""里程碑停在阈值之前 -> 报 no_progress。"""
|
|
old = tasks.STALL_NO_PROGRESS_S + 120
|
|
p = self._make([
|
|
'[INFO] %s [AALC] tasks\\base\\script_task_scheme.py:262: 开始执行 一次纽本' % _stamp(old),
|
|
'[DEBUG] %s [AALC] tasks\\daily\\luxcavation.py:224: 目标图片:home/luxcavation_assets.png'
|
|
% _stamp(0),
|
|
])
|
|
stalls = tasks.log_stalls(p)
|
|
self.assertIn('no_progress', stalls)
|
|
self.assertGreaterEqual(stalls['no_progress'], tasks.STALL_NO_PROGRESS_S)
|
|
|
|
def test_recent_milestone_is_not_reported(self):
|
|
"""里程碑刚刚前进过 -> 不报。"""
|
|
p = self._make([
|
|
'[INFO] %s [AALC] tasks\\daily\\luxcavation.py:9: 结束执行 一次纽本' % _stamp(5),
|
|
'[DEBUG] %s [AALC] tasks\\automation\\automation.py:152: 点击位置:(979,290)' % _stamp(0),
|
|
])
|
|
self.assertNotIn('no_progress', tasks.log_stalls(p))
|
|
|
|
def test_frozen_log_does_not_freeze_stall_clock(self):
|
|
old = tasks.STALL_NO_PROGRESS_S + 120
|
|
p = self._make(['[INFO] %s [AALC] 开始执行 一次纽本' % _stamp(old)])
|
|
frozen = time.time() - old
|
|
os.utime(p, (frozen, frozen))
|
|
self.assertIn('no_progress', tasks.log_stalls(p))
|
|
|
|
def test_no_milestone_at_all_is_not_reported(self):
|
|
"""一条里程碑都没有时不下结论(可能只是还没走到), 避免误杀启动阶段。"""
|
|
p = self._make([
|
|
'[DEBUG] %s [AALC] module\\automation\\automation.py:93: 使用基于窗口移动的后台点击模块'
|
|
% _stamp(99999),
|
|
])
|
|
self.assertNotIn('no_progress', tasks.log_stalls(p))
|
|
|
|
def test_existing_empty_team_marker_still_works(self):
|
|
"""原有判据不能被破坏。"""
|
|
p = self._make([
|
|
'[DEBUG] %s [AALC] tasks\\teams\\team_formation.py:225: 对于配队人数OCR得到:0/5' % _stamp(1),
|
|
] * 12)
|
|
self.assertIn('empty_team', tasks.log_stalls(p))
|
|
|
|
def test_stale_milestone_far_from_tail_is_still_found(self):
|
|
"""卡死期间日志涨得很快, 里程碑可能离文件尾很远, 回溯窗口必须够大。"""
|
|
old = tasks.STALL_NO_PROGRESS_S + 60
|
|
body = ['[INFO] %s [AALC] tasks\\base\\script_task_scheme.py:262: 开始执行 一次经验本' % _stamp(old)]
|
|
body += ['[DEBUG] %s [AALC] tasks\\daily\\luxcavation.py:224: 目标图片:home/luxcavation_assets.png'
|
|
' 相似度:0.836' % _stamp(0)] * 6000
|
|
p = self._make(body)
|
|
self.assertGreater(os.path.getsize(p), 64 * 1024) # 确定超出了 tail() 的 64KB 窗口
|
|
self.assertIn('no_progress', tasks.log_stalls(p))
|
|
|
|
def test_archived_windows_runs_distinguish_stalled_and_healthy_tasks(self):
|
|
root = Path(__file__).resolve().parents[1] / 'docs/feedback/2026-10-13/results/runs'
|
|
for name, stalled in [('task_20261013_124337_daily.log.gz', True),
|
|
('task_20261013_132845_daily.log.gz', False),
|
|
('task_20261013_135050_mirror.log.gz', False)]:
|
|
with self.subTest(log=name):
|
|
with gzip.open(root / name, 'rt', encoding='utf-8') as f:
|
|
lines = f.read().splitlines()
|
|
stamp = next(tasks.TIMESTAMP_RE.search(line).group(1)
|
|
for line in reversed(lines) if tasks.TIMESTAMP_RE.search(line))
|
|
observed_at = time.mktime(time.strptime(stamp, '%Y-%m-%d %H:%M:%S'))
|
|
path = self._make(lines)
|
|
self.assertEqual('no_progress' in tasks.log_stalls(path, now=observed_at), stalled)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|