值守: 新增 audit_runs.py —— 把上一轮验证过的核对办法做成工具
上一轮我照着自己写在手册里的办法手工核对, 结果 6 条全部误报"对不上"(坑在
日志结论在结束时、报告 started 在开始时, 差一个 elapsed_s)。手敲 PowerShell
容易出错, 所以做成脚本: 代码里的时间关系比人记的可靠。
用法: python scripts/audit_runs.py (退出码 0 = 全部吻合)
检查三件事:
1. 每份报告都能在日志里找到对应的"开始行", 且时间关系(开始+耗时=结论时刻)成立
2. 日志里每一轮都有对应报告
3. mock 残留: 同一秒扎堆的结论、或内容自相矛盾的结论
⛔ 写第一版时自己踩了同一类坑: 它拿"报告.started + elapsed"去找日志结论行,
能对上就说明 elapsed 一致 —— 于是"耗时不符"那个分支永远不可能触发, 是**死代码**。
看起来在查、实际查不到的东西比没有更糟(和"读不到就跳过"是同一种病)。
改成按日志里的"=== 开始: 体力换模块 ==="配对后, 时间关系和耗时才能各自独立核对。
13 个用例, 每个都是"必须能抓到某种问题"而不是"必须通过";
实测真实数据 6 轮/6 报告全部吻合, 造的三类坏数据(缺报告/耗时不符/内容矛盾)
全部被点名。全套 310/310 通过。
This commit is contained in:
@@ -0,0 +1,184 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""值守体检: 日志与报告交叉核对, 并挑出可疑记录。
|
||||||
|
|
||||||
|
用法: python audit_runs.py [--dir DIR] [--json]
|
||||||
|
退出码: 0 一切吻合 / 1 发现不一致
|
||||||
|
|
||||||
|
为什么需要这个脚本:
|
||||||
|
上一轮我照着自己写在 docs/WATCH.md 里的办法做核对, 结果 6 条全部报"对不上" ——
|
||||||
|
但数据其实是吻合的。坑在于: **日志的结论行写在一轮结束时, 报告的 started 是
|
||||||
|
开始时**, 两者天然差一个 elapsed_s。拿两个时间戳直接比必然全部失败。
|
||||||
|
|
||||||
|
正确关系是: 报告.started + elapsed ≈ 日志结论时刻。
|
||||||
|
|
||||||
|
代码里的时间关系比人记的可靠, 所以把核对做成脚本而不是让人照着文档敲
|
||||||
|
PowerShell —— 我手敲那次就误判了。
|
||||||
|
|
||||||
|
检查三件事:
|
||||||
|
1. 每份报告都能在日志里找到对应结论(时间对得上, 耗时和结果文本一致)
|
||||||
|
2. 日志里每条结论都有对应报告(没有"跑了但没留报告"的运行)
|
||||||
|
3. 疑似 mock 残留: 同一秒扎堆、或内容自相矛盾的结论行
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
DIR = os.path.expanduser(os.path.join('~', 'lc_mcp', 'modules'))
|
||||||
|
LOG_NAME = 'convert.log'
|
||||||
|
# 报告.started 与日志结论时刻的允许偏差(两边各写一次时间戳)
|
||||||
|
TOL_S = 5
|
||||||
|
|
||||||
|
CONCLUSION_RE = re.compile(
|
||||||
|
r'^\[(?P<t>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s*===\s*(?P<result>.+?)\s*'
|
||||||
|
r'\((?P<ok>\w+)\), 耗时 (?P<secs>\d+)s ===')
|
||||||
|
START_RE = re.compile(
|
||||||
|
r'^\[(?P<t>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s*===\s*开始: 体力换模块\s*===')
|
||||||
|
# 明显不可能出现在真实运行里的说法
|
||||||
|
IMPOSSIBLE = ('powershell not found', 'not found', 'No such file')
|
||||||
|
|
||||||
|
|
||||||
|
def parse_runs(path):
|
||||||
|
"""把日志解析成一轮一轮的 (开始, 结束, 结果, 耗时)。
|
||||||
|
|
||||||
|
⛔ 为什么要配对而不是只取结论行: 第一版只用结论行, 拿"报告.started + elapsed"
|
||||||
|
去和结论时刻比 —— 能对上就说明 elapsed 一致, 所以"耗时不符"那个分支永远不可能
|
||||||
|
触发, 是死代码。看起来在查、实际查不到的东西比没有更糟。
|
||||||
|
改成按日志里的"=== 开始: 体力换模块 ===" 配对, 报告的 started 才能和**开始**时刻
|
||||||
|
真正对照, 耗时也能独立核对。
|
||||||
|
"""
|
||||||
|
out = []
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return out
|
||||||
|
pending = []
|
||||||
|
with open(path, encoding='utf-8') as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
ms = START_RE.match(line)
|
||||||
|
if ms:
|
||||||
|
pending.append(datetime.strptime(ms.group('t'), '%Y-%m-%d %H:%M:%S'))
|
||||||
|
continue
|
||||||
|
mc = CONCLUSION_RE.match(line)
|
||||||
|
if mc:
|
||||||
|
start = pending.pop(0) if pending else None
|
||||||
|
out.append({'start': start,
|
||||||
|
'end': datetime.strptime(mc.group('t'), '%Y-%m-%d %H:%M:%S'),
|
||||||
|
'result': mc.group('result'), 'ok': mc.group('ok'),
|
||||||
|
'secs': int(mc.group('secs')), 'line': line})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# 兼容旧名字: 只要结论行
|
||||||
|
def parse_log(path):
|
||||||
|
return parse_runs(path)
|
||||||
|
|
||||||
|
|
||||||
|
def load_reports(d):
|
||||||
|
out = []
|
||||||
|
for p in sorted(glob.glob(os.path.join(d, 'convert_*.json'))):
|
||||||
|
try:
|
||||||
|
with open(p, encoding='utf-8') as f:
|
||||||
|
r = json.load(f)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
continue
|
||||||
|
out.append({'file': os.path.basename(p),
|
||||||
|
'start': datetime.strptime(r['started'], '%Y-%m-%d %H:%M:%S'),
|
||||||
|
'secs': r.get('elapsed_s'), 'result': r.get('result', ''),
|
||||||
|
'ok': bool(r.get('ok'))})
|
||||||
|
out.sort(key=lambda r: r['start'])
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def same_second_clusters(conclusions):
|
||||||
|
"""同一秒里出现多行结论 = 几乎必然是测试批量产生的。"""
|
||||||
|
bysec = defaultdict(list)
|
||||||
|
for c in conclusions:
|
||||||
|
bysec[c['end']].append(c)
|
||||||
|
return {k: v for k, v in bysec.items() if len(v) > 1}
|
||||||
|
|
||||||
|
|
||||||
|
def impossible_claims(conclusions):
|
||||||
|
"""内容自相矛盾的结论(比如 powershell 明明在却说找不到)。"""
|
||||||
|
out = []
|
||||||
|
for c in conclusions:
|
||||||
|
low = c['result']
|
||||||
|
if any(k.lower() in low.lower() for k in IMPOSSIBLE):
|
||||||
|
out.append(c)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def audit(d):
|
||||||
|
runs = parse_runs(os.path.join(d, LOG_NAME))
|
||||||
|
reports = load_reports(d)
|
||||||
|
problems = []
|
||||||
|
|
||||||
|
# 1. 每份报告都要有对应的一轮
|
||||||
|
for r in reports:
|
||||||
|
hit = [c for c in runs if c['start'] is not None
|
||||||
|
and abs((c['start'] - r['start']).total_seconds()) <= TOL_S]
|
||||||
|
if not hit:
|
||||||
|
problems.append('报告 %s (started %s) 在日志里找不到对应的开始行'
|
||||||
|
% (r['file'], r['start']))
|
||||||
|
continue
|
||||||
|
c = hit[0]
|
||||||
|
# 结论时刻应当 = 开始 + 耗时
|
||||||
|
expect_end = c['start'] + timedelta(seconds=r['secs'] or 0)
|
||||||
|
if abs((c['end'] - expect_end).total_seconds()) > TOL_S:
|
||||||
|
problems.append('%s 时间对不上: 开始 %s + 报告耗时 %ss 应为 %s, 日志结论在 %s'
|
||||||
|
% (r['file'], c['start'].strftime('%H:%M:%S'), r['secs'],
|
||||||
|
expect_end.strftime('%H:%M:%S'), c['end'].strftime('%H:%M:%S')))
|
||||||
|
if c['secs'] != r['secs']:
|
||||||
|
problems.append('%s 耗时不符: 日志 %ss vs 报告 %ss'
|
||||||
|
% (r['file'], c['secs'], r['secs']))
|
||||||
|
if c['result'] != r['result']:
|
||||||
|
problems.append('%s 结果文本不符:\n 日志 %s\n 报告 %s'
|
||||||
|
% (r['file'], c['result'], r['result']))
|
||||||
|
|
||||||
|
# 2. 每一轮都要有对应报告
|
||||||
|
for c in runs:
|
||||||
|
hit = [r for r in reports
|
||||||
|
if c['start'] is not None
|
||||||
|
and abs((r['start'] - c['start']).total_seconds()) <= TOL_S]
|
||||||
|
if not hit:
|
||||||
|
problems.append('日志 %s 那轮没有对应报告: %s'
|
||||||
|
% (c['end'].strftime('%H:%M:%S'), c['result'][:50]))
|
||||||
|
|
||||||
|
# 3. mock 残留
|
||||||
|
for sec, group in same_second_clusters(runs).items():
|
||||||
|
problems.append('同一秒(%s)出现 %d 行结论, 疑似测试残留'
|
||||||
|
% (sec.strftime('%H:%M:%S'), len(group)))
|
||||||
|
for c in impossible_claims(runs):
|
||||||
|
problems.append('可疑内容(%s): %s'
|
||||||
|
% (c['end'].strftime('%H:%M:%S'), c['result'][:60]))
|
||||||
|
|
||||||
|
return {'runs': len(runs), 'reports': len(reports), 'problems': problems}
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
ap = argparse.ArgumentParser(description='值守体检: 日志与报告交叉核对')
|
||||||
|
ap.add_argument('--dir', default=DIR)
|
||||||
|
ap.add_argument('--json', action='store_true')
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
r = audit(args.dir)
|
||||||
|
if args.json:
|
||||||
|
json.dump(r, sys.stdout, ensure_ascii=False, indent=2)
|
||||||
|
print()
|
||||||
|
return 1 if r['problems'] else 0
|
||||||
|
|
||||||
|
print('日志 %d 轮 / 报告 %d 份' % (r['runs'], r['reports']))
|
||||||
|
if not r['problems']:
|
||||||
|
print('✅ 全部吻合, 没有可疑记录')
|
||||||
|
return 0
|
||||||
|
print('⚠️ 发现 %d 处问题:' % len(r['problems']))
|
||||||
|
for p in r['problems']:
|
||||||
|
print(' - %s' % p)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""audit_runs 的核对逻辑。
|
||||||
|
|
||||||
|
这个工具的职责是"把日志和报告对起来", 所以测试重点不是它能通过, 而是
|
||||||
|
**它能不能抓到问题** —— 一个什么都不报的体检工具等于没有。
|
||||||
|
|
||||||
|
⛔ 第一版就踩过这个坑: 它拿"报告.started + elapsed"去找日志结论行, 能对上就说明
|
||||||
|
elapsed 一致, 于是"耗时不符"那个分支永远不可能触发 —— 是死代码。现在改成按日志里
|
||||||
|
的"=== 开始: 体力换模块 ===" 配对, 时间关系和耗时才能各自独立核对。
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||||
|
import audit_runs as ar # noqa: E402
|
||||||
|
|
||||||
|
START = '=== 开始: 体力换模块 ==='
|
||||||
|
|
||||||
|
|
||||||
|
def start_line(t):
|
||||||
|
return '[%s] %s' % (t, START)
|
||||||
|
|
||||||
|
|
||||||
|
def conclusion(t, result, secs, ok='OK'):
|
||||||
|
return '[%s] === %s (%s), 耗时 %ds ===' % (t, result, ok, secs)
|
||||||
|
|
||||||
|
|
||||||
|
def report(start, secs, result, ok=True):
|
||||||
|
return json.dumps({'started': start, 'elapsed_s': secs, 'ok': ok,
|
||||||
|
'result': result, 'steps': []}, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
class AuditTest(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.d = tempfile.mkdtemp()
|
||||||
|
|
||||||
|
def write_log(self, lines):
|
||||||
|
with open(os.path.join(self.d, 'convert.log'), 'w', encoding='utf-8') as f:
|
||||||
|
f.write('\n'.join(lines) + '\n')
|
||||||
|
|
||||||
|
def write_report(self, name, content):
|
||||||
|
with open(os.path.join(self.d, name), 'w', encoding='utf-8') as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
def problems(self):
|
||||||
|
return ar.audit(self.d)['problems']
|
||||||
|
|
||||||
|
# ---------------------------------------------------------- 正常情况
|
||||||
|
|
||||||
|
def test_matching_run_is_clean(self):
|
||||||
|
self.write_log([start_line('2026-01-01 10:00:00'),
|
||||||
|
conclusion('2026-01-01 10:00:30', '合成完成: 模组 1 -> 2 (+1)', 30)])
|
||||||
|
self.write_report('convert_20260101_100000.json',
|
||||||
|
report('2026-01-01 10:00:00', 30, '合成完成: 模组 1 -> 2 (+1)'))
|
||||||
|
self.assertEqual(self.problems(), [])
|
||||||
|
|
||||||
|
def test_empty_dir_is_clean(self):
|
||||||
|
self.assertEqual(self.problems(), [])
|
||||||
|
|
||||||
|
# ---------------------------------------------------------- 抓问题
|
||||||
|
|
||||||
|
def test_report_without_run_is_flagged(self):
|
||||||
|
self.write_report('convert_20260101_110000.json',
|
||||||
|
report('2026-01-01 11:00:00', 40, '体力不足(3<20), 本轮无需合成'))
|
||||||
|
self.assertTrue(any('找不到对应的开始行' in p for p in self.problems()))
|
||||||
|
|
||||||
|
def test_run_without_report_is_flagged(self):
|
||||||
|
self.write_log([start_line('2026-01-01 12:00:00'),
|
||||||
|
conclusion('2026-01-01 12:00:10', '跑了但没留报告', 10)])
|
||||||
|
self.assertTrue(any('没有对应报告' in p for p in self.problems()))
|
||||||
|
|
||||||
|
def test_elapsed_mismatch_is_flagged(self):
|
||||||
|
"""日志说耗时 30s, 报告说 99s —— 必须能发现。
|
||||||
|
|
||||||
|
老版本这里永远查不出来(死代码), 所以这条用例是专门守它的。
|
||||||
|
"""
|
||||||
|
self.write_log([start_line('2026-01-01 10:00:00'),
|
||||||
|
conclusion('2026-01-01 10:00:30', '合成完成', 30)])
|
||||||
|
self.write_report('convert_20260101_100000.json',
|
||||||
|
report('2026-01-01 10:00:00', 99, '合成完成'))
|
||||||
|
got = self.problems()
|
||||||
|
self.assertTrue(any('耗时不符' in p for p in got), got)
|
||||||
|
|
||||||
|
def test_end_time_mismatch_is_flagged(self):
|
||||||
|
# 开始和耗时都对得上, 但结论时刻对不上 -> 时间关系有问题
|
||||||
|
self.write_log([start_line('2026-01-01 10:00:00'),
|
||||||
|
conclusion('2026-01-01 10:05:00', '合成完成', 30)])
|
||||||
|
self.write_report('convert_20260101_100000.json',
|
||||||
|
report('2026-01-01 10:00:00', 30, '合成完成'))
|
||||||
|
self.assertTrue(any('时间对不上' in p for p in self.problems()))
|
||||||
|
|
||||||
|
def test_result_text_mismatch_is_flagged(self):
|
||||||
|
self.write_log([start_line('2026-01-01 10:00:00'),
|
||||||
|
conclusion('2026-01-01 10:00:30', '合成完成: 模组 1 -> 2 (+1)', 30)])
|
||||||
|
self.write_report('convert_20260101_100000.json',
|
||||||
|
report('2026-01-01 10:00:00', 30, '合成完成: 模组 5 -> 9 (+4)'))
|
||||||
|
self.assertTrue(any('结果文本不符' in p for p in self.problems()))
|
||||||
|
|
||||||
|
def test_same_second_cluster_is_flagged(self):
|
||||||
|
"""同一秒多行结论 = 测试批量产生的典型特征。"""
|
||||||
|
self.write_log([conclusion('2026-01-01 12:00:00', '第一条', 0),
|
||||||
|
conclusion('2026-01-01 12:00:00', '第二条', 0)])
|
||||||
|
self.assertTrue(any('同一秒' in p for p in self.problems()))
|
||||||
|
|
||||||
|
def test_impossible_content_is_flagged(self):
|
||||||
|
self.write_log([start_line('2026-01-01 12:00:00'),
|
||||||
|
conclusion('2026-01-01 12:00:05', '看门狗执行失败: powershell not found', 5)])
|
||||||
|
self.assertTrue(any('可疑内容' in p for p in self.problems()))
|
||||||
|
|
||||||
|
# ---------------------------------------------------------- 解析
|
||||||
|
|
||||||
|
def test_parse_pairs_start_with_conclusion(self):
|
||||||
|
self.write_log([
|
||||||
|
'[2026-01-01 09:00:00] === 开始: 体力换模块 ===',
|
||||||
|
'[2026-01-01 09:00:05] 游戏已在运行',
|
||||||
|
conclusion('2026-01-01 09:01:00', '合成完成', 60),
|
||||||
|
])
|
||||||
|
runs = ar.parse_runs(os.path.join(self.d, 'convert.log'))
|
||||||
|
self.assertEqual(len(runs), 1)
|
||||||
|
self.assertEqual(runs[0]['secs'], 60)
|
||||||
|
self.assertEqual(runs[0]['start'].strftime('%H:%M:%S'), '09:00:00')
|
||||||
|
|
||||||
|
def test_parse_handles_missing_file(self):
|
||||||
|
self.assertEqual(ar.parse_runs(os.path.join(self.d, 'nope.log')), [])
|
||||||
|
|
||||||
|
def test_conclusion_without_start_line_still_listed(self):
|
||||||
|
# 没有开始行的结论不该被丢掉(要能报"没有对应报告")
|
||||||
|
self.write_log([conclusion('2026-01-01 12:00:00', '独苗', 10)])
|
||||||
|
runs = ar.parse_runs(os.path.join(self.d, 'convert.log'))
|
||||||
|
self.assertEqual(len(runs), 1)
|
||||||
|
self.assertIsNone(runs[0]['start'])
|
||||||
|
|
||||||
|
def test_skip_broken_report_json(self):
|
||||||
|
with open(os.path.join(self.d, 'convert_20260101_100000.json'), 'w', encoding='utf-8') as f:
|
||||||
|
f.write('{ 半截')
|
||||||
|
self.assertEqual(ar.load_reports(self.d), [])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user