Files
lc-mcp/scripts/analyze_feedback.py

129 lines
4.9 KiB
Python

#!/usr/bin/env python3
"""Summarize recorded MCP actions without starting the game or importing AALC.
Only records containing an action object contribute to action/latency statistics.
These records do not prove that the game accepted an input. Missing timings
remain missing: older logs cannot separate observation and execution.
Positive jev_ms values are recorded request latencies, not proof of a successful
decision or an exhaustive account of all API attempts.
"""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
import math
from pathlib import Path
import statistics
def latency_summary(records, field):
values = []
invalid = 0
for record in records:
if field not in record:
continue
value = record[field]
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
invalid += 1
continue
values.append(value)
positive = sorted(value for value in values if value > 0)
return {
'present_records': len(values),
'missing_records': sum(field not in record for record in records),
'invalid_records': invalid,
'zero_records': sum(value == 0 for value in values),
'positive_records': len(positive),
'sum_ms': round(sum(values), 3) if values else None,
'positive_median_ms': statistics.median(positive) if positive else None,
'positive_p95_nearest_rank_ms': positive[math.ceil(len(positive) * .95) - 1] if positive else None,
'positive_max_ms': positive[-1] if positive else None,
}
def summarize(path):
path = Path(path)
raw = path.read_bytes()
records = []
for line_number, line in enumerate(raw.decode('utf-8-sig').splitlines(), 1):
if not line.strip():
continue
value = json.loads(line)
if not isinstance(value, dict):
raise ValueError(f'{path}:{line_number}: expected a JSON object')
records.append(value)
actions = [row for row in records if isinstance(row.get('action'), dict)]
tools = Counter(str(row['action'].get('tool', 'unknown')) for row in actions)
states = Counter(str(row.get('state', 'missing')) for row in records)
battle_pairs = []
for row in actions:
if row['action'].get('tool') != 'battle':
continue
result = row.get('result')
value = result.get('pairs') if isinstance(result, dict) else None
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
battle_pairs.append(value)
# Split aware/naive timestamps rather than silently invent a timezone.
timestamp_groups = {'naive_local': [], 'aware': []}
bad_timestamps = 0
for row in actions:
stamp = row.get('at')
if not isinstance(stamp, str):
bad_timestamps += 1
continue
try:
parsed = datetime.fromisoformat(stamp.replace('Z', '+00:00'))
except ValueError:
bad_timestamps += 1
continue
group = 'aware' if parsed.utcoffset() is not None else 'naive_local'
timestamp_groups[group].append(parsed)
spans = {}
for name, stamps in timestamp_groups.items():
if not stamps:
continue
ordered = sorted(stamps)
spans[name] = {
'records': len(ordered),
'first': ordered[0].isoformat(),
'last': ordered[-1].isoformat(),
'span_seconds': (ordered[-1] - ordered[0]).total_seconds(),
}
return {
'source': path.as_posix(),
'sha256': hashlib.sha256(raw).hexdigest(),
'records': len(records),
'action_records': len(actions),
'tool_counts': dict(sorted(tools.items())),
'state_counts': dict(sorted(states.items())),
'recorded_latency_ms': {field: latency_summary(actions, field) for field in ('jev_ms', 'ocr_ms', 'action_ms')},
'battle_pair_attempts': {
'reported_batches': len(battle_pairs),
'unreported_batches': tools.get('battle', 0) - len(battle_pairs),
'sum': sum(battle_pairs) if battle_pairs else None,
},
'action_timestamp_spans': spans,
'missing_or_invalid_action_timestamps': bad_timestamps,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('logs', nargs='+', type=Path, help='JSONL action logs')
parser.add_argument('--output', type=Path, help='Save the summary as JSON; otherwise print it')
args = parser.parse_args()
try:
report = {'schema_version': 1, 'logs': [summarize(path) for path in args.logs]}
except (OSError, ValueError) as error:
parser.error(str(error))
payload = json.dumps(report, ensure_ascii=False, indent=2) + '\n'
if args.output:
args.output.write_text(payload, encoding='utf-8')
else:
print(payload, end='')
if __name__ == '__main__':
main()