Add execution-model decision handoff and Jev AALC research

Browse Source
This commit is contained in:
Codex
2026-10-03 07:08:38 +08:00
parent 7188b4ac0f
commit 6c2d26e629
11 changed files with 958 additions and 27 deletions
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""Summarize recorded MCP actions without starting the game or importing AALC.
Only records containing an action object contribute to action/latency statistics.
These records do not prove that the game accepted an input. Missing timings
remain missing: older logs cannot separate observation and execution.
Positive jev_ms values are recorded request latencies, not proof of a successful
decision or an exhaustive account of all API attempts.
"""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
import math
from pathlib import Path
import statistics
def latency_summary(records, field):
values = []
invalid = 0
for record in records:
if field not in record:
continue
value = record[field]
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
invalid += 1
continue
values.append(value)
positive = sorted(value for value in values if value > 0)
return {
'present_records': len(values),
'missing_records': sum(field not in record for record in records),
'invalid_records': invalid,
'zero_records': sum(value == 0 for value in values),
'positive_records': len(positive),
'sum_ms': round(sum(values), 3) if values else None,
'positive_median_ms': statistics.median(positive) if positive else None,
'positive_p95_nearest_rank_ms': positive[math.ceil(len(positive) * .95) - 1] if positive else None,
'positive_max_ms': positive[-1] if positive else None,
}
def summarize(path):
path = Path(path)
raw = path.read_bytes()
records = []
for line_number, line in enumerate(raw.decode('utf-8-sig').splitlines(), 1):
if not line.strip():
continue
value = json.loads(line)
if not isinstance(value, dict):
raise ValueError(f'{path}:{line_number}: expected a JSON object')
records.append(value)
actions = [row for row in records if isinstance(row.get('action'), dict)]
tools = Counter(str(row['action'].get('tool', 'unknown')) for row in actions)
states = Counter(str(row.get('state', 'missing')) for row in records)
battle_pairs = []
for row in actions:
if row['action'].get('tool') != 'battle':
continue
result = row.get('result')
value = result.get('pairs') if isinstance(result, dict) else None
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
battle_pairs.append(value)
# Split aware/naive timestamps rather than silently invent a timezone.
timestamp_groups = {'naive_local': [], 'aware': []}
bad_timestamps = 0
for row in actions:
stamp = row.get('at')
if not isinstance(stamp, str):
bad_timestamps += 1
continue
try:
parsed = datetime.fromisoformat(stamp.replace('Z', '+00:00'))
except ValueError:
bad_timestamps += 1
continue
group = 'aware' if parsed.utcoffset() is not None else 'naive_local'
timestamp_groups[group].append(parsed)
spans = {}
for name, stamps in timestamp_groups.items():
if not stamps:
continue
ordered = sorted(stamps)
spans[name] = {
'records': len(ordered),
'first': ordered[0].isoformat(),
'last': ordered[-1].isoformat(),
'span_seconds': (ordered[-1] - ordered[0]).total_seconds(),
}
return {
'source': path.as_posix(),
'sha256': hashlib.sha256(raw).hexdigest(),
'records': len(records),
'action_records': len(actions),
'tool_counts': dict(sorted(tools.items())),
'state_counts': dict(sorted(states.items())),
'recorded_latency_ms': {field: latency_summary(actions, field) for field in ('jev_ms', 'ocr_ms', 'action_ms')},
'battle_pair_attempts': {
'reported_batches': len(battle_pairs),
'unreported_batches': tools.get('battle', 0) - len(battle_pairs),
'sum': sum(battle_pairs) if battle_pairs else None,
},
'action_timestamp_spans': spans,
'missing_or_invalid_action_timestamps': bad_timestamps,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('logs', nargs='+', type=Path, help='JSONL action logs')
parser.add_argument('--output', type=Path, help='Save the summary as JSON; otherwise print it')
args = parser.parse_args()
try:
report = {'schema_version': 1, 'logs': [summarize(path) for path in args.logs]}
except (OSError, ValueError) as error:
parser.error(str(error))
payload = json.dumps(report, ensure_ascii=False, indent=2) + '\n'
if args.output:
args.output.write_text(payload, encoding='utf-8')
else:
print(payload, end='')
if __name__ == '__main__':
main()
+132
View File
@@ -0,0 +1,132 @@
"""Select a decision provider without giving providers control of game input.
An embedding controller can supply executor_decider(request) for its active
model. Standalone scripts publish DecisionRequired for their caller to handle;
they cannot invoke the caller's model themselves.
"""
from dataclasses import dataclass
import json
import math
import time
import urllib.error
import uuid
class DecisionRequired(RuntimeError):
def __init__(self, request):
super().__init__('The execution model must choose a candidate')
self.request = request
@dataclass
class Decision:
choice: str
engine: str
latency_ms: int
jev_ms: int = 0
confidence: float | None = None
probabilities: dict | None = None
model: str | None = None
fallback_reason: str | None = None
failed_jev_ms: int | None = None
def _answer(answer, options):
if isinstance(answer, str):
answer = {'choice': answer, 'latency_ms': 0}
elif isinstance(answer, tuple) and len(answer) == 2:
answer = {'choice': answer[0], 'latency_ms': answer[1]}
if not isinstance(answer, dict) or not isinstance(answer.get('choice'), str) or answer['choice'] not in options:
raise ValueError('Decision provider returned an unsupported candidate')
latency = answer.get('latency_ms', 0)
if isinstance(latency, bool) or not isinstance(latency, (int, float)) or not math.isfinite(latency) or latency < 0:
raise ValueError('Decision latency must be a finite non-negative number')
confidence = answer.get('confidence')
if confidence is not None and (isinstance(confidence, bool) or not isinstance(confidence, (int, float))
or not math.isfinite(confidence) or not 0 <= confidence <= 1):
raise ValueError('Decision confidence must be between 0 and 1')
probabilities = answer.get('probabilities')
if probabilities is not None:
if not isinstance(probabilities, dict) or set(probabilities) != set(options):
raise ValueError('Decision probabilities must describe the supplied candidates')
if any(isinstance(p, bool) or not isinstance(p, (int, float)) or not math.isfinite(p) or not 0 <= p <= 1
for p in probabilities.values()) or not math.isclose(sum(probabilities.values()), 1, abs_tol=.01):
raise ValueError('Decision probabilities must form a probability distribution')
return answer['choice'], round(latency), confidence, probabilities, answer.get('model')
class DecisionRouter:
def __init__(self, config, jev_decider, executor_decider=None, fixed_decider=None,
rules='', force_fixed=False):
self.config = dict(config)
self.mode = 'fixed' if force_fixed else self.config.get('decisionEngine', 'auto')
if self.mode not in ('auto', 'jev', 'executor', 'fixed'):
raise ValueError('decisionEngine must be auto, jev, executor or fixed')
self.jev_decider = jev_decider
self.executor_decider = executor_decider
self.fixed_decider = fixed_decider
self.rules = rules
def choose(self, state, options, recent=(), run_id=None):
if not options:
raise ValueError('Decision requires at least one candidate')
if self.mode == 'fixed':
if self.fixed_decider is None:
raise ValueError('Fixed decision provider is not configured')
choice, latency, confidence, probabilities, model = _answer(self.fixed_decider(options), options)
return Decision(choice, 'fixed', latency, confidence=confidence, probabilities=probabilities, model=model)
reason = 'configured_executor'
model = self.config.get('model')
configured_model = isinstance(model, str) and bool(model.strip())
if self.mode in ('auto', 'jev'):
if not configured_model:
reason = 'no_jev_model'
elif not self.config.get('apiKey'):
reason = 'missing_jev_credentials'
else:
started = time.monotonic()
try:
answer = self.jev_decider(self.config, state, options, recent)
choice, latency, confidence, probabilities, returned_model = _answer(answer, options)
return Decision(choice, 'jev', latency, jev_ms=latency, confidence=confidence,
probabilities=probabilities, model=returned_model or model)
except urllib.error.HTTPError as error:
reason = 'jev_http_%d' % error.code
except (OSError, TimeoutError, ValueError) as error:
reason = 'jev_unavailable_%s' % type(error).__name__
failed_jev_ms = round((time.monotonic() - started) * 1000)
else:
configured_model = False
request = {
'schema_version': 1,
'request_id': uuid.uuid4().hex,
'run_id': run_id,
'reason': reason,
'requested_engine': 'executor',
'page_state': state,
'candidates': options,
'rules': self.rules,
'recent': list(recent)[-4:],
'execution_contract': {
'response': {'request_id': '<same request_id>', 'choice': '<candidate ID>'},
'requires_fresh_observation_before_external_input': True,
'note': 'The caller handles this decision. No game input is sent while handing off.',
},
}
if self.mode in ('auto', 'jev') and configured_model and self.config.get('apiKey'):
request['failed_jev_ms'] = failed_jev_ms
# The request contains observations and candidates, never provider config
# or credentials. Detach it so the callback cannot alter executable actions.
request = json.loads(json.dumps(request, ensure_ascii=False))
if self.executor_decider is None:
raise DecisionRequired(request)
started = time.monotonic()
expected_request_id = request['request_id']
answer = self.executor_decider(request)
if isinstance(answer, dict) and answer.get('request_id') != expected_request_id:
raise ValueError('Execution-model response belongs to a different decision request')
choice, _, confidence, probabilities, returned_model = _answer(answer, options)
latency = round((time.monotonic() - started) * 1000)
return Decision(choice, 'executor', latency, confidence=confidence, probabilities=probabilities,
model=returned_model, fallback_reason=reason, failed_jev_ms=request.get('failed_jev_ms'))
+40 -19
View File
@@ -14,6 +14,7 @@ from pathlib import Path
from mcp_session import Session as Bridge, SessionCancelled, SessionTimeout
from game_scenes import battle_is_terminal, battle_is_visible, death_retry_screen
from run_state import prepare_run, save_json, record_claim, claim_is_current, complete_run
from decision_router import DecisionRouter, DecisionRequired
REPO_ROOT = Path(__file__).resolve().parents[1]
RUNTIME_HOME = Path(os.environ.get('LC_JEV_HOME', str(Path.home() / 'lc_mcp' / 'jev')))
@@ -540,7 +541,7 @@ def candidates(state, previous_battle=False, refresh_used=0):
def decide(cfg, state, options, recent):
body = {'model': 'jev-1.13-free',
body = {'model': cfg['model'],
'state': '目标:按用户已配置的队伍和速通规则推进当前镜牢。当前楼层:%s\n当前游戏 OCR(坐标仅供辨识):\n' % state.get('floor', '尚未识别') +
'\n'.join('%s @%d,%d' % (i['text'], i['x'], i['y']) for i in text_items(state)
if not i['text'].upper().startswith('STEAM')) +
@@ -572,7 +573,9 @@ def decide(cfg, state, options, recent):
choice = answer.get('choice')
if answer.get('type') != 'choice' or choice not in options:
raise ValueError('Jev returned an unsupported candidate')
return choice, round((time.monotonic() - start) * 1000)
return {'choice': choice, 'latency_ms': round((time.monotonic() - start) * 1000),
'confidence': answer.get('confidence'), 'probabilities': answer.get('probabilities'),
'model': result.get('model') or cfg['model']}
def fixed_fallback(options):
@@ -677,9 +680,10 @@ def execute(bridge, action, deadline=None):
return call(action['tool'], action.get('args'))
def main():
def main(executor_decider=None):
parser = argparse.ArgumentParser()
parser.add_argument('config')
parser.add_argument('config', nargs='?', help='Optional Jev JSON config; omit for execution-model handoff')
parser.add_argument('--decision-engine', choices=('auto', 'jev', 'executor', 'fixed'))
parser.add_argument('--steps', type=int, default=250)
parser.add_argument('--seconds', type=int, default=1200)
mode = parser.add_mutually_exclusive_group()
@@ -710,13 +714,15 @@ def main():
stable_since = time.monotonic()
last_fingerprint = None
unknown_since = None
decision_fallback_reason = None
try:
RUNTIME_HOME.mkdir(parents=True, exist_ok=True)
cfg = json.loads(Path(args.config).read_text(encoding='utf-8-sig'))
cfg = json.loads(Path(args.config).read_text(encoding='utf-8-sig')) if args.config else {}
cfg['apiKey'] = os.environ.get('OPENCODE_API_KEY') or cfg.get('apiKey')
if not cfg['apiKey'] and os.environ.get('LC_FIXED_FALLBACK') != '1':
raise RuntimeError('Set OPENCODE_API_KEY or enable LC_FIXED_FALLBACK=1')
if args.decision_engine:
cfg['decisionEngine'] = args.decision_engine
router = DecisionRouter(cfg, decide, executor_decider=executor_decider,
fixed_decider=fixed_fallback, rules=SYSTEM_RULES,
force_fixed=os.environ.get('LC_FIXED_FALLBACK') == '1')
if STOP_FILE.exists():
save_json(status, {'state': 'stopped', 'reason': 'stop_requested'})
return
@@ -779,6 +785,7 @@ def main():
unknown_since = None
# User explicitly prescribed the complete battle loop. Only scene
# transitions return to Jev; combat does not require new decisions.
decision = None
if any(re.search(r'CON+ECTING|LOADING', i['text'].upper()) for i in text_items(state)):
choice, latency = 'wait', 0
elif len(options)==2 and time.monotonic() - unknown_since < 10:
@@ -791,22 +798,36 @@ def main():
choice, latency = 'event_advance', 0
elif 'battle' in options:
choice, latency = 'battle', 0
else:
if os.environ.get('LC_FIXED_FALLBACK') == '1' or decision_fallback_reason:
choice, latency = fixed_fallback(options), 0
else:
try:
choice, latency = decide(cfg, state, options, recent)
except urllib.error.HTTPError as error:
if error.code not in (401,403,429):
raise
decision_fallback_reason='Jev HTTP %d; deterministic fallback for this session' % error.code
choice, latency=fixed_fallback(options),0
decision = router.choose(state, options, recent, run['run_id'])
choice, latency = decision.choice, decision.jev_ms
except DecisionRequired as required:
record = {'state': 'requires_decision', 'step': step, 'run_id': run['run_id'],
'decision_engine': 'executor', 'decision_request': required.request,
'ocr_ms': observation_ms, 'screen': state.get('screen'),
'at': time.strftime('%Y-%m-%dT%H:%M:%S')}
if 'failed_jev_ms' in required.request:
record['failed_jev_ms'] = required.request['failed_jev_ms']
run['status'] = 'requires_decision'
save_json(RUNTIME_HOME / 'mirror_current_run.json', run)
save_json(RUNTIME_HOME / 'decision_request.json', required.request)
save_json(status, record)
with log.open('a', encoding='utf-8') as output:
output.write(json.dumps(record, ensure_ascii=False) + '\n')
print(json.dumps(record, ensure_ascii=False), flush=True)
return
action = options[choice]['action']
record = {'step': step, 'run_id': run['run_id'], 'choice': choice, 'action': action, 'jev_ms': latency,
'ocr_ms':observation_ms,'screen': state.get('screen'), 'at': time.strftime('%Y-%m-%dT%H:%M:%S')}
if decision_fallback_reason:
record['decision_fallback_reason']=decision_fallback_reason
record['decision_engine'] = decision.engine if decision else 'rules'
if decision:
record.update(decision_ms=decision.latency_ms + (decision.failed_jev_ms or 0), confidence=decision.confidence,
probabilities=decision.probabilities, model=decision.model)
if decision.failed_jev_ms is not None:
record['failed_jev_ms'] = decision.failed_jev_ms
if decision.fallback_reason:
record['decision_fallback_reason'] = decision.fallback_reason
if choice == 'run_complete':
complete_run(RUNTIME_HOME, run, context, state.get('screen'))
record['state'] = 'completed'
+1 -1
View File
@@ -6,7 +6,7 @@ import uuid
from datetime import datetime, timezone
from pathlib import Path
RESUMABLE = {'running', 'requires_analysis', 'stopped', 'user_takeover'}
RESUMABLE = {'running', 'requires_analysis', 'requires_decision', 'stopped', 'user_takeover'}
def save_json(path, value):