Add execution-model decision handoff and Jev AALC research
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize recorded MCP actions without starting the game or importing AALC.
|
||||
|
||||
Only records containing an action object contribute to action/latency statistics.
|
||||
These records do not prove that the game accepted an input. Missing timings
|
||||
remain missing: older logs cannot separate observation and execution.
|
||||
Positive jev_ms values are recorded request latencies, not proof of a successful
|
||||
decision or an exhaustive account of all API attempts.
|
||||
"""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
|
||||
def latency_summary(records, field):
|
||||
values = []
|
||||
invalid = 0
|
||||
for record in records:
|
||||
if field not in record:
|
||||
continue
|
||||
value = record[field]
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
|
||||
invalid += 1
|
||||
continue
|
||||
values.append(value)
|
||||
positive = sorted(value for value in values if value > 0)
|
||||
return {
|
||||
'present_records': len(values),
|
||||
'missing_records': sum(field not in record for record in records),
|
||||
'invalid_records': invalid,
|
||||
'zero_records': sum(value == 0 for value in values),
|
||||
'positive_records': len(positive),
|
||||
'sum_ms': round(sum(values), 3) if values else None,
|
||||
'positive_median_ms': statistics.median(positive) if positive else None,
|
||||
'positive_p95_nearest_rank_ms': positive[math.ceil(len(positive) * .95) - 1] if positive else None,
|
||||
'positive_max_ms': positive[-1] if positive else None,
|
||||
}
|
||||
|
||||
|
||||
def summarize(path):
|
||||
path = Path(path)
|
||||
raw = path.read_bytes()
|
||||
records = []
|
||||
for line_number, line in enumerate(raw.decode('utf-8-sig').splitlines(), 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
value = json.loads(line)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f'{path}:{line_number}: expected a JSON object')
|
||||
records.append(value)
|
||||
actions = [row for row in records if isinstance(row.get('action'), dict)]
|
||||
tools = Counter(str(row['action'].get('tool', 'unknown')) for row in actions)
|
||||
states = Counter(str(row.get('state', 'missing')) for row in records)
|
||||
battle_pairs = []
|
||||
for row in actions:
|
||||
if row['action'].get('tool') != 'battle':
|
||||
continue
|
||||
result = row.get('result')
|
||||
value = result.get('pairs') if isinstance(result, dict) else None
|
||||
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
|
||||
battle_pairs.append(value)
|
||||
# Split aware/naive timestamps rather than silently invent a timezone.
|
||||
timestamp_groups = {'naive_local': [], 'aware': []}
|
||||
bad_timestamps = 0
|
||||
for row in actions:
|
||||
stamp = row.get('at')
|
||||
if not isinstance(stamp, str):
|
||||
bad_timestamps += 1
|
||||
continue
|
||||
try:
|
||||
parsed = datetime.fromisoformat(stamp.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
bad_timestamps += 1
|
||||
continue
|
||||
group = 'aware' if parsed.utcoffset() is not None else 'naive_local'
|
||||
timestamp_groups[group].append(parsed)
|
||||
spans = {}
|
||||
for name, stamps in timestamp_groups.items():
|
||||
if not stamps:
|
||||
continue
|
||||
ordered = sorted(stamps)
|
||||
spans[name] = {
|
||||
'records': len(ordered),
|
||||
'first': ordered[0].isoformat(),
|
||||
'last': ordered[-1].isoformat(),
|
||||
'span_seconds': (ordered[-1] - ordered[0]).total_seconds(),
|
||||
}
|
||||
return {
|
||||
'source': path.as_posix(),
|
||||
'sha256': hashlib.sha256(raw).hexdigest(),
|
||||
'records': len(records),
|
||||
'action_records': len(actions),
|
||||
'tool_counts': dict(sorted(tools.items())),
|
||||
'state_counts': dict(sorted(states.items())),
|
||||
'recorded_latency_ms': {field: latency_summary(actions, field) for field in ('jev_ms', 'ocr_ms', 'action_ms')},
|
||||
'battle_pair_attempts': {
|
||||
'reported_batches': len(battle_pairs),
|
||||
'unreported_batches': tools.get('battle', 0) - len(battle_pairs),
|
||||
'sum': sum(battle_pairs) if battle_pairs else None,
|
||||
},
|
||||
'action_timestamp_spans': spans,
|
||||
'missing_or_invalid_action_timestamps': bad_timestamps,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('logs', nargs='+', type=Path, help='JSONL action logs')
|
||||
parser.add_argument('--output', type=Path, help='Save the summary as JSON; otherwise print it')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
report = {'schema_version': 1, 'logs': [summarize(path) for path in args.logs]}
|
||||
except (OSError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
payload = json.dumps(report, ensure_ascii=False, indent=2) + '\n'
|
||||
if args.output:
|
||||
args.output.write_text(payload, encoding='utf-8')
|
||||
else:
|
||||
print(payload, end='')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Select a decision provider without giving providers control of game input.
|
||||
|
||||
An embedding controller can supply executor_decider(request) for its active
|
||||
model. Standalone scripts publish DecisionRequired for their caller to handle;
|
||||
they cannot invoke the caller's model themselves.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
import urllib.error
|
||||
import uuid
|
||||
|
||||
|
||||
class DecisionRequired(RuntimeError):
|
||||
def __init__(self, request):
|
||||
super().__init__('The execution model must choose a candidate')
|
||||
self.request = request
|
||||
|
||||
|
||||
@dataclass
|
||||
class Decision:
|
||||
choice: str
|
||||
engine: str
|
||||
latency_ms: int
|
||||
jev_ms: int = 0
|
||||
confidence: float | None = None
|
||||
probabilities: dict | None = None
|
||||
model: str | None = None
|
||||
fallback_reason: str | None = None
|
||||
failed_jev_ms: int | None = None
|
||||
|
||||
|
||||
def _answer(answer, options):
|
||||
if isinstance(answer, str):
|
||||
answer = {'choice': answer, 'latency_ms': 0}
|
||||
elif isinstance(answer, tuple) and len(answer) == 2:
|
||||
answer = {'choice': answer[0], 'latency_ms': answer[1]}
|
||||
if not isinstance(answer, dict) or not isinstance(answer.get('choice'), str) or answer['choice'] not in options:
|
||||
raise ValueError('Decision provider returned an unsupported candidate')
|
||||
latency = answer.get('latency_ms', 0)
|
||||
if isinstance(latency, bool) or not isinstance(latency, (int, float)) or not math.isfinite(latency) or latency < 0:
|
||||
raise ValueError('Decision latency must be a finite non-negative number')
|
||||
confidence = answer.get('confidence')
|
||||
if confidence is not None and (isinstance(confidence, bool) or not isinstance(confidence, (int, float))
|
||||
or not math.isfinite(confidence) or not 0 <= confidence <= 1):
|
||||
raise ValueError('Decision confidence must be between 0 and 1')
|
||||
probabilities = answer.get('probabilities')
|
||||
if probabilities is not None:
|
||||
if not isinstance(probabilities, dict) or set(probabilities) != set(options):
|
||||
raise ValueError('Decision probabilities must describe the supplied candidates')
|
||||
if any(isinstance(p, bool) or not isinstance(p, (int, float)) or not math.isfinite(p) or not 0 <= p <= 1
|
||||
for p in probabilities.values()) or not math.isclose(sum(probabilities.values()), 1, abs_tol=.01):
|
||||
raise ValueError('Decision probabilities must form a probability distribution')
|
||||
return answer['choice'], round(latency), confidence, probabilities, answer.get('model')
|
||||
|
||||
|
||||
class DecisionRouter:
|
||||
def __init__(self, config, jev_decider, executor_decider=None, fixed_decider=None,
|
||||
rules='', force_fixed=False):
|
||||
self.config = dict(config)
|
||||
self.mode = 'fixed' if force_fixed else self.config.get('decisionEngine', 'auto')
|
||||
if self.mode not in ('auto', 'jev', 'executor', 'fixed'):
|
||||
raise ValueError('decisionEngine must be auto, jev, executor or fixed')
|
||||
self.jev_decider = jev_decider
|
||||
self.executor_decider = executor_decider
|
||||
self.fixed_decider = fixed_decider
|
||||
self.rules = rules
|
||||
|
||||
def choose(self, state, options, recent=(), run_id=None):
|
||||
if not options:
|
||||
raise ValueError('Decision requires at least one candidate')
|
||||
if self.mode == 'fixed':
|
||||
if self.fixed_decider is None:
|
||||
raise ValueError('Fixed decision provider is not configured')
|
||||
choice, latency, confidence, probabilities, model = _answer(self.fixed_decider(options), options)
|
||||
return Decision(choice, 'fixed', latency, confidence=confidence, probabilities=probabilities, model=model)
|
||||
|
||||
reason = 'configured_executor'
|
||||
model = self.config.get('model')
|
||||
configured_model = isinstance(model, str) and bool(model.strip())
|
||||
if self.mode in ('auto', 'jev'):
|
||||
if not configured_model:
|
||||
reason = 'no_jev_model'
|
||||
elif not self.config.get('apiKey'):
|
||||
reason = 'missing_jev_credentials'
|
||||
else:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
answer = self.jev_decider(self.config, state, options, recent)
|
||||
choice, latency, confidence, probabilities, returned_model = _answer(answer, options)
|
||||
return Decision(choice, 'jev', latency, jev_ms=latency, confidence=confidence,
|
||||
probabilities=probabilities, model=returned_model or model)
|
||||
except urllib.error.HTTPError as error:
|
||||
reason = 'jev_http_%d' % error.code
|
||||
except (OSError, TimeoutError, ValueError) as error:
|
||||
reason = 'jev_unavailable_%s' % type(error).__name__
|
||||
failed_jev_ms = round((time.monotonic() - started) * 1000)
|
||||
else:
|
||||
configured_model = False
|
||||
request = {
|
||||
'schema_version': 1,
|
||||
'request_id': uuid.uuid4().hex,
|
||||
'run_id': run_id,
|
||||
'reason': reason,
|
||||
'requested_engine': 'executor',
|
||||
'page_state': state,
|
||||
'candidates': options,
|
||||
'rules': self.rules,
|
||||
'recent': list(recent)[-4:],
|
||||
'execution_contract': {
|
||||
'response': {'request_id': '<same request_id>', 'choice': '<candidate ID>'},
|
||||
'requires_fresh_observation_before_external_input': True,
|
||||
'note': 'The caller handles this decision. No game input is sent while handing off.',
|
||||
},
|
||||
}
|
||||
if self.mode in ('auto', 'jev') and configured_model and self.config.get('apiKey'):
|
||||
request['failed_jev_ms'] = failed_jev_ms
|
||||
# The request contains observations and candidates, never provider config
|
||||
# or credentials. Detach it so the callback cannot alter executable actions.
|
||||
request = json.loads(json.dumps(request, ensure_ascii=False))
|
||||
if self.executor_decider is None:
|
||||
raise DecisionRequired(request)
|
||||
started = time.monotonic()
|
||||
expected_request_id = request['request_id']
|
||||
answer = self.executor_decider(request)
|
||||
if isinstance(answer, dict) and answer.get('request_id') != expected_request_id:
|
||||
raise ValueError('Execution-model response belongs to a different decision request')
|
||||
choice, _, confidence, probabilities, returned_model = _answer(answer, options)
|
||||
latency = round((time.monotonic() - started) * 1000)
|
||||
return Decision(choice, 'executor', latency, confidence=confidence, probabilities=probabilities,
|
||||
model=returned_model, fallback_reason=reason, failed_jev_ms=request.get('failed_jev_ms'))
|
||||
+40
-19
@@ -14,6 +14,7 @@ from pathlib import Path
|
||||
from mcp_session import Session as Bridge, SessionCancelled, SessionTimeout
|
||||
from game_scenes import battle_is_terminal, battle_is_visible, death_retry_screen
|
||||
from run_state import prepare_run, save_json, record_claim, claim_is_current, complete_run
|
||||
from decision_router import DecisionRouter, DecisionRequired
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUNTIME_HOME = Path(os.environ.get('LC_JEV_HOME', str(Path.home() / 'lc_mcp' / 'jev')))
|
||||
@@ -540,7 +541,7 @@ def candidates(state, previous_battle=False, refresh_used=0):
|
||||
|
||||
|
||||
def decide(cfg, state, options, recent):
|
||||
body = {'model': 'jev-1.13-free',
|
||||
body = {'model': cfg['model'],
|
||||
'state': '目标:按用户已配置的队伍和速通规则推进当前镜牢。当前楼层:%s\n当前游戏 OCR(坐标仅供辨识):\n' % state.get('floor', '尚未识别') +
|
||||
'\n'.join('%s @%d,%d' % (i['text'], i['x'], i['y']) for i in text_items(state)
|
||||
if not i['text'].upper().startswith('STEAM')) +
|
||||
@@ -572,7 +573,9 @@ def decide(cfg, state, options, recent):
|
||||
choice = answer.get('choice')
|
||||
if answer.get('type') != 'choice' or choice not in options:
|
||||
raise ValueError('Jev returned an unsupported candidate')
|
||||
return choice, round((time.monotonic() - start) * 1000)
|
||||
return {'choice': choice, 'latency_ms': round((time.monotonic() - start) * 1000),
|
||||
'confidence': answer.get('confidence'), 'probabilities': answer.get('probabilities'),
|
||||
'model': result.get('model') or cfg['model']}
|
||||
|
||||
|
||||
def fixed_fallback(options):
|
||||
@@ -677,9 +680,10 @@ def execute(bridge, action, deadline=None):
|
||||
return call(action['tool'], action.get('args'))
|
||||
|
||||
|
||||
def main():
|
||||
def main(executor_decider=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('config')
|
||||
parser.add_argument('config', nargs='?', help='Optional Jev JSON config; omit for execution-model handoff')
|
||||
parser.add_argument('--decision-engine', choices=('auto', 'jev', 'executor', 'fixed'))
|
||||
parser.add_argument('--steps', type=int, default=250)
|
||||
parser.add_argument('--seconds', type=int, default=1200)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
@@ -710,13 +714,15 @@ def main():
|
||||
stable_since = time.monotonic()
|
||||
last_fingerprint = None
|
||||
unknown_since = None
|
||||
decision_fallback_reason = None
|
||||
try:
|
||||
RUNTIME_HOME.mkdir(parents=True, exist_ok=True)
|
||||
cfg = json.loads(Path(args.config).read_text(encoding='utf-8-sig'))
|
||||
cfg = json.loads(Path(args.config).read_text(encoding='utf-8-sig')) if args.config else {}
|
||||
cfg['apiKey'] = os.environ.get('OPENCODE_API_KEY') or cfg.get('apiKey')
|
||||
if not cfg['apiKey'] and os.environ.get('LC_FIXED_FALLBACK') != '1':
|
||||
raise RuntimeError('Set OPENCODE_API_KEY or enable LC_FIXED_FALLBACK=1')
|
||||
if args.decision_engine:
|
||||
cfg['decisionEngine'] = args.decision_engine
|
||||
router = DecisionRouter(cfg, decide, executor_decider=executor_decider,
|
||||
fixed_decider=fixed_fallback, rules=SYSTEM_RULES,
|
||||
force_fixed=os.environ.get('LC_FIXED_FALLBACK') == '1')
|
||||
if STOP_FILE.exists():
|
||||
save_json(status, {'state': 'stopped', 'reason': 'stop_requested'})
|
||||
return
|
||||
@@ -779,6 +785,7 @@ def main():
|
||||
unknown_since = None
|
||||
# User explicitly prescribed the complete battle loop. Only scene
|
||||
# transitions return to Jev; combat does not require new decisions.
|
||||
decision = None
|
||||
if any(re.search(r'CON+ECTING|LOADING', i['text'].upper()) for i in text_items(state)):
|
||||
choice, latency = 'wait', 0
|
||||
elif len(options)==2 and time.monotonic() - unknown_since < 10:
|
||||
@@ -791,22 +798,36 @@ def main():
|
||||
choice, latency = 'event_advance', 0
|
||||
elif 'battle' in options:
|
||||
choice, latency = 'battle', 0
|
||||
else:
|
||||
if os.environ.get('LC_FIXED_FALLBACK') == '1' or decision_fallback_reason:
|
||||
choice, latency = fixed_fallback(options), 0
|
||||
else:
|
||||
try:
|
||||
choice, latency = decide(cfg, state, options, recent)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code not in (401,403,429):
|
||||
raise
|
||||
decision_fallback_reason='Jev HTTP %d; deterministic fallback for this session' % error.code
|
||||
choice, latency=fixed_fallback(options),0
|
||||
decision = router.choose(state, options, recent, run['run_id'])
|
||||
choice, latency = decision.choice, decision.jev_ms
|
||||
except DecisionRequired as required:
|
||||
record = {'state': 'requires_decision', 'step': step, 'run_id': run['run_id'],
|
||||
'decision_engine': 'executor', 'decision_request': required.request,
|
||||
'ocr_ms': observation_ms, 'screen': state.get('screen'),
|
||||
'at': time.strftime('%Y-%m-%dT%H:%M:%S')}
|
||||
if 'failed_jev_ms' in required.request:
|
||||
record['failed_jev_ms'] = required.request['failed_jev_ms']
|
||||
run['status'] = 'requires_decision'
|
||||
save_json(RUNTIME_HOME / 'mirror_current_run.json', run)
|
||||
save_json(RUNTIME_HOME / 'decision_request.json', required.request)
|
||||
save_json(status, record)
|
||||
with log.open('a', encoding='utf-8') as output:
|
||||
output.write(json.dumps(record, ensure_ascii=False) + '\n')
|
||||
print(json.dumps(record, ensure_ascii=False), flush=True)
|
||||
return
|
||||
action = options[choice]['action']
|
||||
record = {'step': step, 'run_id': run['run_id'], 'choice': choice, 'action': action, 'jev_ms': latency,
|
||||
'ocr_ms':observation_ms,'screen': state.get('screen'), 'at': time.strftime('%Y-%m-%dT%H:%M:%S')}
|
||||
if decision_fallback_reason:
|
||||
record['decision_fallback_reason']=decision_fallback_reason
|
||||
record['decision_engine'] = decision.engine if decision else 'rules'
|
||||
if decision:
|
||||
record.update(decision_ms=decision.latency_ms + (decision.failed_jev_ms or 0), confidence=decision.confidence,
|
||||
probabilities=decision.probabilities, model=decision.model)
|
||||
if decision.failed_jev_ms is not None:
|
||||
record['failed_jev_ms'] = decision.failed_jev_ms
|
||||
if decision.fallback_reason:
|
||||
record['decision_fallback_reason'] = decision.fallback_reason
|
||||
if choice == 'run_complete':
|
||||
complete_run(RUNTIME_HOME, run, context, state.get('screen'))
|
||||
record['state'] = 'completed'
|
||||
|
||||
@@ -6,7 +6,7 @@ import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
RESUMABLE = {'running', 'requires_analysis', 'stopped', 'user_takeover'}
|
||||
RESUMABLE = {'running', 'requires_analysis', 'requires_decision', 'stopped', 'user_takeover'}
|
||||
|
||||
|
||||
def save_json(path, value):
|
||||
|
||||
Reference in New Issue
Block a user