Add execution-model decision handoff and Jev AALC research

Browse Source
This commit is contained in:
Codex
2026-10-03 07:08:38 +08:00
parent 7188b4ac0f
commit 6c2d26e629
11 changed files with 958 additions and 27 deletions
+57 -4
View File
@@ -77,9 +77,9 @@ class AgentRuntimeTest(unittest.TestCase):
self.kernel.CreateMutexW.return_value = 12345 self.kernel.CreateMutexW.return_value = 12345
self.kernel.GetLastError.return_value = 0 self.kernel.GetLastError.return_value = 0
def run_agent(self, bridge, extra_args=(), constructor_error=None): def run_agent(self, bridge, extra_args=(), constructor_error=None, fixed=True, executor_decider=None):
output = io.StringIO() output = io.StringIO()
environment = dict(os.environ, LC_FIXED_FALLBACK='1') environment = dict(os.environ, LC_FIXED_FALLBACK='1' if fixed else '0')
environment.pop('OPENCODE_API_KEY', None) environment.pop('OPENCODE_API_KEY', None)
with mock.patch.object(agent, 'RUNTIME_HOME', self.home), \ with mock.patch.object(agent, 'RUNTIME_HOME', self.home), \
mock.patch.object(agent, 'STOP_FILE', self.home / 'JEV_STOP'), \ mock.patch.object(agent, 'STOP_FILE', self.home / 'JEV_STOP'), \
@@ -90,8 +90,8 @@ class AgentRuntimeTest(unittest.TestCase):
mock.patch.object(agent.time, 'sleep'), \ mock.patch.object(agent.time, 'sleep'), \
mock.patch.object(agent, 'button_enabled', return_value=False), \ mock.patch.object(agent, 'button_enabled', return_value=False), \
contextlib.redirect_stdout(output): contextlib.redirect_stdout(output):
agent.main() agent.main(executor_decider=executor_decider)
status = json.loads((self.home / 'jev_mirror_status.json').read_text()) status = json.loads((self.home / 'jev_mirror_status.json').read_text(encoding='utf-8'))
return status, factory return status, factory
def test_old_claim_context_does_not_complete_new_run_at_lobby(self): def test_old_claim_context_does_not_complete_new_run_at_lobby(self):
@@ -151,6 +151,59 @@ class AgentRuntimeTest(unittest.TestCase):
self.run_agent(bridge) self.run_agent(bridge)
self.assertEqual(self.kernel.CloseHandle.call_count, 1) self.assertEqual(self.kernel.CloseHandle.call_count, 1)
def test_no_jev_config_publishes_execution_model_request_without_input(self):
bridge = mock.Mock()
bridge.call.return_value = {'ok': True, 'items': [{'text': '路线', 'score': 1}]}
options = {
'wait': {'description': 'wait', 'action': {'tool': 'wait', 'args': {'seconds': .7}}},
'requires_analysis': {'description': 'inspect', 'action': {'tool': 'requires_analysis', 'args': {}}},
'route_w': {'description': 'event above', 'action': {'tool': 'route', 'args': {'key': 'w'}}},
}
with mock.patch.object(agent, 'candidates', return_value=options), mock.patch.object(agent, 'decide') as decide:
status, _ = self.run_agent(bridge, fixed=False)
self.assertEqual(status['state'], 'requires_decision')
self.assertEqual(status['decision_request']['reason'], 'no_jev_model')
self.assertEqual(bridge.call.call_args_list, [mock.call('get_state')])
decide.assert_not_called()
bridge.close.assert_called_once()
self.assertEqual(self.kernel.CloseHandle.call_count, 1)
pending = json.loads((self.home / 'decision_request.json').read_text(encoding='utf-8'))
run = json.loads((self.home / 'mirror_current_run.json').read_text())
self.assertEqual(pending['run_id'], run['run_id'])
self.assertEqual(run['status'], 'requires_decision')
def test_pending_execution_model_decision_can_resume_same_run(self):
save_json(self.home / 'mirror_current_run.json', {
'run_id': 'pending-run', 'start_utc': '2026-10-02T00:00:00+00:00',
'status': 'requires_decision', 'dungeon_completed': False, 'final_reward_claimed': False})
save_json(self.home / 'jev_shop_context.json', {'run_id': 'pending-run', 'floor': 3})
bridge = mock.Mock()
bridge.call.return_value = {'ok': True, 'items': []}
status, _ = self.run_agent(bridge, extra_args=['--resume'])
self.assertEqual(status['state'], 'stopped')
self.assertEqual(status['run_id'], 'pending-run')
def test_embedded_execution_model_selects_and_executes_without_jev(self):
bridge = mock.Mock()
bridge.call.return_value = {'ok': True, 'items': [{'text': '路线', 'score': 1}]}
options = {
'wait': {'description': 'wait', 'action': {'tool': 'wait', 'args': {'seconds': .7}}},
'requires_analysis': {'description': 'inspect', 'action': {'tool': 'requires_analysis', 'args': {}}},
'route_w': {'description': 'event above', 'action': {'tool': 'route', 'args': {'key': 'w'}}},
}
executor = mock.Mock(side_effect=lambda request: {'request_id': request['request_id'], 'choice': 'route_w'})
with mock.patch.object(agent, 'candidates', return_value=options), mock.patch.object(agent, 'decide') as decide:
status, _ = self.run_agent(bridge, fixed=False, executor_decider=executor)
self.assertEqual(status['state'], 'stopped')
executor.assert_called_once()
decide.assert_not_called()
self.assertEqual(bridge.call.call_args_list, [mock.call('get_state'),
mock.call('keypress', {'key': 'w'}), mock.call('keypress', {'key': 'enter'}),
mock.call('keypress', {'key': 'enter'})])
rows = [json.loads(line) for line in (self.home / 'jev_mirror_log.jsonl').read_text(encoding='utf-8').splitlines()]
self.assertEqual(rows[-1]['decision_engine'], 'executor')
self.assertEqual(rows[-1]['jev_ms'], 0)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
+125
View File
@@ -0,0 +1,125 @@
"""Provider routing regressions; no game or network calls."""
import json
from pathlib import Path
import sys
import unittest
from unittest import mock
import urllib.error
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from decision_router import DecisionRequired, DecisionRouter
class DecisionRouterTest(unittest.TestCase):
def setUp(self):
self.options = {
'route_up': {'description': 'Reachable event', 'action': {'tool': 'route', 'args': {'key': 'w'}}},
'wait': {'description': 'Observe again', 'action': {'tool': 'wait', 'args': {'seconds': .7}}},
}
self.state = {'scene': 'map', 'items': [{'text': '事件'}]}
self.jev = mock.Mock(return_value={'choice': 'route_up', 'latency_ms': 10,
'confidence': .8, 'probabilities': {'route_up': .9, 'wait': .1}})
self.config = {'decisionEngine': 'jev', 'model': 'jev-test', 'apiKey': 'private-key'}
def router(self, config=None, **kwargs):
return DecisionRouter(self.config if config is None else config, self.jev, **kwargs)
def test_configured_jev_keeps_model_and_probabilities(self):
result = self.router().choose(self.state, self.options)
self.assertEqual((result.engine, result.choice, result.model, result.jev_ms), ('jev', 'route_up', 'jev-test', 10))
self.assertEqual(result.confidence, .8)
self.assertEqual(result.probabilities, {'route_up': .9, 'wait': .1})
def test_no_model_hands_off_without_using_a_key_from_environment(self):
with self.assertRaises(DecisionRequired) as raised:
self.router({'apiKey': 'private-key'}).choose(self.state, self.options, run_id='run-1')
self.jev.assert_not_called()
request = raised.exception.request
self.assertEqual((request['reason'], request['run_id']), ('no_jev_model', 'run-1'))
self.assertEqual(request['candidates'], self.options)
self.assertNotIn('private-key', json.dumps(request))
def test_model_without_credentials_hands_off(self):
with self.assertRaises(DecisionRequired) as raised:
self.router({'model': 'jev-test'}).choose(self.state, self.options)
self.assertEqual(raised.exception.request['reason'], 'missing_jev_credentials')
self.jev.assert_not_called()
def test_executor_mode_never_calls_jev_even_when_configured(self):
result = self.router(dict(self.config, decisionEngine='executor'), executor_decider=lambda request: 'wait').choose(self.state, self.options)
self.assertEqual((result.engine, result.choice, result.jev_ms), ('executor', 'wait', 0))
self.jev.assert_not_called()
def test_auto_uses_jev_when_model_and_key_are_both_present(self):
executor = mock.Mock()
result = self.router({'model': 'jev-test', 'apiKey': 'private-key'}, executor_decider=executor).choose(self.state, self.options)
self.assertEqual(result.engine, 'jev')
executor.assert_not_called()
def test_callback_receives_request_and_returns_matching_id(self):
def executor(request):
self.assertEqual(request['page_state']['scene'], 'map')
self.assertNotIn('apiKey', request)
return {'request_id': request['request_id'], 'choice': 'route_up', 'model': 'active-executor'}
result = self.router({}, executor_decider=executor).choose(self.state, self.options)
self.assertEqual((result.engine, result.model), ('executor', 'active-executor'))
def test_http_limit_hands_off_with_failed_attempt_timing(self):
self.jev.side_effect = urllib.error.HTTPError('https://example.invalid', 429, 'rate limit', {}, None)
with self.assertRaises(DecisionRequired) as raised:
self.router().choose(self.state, self.options)
request = raised.exception.request
self.assertEqual(request['reason'], 'jev_http_429')
self.assertGreaterEqual(request['failed_jev_ms'], 0)
self.assertNotIn('private-key', json.dumps(request))
def test_timeout_uses_available_execution_model(self):
self.jev.side_effect = TimeoutError('timed out')
result = self.router(executor_decider=lambda request: 'route_up').choose(self.state, self.options)
self.assertEqual((result.engine, result.choice), ('executor', 'route_up'))
self.assertEqual(result.fallback_reason, 'jev_unavailable_TimeoutError')
def test_jev_unknown_candidate_hands_off_instead_of_sending_input(self):
self.jev.return_value = {'choice': 'invented_click'}
result = self.router(executor_decider=lambda request: 'wait').choose(self.state, self.options)
self.assertEqual(result.fallback_reason, 'jev_unavailable_ValueError')
def test_executor_cannot_return_unlisted_action(self):
with self.assertRaisesRegex(ValueError, 'unsupported candidate'):
self.router({}, executor_decider=lambda request: 'invented_click').choose(self.state, self.options)
def test_executor_response_from_another_request_is_rejected(self):
with self.assertRaisesRegex(ValueError, 'different decision request'):
self.router({}, executor_decider=lambda request: {'request_id': 'old-request', 'choice': 'wait'}).choose(self.state, self.options)
def test_callback_cannot_change_request_id_or_executable_actions(self):
def executor(request):
request['request_id'] = 'changed'
request['candidates']['route_up']['action']['args']['key'] = 'enter'
return {'request_id': 'changed', 'choice': 'route_up'}
with self.assertRaisesRegex(ValueError, 'different decision request'):
self.router({}, executor_decider=executor).choose(self.state, self.options)
self.assertEqual(self.options['route_up']['action']['args']['key'], 'w')
def test_forced_fixed_mode_preserves_explicit_legacy_choice(self):
result = self.router(force_fixed=True, fixed_decider=lambda options: 'wait').choose(self.state, self.options)
self.assertEqual((result.engine, result.choice), ('fixed', 'wait'))
self.jev.assert_not_called()
def test_empty_candidates_and_invalid_mode_are_errors(self):
with self.assertRaisesRegex(ValueError, 'at least one candidate'):
self.router().choose(self.state, {})
with self.assertRaisesRegex(ValueError, 'decisionEngine'):
self.router({'decisionEngine': 'unknown'})
def test_non_finite_confidence_or_invalid_probabilities_hands_off(self):
for invalid in ({'confidence': float('nan')}, {'probabilities': {'route_up': 1}},
{'probabilities': {'route_up': .8, 'wait': .8}}):
with self.subTest(invalid=invalid):
self.jev.return_value = dict(choice='route_up', **invalid)
result = self.router(executor_decider=lambda request: 'wait').choose(self.state, self.options)
self.assertEqual(result.engine, 'executor')
if __name__ == '__main__':
unittest.main()
+17 -1
View File
@@ -93,10 +93,26 @@ class SceneRegressionTest(unittest.TestCase):
error=agent.urllib.error.HTTPError('https://example.test',429,'rate limit',{},None) error=agent.urllib.error.HTTPError('https://example.test',429,'rate limit',{},None)
with mock.patch.object(agent.urllib.request,'urlopen',side_effect=error) as request, mock.patch.object(agent.time,'sleep') as sleep: with mock.patch.object(agent.urllib.request,'urlopen',side_effect=error) as request, mock.patch.object(agent.time,'sleep') as sleep:
with self.assertRaises(agent.urllib.error.HTTPError): with self.assertRaises(agent.urllib.error.HTTPError):
agent.decide({'apiKey':'test'}, {'items':[]},{'wait':{'description':'wait'}},[]) agent.decide({'apiKey':'test','model':'jev-test'}, {'items':[]},{'wait':{'description':'wait'}},[])
self.assertEqual(request.call_count,1) self.assertEqual(request.call_count,1)
sleep.assert_not_called() sleep.assert_not_called()
def test_jev_uses_configured_model_and_returns_probabilities_without_images(self):
response = mock.MagicMock()
response.__enter__.return_value = response
response.read.return_value = b'{"model":"jev-test","answers":{"action":{"type":"choice","choice":"wait","confidence":0.5,"probabilities":{"wait":0.75,"inspect":0.25}}}}'
state = {'screen': {'path': 'LOCAL_SCREENSHOT_ONLY.png'},
'items': [{'text': 'EVENT', 'x': 10, 'y': 20}]}
options = {'wait': {'description': 'wait'}, 'inspect': {'description': 'inspect'}}
with mock.patch.object(agent.urllib.request, 'urlopen', return_value=response) as request:
result = agent.decide({'apiKey': 'test', 'model': 'jev-test'}, state, options, [])
payload = agent.json.loads(request.call_args.args[0].data.decode('utf-8'))
self.assertEqual(payload['model'], 'jev-test')
self.assertIsInstance(payload['state'], str)
self.assertNotIn('LOCAL_SCREENSHOT_ONLY', payload['state'])
self.assertEqual(result['confidence'], .5)
self.assertEqual(result['probabilities'], {'wait': .75, 'inspect': .25})
def test_single_extra_gift_decline_preserves_core_and_multi_cards(self): def test_single_extra_gift_decline_preserves_core_and_multi_cards(self):
state = {'items': [{'text':t,'x':100,'y':100,'w':100,'h':40} for t in state = {'items': [{'text':t,'x':100,'y':100,'w':100,'h':40} for t in
('获得E.G.O饰品','石版字符','拒绝饰品','选择')]} ('获得E.G.O饰品','石版字符','拒绝饰品','选择')]}