Add execution-model decision handoff and Jev AALC research

Browse Source
This commit is contained in:
Codex
2026-10-03 07:08:38 +08:00
parent 7188b4ac0f
commit 6c2d26e629
11 changed files with 958 additions and 27 deletions
+125
View File
@@ -0,0 +1,125 @@
"""Provider routing regressions; no game or network calls."""
import json
from pathlib import Path
import sys
import unittest
from unittest import mock
import urllib.error
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from decision_router import DecisionRequired, DecisionRouter
class DecisionRouterTest(unittest.TestCase):
def setUp(self):
self.options = {
'route_up': {'description': 'Reachable event', 'action': {'tool': 'route', 'args': {'key': 'w'}}},
'wait': {'description': 'Observe again', 'action': {'tool': 'wait', 'args': {'seconds': .7}}},
}
self.state = {'scene': 'map', 'items': [{'text': '事件'}]}
self.jev = mock.Mock(return_value={'choice': 'route_up', 'latency_ms': 10,
'confidence': .8, 'probabilities': {'route_up': .9, 'wait': .1}})
self.config = {'decisionEngine': 'jev', 'model': 'jev-test', 'apiKey': 'private-key'}
def router(self, config=None, **kwargs):
return DecisionRouter(self.config if config is None else config, self.jev, **kwargs)
def test_configured_jev_keeps_model_and_probabilities(self):
result = self.router().choose(self.state, self.options)
self.assertEqual((result.engine, result.choice, result.model, result.jev_ms), ('jev', 'route_up', 'jev-test', 10))
self.assertEqual(result.confidence, .8)
self.assertEqual(result.probabilities, {'route_up': .9, 'wait': .1})
def test_no_model_hands_off_without_using_a_key_from_environment(self):
with self.assertRaises(DecisionRequired) as raised:
self.router({'apiKey': 'private-key'}).choose(self.state, self.options, run_id='run-1')
self.jev.assert_not_called()
request = raised.exception.request
self.assertEqual((request['reason'], request['run_id']), ('no_jev_model', 'run-1'))
self.assertEqual(request['candidates'], self.options)
self.assertNotIn('private-key', json.dumps(request))
def test_model_without_credentials_hands_off(self):
with self.assertRaises(DecisionRequired) as raised:
self.router({'model': 'jev-test'}).choose(self.state, self.options)
self.assertEqual(raised.exception.request['reason'], 'missing_jev_credentials')
self.jev.assert_not_called()
def test_executor_mode_never_calls_jev_even_when_configured(self):
result = self.router(dict(self.config, decisionEngine='executor'), executor_decider=lambda request: 'wait').choose(self.state, self.options)
self.assertEqual((result.engine, result.choice, result.jev_ms), ('executor', 'wait', 0))
self.jev.assert_not_called()
def test_auto_uses_jev_when_model_and_key_are_both_present(self):
executor = mock.Mock()
result = self.router({'model': 'jev-test', 'apiKey': 'private-key'}, executor_decider=executor).choose(self.state, self.options)
self.assertEqual(result.engine, 'jev')
executor.assert_not_called()
def test_callback_receives_request_and_returns_matching_id(self):
def executor(request):
self.assertEqual(request['page_state']['scene'], 'map')
self.assertNotIn('apiKey', request)
return {'request_id': request['request_id'], 'choice': 'route_up', 'model': 'active-executor'}
result = self.router({}, executor_decider=executor).choose(self.state, self.options)
self.assertEqual((result.engine, result.model), ('executor', 'active-executor'))
def test_http_limit_hands_off_with_failed_attempt_timing(self):
self.jev.side_effect = urllib.error.HTTPError('https://example.invalid', 429, 'rate limit', {}, None)
with self.assertRaises(DecisionRequired) as raised:
self.router().choose(self.state, self.options)
request = raised.exception.request
self.assertEqual(request['reason'], 'jev_http_429')
self.assertGreaterEqual(request['failed_jev_ms'], 0)
self.assertNotIn('private-key', json.dumps(request))
def test_timeout_uses_available_execution_model(self):
self.jev.side_effect = TimeoutError('timed out')
result = self.router(executor_decider=lambda request: 'route_up').choose(self.state, self.options)
self.assertEqual((result.engine, result.choice), ('executor', 'route_up'))
self.assertEqual(result.fallback_reason, 'jev_unavailable_TimeoutError')
def test_jev_unknown_candidate_hands_off_instead_of_sending_input(self):
self.jev.return_value = {'choice': 'invented_click'}
result = self.router(executor_decider=lambda request: 'wait').choose(self.state, self.options)
self.assertEqual(result.fallback_reason, 'jev_unavailable_ValueError')
def test_executor_cannot_return_unlisted_action(self):
with self.assertRaisesRegex(ValueError, 'unsupported candidate'):
self.router({}, executor_decider=lambda request: 'invented_click').choose(self.state, self.options)
def test_executor_response_from_another_request_is_rejected(self):
with self.assertRaisesRegex(ValueError, 'different decision request'):
self.router({}, executor_decider=lambda request: {'request_id': 'old-request', 'choice': 'wait'}).choose(self.state, self.options)
def test_callback_cannot_change_request_id_or_executable_actions(self):
def executor(request):
request['request_id'] = 'changed'
request['candidates']['route_up']['action']['args']['key'] = 'enter'
return {'request_id': 'changed', 'choice': 'route_up'}
with self.assertRaisesRegex(ValueError, 'different decision request'):
self.router({}, executor_decider=executor).choose(self.state, self.options)
self.assertEqual(self.options['route_up']['action']['args']['key'], 'w')
def test_forced_fixed_mode_preserves_explicit_legacy_choice(self):
result = self.router(force_fixed=True, fixed_decider=lambda options: 'wait').choose(self.state, self.options)
self.assertEqual((result.engine, result.choice), ('fixed', 'wait'))
self.jev.assert_not_called()
def test_empty_candidates_and_invalid_mode_are_errors(self):
with self.assertRaisesRegex(ValueError, 'at least one candidate'):
self.router().choose(self.state, {})
with self.assertRaisesRegex(ValueError, 'decisionEngine'):
self.router({'decisionEngine': 'unknown'})
def test_non_finite_confidence_or_invalid_probabilities_hands_off(self):
for invalid in ({'confidence': float('nan')}, {'probabilities': {'route_up': 1}},
{'probabilities': {'route_up': .8, 'wait': .8}}):
with self.subTest(invalid=invalid):
self.jev.return_value = dict(choice='route_up', **invalid)
result = self.router(executor_decider=lambda request: 'wait').choose(self.state, self.options)
self.assertEqual(result.engine, 'executor')
if __name__ == '__main__':
unittest.main()