Fix Jev mirror runtime boundaries for English UI and fast confirmations

Browse Source
This commit is contained in:
Codex Agent
2026-10-03 15:35:32 +08:00
parent 95ae060118
commit bb549c0265
6 changed files with 359 additions and 15 deletions
+3 -1
View File
@@ -29,6 +29,7 @@ class Decision:
model: str | None = None model: str | None = None
fallback_reason: str | None = None fallback_reason: str | None = None
failed_jev_ms: int | None = None failed_jev_ms: int | None = None
candidate_scope: list | None = None
def _answer(answer, options): def _answer(answer, options):
@@ -90,7 +91,8 @@ class DecisionRouter:
answer = self.jev_decider(self.config, state, options, recent) answer = self.jev_decider(self.config, state, options, recent)
choice, latency, confidence, probabilities, returned_model = _answer(answer, options) choice, latency, confidence, probabilities, returned_model = _answer(answer, options)
return Decision(choice, 'jev', latency, jev_ms=latency, confidence=confidence, return Decision(choice, 'jev', latency, jev_ms=latency, confidence=confidence,
probabilities=probabilities, model=returned_model or model) probabilities=probabilities, model=returned_model or model,
candidate_scope=answer.get('candidate_scope') if isinstance(answer, dict) else None)
except urllib.error.HTTPError as error: except urllib.error.HTTPError as error:
reason = 'jev_http_%d' % error.code reason = 'jev_http_%d' % error.code
except (OSError, TimeoutError, ValueError) as error: except (OSError, TimeoutError, ValueError) as error:
+86 -14
View File
@@ -12,6 +12,7 @@ from game_scenes import battle_is_terminal, battle_is_visible, death_retry_scree
from run_state import prepare_run, save_json, record_claim, claim_is_current, complete_run from run_state import prepare_run, save_json, record_claim, claim_is_current, complete_run
from decision_router import DecisionRouter, DecisionRequired from decision_router import DecisionRouter, DecisionRequired
from jev_client import request_answer from jev_client import request_answer
from lc_scene_aliases import normalize
REPO_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[1]
MIRROR_ASSETS = REPO_ROOT / 'vendor' / 'aalc' / 'assets' / 'images' / 'default' / 'share' / 'mirror' MIRROR_ASSETS = REPO_ROOT / 'vendor' / 'aalc' / 'assets' / 'images' / 'default' / 'share' / 'mirror'
@@ -26,6 +27,7 @@ SYSTEM_RULES = (
'事件当前有“选择后获得”且无需判定、无需战斗的选项时直接选择它,不反复滚动;避开写着战斗胜利后的选项。' '事件当前有“选择后获得”且无需判定、无需战斗的选项时直接选择它,不反复滚动;避开写着战斗胜利后的选项。'
'确定确认/进入/开始战斗直接执行。地图优先可达事件和商店,减少战斗。' '确定确认/进入/开始战斗直接执行。地图优先可达事件和商店,减少战斗。'
'选好路线直接回车进入,关卡编队界面也直接回车开战;保留原编队,不再检查人员。' '选好路线直接回车进入,关卡编队界面也直接回车开战;保留原编队,不再检查人员。'
'候选路线已由程序核对可达和节点类型,无需再确认已完成的开局配置。多个同类型节点都可达时默认正前方,不因没有事件节点而要求分析。'
'保留当前编队。开局三级饰品三选一必须选择标准负载电池,再确认领取;不能当成额外观测拒绝。' '保留当前编队。开局三级饰品三选一必须选择标准负载电池,再确认领取;不能当成额外观测拒绝。'
'成捆的符咒++、标准负载电池和鸡胸为核心,保留不可出售或合成;仅略过单独的额外观测。其他苦难避开敌方体力上限和防御等级。' '成捆的符咒++、标准负载电池和鸡胸为核心,保留不可出售或合成;仅略过单独的额外观测。其他苦难避开敌方体力上限和防御等级。'
'3层起辛克莱1换3,黑兽奥提斯2换3,其他出战1换3。只在商店强化饰品。' '3层起辛克莱1换3,黑兽奥提斯2换3,其他出战1换3。只在商店强化饰品。'
@@ -41,6 +43,13 @@ def text_items(state):
return state.get('items') or [] return state.get('items') or []
def scene_fingerprint(state):
"""Ignore changing clocks and counters when checking failed confirmations."""
return tuple(item['text'] for item in text_items(state)
if item.get('score', 0) > .93
and not re.fullmatch(r'[0-9:.]+', item['text'].replace(' ', '')))
_SCREEN_CACHE = {} _SCREEN_CACHE = {}
@@ -234,7 +243,7 @@ def candidates(state, previous_battle=False, refresh_used=0):
'y': item['y'] + item['h']//2}}, 'y': item['y'] + item['h']//2}},
} }
return options return options
if any('REC' in t.upper() for t in texts) and '选项' not in texts and not any(t in combined for t in ('选择一位罪人', '进行判定', '判定成功', '判定失败', '结果')): if any(re.fullmatch(r'REC[●.]*', t, re.I) for t in texts) and '选项' not in texts and not any(t in combined for t in ('选择一位罪人', '进行判定', '判定成功', '判定失败', '结果')):
options['event_advance'] = {'description':'事件选项已经确定,按回车快速推进后续文本。', 'action':{'tool':'advance_dialog','args':{'presses':5}}} options['event_advance'] = {'description':'事件选项已经确定,按回车快速推进后续文本。', 'action':{'tool':'advance_dialog','args':{'presses':5}}}
return options return options
if '应该让谁' in combined and '选择一位罪人' not in combined and '进行判定' not in combined: if '应该让谁' in combined and '选择一位罪人' not in combined and '进行判定' not in combined:
@@ -516,10 +525,14 @@ def candidates(state, previous_battle=False, refresh_used=0):
options['button_%d' % index] = { options['button_%d' % index] = {
'description': '当前可见且非灰色按钮:%s;点击 OCR 中心。' % label, 'description': '当前可见且非灰色按钮:%s;点击 OCR 中心。' % label,
# 直接点已核对过的这一项: 重新按文字搜索要多一次截图+OCR, 还可能命中背景里的同名按钮 # 直接点已核对过的这一项: 重新按文字搜索要多一次截图+OCR, 还可能命中背景里的同名按钮
'action': ({'tool': 'keypress', 'args': {'key': 'enter'}} if label in ('进入', '开始战斗') 'action': ({'tool': 'keypress', 'args': {'key': 'enter'}} if label in ('进入', '开始战斗', '进行判定')
else {'tool': 'click', 'args': {'x': item['x'] + item['w']//2, else {'tool': 'click', 'args': {'x': item['x'] + item['w']//2,
'y': item['y'] + item['h']//2}}), 'y': item['y'] + item['h']//2}}),
} }
if options['button_%d' % index]['action']['tool'] == 'keypress':
options['button_%d' % index]['fallback_action'] = {
'tool': 'click', 'args': {'x': item['x'] + item['w']//2,
'y': item['y'] + item['h']//2}}
if (not battle and not in_shop and state.get('screen', {}).get('path') if (not battle and not in_shop and state.get('screen', {}).get('path')
and Path(state['screen']['path']).is_file() and Path(state['screen']['path']).is_file()
and not any(t in combined for t in ('确认', '继续', '获得E.G.O', 'SKIP', '选项'))): and not any(t in combined for t in ('确认', '继续', '获得E.G.O', 'SKIP', '选项'))):
@@ -558,7 +571,9 @@ def candidates(state, previous_battle=False, refresh_used=0):
} }
for index, item in enumerate(items): for index, item in enumerate(items):
if item['x'] > width * 0.52 and height * 0.23 < item['y'] < height * 0.75 and item['h'] > height * 0.025 and item.get('score', 0) > 0.93: if item['x'] > width * 0.52 and height * 0.23 < item['y'] < height * 0.75 and item['h'] > height * 0.025 and item.get('score', 0) > 0.93:
if not any(t in item['text'] for t in ('判定', '获得', '成功', '全体', '战斗胜利')): if (not any(t in item['text'] for t in ('判定', '获得', '成功', '全体', '战斗胜利'))
and not re.search(r'check\b|\bgain\b|\be\.?g\.?o\.?\s*gift|(?:advantage|guilt)check',
item['text'], re.I)):
outcome = ';'.join(i['text'] for i in items if i['x'] > width*.52 outcome = ';'.join(i['text'] for i in items if i['x'] > width*.52
and item['y']+item['h'] < i['y'] < item['y']+height*.085) and item['y']+item['h'] < i['y'] < item['y']+height*.085)
options['event_%d' % index] = {'description': '当前事件选项:' + item['text'] + ';结果:' + outcome, options['event_%d' % index] = {'description': '当前事件选项:' + item['text'] + ';结果:' + outcome,
@@ -567,16 +582,69 @@ def candidates(state, previous_battle=False, refresh_used=0):
def decide(cfg, state, options, recent): def decide(cfg, state, options, recent):
description = ('目标:按用户已配置的队伍和速通规则推进当前镜牢。当前楼层:%s\n当前游戏 OCR(坐标仅供辨识):\n' % state.get('floor', '尚未识别') + actions = [key for key in options if key not in ('wait', 'requires_analysis')]
'\n'.join('%s @%d,%d' % (i['text'], i['x'], i['y']) for i in text_items(state) route_scene = actions and all(key.startswith('route_') for key in actions)
if not i['text'].upper().startswith('STEAM')) + sinner_scene = actions and all(key.startswith(('sinner_', 'button_')) for key in actions) and any(key.startswith('sinner_') for key in actions)
'\n截图视觉与可用控件观察(不是预先做出的决定):\n' + if route_scene or sinner_scene:
'\n'.join(val['description'] for key,val in options.items() if key != 'requires_analysis') + permitted = options
'\n最近动作和执行结果:' + json.dumps(recent[-4:], ensure_ascii=False)) if sinner_scene:
# All explicit maximum-probability cards are legal. An uncertain
# colour-only estimate must not compete with an OCR-labelled rate.
rated = [(key, re.search(r'预计成功率(极高|非常高|高)', options[key]['description']))
for key in actions if key.startswith('sinner_')]
rated = [(key, match[1]) for key, match in rated if match]
if rated:
best = 0 if any(rate != '高' for _, rate in rated) else 1
permitted = {key: options[key] for key, rate in rated
if (0 if rate != '高' else 1) == best}
# These candidates have already been validated against the live scene.
# Party setup, gifts and fusion do not affect this immediate choice.
instructions = (
'目标是最快完成镜牢,只选择所给候选。候选已核对当前画面,不需要再次核对编队或开局配置。'
+ ('地图优先商店、事件,再普通战斗,避免精英或集中战斗。同类型默认正前方 route_d;没有事件也直接选择可达战斗。'
if route_scene else
'当前是事件判定。已有进行判定按钮则选择该按钮;否则选择明确标注极高成功率的 sinner 候选,极高优于高;同档任选一个。不需要额外角色技能信息。')
+ '仅在候选互相矛盾或没有有效动作时选择 requires_analysis。')
scene = ('镜牢地图,当前高亮可达节点如下:' if route_scene else '事件选择罪人界面,已识别当前可选择的罪人和成功率如下:')
description = scene + ';'.join(options[key]['description'] for key in actions)
answer = request_answer(cfg, description,
{key: val['description'] for key, val in permitted.items()}, instructions)
if permitted is not options and answer.get('probabilities') is not None:
# The provider only scored permitted cards. The surrounding router
# validates a distribution across its original candidate set.
answer['candidate_scope'] = list(permitted)
answer['probabilities'] = {key: answer['probabilities'].get(key, 0) for key in options}
return answer
description = ('当前楼层:%s\n当前游戏文字:\n' % state.get('floor', '尚未识别') +
'\n'.join(dict.fromkeys(i['text'] for i in text_items(state)
if not i['text'].upper().startswith('STEAM'))) +
'\n最近执行结果:' + json.dumps(recent[-2:], ensure_ascii=False))
return request_answer(cfg, description, return request_answer(cfg, description,
{key: val['description'] for key, val in options.items()}, SYSTEM_RULES) {key: val['description'] for key, val in options.items()}, SYSTEM_RULES)
def unambiguous_routine(options):
"""Execute one validated confirmation; choices still go through the model."""
judges = [key for key, value in options.items() if key.startswith('button_')
and '进行判定' in value['description']]
if len(judges) == 1:
return judges[0]
actions = [key for key in options if key not in ('wait', 'requires_analysis')]
if len(actions) != 1:
return None
key = actions[0]
safe = ('modal_confirm', 'select_startup_battery', 'confirm_startup_battery',
'confirm_thunderbranch', 'confirm_reward', 'hardship_confirm',
'confirm_swap', 'buy_swap', 'claim_final', 'run_complete', 'retry_network')
if key in safe:
return key
if key.startswith('button_') and options[key]['action']['tool'] in ('click', 'keypress'):
description = options[key]['description']
if any(label in description for label in ('确认', '继续', '进入', '开始战斗', '离开', '下一层', '进行判定')):
return key
return None
def fixed_fallback(options): def fixed_fallback(options):
for key in ('run_complete','required_pack','pack_refresh','select_startup_battery','confirm_startup_battery','select_thunderbranch','confirm_thunderbranch','prepare_thrill','prepare_thunderbranch','decline_extra_observation','retry_network','buy_swap','confirm_swap','confirm_reward','hardship_confirm','modal_confirm','claim_final','close_progress'): for key in ('run_complete','required_pack','pack_refresh','select_startup_battery','confirm_startup_battery','select_thunderbranch','confirm_thunderbranch','prepare_thrill','prepare_thunderbranch','decline_extra_observation','retry_network','buy_swap','confirm_swap','confirm_reward','hardship_confirm','modal_confirm','claim_final','close_progress'):
if key in options: if key in options:
@@ -630,7 +698,8 @@ def execute(bridge, action, deadline=None):
def call(tool, args=None): def call(tool, args=None):
if STOP_FILE.exists() or deadline is not None and time.monotonic() >= deadline: if STOP_FILE.exists() or deadline is not None and time.monotonic() >= deadline:
return {'ok':False,'stopped':True} return {'ok':False,'stopped':True}
return bridge.call(tool, args) result = bridge.call(tool, args)
return normalize(result) if tool == 'get_state' else result
if action['tool'] == 'claim_final': if action['tool'] == 'claim_final':
result = call('click', action['args']) result = call('click', action['args'])
if not result.get('ok'): if not result.get('ok'):
@@ -748,13 +817,12 @@ def main(executor_decider=None):
save_json(status, {'state': 'stopped', 'step': step, 'run_id': run['run_id']}) save_json(status, {'state': 'stopped', 'step': step, 'run_id': run['run_id']})
return return
observation_started = time.monotonic() observation_started = time.monotonic()
state = bridge.call('get_state') state = normalize(bridge.call('get_state'))
observation_ms = round((time.monotonic()-observation_started)*1000) observation_ms = round((time.monotonic()-observation_started)*1000)
if not state.get('ok'): if not state.get('ok'):
state = {'ok': False, 'items': [{'text': 'MCP 截图暂不可用;游戏窗口可能正在重建。' + str(state.get('error','')), state = {'ok': False, 'items': [{'text': 'MCP 截图暂不可用;游戏窗口可能正在重建。' + str(state.get('error','')),
'x':0,'y':0,'w':0,'h':0,'score':1.0}]} 'x':0,'y':0,'w':0,'h':0,'score':1.0}]}
fingerprint = tuple(item['text'] for item in text_items(state) fingerprint = scene_fingerprint(state)
if item.get('score', 0) > 0.93 and not item['text'].replace('.', '').isdigit() and not re.fullmatch(r'[0-9:]+', item['text']))
if any('正在探' in i['text'] or '主题卡包' in i['text'] for i in text_items(state)): if any('正在探' in i['text'] or '主题卡包' in i['text'] for i in text_items(state)):
context['refresh_used'] = 0 context['refresh_used'] = 0
for item in text_items(state): for item in text_items(state):
@@ -782,7 +850,7 @@ def main(executor_decider=None):
and repeats.get((fingerprint, action_id), 0) >= 1): and repeats.get((fingerprint, action_id), 0) >= 1):
fallback = options[key].get('fallback_action') fallback = options[key].get('fallback_action')
button = next((i for i in text_items(state) button = next((i for i in text_items(state)
if i['text'].replace(' ', '') in ('进入', '开始战斗', '确认', '继续') if i['text'].replace(' ', '') in ('进入', '开始战斗', '确认', '继续', '进行判定')
and button_enabled(state, i)), None) and button_enabled(state, i)), None)
if fallback or button: if fallback or button:
options[key]['action'] = fallback or {'tool': 'click', 'args': { options[key]['action'] = fallback or {'tool': 'click', 'args': {
@@ -810,6 +878,8 @@ def main(executor_decider=None):
choice, latency = 'event_advance', 0 choice, latency = 'event_advance', 0
elif 'battle' in options: elif 'battle' in options:
choice, latency = 'battle', 0 choice, latency = 'battle', 0
elif unambiguous_routine(options):
choice, latency = unambiguous_routine(options), 0
else: else:
try: try:
decision = router.choose(state, options, recent, run['run_id']) decision = router.choose(state, options, recent, run['run_id'])
@@ -840,6 +910,8 @@ def main(executor_decider=None):
record['failed_jev_ms'] = decision.failed_jev_ms record['failed_jev_ms'] = decision.failed_jev_ms
if decision.fallback_reason: if decision.fallback_reason:
record['decision_fallback_reason'] = decision.fallback_reason record['decision_fallback_reason'] = decision.fallback_reason
if decision.candidate_scope is not None:
record['provider_candidate_scope'] = decision.candidate_scope
if choice == 'run_complete': if choice == 'run_complete':
complete_run(RUNTIME_HOME, run, context, state.get('screen')) complete_run(RUNTIME_HOME, run, context, state.get('screen'))
record['state'] = 'completed' record['state'] = 'completed'
+81
View File
@@ -0,0 +1,81 @@
"""Normalize observed English UI labels for the existing mirror controller."""
import re
LABELS = {
'confirm':'确认', 'continue':'继续', 'proceed':'继续', 'enter':'进入', 'cancel':'取消',
'leave':'离开', 'close':'关闭', 'startbattle':'开始战斗', 'startcombat':'开始战斗',
'battle!':'开始战斗', 'chainbattle!':'开始战斗',
'select':'选择', 'selectasinner':'选择一位罪人', 'chooseasinner':'选择一位罪人',
'commence':'进行判定', 'commencecheck':'进行判定', 'performcheck':'进行判定', 'checksuccess':'判定成功',
'checkfailure':'判定失败', 'checkpassed':'判定成功', 'checkfailed':'判定失败',
'result':'结果', 'options':'选项', 'choices':'选项', 'veryhigh':'极高',
'high':'高', 'shop':'商店', 'specialshop':'特殊商店', 'refresh':'刷新',
'enhancegifts':'强化饰品', 'enhancee.g.ogifts':'强化饰品', 'purchase':'购买',
'buy':'购买', 'purchased':'已购买', 'currentownedskills':'当前持有技能',
'thunderbranch':'雷枝', '雷击木':'雷枝', 'talismanbundle':'成捆的符咒',
'standard-dutybattery':'标准负载电池', 'standarddutybattery':'标准负载电池',
'thrill':'快感', 'selecte.g.ogifts':'选择E.G.O饰品',
'selecte.g.ogift':'选择E.G.O饰品', 'declinegifts':'拒绝饰品',
'declinegift':'拒绝饰品', 'refusegifts':'拒绝饰品',
'refusegift':'拒绝饰品', 'selectede.g.ogift':'选择E.G.O饰品',
'gaine.g.ogift':'获得E.G.O饰品', 'gaine.g.ogift!':'获得E.G.O饰品!',
'obtaine.g.ogift!':'获得E.G.O饰品!', 'e.g.ogiftobservation':'饰品观测',
'e.g.ogiftget!':'获得E.G.O饰品!',
'e.g.ogiftsearch':'额外观测', 'eg.dgiftsearch':'额外观测',
'nextfloor':'下一层', 'gotonextfloor':'前往下一层', 'proceedtonextfloor':'前往下一层', 'claim':'领取',
'additionalhardship':'追加的苦难', 'additionalhardships':'追加的苦难',
'obtaine.g.ogift':'获得E.G.O饰品', 'obtaine.g.ogifts':'获得E.G.O饰品',
'claimrewards':'领取奖励', 'explorationcomplete':'探索完成',
'explorationcompleted':'探索完成', 'dungeoncomplete':'探索完成',
'drive':'驾驶席', 'sinners':'罪人', 'exploring:':'探索状态',
'beforeentry':'入场前', 'retry':'重试', 'dungeonprogress':'迷宫进度',
}
def normalize(state):
if state.get('_lc_labels_normalized'):
return state
rate_items = []
for item in state.get('items') or []:
raw = item['text']
compact = re.sub(r'\s+', '', raw).lower().lstrip('×x')
compact = compact.replace('e.g.0', 'e.g.o')
text = LABELS.get(compact, raw)
if text == raw:
match = re.search(r'select(?:the)?(?:floor)?(\d+)(?:st|nd|rd|th)?floorthemepack', compact)
if not match:
match = re.search(r'selectfloorthemepack(?:for)?floor(\d+)', compact)
if not match:
match = re.search(r'selectfloor(\d+)themepack', compact)
if match:
text = '选择%s层主题卡包' % match[1]
elif re.search(r'(?:select|choose).*encounterreward', compact):
text = '选择遭遇战奖励卡'
elif re.search(r'exploring(?:floor)?\d+', compact):
text = '正在探索第%s层' % re.search(r'\d+', compact)[0]
elif compact.startswith('select') and re.search(r'\d+/\d+', compact):
text = '选择' + re.search(r'\d+/\d+', compact)[0]
elif compact.startswith('refresh'):
text = raw.replace(raw[:7], '刷新')
elif compact.startswith('chooseacharactertoperform'):
text = '选择一位罪人'
elif re.match(r'who(?:will|should)', compact):
text = '应该让谁 ' + raw
elif compact.startswith('skillreplacement') or compact.startswith('replaceskill'):
text = '技能替换 ' + raw
# English OCR sometimes merges several adjacent probability labels.
# Keep each explicit Very High span as a separate observed candidate.
if item.get('y', 0) > state.get('screen', {}).get('h', 1080) * .8 and 'veryhigh' in compact:
for match in re.finditer('veryhigh', compact):
center = item['x'] + (match.start() + len(match[0]) / 2) * item['w'] / len(compact)
rate_items.append(dict(item, text='极高', original_text=raw,
x=round(center - 25), w=50))
for english, chinese in [('Sinclair','辛克莱'), ('Hong Lu','鸿璐'),
('Heathcliff','希斯克利夫'), ('Outis','奥提斯'), ('Gregor','格里高尔'),
('Ryoshu','良秀'), ('Faust','浮士德')]:
text = text.replace(english, chinese)
if text != raw:
item['original_text'] = raw
item['text'] = text
state['items'] = (state.get('items') or []) + rate_items
state['_lc_labels_normalized'] = True
return state