修复 v0.5 审查问题: platform 改名, 协议健壮性, CJK 匹配, VBS 引号, interception API

Browse Source
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yws
2026-10-02 01:39:06 +00:00
co-authored by Claude Opus 5.5
parent 176570e7cd
commit 809a7fb017
12 changed files with 667 additions and 360 deletions
+167 -73
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""Limbus Company 语义化操控 MCP server v0.5 (跨平台)。 """Limbus Company 语义化操控 MCP server v0.6 (跨平台)。
核心思路: 不让 AI 盲猜坐标。工具层做"文本化 + 语义点击"—— 核心思路: 不让 AI 盲猜坐标。工具层做"文本化 + 语义点击"——
get_state : 截图 + OCR -> 全页文字元素(带坐标) get_state : 截图 + OCR -> 全页文字元素(带坐标)
@@ -10,22 +10,32 @@
win : 截图=交互计划任务; 输入=sendinput|interception win : 截图=交互计划任务; 输入=sendinput|interception
linux : 截图=X11 PIL/scrot; 输入=xdotool|uinput(物理) linux : 截图=X11 PIL/scrot; 输入=xdotool|uinput(物理)
mac : 截图=screencapture; 输入=cg mac : 截图=screencapture; 输入=cg
OCR 服务: 跑在远端(默认 http://35.212.193.150:8599/), 客户端机器零安装。 OCR: 纯本地 (Windows.Media.Ocr / winsdk)。
协议: MCP stdio (newline-delimited JSON-RPC)。 协议: MCP stdio (newline-delimited JSON-RPC)。
""" """
import glob
import os import os
import re
import struct
import sys import sys
import json import json
import time import time
from platform import detect_platform, get_input_backend, get_screenshot_provider from lcplatform import detect_platform, get_input_backend, get_screenshot_provider
VERSION = '0.6.0'
BACKEND_NAME = os.environ.get('INPUT_BACKEND', '').strip().lower() BACKEND_NAME = os.environ.get('INPUT_BACKEND', '').strip().lower()
OCR_BACKEND = os.environ.get('OCR_BACKEND', 'local').strip().lower() OCR_BACKEND = os.environ.get('OCR_BACKEND', 'local').strip().lower()
MIN_SCORE = float(os.environ.get('OCR_MIN_SCORE', '0.5')) MIN_SCORE = float(os.environ.get('OCR_MIN_SCORE', '0.5'))
KEEP_SHOTS = int(os.environ.get('LCP_KEEP_SHOTS', '30'))
MAX_WAIT = 300.0
WAIT_INTERVAL = 1.2
BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp') BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp')
os.makedirs(BASE, exist_ok=True) os.makedirs(BASE, exist_ok=True)
NEED_SCREEN = {'screenshot', 'get_state', 'wait_for_text', 'wait_for_text_gone', 'click_text'}
NEED_INPUT = {'click_text', 'click', 'move', 'keypress', 'type_text'}
def log(msg): def log(msg):
try: try:
@@ -46,19 +56,59 @@ def ocr_image(img_path):
items, err = ocr_local.ocr_image(img_path) items, err = ocr_local.ocr_image(img_path)
if items is None: if items is None:
return None, err return None, err
# Windows.Media.Ocr 不给置信度; 只有带 score 的后端才会被过滤
items = [it for it in items if float(it.get('score', 1)) >= MIN_SCORE] items = [it for it in items if float(it.get('score', 1)) >= MIN_SCORE]
return items, None return items, None
def png_size(path):
"""读 PNG IHDR 拿宽高(纯标准库)。失败返回 None。"""
try:
with open(path, 'rb') as f:
head = f.read(24)
if head[:8] != b'\x89PNG\r\n\x1a\n':
return None
return struct.unpack('>II', head[16:24])
except Exception:
return None
def prune_shots(path, keep=KEEP_SHOTS):
"""只保留同目录下最新 keep 张截图,防止轮询把磁盘写满。"""
if keep <= 0:
return
d = os.path.dirname(path)
shots = glob.glob(os.path.join(d, 'shot_*.png')) + glob.glob(os.path.join(d, 'lc_shot_*.png'))
shots.sort(key=lambda p: os.path.getmtime(p) if os.path.exists(p) else 0)
for p in shots[:-keep]:
try:
os.remove(p)
except Exception:
pass
_WS = re.compile(r'\s+')
def norm(s):
"""匹配用归一化: 去全部空白 + 小写。
Windows OCR 对中文按单字切词, 拼行时会变成 "开 始"。"""
return _WS.sub('', str(s)).lower()
def calc_click(txt, items): def calc_click(txt, items):
"""在 OCR items 里找包含 txt 的项,返回中心和候选列表。""" """在 OCR items 里找包含 txt 的项,返回中心列表。完全相等的排前面。"""
hits = [] key = norm(txt)
if not key:
return []
exact, partial = [], []
for it in items: for it in items:
if txt.lower() in it['text'].lower(): t = norm(it['text'])
cx = it['x'] + it['w'] // 2 if key not in t:
cy = it['y'] + it['h'] // 2 continue
hits.append({'text': it['text'], 'x': cx, 'y': cy}) hit = {'text': it['text'], 'x': it['x'] + it['w'] // 2, 'y': it['y'] + it['h'] // 2}
return hits (exact if t == key else partial).append(hit)
return exact + partial
class MCPServer: class MCPServer:
@@ -70,94 +120,132 @@ class MCPServer:
self.tools = [ self.tools = [
{"name": "get_state", "description": "截图并OCR, 返回当前屏幕全部文字元素(含坐标)。AI 决策前先调这个, 不需要自己猜坐标", {"name": "get_state", "description": "截图并OCR, 返回当前屏幕全部文字元素(含坐标)。AI 决策前先调这个, 不需要自己猜坐标",
"inputSchema": {"type": "object", "properties": {}}}, "inputSchema": {"type": "object", "properties": {}}},
{"name": "click_text", "description": "语义点击: 给按钮文字(支持子串), 内部OCR定位后点击其中心。例: click_text('开始')", {"name": "click_text", "description": "语义点击: 给按钮文字(支持子串, 忽略空格; 完全匹配优先), 内部OCR定位后点击其中心。例: click_text('开始')",
"inputSchema": {"type": "object", "properties": { "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}, "index": {"type": "number", "default": 0}}, "text": {"type": "string"}, "index": {"type": "number", "default": 0}},
"required": ["text"]}}, "required": ["text"]}},
{"name": "wait_for_text", "description": "循环截图+OCR, 等到指定文字出现(默认最多30秒)", "inputSchema": {"type": "object", "properties": { {"name": "wait_for_text", "description": "循环截图+OCR, 等到指定文字出现(默认30秒, 上限300秒)", "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}, "timeout": {"type": "number", "default": 30}}, "text": {"type": "string"}, "timeout": {"type": "number", "default": 30}},
"required": ["text"]}}, "required": ["text"]}},
{"name": "wait_for_text_gone", "description": "循环截图+OCR, 等到指定文字消失(默认最多30秒)", "inputSchema": {"type": "object", "properties": { {"name": "wait_for_text_gone", "description": "循环截图+OCR, 等到指定文字消失(默认30秒, 上限300秒)", "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}, "timeout": {"type": "number", "default": 30}}, "text": {"type": "string"}, "timeout": {"type": "number", "default": 30}},
"required": ["text"]}}, "required": ["text"]}},
{"name": "screenshot", "description": "截图并返回PNG路径(用于模板/人工查看)", "inputSchema": {"type": "object", "properties": {}}}, {"name": "screenshot", "description": "截图并返回PNG路径(用于模板/人工查看)", "inputSchema": {"type": "object", "properties": {}}},
{"name": "click", "description": "直接按坐标点击(一般用 click_text 更稳)", "inputSchema": {"type": "object", "properties": { {"name": "click", "description": "直接按坐标点击(截图像素坐标; 一般用 click_text 更稳)", "inputSchema": {"type": "object", "properties": {
"x": {"type": "number"}, "y": {"type": "number"}, "x": {"type": "number"}, "y": {"type": "number"},
"button": {"type": "string", "enum": ["left", "right"], "default": "left"}}, "button": {"type": "string", "enum": ["left", "right"], "default": "left"}},
"required": ["x", "y"]}}, "required": ["x", "y"]}},
{"name": "move", "description": "移动鼠标到(x,y)", "inputSchema": {"type": "object", "properties": { {"name": "move", "description": "移动鼠标到(x,y)", "inputSchema": {"type": "object", "properties": {
"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}},
{"name": "keypress", "description": "按一次键(p/enter/esc/space/tab/alt/f4/单字符等)", "inputSchema": {"type": "object", "properties": { {"name": "keypress", "description": "按一次键(p/enter/esc/space/tab/alt/f1-f12/单字符等)", "inputSchema": {"type": "object", "properties": {
"key": {"type": "string"}}, "required": ["key"]}}, "key": {"type": "string"}}, "required": ["key"]}},
{"name": "type_text", "description": "输入一串文本", "inputSchema": {"type": "object", "properties": { {"name": "type_text", "description": "输入一串文本", "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}}, "required": ["text"]}}, "text": {"type": "string"}}, "required": ["text"]}},
] ]
def _grab(self):
"""截图 + 清理旧图 + 把真实屏幕像素尺寸告诉输入后端(坐标换算以截图为准)。"""
sh = self.provider.grab()
if sh.get('ok'):
prune_shots(sh['path'])
size = png_size(sh['path'])
if size and hasattr(self.backend, 'set_screen'):
self.backend.set_screen(*size)
return sh
def _ocr_now(self):
sh = self._grab()
if not sh.get('ok'):
return sh, None
items, err = ocr_image(sh['path'])
if err:
return {'ok': False, 'error': err, 'screen': sh['path']}, None
return sh, items or []
def call(self, name, args): def call(self, name, args):
args = args or {} args = args or {}
if name == 'screenshot': if name == 'screenshot':
return self.provider.grab() return self._grab()
if name == 'get_state': if name == 'get_state':
sh = self.provider.grab() sh, items = self._ocr_now()
if not sh.get('ok'): if items is None:
return sh return sh
items, err = ocr_image(sh['path'])
if err:
return {'ok': False, 'error': err}
return {'ok': True, 'screen': sh['path'], 'items': items} return {'ok': True, 'screen': sh['path'], 'items': items}
if name == 'click_text': if name == 'click_text':
sh = self.provider.grab() txt = str(args.get('text', ''))
if not sh.get('ok'): if not norm(txt):
return {'ok': False, 'error': 'text must be non-empty'}
sh, items = self._ocr_now()
if items is None:
return sh return sh
items, err = ocr_image(sh['path']) hits = calc_click(txt, items)
if err:
return {'ok': False, 'error': err}
hits = calc_click(args.get('text', ''), items or [])
if not hits: if not hits:
return {'ok': False, 'error': 'text not found: %s' % args.get('text', ''), return {'ok': False, 'error': 'text not found: %s' % txt,
'screen': sh['path'], 'items': items} 'screen': sh['path'], 'items': items}
idx = min(int(args.get('index', 0)), len(hits) - 1) idx = int(args.get('index', 0))
if not 0 <= idx < len(hits):
return {'ok': False, 'error': 'index %d out of range (%d hits)' % (idx, len(hits)),
'hits': hits[:10]}
h = hits[idx] h = hits[idx]
r = self.backend.click(h['x'], h['y']) r = self.backend.click(h['x'], h['y'])
r['on'] = h['text'] r['on'] = h['text']
if len(hits) > 1:
r['other_hits'] = len(hits) - 1
return r return r
if name == 'wait_for_text': if name in ('wait_for_text', 'wait_for_text_gone'):
return self._wait(args.get('text', ''), float(args.get('timeout', 30)), True) txt = str(args.get('text', ''))
if name == 'wait_for_text_gone': if not norm(txt):
return self._wait(args.get('text', ''), float(args.get('timeout', 30)), False) return {'ok': False, 'error': 'text must be non-empty'}
timeout = max(0.0, min(float(args.get('timeout', 30)), MAX_WAIT))
return self._wait(txt, timeout, name == 'wait_for_text')
if name == 'click': if name == 'click':
return self.backend.click(int(args.get('x', 0)), int(args.get('y', 0)), args.get('button', 'left'), 1) button = args.get('button', 'left')
if button not in ('left', 'right'):
return {'ok': False, 'error': 'button must be left/right'}
return self.backend.click(int(args['x']), int(args['y']), button, 1)
if name == 'move': if name == 'move':
return self.backend.move(int(args.get('x', 0)), int(args.get('y', 0))) return self.backend.move(int(args['x']), int(args['y']))
if name == 'keypress': if name == 'keypress':
return self.backend.keypress(args.get('key', '')) return self.backend.keypress(str(args.get('key', '')))
if name == 'type_text': if name == 'type_text':
return self.backend.type_text(args.get('text', '')) return self.backend.type_text(str(args.get('text', '')))
return {'ok': False, 'error': 'unknown tool %s' % name} return {'ok': False, 'error': 'unknown tool %s' % name}
def _wait(self, txt, timeout, want_present): def _wait(self, txt, timeout, want_present):
deadline = time.time() + timeout deadline = time.time() + timeout
last_items = [] last_items, last_err = [], None
while time.time() < deadline: while True:
sh = self.provider.grab() sh, items = self._ocr_now()
if sh.get('ok'): if items is not None:
items, err = ocr_image(sh['path']) last_items = items
if not err: hits = calc_click(txt, items)
last_items = items or []
hits = calc_click(txt, last_items)
if bool(hits) == want_present: if bool(hits) == want_present:
return {'ok': True, 'present': bool(hits), return {'ok': True, 'present': bool(hits),
'hits': hits[:5], 'screen': sh['path']} 'hits': hits[:5], 'screen': sh['path']}
time.sleep(1.2) else:
return {'ok': False, 'error': 'timeout waiting %s (present=%s)' % (txt, want_present), last_err = sh.get('error')
if time.time() + WAIT_INTERVAL > deadline:
break
time.sleep(WAIT_INTERVAL)
r = {'ok': False, 'error': 'timeout waiting %s (present=%s)' % (txt, want_present),
'last_items': last_items[:10]} 'last_items': last_items[:10]}
if last_err:
r['last_error'] = last_err
return r
def read_msg(): def read_msg():
"""读一行 JSON-RPC。EOF 返回 None; 空行/坏 JSON 抛 ValueError 由调用方回错误后继续。"""
line = sys.stdin.readline() line = sys.stdin.readline()
if not line: if not line:
return None return None
return json.loads(line.strip()) line = line.strip()
if not line:
return {}
msg = json.loads(line)
if not isinstance(msg, dict):
raise ValueError('request must be a JSON object')
return msg
def send(obj): def send(obj):
@@ -165,62 +253,68 @@ def send(obj):
sys.stdout.flush() sys.stdout.flush()
def tool_result(mid, result, is_error=False):
send({"jsonrpc": "2.0", "id": mid, "result": {
"content": [{"type": "text", "text": json.dumps(result, ensure_ascii=False)}],
"isError": bool(is_error)}})
def main(): def main():
platform = detect_platform() platform = detect_platform()
log('lc_mcp v0.5 platform=%s backend=%s ocr=%s' % (platform, BACKEND_NAME or 'auto', OCR_BACKEND)) log('lc_mcp v%s platform=%s backend=%s ocr=%s' % (VERSION, platform, BACKEND_NAME or 'auto', OCR_BACKEND))
backend = None provider = backend = None
try: try:
provider = get_screenshot_provider(platform) provider = get_screenshot_provider(platform)
except Exception as e:
log('screenshot provider init fail: %s' % e)
try:
backend = get_input_backend(BACKEND_NAME, platform) backend = get_input_backend(BACKEND_NAME, platform)
except Exception as e: except Exception as e:
log('init fail: %s' % e) log('input backend init fail: %s' % e)
provider = None
backend = None
if backend is None:
log('WARN: input backend unavailable, semantic/click tools disabled')
server = MCPServer(platform, provider, backend) server = MCPServer(platform, provider, backend)
while True: while True:
try: try:
msg = read_msg() msg = read_msg()
except ValueError as e:
log('parse err %s' % e)
send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error: %s" % e}})
continue
if msg is None: if msg is None:
break break
except Exception as e: if not msg:
log('read err %s' % e) continue
break
method = msg.get('method') method = msg.get('method')
mid = msg.get('id') mid = msg.get('id')
if method == 'initialize': if method == 'initialize':
send({"jsonrpc": "2.0", "id": mid, "result": { send({"jsonrpc": "2.0", "id": mid, "result": {
"protocolVersion": "2024-11-05", "protocolVersion": "2024-11-05",
"capabilities": {"tools": {"listChanged": False}}, "capabilities": {"tools": {"listChanged": False}},
"serverInfo": {"name": "lc-mcp", "version": "0.5.0"}}}) "serverInfo": {"name": "lc-mcp", "version": VERSION}}})
elif method == 'notifications/initialized':
continue
elif method == 'ping': elif method == 'ping':
send({"jsonrpc": "2.0", "id": mid, "result": {}}) send({"jsonrpc": "2.0", "id": mid, "result": {}})
elif method == 'tools/list': elif method == 'tools/list':
send({"jsonrpc": "2.0", "id": mid, "result": {"tools": server.tools}}) send({"jsonrpc": "2.0", "id": mid, "result": {"tools": server.tools}})
elif method == 'tools/call': elif method == 'tools/call':
params = msg.get('params', {}) params = msg.get('params') or {}
name = params.get('name') name = params.get('name')
args = params.get('arguments', {}) args = params.get('arguments') or {}
log('call %s %s' % (name, args)) log('call %s %s' % (name, args))
if server.backend is None and name not in ('screenshot', 'get_state'): if name in NEED_SCREEN and server.provider is None:
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [ tool_result(mid, {'ok': False, 'error': 'screenshot provider unavailable (see lc_mcp.log)'}, True)
{"type": "text", "text": json.dumps({'ok': False, 'error': 'input backend unavailable'})}]}}) continue
if name in NEED_INPUT and server.backend is None:
tool_result(mid, {'ok': False, 'error': 'input backend unavailable (see lc_mcp.log)'}, True)
continue continue
try: try:
result = server.call(name, args) result = server.call(name, args)
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [ tool_result(mid, result, not result.get('ok', True))
{"type": "text", "text": json.dumps(result, ensure_ascii=False)}]}})
except Exception as e: except Exception as e:
log('tool err %s' % e) log('tool err %s: %r' % (name, e))
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [ tool_result(mid, {'ok': False, 'error': '%s: %s' % (type(e).__name__, e)}, True)
{"type": "text", "text": json.dumps({'ok': False, 'error': str(e)})}]}, "isError": True}) elif mid is not None:
else:
if mid is not None:
send({"jsonrpc": "2.0", "id": mid, send({"jsonrpc": "2.0", "id": mid,
"error": {"code": -32601, "message": "method not found"}}) "error": {"code": -32601, "message": "method not found"}})
# 其余无 id 的通知(notifications/*)按规范忽略
log('lc_mcp exit') log('lc_mcp exit')