#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Limbus Company 语义化操控 MCP server v0.5 (跨平台)。 核心思路: 不让 AI 盲猜坐标。工具层做"文本化 + 语义点击"—— get_state : 截图 + OCR -> 全页文字元素(带坐标) click_text : 说文字/按钮名, 内部 OCR 定位后点击 wait_for_text / wait_for_text_gone : 等条件出现/消失再回报 平台无关: 截图/输入按 LCP_PLATFORM (win|linux|mac, 默认自动检测) 加载实现。 win : 截图=交互计划任务; 输入=sendinput|interception linux : 截图=X11 PIL/scrot; 输入=xdotool|uinput(物理) mac : 截图=screencapture; 输入=cg OCR 服务: 跑在远端(默认 http://35.212.193.150:8599/), 客户端机器零安装。 协议: MCP stdio (newline-delimited JSON-RPC)。 """ import os import sys import json import time import urllib.request from platform import detect_platform, get_input_backend, get_screenshot_provider BACKEND_NAME = os.environ.get('INPUT_BACKEND', '').strip().lower() OCR_URL = os.environ.get('OCR_URL', 'http://35.212.193.150:8599/').strip().rstrip('/') + '/' OCR_BACKEND = os.environ.get('OCR_BACKEND', 'auto').strip().lower() MIN_SCORE = float(os.environ.get('OCR_MIN_SCORE', '0.5')) BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp') os.makedirs(BASE, exist_ok=True) def log(msg): try: with open(os.path.join(BASE, 'lc_mcp.log'), 'a', encoding='utf-8') as f: f.write('[%s] %s\n' % (time.strftime('%H:%M:%S'), msg)) except Exception: pass def ocr_image(img_path): """OCR 分发:local(winsdk/Win自带) / remote(gcp1)。 OCR_BACKEND=auto: 本地可用则本地, 否则远端。""" impl = None if OCR_BACKEND == 'local' or OCR_BACKEND == 'auto': try: import ocr_local if ocr_local.available(): impl = ('local', ocr_local) except Exception: impl = None if impl is None and OCR_BACKEND != 'local': impl = ('remote', None) if impl is None: return None, 'no usable OCR backend (OCR_BACKEND=%s)' % OCR_BACKEND kind, mod = impl if kind == 'local': items, err = mod.ocr_image(img_path) else: with open(img_path, 'rb') as f: body = f.read() req = urllib.request.Request(OCR_URL, data=body, headers={'Content-Type': 'image/png'}) try: with urllib.request.urlopen(req, timeout=30) as resp: data = json.loads(resp.read().decode('utf-8')) except Exception as e: return None, 'ocr call fail: %s' % e if not data.get('ok'): return None, data.get('error', 'ocr error') items = data.get('items', []) if items is None: return None, err items = [it for it in items if float(it.get('score', 1)) >= MIN_SCORE] return items, None def calc_click(txt, items): """在 OCR items 里找包含 txt 的项,返回中心和候选列表。""" hits = [] for it in items: if txt.lower() in it['text'].lower(): cx = it['x'] + it['w'] // 2 cy = it['y'] + it['h'] // 2 hits.append({'text': it['text'], 'x': cx, 'y': cy}) return hits class MCPServer: def __init__(self, platform, provider, backend): self.platform = platform self.provider = provider self.backend = backend self.tools = [ {"name": "get_state", "description": "截图并OCR, 返回当前屏幕全部文字元素(含坐标)。AI 决策前先调这个, 不需要自己猜坐标", "inputSchema": {"type": "object", "properties": {}}}, {"name": "click_text", "description": "语义点击: 给按钮文字(支持子串), 内部OCR定位后点击其中心。例: click_text('开始')", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}, "index": {"type": "number", "default": 0}}, "required": ["text"]}}, {"name": "wait_for_text", "description": "循环截图+OCR, 等到指定文字出现(默认最多30秒)", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}, "timeout": {"type": "number", "default": 30}}, "required": ["text"]}}, {"name": "wait_for_text_gone", "description": "循环截图+OCR, 等到指定文字消失(默认最多30秒)", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}, "timeout": {"type": "number", "default": 30}}, "required": ["text"]}}, {"name": "screenshot", "description": "截图并返回PNG路径(用于模板/人工查看)", "inputSchema": {"type": "object", "properties": {}}}, {"name": "click", "description": "直接按坐标点击(一般用 click_text 更稳)", "inputSchema": {"type": "object", "properties": { "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right"], "default": "left"}}, "required": ["x", "y"]}}, {"name": "move", "description": "移动鼠标到(x,y)", "inputSchema": {"type": "object", "properties": { "x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, {"name": "keypress", "description": "按一次键(p/enter/esc/space/tab/alt/f4/单字符等)", "inputSchema": {"type": "object", "properties": { "key": {"type": "string"}}, "required": ["key"]}}, {"name": "type_text", "description": "输入一串文本", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}}, "required": ["text"]}}, ] def call(self, name, args): args = args or {} if name == 'screenshot': return self.provider.grab() if name == 'get_state': sh = self.provider.grab() if not sh.get('ok'): return sh items, err = ocr_image(sh['path']) if err: return {'ok': False, 'error': err} return {'ok': True, 'screen': sh['path'], 'items': items} if name == 'click_text': sh = self.provider.grab() if not sh.get('ok'): return sh items, err = ocr_image(sh['path']) if err: return {'ok': False, 'error': err} hits = calc_click(args.get('text', ''), items or []) if not hits: return {'ok': False, 'error': 'text not found: %s' % args.get('text', ''), 'screen': sh['path'], 'items': items} idx = min(int(args.get('index', 0)), len(hits) - 1) h = hits[idx] r = self.backend.click(h['x'], h['y']) r['on'] = h['text'] return r if name == 'wait_for_text': return self._wait(args.get('text', ''), float(args.get('timeout', 30)), True) if name == 'wait_for_text_gone': return self._wait(args.get('text', ''), float(args.get('timeout', 30)), False) if name == 'click': return self.backend.click(int(args.get('x', 0)), int(args.get('y', 0)), args.get('button', 'left'), 1) if name == 'move': return self.backend.move(int(args.get('x', 0)), int(args.get('y', 0))) if name == 'keypress': return self.backend.keypress(args.get('key', '')) if name == 'type_text': return self.backend.type_text(args.get('text', '')) return {'ok': False, 'error': 'unknown tool %s' % name} def _wait(self, txt, timeout, want_present): deadline = time.time() + timeout last_items = [] while time.time() < deadline: sh = self.provider.grab() if sh.get('ok'): items, err = ocr_image(sh['path']) if not err: last_items = items or [] hits = calc_click(txt, last_items) if bool(hits) == want_present: return {'ok': True, 'present': bool(hits), 'hits': hits[:5], 'screen': sh['path']} time.sleep(1.2) return {'ok': False, 'error': 'timeout waiting %s (present=%s)' % (txt, want_present), 'last_items': last_items[:10]} def read_msg(): line = sys.stdin.readline() if not line: return None return json.loads(line.strip()) def send(obj): sys.stdout.write(json.dumps(obj, ensure_ascii=False) + '\n') sys.stdout.flush() def main(): platform = detect_platform() log('lc_mcp v0.5 platform=%s backend=%s ocr=%s' % (platform, BACKEND_NAME or 'auto', OCR_URL)) backend = None try: provider = get_screenshot_provider(platform) backend = get_input_backend(BACKEND_NAME, platform) except Exception as e: log('init fail: %s' % e) provider = None backend = None if backend is None: log('WARN: input backend unavailable, semantic/click tools disabled') server = MCPServer(platform, provider, backend) while True: try: msg = read_msg() if msg is None: break except Exception as e: log('read err %s' % e) break method = msg.get('method') mid = msg.get('id') if method == 'initialize': send({"jsonrpc": "2.0", "id": mid, "result": { "protocolVersion": "2024-11-05", "capabilities": {"tools": {"listChanged": False}}, "serverInfo": {"name": "lc-mcp", "version": "0.5.0"}}}) elif method == 'notifications/initialized': continue elif method == 'ping': send({"jsonrpc": "2.0", "id": mid, "result": {}}) elif method == 'tools/list': send({"jsonrpc": "2.0", "id": mid, "result": {"tools": server.tools}}) elif method == 'tools/call': params = msg.get('params', {}) name = params.get('name') args = params.get('arguments', {}) log('call %s %s' % (name, args)) if server.backend is None and name not in ('screenshot', 'get_state'): send({"jsonrpc": "2.0", "id": mid, "result": {"content": [ {"type": "text", "text": json.dumps({'ok': False, 'error': 'input backend unavailable'})}]}}) continue try: result = server.call(name, args) send({"jsonrpc": "2.0", "id": mid, "result": {"content": [ {"type": "text", "text": json.dumps(result, ensure_ascii=False)}]}}) except Exception as e: log('tool err %s' % e) send({"jsonrpc": "2.0", "id": mid, "result": {"content": [ {"type": "text", "text": json.dumps({'ok': False, 'error': str(e)})}]}, "isError": True}) else: if mid is not None: send({"jsonrpc": "2.0", "id": mid, "error": {"code": -32601, "message": "method not found"}}) log('lc_mcp exit') if __name__ == '__main__': main()