#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Limbus Company 语义化操控 MCP server v0.6 (跨平台)。 核心思路: 不让 AI 盲猜坐标。工具层做"文本化 + 语义点击"—— get_state : 截图 + OCR -> 全页文字元素(带坐标) click_text : 说文字/按钮名, 内部 OCR 定位后点击 wait_for_text / wait_for_text_gone : 等条件出现/消失再回报 平台无关: 截图/输入按 LCP_PLATFORM (win|linux|mac, 默认自动检测) 加载实现。 win : 截图=交互计划任务; 输入=sendinput|interception linux : 截图=X11 PIL/scrot; 输入=xdotool|uinput(物理) mac : 截图=screencapture; 输入=cg OCR: 纯本地 (Windows.Media.Ocr / winsdk)。 协议: MCP stdio (newline-delimited JSON-RPC)。 """ import glob import os import re import struct import sys import json import time from lcplatform import detect_platform, get_input_backend, get_screenshot_provider VERSION = '0.6.0' BACKEND_NAME = os.environ.get('INPUT_BACKEND', '').strip().lower() OCR_BACKEND = os.environ.get('OCR_BACKEND', 'local').strip().lower() MIN_SCORE = float(os.environ.get('OCR_MIN_SCORE', '0.5')) KEEP_SHOTS = int(os.environ.get('LCP_KEEP_SHOTS', '30')) MAX_WAIT = 300.0 WAIT_INTERVAL = 1.2 BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp') os.makedirs(BASE, exist_ok=True) NEED_SCREEN = {'screenshot', 'get_state', 'wait_for_text', 'wait_for_text_gone', 'click_text'} NEED_INPUT = {'click_text', 'click', 'move', 'keypress', 'type_text'} def log(msg): try: with open(os.path.join(BASE, 'lc_mcp.log'), 'a', encoding='utf-8') as f: f.write('[%s] %s\n' % (time.strftime('%H:%M:%S'), msg)) except Exception: pass def ocr_image(img_path): """本地 OCR(Windows 自带 OCR,winsdk;无任何云端依赖)。""" try: import ocr_local except Exception as e: return None, 'ocr_local import fail: %s' % e if not ocr_local.available(): return None, 'winsdk 不可用: pip install --user winsdk' items, err = ocr_local.ocr_image(img_path) if items is None: return None, err # Windows.Media.Ocr 不给置信度; 只有带 score 的后端才会被过滤 items = [it for it in items if float(it.get('score', 1)) >= MIN_SCORE] return items, None def png_size(path): """读 PNG IHDR 拿宽高(纯标准库)。失败返回 None。""" try: with open(path, 'rb') as f: head = f.read(24) if head[:8] != b'\x89PNG\r\n\x1a\n': return None return struct.unpack('>II', head[16:24]) except Exception: return None def prune_shots(path, keep=KEEP_SHOTS): """只保留同目录下最新 keep 张截图,防止轮询把磁盘写满。""" if keep <= 0: return d = os.path.dirname(path) shots = glob.glob(os.path.join(d, 'shot_*.png')) + glob.glob(os.path.join(d, 'lc_shot_*.png')) shots.sort(key=lambda p: os.path.getmtime(p) if os.path.exists(p) else 0) for p in shots[:-keep]: try: os.remove(p) except Exception: pass _WS = re.compile(r'\s+') def norm(s): """匹配用归一化: 去全部空白 + 小写。 Windows OCR 对中文按单字切词, 拼行时会变成 "开 始"。""" return _WS.sub('', str(s)).lower() def calc_click(txt, items): """在 OCR items 里找包含 txt 的项,返回中心列表。完全相等的排前面。""" key = norm(txt) if not key: return [] exact, partial = [], [] for it in items: t = norm(it['text']) if key not in t: continue hit = {'text': it['text'], 'x': it['x'] + it['w'] // 2, 'y': it['y'] + it['h'] // 2} (exact if t == key else partial).append(hit) return exact + partial class MCPServer: def __init__(self, platform, provider, backend): self.platform = platform self.provider = provider self.backend = backend self.tools = [ {"name": "get_state", "description": "截图并OCR, 返回当前屏幕全部文字元素(含坐标)。AI 决策前先调这个, 不需要自己猜坐标", "inputSchema": {"type": "object", "properties": {}}}, {"name": "click_text", "description": "语义点击: 给按钮文字(支持子串, 忽略空格; 完全匹配优先), 内部OCR定位后点击其中心。例: click_text('开始')", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}, "index": {"type": "number", "default": 0}}, "required": ["text"]}}, {"name": "wait_for_text", "description": "循环截图+OCR, 等到指定文字出现(默认30秒, 上限300秒)", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}, "timeout": {"type": "number", "default": 30}}, "required": ["text"]}}, {"name": "wait_for_text_gone", "description": "循环截图+OCR, 等到指定文字消失(默认30秒, 上限300秒)", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}, "timeout": {"type": "number", "default": 30}}, "required": ["text"]}}, {"name": "screenshot", "description": "截图并返回PNG路径(用于模板/人工查看)", "inputSchema": {"type": "object", "properties": {}}}, {"name": "click", "description": "直接按坐标点击(截图像素坐标; 一般用 click_text 更稳)", "inputSchema": {"type": "object", "properties": { "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right"], "default": "left"}}, "required": ["x", "y"]}}, {"name": "move", "description": "移动鼠标到(x,y)", "inputSchema": {"type": "object", "properties": { "x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, {"name": "keypress", "description": "按一次键(p/enter/esc/space/tab/alt/f1-f12/单字符等)", "inputSchema": {"type": "object", "properties": { "key": {"type": "string"}}, "required": ["key"]}}, {"name": "type_text", "description": "输入一串文本", "inputSchema": {"type": "object", "properties": { "text": {"type": "string"}}, "required": ["text"]}}, ] def _grab(self): """截图 + 清理旧图 + 把真实屏幕像素尺寸告诉输入后端(坐标换算以截图为准)。""" sh = self.provider.grab() if sh.get('ok'): prune_shots(sh['path']) size = png_size(sh['path']) if size and hasattr(self.backend, 'set_screen'): self.backend.set_screen(*size) return sh def _ocr_now(self): sh = self._grab() if not sh.get('ok'): return sh, None items, err = ocr_image(sh['path']) if err: return {'ok': False, 'error': err, 'screen': sh['path']}, None return sh, items or [] def call(self, name, args): args = args or {} if name == 'screenshot': return self._grab() if name == 'get_state': sh, items = self._ocr_now() if items is None: return sh return {'ok': True, 'screen': sh['path'], 'items': items} if name == 'click_text': txt = str(args.get('text', '')) if not norm(txt): return {'ok': False, 'error': 'text must be non-empty'} sh, items = self._ocr_now() if items is None: return sh hits = calc_click(txt, items) if not hits: return {'ok': False, 'error': 'text not found: %s' % txt, 'screen': sh['path'], 'items': items} idx = int(args.get('index', 0)) if not 0 <= idx < len(hits): return {'ok': False, 'error': 'index %d out of range (%d hits)' % (idx, len(hits)), 'hits': hits[:10]} h = hits[idx] r = self.backend.click(h['x'], h['y']) r['on'] = h['text'] if len(hits) > 1: r['other_hits'] = len(hits) - 1 return r if name in ('wait_for_text', 'wait_for_text_gone'): txt = str(args.get('text', '')) if not norm(txt): return {'ok': False, 'error': 'text must be non-empty'} timeout = max(0.0, min(float(args.get('timeout', 30)), MAX_WAIT)) return self._wait(txt, timeout, name == 'wait_for_text') if name == 'click': button = args.get('button', 'left') if button not in ('left', 'right'): return {'ok': False, 'error': 'button must be left/right'} return self.backend.click(int(args['x']), int(args['y']), button, 1) if name == 'move': return self.backend.move(int(args['x']), int(args['y'])) if name == 'keypress': return self.backend.keypress(str(args.get('key', ''))) if name == 'type_text': return self.backend.type_text(str(args.get('text', ''))) return {'ok': False, 'error': 'unknown tool %s' % name} def _wait(self, txt, timeout, want_present): deadline = time.time() + timeout last_items, last_err = [], None while True: sh, items = self._ocr_now() if items is not None: last_items = items hits = calc_click(txt, items) if bool(hits) == want_present: return {'ok': True, 'present': bool(hits), 'hits': hits[:5], 'screen': sh['path']} else: last_err = sh.get('error') if time.time() + WAIT_INTERVAL > deadline: break time.sleep(WAIT_INTERVAL) r = {'ok': False, 'error': 'timeout waiting %s (present=%s)' % (txt, want_present), 'last_items': last_items[:10]} if last_err: r['last_error'] = last_err return r def read_msg(): """读一行 JSON-RPC。EOF 返回 None; 空行/坏 JSON 抛 ValueError 由调用方回错误后继续。""" line = sys.stdin.readline() if not line: return None line = line.strip() if not line: return {} msg = json.loads(line) if not isinstance(msg, dict): raise ValueError('request must be a JSON object') return msg def send(obj): sys.stdout.write(json.dumps(obj, ensure_ascii=False) + '\n') sys.stdout.flush() def tool_result(mid, result, is_error=False): send({"jsonrpc": "2.0", "id": mid, "result": { "content": [{"type": "text", "text": json.dumps(result, ensure_ascii=False)}], "isError": bool(is_error)}}) def main(): platform = detect_platform() log('lc_mcp v%s platform=%s backend=%s ocr=%s' % (VERSION, platform, BACKEND_NAME or 'auto', OCR_BACKEND)) provider = backend = None try: provider = get_screenshot_provider(platform) except Exception as e: log('screenshot provider init fail: %s' % e) try: backend = get_input_backend(BACKEND_NAME, platform) except Exception as e: log('input backend init fail: %s' % e) server = MCPServer(platform, provider, backend) while True: try: msg = read_msg() except ValueError as e: log('parse err %s' % e) send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error: %s" % e}}) continue if msg is None: break if not msg: continue method = msg.get('method') mid = msg.get('id') if method == 'initialize': send({"jsonrpc": "2.0", "id": mid, "result": { "protocolVersion": "2024-11-05", "capabilities": {"tools": {"listChanged": False}}, "serverInfo": {"name": "lc-mcp", "version": VERSION}}}) elif method == 'ping': send({"jsonrpc": "2.0", "id": mid, "result": {}}) elif method == 'tools/list': send({"jsonrpc": "2.0", "id": mid, "result": {"tools": server.tools}}) elif method == 'tools/call': params = msg.get('params') or {} name = params.get('name') args = params.get('arguments') or {} log('call %s %s' % (name, args)) if name in NEED_SCREEN and server.provider is None: tool_result(mid, {'ok': False, 'error': 'screenshot provider unavailable (see lc_mcp.log)'}, True) continue if name in NEED_INPUT and server.backend is None: tool_result(mid, {'ok': False, 'error': 'input backend unavailable (see lc_mcp.log)'}, True) continue try: result = server.call(name, args) tool_result(mid, result, not result.get('ok', True)) except Exception as e: log('tool err %s: %r' % (name, e)) tool_result(mid, {'ok': False, 'error': '%s: %s' % (type(e).__name__, e)}, True) elif mid is not None: send({"jsonrpc": "2.0", "id": mid, "error": {"code": -32601, "message": "method not found"}}) # 其余无 id 的通知(notifications/*)按规范忽略 log('lc_mcp exit') if __name__ == '__main__': main()