lc-mcp v0.5: 语义化操控(MCP stdio) — OCR文本化+语义点击, win/linux/mac 截图与输入后端
This commit is contained in:
@@ -0,0 +1,230 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""Limbus Company 语义化操控 MCP server v0.5 (跨平台)。
|
||||||
|
|
||||||
|
核心思路: 不让 AI 盲猜坐标。工具层做"文本化 + 语义点击"——
|
||||||
|
get_state : 截图 + OCR -> 全页文字元素(带坐标)
|
||||||
|
click_text : 说文字/按钮名, 内部 OCR 定位后点击
|
||||||
|
wait_for_text / wait_for_text_gone : 等条件出现/消失再回报
|
||||||
|
平台无关: 截图/输入按 LCP_PLATFORM (win|linux|mac, 默认自动检测) 加载实现。
|
||||||
|
win : 截图=交互计划任务; 输入=sendinput|interception
|
||||||
|
linux : 截图=X11 PIL/scrot; 输入=xdotool|uinput(物理)
|
||||||
|
mac : 截图=screencapture; 输入=cg
|
||||||
|
OCR 服务: 跑在远端(默认 http://35.212.193.150:8599/), 客户端机器零安装。
|
||||||
|
协议: MCP stdio (newline-delimited JSON-RPC)。
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from platform import detect_platform, get_input_backend, get_screenshot_provider
|
||||||
|
|
||||||
|
BACKEND_NAME = os.environ.get('INPUT_BACKEND', '').strip().lower()
|
||||||
|
OCR_URL = os.environ.get('OCR_URL', 'http://35.212.193.150:8599/').strip().rstrip('/') + '/'
|
||||||
|
MIN_SCORE = float(os.environ.get('OCR_MIN_SCORE', '0.5'))
|
||||||
|
BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp')
|
||||||
|
os.makedirs(BASE, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
try:
|
||||||
|
with open(os.path.join(BASE, 'lc_mcp.log'), 'a', encoding='utf-8') as f:
|
||||||
|
f.write('[%s] %s\n' % (time.strftime('%H:%M:%S'), msg))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def ocr_image(img_path):
|
||||||
|
"""把本地 png POST 给远端 OCR,返回 items 列表。"""
|
||||||
|
with open(img_path, 'rb') as f:
|
||||||
|
body = f.read()
|
||||||
|
req = urllib.request.Request(OCR_URL, data=body, headers={'Content-Type': 'image/png'})
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = json.loads(resp.read().decode('utf-8'))
|
||||||
|
except Exception as e:
|
||||||
|
return None, 'ocr call fail: %s' % e
|
||||||
|
if not data.get('ok'):
|
||||||
|
return None, data.get('error', 'ocr error')
|
||||||
|
items = [it for it in data.get('items', []) if float(it.get('score', 0)) >= MIN_SCORE]
|
||||||
|
return items, None
|
||||||
|
|
||||||
|
|
||||||
|
def calc_click(txt, items):
|
||||||
|
"""在 OCR items 里找包含 txt 的项,返回中心和候选列表。"""
|
||||||
|
hits = []
|
||||||
|
for it in items:
|
||||||
|
if txt.lower() in it['text'].lower():
|
||||||
|
cx = it['x'] + it['w'] // 2
|
||||||
|
cy = it['y'] + it['h'] // 2
|
||||||
|
hits.append({'text': it['text'], 'x': cx, 'y': cy})
|
||||||
|
return hits
|
||||||
|
|
||||||
|
|
||||||
|
class MCPServer:
|
||||||
|
|
||||||
|
def __init__(self, platform, provider, backend):
|
||||||
|
self.platform = platform
|
||||||
|
self.provider = provider
|
||||||
|
self.backend = backend
|
||||||
|
self.tools = [
|
||||||
|
{"name": "get_state", "description": "截图并OCR, 返回当前屏幕全部文字元素(含坐标)。AI 决策前先调这个, 不需要自己猜坐标",
|
||||||
|
"inputSchema": {"type": "object", "properties": {}}},
|
||||||
|
{"name": "click_text", "description": "语义点击: 给按钮文字(支持子串), 内部OCR定位后点击其中心。例: click_text('开始')",
|
||||||
|
"inputSchema": {"type": "object", "properties": {
|
||||||
|
"text": {"type": "string"}, "index": {"type": "number", "default": 0}},
|
||||||
|
"required": ["text"]}},
|
||||||
|
{"name": "wait_for_text", "description": "循环截图+OCR, 等到指定文字出现(默认最多30秒)", "inputSchema": {"type": "object", "properties": {
|
||||||
|
"text": {"type": "string"}, "timeout": {"type": "number", "default": 30}},
|
||||||
|
"required": ["text"]}},
|
||||||
|
{"name": "wait_for_text_gone", "description": "循环截图+OCR, 等到指定文字消失(默认最多30秒)", "inputSchema": {"type": "object", "properties": {
|
||||||
|
"text": {"type": "string"}, "timeout": {"type": "number", "default": 30}},
|
||||||
|
"required": ["text"]}},
|
||||||
|
{"name": "screenshot", "description": "截图并返回PNG路径(用于模板/人工查看)", "inputSchema": {"type": "object", "properties": {}}},
|
||||||
|
{"name": "click", "description": "直接按坐标点击(一般用 click_text 更稳)", "inputSchema": {"type": "object", "properties": {
|
||||||
|
"x": {"type": "number"}, "y": {"type": "number"},
|
||||||
|
"button": {"type": "string", "enum": ["left", "right"], "default": "left"}},
|
||||||
|
"required": ["x", "y"]}},
|
||||||
|
{"name": "move", "description": "移动鼠标到(x,y)", "inputSchema": {"type": "object", "properties": {
|
||||||
|
"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}},
|
||||||
|
{"name": "keypress", "description": "按一次键(p/enter/esc/space/tab/alt/f4/单字符等)", "inputSchema": {"type": "object", "properties": {
|
||||||
|
"key": {"type": "string"}}, "required": ["key"]}},
|
||||||
|
{"name": "type_text", "description": "输入一串文本", "inputSchema": {"type": "object", "properties": {
|
||||||
|
"text": {"type": "string"}}, "required": ["text"]}},
|
||||||
|
]
|
||||||
|
|
||||||
|
def call(self, name, args):
|
||||||
|
args = args or {}
|
||||||
|
if name == 'screenshot':
|
||||||
|
return self.provider.grab()
|
||||||
|
if name == 'get_state':
|
||||||
|
sh = self.provider.grab()
|
||||||
|
if not sh.get('ok'):
|
||||||
|
return sh
|
||||||
|
items, err = ocr_image(sh['path'])
|
||||||
|
if err:
|
||||||
|
return {'ok': False, 'error': err}
|
||||||
|
return {'ok': True, 'screen': sh['path'], 'items': items}
|
||||||
|
if name == 'click_text':
|
||||||
|
sh = self.provider.grab()
|
||||||
|
if not sh.get('ok'):
|
||||||
|
return sh
|
||||||
|
items, err = ocr_image(sh['path'])
|
||||||
|
if err:
|
||||||
|
return {'ok': False, 'error': err}
|
||||||
|
hits = calc_click(args.get('text', ''), items or [])
|
||||||
|
if not hits:
|
||||||
|
return {'ok': False, 'error': 'text not found: %s' % args.get('text', ''),
|
||||||
|
'screen': sh['path'], 'items': items}
|
||||||
|
idx = min(int(args.get('index', 0)), len(hits) - 1)
|
||||||
|
h = hits[idx]
|
||||||
|
r = self.backend.click(h['x'], h['y'])
|
||||||
|
r['on'] = h['text']
|
||||||
|
return r
|
||||||
|
if name == 'wait_for_text':
|
||||||
|
return self._wait(args.get('text', ''), float(args.get('timeout', 30)), True)
|
||||||
|
if name == 'wait_for_text_gone':
|
||||||
|
return self._wait(args.get('text', ''), float(args.get('timeout', 30)), False)
|
||||||
|
if name == 'click':
|
||||||
|
return self.backend.click(int(args.get('x', 0)), int(args.get('y', 0)), args.get('button', 'left'), 1)
|
||||||
|
if name == 'move':
|
||||||
|
return self.backend.move(int(args.get('x', 0)), int(args.get('y', 0)))
|
||||||
|
if name == 'keypress':
|
||||||
|
return self.backend.keypress(args.get('key', ''))
|
||||||
|
if name == 'type_text':
|
||||||
|
return self.backend.type_text(args.get('text', ''))
|
||||||
|
return {'ok': False, 'error': 'unknown tool %s' % name}
|
||||||
|
|
||||||
|
def _wait(self, txt, timeout, want_present):
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
last_items = []
|
||||||
|
while time.time() < deadline:
|
||||||
|
sh = self.provider.grab()
|
||||||
|
if sh.get('ok'):
|
||||||
|
items, err = ocr_image(sh['path'])
|
||||||
|
if not err:
|
||||||
|
last_items = items or []
|
||||||
|
hits = calc_click(txt, last_items)
|
||||||
|
if bool(hits) == want_present:
|
||||||
|
return {'ok': True, 'present': bool(hits),
|
||||||
|
'hits': hits[:5], 'screen': sh['path']}
|
||||||
|
time.sleep(1.2)
|
||||||
|
return {'ok': False, 'error': 'timeout waiting %s (present=%s)' % (txt, want_present),
|
||||||
|
'last_items': last_items[:10]}
|
||||||
|
|
||||||
|
|
||||||
|
def read_msg():
|
||||||
|
line = sys.stdin.readline()
|
||||||
|
if not line:
|
||||||
|
return None
|
||||||
|
return json.loads(line.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def send(obj):
|
||||||
|
sys.stdout.write(json.dumps(obj, ensure_ascii=False) + '\n')
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
platform = detect_platform()
|
||||||
|
log('lc_mcp v0.5 platform=%s backend=%s ocr=%s' % (platform, BACKEND_NAME or 'auto', OCR_URL))
|
||||||
|
backend = None
|
||||||
|
try:
|
||||||
|
provider = get_screenshot_provider(platform)
|
||||||
|
backend = get_input_backend(BACKEND_NAME, platform)
|
||||||
|
except Exception as e:
|
||||||
|
log('init fail: %s' % e)
|
||||||
|
provider = None
|
||||||
|
backend = None
|
||||||
|
if backend is None:
|
||||||
|
log('WARN: input backend unavailable, semantic/click tools disabled')
|
||||||
|
server = MCPServer(platform, provider, backend)
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
msg = read_msg()
|
||||||
|
if msg is None:
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
log('read err %s' % e)
|
||||||
|
break
|
||||||
|
method = msg.get('method')
|
||||||
|
mid = msg.get('id')
|
||||||
|
if method == 'initialize':
|
||||||
|
send({"jsonrpc": "2.0", "id": mid, "result": {
|
||||||
|
"protocolVersion": "2024-11-05",
|
||||||
|
"capabilities": {"tools": {"listChanged": False}},
|
||||||
|
"serverInfo": {"name": "lc-mcp", "version": "0.5.0"}}})
|
||||||
|
elif method == 'notifications/initialized':
|
||||||
|
continue
|
||||||
|
elif method == 'ping':
|
||||||
|
send({"jsonrpc": "2.0", "id": mid, "result": {}})
|
||||||
|
elif method == 'tools/list':
|
||||||
|
send({"jsonrpc": "2.0", "id": mid, "result": {"tools": server.tools}})
|
||||||
|
elif method == 'tools/call':
|
||||||
|
params = msg.get('params', {})
|
||||||
|
name = params.get('name')
|
||||||
|
args = params.get('arguments', {})
|
||||||
|
log('call %s %s' % (name, args))
|
||||||
|
if server.backend is None and name not in ('screenshot', 'get_state'):
|
||||||
|
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [
|
||||||
|
{"type": "text", "text": json.dumps({'ok': False, 'error': 'input backend unavailable'})}]}})
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result = server.call(name, args)
|
||||||
|
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [
|
||||||
|
{"type": "text", "text": json.dumps(result, ensure_ascii=False)}]}})
|
||||||
|
except Exception as e:
|
||||||
|
log('tool err %s' % e)
|
||||||
|
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [
|
||||||
|
{"type": "text", "text": json.dumps({'ok': False, 'error': str(e)})}]}, "isError": True})
|
||||||
|
else:
|
||||||
|
if mid is not None:
|
||||||
|
send({"jsonrpc": "2.0", "id": mid,
|
||||||
|
"error": {"code": -32601, "message": "method not found"}})
|
||||||
|
log('lc_mcp exit')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user