lc-mcp v0.5: 语义化操控(MCP stdio) — OCR文本化+语义点击, win/linux/mac 截图与输入后端

Browse Source
This commit is contained in:
yws
2026-10-01 16:37:33 +00:00
commit 0db14040b7
15 changed files with 1124 additions and 0 deletions
+6
View File
@@ -0,0 +1,6 @@
__pycache__/
*.pyc
*.log
shot_*.png
lc_mcp/job.ps1
lc_mcp/run.vbs
+68
View File
@@ -0,0 +1,68 @@
# lc-mcp — Limbus Company 语义化操控 MCP server (跨平台)
给 AI agent (codex / deepseek / 任意 MCP 客户端) 提供**语义化**游戏操控,
解决"盲看屏幕猜坐标→思考慢、点错率高"。
## 核心设计
AI 不用猜坐标。工具层做"文本化 + 语义点击":
```
get_state() -> 截图+远端OCR:全屏文字元素 + 坐标
click_text("开始") -> 内部 OCR 定位"开始"并点击中心
wait_for_text("结算") -> 轮询直到文字出现/消失
```
任务循环: `get_state -> 决策 -> click_text -> wait_for_text -> 确认 -> 下一步`
## 平台矩阵
| 平台 | 截图 | 输入后端 | 备注 |
|------|------|----------|------|
| win | 交互计划任务 | `sendinput`(默认) / `interception`(物理HID级) | interception 需装驱动+重启 |
| linux | X11 PIL / scrot | `xdotool` / `uinput`(内核虚拟设备,物理级) | uinput 需 root 或 uinput 组 |
| mac | screencapture | `cg`(CGEvent, PyObjC) | 需辅助功能权限 |
`LCP_PLATFORM=win|linux|mac` 强制指定,默认按系统自动检测。
OCR 服务默认 http://35.212.193.150:8599/(gcp1 上的 RapidOCR,客户端零安装),
可 `OCR_URL` 覆盖。
## 部署
1. 拷贝 `lc-mcp-dev/` 到目标机器
2. 装依赖(只用标准库;linux xdotool 后端需 `apt install xdotool`;mac cg 需 `pip install pyobjc-framework-Quartz`)
3. 启动:`python lc_mcp.py`(stdio),MCP 客户端 spawn
```jsonc
// codex MCP 配置 (codex/mcp.json 或项目 .mcp.json)
{
"mcpServers": {
"limbus": {
"type": "stdio",
"command": "ssh",
"args": ["win", "python", "C:/Users/yws-hhh/lc-mcp-dev/lc_mcp.py"],
"env": { "INPUT_BACKEND": "interception" }
}
}
}
```
## 环境变量
| 变量 | 默认 | 说明 |
|------|------|------|
| `LCP_PLATFORM` | 自动 | win/linux/mac |
| `INPUT_BACKEND` | 平台默认 | mock/sendinput/interception/xdotool/uinput/cg |
| `OCR_URL` | gcp1:8599 | 远端 OCR 服务 |
| `OCR_MIN_SCORE` | 0.5 | 过滤低置信度 OCR 结果 |
## 工具
get_state · click_text · wait_for_text · wait_for_text_gone · screenshot · click · move · keypress · type_text
## 安全提醒
- 输入会真实操作你已登录的桌面,跑任务时别动鼠标
- win 的截图走交互计划任务(Registry + wscript),杀软可能告警
- interception/uinput 是"物理化"输入,用于降低虚拟输入痕迹;真·硬件级可外接 Arduino/RPi Pico HID
+35
View File
@@ -0,0 +1,35 @@
# -*- coding: utf-8 -*-
"""输入后端接口。每种后端实现同一套动作:click/move/keypress/type_text。"""
from abc import ABC, abstractmethod
class InputBackend(ABC):
@abstractmethod
def click(self, x, y, button='left', clicks=1):
...
@abstractmethod
def move(self, x, y):
...
@abstractmethod
def keypress(self, key):
...
@abstractmethod
def type_text(self, text):
...
def backend_factory(name, **kwargs):
if name == 'mock':
from .mock import MockBackend
return MockBackend(**kwargs)
if name == 'sendinput':
from .sendinput_win import SendInputBackend
return SendInputBackend(**kwargs)
if name == 'interception':
from .interception_win import InterceptionBackend
return InterceptionBackend(**kwargs)
raise ValueError('unknown backend: %s' % name)
+64
View File
@@ -0,0 +1,64 @@
# -*- coding: utf-8 -*-
"""macOS 输入后端:Quartz CGEvent (需 PyObjC: pip install pyobjc-framework-Quartz)。"""
import time
class CGBackend:
def __init__(self):
try:
import Quartz
self.q = Quartz
except ImportError:
raise RuntimeError('PyObjC Quartz not installed (pip install pyobjc-framework-Quartz)')
def _post(self, event):
self.q.CGEventPost(self.q.kCGHIDEventTap, event)
time.sleep(0.02)
def move(self, x, y):
e = self.q.CGEventCreateMouseEvent(None, self.q.kCGEventMouseMoved, (float(x), float(y)), self.q.kCGMouseButtonLeft)
self._post(e)
return {'ok': True}
def click(self, x, y, button='left', times=1):
btn = self.q.kCGMouseButtonLeft if button == 'left' else self.q.kCGMouseButtonRight
self.move(x, y)
for _ in range(times):
down = self.q.CGEventCreateMouseEvent(None, self.q.kCGEventLeftMouseDown if button == 'left' else self.q.kCGEventRightMouseDown, (float(x), float(y)), btn)
up = self.q.CGEventCreateMouseEvent(None, self.q.kCGEventLeftMouseUp if button == 'left' else self.q.kCGEventRightMouseUp, (float(x), float(y)), btn)
self._post(down)
self._post(up)
return {'ok': True}
def keypress(self, key):
mapping = {'enter': 36, 'esc': 53, 'space': 49, 'tab': 48, 'f4': 96,
'ctrl': 59, 'shift': 56, 'alt': 58, 'backspace': 51}
k = key.lower()
if k in mapping:
self._key(mapping[k])
return {'ok': True}
if len(k) == 1 and 'a' <= k <= 'z':
# mac 虚拟键码: a=0, b=11, ... 非连续, 用简单表
table = {'a': 0, 'b': 11, 'c': 8, 'd': 2, 'e': 14, 'f': 3, 'g': 5, 'h': 4,
'i': 34, 'j': 38, 'k': 40, 'l': 37, 'm': 46, 'n': 45, 'o': 31,
'p': 35, 'q': 12, 'r': 15, 's': 1, 't': 17, 'u': 32, 'v': 9,
'w': 13, 'x': 7, 'y': 16, 'z': 6}
if k in table:
self._key(table[k])
return {'ok': True}
return {'ok': False, 'error': 'unsupported key %s' % key}
def _key(self, vk):
down = self.q.CGEventCreateKeyboardEvent(None, vk, True)
up = self.q.CGEventCreateKeyboardEvent(None, vk, False)
self._post(down)
self._post(up)
def type_text(self, text):
for ch in text:
r = self.keypress(ch)
if not r.get('ok'):
return r
time.sleep(0.03)
return {'ok': True}
+195
View File
@@ -0,0 +1,195 @@
# -*- coding: utf-8 -*-
"""Interception 物理输入后端(Windows + Interception 驱动)。
输入在驱动层转为真实 HID 事件(对应用透明,非 SendInput 虚拟输入)。
需要: install-interception.exe /install 后重启系统,且本 dll 与驱动共存。
"""
import ctypes
import time
from ctypes import wintypes
# ---- Interception 常量 ----
MOUSE_LEFTBUTTON_DOWN = 0x001
MOUSE_LEFTBUTTON_UP = 0x002
MOUSE_RIGHTBUTTON_DOWN = 0x004
MOUSE_RIGHTBUTTON_UP = 0x008
MOUSE_MIDDLEBUTTON_DOWN = 0x010
MOUSE_MIDDLEBUTTON_UP = 0x020
MOUSE_MOVE_RELATIVE = 0x000
MOUSE_MOVE_ABSOLUTE = 0x001
MOUSE_VIRTUAL_DESKTOP = 0x002
KEY_DOWN = 0x00
KEY_UP = 0x01
KEY_E0 = 0x02
# ---- Set 1 扫描码 ----
SCAN_ALPHA = {chr(c): 0x1E + i for i, c in enumerate(range(ord('A'), ord('Z') + 1))}
# 上面的映射只在 A-P 连续,Q-Z 不连续,改为显式表
SCAN = {
'a': 0x1E, 'b': 0x30, 'c': 0x2E, 'd': 0x20, 'e': 0x12, 'f': 0x21,
'g': 0x22, 'h': 0x23, 'i': 0x17, 'j': 0x24, 'k': 0x25, 'l': 0x26,
'm': 0x32, 'n': 0x31, 'o': 0x18, 'p': 0x19, 'q': 0x10, 'r': 0x13,
's': 0x1F, 't': 0x14, 'u': 0x16, 'v': 0x2F, 'w': 0x11, 'x': 0x2D,
'y': 0x15, 'z': 0x2C,
'1': 0x02, '2': 0x03, '3': 0x04, '4': 0x05, '5': 0x06,
'6': 0x07, '7': 0x08, '8': 0x09, '9': 0x0A, '0': 0x0B,
'enter': 0x1C, 'return': 0x1C, 'esc': 0x01, 'space': 0x39, 'tab': 0x0F,
'ctrl': 0x1D, 'alt': 0x38, 'lshift': 0x2A, 'rshift': 0x36, 'shift': 0x2A,
'f1': 0x3B, 'f2': 0x3C, 'f3': 0x3D, 'f4': 0x3E, 'f5': 0x3F, 'f6': 0x40,
'f7': 0x41, 'f8': 0x42, 'f9': 0x43, 'f10': 0x44, 'f11': 0x57, 'f12': 0x58,
',': 0x33, '.': 0x34, '/': 0x35, ';': 0x27, "'": 0x28, '[': 0x1A,
']': 0x1B, '-': 0x0C, '=': 0x0D, '`': 0x29, '\\': 0x2B,
}
UPPER_SCAN = { # 大写所需符号的扫描码(按住 Shift 时)
'!': 0x02, '@': 0x03, '#': 0x04, '$': 0x05, '%': 0x06, '^': 0x07,
'&': 0x08, '*': 0x09, '(': 0x0A, ')': 0x0B, '_': 0x0C, '+': 0x0D,
'{': 0x1A, '}': 0x1B, ':': 0x27, '"': 0x28, '<': 0x33, '>': 0x34,
'?': 0x35, '~': 0x29, '|': 0x2B,
}
MAX_DEVICES = 64 # 键盘下游设备数量通常 <= 16,鼠标 <= 16;放宽到 64 覆盖异常
class MouseStroke(ctypes.Structure):
_fields_ = [("state", ctypes.c_ushort), ("flags", ctypes.c_ushort),
("rolling", ctypes.c_short), ("x", ctypes.c_ulong),
("y", ctypes.c_ulong), ("information", ctypes.c_ulong)]
class KeyStroke(ctypes.Structure):
_fields_ = [("code", ctypes.c_ushort), ("state", ctypes.c_ushort),
("information", ctypes.c_ulong)]
class InterceptionBackend:
def __init__(self, dll_path=None):
if dll_path is None:
dll_path = r'C:\Program Files\Interception\interception.dll'
try:
self.dll = ctypes.CDLL(dll_path)
except OSError:
# 兜底:尝试系统 PATH / 常见目录
self.dll = ctypes.CDLL('interception.dll')
self._api()
self._user32 = ctypes.WinDLL('user32', use_last_error=True)
self.ctx = self.dll.interception_create_context()
if not self.ctx:
raise RuntimeError('interception_create_context failed (驱动未安装/未重启?)')
self._kbd_devs, self._mou_devs = self._discover()
self._scr_w = self._user32.GetSystemMetrics(0)
self._scr_h = self._user32.GetSystemMetrics(1)
def _api(self):
d = self.dll
d.interception_create_context.restype = ctypes.c_void_p
d.interception_create_context.argtypes = []
d.interception_destroy_context.argtypes = [ctypes.c_void_p]
d.interception_open_device.restype = ctypes.c_bool
d.interception_open_device.argtypes = [ctypes.c_void_p, ctypes.c_int]
d.interception_close_device.argtypes = [ctypes.c_void_p, ctypes.c_int]
d.interception_get_hardware_id.restype = ctypes.c_int
d.interception_get_hardware_id.argtypes = [ctypes.c_void_p, ctypes.c_int,
ctypes.c_char_p, ctypes.c_int]
d.interception_send.restype = ctypes.c_int
d.interception_send.argtypes = [ctypes.c_void_p, ctypes.c_int,
ctypes.c_void_p, ctypes.c_size_t]
def _discover(self):
kbd, mou = [], []
buf = ctypes.create_string_buffer(256)
for dev in range(1, MAX_DEVICES + 1):
if not self.dll.interception_open_device(self.ctx, dev):
continue
n = self.dll.interception_get_hardware_id(self.ctx, dev, buf, 256)
hid = buf.value.decode('ascii', 'replace').lower() if n > 0 else ''
if 'keyboard' in hid:
kbd.append(dev)
elif 'mouse' in hid:
mou.append(dev)
return kbd, mou
# ---- 底层发送 ----
def _kbd_stroke(self, code, state):
ks = KeyStroke()
ks.code = ctypes.c_ushort(code)
ks.state = ctypes.c_ushort(state)
for dev in self._kbd_devs or [1]:
try:
self.dll.interception_send(self.ctx, dev, ctypes.byref(ks), ctypes.sizeof(ks))
except Exception:
pass
def _mouse_stroke(self, state=0, flags=0, x=0, y=0):
ms = MouseStroke()
ms.state = ctypes.c_ushort(state)
ms.flags = ctypes.c_ushort(flags)
ms.x = ctypes.c_ulong(int(x))
ms.y = ctypes.c_ulong(int(y))
for dev in self._mou_devs or [17]:
try:
self.dll.interception_send(self.ctx, dev, ctypes.byref(ms), ctypes.sizeof(ms))
except Exception:
pass
# ---- 公开动作 ----
def move(self, x, y):
nx = int(int(x) * 65535 / self._scr_w) if self._scr_w else 0
ny = int(int(y) * 65535 / self._scr_h) if self._scr_h else 0
self._mouse_stroke(state=0, flags=MOUSE_MOVE_ABSOLUTE, x=nx, y=ny)
time.sleep(0.03)
return {'ok': True, 'backend': 'interception'}
def click(self, x, y, button='left', clicks=1):
self.move(x, y)
time.sleep(0.05)
if button == 'left':
down, up = MOUSE_LEFTBUTTON_DOWN, MOUSE_LEFTBUTTON_UP
else:
down, up = MOUSE_RIGHTBUTTON_DOWN, MOUSE_RIGHTBUTTON_UP
for _ in range(max(1, int(clicks))):
self._mouse_stroke(state=down)
time.sleep(0.04)
self._mouse_stroke(state=up)
time.sleep(0.05)
return {'ok': True, 'clicked': [int(x), int(y)], 'backend': 'interception'}
def _scancode(self, key):
k = str(key).lower()
return SCAN.get(k)
def keypress(self, key):
k = str(key)
code = SCAN.get(k.lower())
if code is None:
return {'ok': False, 'error': 'unknown key %s' % key}
self._kbd_stroke(code, KEY_DOWN)
time.sleep(0.04)
self._kbd_stroke(code, KEY_UP)
time.sleep(0.02)
return {'ok': True, 'key': k, 'backend': 'interception'}
def type_text(self, text):
for ch in str(text):
low = ch.lower()
if ch.isupper() and low in SCAN:
self._kbd_stroke(SCAN['shift'], KEY_DOWN)
self._kbd_stroke(SCAN[low], KEY_DOWN)
self._kbd_stroke(SCAN[low], KEY_UP)
self._kbd_stroke(SCAN['shift'], KEY_UP)
elif ch.lower() in SCAN:
code = SCAN[low]
self._kbd_stroke(code, KEY_DOWN)
self._kbd_stroke(code, KEY_UP)
elif ch in UPPER_SCAN:
self._kbd_stroke(SCAN['shift'], KEY_DOWN)
self._kbd_stroke(UPPER_SCAN[ch], KEY_DOWN)
self._kbd_stroke(UPPER_SCAN[ch], KEY_UP)
self._kbd_stroke(SCAN['shift'], KEY_UP)
else:
return {'ok': False, 'error': 'untypable char %r' % ch}
time.sleep(0.015)
return {'ok': True, 'typed': str(text), 'backend': 'interception'}
+31
View File
@@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
"""Mock 后端:记录动作,无真实输入。开发/测试用。"""
import time
class MockBackend:
def __init__(self, log=None):
self.log = log if log is not None else []
self._screen = (1920, 1080)
def click(self, x, y, button='left', clicks=1):
self.log.append({'act': 'click', 'x': int(x), 'y': int(y),
'button': button, 'clicks': int(clicks)})
time.sleep(0.02)
return {'ok': True, 'clicked': [int(x), int(y)], 'backend': 'mock'}
def move(self, x, y):
self.log.append({'act': 'move', 'x': int(x), 'y': int(y)})
time.sleep(0.01)
return {'ok': True, 'backend': 'mock'}
def keypress(self, key):
self.log.append({'act': 'keypress', 'key': str(key)})
time.sleep(0.01)
return {'ok': True, 'key': str(key), 'backend': 'mock'}
def type_text(self, text):
self.log.append({'act': 'type_text', 'text': str(text)})
time.sleep(0.01 * len(text))
return {'ok': True, 'typed': str(text), 'backend': 'mock'}
+100
View File
@@ -0,0 +1,100 @@
# -*- coding: utf-8 -*-
"""SendInput 后端(Windows,ctypes user32)。
当前线上 lc_mcp.py 用的机制;作为 interception 不可用时的兜底。
"""
import ctypes
import time
from ctypes import wintypes
INPUT_MOUSE = 0
INPUT_KEYBOARD = 1
KEYEVENTF_KEYUP = 0x0002
KEYEVENTF_UNICODE = 0x0004
MOUSEEVENTF_LEFTDOWN = 0x0002
MOUSEEVENTF_LEFTUP = 0x0004
MOUSEEVENTF_RIGHTDOWN = 0x0008
MOUSEEVENTF_RIGHTUP = 0x0010
class MOUSEINPUT(ctypes.Structure):
_fields_ = [("dx", wintypes.LONG), ("dy", wintypes.LONG),
("mouseData", wintypes.DWORD), ("dwFlags", wintypes.DWORD),
("time", wintypes.DWORD), ("dwExtraInfo", ctypes.POINTER(ctypes.c_ulong))]
class KEYBDINPUT(ctypes.Structure):
_fields_ = [("wVk", wintypes.WORD), ("wScan", wintypes.WORD),
("dwFlags", wintypes.DWORD), ("time", wintypes.DWORD),
("dwExtraInfo", ctypes.POINTER(ctypes.c_ulong))]
class INPUT(ctypes.Structure):
class _I(ctypes.Union):
_fields_ = [("mi", MOUSEINPUT), ("ki", KEYBDINPUT)]
_anonymous_ = ("i",)
_fields_ = [("type", wintypes.DWORD), ("i", _I)]
KEYMAP = {'enter': 0x0D, 'return': 0x0D, 'esc': 0x1B, 'space': 0x20,
'tab': 0x09, 'ctrl': 0x11, 'alt': 0x12, 'shift': 0x10,
'p': 0x50, 'f4': 0x73, 'f5': 0x74, 'f8': 0x77, 'f12': 0x7B,
'1': 0x31, '2': 0x32, '3': 0x33, '4': 0x34, '5': 0x35,
'6': 0x36, '7': 0x37, '8': 0x38, '9': 0x39, '0': 0x30}
class SendInputBackend:
def __init__(self):
self._user32 = ctypes.WinDLL('user32', use_last_error=True)
def _send_input(self, *inputs):
n = len(inputs)
arr = (INPUT * n)(*inputs)
return self._user32.SendInput(n, arr, ctypes.sizeof(INPUT))
def click(self, x, y, button='left', clicks=1):
self._user32.SetCursorPos(int(x), int(y))
time.sleep(0.03)
down_flag = MOUSEEVENTF_LEFTDOWN if button == 'left' else MOUSEEVENTF_RIGHTDOWN
up_flag = MOUSEEVENTF_LEFTUP if button == 'left' else MOUSEEVENTF_RIGHTUP
for _ in range(max(1, int(clicks))):
down = INPUT(); down.type = INPUT_MOUSE; down.mi.dwFlags = down_flag
up = INPUT(); up.type = INPUT_MOUSE; up.mi.dwFlags = up_flag
self._send_input(down, up)
time.sleep(0.05)
return {'ok': True, 'clicked': [int(x), int(y)], 'backend': 'sendinput'}
def move(self, x, y):
self._user32.SetCursorPos(int(x), int(y))
time.sleep(0.01)
return {'ok': True, 'backend': 'sendinput'}
def keypress(self, key):
name = str(key).lower()
vk = KEYMAP.get(name)
if vk is None and len(name) == 1:
vk = ord(name.upper()) if name.isalpha() else ord(name)
if vk is None:
return {'ok': False, 'error': 'unknown key %s' % key}
self._press(vk)
return {'ok': True, 'key': str(key), 'backend': 'sendinput'}
def _press(self, vk):
ki = INPUT(); ki.type = INPUT_KEYBOARD; ki.ki.wVk = vk
self._send_input(ki)
time.sleep(0.04)
ki2 = INPUT(); ki2.type = INPUT_KEYBOARD; ki2.ki.wVk = vk
ki2.ki.dwFlags = KEYEVENTF_KEYUP
self._send_input(ki2)
def type_text(self, text):
for ch in str(text):
ki = INPUT(); ki.type = INPUT_KEYBOARD
ki.ki.wScan = ord(ch); ki.ki.dwFlags = KEYEVENTF_UNICODE
ki2 = INPUT(); ki2.type = INPUT_KEYBOARD
ki2.ki.wScan = ord(ch)
ki2.ki.dwFlags = KEYEVENTF_UNICODE | KEYEVENTF_KEYUP
self._send_input(ki, ki2)
time.sleep(0.01)
return {'ok': True, 'typed': str(text), 'backend': 'sendinput'}
+94
View File
@@ -0,0 +1,94 @@
# -*- coding: utf-8 -*-
"""Linux uinput 物理输入后端(虚拟内核 evdev 设备,需 root 或 uinput 组权限)。
比 xdotool 更接近硬件:内核层面人为物理设备,无 X 层痕迹。
依赖: python3-uinput 或手工 ioctl。
"""
import os
import struct
try:
import uinput
HAVE_UINPUT = True
except ImportError:
HAVE_UINPUT = False
class UinputBackend:
def __init__(self):
if not HAVE_UINPUT:
raise RuntimeError('python3-uinput not installed')
self._ctx = None
def _ensure(self):
if self._ctx is None:
from uinput import Device, KEY_A, KEY_Z, KEY_ENTER, KEY_ESC, KEY_SPACE, KEY_TAB, KEY_F4, KEY_LEFTCTRL, KEY_LEFTSHIFT, KEY_LEFTALT, KEY_BACKSPACE, REL_X, REL_Y, BTN_LEFT, BTN_RIGHT
events = (
# 全键盘范围(够用): A-Z + 数字 + 常用
*[key for key in range(KEY_A, KEY_Z + 1)],
KEY_ENTER, KEY_ESC, KEY_SPACE, KEY_TAB, KEY_F4,
KEY_LEFTCTRL, KEY_LEFTSHIFT, KEY_LEFTALT, KEY_BACKSPACE,
REL_X, REL_Y, BTN_LEFT, BTN_RIGHT,
)
self._dev = Device(events)
return self._dev
def _key(self, code):
dev = self._ensure()
dev.emit(code, 1)
dev.emit(code, 0)
time.sleep(0.03)
def move(self, x, y):
# uinput 只有相对位移; 需要知道当前位置。先简单相对 0 通过 xdotool 定位, 或 https 轮询
# 简化: 用 python-xlib 查指针位置 -> 计算 delta
try:
from Xlib import display
d = display.Display()
root = d.screen().root
qp = root.query_pointer()
curx, cury = qp.root_x, qp.root_y
dx, dy = int(x) - curx, int(y) - cury
dev = self._ensure()
dev.emit(REL_X, dx)
dev.emit(REL_Y, dy)
d.sync()
except Exception:
return {'ok': False, 'error': 'uinput move needs python-xlib'}
return {'ok': True}
def click(self, x, y, button='left', times=1):
self.move(x, y)
dev = self._ensure()
btn = BTN_LEFT if button == 'left' else BTN_RIGHT
for _ in range(times):
dev.emit(btn, 1)
time.sleep(0.05)
dev.emit(btn, 0)
time.sleep(0.05)
return {'ok': True}
def keypress(self, key):
mapping = {'enter': KEY_ENTER, 'esc': KEY_ESC, 'space': KEY_SPACE,
'tab': KEY_TAB, 'f4': KEY_F4, 'ctrl': KEY_LEFTCTRL,
'shift': KEY_LEFTSHIFT, 'alt': KEY_LEFTALT, 'backspace': KEY_BACKSPACE}
k = key.lower()
if k in mapping:
self._key(mapping[k])
return {'ok': True}
if len(k) == 1 and 'a' <= k <= 'z':
from uinput import KEY_A
self._key(KEY_A + (ord(k) - ord('a')))
return {'ok': True}
return {'ok': False, 'error': 'unsupported key %s' % key}
def type_text(self, text):
for ch in text:
if 'a' <= ch.lower() <= 'z':
self.keypress(ch.lower())
elif ch == ' ':
self.keypress('space')
else:
return {'ok': False, 'error': 'unsupported char %r' % ch}
time.sleep(0.05)
return {'ok': True}
+50
View File
@@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-
"""Linux X11 输入后端:xdotool(X 事件注入,xdotool 需已安装)。
物理化升级路径: uinput 后端(虚拟内核 evdev 设备)。
"""
import shutil
import subprocess
import time
def _run(args):
return subprocess.run(args, capture_output=True, text=True, timeout=30)
class XdotoolBackend:
def __init__(self):
if not shutil.which('xdotool'):
raise RuntimeError('xdotool not found (apt install xdotool)')
def _screen(self):
try:
r = _run(['xdotool', 'getdisplaygeometry'])
w, h = r.stdout.strip().split()
return int(w), int(h)
except Exception:
return 1920, 1080
def move(self, x, y):
_run(['xdotool', 'mousemove', str(int(x)), str(int(y))])
return {'ok': True}
def click(self, x, y, button='left', times=1):
btn = {'left': '1', 'right': '3', 'middle': '2'}.get(button, '1')
_run(['xdotool', 'mousemove', str(int(x)), str(int(y))])
_run(['xdotool', 'click', '--repeat', str(times), '--delay', '60', btn])
return {'ok': True}
def keypress(self, key):
key = key.lower()
# xdotool 键名映射
mapping = {'enter': 'Return', 'esc': 'Escape', 'space': 'space',
'tab': 'Tab', 'alt': 'Alt_L', 'f4': 'F4', 'ctrl': 'Control_L',
'shift': 'Shift_L', 'backspace': 'BackSpace'}
k = mapping.get(key, key)
_run(['xdotool', 'key', k])
return {'ok': True}
def type_text(self, text):
_run(['xdotool', 'type', '--delay', '60', text])
return {'ok': True}
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Limbus Company 语义化操控 MCP server v0.5 (跨平台)。
核心思路: 不让 AI 盲猜坐标。工具层做"文本化 + 语义点击"——
get_state : 截图 + OCR -> 全页文字元素(带坐标)
click_text : 说文字/按钮名, 内部 OCR 定位后点击
wait_for_text / wait_for_text_gone : 等条件出现/消失再回报
平台无关: 截图/输入按 LCP_PLATFORM (win|linux|mac, 默认自动检测) 加载实现。
win : 截图=交互计划任务; 输入=sendinput|interception
linux : 截图=X11 PIL/scrot; 输入=xdotool|uinput(物理)
mac : 截图=screencapture; 输入=cg
OCR 服务: 跑在远端(默认 http://35.212.193.150:8599/), 客户端机器零安装。
协议: MCP stdio (newline-delimited JSON-RPC)。
"""
import os
import sys
import json
import time
import urllib.request
from platform import detect_platform, get_input_backend, get_screenshot_provider
BACKEND_NAME = os.environ.get('INPUT_BACKEND', '').strip().lower()
OCR_URL = os.environ.get('OCR_URL', 'http://35.212.193.150:8599/').strip().rstrip('/') + '/'
MIN_SCORE = float(os.environ.get('OCR_MIN_SCORE', '0.5'))
BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp')
os.makedirs(BASE, exist_ok=True)
def log(msg):
try:
with open(os.path.join(BASE, 'lc_mcp.log'), 'a', encoding='utf-8') as f:
f.write('[%s] %s\n' % (time.strftime('%H:%M:%S'), msg))
except Exception:
pass
def ocr_image(img_path):
"""把本地 png POST 给远端 OCR,返回 items 列表。"""
with open(img_path, 'rb') as f:
body = f.read()
req = urllib.request.Request(OCR_URL, data=body, headers={'Content-Type': 'image/png'})
try:
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read().decode('utf-8'))
except Exception as e:
return None, 'ocr call fail: %s' % e
if not data.get('ok'):
return None, data.get('error', 'ocr error')
items = [it for it in data.get('items', []) if float(it.get('score', 0)) >= MIN_SCORE]
return items, None
def calc_click(txt, items):
"""在 OCR items 里找包含 txt 的项,返回中心和候选列表。"""
hits = []
for it in items:
if txt.lower() in it['text'].lower():
cx = it['x'] + it['w'] // 2
cy = it['y'] + it['h'] // 2
hits.append({'text': it['text'], 'x': cx, 'y': cy})
return hits
class MCPServer:
def __init__(self, platform, provider, backend):
self.platform = platform
self.provider = provider
self.backend = backend
self.tools = [
{"name": "get_state", "description": "截图并OCR, 返回当前屏幕全部文字元素(含坐标)。AI 决策前先调这个, 不需要自己猜坐标",
"inputSchema": {"type": "object", "properties": {}}},
{"name": "click_text", "description": "语义点击: 给按钮文字(支持子串), 内部OCR定位后点击其中心。例: click_text('开始')",
"inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}, "index": {"type": "number", "default": 0}},
"required": ["text"]}},
{"name": "wait_for_text", "description": "循环截图+OCR, 等到指定文字出现(默认最多30秒)", "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}, "timeout": {"type": "number", "default": 30}},
"required": ["text"]}},
{"name": "wait_for_text_gone", "description": "循环截图+OCR, 等到指定文字消失(默认最多30秒)", "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}, "timeout": {"type": "number", "default": 30}},
"required": ["text"]}},
{"name": "screenshot", "description": "截图并返回PNG路径(用于模板/人工查看)", "inputSchema": {"type": "object", "properties": {}}},
{"name": "click", "description": "直接按坐标点击(一般用 click_text 更稳)", "inputSchema": {"type": "object", "properties": {
"x": {"type": "number"}, "y": {"type": "number"},
"button": {"type": "string", "enum": ["left", "right"], "default": "left"}},
"required": ["x", "y"]}},
{"name": "move", "description": "移动鼠标到(x,y)", "inputSchema": {"type": "object", "properties": {
"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}},
{"name": "keypress", "description": "按一次键(p/enter/esc/space/tab/alt/f4/单字符等)", "inputSchema": {"type": "object", "properties": {
"key": {"type": "string"}}, "required": ["key"]}},
{"name": "type_text", "description": "输入一串文本", "inputSchema": {"type": "object", "properties": {
"text": {"type": "string"}}, "required": ["text"]}},
]
def call(self, name, args):
args = args or {}
if name == 'screenshot':
return self.provider.grab()
if name == 'get_state':
sh = self.provider.grab()
if not sh.get('ok'):
return sh
items, err = ocr_image(sh['path'])
if err:
return {'ok': False, 'error': err}
return {'ok': True, 'screen': sh['path'], 'items': items}
if name == 'click_text':
sh = self.provider.grab()
if not sh.get('ok'):
return sh
items, err = ocr_image(sh['path'])
if err:
return {'ok': False, 'error': err}
hits = calc_click(args.get('text', ''), items or [])
if not hits:
return {'ok': False, 'error': 'text not found: %s' % args.get('text', ''),
'screen': sh['path'], 'items': items}
idx = min(int(args.get('index', 0)), len(hits) - 1)
h = hits[idx]
r = self.backend.click(h['x'], h['y'])
r['on'] = h['text']
return r
if name == 'wait_for_text':
return self._wait(args.get('text', ''), float(args.get('timeout', 30)), True)
if name == 'wait_for_text_gone':
return self._wait(args.get('text', ''), float(args.get('timeout', 30)), False)
if name == 'click':
return self.backend.click(int(args.get('x', 0)), int(args.get('y', 0)), args.get('button', 'left'), 1)
if name == 'move':
return self.backend.move(int(args.get('x', 0)), int(args.get('y', 0)))
if name == 'keypress':
return self.backend.keypress(args.get('key', ''))
if name == 'type_text':
return self.backend.type_text(args.get('text', ''))
return {'ok': False, 'error': 'unknown tool %s' % name}
def _wait(self, txt, timeout, want_present):
deadline = time.time() + timeout
last_items = []
while time.time() < deadline:
sh = self.provider.grab()
if sh.get('ok'):
items, err = ocr_image(sh['path'])
if not err:
last_items = items or []
hits = calc_click(txt, last_items)
if bool(hits) == want_present:
return {'ok': True, 'present': bool(hits),
'hits': hits[:5], 'screen': sh['path']}
time.sleep(1.2)
return {'ok': False, 'error': 'timeout waiting %s (present=%s)' % (txt, want_present),
'last_items': last_items[:10]}
def read_msg():
line = sys.stdin.readline()
if not line:
return None
return json.loads(line.strip())
def send(obj):
sys.stdout.write(json.dumps(obj, ensure_ascii=False) + '\n')
sys.stdout.flush()
def main():
platform = detect_platform()
log('lc_mcp v0.5 platform=%s backend=%s ocr=%s' % (platform, BACKEND_NAME or 'auto', OCR_URL))
backend = None
try:
provider = get_screenshot_provider(platform)
backend = get_input_backend(BACKEND_NAME, platform)
except Exception as e:
log('init fail: %s' % e)
provider = None
backend = None
if backend is None:
log('WARN: input backend unavailable, semantic/click tools disabled')
server = MCPServer(platform, provider, backend)
while True:
try:
msg = read_msg()
if msg is None:
break
except Exception as e:
log('read err %s' % e)
break
method = msg.get('method')
mid = msg.get('id')
if method == 'initialize':
send({"jsonrpc": "2.0", "id": mid, "result": {
"protocolVersion": "2024-11-05",
"capabilities": {"tools": {"listChanged": False}},
"serverInfo": {"name": "lc-mcp", "version": "0.5.0"}}})
elif method == 'notifications/initialized':
continue
elif method == 'ping':
send({"jsonrpc": "2.0", "id": mid, "result": {}})
elif method == 'tools/list':
send({"jsonrpc": "2.0", "id": mid, "result": {"tools": server.tools}})
elif method == 'tools/call':
params = msg.get('params', {})
name = params.get('name')
args = params.get('arguments', {})
log('call %s %s' % (name, args))
if server.backend is None and name not in ('screenshot', 'get_state'):
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [
{"type": "text", "text": json.dumps({'ok': False, 'error': 'input backend unavailable'})}]}})
continue
try:
result = server.call(name, args)
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [
{"type": "text", "text": json.dumps(result, ensure_ascii=False)}]}})
except Exception as e:
log('tool err %s' % e)
send({"jsonrpc": "2.0", "id": mid, "result": {"content": [
{"type": "text", "text": json.dumps({'ok': False, 'error': str(e)})}]}, "isError": True})
else:
if mid is not None:
send({"jsonrpc": "2.0", "id": mid,
"error": {"code": -32601, "message": "method not found"}})
log('lc_mcp exit')
if __name__ == '__main__':
main()
+58
View File
@@ -0,0 +1,58 @@
# -*- coding: utf-8 -*-
"""平台抽象:截图 + 输入后端,按平台注册。
core (MCP协议/OCR/语义工具) 平台无关。"""
import os
import sys
def detect_platform():
p = os.environ.get('LCP_PLATFORM', '')
if p:
return p.lower()
if sys.platform.startswith('win'):
return 'win'
if sys.platform == 'darwin':
return 'mac'
return 'linux'
def get_screenshot_provider(platform=None):
platform = platform or detect_platform()
if platform == 'win':
from .win import WinScreenshotProvider
return WinScreenshotProvider()
if platform == 'linux':
from .linux import LinuxScreenshotProvider
return LinuxScreenshotProvider()
if platform == 'mac':
from .mac import MacScreenshotProvider
return MacScreenshotProvider()
raise ValueError('no screenshot provider for %s' % platform)
def get_input_backend(name=None, platform=None):
platform = platform or detect_platform()
name = (name or os.environ.get('INPUT_BACKEND', '') or
{'win': 'sendinput', 'linux': 'xdotool', 'mac': 'cg'}[platform]).lower()
if name == 'mock':
from backends.mock import MockBackend
return MockBackend()
if platform == 'win':
if name == 'sendinput':
from backends.sendinput_win import SendInputBackend
return SendInputBackend()
if name == 'interception':
from backends.interception_win import InterceptionBackend
return InterceptionBackend()
if platform == 'linux':
if name == 'xdotool':
from backends.xdotool_linux import XdotoolBackend
return XdotoolBackend()
if name == 'uinput':
from backends.uinput_linux import UinputBackend
return UinputBackend()
if platform == 'mac' and name == 'cg':
from backends.cg_mac import CGBackend
return CGBackend()
raise ValueError('no input backend %s for %s' % (name, platform))
+34
View File
@@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
"""Linux 截图提供者:X11 (PIL ImageGrab / scrot 兜底)。
Wayland 下 ImageGrab 不可用,需额外方案(见 README)。
"""
import os
import shutil
import subprocess
import tempfile
import time
class LinuxScreenshotProvider:
def grab(self):
# 1) PIL ImageGrab (X11)
try:
from PIL import ImageGrab
img = ImageGrab.grab()
path = os.path.join(tempfile.gettempdir(), 'lc_shot_%s.png' % time.strftime('%H%M%S'))
img.save(path)
return {'ok': True, 'path': path}
except Exception:
pass
# 2) scrot / import (ImageMagick) / gnome-screenshot 兜底
for cmd in (['scrot', '-z'], ['import', '-window', 'root'], ['gnome-screenshot', '-f']):
if shutil.which(cmd[0]):
path = os.path.join(tempfile.gettempdir(), 'lc_shot_%s.png' % time.strftime('%H%M%S'))
try:
subprocess.run(cmd + [path], check=True, timeout=20)
if os.path.exists(path):
return {'ok': True, 'path': path}
except Exception:
continue
return {'ok': False, 'error': 'no working screenshot method on this Linux (need X11 + PIL/scrot)'}
+20
View File
@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
"""macOS 截图提供者:screencapture CLI (系统自带)。"""
import os
import subprocess
import tempfile
import time
class MacScreenshotProvider:
def grab(self):
path = os.path.join(tempfile.gettempdir(), 'lc_shot_%s.png' % time.strftime('%H%M%S'))
try:
subprocess.run(['screencapture', '-x', path], check=True, timeout=20)
if os.path.exists(path):
return {'ok': True, 'path': path}
except Exception as e:
return {'ok': False, 'error': 'screencapture fail: %s' % e}
return {'ok': False, 'error': 'no png produced'}
+76
View File
@@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
"""Windows 截图提供者:交互计划任务(LogonType Interactive)。
OpenSSH 服务会话(session 0)没有交互桌面,截图/前端输入必须在交互会话执行。
"""
import os
import subprocess
import time
USER = os.environ.get('USERNAME', 'yws-hhh')
BASE = os.path.join(os.path.expanduser('~'), 'lc_mcp')
os.makedirs(BASE, exist_ok=True)
OUT_DIR = r'C:\ProgramData\hduty'
os.makedirs(OUT_DIR, exist_ok=True)
JOB_PS1 = os.path.join(BASE, 'job.ps1')
RUN_VBS = os.path.join(BASE, 'run.vbs')
TASK = 'LcMCP'
if not os.path.exists(RUN_VBS):
with open(RUN_VBS, 'w', encoding='utf-8') as f:
f.write('CreateObject("WScript.Shell").Run "powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%s"", 0, False\n' % JOB_PS1)
def _pshell(code):
r = subprocess.run(['powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', code],
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=60)
return r.returncode, (r.stdout or ''), (r.stderr or '')
def _run_interactive(ps1_code, expect_file, timeout=15):
with open(JOB_PS1, 'w', encoding='utf-8') as f:
f.write('$ErrorActionPreference="Stop"\n' + ps1_code)
try:
if os.path.exists(expect_file):
os.remove(expect_file)
except Exception:
pass
reg = ("Unregister-ScheduledTask -TaskName %s -Confirm:$false -ErrorAction SilentlyContinue; "
"$A=New-ScheduledTaskAction -Execute 'wscript.exe' -Argument '%s'; "
"$P=New-ScheduledTaskPrincipal -UserId '%s' -LogonType Interactive -RunLevel Highest; "
"Register-ScheduledTask -TaskName %s -Action $A -Principal $P -Force | Out-Null; "
"Start-ScheduledTask -TaskName %s; 'OK'"
% (TASK, RUN_VBS.replace("'", "''"), USER, TASK, TASK))
rc, out, err = _pshell(reg)
if 'OK' not in out:
return {"ok": False, "error": "task start fail: %s %s" % (out[-150:], err[-150:])}
deadline = time.time() + timeout
while time.time() < deadline:
if os.path.exists(expect_file):
try:
with open(expect_file, 'r', encoding='utf-8') as f:
return {"ok": True, "out": f.read().strip()}
except Exception:
return {"ok": True, "out": ''}
time.sleep(0.4)
return {"ok": False, "error": "timeout waiting %s" % expect_file}
class WinScreenshotProvider:
def grab(self):
name = 'shot_%s.png' % time.strftime('%Y%m%d_%H%M%S')
img_path = os.path.join(OUT_DIR, name)
ps = ("Add-Type -AssemblyName System.Windows.Forms,System.Drawing; "
"$b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; "
"$bmp=New-Object System.Drawing.Bitmap $b.Width,$b.Height; "
"$g=[System.Drawing.Graphics]::FromImage($bmp); "
"$g.CopyFromScreen($b.X,$b.Y,0,0,$b.Size); "
"$bmp.Save('%s'); $g.Dispose(); $bmp.Dispose(); "
"$f=Join-Path '%s' 'done.txt'; 'DONE' | Set-Content $f"
% (img_path, OUT_DIR))
r = _run_interactive(ps, os.path.join(OUT_DIR, 'done.txt'), timeout=20)
if not r.get('ok'):
return {'ok': False, 'error': r.get('error', 'screenshot fail')}
if not os.path.exists(img_path):
return {'ok': False, 'error': 'png not produced'}
return {'ok': True, 'path': img_path.replace('\\', '/')}
+63
View File
@@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
"""后端/协议测试(本机直接跑,无需 Windows)。"""
import json
import os
import subprocess
import sys
HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, HERE)
from platform import detect_platform, get_input_backend # noqa: E402
def test_mock_backend():
b = get_input_backend('mock')
assert b.click(100, 200)['ok']
assert b.move(10, 10)['ok']
assert b.keypress('p')['ok']
assert b.type_text('ok')['ok']
acts = [x['act'] for x in b.log]
assert acts == ['click', 'move', 'keypress', 'type_text'], acts
print('mock backend: OK', acts)
def test_scancode_table():
from backends.interception_win import SCAN, UPPER_SCAN
assert SCAN['a'] == 0x1E and SCAN['p'] == 0x19 and SCAN['z'] == 0x2C
assert SCAN['enter'] == 0x1C and SCAN['esc'] == 0x01
assert UPPER_SCAN['!'] == 0x02
# 连续无缺失
for ch in 'abcdefghijklmnopqrstuvwxyz0123456789':
assert ch in SCAN, 'missing %s' % ch
print('scancode table: OK')
def test_mcp_roundtrip_mock():
env = dict(os.environ, INPUT_BACKEND='mock')
p = subprocess.Popen([sys.executable, os.path.join(HERE, 'lc_mcp.py')],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, encoding='utf-8',
errors='replace', env=env)
def call(obj):
p.stdin.write(json.dumps(obj) + '\n')
p.stdin.flush()
return json.loads(p.stdout.readline())
r = call({'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {}})
assert r['result']['serverInfo']['name'] == 'lc-mcp'
r = call({'jsonrpc': '2.0', 'id': 2, 'method': 'tools/list', 'params': {}})
names = [t['name'] for t in r['result']['tools']]
assert 'get_state' in names and 'click_text' in names and 'wait_for_text' in names and 'wait_for_text_gone' in names, names
assert 'screenshot' in names and 'keypress' in names, names
r = call({'jsonrpc': '2.0', 'id': 3, 'method': 'tools/call',
'params': {'name': 'move', 'arguments': {'x': 5, 'y': 5}}})
assert 'mock' in r['result']['content'][0]['text']
p.kill()
print('mcp roundtrip(mock): OK', names)
if __name__ == '__main__':
test_mock_backend()
test_scancode_table()
test_mcp_roundtrip_mock()
print('ALL TESTS PASSED')