#!/usr/bin/env python3 # -*- coding: utf-8 -*- """金钻工坊 · 数据层(SQLite,标准库自带,零依赖) 2026-10-05 起,原来散在 下的这些「拿 JSON 文件当数据库」的东西,统一进一个库: /ys.db | 表 | 原来在哪 | 里面是什么 | |-------------|-----------------------------------|------------| | codes | codes.json | 卡密池(一行一张码) | | orders | orders/<单号>/meta.json | 订单信息(一行一单;皮肤 / 成品**文件**还在 orders/<单号>/ 目录里) | | pay_orders | payplatform-orders.json | 爱发电回调账本(一行一笔,按平台单号幂等) | | calls | calls.jsonl | 「呼叫金钻」/ 反馈 / 部分失败 记录 | | kv | gifts.json / heartbeat.json / home_ips.json / tpl_times.json | 小块状态(一个键一份 JSON) | **还是文件、没进库的**(都是人手改的配置或二进制): payplatform.json(凭据)、payplatform-map.json、prices.json、payinfo.json、 worker_token / admin_token、皮肤与成品图片、feedback/ 里的反馈图、各种 .log 设计取舍(为什么这么做): * 每张表都是「几个能查的列 + 一列 data(整条记录的 JSON)」。上层代码拿到的还是原来的 dict, **字段想加就加,不用改表**;能查的列(state / order_id / batch …)是写入时从 data 里抽出来的。 * 写整份 dict 的老写法(load → 改 → save)保留:save 时在**一个事务里**只改有变化的行。 所以 server.py 里的路由逻辑一行都不用动,只是存取换了地方。 * 并发:上层照旧 `with LOCK:`(线程锁 + 跨进程 flock);库本身开 WAL + busy_timeout, 读不挡写、写不挡读,另一个进程(admin.py)在写也只是等一会儿,不会报 locked。 * 断电:SQLite 事务要么全写进去、要么全没写,不会再有「半截 codes.json」。 第一次启动自动从旧 JSON 导入(只导一次,旧文件原样留着不删);要回退到旧版: python3 store.py export 把库里的数据导回成旧版的 JSON 文件 python3 store.py check 完整性检查 + 每张表几行 python3 store.py backup 在线热备份一份到 /backup/ys-<时间>.db """ import json, os, sqlite3, sys, threading, time SCHEMA_VERSION = 1 # 哪些 kv 键原来是哪个文件(导入 / 导出用) KV_FILES = { "gifts": "gifts.json", "heartbeat": "heartbeat.json", "home_ips": "home_ips.json", "tpl_times": "tpl_times.json", } _SCHEMA = """ CREATE TABLE IF NOT EXISTS codes ( code TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active', order_id TEXT, batch TEXT, platform TEXT, gift TEXT, data TEXT NOT NULL, updated REAL NOT NULL ); CREATE INDEX IF NOT EXISTS codes_state ON codes(state); CREATE INDEX IF NOT EXISTS codes_order ON codes(order_id); CREATE INDEX IF NOT EXISTS codes_batch ON codes(batch); CREATE INDEX IF NOT EXISTS codes_gift ON codes(gift); CREATE TABLE IF NOT EXISTS orders ( oid TEXT PRIMARY KEY, state TEXT, created TEXT, data TEXT NOT NULL, updated REAL NOT NULL, purged REAL -- 文件已清理的时间;NULL = 还在。清掉的单留一行做账,不再当成「存在」 ); CREATE INDEX IF NOT EXISTS orders_state ON orders(state) WHERE purged IS NULL; CREATE TABLE IF NOT EXISTS pay_orders ( out_trade_no TEXT PRIMARY KEY, data TEXT NOT NULL, updated REAL NOT NULL ); CREATE TABLE IF NOT EXISTS calls ( id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT, ip TEXT, order_id TEXT, data TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS calls_ip ON calls(ip, ts); CREATE TABLE IF NOT EXISTS kv ( k TEXT PRIMARY KEY, data TEXT NOT NULL, updated REAL NOT NULL ); CREATE TABLE IF NOT EXISTS schema_info (k TEXT PRIMARY KEY, v TEXT); """ def _dump(obj): return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":")) def _code_cols(code, rec): st = rec.get("state") if st not in ("active", "inactive", "used", "bound"): st = "used" if rec.get("used") else "active" return (code, st, rec.get("order") or None, rec.get("batch") or None, rec.get("platform") or None, rec.get("gift") or None) class Store(object): """一个库文件一个实例。每个线程自己一条连接(sqlite3 连接不能跨线程用)。""" def __init__(self, root): self.root = root self.path = os.path.join(root, "ys.db") self._local = threading.local() os.makedirs(root, exist_ok=True) fresh = not os.path.exists(self.path) c = self.conn() c.executescript(_SCHEMA) if fresh: try: os.chmod(self.path, 0o640) # 和 codes.json 一样:别的账号读不到 except OSError: pass if self._info("schema") is None: with self.tx() as c: c.execute("INSERT OR REPLACE INTO schema_info VALUES ('schema', ?)", (str(SCHEMA_VERSION),)) if self._info("imported") is None: self.import_legacy() # ------------------------------------------------------------ 连接 / 事务 def conn(self): c = getattr(self._local, "c", None) if c is None: c = sqlite3.connect(self.path, timeout=30, isolation_level=None, check_same_thread=False) c.execute("PRAGMA journal_mode=WAL") c.execute("PRAGMA synchronous=NORMAL") # WAL 下 NORMAL:断电最多丢最后一个事务,不会写坏库 c.execute("PRAGMA busy_timeout=30000") c.execute("PRAGMA foreign_keys=ON") self._local.c = c return c class _Tx(object): def __init__(self, c): self.c = c def __enter__(self): self.c.execute("BEGIN IMMEDIATE") return self.c def __exit__(self, et, ev, tb): self.c.execute("COMMIT" if et is None else "ROLLBACK") return False def tx(self): """写事务:`with store.tx() as c:` 里的语句要么全成、要么全不成。""" return Store._Tx(self.conn()) def _info(self, k): r = self.conn().execute("SELECT v FROM schema_info WHERE k=?", (k,)).fetchone() return r[0] if r else None # ------------------------------------------------------------ 卡密 def codes_all(self): return {k: json.loads(d) for k, d in self.conn().execute("SELECT code, data FROM codes")} def code_get(self, code): r = self.conn().execute("SELECT data FROM codes WHERE code=?", (code,)).fetchone() return json.loads(r[0]) if r else None def codes_where(self, **cols): """按列查:codes_where(order_id='YS…') / codes_where(gift='g1') / codes_where(state='active')""" ok = ("state", "order_id", "batch", "platform", "gift") ks = [k for k in cols if k in ok] sql = "SELECT code, data FROM codes" + (" WHERE " + " AND ".join("%s=?" % k for k in ks) if ks else "") return {k: json.loads(d) for k, d in self.conn().execute(sql, [cols[k] for k in ks])} def codes_save(self, codes): """整份写回(老写法 load → 改 → save 用)。一个事务里:变了的更新、没了的删掉、新的插入。""" now = time.time() with self.tx() as c: old = dict(c.execute("SELECT code, data FROM codes")) for k in set(old) - set(codes): c.execute("DELETE FROM codes WHERE code=?", (k,)) for k, rec in codes.items(): d = _dump(rec) if old.get(k) == d: continue c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)" " VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (d, now)) def code_put(self, code, rec): with self.tx() as c: c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)" " VALUES (?,?,?,?,?,?,?,?)", _code_cols(code, rec) + (_dump(rec), time.time())) def code_counts(self): return dict(self.conn().execute("SELECT state, COUNT(*) FROM codes GROUP BY state")) # ------------------------------------------------------------ 订单 def order_get(self, oid): r = self.conn().execute("SELECT data FROM orders WHERE oid=? AND purged IS NULL", (oid,)).fetchone() return json.loads(r[0]) if r else None def order_put(self, oid, meta): with self.tx() as c: c.execute("INSERT INTO orders (oid, state, created, data, updated, purged) VALUES (?,?,?,?,?,NULL)" " ON CONFLICT(oid) DO UPDATE SET state=excluded.state, created=excluded.created," " data=excluded.data, updated=excluded.updated, purged=NULL", (oid, meta.get("state"), meta.get("created"), _dump(meta), time.time())) def order_known(self, oid): """这个单号用过没有(包括已清理的)—— 发新单号时防撞号。""" return self.conn().execute("SELECT 1 FROM orders WHERE oid=?", (oid,)).fetchone() is not None def orders(self, states=None, include_purged=False): sql, args = "SELECT data FROM orders", [] cond = [] if include_purged else ["purged IS NULL"] if states is not None: states = list(states) if not states: return [] cond.append("state IN (%s)" % ",".join("?" * len(states))) args += states if cond: sql += " WHERE " + " AND ".join(cond) sql += " ORDER BY oid" return [json.loads(d) for (d,) in self.conn().execute(sql, args)] def order_ids(self): return [r[0] for r in self.conn().execute("SELECT oid FROM orders WHERE purged IS NULL ORDER BY oid")] def order_purge(self, oid): """文件清掉了:行留着(记账 / 统计用),但之后 order_get 当它不存在。""" with self.tx() as c: c.execute("UPDATE orders SET purged=? WHERE oid=?", (time.time(), oid)) def order_drop(self, oid): """单根本没建成(提交中途失败):连行一起删。""" with self.tx() as c: c.execute("DELETE FROM orders WHERE oid=?", (oid,)) def order_counts(self): return dict(self.conn().execute("SELECT state, COUNT(*) FROM orders WHERE purged IS NULL GROUP BY state")) # ------------------------------------------------------------ 爱发电账本 def ledger_all(self): return {k: json.loads(d) for k, d in self.conn().execute("SELECT out_trade_no, data FROM pay_orders")} def ledger_get(self, no): r = self.conn().execute("SELECT data FROM pay_orders WHERE out_trade_no=?", (str(no),)).fetchone() return json.loads(r[0]) if r else None def ledger_save(self, led): now = time.time() with self.tx() as c: old = dict(c.execute("SELECT out_trade_no, data FROM pay_orders")) for k in set(old) - set(led): c.execute("DELETE FROM pay_orders WHERE out_trade_no=?", (k,)) for k, rec in led.items(): d = _dump(rec) if old.get(k) != d: c.execute("INSERT OR REPLACE INTO pay_orders VALUES (?,?,?)", (str(k), d, now)) # ------------------------------------------------------------ 呼叫记录 def call_add(self, rec): with self.tx() as c: cur = c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)", (rec.get("ts"), rec.get("ip"), rec.get("order") or None, _dump(rec))) return cur.lastrowid def calls(self, limit=None, ip=None, since=None): """最近的在后。ip / since(ISO 时间串)可以筛。""" sql, cond, args = "SELECT id, data FROM calls", [], [] if ip is not None: cond.append("ip=?"); args.append(ip) if since is not None: cond.append("ts>=?"); args.append(since) if cond: sql += " WHERE " + " AND ".join(cond) sql += " ORDER BY id DESC" if limit: sql += " LIMIT %d" % int(limit) out = [] for i, d in self.conn().execute(sql, args): r = json.loads(d); r["id"] = i; out.append(r) return out[::-1] def call_mark(self, cid, **fields): with self.tx() as c: r = c.execute("SELECT data FROM calls WHERE id=?", (cid,)).fetchone() if not r: return False d = json.loads(r[0]); d.update(fields) c.execute("UPDATE calls SET data=? WHERE id=?", (_dump(d), cid)) return True # ------------------------------------------------------------ 小块状态 def kv_get(self, k, default=None): r = self.conn().execute("SELECT data FROM kv WHERE k=?", (k,)).fetchone() return json.loads(r[0]) if r else default def kv_put(self, k, obj): with self.tx() as c: c.execute("INSERT OR REPLACE INTO kv VALUES (?,?,?)", (k, _dump(obj), time.time())) # ------------------------------------------------------------ 导入 / 导出 / 体检 def import_legacy(self): """第一次启动:把旧 JSON 导进来(旧文件不动)。整个导入一个事务,失败就整体回滚、下次再试。""" R = self.root n = {"codes": 0, "orders": 0, "pay_orders": 0, "calls": 0, "kv": 0} def jl(p, default): try: with open(p, encoding="utf-8") as f: return json.load(f) except (OSError, ValueError): return default now = time.time() with self.tx() as c: for k, rec in (jl(os.path.join(R, "codes.json"), {}) or {}).items(): if isinstance(rec, dict): c.execute("INSERT OR IGNORE INTO codes (code, state, order_id, batch, platform, gift, data, updated)" " VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (_dump(rec), now)) n["codes"] += 1 od = os.path.join(R, "orders") if os.path.isdir(od): for oid in sorted(os.listdir(od)): m = jl(os.path.join(od, oid, "meta.json"), None) if isinstance(m, dict): c.execute("INSERT OR IGNORE INTO orders (oid, state, created, data, updated) VALUES (?,?,?,?,?)", (oid, m.get("state"), m.get("created"), _dump(m), now)) n["orders"] += 1 for k, rec in (jl(os.path.join(R, "payplatform-orders.json"), {}) or {}).items(): c.execute("INSERT OR IGNORE INTO pay_orders VALUES (?,?,?)", (str(k), _dump(rec), now)) n["pay_orders"] += 1 try: with open(os.path.join(R, "calls.jsonl"), encoding="utf-8") as f: for line in f: try: r = json.loads(line) except ValueError: continue # 旧文件里写了半行的那条跳过 if isinstance(r, dict): c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)", (r.get("ts"), r.get("ip"), r.get("order") or None, _dump(r))) n["calls"] += 1 except OSError: pass for k, fn in KV_FILES.items(): v = jl(os.path.join(R, fn), None) if v is not None: c.execute("INSERT OR IGNORE INTO kv VALUES (?,?,?)", (k, _dump(v), now)) n["kv"] += 1 c.execute("INSERT OR REPLACE INTO schema_info VALUES ('imported', ?)", (json.dumps({"at": time.strftime("%Y-%m-%d %H:%M:%S"), "n": n}, ensure_ascii=False),)) return n def export_legacy(self, dest=None): """把库导回旧版 JSON(回退用)。dest 不给 = 写回 (会覆盖那几个旧文件)。""" D = dest or self.root os.makedirs(os.path.join(D, "orders"), exist_ok=True) def js(p, obj): tmp = p + ".export.tmp" with open(tmp, "w", encoding="utf-8") as f: json.dump(obj, f, ensure_ascii=False, indent=1) os.replace(tmp, p) js(os.path.join(D, "codes.json"), self.codes_all()) js(os.path.join(D, "payplatform-orders.json"), self.ledger_all()) for m in self.orders(): od = os.path.join(D, "orders", m["oid"]) os.makedirs(od, exist_ok=True) js(os.path.join(od, "meta.json"), m) with open(os.path.join(D, "calls.jsonl"), "w", encoding="utf-8") as f: for r in self.calls(): r.pop("id", None) f.write(json.dumps(r, ensure_ascii=False) + "\n") for k, fn in KV_FILES.items(): v = self.kv_get(k) if v is not None: js(os.path.join(D, fn), v) return D def check(self): c = self.conn() ok = c.execute("PRAGMA integrity_check").fetchone()[0] n = {t: c.execute("SELECT COUNT(*) FROM %s" % t).fetchone()[0] for t in ("codes", "orders", "pay_orders", "calls", "kv")} return ok, n, self._info("imported") def backup(self, dest=None): """在线热备份(服务器不用停)。""" d = dest or os.path.join(self.root, "backup") os.makedirs(d, exist_ok=True) p = os.path.join(d, "ys-%s.db" % time.strftime("%Y%m%d-%H%M%S")) b = sqlite3.connect(p) with b: self.conn().backup(b) b.close() os.chmod(p, 0o600) return p if __name__ == "__main__": root = os.environ.get("YS_ROOT", "") cmd = (sys.argv[1:] or ["check"])[0] st = Store(root) if cmd == "check": ok, n, imp = st.check() print("完整性:%s" % ok) for k, v in n.items(): print(" %-10s %d 行" % (k, v)) print("旧数据导入:%s" % (imp or "(没导过)")) elif cmd == "export": print("已导出到 %s" % st.export_legacy(sys.argv[2] if len(sys.argv) > 2 else None)) elif cmd == "backup": print("备份好了:%s" % st.backup()) else: print(__doc__) sys.exit(2)