backend: 接入 SQLite 数据库 + 开单后台按功能重写

Browse Source
- store.py(新):卡密 / 订单信息 / 爱发电账本 / 呼叫记录 / 小块状态进 <DATA_DIR>/ys.db;
  第一次启动自动建库并导入旧 JSON(旧文件不动),admin.py db check/backup/export
- server.py / admin.py 存取换到库上,路由与接口字段不变;calls.jsonl 照旧追加
- 管理接口新增 calls / call-done / retry,codes 支持 q 搜索
- ys_admin.py 页面重写:待处理 / 订单 / 卡密 / 福利券 / 呼叫·反馈 / 开单工具
- tests/数据库测试.py(37 条,假数据);回归 27 条照旧全过
- 交接说明(数据库一节、面板一节)、回执第二轮、交付清单更新

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yws
2026-10-05 16:20:45 +00:00
co-authored by Claude Opus 5.5
parent 651d21d882
commit 51212a07c0
11 changed files with 1743 additions and 798 deletions
+421
View File
@@ -0,0 +1,421 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""金钻工坊 · 数据层(SQLite,标准库自带,零依赖)
2026-10-05 起,原来散在 <DATA_DIR> 下的这些「拿 JSON 文件当数据库」的东西,统一进一个库:
<DATA_DIR>/ys.db
| 表 | 原来在哪 | 里面是什么 |
|-------------|-----------------------------------|------------|
| codes | codes.json | 卡密池(一行一张码) |
| orders | orders/<单号>/meta.json | 订单信息(一行一单;皮肤 / 成品**文件**还在 orders/<单号>/ 目录里) |
| pay_orders | payplatform-orders.json | 爱发电回调账本(一行一笔,按平台单号幂等) |
| calls | calls.jsonl | 「呼叫金钻」/ 反馈 / 部分失败 记录 |
| kv | gifts.json / heartbeat.json / home_ips.json / tpl_times.json | 小块状态(一个键一份 JSON) |
**还是文件、没进库的**(都是人手改的配置或二进制):
payplatform.json(凭据)、payplatform-map.json、prices.json、payinfo.json、
worker_token / admin_token、皮肤与成品图片、feedback/ 里的反馈图、各种 .log
设计取舍(为什么这么做):
* 每张表都是「几个能查的列 + 一列 data(整条记录的 JSON)」。上层代码拿到的还是原来的 dict,
**字段想加就加,不用改表**;能查的列(state / order_id / batch …)是写入时从 data 里抽出来的。
* 写整份 dict 的老写法(load → 改 → save)保留:save 时在**一个事务里**只改有变化的行。
所以 server.py 里的路由逻辑一行都不用动,只是存取换了地方。
* 并发:上层照旧 `with LOCK:`(线程锁 + 跨进程 flock);库本身开 WAL + busy_timeout,
读不挡写、写不挡读,另一个进程(admin.py)在写也只是等一会儿,不会报 locked。
* 断电:SQLite 事务要么全写进去、要么全没写,不会再有「半截 codes.json」。
第一次启动自动从旧 JSON 导入(只导一次,旧文件原样留着不删);要回退到旧版:
python3 store.py export 把库里的数据导回成旧版的 JSON 文件
python3 store.py check 完整性检查 + 每张表几行
python3 store.py backup 在线热备份一份到 <DATA_DIR>/backup/ys-<时间>.db
"""
import json, os, sqlite3, sys, threading, time
SCHEMA_VERSION = 1
# 哪些 kv 键原来是哪个文件(导入 / 导出用)
KV_FILES = {
"gifts": "gifts.json",
"heartbeat": "heartbeat.json",
"home_ips": "home_ips.json",
"tpl_times": "tpl_times.json",
}
_SCHEMA = """
CREATE TABLE IF NOT EXISTS codes (
code TEXT PRIMARY KEY,
state TEXT NOT NULL DEFAULT 'active',
order_id TEXT,
batch TEXT,
platform TEXT,
gift TEXT,
data TEXT NOT NULL,
updated REAL NOT NULL
);
CREATE INDEX IF NOT EXISTS codes_state ON codes(state);
CREATE INDEX IF NOT EXISTS codes_order ON codes(order_id);
CREATE INDEX IF NOT EXISTS codes_batch ON codes(batch);
CREATE INDEX IF NOT EXISTS codes_gift ON codes(gift);
CREATE TABLE IF NOT EXISTS orders (
oid TEXT PRIMARY KEY,
state TEXT,
created TEXT,
data TEXT NOT NULL,
updated REAL NOT NULL,
purged REAL -- 文件已清理的时间;NULL = 还在。清掉的单留一行做账,不再当成「存在」
);
CREATE INDEX IF NOT EXISTS orders_state ON orders(state) WHERE purged IS NULL;
CREATE TABLE IF NOT EXISTS pay_orders (
out_trade_no TEXT PRIMARY KEY,
data TEXT NOT NULL,
updated REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS calls (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts TEXT,
ip TEXT,
order_id TEXT,
data TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS calls_ip ON calls(ip, ts);
CREATE TABLE IF NOT EXISTS kv (
k TEXT PRIMARY KEY,
data TEXT NOT NULL,
updated REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS schema_info (k TEXT PRIMARY KEY, v TEXT);
"""
def _dump(obj):
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def _code_cols(code, rec):
st = rec.get("state")
if st not in ("active", "inactive", "used", "bound"):
st = "used" if rec.get("used") else "active"
return (code, st, rec.get("order") or None, rec.get("batch") or None,
rec.get("platform") or None, rec.get("gift") or None)
class Store(object):
"""一个库文件一个实例。每个线程自己一条连接(sqlite3 连接不能跨线程用)。"""
def __init__(self, root):
self.root = root
self.path = os.path.join(root, "ys.db")
self._local = threading.local()
os.makedirs(root, exist_ok=True)
fresh = not os.path.exists(self.path)
c = self.conn()
c.executescript(_SCHEMA)
if fresh:
try:
os.chmod(self.path, 0o640) # 和 codes.json 一样:别的账号读不到
except OSError:
pass
if self._info("schema") is None:
with self.tx() as c:
c.execute("INSERT OR REPLACE INTO schema_info VALUES ('schema', ?)", (str(SCHEMA_VERSION),))
if self._info("imported") is None:
self.import_legacy()
# ------------------------------------------------------------ 连接 / 事务
def conn(self):
c = getattr(self._local, "c", None)
if c is None:
c = sqlite3.connect(self.path, timeout=30, isolation_level=None, check_same_thread=False)
c.execute("PRAGMA journal_mode=WAL")
c.execute("PRAGMA synchronous=NORMAL") # WAL 下 NORMAL:断电最多丢最后一个事务,不会写坏库
c.execute("PRAGMA busy_timeout=30000")
c.execute("PRAGMA foreign_keys=ON")
self._local.c = c
return c
class _Tx(object):
def __init__(self, c):
self.c = c
def __enter__(self):
self.c.execute("BEGIN IMMEDIATE")
return self.c
def __exit__(self, et, ev, tb):
self.c.execute("COMMIT" if et is None else "ROLLBACK")
return False
def tx(self):
"""写事务:`with store.tx() as c:` 里的语句要么全成、要么全不成。"""
return Store._Tx(self.conn())
def _info(self, k):
r = self.conn().execute("SELECT v FROM schema_info WHERE k=?", (k,)).fetchone()
return r[0] if r else None
# ------------------------------------------------------------ 卡密
def codes_all(self):
return {k: json.loads(d) for k, d in self.conn().execute("SELECT code, data FROM codes")}
def code_get(self, code):
r = self.conn().execute("SELECT data FROM codes WHERE code=?", (code,)).fetchone()
return json.loads(r[0]) if r else None
def codes_where(self, **cols):
"""按列查:codes_where(order_id='YS…') / codes_where(gift='g1') / codes_where(state='active')"""
ok = ("state", "order_id", "batch", "platform", "gift")
ks = [k for k in cols if k in ok]
sql = "SELECT code, data FROM codes" + (" WHERE " + " AND ".join("%s=?" % k for k in ks) if ks else "")
return {k: json.loads(d) for k, d in self.conn().execute(sql, [cols[k] for k in ks])}
def codes_save(self, codes):
"""整份写回(老写法 load → 改 → save 用)。一个事务里:变了的更新、没了的删掉、新的插入。"""
now = time.time()
with self.tx() as c:
old = dict(c.execute("SELECT code, data FROM codes"))
for k in set(old) - set(codes):
c.execute("DELETE FROM codes WHERE code=?", (k,))
for k, rec in codes.items():
d = _dump(rec)
if old.get(k) == d:
continue
c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (d, now))
def code_put(self, code, rec):
with self.tx() as c:
c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(code, rec) + (_dump(rec), time.time()))
def code_counts(self):
return dict(self.conn().execute("SELECT state, COUNT(*) FROM codes GROUP BY state"))
# ------------------------------------------------------------ 订单
def order_get(self, oid):
r = self.conn().execute("SELECT data FROM orders WHERE oid=? AND purged IS NULL", (oid,)).fetchone()
return json.loads(r[0]) if r else None
def order_put(self, oid, meta):
with self.tx() as c:
c.execute("INSERT INTO orders (oid, state, created, data, updated, purged) VALUES (?,?,?,?,?,NULL)"
" ON CONFLICT(oid) DO UPDATE SET state=excluded.state, created=excluded.created,"
" data=excluded.data, updated=excluded.updated, purged=NULL",
(oid, meta.get("state"), meta.get("created"), _dump(meta), time.time()))
def order_known(self, oid):
"""这个单号用过没有(包括已清理的)—— 发新单号时防撞号。"""
return self.conn().execute("SELECT 1 FROM orders WHERE oid=?", (oid,)).fetchone() is not None
def orders(self, states=None, include_purged=False):
sql, args = "SELECT data FROM orders", []
cond = [] if include_purged else ["purged IS NULL"]
if states is not None:
states = list(states)
if not states:
return []
cond.append("state IN (%s)" % ",".join("?" * len(states)))
args += states
if cond:
sql += " WHERE " + " AND ".join(cond)
sql += " ORDER BY oid"
return [json.loads(d) for (d,) in self.conn().execute(sql, args)]
def order_ids(self):
return [r[0] for r in self.conn().execute("SELECT oid FROM orders WHERE purged IS NULL ORDER BY oid")]
def order_purge(self, oid):
"""文件清掉了:行留着(记账 / 统计用),但之后 order_get 当它不存在。"""
with self.tx() as c:
c.execute("UPDATE orders SET purged=? WHERE oid=?", (time.time(), oid))
def order_drop(self, oid):
"""单根本没建成(提交中途失败):连行一起删。"""
with self.tx() as c:
c.execute("DELETE FROM orders WHERE oid=?", (oid,))
def order_counts(self):
return dict(self.conn().execute("SELECT state, COUNT(*) FROM orders WHERE purged IS NULL GROUP BY state"))
# ------------------------------------------------------------ 爱发电账本
def ledger_all(self):
return {k: json.loads(d) for k, d in self.conn().execute("SELECT out_trade_no, data FROM pay_orders")}
def ledger_get(self, no):
r = self.conn().execute("SELECT data FROM pay_orders WHERE out_trade_no=?", (str(no),)).fetchone()
return json.loads(r[0]) if r else None
def ledger_save(self, led):
now = time.time()
with self.tx() as c:
old = dict(c.execute("SELECT out_trade_no, data FROM pay_orders"))
for k in set(old) - set(led):
c.execute("DELETE FROM pay_orders WHERE out_trade_no=?", (k,))
for k, rec in led.items():
d = _dump(rec)
if old.get(k) != d:
c.execute("INSERT OR REPLACE INTO pay_orders VALUES (?,?,?)", (str(k), d, now))
# ------------------------------------------------------------ 呼叫记录
def call_add(self, rec):
with self.tx() as c:
cur = c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)",
(rec.get("ts"), rec.get("ip"), rec.get("order") or None, _dump(rec)))
return cur.lastrowid
def calls(self, limit=None, ip=None, since=None):
"""最近的在后。ip / since(ISO 时间串)可以筛。"""
sql, cond, args = "SELECT id, data FROM calls", [], []
if ip is not None:
cond.append("ip=?"); args.append(ip)
if since is not None:
cond.append("ts>=?"); args.append(since)
if cond:
sql += " WHERE " + " AND ".join(cond)
sql += " ORDER BY id DESC"
if limit:
sql += " LIMIT %d" % int(limit)
out = []
for i, d in self.conn().execute(sql, args):
r = json.loads(d); r["id"] = i; out.append(r)
return out[::-1]
def call_mark(self, cid, **fields):
with self.tx() as c:
r = c.execute("SELECT data FROM calls WHERE id=?", (cid,)).fetchone()
if not r:
return False
d = json.loads(r[0]); d.update(fields)
c.execute("UPDATE calls SET data=? WHERE id=?", (_dump(d), cid))
return True
# ------------------------------------------------------------ 小块状态
def kv_get(self, k, default=None):
r = self.conn().execute("SELECT data FROM kv WHERE k=?", (k,)).fetchone()
return json.loads(r[0]) if r else default
def kv_put(self, k, obj):
with self.tx() as c:
c.execute("INSERT OR REPLACE INTO kv VALUES (?,?,?)", (k, _dump(obj), time.time()))
# ------------------------------------------------------------ 导入 / 导出 / 体检
def import_legacy(self):
"""第一次启动:把旧 JSON 导进来(旧文件不动)。整个导入一个事务,失败就整体回滚、下次再试。"""
R = self.root
n = {"codes": 0, "orders": 0, "pay_orders": 0, "calls": 0, "kv": 0}
def jl(p, default):
try:
with open(p, encoding="utf-8") as f:
return json.load(f)
except (OSError, ValueError):
return default
now = time.time()
with self.tx() as c:
for k, rec in (jl(os.path.join(R, "codes.json"), {}) or {}).items():
if isinstance(rec, dict):
c.execute("INSERT OR IGNORE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (_dump(rec), now))
n["codes"] += 1
od = os.path.join(R, "orders")
if os.path.isdir(od):
for oid in sorted(os.listdir(od)):
m = jl(os.path.join(od, oid, "meta.json"), None)
if isinstance(m, dict):
c.execute("INSERT OR IGNORE INTO orders (oid, state, created, data, updated) VALUES (?,?,?,?,?)",
(oid, m.get("state"), m.get("created"), _dump(m), now))
n["orders"] += 1
for k, rec in (jl(os.path.join(R, "payplatform-orders.json"), {}) or {}).items():
c.execute("INSERT OR IGNORE INTO pay_orders VALUES (?,?,?)", (str(k), _dump(rec), now))
n["pay_orders"] += 1
try:
with open(os.path.join(R, "calls.jsonl"), encoding="utf-8") as f:
for line in f:
try:
r = json.loads(line)
except ValueError:
continue # 旧文件里写了半行的那条跳过
if isinstance(r, dict):
c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)",
(r.get("ts"), r.get("ip"), r.get("order") or None, _dump(r)))
n["calls"] += 1
except OSError:
pass
for k, fn in KV_FILES.items():
v = jl(os.path.join(R, fn), None)
if v is not None:
c.execute("INSERT OR IGNORE INTO kv VALUES (?,?,?)", (k, _dump(v), now))
n["kv"] += 1
c.execute("INSERT OR REPLACE INTO schema_info VALUES ('imported', ?)",
(json.dumps({"at": time.strftime("%Y-%m-%d %H:%M:%S"), "n": n}, ensure_ascii=False),))
return n
def export_legacy(self, dest=None):
"""把库导回旧版 JSON(回退用)。dest 不给 = 写回 <DATA_DIR>(会覆盖那几个旧文件)。"""
D = dest or self.root
os.makedirs(os.path.join(D, "orders"), exist_ok=True)
def js(p, obj):
tmp = p + ".export.tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(obj, f, ensure_ascii=False, indent=1)
os.replace(tmp, p)
js(os.path.join(D, "codes.json"), self.codes_all())
js(os.path.join(D, "payplatform-orders.json"), self.ledger_all())
for m in self.orders():
od = os.path.join(D, "orders", m["oid"])
os.makedirs(od, exist_ok=True)
js(os.path.join(od, "meta.json"), m)
with open(os.path.join(D, "calls.jsonl"), "w", encoding="utf-8") as f:
for r in self.calls():
r.pop("id", None)
f.write(json.dumps(r, ensure_ascii=False) + "\n")
for k, fn in KV_FILES.items():
v = self.kv_get(k)
if v is not None:
js(os.path.join(D, fn), v)
return D
def check(self):
c = self.conn()
ok = c.execute("PRAGMA integrity_check").fetchone()[0]
n = {t: c.execute("SELECT COUNT(*) FROM %s" % t).fetchone()[0]
for t in ("codes", "orders", "pay_orders", "calls", "kv")}
return ok, n, self._info("imported")
def backup(self, dest=None):
"""在线热备份(服务器不用停)。"""
d = dest or os.path.join(self.root, "backup")
os.makedirs(d, exist_ok=True)
p = os.path.join(d, "ys-%s.db" % time.strftime("%Y%m%d-%H%M%S"))
b = sqlite3.connect(p)
with b:
self.conn().backup(b)
b.close()
os.chmod(p, 0o600)
return p
if __name__ == "__main__":
root = os.environ.get("YS_ROOT", "<DATA_DIR>")
cmd = (sys.argv[1:] or ["check"])[0]
st = Store(root)
if cmd == "check":
ok, n, imp = st.check()
print("完整性:%s" % ok)
for k, v in n.items():
print(" %-10s %d 行" % (k, v))
print("旧数据导入:%s" % (imp or "(没导过)"))
elif cmd == "export":
print("已导出到 %s" % st.export_legacy(sys.argv[2] if len(sys.argv) > 2 else None))
elif cmd == "backup":
print("备份好了:%s" % st.backup())
else:
print(__doc__)
sys.exit(2)