- store.py(新):卡密 / 订单信息 / 爱发电账本 / 呼叫记录 / 小块状态进 <DATA_DIR>/ys.db; 第一次启动自动建库并导入旧 JSON(旧文件不动),admin.py db check/backup/export - server.py / admin.py 存取换到库上,路由与接口字段不变;calls.jsonl 照旧追加 - 管理接口新增 calls / call-done / retry,codes 支持 q 搜索 - ys_admin.py 页面重写:待处理 / 订单 / 卡密 / 福利券 / 呼叫·反馈 / 开单工具 - tests/数据库测试.py(37 条,假数据);回归 27 条照旧全过 - 交接说明(数据库一节、面板一节)、回执第二轮、交付清单更新 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
422 lines
18 KiB
Python
422 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""金钻工坊 · 数据层(SQLite,标准库自带,零依赖)
|
||
|
||
2026-10-05 起,原来散在 <DATA_DIR> 下的这些「拿 JSON 文件当数据库」的东西,统一进一个库:
|
||
<DATA_DIR>/ys.db
|
||
|
||
| 表 | 原来在哪 | 里面是什么 |
|
||
|-------------|-----------------------------------|------------|
|
||
| codes | codes.json | 卡密池(一行一张码) |
|
||
| orders | orders/<单号>/meta.json | 订单信息(一行一单;皮肤 / 成品**文件**还在 orders/<单号>/ 目录里) |
|
||
| pay_orders | payplatform-orders.json | 爱发电回调账本(一行一笔,按平台单号幂等) |
|
||
| calls | calls.jsonl | 「呼叫金钻」/ 反馈 / 部分失败 记录 |
|
||
| kv | gifts.json / heartbeat.json / home_ips.json / tpl_times.json | 小块状态(一个键一份 JSON) |
|
||
|
||
**还是文件、没进库的**(都是人手改的配置或二进制):
|
||
payplatform.json(凭据)、payplatform-map.json、prices.json、payinfo.json、
|
||
worker_token / admin_token、皮肤与成品图片、feedback/ 里的反馈图、各种 .log
|
||
|
||
设计取舍(为什么这么做):
|
||
* 每张表都是「几个能查的列 + 一列 data(整条记录的 JSON)」。上层代码拿到的还是原来的 dict,
|
||
**字段想加就加,不用改表**;能查的列(state / order_id / batch …)是写入时从 data 里抽出来的。
|
||
* 写整份 dict 的老写法(load → 改 → save)保留:save 时在**一个事务里**只改有变化的行。
|
||
所以 server.py 里的路由逻辑一行都不用动,只是存取换了地方。
|
||
* 并发:上层照旧 `with LOCK:`(线程锁 + 跨进程 flock);库本身开 WAL + busy_timeout,
|
||
读不挡写、写不挡读,另一个进程(admin.py)在写也只是等一会儿,不会报 locked。
|
||
* 断电:SQLite 事务要么全写进去、要么全没写,不会再有「半截 codes.json」。
|
||
|
||
第一次启动自动从旧 JSON 导入(只导一次,旧文件原样留着不删);要回退到旧版:
|
||
python3 store.py export 把库里的数据导回成旧版的 JSON 文件
|
||
python3 store.py check 完整性检查 + 每张表几行
|
||
python3 store.py backup 在线热备份一份到 <DATA_DIR>/backup/ys-<时间>.db
|
||
"""
|
||
import json, os, sqlite3, sys, threading, time
|
||
|
||
SCHEMA_VERSION = 1
|
||
|
||
# 哪些 kv 键原来是哪个文件(导入 / 导出用)
|
||
KV_FILES = {
|
||
"gifts": "gifts.json",
|
||
"heartbeat": "heartbeat.json",
|
||
"home_ips": "home_ips.json",
|
||
"tpl_times": "tpl_times.json",
|
||
}
|
||
|
||
_SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS codes (
|
||
code TEXT PRIMARY KEY,
|
||
state TEXT NOT NULL DEFAULT 'active',
|
||
order_id TEXT,
|
||
batch TEXT,
|
||
platform TEXT,
|
||
gift TEXT,
|
||
data TEXT NOT NULL,
|
||
updated REAL NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS codes_state ON codes(state);
|
||
CREATE INDEX IF NOT EXISTS codes_order ON codes(order_id);
|
||
CREATE INDEX IF NOT EXISTS codes_batch ON codes(batch);
|
||
CREATE INDEX IF NOT EXISTS codes_gift ON codes(gift);
|
||
|
||
CREATE TABLE IF NOT EXISTS orders (
|
||
oid TEXT PRIMARY KEY,
|
||
state TEXT,
|
||
created TEXT,
|
||
data TEXT NOT NULL,
|
||
updated REAL NOT NULL,
|
||
purged REAL -- 文件已清理的时间;NULL = 还在。清掉的单留一行做账,不再当成「存在」
|
||
);
|
||
CREATE INDEX IF NOT EXISTS orders_state ON orders(state) WHERE purged IS NULL;
|
||
|
||
CREATE TABLE IF NOT EXISTS pay_orders (
|
||
out_trade_no TEXT PRIMARY KEY,
|
||
data TEXT NOT NULL,
|
||
updated REAL NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS calls (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
ts TEXT,
|
||
ip TEXT,
|
||
order_id TEXT,
|
||
data TEXT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS calls_ip ON calls(ip, ts);
|
||
|
||
CREATE TABLE IF NOT EXISTS kv (
|
||
k TEXT PRIMARY KEY,
|
||
data TEXT NOT NULL,
|
||
updated REAL NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS schema_info (k TEXT PRIMARY KEY, v TEXT);
|
||
"""
|
||
|
||
|
||
def _dump(obj):
|
||
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||
|
||
|
||
def _code_cols(code, rec):
|
||
st = rec.get("state")
|
||
if st not in ("active", "inactive", "used", "bound"):
|
||
st = "used" if rec.get("used") else "active"
|
||
return (code, st, rec.get("order") or None, rec.get("batch") or None,
|
||
rec.get("platform") or None, rec.get("gift") or None)
|
||
|
||
|
||
class Store(object):
|
||
"""一个库文件一个实例。每个线程自己一条连接(sqlite3 连接不能跨线程用)。"""
|
||
|
||
def __init__(self, root):
|
||
self.root = root
|
||
self.path = os.path.join(root, "ys.db")
|
||
self._local = threading.local()
|
||
os.makedirs(root, exist_ok=True)
|
||
fresh = not os.path.exists(self.path)
|
||
c = self.conn()
|
||
c.executescript(_SCHEMA)
|
||
if fresh:
|
||
try:
|
||
os.chmod(self.path, 0o640) # 和 codes.json 一样:别的账号读不到
|
||
except OSError:
|
||
pass
|
||
if self._info("schema") is None:
|
||
with self.tx() as c:
|
||
c.execute("INSERT OR REPLACE INTO schema_info VALUES ('schema', ?)", (str(SCHEMA_VERSION),))
|
||
if self._info("imported") is None:
|
||
self.import_legacy()
|
||
|
||
# ------------------------------------------------------------ 连接 / 事务
|
||
def conn(self):
|
||
c = getattr(self._local, "c", None)
|
||
if c is None:
|
||
c = sqlite3.connect(self.path, timeout=30, isolation_level=None, check_same_thread=False)
|
||
c.execute("PRAGMA journal_mode=WAL")
|
||
c.execute("PRAGMA synchronous=NORMAL") # WAL 下 NORMAL:断电最多丢最后一个事务,不会写坏库
|
||
c.execute("PRAGMA busy_timeout=30000")
|
||
c.execute("PRAGMA foreign_keys=ON")
|
||
self._local.c = c
|
||
return c
|
||
|
||
class _Tx(object):
|
||
def __init__(self, c):
|
||
self.c = c
|
||
|
||
def __enter__(self):
|
||
self.c.execute("BEGIN IMMEDIATE")
|
||
return self.c
|
||
|
||
def __exit__(self, et, ev, tb):
|
||
self.c.execute("COMMIT" if et is None else "ROLLBACK")
|
||
return False
|
||
|
||
def tx(self):
|
||
"""写事务:`with store.tx() as c:` 里的语句要么全成、要么全不成。"""
|
||
return Store._Tx(self.conn())
|
||
|
||
def _info(self, k):
|
||
r = self.conn().execute("SELECT v FROM schema_info WHERE k=?", (k,)).fetchone()
|
||
return r[0] if r else None
|
||
|
||
# ------------------------------------------------------------ 卡密
|
||
def codes_all(self):
|
||
return {k: json.loads(d) for k, d in self.conn().execute("SELECT code, data FROM codes")}
|
||
|
||
def code_get(self, code):
|
||
r = self.conn().execute("SELECT data FROM codes WHERE code=?", (code,)).fetchone()
|
||
return json.loads(r[0]) if r else None
|
||
|
||
def codes_where(self, **cols):
|
||
"""按列查:codes_where(order_id='YS…') / codes_where(gift='g1') / codes_where(state='active')"""
|
||
ok = ("state", "order_id", "batch", "platform", "gift")
|
||
ks = [k for k in cols if k in ok]
|
||
sql = "SELECT code, data FROM codes" + (" WHERE " + " AND ".join("%s=?" % k for k in ks) if ks else "")
|
||
return {k: json.loads(d) for k, d in self.conn().execute(sql, [cols[k] for k in ks])}
|
||
|
||
def codes_save(self, codes):
|
||
"""整份写回(老写法 load → 改 → save 用)。一个事务里:变了的更新、没了的删掉、新的插入。"""
|
||
now = time.time()
|
||
with self.tx() as c:
|
||
old = dict(c.execute("SELECT code, data FROM codes"))
|
||
for k in set(old) - set(codes):
|
||
c.execute("DELETE FROM codes WHERE code=?", (k,))
|
||
for k, rec in codes.items():
|
||
d = _dump(rec)
|
||
if old.get(k) == d:
|
||
continue
|
||
c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
|
||
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (d, now))
|
||
|
||
def code_put(self, code, rec):
|
||
with self.tx() as c:
|
||
c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
|
||
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(code, rec) + (_dump(rec), time.time()))
|
||
|
||
def code_counts(self):
|
||
return dict(self.conn().execute("SELECT state, COUNT(*) FROM codes GROUP BY state"))
|
||
|
||
# ------------------------------------------------------------ 订单
|
||
def order_get(self, oid):
|
||
r = self.conn().execute("SELECT data FROM orders WHERE oid=? AND purged IS NULL", (oid,)).fetchone()
|
||
return json.loads(r[0]) if r else None
|
||
|
||
def order_put(self, oid, meta):
|
||
with self.tx() as c:
|
||
c.execute("INSERT INTO orders (oid, state, created, data, updated, purged) VALUES (?,?,?,?,?,NULL)"
|
||
" ON CONFLICT(oid) DO UPDATE SET state=excluded.state, created=excluded.created,"
|
||
" data=excluded.data, updated=excluded.updated, purged=NULL",
|
||
(oid, meta.get("state"), meta.get("created"), _dump(meta), time.time()))
|
||
|
||
def order_known(self, oid):
|
||
"""这个单号用过没有(包括已清理的)—— 发新单号时防撞号。"""
|
||
return self.conn().execute("SELECT 1 FROM orders WHERE oid=?", (oid,)).fetchone() is not None
|
||
|
||
def orders(self, states=None, include_purged=False):
|
||
sql, args = "SELECT data FROM orders", []
|
||
cond = [] if include_purged else ["purged IS NULL"]
|
||
if states is not None:
|
||
states = list(states)
|
||
if not states:
|
||
return []
|
||
cond.append("state IN (%s)" % ",".join("?" * len(states)))
|
||
args += states
|
||
if cond:
|
||
sql += " WHERE " + " AND ".join(cond)
|
||
sql += " ORDER BY oid"
|
||
return [json.loads(d) for (d,) in self.conn().execute(sql, args)]
|
||
|
||
def order_ids(self):
|
||
return [r[0] for r in self.conn().execute("SELECT oid FROM orders WHERE purged IS NULL ORDER BY oid")]
|
||
|
||
def order_purge(self, oid):
|
||
"""文件清掉了:行留着(记账 / 统计用),但之后 order_get 当它不存在。"""
|
||
with self.tx() as c:
|
||
c.execute("UPDATE orders SET purged=? WHERE oid=?", (time.time(), oid))
|
||
|
||
def order_drop(self, oid):
|
||
"""单根本没建成(提交中途失败):连行一起删。"""
|
||
with self.tx() as c:
|
||
c.execute("DELETE FROM orders WHERE oid=?", (oid,))
|
||
|
||
def order_counts(self):
|
||
return dict(self.conn().execute("SELECT state, COUNT(*) FROM orders WHERE purged IS NULL GROUP BY state"))
|
||
|
||
# ------------------------------------------------------------ 爱发电账本
|
||
def ledger_all(self):
|
||
return {k: json.loads(d) for k, d in self.conn().execute("SELECT out_trade_no, data FROM pay_orders")}
|
||
|
||
def ledger_get(self, no):
|
||
r = self.conn().execute("SELECT data FROM pay_orders WHERE out_trade_no=?", (str(no),)).fetchone()
|
||
return json.loads(r[0]) if r else None
|
||
|
||
def ledger_save(self, led):
|
||
now = time.time()
|
||
with self.tx() as c:
|
||
old = dict(c.execute("SELECT out_trade_no, data FROM pay_orders"))
|
||
for k in set(old) - set(led):
|
||
c.execute("DELETE FROM pay_orders WHERE out_trade_no=?", (k,))
|
||
for k, rec in led.items():
|
||
d = _dump(rec)
|
||
if old.get(k) != d:
|
||
c.execute("INSERT OR REPLACE INTO pay_orders VALUES (?,?,?)", (str(k), d, now))
|
||
|
||
# ------------------------------------------------------------ 呼叫记录
|
||
def call_add(self, rec):
|
||
with self.tx() as c:
|
||
cur = c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)",
|
||
(rec.get("ts"), rec.get("ip"), rec.get("order") or None, _dump(rec)))
|
||
return cur.lastrowid
|
||
|
||
def calls(self, limit=None, ip=None, since=None):
|
||
"""最近的在后。ip / since(ISO 时间串)可以筛。"""
|
||
sql, cond, args = "SELECT id, data FROM calls", [], []
|
||
if ip is not None:
|
||
cond.append("ip=?"); args.append(ip)
|
||
if since is not None:
|
||
cond.append("ts>=?"); args.append(since)
|
||
if cond:
|
||
sql += " WHERE " + " AND ".join(cond)
|
||
sql += " ORDER BY id DESC"
|
||
if limit:
|
||
sql += " LIMIT %d" % int(limit)
|
||
out = []
|
||
for i, d in self.conn().execute(sql, args):
|
||
r = json.loads(d); r["id"] = i; out.append(r)
|
||
return out[::-1]
|
||
|
||
def call_mark(self, cid, **fields):
|
||
with self.tx() as c:
|
||
r = c.execute("SELECT data FROM calls WHERE id=?", (cid,)).fetchone()
|
||
if not r:
|
||
return False
|
||
d = json.loads(r[0]); d.update(fields)
|
||
c.execute("UPDATE calls SET data=? WHERE id=?", (_dump(d), cid))
|
||
return True
|
||
|
||
# ------------------------------------------------------------ 小块状态
|
||
def kv_get(self, k, default=None):
|
||
r = self.conn().execute("SELECT data FROM kv WHERE k=?", (k,)).fetchone()
|
||
return json.loads(r[0]) if r else default
|
||
|
||
def kv_put(self, k, obj):
|
||
with self.tx() as c:
|
||
c.execute("INSERT OR REPLACE INTO kv VALUES (?,?,?)", (k, _dump(obj), time.time()))
|
||
|
||
# ------------------------------------------------------------ 导入 / 导出 / 体检
|
||
def import_legacy(self):
|
||
"""第一次启动:把旧 JSON 导进来(旧文件不动)。整个导入一个事务,失败就整体回滚、下次再试。"""
|
||
R = self.root
|
||
n = {"codes": 0, "orders": 0, "pay_orders": 0, "calls": 0, "kv": 0}
|
||
|
||
def jl(p, default):
|
||
try:
|
||
with open(p, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except (OSError, ValueError):
|
||
return default
|
||
now = time.time()
|
||
with self.tx() as c:
|
||
for k, rec in (jl(os.path.join(R, "codes.json"), {}) or {}).items():
|
||
if isinstance(rec, dict):
|
||
c.execute("INSERT OR IGNORE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
|
||
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (_dump(rec), now))
|
||
n["codes"] += 1
|
||
od = os.path.join(R, "orders")
|
||
if os.path.isdir(od):
|
||
for oid in sorted(os.listdir(od)):
|
||
m = jl(os.path.join(od, oid, "meta.json"), None)
|
||
if isinstance(m, dict):
|
||
c.execute("INSERT OR IGNORE INTO orders (oid, state, created, data, updated) VALUES (?,?,?,?,?)",
|
||
(oid, m.get("state"), m.get("created"), _dump(m), now))
|
||
n["orders"] += 1
|
||
for k, rec in (jl(os.path.join(R, "payplatform-orders.json"), {}) or {}).items():
|
||
c.execute("INSERT OR IGNORE INTO pay_orders VALUES (?,?,?)", (str(k), _dump(rec), now))
|
||
n["pay_orders"] += 1
|
||
try:
|
||
with open(os.path.join(R, "calls.jsonl"), encoding="utf-8") as f:
|
||
for line in f:
|
||
try:
|
||
r = json.loads(line)
|
||
except ValueError:
|
||
continue # 旧文件里写了半行的那条跳过
|
||
if isinstance(r, dict):
|
||
c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)",
|
||
(r.get("ts"), r.get("ip"), r.get("order") or None, _dump(r)))
|
||
n["calls"] += 1
|
||
except OSError:
|
||
pass
|
||
for k, fn in KV_FILES.items():
|
||
v = jl(os.path.join(R, fn), None)
|
||
if v is not None:
|
||
c.execute("INSERT OR IGNORE INTO kv VALUES (?,?,?)", (k, _dump(v), now))
|
||
n["kv"] += 1
|
||
c.execute("INSERT OR REPLACE INTO schema_info VALUES ('imported', ?)",
|
||
(json.dumps({"at": time.strftime("%Y-%m-%d %H:%M:%S"), "n": n}, ensure_ascii=False),))
|
||
return n
|
||
|
||
def export_legacy(self, dest=None):
|
||
"""把库导回旧版 JSON(回退用)。dest 不给 = 写回 <DATA_DIR>(会覆盖那几个旧文件)。"""
|
||
D = dest or self.root
|
||
os.makedirs(os.path.join(D, "orders"), exist_ok=True)
|
||
|
||
def js(p, obj):
|
||
tmp = p + ".export.tmp"
|
||
with open(tmp, "w", encoding="utf-8") as f:
|
||
json.dump(obj, f, ensure_ascii=False, indent=1)
|
||
os.replace(tmp, p)
|
||
js(os.path.join(D, "codes.json"), self.codes_all())
|
||
js(os.path.join(D, "payplatform-orders.json"), self.ledger_all())
|
||
for m in self.orders():
|
||
od = os.path.join(D, "orders", m["oid"])
|
||
os.makedirs(od, exist_ok=True)
|
||
js(os.path.join(od, "meta.json"), m)
|
||
with open(os.path.join(D, "calls.jsonl"), "w", encoding="utf-8") as f:
|
||
for r in self.calls():
|
||
r.pop("id", None)
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
for k, fn in KV_FILES.items():
|
||
v = self.kv_get(k)
|
||
if v is not None:
|
||
js(os.path.join(D, fn), v)
|
||
return D
|
||
|
||
def check(self):
|
||
c = self.conn()
|
||
ok = c.execute("PRAGMA integrity_check").fetchone()[0]
|
||
n = {t: c.execute("SELECT COUNT(*) FROM %s" % t).fetchone()[0]
|
||
for t in ("codes", "orders", "pay_orders", "calls", "kv")}
|
||
return ok, n, self._info("imported")
|
||
|
||
def backup(self, dest=None):
|
||
"""在线热备份(服务器不用停)。"""
|
||
d = dest or os.path.join(self.root, "backup")
|
||
os.makedirs(d, exist_ok=True)
|
||
p = os.path.join(d, "ys-%s.db" % time.strftime("%Y%m%d-%H%M%S"))
|
||
b = sqlite3.connect(p)
|
||
with b:
|
||
self.conn().backup(b)
|
||
b.close()
|
||
os.chmod(p, 0o600)
|
||
return p
|
||
|
||
|
||
if __name__ == "__main__":
|
||
root = os.environ.get("YS_ROOT", "<DATA_DIR>")
|
||
cmd = (sys.argv[1:] or ["check"])[0]
|
||
st = Store(root)
|
||
if cmd == "check":
|
||
ok, n, imp = st.check()
|
||
print("完整性:%s" % ok)
|
||
for k, v in n.items():
|
||
print(" %-10s %d 行" % (k, v))
|
||
print("旧数据导入:%s" % (imp or "(没导过)"))
|
||
elif cmd == "export":
|
||
print("已导出到 %s" % st.export_legacy(sys.argv[2] if len(sys.argv) > 2 else None))
|
||
elif cmd == "backup":
|
||
print("备份好了:%s" % st.backup())
|
||
else:
|
||
print(__doc__)
|
||
sys.exit(2)
|