Files
jinzuanwork/backend/backend-site/store.py
T
ywsandClaude Opus 5.5 51212a07c0 backend: 接入 SQLite 数据库 + 开单后台按功能重写
- store.py(新):卡密 / 订单信息 / 爱发电账本 / 呼叫记录 / 小块状态进 <DATA_DIR>/ys.db;
  第一次启动自动建库并导入旧 JSON(旧文件不动),admin.py db check/backup/export
- server.py / admin.py 存取换到库上,路由与接口字段不变;calls.jsonl 照旧追加
- 管理接口新增 calls / call-done / retry,codes 支持 q 搜索
- ys_admin.py 页面重写:待处理 / 订单 / 卡密 / 福利券 / 呼叫·反馈 / 开单工具
- tests/数据库测试.py(37 条,假数据);回归 27 条照旧全过
- 交接说明(数据库一节、面板一节)、回执第二轮、交付清单更新

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 16:20:45 +00:00

422 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""金钻工坊 · 数据层(SQLite,标准库自带,零依赖)
2026-10-05 起,原来散在 <DATA_DIR> 下的这些「拿 JSON 文件当数据库」的东西,统一进一个库:
<DATA_DIR>/ys.db
| 表 | 原来在哪 | 里面是什么 |
|-------------|-----------------------------------|------------|
| codes | codes.json | 卡密池(一行一张码) |
| orders | orders/<单号>/meta.json | 订单信息(一行一单;皮肤 / 成品**文件**还在 orders/<单号>/ 目录里) |
| pay_orders | payplatform-orders.json | 爱发电回调账本(一行一笔,按平台单号幂等) |
| calls | calls.jsonl | 「呼叫金钻」/ 反馈 / 部分失败 记录 |
| kv | gifts.json / heartbeat.json / home_ips.json / tpl_times.json | 小块状态(一个键一份 JSON) |
**还是文件、没进库的**(都是人手改的配置或二进制):
payplatform.json(凭据)、payplatform-map.json、prices.json、payinfo.json、
worker_token / admin_token、皮肤与成品图片、feedback/ 里的反馈图、各种 .log
设计取舍(为什么这么做):
* 每张表都是「几个能查的列 + 一列 data(整条记录的 JSON)」。上层代码拿到的还是原来的 dict,
**字段想加就加,不用改表**;能查的列(state / order_id / batch …)是写入时从 data 里抽出来的。
* 写整份 dict 的老写法(load → 改 → save)保留:save 时在**一个事务里**只改有变化的行。
所以 server.py 里的路由逻辑一行都不用动,只是存取换了地方。
* 并发:上层照旧 `with LOCK:`(线程锁 + 跨进程 flock);库本身开 WAL + busy_timeout,
读不挡写、写不挡读,另一个进程(admin.py)在写也只是等一会儿,不会报 locked。
* 断电:SQLite 事务要么全写进去、要么全没写,不会再有「半截 codes.json」。
第一次启动自动从旧 JSON 导入(只导一次,旧文件原样留着不删);要回退到旧版:
python3 store.py export 把库里的数据导回成旧版的 JSON 文件
python3 store.py check 完整性检查 + 每张表几行
python3 store.py backup 在线热备份一份到 <DATA_DIR>/backup/ys-<时间>.db
"""
import json, os, sqlite3, sys, threading, time
SCHEMA_VERSION = 1
# 哪些 kv 键原来是哪个文件(导入 / 导出用)
KV_FILES = {
"gifts": "gifts.json",
"heartbeat": "heartbeat.json",
"home_ips": "home_ips.json",
"tpl_times": "tpl_times.json",
}
_SCHEMA = """
CREATE TABLE IF NOT EXISTS codes (
code TEXT PRIMARY KEY,
state TEXT NOT NULL DEFAULT 'active',
order_id TEXT,
batch TEXT,
platform TEXT,
gift TEXT,
data TEXT NOT NULL,
updated REAL NOT NULL
);
CREATE INDEX IF NOT EXISTS codes_state ON codes(state);
CREATE INDEX IF NOT EXISTS codes_order ON codes(order_id);
CREATE INDEX IF NOT EXISTS codes_batch ON codes(batch);
CREATE INDEX IF NOT EXISTS codes_gift ON codes(gift);
CREATE TABLE IF NOT EXISTS orders (
oid TEXT PRIMARY KEY,
state TEXT,
created TEXT,
data TEXT NOT NULL,
updated REAL NOT NULL,
purged REAL -- 文件已清理的时间;NULL = 还在。清掉的单留一行做账,不再当成「存在」
);
CREATE INDEX IF NOT EXISTS orders_state ON orders(state) WHERE purged IS NULL;
CREATE TABLE IF NOT EXISTS pay_orders (
out_trade_no TEXT PRIMARY KEY,
data TEXT NOT NULL,
updated REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS calls (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts TEXT,
ip TEXT,
order_id TEXT,
data TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS calls_ip ON calls(ip, ts);
CREATE TABLE IF NOT EXISTS kv (
k TEXT PRIMARY KEY,
data TEXT NOT NULL,
updated REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS schema_info (k TEXT PRIMARY KEY, v TEXT);
"""
def _dump(obj):
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def _code_cols(code, rec):
st = rec.get("state")
if st not in ("active", "inactive", "used", "bound"):
st = "used" if rec.get("used") else "active"
return (code, st, rec.get("order") or None, rec.get("batch") or None,
rec.get("platform") or None, rec.get("gift") or None)
class Store(object):
"""一个库文件一个实例。每个线程自己一条连接(sqlite3 连接不能跨线程用)。"""
def __init__(self, root):
self.root = root
self.path = os.path.join(root, "ys.db")
self._local = threading.local()
os.makedirs(root, exist_ok=True)
fresh = not os.path.exists(self.path)
c = self.conn()
c.executescript(_SCHEMA)
if fresh:
try:
os.chmod(self.path, 0o640) # 和 codes.json 一样:别的账号读不到
except OSError:
pass
if self._info("schema") is None:
with self.tx() as c:
c.execute("INSERT OR REPLACE INTO schema_info VALUES ('schema', ?)", (str(SCHEMA_VERSION),))
if self._info("imported") is None:
self.import_legacy()
# ------------------------------------------------------------ 连接 / 事务
def conn(self):
c = getattr(self._local, "c", None)
if c is None:
c = sqlite3.connect(self.path, timeout=30, isolation_level=None, check_same_thread=False)
c.execute("PRAGMA journal_mode=WAL")
c.execute("PRAGMA synchronous=NORMAL") # WAL 下 NORMAL:断电最多丢最后一个事务,不会写坏库
c.execute("PRAGMA busy_timeout=30000")
c.execute("PRAGMA foreign_keys=ON")
self._local.c = c
return c
class _Tx(object):
def __init__(self, c):
self.c = c
def __enter__(self):
self.c.execute("BEGIN IMMEDIATE")
return self.c
def __exit__(self, et, ev, tb):
self.c.execute("COMMIT" if et is None else "ROLLBACK")
return False
def tx(self):
"""写事务:`with store.tx() as c:` 里的语句要么全成、要么全不成。"""
return Store._Tx(self.conn())
def _info(self, k):
r = self.conn().execute("SELECT v FROM schema_info WHERE k=?", (k,)).fetchone()
return r[0] if r else None
# ------------------------------------------------------------ 卡密
def codes_all(self):
return {k: json.loads(d) for k, d in self.conn().execute("SELECT code, data FROM codes")}
def code_get(self, code):
r = self.conn().execute("SELECT data FROM codes WHERE code=?", (code,)).fetchone()
return json.loads(r[0]) if r else None
def codes_where(self, **cols):
"""按列查:codes_where(order_id='YS…') / codes_where(gift='g1') / codes_where(state='active')"""
ok = ("state", "order_id", "batch", "platform", "gift")
ks = [k for k in cols if k in ok]
sql = "SELECT code, data FROM codes" + (" WHERE " + " AND ".join("%s=?" % k for k in ks) if ks else "")
return {k: json.loads(d) for k, d in self.conn().execute(sql, [cols[k] for k in ks])}
def codes_save(self, codes):
"""整份写回(老写法 load → 改 → save 用)。一个事务里:变了的更新、没了的删掉、新的插入。"""
now = time.time()
with self.tx() as c:
old = dict(c.execute("SELECT code, data FROM codes"))
for k in set(old) - set(codes):
c.execute("DELETE FROM codes WHERE code=?", (k,))
for k, rec in codes.items():
d = _dump(rec)
if old.get(k) == d:
continue
c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (d, now))
def code_put(self, code, rec):
with self.tx() as c:
c.execute("INSERT OR REPLACE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(code, rec) + (_dump(rec), time.time()))
def code_counts(self):
return dict(self.conn().execute("SELECT state, COUNT(*) FROM codes GROUP BY state"))
# ------------------------------------------------------------ 订单
def order_get(self, oid):
r = self.conn().execute("SELECT data FROM orders WHERE oid=? AND purged IS NULL", (oid,)).fetchone()
return json.loads(r[0]) if r else None
def order_put(self, oid, meta):
with self.tx() as c:
c.execute("INSERT INTO orders (oid, state, created, data, updated, purged) VALUES (?,?,?,?,?,NULL)"
" ON CONFLICT(oid) DO UPDATE SET state=excluded.state, created=excluded.created,"
" data=excluded.data, updated=excluded.updated, purged=NULL",
(oid, meta.get("state"), meta.get("created"), _dump(meta), time.time()))
def order_known(self, oid):
"""这个单号用过没有(包括已清理的)—— 发新单号时防撞号。"""
return self.conn().execute("SELECT 1 FROM orders WHERE oid=?", (oid,)).fetchone() is not None
def orders(self, states=None, include_purged=False):
sql, args = "SELECT data FROM orders", []
cond = [] if include_purged else ["purged IS NULL"]
if states is not None:
states = list(states)
if not states:
return []
cond.append("state IN (%s)" % ",".join("?" * len(states)))
args += states
if cond:
sql += " WHERE " + " AND ".join(cond)
sql += " ORDER BY oid"
return [json.loads(d) for (d,) in self.conn().execute(sql, args)]
def order_ids(self):
return [r[0] for r in self.conn().execute("SELECT oid FROM orders WHERE purged IS NULL ORDER BY oid")]
def order_purge(self, oid):
"""文件清掉了:行留着(记账 / 统计用),但之后 order_get 当它不存在。"""
with self.tx() as c:
c.execute("UPDATE orders SET purged=? WHERE oid=?", (time.time(), oid))
def order_drop(self, oid):
"""单根本没建成(提交中途失败):连行一起删。"""
with self.tx() as c:
c.execute("DELETE FROM orders WHERE oid=?", (oid,))
def order_counts(self):
return dict(self.conn().execute("SELECT state, COUNT(*) FROM orders WHERE purged IS NULL GROUP BY state"))
# ------------------------------------------------------------ 爱发电账本
def ledger_all(self):
return {k: json.loads(d) for k, d in self.conn().execute("SELECT out_trade_no, data FROM pay_orders")}
def ledger_get(self, no):
r = self.conn().execute("SELECT data FROM pay_orders WHERE out_trade_no=?", (str(no),)).fetchone()
return json.loads(r[0]) if r else None
def ledger_save(self, led):
now = time.time()
with self.tx() as c:
old = dict(c.execute("SELECT out_trade_no, data FROM pay_orders"))
for k in set(old) - set(led):
c.execute("DELETE FROM pay_orders WHERE out_trade_no=?", (k,))
for k, rec in led.items():
d = _dump(rec)
if old.get(k) != d:
c.execute("INSERT OR REPLACE INTO pay_orders VALUES (?,?,?)", (str(k), d, now))
# ------------------------------------------------------------ 呼叫记录
def call_add(self, rec):
with self.tx() as c:
cur = c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)",
(rec.get("ts"), rec.get("ip"), rec.get("order") or None, _dump(rec)))
return cur.lastrowid
def calls(self, limit=None, ip=None, since=None):
"""最近的在后。ip / since(ISO 时间串)可以筛。"""
sql, cond, args = "SELECT id, data FROM calls", [], []
if ip is not None:
cond.append("ip=?"); args.append(ip)
if since is not None:
cond.append("ts>=?"); args.append(since)
if cond:
sql += " WHERE " + " AND ".join(cond)
sql += " ORDER BY id DESC"
if limit:
sql += " LIMIT %d" % int(limit)
out = []
for i, d in self.conn().execute(sql, args):
r = json.loads(d); r["id"] = i; out.append(r)
return out[::-1]
def call_mark(self, cid, **fields):
with self.tx() as c:
r = c.execute("SELECT data FROM calls WHERE id=?", (cid,)).fetchone()
if not r:
return False
d = json.loads(r[0]); d.update(fields)
c.execute("UPDATE calls SET data=? WHERE id=?", (_dump(d), cid))
return True
# ------------------------------------------------------------ 小块状态
def kv_get(self, k, default=None):
r = self.conn().execute("SELECT data FROM kv WHERE k=?", (k,)).fetchone()
return json.loads(r[0]) if r else default
def kv_put(self, k, obj):
with self.tx() as c:
c.execute("INSERT OR REPLACE INTO kv VALUES (?,?,?)", (k, _dump(obj), time.time()))
# ------------------------------------------------------------ 导入 / 导出 / 体检
def import_legacy(self):
"""第一次启动:把旧 JSON 导进来(旧文件不动)。整个导入一个事务,失败就整体回滚、下次再试。"""
R = self.root
n = {"codes": 0, "orders": 0, "pay_orders": 0, "calls": 0, "kv": 0}
def jl(p, default):
try:
with open(p, encoding="utf-8") as f:
return json.load(f)
except (OSError, ValueError):
return default
now = time.time()
with self.tx() as c:
for k, rec in (jl(os.path.join(R, "codes.json"), {}) or {}).items():
if isinstance(rec, dict):
c.execute("INSERT OR IGNORE INTO codes (code, state, order_id, batch, platform, gift, data, updated)"
" VALUES (?,?,?,?,?,?,?,?)", _code_cols(k, rec) + (_dump(rec), now))
n["codes"] += 1
od = os.path.join(R, "orders")
if os.path.isdir(od):
for oid in sorted(os.listdir(od)):
m = jl(os.path.join(od, oid, "meta.json"), None)
if isinstance(m, dict):
c.execute("INSERT OR IGNORE INTO orders (oid, state, created, data, updated) VALUES (?,?,?,?,?)",
(oid, m.get("state"), m.get("created"), _dump(m), now))
n["orders"] += 1
for k, rec in (jl(os.path.join(R, "payplatform-orders.json"), {}) or {}).items():
c.execute("INSERT OR IGNORE INTO pay_orders VALUES (?,?,?)", (str(k), _dump(rec), now))
n["pay_orders"] += 1
try:
with open(os.path.join(R, "calls.jsonl"), encoding="utf-8") as f:
for line in f:
try:
r = json.loads(line)
except ValueError:
continue # 旧文件里写了半行的那条跳过
if isinstance(r, dict):
c.execute("INSERT INTO calls (ts, ip, order_id, data) VALUES (?,?,?,?)",
(r.get("ts"), r.get("ip"), r.get("order") or None, _dump(r)))
n["calls"] += 1
except OSError:
pass
for k, fn in KV_FILES.items():
v = jl(os.path.join(R, fn), None)
if v is not None:
c.execute("INSERT OR IGNORE INTO kv VALUES (?,?,?)", (k, _dump(v), now))
n["kv"] += 1
c.execute("INSERT OR REPLACE INTO schema_info VALUES ('imported', ?)",
(json.dumps({"at": time.strftime("%Y-%m-%d %H:%M:%S"), "n": n}, ensure_ascii=False),))
return n
def export_legacy(self, dest=None):
"""把库导回旧版 JSON(回退用)。dest 不给 = 写回 <DATA_DIR>(会覆盖那几个旧文件)。"""
D = dest or self.root
os.makedirs(os.path.join(D, "orders"), exist_ok=True)
def js(p, obj):
tmp = p + ".export.tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(obj, f, ensure_ascii=False, indent=1)
os.replace(tmp, p)
js(os.path.join(D, "codes.json"), self.codes_all())
js(os.path.join(D, "payplatform-orders.json"), self.ledger_all())
for m in self.orders():
od = os.path.join(D, "orders", m["oid"])
os.makedirs(od, exist_ok=True)
js(os.path.join(od, "meta.json"), m)
with open(os.path.join(D, "calls.jsonl"), "w", encoding="utf-8") as f:
for r in self.calls():
r.pop("id", None)
f.write(json.dumps(r, ensure_ascii=False) + "\n")
for k, fn in KV_FILES.items():
v = self.kv_get(k)
if v is not None:
js(os.path.join(D, fn), v)
return D
def check(self):
c = self.conn()
ok = c.execute("PRAGMA integrity_check").fetchone()[0]
n = {t: c.execute("SELECT COUNT(*) FROM %s" % t).fetchone()[0]
for t in ("codes", "orders", "pay_orders", "calls", "kv")}
return ok, n, self._info("imported")
def backup(self, dest=None):
"""在线热备份(服务器不用停)。"""
d = dest or os.path.join(self.root, "backup")
os.makedirs(d, exist_ok=True)
p = os.path.join(d, "ys-%s.db" % time.strftime("%Y%m%d-%H%M%S"))
b = sqlite3.connect(p)
with b:
self.conn().backup(b)
b.close()
os.chmod(p, 0o600)
return p
if __name__ == "__main__":
root = os.environ.get("YS_ROOT", "<DATA_DIR>")
cmd = (sys.argv[1:] or ["check"])[0]
st = Store(root)
if cmd == "check":
ok, n, imp = st.check()
print("完整性:%s" % ok)
for k, v in n.items():
print(" %-10s %d 行" % (k, v))
print("旧数据导入:%s" % (imp or "(没导过)"))
elif cmd == "export":
print("已导出到 %s" % st.export_legacy(sys.argv[2] if len(sys.argv) > 2 else None))
elif cmd == "backup":
print("备份好了:%s" % st.backup())
else:
print(__doc__)
sys.exit(2)