渲染机看门狗按分辨率放宽 + 静默检测;开单后台 Windows 下禁止双开
- ys_worker: 总时长上限随分辨率放宽(1000→600s,4096→~84min),另加 900s 无输出判卡死; 超时时把 Blender 最后几行输出写进日志。修复空艇 4096 每次在第 10 分钟被误杀(订单 YSPPVNXQ)。 - ys_worker: 读输出改为按字节 read1 + 增量解码,文本模式 read(512) 会攒满才返回。 - ys_local: 本机单同样按分辨率给上限。 - ys_admin: Windows 上关闭 SO_REUSEADDR —— 否则再双击会起第二个实例绑同一端口(宿主机实测复现)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1071,6 +1071,13 @@ class H(BaseHTTPRequestHandler):
|
||||
"codes": (cd.get("codes") if ok3 else []) or []})
|
||||
|
||||
|
||||
class _Server(ThreadingHTTPServer):
|
||||
# ⚠️ 2026-10-07 实测:HTTPServer 默认开 SO_REUSEADDR,**Windows 上这等于允许两个进程绑同一个端口**——
|
||||
# 金钻再双击一次「开单后台」,第二个实例照样起来了(没走"已经有一个在跑"分支),两个面板并存。
|
||||
# Windows 上关掉它,第二个实例 bind 就会失败 → 只开浏览器。Linux 上留着(重启时不用等 TIME_WAIT)。
|
||||
allow_reuse_address = (os.name != "nt")
|
||||
|
||||
|
||||
def single_instance():
|
||||
s = socket.socket()
|
||||
try:
|
||||
@@ -1091,7 +1098,7 @@ def main():
|
||||
# 两个地址在同一个端口上会 EADDRINUSE(2026-09-30 自测实踩)。
|
||||
# 直接起服务:起不来就说明已经有一个在跑,开个浏览器就完事。
|
||||
try:
|
||||
srv = ThreadingHTTPServer(("<ALL-INTERFACES>", PORT), H) # <ALL-INTERFACES> = 局域网(手机同 WiFi 也能开)
|
||||
srv = _Server(("<ALL-INTERFACES>", PORT), H) # <ALL-INTERFACES> = 局域网(手机同 WiFi 也能开)
|
||||
except OSError:
|
||||
log("已经有一个开单后台在跑了%s" % (",这次只开浏览器" if OPEN_BROWSER else ",这次什么都不做"))
|
||||
if OPEN_BROWSER:
|
||||
|
||||
@@ -203,7 +203,7 @@ def run_job(job_id):
|
||||
"--template", tpl, "--skin", skin,
|
||||
"--orders-dir", root, "--out-dir", d, "--res", str(res)]
|
||||
W.log("[本机单] 渲染 %s → %s(%d px)" % (fname, os.path.basename(d), res))
|
||||
okr, tail, dt, note = W.run_blender(cmd)
|
||||
okr, tail, dt, note = W.run_blender(cmd, timeout=W.blender_timeout(res))
|
||||
if not okr:
|
||||
j["errors"].append("%s:%s" % (fname, note))
|
||||
continue
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
|
||||
只在需要时连服务器(出站),服务器从不主动连你。
|
||||
"""
|
||||
import base64, json, os, re, shutil, signal, socket, subprocess, sys, threading, time, traceback
|
||||
import base64, codecs, json, os, re, shutil, signal, socket, subprocess, sys, threading, time, traceback
|
||||
import urllib.request, urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
@@ -214,10 +214,22 @@ def next_seq(rec=None):
|
||||
|
||||
|
||||
# ===== Blender 看门狗(金钻 2026-09-30:卡到十分钟就关掉重开再渲)=====
|
||||
BLENDER_TIMEOUT = 600 # 单张 10 分钟;4096 实测 ~10 分钟内,够用
|
||||
BLENDER_TIMEOUT = 600 # 1000 px 单张的总时长上限(10 分钟);更大的图按像素放宽,见 blender_timeout()
|
||||
BLENDER_STALL = 900 # 连续这么久 Blender **一个字都没吐** 才算真卡住(不分分辨率)
|
||||
BLENDER_RETRY = 1 # 卡住就杀掉重开一次;再卡就判失败,别死循环
|
||||
|
||||
|
||||
def blender_timeout(res):
|
||||
"""总时长上限随分辨率放宽:1000 → 600s,2048 → ~21 分钟,4096 → ~84 分钟。
|
||||
|
||||
⚠️ 2026-10-07:以前不分分辨率一律 600 秒。空艇 1000 px 要 70~130 秒,4096 像素多 16.8 倍,
|
||||
每次都在第 10 分钟被当成"卡住"杀掉、重开再杀 → 订单 YSPPVNXQ(空艇 4096)白白失败。
|
||||
真卡死靠 BLENDER_STALL 抓(渲染中 Blender 每隔几十秒就会刷一次 Sample 进度)。
|
||||
"""
|
||||
r = (int(res or DEFAULT_RES) / 1000.0) ** 2
|
||||
return int(BLENDER_TIMEOUT * max(1.0, r * 0.5))
|
||||
|
||||
|
||||
def _kill_tree(proc):
|
||||
"""连子进程一起杀。Blender 会开子进程,只杀父进程会留孤儿继续吃 GPU。
|
||||
|
||||
@@ -280,7 +292,7 @@ def blender_pct(line):
|
||||
return None
|
||||
|
||||
|
||||
def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
||||
def run_blender(cmd, timeout=None, retry=None, on_progress=None, stall=None):
|
||||
"""带超时看门狗的渲染。返回 (ok, 输出尾巴, 耗时秒, 说明)。
|
||||
|
||||
「关掉重新打开」= 直接重跑同一条命令 —— 工程每次都是**从干净模板新复制的副本**,
|
||||
@@ -290,6 +302,7 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
||||
所以:不许抛异常、不许阻塞太久(回调里已经有节流 + 超时)。
|
||||
"""
|
||||
timeout = timeout or BLENDER_TIMEOUT
|
||||
stall = stall or BLENDER_STALL
|
||||
retry =BLENDER_RETRY if retry is None else retry
|
||||
t_all = time.time()
|
||||
last = ""
|
||||
@@ -299,20 +312,26 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
||||
if os.name != "nt":
|
||||
kw["start_new_session"] = True # 自成进程组,超时能整组杀干净
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, encoding="utf-8", errors="replace", bufsize=1, **kw)
|
||||
**kw)
|
||||
except OSError as e:
|
||||
return False, "", time.time() - t_all, "启动 Blender 失败:%r" % e
|
||||
# 输出交给**独立线程**读:一边攒全文,一边挑进度行回调。
|
||||
# 主线程只负责等它结束 / 超时 —— 超时看门狗的行为和以前完全一样。
|
||||
# 输出交给**独立线程**读:一边攒全文,一边挑进度行回调,一边记"最后一次有输出"的时刻。
|
||||
# 主线程只负责等它结束 / 超时(总时长超限 或 太久没输出)。
|
||||
buf = []
|
||||
last_out = [time.time()]
|
||||
|
||||
def _pump():
|
||||
try:
|
||||
tail = ""
|
||||
# ⚠️ 按**字节**有多少读多少(read1)再增量解码:文本模式的 read(512) 会憋到凑满
|
||||
# 512 个字才返回,"最后一次有输出"就不准了(2026-10-07 测出来的)
|
||||
dec = codecs.getincrementaldecoder("utf-8")("replace")
|
||||
while True:
|
||||
chunk = p.stdout.read(512)
|
||||
if not chunk:
|
||||
raw = p.stdout.read1(4096)
|
||||
if not raw:
|
||||
break
|
||||
chunk = dec.decode(raw)
|
||||
last_out[0] = time.time()
|
||||
tail += chunk
|
||||
# ⚠️ Blender 的进度条是**回车原地刷新**的(`... Sample 4/128\r`),
|
||||
# 按"行"读(for line in stdout)压根读不到 —— 2026-10-04 金钻要
|
||||
@@ -337,8 +356,21 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
||||
pass
|
||||
th = threading.Thread(target=_pump, daemon=True)
|
||||
th.start()
|
||||
t_run = time.time()
|
||||
try:
|
||||
p.wait(timeout=timeout)
|
||||
while True:
|
||||
try:
|
||||
p.wait(timeout=5)
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
now = time.time()
|
||||
if now - t_run > timeout:
|
||||
why = "总时长超过 %d 秒" % timeout
|
||||
elif now - last_out[0] > stall:
|
||||
why = "%d 秒没有任何输出" % stall
|
||||
else:
|
||||
continue
|
||||
raise subprocess.TimeoutExpired(cmd, timeout, output=why)
|
||||
th.join(timeout=15)
|
||||
out = "".join(buf)
|
||||
if p.returncode == 0:
|
||||
@@ -351,14 +383,17 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
||||
p.returncode,
|
||||
(" | ".join(_keys[-6:]) + " | " if _keys else "") + _txt[-1200:])
|
||||
log("渲染失败:%s" % last)
|
||||
except subprocess.TimeoutExpired:
|
||||
log("⚠️ 渲染卡住超过 %d 秒 —— 杀掉,重新打开再渲" % timeout)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
why = e.output or ("总时长超过 %d 秒" % timeout)
|
||||
# 卡在哪一步留个底(以前超时直接丢掉输出,事后完全查不到)
|
||||
_seen = [ln.strip() for ln in "".join(buf).replace("\r", "\n").splitlines() if ln.strip()]
|
||||
log("⚠️ 渲染卡住(%s)—— 杀掉,重新打开再渲|最后输出:%s" % (why, " | ".join(_seen[-3:])[-600:]))
|
||||
_kill_tree(p)
|
||||
try:
|
||||
p.wait(timeout=10) # 收尸;整组杀过就该立刻返回
|
||||
except Exception:
|
||||
pass
|
||||
last = "渲染卡住超过 %d 秒(已杀掉重开)" % timeout
|
||||
last = "渲染卡住(%s,已杀掉重开)" % why
|
||||
if attempt < retry:
|
||||
log("↻ 重开渲染(第 %d 次尝试)" % (attempt + 2))
|
||||
return False, "", time.time() - t_all, last
|
||||
@@ -394,8 +429,8 @@ def render_one(tpl_key, skin_path, order_meta, res=DEFAULT_RES, arm="auto", on_p
|
||||
_sm = (CFG.get("skin_materials") or {}).get(tpl_key)
|
||||
if _sm:
|
||||
cmd += ["--skin-material", _sm]
|
||||
log("渲染 %s(模板 %s,臂型 %s)" % (tpl_key, fname, arm or "auto"))
|
||||
okr, tail, dt, note = run_blender(cmd, on_progress=on_progress)
|
||||
log("渲染 %s(模板 %s,臂型 %s,%d px)" % (tpl_key, fname, arm or "auto", int(res)))
|
||||
okr, tail, dt, note = run_blender(cmd, timeout=blender_timeout(res), on_progress=on_progress)
|
||||
note_skin_diag(tpl_key, tail) # 不管成没成,先把"皮肤节点认了谁"记下来
|
||||
if not okr:
|
||||
return False, note
|
||||
|
||||
Reference in New Issue
Block a user