渲染机看门狗按分辨率放宽 + 静默检测;开单后台 Windows 下禁止双开
- ys_worker: 总时长上限随分辨率放宽(1000→600s,4096→~84min),另加 900s 无输出判卡死; 超时时把 Blender 最后几行输出写进日志。修复空艇 4096 每次在第 10 分钟被误杀(订单 YSPPVNXQ)。 - ys_worker: 读输出改为按字节 read1 + 增量解码,文本模式 read(512) 会攒满才返回。 - ys_local: 本机单同样按分辨率给上限。 - ys_admin: Windows 上关闭 SO_REUSEADDR —— 否则再双击会起第二个实例绑同一端口(宿主机实测复现)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1071,6 +1071,13 @@ class H(BaseHTTPRequestHandler):
|
|||||||
"codes": (cd.get("codes") if ok3 else []) or []})
|
"codes": (cd.get("codes") if ok3 else []) or []})
|
||||||
|
|
||||||
|
|
||||||
|
class _Server(ThreadingHTTPServer):
|
||||||
|
# ⚠️ 2026-10-07 实测:HTTPServer 默认开 SO_REUSEADDR,**Windows 上这等于允许两个进程绑同一个端口**——
|
||||||
|
# 金钻再双击一次「开单后台」,第二个实例照样起来了(没走"已经有一个在跑"分支),两个面板并存。
|
||||||
|
# Windows 上关掉它,第二个实例 bind 就会失败 → 只开浏览器。Linux 上留着(重启时不用等 TIME_WAIT)。
|
||||||
|
allow_reuse_address = (os.name != "nt")
|
||||||
|
|
||||||
|
|
||||||
def single_instance():
|
def single_instance():
|
||||||
s = socket.socket()
|
s = socket.socket()
|
||||||
try:
|
try:
|
||||||
@@ -1091,7 +1098,7 @@ def main():
|
|||||||
# 两个地址在同一个端口上会 EADDRINUSE(2026-09-30 自测实踩)。
|
# 两个地址在同一个端口上会 EADDRINUSE(2026-09-30 自测实踩)。
|
||||||
# 直接起服务:起不来就说明已经有一个在跑,开个浏览器就完事。
|
# 直接起服务:起不来就说明已经有一个在跑,开个浏览器就完事。
|
||||||
try:
|
try:
|
||||||
srv = ThreadingHTTPServer(("<ALL-INTERFACES>", PORT), H) # <ALL-INTERFACES> = 局域网(手机同 WiFi 也能开)
|
srv = _Server(("<ALL-INTERFACES>", PORT), H) # <ALL-INTERFACES> = 局域网(手机同 WiFi 也能开)
|
||||||
except OSError:
|
except OSError:
|
||||||
log("已经有一个开单后台在跑了%s" % (",这次只开浏览器" if OPEN_BROWSER else ",这次什么都不做"))
|
log("已经有一个开单后台在跑了%s" % (",这次只开浏览器" if OPEN_BROWSER else ",这次什么都不做"))
|
||||||
if OPEN_BROWSER:
|
if OPEN_BROWSER:
|
||||||
|
|||||||
@@ -203,7 +203,7 @@ def run_job(job_id):
|
|||||||
"--template", tpl, "--skin", skin,
|
"--template", tpl, "--skin", skin,
|
||||||
"--orders-dir", root, "--out-dir", d, "--res", str(res)]
|
"--orders-dir", root, "--out-dir", d, "--res", str(res)]
|
||||||
W.log("[本机单] 渲染 %s → %s(%d px)" % (fname, os.path.basename(d), res))
|
W.log("[本机单] 渲染 %s → %s(%d px)" % (fname, os.path.basename(d), res))
|
||||||
okr, tail, dt, note = W.run_blender(cmd)
|
okr, tail, dt, note = W.run_blender(cmd, timeout=W.blender_timeout(res))
|
||||||
if not okr:
|
if not okr:
|
||||||
j["errors"].append("%s:%s" % (fname, note))
|
j["errors"].append("%s:%s" % (fname, note))
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
|
|
||||||
只在需要时连服务器(出站),服务器从不主动连你。
|
只在需要时连服务器(出站),服务器从不主动连你。
|
||||||
"""
|
"""
|
||||||
import base64, json, os, re, shutil, signal, socket, subprocess, sys, threading, time, traceback
|
import base64, codecs, json, os, re, shutil, signal, socket, subprocess, sys, threading, time, traceback
|
||||||
import urllib.request, urllib.error
|
import urllib.request, urllib.error
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
@@ -214,10 +214,22 @@ def next_seq(rec=None):
|
|||||||
|
|
||||||
|
|
||||||
# ===== Blender 看门狗(金钻 2026-09-30:卡到十分钟就关掉重开再渲)=====
|
# ===== Blender 看门狗(金钻 2026-09-30:卡到十分钟就关掉重开再渲)=====
|
||||||
BLENDER_TIMEOUT = 600 # 单张 10 分钟;4096 实测 ~10 分钟内,够用
|
BLENDER_TIMEOUT = 600 # 1000 px 单张的总时长上限(10 分钟);更大的图按像素放宽,见 blender_timeout()
|
||||||
|
BLENDER_STALL = 900 # 连续这么久 Blender **一个字都没吐** 才算真卡住(不分分辨率)
|
||||||
BLENDER_RETRY = 1 # 卡住就杀掉重开一次;再卡就判失败,别死循环
|
BLENDER_RETRY = 1 # 卡住就杀掉重开一次;再卡就判失败,别死循环
|
||||||
|
|
||||||
|
|
||||||
|
def blender_timeout(res):
|
||||||
|
"""总时长上限随分辨率放宽:1000 → 600s,2048 → ~21 分钟,4096 → ~84 分钟。
|
||||||
|
|
||||||
|
⚠️ 2026-10-07:以前不分分辨率一律 600 秒。空艇 1000 px 要 70~130 秒,4096 像素多 16.8 倍,
|
||||||
|
每次都在第 10 分钟被当成"卡住"杀掉、重开再杀 → 订单 YSPPVNXQ(空艇 4096)白白失败。
|
||||||
|
真卡死靠 BLENDER_STALL 抓(渲染中 Blender 每隔几十秒就会刷一次 Sample 进度)。
|
||||||
|
"""
|
||||||
|
r = (int(res or DEFAULT_RES) / 1000.0) ** 2
|
||||||
|
return int(BLENDER_TIMEOUT * max(1.0, r * 0.5))
|
||||||
|
|
||||||
|
|
||||||
def _kill_tree(proc):
|
def _kill_tree(proc):
|
||||||
"""连子进程一起杀。Blender 会开子进程,只杀父进程会留孤儿继续吃 GPU。
|
"""连子进程一起杀。Blender 会开子进程,只杀父进程会留孤儿继续吃 GPU。
|
||||||
|
|
||||||
@@ -280,7 +292,7 @@ def blender_pct(line):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
def run_blender(cmd, timeout=None, retry=None, on_progress=None, stall=None):
|
||||||
"""带超时看门狗的渲染。返回 (ok, 输出尾巴, 耗时秒, 说明)。
|
"""带超时看门狗的渲染。返回 (ok, 输出尾巴, 耗时秒, 说明)。
|
||||||
|
|
||||||
「关掉重新打开」= 直接重跑同一条命令 —— 工程每次都是**从干净模板新复制的副本**,
|
「关掉重新打开」= 直接重跑同一条命令 —— 工程每次都是**从干净模板新复制的副本**,
|
||||||
@@ -290,6 +302,7 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
|||||||
所以:不许抛异常、不许阻塞太久(回调里已经有节流 + 超时)。
|
所以:不许抛异常、不许阻塞太久(回调里已经有节流 + 超时)。
|
||||||
"""
|
"""
|
||||||
timeout = timeout or BLENDER_TIMEOUT
|
timeout = timeout or BLENDER_TIMEOUT
|
||||||
|
stall = stall or BLENDER_STALL
|
||||||
retry =BLENDER_RETRY if retry is None else retry
|
retry =BLENDER_RETRY if retry is None else retry
|
||||||
t_all = time.time()
|
t_all = time.time()
|
||||||
last = ""
|
last = ""
|
||||||
@@ -299,20 +312,26 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
|||||||
if os.name != "nt":
|
if os.name != "nt":
|
||||||
kw["start_new_session"] = True # 自成进程组,超时能整组杀干净
|
kw["start_new_session"] = True # 自成进程组,超时能整组杀干净
|
||||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
text=True, encoding="utf-8", errors="replace", bufsize=1, **kw)
|
**kw)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
return False, "", time.time() - t_all, "启动 Blender 失败:%r" % e
|
return False, "", time.time() - t_all, "启动 Blender 失败:%r" % e
|
||||||
# 输出交给**独立线程**读:一边攒全文,一边挑进度行回调。
|
# 输出交给**独立线程**读:一边攒全文,一边挑进度行回调,一边记"最后一次有输出"的时刻。
|
||||||
# 主线程只负责等它结束 / 超时 —— 超时看门狗的行为和以前完全一样。
|
# 主线程只负责等它结束 / 超时(总时长超限 或 太久没输出)。
|
||||||
buf = []
|
buf = []
|
||||||
|
last_out = [time.time()]
|
||||||
|
|
||||||
def _pump():
|
def _pump():
|
||||||
try:
|
try:
|
||||||
tail = ""
|
tail = ""
|
||||||
|
# ⚠️ 按**字节**有多少读多少(read1)再增量解码:文本模式的 read(512) 会憋到凑满
|
||||||
|
# 512 个字才返回,"最后一次有输出"就不准了(2026-10-07 测出来的)
|
||||||
|
dec = codecs.getincrementaldecoder("utf-8")("replace")
|
||||||
while True:
|
while True:
|
||||||
chunk = p.stdout.read(512)
|
raw = p.stdout.read1(4096)
|
||||||
if not chunk:
|
if not raw:
|
||||||
break
|
break
|
||||||
|
chunk = dec.decode(raw)
|
||||||
|
last_out[0] = time.time()
|
||||||
tail += chunk
|
tail += chunk
|
||||||
# ⚠️ Blender 的进度条是**回车原地刷新**的(`... Sample 4/128\r`),
|
# ⚠️ Blender 的进度条是**回车原地刷新**的(`... Sample 4/128\r`),
|
||||||
# 按"行"读(for line in stdout)压根读不到 —— 2026-10-04 金钻要
|
# 按"行"读(for line in stdout)压根读不到 —— 2026-10-04 金钻要
|
||||||
@@ -337,8 +356,21 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
|||||||
pass
|
pass
|
||||||
th = threading.Thread(target=_pump, daemon=True)
|
th = threading.Thread(target=_pump, daemon=True)
|
||||||
th.start()
|
th.start()
|
||||||
|
t_run = time.time()
|
||||||
try:
|
try:
|
||||||
p.wait(timeout=timeout)
|
while True:
|
||||||
|
try:
|
||||||
|
p.wait(timeout=5)
|
||||||
|
break
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
now = time.time()
|
||||||
|
if now - t_run > timeout:
|
||||||
|
why = "总时长超过 %d 秒" % timeout
|
||||||
|
elif now - last_out[0] > stall:
|
||||||
|
why = "%d 秒没有任何输出" % stall
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
raise subprocess.TimeoutExpired(cmd, timeout, output=why)
|
||||||
th.join(timeout=15)
|
th.join(timeout=15)
|
||||||
out = "".join(buf)
|
out = "".join(buf)
|
||||||
if p.returncode == 0:
|
if p.returncode == 0:
|
||||||
@@ -351,14 +383,17 @@ def run_blender(cmd, timeout=None, retry=None, on_progress=None):
|
|||||||
p.returncode,
|
p.returncode,
|
||||||
(" | ".join(_keys[-6:]) + " | " if _keys else "") + _txt[-1200:])
|
(" | ".join(_keys[-6:]) + " | " if _keys else "") + _txt[-1200:])
|
||||||
log("渲染失败:%s" % last)
|
log("渲染失败:%s" % last)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired as e:
|
||||||
log("⚠️ 渲染卡住超过 %d 秒 —— 杀掉,重新打开再渲" % timeout)
|
why = e.output or ("总时长超过 %d 秒" % timeout)
|
||||||
|
# 卡在哪一步留个底(以前超时直接丢掉输出,事后完全查不到)
|
||||||
|
_seen = [ln.strip() for ln in "".join(buf).replace("\r", "\n").splitlines() if ln.strip()]
|
||||||
|
log("⚠️ 渲染卡住(%s)—— 杀掉,重新打开再渲|最后输出:%s" % (why, " | ".join(_seen[-3:])[-600:]))
|
||||||
_kill_tree(p)
|
_kill_tree(p)
|
||||||
try:
|
try:
|
||||||
p.wait(timeout=10) # 收尸;整组杀过就该立刻返回
|
p.wait(timeout=10) # 收尸;整组杀过就该立刻返回
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
last = "渲染卡住超过 %d 秒(已杀掉重开)" % timeout
|
last = "渲染卡住(%s,已杀掉重开)" % why
|
||||||
if attempt < retry:
|
if attempt < retry:
|
||||||
log("↻ 重开渲染(第 %d 次尝试)" % (attempt + 2))
|
log("↻ 重开渲染(第 %d 次尝试)" % (attempt + 2))
|
||||||
return False, "", time.time() - t_all, last
|
return False, "", time.time() - t_all, last
|
||||||
@@ -394,8 +429,8 @@ def render_one(tpl_key, skin_path, order_meta, res=DEFAULT_RES, arm="auto", on_p
|
|||||||
_sm = (CFG.get("skin_materials") or {}).get(tpl_key)
|
_sm = (CFG.get("skin_materials") or {}).get(tpl_key)
|
||||||
if _sm:
|
if _sm:
|
||||||
cmd += ["--skin-material", _sm]
|
cmd += ["--skin-material", _sm]
|
||||||
log("渲染 %s(模板 %s,臂型 %s)" % (tpl_key, fname, arm or "auto"))
|
log("渲染 %s(模板 %s,臂型 %s,%d px)" % (tpl_key, fname, arm or "auto", int(res)))
|
||||||
okr, tail, dt, note = run_blender(cmd, on_progress=on_progress)
|
okr, tail, dt, note = run_blender(cmd, timeout=blender_timeout(res), on_progress=on_progress)
|
||||||
note_skin_diag(tpl_key, tail) # 不管成没成,先把"皮肤节点认了谁"记下来
|
note_skin_diag(tpl_key, tail) # 不管成没成,先把"皮肤节点认了谁"记下来
|
||||||
if not okr:
|
if not okr:
|
||||||
return False, note
|
return False, note
|
||||||
|
|||||||
Reference in New Issue
Block a user