fix: 优化架构

This commit is contained in:
Daniel
2026-03-25 19:35:37 +08:00
parent 34786b37c7
commit 508c28ce31
184 changed files with 2199 additions and 241 deletions

View File

@@ -63,78 +63,82 @@
const [globalStyle, setGlobalStyle] = useState("电影感");
const [characterPreset, setCharacterPreset] = useState("");
const [mock, setMock] = useState(true);
const [logs, setLogs] = useState("");
const [scenes, setScenes] = useState([null, null, null]);
const [llmProvider, setLlmProvider] = useState("mock");
const [imageProvider, setImageProvider] = useState("mock");
const [imageFallbackProvider, setImageFallbackProvider] = useState("mock");
const [scenes, setScenes] = useState([]);
const [stageLogs, setStageLogs] = useState({ Script: [], Refine: [], Render: [] });
const [stageState, setStageState] = useState({
Script: "pending",
Refine: "pending",
Render: "pending",
});
const [activeLogStage, setActiveLogStage] = useState("Script");
const [renderProgress, setRenderProgress] = useState(0);
const [canRender, setCanRender] = useState(false);
const [finalVideoUrl, setFinalVideoUrl] = useState("");
const [taskId, setTaskId] = useState("");
const [toast, setToast] = useState("");
const [scriptRunning, setScriptRunning] = useState(false);
const esRef = useRef(null);
const logRef = useRef(null);
const autoTimerRef = useRef(null);
const appendLog = (line) => {
setLogs((prev) => prev + line + "\n");
const appendStageLog = (stage, line) => {
setStageLogs((prev) => ({ ...prev, [stage]: [...(prev[stage] || []), String(line)] }));
};
const showToast = (msg) => {
setToast(String(msg || "发生错误"));
// auto hide
setTimeout(() => setToast(""), 6000);
};
useEffect(() => {
if (!logRef.current) return;
logRef.current.scrollTop = logRef.current.scrollHeight;
}, [logs]);
}, [stageLogs]);
const startScript = () => {
stopScript();
setLogs("");
setScenes([null, null, null]);
setCanRender(false);
setFinalVideoUrl("");
setTaskId("");
const normalizeScene = (raw, idx) => {
const s = raw || {};
return {
index: Number(s.index || idx + 1),
image_prompt: String(s.image_prompt || ""),
video_motion: String(s.video_motion || ""),
narration: String(s.narration || ""),
preview_url: String(s.preview_url || ""),
motion_camera: String(s.motion_camera || ""),
motion_direction: String(s.motion_direction || ""),
motion_speed: String(s.motion_speed || "normal"),
};
};
const url = `/api/script?prompt=${encodeURIComponent(prompt.trim())}&mock=${mock ? "1" : "0"}&global_style=${encodeURIComponent(globalStyle)}&character=${encodeURIComponent(characterPreset)}`;
const es = new EventSource(url);
esRef.current = es;
const parseStageUpdate = (raw) => {
if (!raw || typeof raw !== "object") return null;
if (Number(raw.schema_version) !== 1) return null;
const stage = String(raw.stage || "");
if (!["Script", "Refine", "Render"].includes(stage)) return null;
return raw;
};
es.addEventListener("status", (e) => appendLog("[status] " + e.data));
es.addEventListener("error", (e) => {
const m = (e && e.data) ? e.data : "连接或后端错误";
appendLog("[ERROR] " + m);
showToast(m);
});
es.addEventListener("task", (e) => {
try { setTaskId(JSON.parse(e.data).task_id || ""); } catch (err) { }
});
es.addEventListener("done", (e) => {
appendLog("[done] exit_code=" + e.data);
stopScript();
});
es.addEventListener("scene", (e) => {
try {
const obj = JSON.parse(e.data);
setScenes((prev) => {
const next = [...prev];
next[obj.index - 1] = {
index: obj.index,
image_prompt: obj.image_prompt || "",
video_motion: obj.video_motion || "",
narration: obj.narration || "",
};
return next;
});
} catch (err) {
appendLog("[parse_error] " + err);
}
});
es.addEventListener("line", (e) => {
appendLog(e.data);
if (e.data === "SCRIPT_END") setCanRender(true);
});
es.onerror = () => appendLog("[error] connection error");
const applyStageUpdate = (u) => {
const stage = String(u.stage || "Script");
setStageState((prev) => ({ ...prev, [stage]: prev[stage] === "done" ? "done" : "running" }));
if (typeof u.progress === "number" && stage === "Render") {
setRenderProgress(Math.max(0, Math.min(1, u.progress)));
}
if (u.scene_json) {
const scene = u.scene_json;
const idx = Math.max(0, Number(scene.index || 1) - 1);
setScenes((prev) => {
const next = [...prev];
next[idx] = normalizeScene(scene, idx);
return next;
});
setCanRender(true);
}
if (u.message) appendStageLog(stage, u.message);
if (u.shot_id && u.shot_status) appendStageLog(stage, `${u.shot_id} -> ${u.shot_status}`);
};
const stopScript = () => {
@@ -142,68 +146,116 @@
esRef.current.close();
esRef.current = null;
}
setScriptRunning(false);
};
const startScript = () => {
const p = prompt.trim();
if (!p) return;
stopScript();
setScriptRunning(true);
setCanRender(false);
setFinalVideoUrl("");
setRenderProgress(0);
setScenes([]);
setStageLogs({ Script: [], Refine: [], Render: [] });
setStageState({ Script: "running", Refine: "pending", Render: "pending" });
setActiveLogStage("Script");
const q = new URLSearchParams({
prompt: p,
mock: mock ? "1" : "0",
global_style: globalStyle,
character: characterPreset,
llm_provider: llmProvider,
image_provider: imageProvider,
image_fallback_provider: imageFallbackProvider,
});
const es = new EventSource(`/api/script?${q.toString()}`);
esRef.current = es;
es.addEventListener("task", (e) => {
try { setTaskId(JSON.parse(e.data).task_id || ""); } catch (err) { }
});
es.addEventListener("stage_update", (e) => {
try {
const parsed = parseStageUpdate(JSON.parse(e.data));
if (!parsed) return appendStageLog("Script", "[schema_error] invalid stage_update payload");
applyStageUpdate(parsed);
} catch (err) { appendStageLog("Script", "[parse_error] " + err); }
});
es.addEventListener("line", (e) => appendStageLog("Script", e.data));
es.addEventListener("error", (e) => {
setStageState((prev) => ({ ...prev, Script: "failed" }));
showToast((e && e.data) ? e.data : "script连接错误");
});
es.addEventListener("done", () => {
stopScript();
setCanRender(true);
setStageState((prev) => ({ ...prev, Script: "done" }));
});
};
useEffect(() => {
if (autoTimerRef.current) clearTimeout(autoTimerRef.current);
autoTimerRef.current = setTimeout(() => {
if (!prompt.trim()) return;
startScript();
}, 700);
return () => {
if (autoTimerRef.current) clearTimeout(autoTimerRef.current);
};
}, [prompt, globalStyle, characterPreset, llmProvider, imageProvider, imageFallbackProvider, mock]);
const updateMotionField = (idx, field, value) => {
setScenes((prev) => {
const next = [...prev];
const cur = normalizeScene(next[idx], idx);
const merged = { ...cur, [field]: value };
merged.video_motion = [merged.motion_camera, merged.motion_direction, merged.motion_speed].filter(Boolean).join(" | ");
next[idx] = merged;
return next;
});
};
const onEdit = (idx, field, value) => {
setScenes((prev) => {
const next = [...prev];
const cur = next[idx] || { index: idx + 1, image_prompt: "", video_motion: "", narration: "" };
const cur = normalizeScene(next[idx], idx);
next[idx] = { ...cur, [field]: value };
return next;
});
};
const refineOne = async (sceneIndex) => {
appendLog(`[refine] scene ${sceneIndex}...`);
setStageState((prev) => ({ ...prev, Refine: "running" }));
appendStageLog("Refine", `scene ${sceneIndex} refining...`);
const s0 = scenes[sceneIndex - 1] || {};
const payloadScene = {
image_prompt: s0.image_prompt || "",
video_motion: s0.video_motion || "",
narration: s0.narration || "",
};
const resp = await fetch("/api/refine", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ prompt, scene: payloadScene, scene_index: sceneIndex, mock, global_style: globalStyle, character: characterPreset, task_id: taskId }),
});
const data = await resp.json();
if (!resp.ok) {
appendLog("[refine_error] " + JSON.stringify(data));
showToast((data && (data.error || data.msg)) || "润色失败");
return;
}
const s = data.scene;
setScenes((prev) => {
const next = [...prev];
next[s.index - 1] = {
index: s.index,
image_prompt: s.image_prompt || "",
video_motion: s.video_motion || "",
narration: s.narration || "",
};
return next;
});
appendLog(`[refine] scene ${sceneIndex} done`);
};
const renderVideo = async () => {
appendLog("[render] start...");
const payloadScenes = scenes.map((s, i) => ({
image_prompt: (s && s.image_prompt) || "",
video_motion: (s && s.video_motion) || "",
narration: (s && s.narration) || "",
}));
const resp = await fetch("/api/render", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ prompt, scenes: payloadScenes, mock, global_style: globalStyle, character: characterPreset, task_id: taskId }),
body: JSON.stringify({
prompt,
scene: {
image_prompt: s0.image_prompt || "",
video_motion: s0.video_motion || "",
narration: s0.narration || "",
},
scene_index: sceneIndex,
mock,
global_style: globalStyle,
character: characterPreset,
task_id: taskId,
llm_provider: llmProvider,
image_provider: imageProvider,
image_fallback_provider: imageFallbackProvider,
}),
});
if (!resp.ok) {
appendLog("[render_error] http " + resp.status);
showToast("渲染请求失败HTTP " + resp.status + "");
setStageState((prev) => ({ ...prev, Refine: "failed" }));
showToast("润色请求失败HTTP " + resp.status + "");
return;
}
// Parse SSE from fetch (POST)
const reader = resp.body.getReader();
const decoder = new TextDecoder("utf-8");
let buf = "";
@@ -224,40 +276,146 @@
const data = dataLines.join("\n");
if (event === "task") {
try { setTaskId(JSON.parse(data).task_id || ""); } catch (err) { }
} else if (event === "prog") {
appendLog("[prog] " + data);
} else if (event === "stage_update") {
try {
const parsed = parseStageUpdate(JSON.parse(data));
if (!parsed) return appendStageLog("Refine", "[schema_error] invalid stage_update payload");
applyStageUpdate(parsed);
} catch (err) { appendStageLog("Refine", "[parse_error] " + err); }
} else if (event === "error") {
appendLog("[ERROR] " + data);
appendStageLog("Refine", "[ERROR] " + data);
setStageState((prev) => ({ ...prev, Refine: "failed" }));
showToast(data);
} else if (event === "done") {
appendStageLog("Refine", `scene ${sceneIndex} done`);
setStageState((prev) => ({ ...prev, Refine: "done" }));
} else if (event === "line") {
appendStageLog("Refine", data);
}
}
}
};
const renderVideo = async () => {
setStageState((prev) => ({ ...prev, Render: "running" }));
setRenderProgress(0);
appendStageLog("Render", "render start...");
const payloadScenes = scenes.map((s, i) => ({
image_prompt: (s && s.image_prompt) || "",
video_motion: (s && s.video_motion) || "",
narration: (s && s.narration) || "",
}));
const resp = await fetch("/api/render", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
prompt,
scenes: payloadScenes,
mock,
global_style: globalStyle,
character: characterPreset,
task_id: taskId,
llm_provider: llmProvider,
image_provider: imageProvider,
image_fallback_provider: imageFallbackProvider,
}),
});
if (!resp.ok) {
setStageState((prev) => ({ ...prev, Render: "failed" }));
showToast("渲染请求失败HTTP " + resp.status + "");
return;
}
const reader = resp.body.getReader();
const decoder = new TextDecoder("utf-8");
let buf = "";
while (true) {
const { value, done } = await reader.read();
if (done) break;
buf += decoder.decode(value, { stream: true });
const chunks = buf.split("\n\n");
buf = chunks.pop() || "";
for (const c of chunks) {
const lines = c.split("\n").filter(Boolean);
let event = "message";
const dataLines = [];
for (const line of lines) {
if (line.startsWith("event:")) event = line.slice(6).trim();
else if (line.startsWith("data:")) dataLines.push(line.slice(5).trim());
}
const data = dataLines.join("\n");
if (event === "task") {
try { setTaskId(JSON.parse(data).task_id || ""); } catch (err) { }
} else if (event === "stage_update") {
try {
const parsed = parseStageUpdate(JSON.parse(data));
if (!parsed) return appendStageLog("Render", "[schema_error] invalid stage_update payload");
applyStageUpdate(parsed);
} catch (err) { appendStageLog("Render", "[parse_error] " + err); }
} else if (event === "error") {
appendStageLog("Render", "[ERROR] " + data);
setStageState((prev) => ({ ...prev, Render: "failed" }));
showToast(data);
} else if (event === "done") {
try {
const obj = JSON.parse(data);
const file = String(obj.output || "").split("/").pop() || "final.mp4";
const tid = taskId || (obj.task_id || "");
appendLog("[render] done: " + file);
if (tid) setFinalVideoUrl(`/api/static/${encodeURIComponent(tid)}/${encodeURIComponent(file)}?t=${Date.now()}`);
setRenderProgress(1);
setStageState((prev) => ({ ...prev, Render: "done" }));
} catch (e) {
appendLog("[render_done_parse_error] " + e);
setStageState((prev) => ({ ...prev, Render: "failed" }));
showToast("渲染完成消息解析失败");
}
} else {
appendLog(data);
} else if (event === "line") {
appendStageLog("Render", data);
}
}
}
};
const activeLogs = useMemo(() => (stageLogs[activeLogStage] || []).join("\n"), [stageLogs, activeLogStage]);
const stageColor = (s) => {
if (s === "done") return "#16a34a";
if (s === "running") return "#2563eb";
if (s === "failed") return "#dc2626";
return "#9ca3af";
};
return (
<div>
<h2>AiVideo POC双向交互手搓平台</h2>
<p className="muted">分镜可编辑可单条润色渲染完成后可直接预览与下载</p>
<h2>AiVideo POC多后端模型平台</h2>
<p className="muted">自动拆分 PromptSSE 实时反馈Render 全链路可视化</p>
<div className="row">
<input
type="text"
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
/>
<input type="text" value={prompt} onChange={(e) => setPrompt(e.target.value)} />
</div>
<div className="row">
<label>LLM Provider:
<select value={llmProvider} onChange={(e) => setLlmProvider(e.target.value)} style={{ marginLeft: 6, padding: 8 }}>
<option value="mock">mock</option>
<option value="openai">openai</option>
</select>
</label>
<label>Image Provider:
<select value={imageProvider} onChange={(e) => setImageProvider(e.target.value)} style={{ marginLeft: 6, padding: 8 }}>
<option value="mock">mock</option>
<option value="comfy">comfy</option>
<option value="replicate">replicate</option>
<option value="openai">openai</option>
</select>
</label>
<label>Fallback:
<select value={imageFallbackProvider} onChange={(e) => setImageFallbackProvider(e.target.value)} style={{ marginLeft: 6, padding: 8 }}>
<option value="mock">mock</option>
<option value="comfy">comfy</option>
</select>
</label>
<label className="row" style={{ gap: 6 }}>
<input type="checkbox" checked={mock} onChange={(e) => setMock(e.target.checked)} />
mock flag
</label>
</div>
<div className="row">
@@ -271,26 +429,42 @@
</label>
<label className="row" style={{ gap: 6 }}>
Character Preset:
<input
type="text"
value={characterPreset}
onChange={(e) => setCharacterPreset(e.target.value)}
placeholder="例如:黑发短发、穿风衣的年轻侦探、冷静目光"
style={{ width: "min(640px, 100%)", padding: "10px 12px", fontSize: 14 }}
/>
<input type="text" value={characterPreset} onChange={(e) => setCharacterPreset(e.target.value)} style={{ width: "min(520px, 100%)", padding: "10px 12px", fontSize: 14 }} />
</label>
<label className="row" style={{ gap: 6 }}>
<input type="checkbox" checked={mock} onChange={(e) => setMock(e.target.checked)} />
mock ComfyUI / Key 也能跑
</label>
<button onClick={startScript}>生成分镜</button>
<button onClick={startScript}>立即重生分镜</button>
<button onClick={stopScript}>停止</button>
{canRender ? (
<button onClick={renderVideo}>确认并开始渲染视频</button>
) : null}
{canRender ? <button onClick={renderVideo}>开始渲染</button> : null}
{taskId ? <span className="muted">task_id: {taskId}</span> : null}
</div>
<div className="row" style={{ marginTop: 10, alignItems: "stretch" }}>
{["Script", "Refine", "Render"].map((s) => (
<button
key={s}
onClick={() => setActiveLogStage(s)}
style={{
flex: 1,
border: `1px solid ${stageColor(stageState[s])}`,
borderRadius: 10,
padding: 10,
textAlign: "left",
background: activeLogStage === s ? "#eef2ff" : "#fff",
cursor: "pointer",
}}
>
<div style={{ fontWeight: 700 }}>{s}</div>
<div className="muted" style={{ color: stageColor(stageState[s]) }}>{stageState[s]}</div>
</button>
))}
</div>
<div style={{ marginTop: 10 }}>
<div className="muted">Render Progress: {(renderProgress * 100).toFixed(0)}%</div>
<div style={{ width: "100%", height: 10, background: "#e5e7eb", borderRadius: 8, overflow: "hidden" }}>
<div style={{ width: `${Math.max(0, Math.min(100, renderProgress * 100))}%`, height: "100%", background: "#2563eb" }} />
</div>
</div>
<div className="scenes">
{scenes.map((s, idx) => (
<div className="card" key={idx}>
@@ -298,21 +472,27 @@
<strong>Scene {idx + 1}</strong>
<button style={{ padding: "6px 10px" }} onClick={() => refineOne(idx + 1)}>🔄 重新润色</button>
</div>
{s && s.preview_url ? (
<div style={{ marginTop: 8 }}>
<img src={s.preview_url + `?t=${Date.now()}`} style={{ width: "100%", borderRadius: 8 }} />
</div>
) : null}
<div className="k">image_prompt</div>
<textarea rows="3" style={{ width: "100%", padding: 8 }}
value={(s && s.image_prompt) || ""}
onChange={(e) => onEdit(idx, "image_prompt", e.target.value)}
/>
<div className="k">video_motion</div>
<textarea rows="2" style={{ width: "100%", padding: 8 }}
value={(s && s.video_motion) || ""}
onChange={(e) => onEdit(idx, "video_motion", e.target.value)}
/>
<textarea rows="3" style={{ width: "100%", padding: 8 }} value={(s && s.image_prompt) || ""} onChange={(e) => onEdit(idx, "image_prompt", e.target.value)} />
<div className="k">motion</div>
<div className="row">
<input type="text" placeholder="camera(如 push-in)" value={(s && s.motion_camera) || ""} onChange={(e) => updateMotionField(idx, "motion_camera", e.target.value)} style={{ padding: 8 }} />
<input type="text" placeholder="direction(如 left->right)" value={(s && s.motion_direction) || ""} onChange={(e) => updateMotionField(idx, "motion_direction", e.target.value)} style={{ padding: 8 }} />
<select value={(s && s.motion_speed) || "normal"} onChange={(e) => updateMotionField(idx, "motion_speed", e.target.value)} style={{ padding: 8 }}>
<option value="slow">slow</option>
<option value="normal">normal</option>
<option value="fast">fast</option>
</select>
</div>
<div className="k">video_motion (raw)</div>
<textarea rows="2" style={{ width: "100%", padding: 8 }} value={(s && s.video_motion) || ""} onChange={(e) => onEdit(idx, "video_motion", e.target.value)} />
<div className="k">narration</div>
<textarea rows="2" style={{ width: "100%", padding: 8 }}
value={(s && s.narration) || ""}
onChange={(e) => onEdit(idx, "narration", e.target.value)}
/>
<textarea rows="2" style={{ width: "100%", padding: 8 }} value={(s && s.narration) || ""} onChange={(e) => onEdit(idx, "narration", e.target.value)} />
</div>
))}
</div>
@@ -323,19 +503,16 @@
<div>
<video controls src={finalVideoUrl}></video>
<div className="row" style={{ marginTop: 10 }}>
<a href={finalVideoUrl} download>
<button>下载视频</button>
</a>
<a href={finalVideoUrl} download><button>下载 final.mp4</button></a>
</div>
<div className="muted">URL: {finalVideoUrl}</div>
</div>
) : (
<div className="muted">尚未渲染完成</div>
)}
</div>
<h3>原始日志stdout/stderr</h3>
<pre ref={logRef}>{logs}</pre>
<h3>分阶段日志{activeLogStage}</h3>
<pre ref={logRef}>{activeLogs}</pre>
{toast ? (
<div className="toast" role="alert">