SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix: recover failed Gaussian project builds

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-26 09:34:23 +03:00
parent cb1e1494af
commit 385082c8ca
10 changed files with 224 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/control-station/src/components/simulation/SimulationProjectWindow.tsx
+1 -1
View File
@@ -111,7 +111,7 @@ export function SimulationProjectWindow({
footer={(
<>
<span className="simulation-project-window__footer-state">
{pending ? <><ActivityIndicator /><span>{project ? "Сохраняем" : "Загружаем источник"}</span></> : null}
{pending && project ? <><ActivityIndicator /><span>Сохраняем</span></> : null}
</span>
<WindowFooterActions>
<Button disabled={pending} onClick={onClose}>Отмена</Button>
apps/control-station/src/core/simulation/projects.ts
+9
View File
@@ -223,6 +223,15 @@ export async function updateSimulationProject(
return parseProject(await jsonResponse(response));
}
export async function retrySimulationProject(projectId: string): Promise<SimulationProject> {
const response = await fetch(`${API_ROOT}/${encodeURIComponent(projectId)}/build`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: "{}",
});
return parseProject(await jsonResponse(response));
}
export async function deleteSimulationProject(projectId: string): Promise<void> {
const response = await fetch(`${API_ROOT}/${encodeURIComponent(projectId)}`, { method: "DELETE" });
if (!response.ok) await jsonResponse(response);
apps/control-station/src/styles/simulation.css
+31 -17
View File
@@ -60,28 +60,30 @@
}
.simulation-catalog {
overflow: hidden;
border: 1px solid var(--station-hairline);
border-radius: 1rem;
background: rgb(255 255 255 / 0.022);
display: grid;
gap: 0.55rem;
min-width: 0;
}
.simulation-catalog__summary {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 1px;
border-bottom: 1px solid var(--station-hairline);
background: var(--station-hairline);
gap: 0.45rem;
}
.simulation-catalog__summary > div {
display: grid;
gap: 0.24rem;
background: var(--nodedc-canvas);
padding: 0.85rem 1rem;
gap: 0.28rem;
border-radius: 0.85rem;
background: rgb(255 255 255 / 0.03);
padding: 0.85rem;
}
.simulation-catalog__summary span {
color: var(--nodedc-text-muted);
font-size: 0.58rem;
}
.simulation-catalog__summary span,
.simulation-catalog__table th {
color: var(--nodedc-text-muted);
font-size: 0.52rem;
@@ -92,11 +94,13 @@
.simulation-catalog__summary strong {
color: var(--nodedc-text-primary);
font-size: 0.88rem;
font-size: 1.15rem;
}
.simulation-catalog__table-wrap {
overflow: auto;
border-radius: 1rem;
background: rgb(255 255 255 / 0.025);
}
.simulation-catalog__table {
@@ -178,7 +182,6 @@
height: 2rem;
flex: 0 0 auto;
place-items: center;
border: 1px solid var(--station-hairline);
border-radius: 0.62rem;
background: rgb(var(--nodedc-accent-rgb) / 0.07);
color: var(--nodedc-text-secondary);
@@ -195,6 +198,8 @@
justify-content: center;
gap: 0.75rem;
color: var(--nodedc-text-muted);
border-radius: 1rem;
background: rgb(255 255 255 / 0.025);
font-size: 0.65rem;
}
@@ -275,7 +280,8 @@
.simulation-upload-progress > span {
position: relative;
overflow: hidden;
width: 8rem;
min-width: 12rem;
flex: 1 1 auto;
height: 0.28rem;
border-radius: 999px;
background: rgb(255 255 255 / 0.08);
@@ -290,7 +296,9 @@
.simulation-upload-progress > div {
display: grid;
min-width: 0;
min-width: 9rem;
max-width: 13rem;
flex: 0 1 13rem;
gap: 0.14rem;
}
@@ -328,7 +336,6 @@
min-height: 0;
grid-template-rows: auto minmax(0, 1fr) auto;
overflow: hidden;
border: 1px solid var(--station-hairline);
border-radius: 1rem;
background: #07090d;
}
@@ -408,12 +415,19 @@
gap: 0.8rem;
}
.simulation-workspace__processing > div {
.simulation-workspace__processing-copy {
display: grid;
max-width: 40rem;
gap: 0.25rem;
}
.simulation-workspace__processing-actions {
display: flex;
flex: 0 0 auto;
align-items: center;
gap: 0.55rem;
}
.simulation-workspace__processing strong {
color: var(--nodedc-text-primary);
font-size: 0.75rem;
apps/control-station/src/workspaces/simulation/SimulationWorkspace.tsx
+37 -5
View File
@@ -14,6 +14,7 @@ import { SimulationViewport } from "../../components/simulation/SimulationViewpo
import {
deleteSimulationProject,
fetchSimulationProjects,
retrySimulationProject,
type SimulationProject,
type SimulationProjectStatus,
} from "../../core/simulation/projects";
@@ -28,6 +29,7 @@ export function SimulationWorkspace() {
const [windowOpen, setWindowOpen] = useState(false);
const [editing, setEditing] = useState<SimulationProject | null>(null);
const [deleting, setDeleting] = useState<SimulationProject | null>(null);
const [retryingId, setRetryingId] = useState<string | null>(null);
const load = useCallback(async (signal?: AbortSignal) => {
try {
@@ -84,6 +86,21 @@ export function SimulationWorkspace() {
setDeleting(null);
};
const retry = async (project: SimulationProject) => {
setRetryingId(project.projectId);
setError(null);
try {
const queued = await retrySimulationProject(project.projectId);
setProjects((current) => current.map((item) => (
item.projectId === queued.projectId ? queued : item
)));
} catch (caught) {
setError(caught instanceof Error ? caught.message : "Не удалось повторить сборку.");
} finally {
setRetryingId(null);
}
};
if (selected) {
return (
<div className="simulation-workspace simulation-workspace--scene">
@@ -109,13 +126,28 @@ export function SimulationWorkspace() {
) : (
<GlassSurface className="simulation-workspace__processing" padding="lg">
{selected.status === "failed" ? <Icon name="alert" size={20} /> : <ActivityIndicator label="Сборка мира" />}
<div>
<div className="simulation-workspace__processing-copy">
<strong>{selected.status === "failed" ? "Сборка остановлена" : "Worker 006 собирает мир"}</strong>
<p>{selected.error ?? processingMessage(selected)}</p>
<p>{error ?? selected.error ?? processingMessage(selected)}</p>
</div>
<div className="simulation-workspace__processing-actions">
<StatusBadge tone={selected.status === "failed" ? "warning" : "accent"}>
{selected.provider.state ?? selected.status}
</StatusBadge>
{selected.status === "failed" ? (
<Button
size="compact"
variant="secondary"
disabled={retryingId === selected.projectId}
icon={retryingId === selected.projectId
? <ActivityIndicator size="compact" />
: <Icon name="refresh" size={14} />}
onClick={() => void retry(selected)}
>
Повторить сборку
</Button>
) : null}
</div>
<StatusBadge tone={selected.status === "failed" ? "warning" : "accent"}>
{selected.provider.state ?? selected.status}
</StatusBadge>
</GlassSurface>
)}
</div>
apps/control-station/test/simulationWorkspace.test.mjs
+12
View File
@@ -39,6 +39,18 @@ test("browser upload remains same-origin, resumable and token-free", async () =>
assert.match(window, /webkitdirectory/);
assert.match(window, /webkitGetAsEntry/);
assert.match(window, /ровно одна сцена \.lcc или \.lcc2/);
assert.doesNotMatch(window, /Загружаем источник/);
});
test("failed simulation can restart from its retained Mission Core source", async () => {
const [core, workspace] = await Promise.all([
read("core/simulation/projects.ts"),
read("workspaces/simulation/SimulationWorkspace.tsx"),
]);
assert.match(core, /retrySimulationProject/);
assert.match(core, /\$\{API_ROOT\}\/\$\{encodeURIComponent\(projectId\)\}\/build/);
assert.match(workspace, /Повторить сборку/);
});
test("PlayCanvas owns the realtime scene graph without an iframe or React entity tree", async () => {
src/k1link/simulation/gaussian_pipeline_gateway.py
+42 -2
View File
@@ -29,6 +29,7 @@ IMAGE_DIGEST_PATTERN: Final = re.compile(r"^sha256:[a-f0-9]{64}$")
DEFAULT_CHUNK_BYTES: Final = 8 * 1024 * 1024
MAX_JSON_RESPONSE_BYTES: Final = 32 * 1024 * 1024
MAX_RETRIES: Final = 3
DEFAULT_INGEST_TIMEOUT_SECONDS: Final = 30 * 60.0
class GaussianPipelineGatewayError(RuntimeError):
@@ -108,6 +109,7 @@ class GaussianPipelineGateway:
token_file: Path,
*,
timeout_seconds: float = 30.0,
ingest_timeout_seconds: float = DEFAULT_INGEST_TIMEOUT_SECONDS,
chunk_bytes: int = DEFAULT_CHUNK_BYTES,
transport: httpx.BaseTransport | None = None,
) -> None:
@@ -116,9 +118,14 @@ class GaussianPipelineGateway:
self.token = _read_token(self.token_file)
if timeout_seconds <= 0:
raise GaussianPipelineConfigurationError("provider timeout must be positive")
if ingest_timeout_seconds <= 0:
raise GaussianPipelineConfigurationError(
"provider archive ingest timeout must be positive"
)
if chunk_bytes <= 0 or chunk_bytes > 64 * 1024 * 1024:
raise GaussianPipelineConfigurationError("provider upload chunk size is invalid")
self.chunk_bytes = chunk_bytes
self.ingest_timeout_seconds = ingest_timeout_seconds
self._client = httpx.Client(
base_url=self.endpoint,
headers={"Authorization": f"Bearer {self.token}"},
@@ -271,6 +278,7 @@ class GaussianPipelineGateway:
"schema_version": ARCHIVE_INGEST_REQUEST_SCHEMA,
"archive": archive.to_dict(),
},
timeout_seconds=self.ingest_timeout_seconds,
)
if document.get("schema_version") != ARCHIVE_INGEST_SCHEMA:
raise GaussianPipelineGatewayError(
@@ -480,11 +488,22 @@ class GaussianPipelineGateway:
path: str,
*,
document: Mapping[str, object] | None = None,
timeout_seconds: float | None = None,
) -> dict[str, Any]:
try:
response = self._client.request(method, path, json=document)
if timeout_seconds is None:
response = self._client.request(method, path, json=document)
else:
response = self._client.request(
method,
path,
json=document,
timeout=httpx.Timeout(timeout_seconds),
)
response.raise_for_status()
except httpx.HTTPError as exc:
except httpx.HTTPStatusError as exc:
raise _provider_rejection(exc.response) from exc
except httpx.TransportError as exc:
raise _unavailable("Gaussian provider request failed", exc) from exc
if len(response.content) > MAX_JSON_RESPONSE_BYTES:
raise GaussianPipelineGatewayError("Gaussian provider JSON response is too large")
@@ -853,3 +872,24 @@ def _unavailable(message: str, error: httpx.HTTPError) -> GaussianPipelineGatewa
if isinstance(error, httpx.TransportError):
return GaussianPipelineUnavailableError(message)
return GaussianPipelineGatewayError(message)
def _provider_rejection(response: httpx.Response) -> GaussianPipelineGatewayError:
detail: str | None = None
if len(response.content) <= MAX_JSON_RESPONSE_BYTES:
try:
document = response.json()
except json.JSONDecodeError:
document = None
if isinstance(document, dict):
raw_detail = document.get("message")
if not isinstance(raw_detail, str) or not raw_detail.strip():
raw_detail = document.get("error")
if isinstance(raw_detail, str):
normalized = " ".join(raw_detail.split())
if normalized:
detail = normalized[:1024]
message = f"Gaussian provider rejected request (HTTP {response.status_code})"
if detail is not None:
message = f"{message}: {detail}"
return GaussianPipelineGatewayError(message)
src/k1link/simulation/projects.py
+26 -1
View File
@@ -239,7 +239,7 @@ class SimulationProjectStore:
def begin_build(self, project_id: str) -> dict[str, Any]:
with self._lock:
document = self._read(project_id)
if document["status"] != "uploading":
if document["status"] not in {"uploading", "failed"}:
raise SimulationProjectConflictError("simulation project cannot start a build")
if any(
int(item["uploaded_bytes"]) != int(item["byte_length"])
@@ -247,6 +247,15 @@ class SimulationProjectStore:
):
raise SimulationProjectConflictError("simulation source upload is incomplete")
document["status"] = "queued"
document["provider"] = {
"provider_id": "gaussian-pipeline",
"job_id": None,
"state": None,
"progress": None,
"runtime": None,
}
document["artifacts"] = []
document["world_manifest"] = None
document["error"] = None
document["updated_at_utc"] = utc_now_iso()
self._write(document)
@@ -402,6 +411,22 @@ class SimulationProjectService:
).start()
return len(pending)
def begin_build(self, project_id: str) -> dict[str, Any]:
project = self.store.get(project_id)
if project.get("status") == "failed":
job_id = project["provider"].get("job_id")
if isinstance(job_id, str):
provider = self.provider_factory()
if provider is None:
raise SimulationProjectConflictError(
"Gaussian Pipeline недоступен для повторной сборки."
)
try:
provider.delete_job(job_id)
finally:
provider.close()
return self.store.begin_build(project_id)
def process(self, project_id: str) -> None:
provider: GaussianPipelineGateway | None = None
try:
src/k1link/web/simulation_projects_api.py
+3 -1
View File
@@ -176,13 +176,15 @@ def build_simulation_projects_router(
background_tasks: BackgroundTasks,
) -> dict[str, Any]:
try:
project = store.begin_build(project_id)
project = service.begin_build(project_id)
background_tasks.add_task(service.process, project_id)
return project
except SimulationProjectNotFoundError as exc:
raise HTTPException(status_code=404, detail="Проект симуляции не найден.") from exc
except SimulationProjectConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except GaussianPipelineGatewayError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
@router.delete("/projects/{project_id}", status_code=204)
def delete_project(project_id: str) -> Response:
tests/test_gaussian_pipeline_gateway.py
+32
View File
@@ -10,6 +10,7 @@ import pytest
from k1link.simulation.gaussian_pipeline_gateway import (
BUILD_REQUEST_SCHEMA,
GaussianArchiveUpload,
GaussianPipelineGateway,
GaussianPipelineGatewayError,
GaussianPipelineIntegrityError,
@@ -163,6 +164,7 @@ def test_gateway_uploads_and_normalizes_archive_with_tus(tmp_path: Path) -> None
received.extend(request.content)
return httpx.Response(204, headers={"Upload-Offset": str(len(received))})
if request.method == "POST" and request.url.path == "/v1/ingests":
assert request.extensions["timeout"]["read"] == 30 * 60.0
submitted = json.loads(request.content)
assert submitted["archive"] == {
"upload_id": "archive-001",
@@ -202,6 +204,36 @@ def test_gateway_uploads_and_normalizes_archive_with_tus(tmp_path: Path) -> None
assert source.bundle_sha256 == bundle_sha
def test_gateway_surfaces_bounded_provider_rejection_detail(tmp_path: Path) -> None:
def handler(_request: httpx.Request) -> httpx.Response:
return httpx.Response(
400,
json={
"error": "archive_link",
"message": "archive links are not supported",
},
)
with (
GaussianPipelineGateway(
"http://gaussian.test",
_token_file(tmp_path),
transport=httpx.MockTransport(handler),
) as gateway,
pytest.raises(
GaussianPipelineGatewayError,
match=r"HTTP 400.*archive links are not supported",
),
):
gateway.normalize_archive(GaussianArchiveUpload(
upload_id="archive-001",
archive_name="source.rar",
format="rar",
sha256="a" * 64,
byte_length=128,
))
def test_gateway_rejects_incomplete_lcc_bundle(tmp_path: Path) -> None:
bundle = tmp_path / "bundle"
bundle.mkdir()
tests/test_simulation_projects.py
+31
View File
@@ -228,6 +228,37 @@ def test_service_resumes_a_persisted_provider_job_without_reupload(tmp_path: Pat
assert provider.submit_calls == 0
def test_failed_project_retries_from_retained_source_and_releases_old_job(tmp_path: Path) -> None:
store = SimulationProjectStore(tmp_path)
project = store.create(
name="Retry scene",
scene_type="outdoor",
source_kind="folder",
files=_folder_files(),
)
_upload_all(store, project)
store.begin_build(project["project_id"])
store.update_processing(
project["project_id"],
status="processing",
provider_job_id="gsp-20260826000000-deadbeef",
provider_state="failed",
progress={"completed_steps": 1, "total_steps": 4},
bundle_sha256="d" * 64,
)
store.fail(project["project_id"], "provider failure")
provider = _ReadyProvider()
service = SimulationProjectService(store, provider_factory=lambda: provider) # type: ignore[arg-type]
queued = service.begin_build(project["project_id"])
assert provider.deleted == ["gsp-20260826000000-deadbeef"]
assert queued["status"] == "queued"
assert queued["error"] is None
assert queued["provider"]["job_id"] is None
assert queued["source"]["uploaded_byte_length"] == queued["source"]["total_byte_length"]
def test_api_exposes_same_origin_resumable_upload_contract(tmp_path: Path) -> None:
store = SimulationProjectStore(tmp_path)
service = SimulationProjectService(store, provider_factory=lambda: None)