fix: recover failed Gaussian project builds

src/k1link/simulation/gaussian_pipeline_gateway.py

@@ -29,6 +29,7 @@ IMAGE_DIGEST_PATTERN: Final = re.compile(r"^sha256:[a-f0-9]{64}$")
 DEFAULT_CHUNK_BYTES: Final = 8 * 1024 * 1024
 MAX_JSON_RESPONSE_BYTES: Final = 32 * 1024 * 1024
 MAX_RETRIES: Final = 3
+DEFAULT_INGEST_TIMEOUT_SECONDS: Final = 30 * 60.0
 
 
 class GaussianPipelineGatewayError(RuntimeError):
@@ -108,6 +109,7 @@ class GaussianPipelineGateway:
         token_file: Path,
         *,
         timeout_seconds: float = 30.0,
+        ingest_timeout_seconds: float = DEFAULT_INGEST_TIMEOUT_SECONDS,
         chunk_bytes: int = DEFAULT_CHUNK_BYTES,
         transport: httpx.BaseTransport | None = None,
     ) -> None:
@@ -116,9 +118,14 @@ class GaussianPipelineGateway:
         self.token = _read_token(self.token_file)
         if timeout_seconds <= 0:
             raise GaussianPipelineConfigurationError("provider timeout must be positive")
+        if ingest_timeout_seconds <= 0:
+            raise GaussianPipelineConfigurationError(
+                "provider archive ingest timeout must be positive"
+            )
         if chunk_bytes <= 0 or chunk_bytes > 64 * 1024 * 1024:
             raise GaussianPipelineConfigurationError("provider upload chunk size is invalid")
         self.chunk_bytes = chunk_bytes
+        self.ingest_timeout_seconds = ingest_timeout_seconds
         self._client = httpx.Client(
             base_url=self.endpoint,
             headers={"Authorization": f"Bearer {self.token}"},
@@ -271,6 +278,7 @@ class GaussianPipelineGateway:
                 "schema_version": ARCHIVE_INGEST_REQUEST_SCHEMA,
                 "archive": archive.to_dict(),
             },
+            timeout_seconds=self.ingest_timeout_seconds,
         )
         if document.get("schema_version") != ARCHIVE_INGEST_SCHEMA:
             raise GaussianPipelineGatewayError(
@@ -480,11 +488,22 @@ class GaussianPipelineGateway:
         path: str,
         *,
         document: Mapping[str, object] | None = None,
+        timeout_seconds: float | None = None,
     ) -> dict[str, Any]:
         try:
-            response = self._client.request(method, path, json=document)
+            if timeout_seconds is None:
+                response = self._client.request(method, path, json=document)
+            else:
+                response = self._client.request(
+                    method,
+                    path,
+                    json=document,
+                    timeout=httpx.Timeout(timeout_seconds),
+                )
             response.raise_for_status()
-        except httpx.HTTPError as exc:
+        except httpx.HTTPStatusError as exc:
+            raise _provider_rejection(exc.response) from exc
+        except httpx.TransportError as exc:
             raise _unavailable("Gaussian provider request failed", exc) from exc
         if len(response.content) > MAX_JSON_RESPONSE_BYTES:
             raise GaussianPipelineGatewayError("Gaussian provider JSON response is too large")
@@ -853,3 +872,24 @@ def _unavailable(message: str, error: httpx.HTTPError) -> GaussianPipelineGatewa
     if isinstance(error, httpx.TransportError):
         return GaussianPipelineUnavailableError(message)
     return GaussianPipelineGatewayError(message)
+
+
+def _provider_rejection(response: httpx.Response) -> GaussianPipelineGatewayError:
+    detail: str | None = None
+    if len(response.content) <= MAX_JSON_RESPONSE_BYTES:
+        try:
+            document = response.json()
+        except json.JSONDecodeError:
+            document = None
+        if isinstance(document, dict):
+            raw_detail = document.get("message")
+            if not isinstance(raw_detail, str) or not raw_detail.strip():
+                raw_detail = document.get("error")
+            if isinstance(raw_detail, str):
+                normalized = " ".join(raw_detail.split())
+                if normalized:
+                    detail = normalized[:1024]
+    message = f"Gaussian provider rejected request (HTTP {response.status_code})"
+    if detail is not None:
+        message = f"{message}: {detail}"
+    return GaussianPipelineGatewayError(message)

src/k1link/simulation/projects.py

@@ -239,7 +239,7 @@ class SimulationProjectStore:
     def begin_build(self, project_id: str) -> dict[str, Any]:
         with self._lock:
             document = self._read(project_id)
-            if document["status"] != "uploading":
+            if document["status"] not in {"uploading", "failed"}:
                 raise SimulationProjectConflictError("simulation project cannot start a build")
             if any(
                 int(item["uploaded_bytes"]) != int(item["byte_length"])
@@ -247,6 +247,15 @@ class SimulationProjectStore:
             ):
                 raise SimulationProjectConflictError("simulation source upload is incomplete")
             document["status"] = "queued"
+            document["provider"] = {
+                "provider_id": "gaussian-pipeline",
+                "job_id": None,
+                "state": None,
+                "progress": None,
+                "runtime": None,
+            }
+            document["artifacts"] = []
+            document["world_manifest"] = None
             document["error"] = None
             document["updated_at_utc"] = utc_now_iso()
             self._write(document)
@@ -402,6 +411,22 @@ class SimulationProjectService:
             ).start()
         return len(pending)
 
+    def begin_build(self, project_id: str) -> dict[str, Any]:
+        project = self.store.get(project_id)
+        if project.get("status") == "failed":
+            job_id = project["provider"].get("job_id")
+            if isinstance(job_id, str):
+                provider = self.provider_factory()
+                if provider is None:
+                    raise SimulationProjectConflictError(
+                        "Gaussian Pipeline недоступен для повторной сборки."
+                    )
+                try:
+                    provider.delete_job(job_id)
+                finally:
+                    provider.close()
+        return self.store.begin_build(project_id)
+
     def process(self, project_id: str) -> None:
         provider: GaussianPipelineGateway | None = None
         try: