fix(simulation): retry builds across worker tunnel outages

src/k1link/simulation/projects.py

@@ -14,7 +14,7 @@ from collections import deque
 from collections.abc import Callable
 from contextlib import suppress
 from pathlib import Path
-from typing import Any, Final, TypeVar
+from typing import Any, Final
 from urllib.parse import quote
 from uuid import uuid4
 
@@ -55,7 +55,6 @@ PROVIDER_JOB_STATES: Final = {
 }
 PROVIDER_POLL_TIMEOUT_SECONDS: Final = 2 * 60 * 60 + 5 * 60
 PROVIDER_UNAVAILABLE_RETRY_LIMIT: Final = 6
-_T = TypeVar("_T")
 
 
 class SimulationProjectError(RuntimeError):
@@ -722,6 +721,8 @@ class SimulationProjectService:
             )
         except _SimulationProcessingCancelled:
             pass
+        except GaussianPipelineUnavailableError:
+            self._requeue_provider_unavailable(project_id)
         except (GaussianPipelineGatewayError, SimulationProjectError, OSError) as exc:
             with suppress(SimulationProjectError):
                 self.store.fail(project_id, str(exc))
@@ -731,6 +732,21 @@ class SimulationProjectService:
             if provider is not None:
                 provider.close()
 
+    def _requeue_provider_unavailable(self, project_id: str) -> None:
+        """Keep a retained source pending while its worker transport is unavailable."""
+        try:
+            self.store.update_processing(project_id, status="queued")
+        except SimulationProjectError:
+            return
+        with self._condition:
+            cancel = self._cancel_events.get(project_id)
+            if cancel is not None and cancel.is_set():
+                return
+            if self._active_project_id == project_id and project_id not in self._queued_ids:
+                self._queue.append(project_id)
+                self._queued_ids.add(project_id)
+                self._condition.notify_all()
+
     def delete(self, project_id: str) -> None:
         project = self.store.get(project_id)
         with self._condition:
@@ -794,7 +810,7 @@ class SimulationProjectService:
             raise _SimulationProcessingCancelled(project_id)
 
 
-def _retry_provider_unavailable(operation: Callable[[], _T]) -> _T:
+def _retry_provider_unavailable[T](operation: Callable[[], T]) -> T:
     delay_seconds = 1.0
     for attempt in range(PROVIDER_UNAVAILABLE_RETRY_LIMIT):
         try: