fix(simulation): retry builds across worker tunnel outages

tests/test_simulation_projects.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 from pathlib import Path
 from threading import Event
+from time import monotonic, sleep
 from typing import Any
 
 import pytest
@@ -444,6 +445,55 @@ def test_service_queue_processes_projects_strictly_one_at_a_time(tmp_path: Path)
     assert order == [projects[0]["project_id"], projects[1]["project_id"]]
 
 
+def test_service_keeps_retained_source_queued_across_temporary_worker_outage(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    store = SimulationProjectStore(tmp_path)
+    project = store.create(
+        name="Reconnect without browser reupload",
+        scene_type="outdoor",
+        source_kind="folder",
+        files=_folder_files(),
+    )
+    _upload_all(store, project)
+    store.begin_build(project["project_id"])
+
+    class _ReconnectProvider(_ReadyProvider):
+        def __init__(self) -> None:
+            super().__init__()
+            self.capability_calls = 0
+
+        def capabilities(self) -> dict[str, object]:
+            self.capability_calls += 1
+            if self.capability_calls == 1:
+                raise GaussianPipelineUnavailableError("temporary tunnel failure")
+            return super().capabilities()
+
+    provider = _ReconnectProvider()
+    monkeypatch.setattr(
+        "k1link.simulation.projects.PROVIDER_UNAVAILABLE_RETRY_LIMIT",
+        1,
+    )
+    service = SimulationProjectService(
+        store,
+        provider_factory=lambda: provider,
+    )  # type: ignore[arg-type]
+
+    service.enqueue(project["project_id"])
+
+    deadline = monotonic() + 1.0
+    while store.get(project["project_id"])["status"] != "ready" and monotonic() < deadline:
+        sleep(0.01)
+    recovered = store.get(project["project_id"])
+    assert recovered["status"] == "ready"
+    assert recovered["error"] is None
+    assert recovered["source"]["uploaded_byte_length"] == recovered["source"]["total_byte_length"]
+    assert provider.capability_calls == 2
+    assert provider.upload_calls == 1
+    assert provider.submit_calls == 1
+
+
 def test_service_deletes_a_queued_project_before_worker_submission(tmp_path: Path) -> None:
     store = SimulationProjectStore(tmp_path)
     project = store.create(