FIX - NAS DEPLOY: wait for Foundry runtime health

infra/deploy-runner/README.md

@@ -435,6 +435,11 @@ The check is source-aware: if an apply rolls back to the prior source, rollback
 acceptance uses that prior health contract instead of falsely requiring a
 feature which the restored generation does not contain.
 
+Module Foundry activation and rollback wait for the Compose container health
+barrier before strict image/container inventory acceptance. The HTTP health
+endpoint may become ready while Docker still reports `starting`; that timing
+window is not treated as an application or rollback failure.
+
 The Foundry ↔ Map Gateway signing key is not an application or `.env` setting.
 On the first relevant `platform` or `module-foundry` apply, the root-owned
 runner creates `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret`

infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs

@@ -28,7 +28,7 @@ const artifactDir = resolve(
     || resolve(scriptDir, "../deploy-artifacts"),
 );
 const [
-  patchId = "module-foundry-map-runtime-recovery-20260809-002",
+  patchId = "module-foundry-map-runtime-recovery-20260809-003",
   ...extra
 ] = process.argv.slice(2);
 

infra/deploy-runner/nodedc-deploy

@@ -39,7 +39,7 @@ MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
 MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
 MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
 MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = (
-    "module-foundry-map-runtime-recovery-20260809-002"
+    "module-foundry-map-runtime-recovery-20260809-003"
 )
 MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = (
     "package.json",
@@ -56,13 +56,7 @@ MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = (
 MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = (
     "server/map-grid-persistence.test.mjs",
 )
-MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = (
-    "8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b"
-)
-MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = (
-    "b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3"
-)
-MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = (
+MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES = (
     "apps/catalog/src/MapFixturePreview.tsx",
     "apps/catalog/src/mapPresentationProfile.ts",
     "apps/catalog/src/mapSectorGrid.d.mts",
@@ -72,44 +66,47 @@ MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = (
     "scripts/map-presentation-filters.test.mjs",
     "scripts/map-sector-grid.test.mjs",
 )
+MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = (
+    "8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b"
+)
+MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = (
+    "b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3"
+)
+MODULE_FOUNDRY_MAP_FAILED_ENTRIES = MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
 MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = (
-    "module-foundry-map-sector-workspace-20260809-001"
+    "module-foundry-map-runtime-recovery-20260809-002"
 )
 MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = (
-    "nodedc-module-foundry-map-sector-workspace-20260809-001.tgz."
-    "20260809-022707"
+    "nodedc-module-foundry-map-runtime-recovery-20260809-002.tgz."
+    "20260809-092439"
 )
 MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = (
-    "96f67297efe6033f545e797dc4e341983b2ce9ca6ebf6ff48a00f6ee8b8ea23b"
+    "bcaffc5dc6098a6c16f8f9362e6e6a89f8eb2745b898bf1964db8ba1f921f603"
 )
 MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = (
-    "module-foundry-module-foundry-map-sector-workspace-20260809-001-"
-    "20260809-022707"
-)
-MODULE_FOUNDRY_MAP_FAILED_MESSAGE = (
-    "Command '['/usr/local/bin/docker', 'compose', '-p', "
-    "'nodedc-module-foundry', '--env-file', "
-    "'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
-    "'/volume1/docker/nodedc-platform/module-foundry/source/infra/"
-    "docker-compose.module-foundry.yml', 'up', '-d', '--force-recreate', "
-    "'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
-    "exit status 17."
+    "module-foundry-module-foundry-map-runtime-recovery-20260809-002-"
+    "20260809-092439"
 )
+MODULE_FOUNDRY_MAP_FAILED_MESSAGE = "Module Foundry runtime inventory mismatch"
+MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS = "failed:DeployError"
 MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = {
     "existing-files.txt": (
-        "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
+        "eae18c8687a8b163f5b7bcf362c44c0be53a107c08f7c8799ee4dc6af9fb4e16"
     ),
     "files.txt": (
-        "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
+        "cde91865f65a8d79fee66b3872a8067caa0c988a4a61f75e3dac2904ff814b10"
     ),
     "manifest.env": (
-        "fd1ea43c173d89e358731415e528220aa72b1a7c27b6ee8ba408292d8ddf41a2"
+        "cefce93cb613d771a4814af100f9cda7c2c020143268913905a233324edd96d9"
     ),
     "missing-files.txt": (
-        "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+        "bec2dcad43b34b2f040ad08414117e0654238a1bc051b37257f1b52f288ce689"
+    ),
+    "module-foundry-runtime-before.json": (
+        "bf2ec7678af00349c15d6b8dd9d4b0a43e16f427ed92c5adc842c59fab07e4fe"
     ),
     "source-before.tgz": (
-        "e3f14d55d5eaff456a9001fb4a1234b5b584afb93cf4d0c929b30312bd7b5668"
+        "4d34654d32fcd8b328bf83d0f9f5f121fed094fe7bc2e9b1e4b322ccd77da8aa"
     ),
 }
 MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
@@ -17467,7 +17464,8 @@ def validate_module_foundry_map_recovery_evidence(
         or record.get("sha256")
         != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
         or record.get("started_apply") is not True
-        or record.get("rollback_status") != "not-required"
+        or record.get("rollback_status")
+        != MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS
         or record.get("status") != "failed"
         or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE
     ):
@@ -17487,7 +17485,7 @@ def validate_module_foundry_map_recovery_evidence(
             or failed_manifest.get("component") != "module-foundry"
             or failed_manifest.get("type") != "app-overlay"
             or tuple(failed_entries)
-            != MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES
+            != MODULE_FOUNDRY_MAP_FAILED_ENTRIES
         ):
             die("Module Foundry map failed artifact contract mismatch")
         failed_payload_sha256 = collect_exact_files(
@@ -17535,10 +17533,26 @@ def validate_module_foundry_map_recovery_evidence(
         != MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
     ):
         die("Module Foundry partial map predecessor drift detected")
+    installed_failed_payload_sha256 = {
+        rel: digest
+        for rel, digest in failed_payload_sha256.items()
+        if any(
+            rel == installed or rel.startswith(f"{installed}/")
+            for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES
+        )
+    }
+    if any(
+        not any(
+            rel == installed or rel.startswith(f"{installed}/")
+            for rel in installed_failed_payload_sha256
+        )
+        for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES
+    ):
+        die("Module Foundry failed installed overlap evidence mismatch")
     if {
         rel: installed_payload_sha256.get(rel)
-        for rel in failed_payload_sha256
-    } != failed_payload_sha256:
+        for rel in installed_failed_payload_sha256
+    } != installed_failed_payload_sha256:
         die("Module Foundry installed failed map payload drift detected")
 
     runtime = module_foundry_runtime_inventory()
@@ -19492,6 +19506,17 @@ def run_healthchecks(component, entries=None, services=None):
         # successful health probe. Wait for the Compose health barrier before
         # the strict immutable-runtime preflight, both on apply and retry.
         healthcheck_compose_service("engine", "nodedc-backend")
+    if component == "module-foundry":
+        if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
+            die("Module Foundry healthcheck service set mismatch")
+        # The HTTP endpoint can answer before Docker publishes the first
+        # successful health probe. Inventory acceptance is intentionally
+        # strict, so wait for the Compose health barrier on both activation
+        # and rollback before inspecting the immutable runtime identity.
+        healthcheck_compose_service(
+            "module-foundry",
+            MODULE_FOUNDRY_SERVICE,
+        )
     for url in component_healthchecks(component, entries, services):
         healthcheck_url(url)
     if (

infra/deploy-runner/test_module_foundry_runtime_recovery.py

@@ -44,6 +44,45 @@ def runtime(container_char="a", image_char="b"):
 
 
 class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
+    def test_healthchecks_wait_for_compose_health_before_http(self):
+        events = []
+        with (
+            mock.patch.object(
+                RUNNER,
+                "healthcheck_compose_service",
+                side_effect=lambda component, service: events.append(
+                    ("compose", component, service)
+                ),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "component_healthchecks",
+                return_value=("http://foundry/healthz",),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "healthcheck_url",
+                side_effect=lambda check: events.append(("http", check)),
+            ),
+        ):
+            RUNNER.run_healthchecks(
+                "module-foundry",
+                ("package.json",),
+                (RUNNER.MODULE_FOUNDRY_SERVICE,),
+            )
+
+        self.assertEqual(
+            events,
+            [
+                (
+                    "compose",
+                    "module-foundry",
+                    RUNNER.MODULE_FOUNDRY_SERVICE,
+                ),
+                ("http", "http://foundry/healthz"),
+            ],
+        )
+
     def test_build_retries_before_any_runtime_recreate(self):
         first_failure = subprocess.CalledProcessError(17, ["docker", "compose"])
         completed = subprocess.CompletedProcess([], 0)
@@ -290,6 +329,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
             failed_entries = (
                 "apps/catalog/src/MapFixturePreview.tsx",
                 "scripts/map-sector-grid.test.mjs",
+                "server/map-grid-persistence.test.mjs",
             )
             candidate_files = {
                 "apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
@@ -326,6 +366,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
                 "files.txt",
                 "manifest.env",
                 "missing-files.txt",
+                "module-foundry-runtime-before.json",
                 "source-before.tgz",
             }
             for name in backup_names:
@@ -377,7 +418,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
                         "component": "module-foundry",
                         "id": failed_patch_id,
                         "message": failed_message,
-                        "rollback_status": "not-required",
+                        "rollback_status": "failed:DeployError",
                         "sha256": failed_sha256,
                         "started_apply": True,
                         "status": "failed",
@@ -416,18 +457,24 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
                 "MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES": (
                     missing_entries
                 ),
+                "MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES": (
+                    failed_entries[:-1]
+                ),
                 "MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256": (
                     installed_tree
                 ),
                 "MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256": (
                     candidate_tree
                 ),
-                "MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES": failed_entries,
+                "MODULE_FOUNDRY_MAP_FAILED_ENTRIES": failed_entries,
                 "MODULE_FOUNDRY_MAP_FAILED_PATCH_ID": failed_patch_id,
                 "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT": failed_artifact_name,
                 "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256": failed_sha256,
                 "MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID": failed_backup_id,
                 "MODULE_FOUNDRY_MAP_FAILED_MESSAGE": failed_message,
+                "MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS": (
+                    "failed:DeployError"
+                ),
                 "MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256": backup_hashes,
             }
             with ExitStack() as stack: