From b511e92cf979f84300de5f9c29809690792c4238 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 9 Aug 2026 09:44:27 +0300 Subject: [PATCH] FIX - NAS DEPLOY: wait for Foundry runtime health --- infra/deploy-runner/README.md | 5 ++ ...-foundry-map-runtime-recovery-artifact.mjs | 2 +- infra/deploy-runner/nodedc-deploy | 89 ++++++++++++------- .../test_module_foundry_runtime_recovery.py | 51 ++++++++++- 4 files changed, 112 insertions(+), 35 deletions(-) diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index 7ccf280..e02c2b4 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -435,6 +435,11 @@ The check is source-aware: if an apply rolls back to the prior source, rollback acceptance uses that prior health contract instead of falsely requiring a feature which the restored generation does not contain. +Module Foundry activation and rollback wait for the Compose container health +barrier before strict image/container inventory acceptance. The HTTP health +endpoint may become ready while Docker still reports `starting`; that timing +window is not treated as an application or rollback failure. + The Foundry ↔ Map Gateway signing key is not an application or `.env` setting. On the first relevant `platform` or `module-foundry` apply, the root-owned runner creates `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret` diff --git a/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs b/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs index 01fcafd..7fa2296 100644 --- a/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs +++ b/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs @@ -28,7 +28,7 @@ const artifactDir = resolve( || resolve(scriptDir, "../deploy-artifacts"), ); const [ - patchId = "module-foundry-map-runtime-recovery-20260809-002", + patchId = "module-foundry-map-runtime-recovery-20260809-003", ...extra ] = process.argv.slice(2); diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 2ff2403..33ff770 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -39,7 +39,7 @@ MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry" MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json" MODULE_FOUNDRY_BUILD_ATTEMPTS = 2 MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = ( - "module-foundry-map-runtime-recovery-20260809-002" + "module-foundry-map-runtime-recovery-20260809-003" ) MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = ( "package.json", @@ -56,13 +56,7 @@ MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = ( MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = ( "server/map-grid-persistence.test.mjs", ) -MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = ( - "8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b" -) -MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = ( - "b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3" -) -MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = ( +MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES = ( "apps/catalog/src/MapFixturePreview.tsx", "apps/catalog/src/mapPresentationProfile.ts", "apps/catalog/src/mapSectorGrid.d.mts", @@ -72,44 +66,47 @@ MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = ( "scripts/map-presentation-filters.test.mjs", "scripts/map-sector-grid.test.mjs", ) +MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = ( + "8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b" +) +MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = ( + "b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3" +) +MODULE_FOUNDRY_MAP_FAILED_ENTRIES = MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = ( - "module-foundry-map-sector-workspace-20260809-001" + "module-foundry-map-runtime-recovery-20260809-002" ) MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = ( - "nodedc-module-foundry-map-sector-workspace-20260809-001.tgz." - "20260809-022707" + "nodedc-module-foundry-map-runtime-recovery-20260809-002.tgz." + "20260809-092439" ) MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = ( - "96f67297efe6033f545e797dc4e341983b2ce9ca6ebf6ff48a00f6ee8b8ea23b" + "bcaffc5dc6098a6c16f8f9362e6e6a89f8eb2745b898bf1964db8ba1f921f603" ) MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = ( - "module-foundry-module-foundry-map-sector-workspace-20260809-001-" - "20260809-022707" -) -MODULE_FOUNDRY_MAP_FAILED_MESSAGE = ( - "Command '['/usr/local/bin/docker', 'compose', '-p', " - "'nodedc-module-foundry', '--env-file', " - "'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', " - "'/volume1/docker/nodedc-platform/module-foundry/source/infra/" - "docker-compose.module-foundry.yml', 'up', '-d', '--force-recreate', " - "'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero " - "exit status 17." + "module-foundry-module-foundry-map-runtime-recovery-20260809-002-" + "20260809-092439" ) +MODULE_FOUNDRY_MAP_FAILED_MESSAGE = "Module Foundry runtime inventory mismatch" +MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS = "failed:DeployError" MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = { "existing-files.txt": ( - "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3" + "eae18c8687a8b163f5b7bcf362c44c0be53a107c08f7c8799ee4dc6af9fb4e16" ), "files.txt": ( - "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3" + "cde91865f65a8d79fee66b3872a8067caa0c988a4a61f75e3dac2904ff814b10" ), "manifest.env": ( - "fd1ea43c173d89e358731415e528220aa72b1a7c27b6ee8ba408292d8ddf41a2" + "cefce93cb613d771a4814af100f9cda7c2c020143268913905a233324edd96d9" ), "missing-files.txt": ( - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + "bec2dcad43b34b2f040ad08414117e0654238a1bc051b37257f1b52f288ce689" + ), + "module-foundry-runtime-before.json": ( + "bf2ec7678af00349c15d6b8dd9d4b0a43e16f427ed92c5adc842c59fab07e4fe" ), "source-before.tgz": ( - "e3f14d55d5eaff456a9001fb4a1234b5b584afb93cf4d0c929b30312bd7b5668" + "4d34654d32fcd8b328bf83d0f9f5f121fed094fe7bc2e9b1e4b322ccd77da8aa" ), } MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets") @@ -17467,7 +17464,8 @@ def validate_module_foundry_map_recovery_evidence( or record.get("sha256") != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 or record.get("started_apply") is not True - or record.get("rollback_status") != "not-required" + or record.get("rollback_status") + != MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS or record.get("status") != "failed" or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE ): @@ -17487,7 +17485,7 @@ def validate_module_foundry_map_recovery_evidence( or failed_manifest.get("component") != "module-foundry" or failed_manifest.get("type") != "app-overlay" or tuple(failed_entries) - != MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES + != MODULE_FOUNDRY_MAP_FAILED_ENTRIES ): die("Module Foundry map failed artifact contract mismatch") failed_payload_sha256 = collect_exact_files( @@ -17535,10 +17533,26 @@ def validate_module_foundry_map_recovery_evidence( != MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 ): die("Module Foundry partial map predecessor drift detected") + installed_failed_payload_sha256 = { + rel: digest + for rel, digest in failed_payload_sha256.items() + if any( + rel == installed or rel.startswith(f"{installed}/") + for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES + ) + } + if any( + not any( + rel == installed or rel.startswith(f"{installed}/") + for rel in installed_failed_payload_sha256 + ) + for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES + ): + die("Module Foundry failed installed overlap evidence mismatch") if { rel: installed_payload_sha256.get(rel) - for rel in failed_payload_sha256 - } != failed_payload_sha256: + for rel in installed_failed_payload_sha256 + } != installed_failed_payload_sha256: die("Module Foundry installed failed map payload drift detected") runtime = module_foundry_runtime_inventory() @@ -19492,6 +19506,17 @@ def run_healthchecks(component, entries=None, services=None): # successful health probe. Wait for the Compose health barrier before # the strict immutable-runtime preflight, both on apply and retry. healthcheck_compose_service("engine", "nodedc-backend") + if component == "module-foundry": + if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,): + die("Module Foundry healthcheck service set mismatch") + # The HTTP endpoint can answer before Docker publishes the first + # successful health probe. Inventory acceptance is intentionally + # strict, so wait for the Compose health barrier on both activation + # and rollback before inspecting the immutable runtime identity. + healthcheck_compose_service( + "module-foundry", + MODULE_FOUNDRY_SERVICE, + ) for url in component_healthchecks(component, entries, services): healthcheck_url(url) if ( diff --git a/infra/deploy-runner/test_module_foundry_runtime_recovery.py b/infra/deploy-runner/test_module_foundry_runtime_recovery.py index 9a4da79..67f0ab9 100644 --- a/infra/deploy-runner/test_module_foundry_runtime_recovery.py +++ b/infra/deploy-runner/test_module_foundry_runtime_recovery.py @@ -44,6 +44,45 @@ def runtime(container_char="a", image_char="b"): class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): + def test_healthchecks_wait_for_compose_health_before_http(self): + events = [] + with ( + mock.patch.object( + RUNNER, + "healthcheck_compose_service", + side_effect=lambda component, service: events.append( + ("compose", component, service) + ), + ), + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=("http://foundry/healthz",), + ), + mock.patch.object( + RUNNER, + "healthcheck_url", + side_effect=lambda check: events.append(("http", check)), + ), + ): + RUNNER.run_healthchecks( + "module-foundry", + ("package.json",), + (RUNNER.MODULE_FOUNDRY_SERVICE,), + ) + + self.assertEqual( + events, + [ + ( + "compose", + "module-foundry", + RUNNER.MODULE_FOUNDRY_SERVICE, + ), + ("http", "http://foundry/healthz"), + ], + ) + def test_build_retries_before_any_runtime_recreate(self): first_failure = subprocess.CalledProcessError(17, ["docker", "compose"]) completed = subprocess.CompletedProcess([], 0) @@ -290,6 +329,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): failed_entries = ( "apps/catalog/src/MapFixturePreview.tsx", "scripts/map-sector-grid.test.mjs", + "server/map-grid-persistence.test.mjs", ) candidate_files = { "apps/catalog/src/MapFixturePreview.tsx": "sector-current\n", @@ -326,6 +366,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): "files.txt", "manifest.env", "missing-files.txt", + "module-foundry-runtime-before.json", "source-before.tgz", } for name in backup_names: @@ -377,7 +418,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): "component": "module-foundry", "id": failed_patch_id, "message": failed_message, - "rollback_status": "not-required", + "rollback_status": "failed:DeployError", "sha256": failed_sha256, "started_apply": True, "status": "failed", @@ -416,18 +457,24 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): "MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES": ( missing_entries ), + "MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES": ( + failed_entries[:-1] + ), "MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256": ( installed_tree ), "MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256": ( candidate_tree ), - "MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES": failed_entries, + "MODULE_FOUNDRY_MAP_FAILED_ENTRIES": failed_entries, "MODULE_FOUNDRY_MAP_FAILED_PATCH_ID": failed_patch_id, "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT": failed_artifact_name, "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256": failed_sha256, "MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID": failed_backup_id, "MODULE_FOUNDRY_MAP_FAILED_MESSAGE": failed_message, + "MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS": ( + "failed:DeployError" + ), "MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256": backup_hashes, } with ExitStack() as stack: