From 0565cd26421d00ca4312d818800bab259063b93b Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 9 Aug 2026 02:18:07 +0300 Subject: [PATCH] FIX - NAS DEPLOY: recover Module Foundry runtime safely --- infra/deploy-runner/nodedc-deploy | 558 ++++++++++++++++++ .../test_module_foundry_runtime_recovery.py | 403 +++++++++++++ 2 files changed, 961 insertions(+) create mode 100644 infra/deploy-runner/test_module_foundry_runtime_recovery.py diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index df125a9..4ffe3e1 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -35,6 +35,62 @@ STATE_FILE = STATE_DIR / "applied.jsonl" FAILED_STATE_FILE = STATE_DIR / "failed.jsonl" LOCK_DIR = STATE_DIR / "deploy.lock" DOCKER = Path("/usr/local/bin/docker") +MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry" +MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json" +MODULE_FOUNDRY_BUILD_ATTEMPTS = 2 +MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = ( + "module-foundry-map-sector-workspace-20260809-001" +) +MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = ( + "apps/catalog/src/MapFixturePreview.tsx", + "apps/catalog/src/mapPresentationProfile.ts", + "apps/catalog/src/mapSectorGrid.d.mts", + "apps/catalog/src/mapSectorGrid.mjs", + "apps/catalog/src/styles.css", + "scripts/map-object-layers.test.mjs", + "scripts/map-presentation-filters.test.mjs", + "scripts/map-sector-grid.test.mjs", +) +MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = ( + "module-foundry-map-sector-workspace-20260808-001" +) +MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = ( + "nodedc-module-foundry-map-sector-workspace-20260808-001.tgz." + "20260809-014720" +) +MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = ( + "0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740" +) +MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = ( + "module-foundry-module-foundry-map-sector-workspace-20260808-001-" + "20260809-014720" +) +MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = ( + "Command '['/usr/local/bin/docker', 'compose', '-p', " + "'nodedc-module-foundry', '--env-file', " + "'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', " + "'/volume1/docker/nodedc-platform/module-foundry/source/infra/" + "docker-compose.module-foundry.yml', 'up', '-d', '--force-recreate', " + "'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero " + "exit status 17." +) +MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = { + "existing-files.txt": ( + "57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c" + ), + "files.txt": ( + "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3" + ), + "manifest.env": ( + "cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2" + ), + "missing-files.txt": ( + "1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f" + ), + "source-before.tgz": ( + "bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562" + ), +} MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets") MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret" MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token" @@ -14961,11 +15017,19 @@ def plan_artifact(artifact): provider_catalog_preflight = None device_plane_postgres_preflight = None device_plane_foundation_recovery_preflight = None + module_foundry_sector_recovery_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_device_plane_foundation_artifact(manifest, sha) reject_terminal_device_plane_backhaul_artifact(manifest, sha) + module_foundry_sector_recovery_preflight = ( + validate_module_foundry_sector_recovery_evidence( + manifest, + entries, + payload_dir, + ) + ) transition_descriptor = None transition_preflight = None if is_engine_n8n_transition(manifest["component"], entries): @@ -16520,6 +16584,37 @@ def plan_artifact(artifact): print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}") print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}") print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}") + print( + "module_foundry_build=" + f"separate-before-recreate:attempts={MODULE_FOUNDRY_BUILD_ATTEMPTS}" + ) + print("module_foundry_runtime_pull=never") + print("module_foundry_rollback=source+image+runtime") + if module_foundry_sector_recovery_preflight is not None: + print( + "module_foundry_transition=" + f"{module_foundry_sector_recovery_preflight['mode']}" + ) + print( + "module_foundry_failed_predecessor=" + f"{module_foundry_sector_recovery_preflight['failed_patch_id']}" + ) + print( + "module_foundry_failed_predecessor_sha256=" + f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}" + ) + print( + "module_foundry_failed_backup=" + f"{module_foundry_sector_recovery_preflight['failed_backup_id']}" + ) + print( + "module_foundry_predecessor_container=" + f"{module_foundry_sector_recovery_preflight['runtime_container_id']}" + ) + print( + "module_foundry_predecessor_image=" + f"{module_foundry_sector_recovery_preflight['runtime_image_id']}" + ) if component == "device-plane": print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}") print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}") @@ -17241,6 +17336,236 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star return f"source+runtime-restored:{restored_count}" +def is_module_foundry_sector_recovery_slice(manifest, entries): + return ( + manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID + and manifest.get("component") == "module-foundry" + and manifest.get("type") == "app-overlay" + and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES + ) + + +def validate_module_foundry_sector_recovery_evidence( + manifest, + entries, + payload_dir, +): + if not is_module_foundry_sector_recovery_slice(manifest, entries): + return None + + backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Module Foundry sector recovery backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Module Foundry sector recovery backup is unsafe") + if {child.name for child in backup_dir.iterdir()} != set( + MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 + ): + die("Module Foundry sector recovery backup file set mismatch") + for name, expected_sha256 in ( + MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items() + ): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Module Foundry sector recovery backup drift detected: " + f"{name}" + ) + + failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Module Foundry sector failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 + ): + die("Module Foundry sector failed artifact evidence mismatch") + + try: + state_stat = FAILED_STATE_FILE.lstat() + state_lines = FAILED_STATE_FILE.read_text( + encoding="utf-8" + ).splitlines() + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Module Foundry sector failed journal is unreadable") + if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG( + state_stat.st_mode + ): + die("Module Foundry sector failed journal is unsafe") + records = [] + for line in state_lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + die("Module Foundry sector failed journal contains invalid JSON") + if ( + isinstance(value, dict) + and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID + ): + records.append(value) + if len(records) != 1: + die("Module Foundry sector failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT + or record.get("backup_id") + != MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID + or record.get("component") != "module-foundry" + or record.get("sha256") + != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "not-required" + or record.get("status") != "failed" + or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE + ): + die("Module Foundry sector failed journal evidence mismatch") + + with tempfile.TemporaryDirectory( + prefix="module-foundry-sector-failed-", + dir=TMP_DIR, + ) as directory: + failed_work = Path(directory) + safe_extract(failed_artifact, failed_work) + failed_manifest = parse_manifest(failed_work / "manifest.env") + failed_entries = parse_files_list(failed_work / "files.txt") + if ( + failed_manifest.get("id") + != MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID + or failed_manifest.get("component") != "module-foundry" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES + ): + die("Module Foundry sector failed artifact contract mismatch") + failed_payload_sha256 = collect_exact_files( + failed_work / "payload", + failed_entries, + "Module Foundry failed sector payload", + ) + + candidate_payload_sha256 = collect_exact_files( + payload_dir, + entries, + "Module Foundry sector recovery payload", + ) + if candidate_payload_sha256 != failed_payload_sha256: + die("Module Foundry sector recovery payload changed unexpectedly") + installed_payload_sha256 = collect_exact_files( + component_root("module-foundry"), + entries, + "Module Foundry partial sector predecessor", + ) + if installed_payload_sha256 != candidate_payload_sha256: + die("Module Foundry partial sector predecessor drift detected") + + runtime = module_foundry_runtime_inventory() + return { + "mode": "failed-overlay-runtime-reconciliation", + "failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID, + "failed_artifact_sha256": ( + MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 + ), + "failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID, + "runtime_container_id": runtime["containerId"], + "runtime_image_id": runtime["imageId"], + } + + +def read_module_foundry_runtime_before(backup_dir): + runtime = read_strict_json( + backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE, + "Module Foundry pre-apply runtime inventory", + max_bytes=16 * 1024, + ) + if ( + runtime.get("schemaVersion") + != "nodedc.module-foundry.runtime-inventory.v1" + or runtime.get("composeProject") != "nodedc-module-foundry" + or runtime.get("service") != MODULE_FOUNDRY_SERVICE + or not re.fullmatch( + r"[a-f0-9]{12,64}", + str(runtime.get("containerId", "")), + ) + or not re.fullmatch( + r"sha256:[a-f0-9]{64}", + str(runtime.get("imageId", "")), + ) + or not isinstance(runtime.get("imageRef"), str) + or not re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}", + runtime["imageRef"], + ) + or runtime["imageRef"].startswith("sha256:") + or "@" in runtime["imageRef"] + ): + die("Module Foundry pre-apply runtime inventory mismatch") + return runtime + + +def rollback_module_foundry_apply( + root, + backup_dir, + entries, + current_stamp, + applied_services, +): + if tuple(applied_services or ()) != (MODULE_FOUNDRY_SERVICE,): + die("Module Foundry rollback service set mismatch") + runtime_before = read_module_foundry_runtime_before(backup_dir) + restored_count = restore_platform_overlay( + root, + backup_dir, + entries, + current_stamp, + ) + current = module_foundry_runtime_inventory(required=False) + if ( + current is not None + and current["containerId"] == runtime_before["containerId"] + and current["imageId"] == runtime_before["imageId"] + ): + run_healthchecks("module-foundry", entries, applied_services) + return f"source-restored-runtime-unchanged:{restored_count}" + + subprocess.run( + [ + str(DOCKER), + "image", + "tag", + runtime_before["imageId"], + runtime_before["imageRef"], + ], + check=True, + ) + if ( + inspect_local_image( + runtime_before["imageRef"], + "Module Foundry rollback image", + ) + != runtime_before["imageId"] + ): + die("Module Foundry rollback image tag mismatch") + run_module_foundry_compose_up(applied_services) + run_healthchecks("module-foundry", entries, applied_services) + restored = module_foundry_runtime_inventory() + if restored["imageId"] != runtime_before["imageId"]: + die("Module Foundry rollback runtime image mismatch") + return f"source+runtime-restored:{restored_count}" + + def validate_engine_l2_closed_loop_stable_source(root): stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256) actual = collect_exact_files( @@ -17728,6 +18053,67 @@ def run_compose(component, services, entries=None): ) +def run_module_foundry_compose_up(services): + if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,): + die("Module Foundry Compose service set mismatch") + compose_root = component_compose_root("module-foundry") + cmd = [ + *compose_base_cmd("module-foundry"), + "up", + "-d", + "--force-recreate", + "--pull", + "never", + "--no-deps", + *services, + ] + try: + subprocess.run(cmd, cwd=str(compose_root), check=True) + except subprocess.CalledProcessError: + subprocess.run( + [ + *compose_base_cmd("module-foundry"), + "logs", + "--no-color", + "--tail=180", + *services, + ], + cwd=str(compose_root), + check=False, + ) + raise + subprocess.run( + [*compose_base_cmd("module-foundry"), "ps"], + cwd=str(compose_root), + check=True, + ) + + +def run_module_foundry_compose(services): + if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,): + die("Module Foundry build service set mismatch") + compose_root = component_compose_root("module-foundry") + build_cmd = [ + *compose_base_cmd("module-foundry"), + "build", + *services, + ] + for attempt in range(1, MODULE_FOUNDRY_BUILD_ATTEMPTS + 1): + try: + subprocess.run(build_cmd, cwd=str(compose_root), check=True) + break + except subprocess.CalledProcessError: + if attempt >= MODULE_FOUNDRY_BUILD_ATTEMPTS: + raise + print( + "module-foundry-build-retry=" + f"{attempt}/{MODULE_FOUNDRY_BUILD_ATTEMPTS - 1}", + file=sys.stderr, + ) + time.sleep(2) + run_module_foundry_compose_up(services) + + def run_engine_node_intelligence_compose(services, entries): if not is_engine_node_intelligence_transition("engine", entries): die("Engine node-intelligence Compose called for unrelated artifact") @@ -18024,6 +18410,8 @@ def run_component_runtime(component, entries, services): prepare_component_runtime(component, entries) if is_engine_node_intelligence_transition(component, entries): run_engine_node_intelligence_compose(services, entries) + elif component == "module-foundry": + run_module_foundry_compose(services) else: run_compose(component, services, entries) @@ -18423,6 +18811,94 @@ def compose_service_container_id(component, service): return container_ids[0] +def module_foundry_runtime_inventory(required=True): + result = subprocess.run( + [ + *compose_base_cmd("module-foundry"), + "ps", + "-q", + MODULE_FOUNDRY_SERVICE, + ], + cwd=str(component_compose_root("module-foundry")), + check=False, + capture_output=True, + text=True, + ) + container_ids = [ + line.strip() + for line in result.stdout.splitlines() + if line.strip() + ] + if ( + result.returncode != 0 + or len(container_ids) > 1 + or any( + not re.fullmatch(r"[a-f0-9]{12,64}", value) + for value in container_ids + ) + ): + die("Module Foundry runtime topology is unproven") + if not container_ids: + if required: + die("Module Foundry runtime is missing") + return None + container_id = container_ids[0] + containers = docker_json( + ["container", "inspect", container_id], + "Module Foundry runtime container inspect", + ) + if ( + not isinstance(containers, list) + or len(containers) != 1 + or not isinstance(containers[0], dict) + ): + die("Module Foundry runtime container inspect shape mismatch") + container = containers[0] + config = container.get("Config") or {} + labels = config.get("Labels") or {} + state = container.get("State") or {} + image_id = container.get("Image") + image_ref = config.get("Image") + health = (state.get("Health") or {}).get("Status") + if ( + labels.get("com.docker.compose.project") + != "nodedc-module-foundry" + or labels.get("com.docker.compose.service") + != MODULE_FOUNDRY_SERVICE + or not re.fullmatch(r"sha256:[a-f0-9]{64}", str(image_id or "")) + or not isinstance(image_ref, str) + or not re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}", + image_ref, + ) + or image_ref.startswith("sha256:") + or "@" in image_ref + or state.get("Running") is not True + or state.get("Status") != "running" + or health not in (None, "healthy") + ): + die("Module Foundry runtime inventory mismatch") + return { + "schemaVersion": "nodedc.module-foundry.runtime-inventory.v1", + "composeProject": "nodedc-module-foundry", + "service": MODULE_FOUNDRY_SERVICE, + "containerId": container_id, + "imageId": image_id, + "imageRef": image_ref, + "health": health or "not-configured", + } + + +def validate_module_foundry_candidate_runtime(runtime_before): + current = module_foundry_runtime_inventory() + if ( + current["containerId"] == runtime_before["containerId"] + or current["imageId"] == runtime_before["imageId"] + ): + die("Module Foundry candidate generation was not activated") + return current + + def healthcheck_compose_service(component, service): healthcheck_container(compose_service_container_id(component, service)) @@ -19555,6 +20031,8 @@ def apply_artifact(artifact): apply_started = False engine_backend_recreated = False engine_backend_initial_mode = None + module_foundry_runtime_before = None + module_foundry_sector_recovery_preflight = None runtime_started = False current_stamp = stamp() @@ -19633,6 +20111,13 @@ def apply_artifact(artifact): root.mkdir(parents=True, exist_ok=True) else: die(f"component payload root not found: {root}") + module_foundry_sector_recovery_preflight = ( + validate_module_foundry_sector_recovery_evidence( + manifest, + entries, + payload_dir, + ) + ) if is_engine_data_product_publish_grant_slice(component, entries): preflight_engine_data_product_publish_grant_predecessor(payload_dir) publish_backend_preflight = preflight_engine_credential_backend_runtime() @@ -19912,6 +20397,27 @@ def apply_artifact(artifact): DEVICE_PLANE_RUNTIME_SERVICES ) ) + if component == "module-foundry": + module_foundry_runtime_before = ( + module_foundry_runtime_inventory() + ) + if ( + module_foundry_sector_recovery_preflight is not None + and ( + module_foundry_runtime_before["containerId"] + != module_foundry_sector_recovery_preflight[ + "runtime_container_id" + ] + or module_foundry_runtime_before["imageId"] + != module_foundry_sector_recovery_preflight[ + "runtime_image_id" + ] + ) + ): + die( + "Module Foundry sector predecessor runtime " + "changed during preflight" + ) if is_engine_n8n_transition(component, entries): transition_descriptor = read_engine_n8n_transition_descriptor( @@ -19938,6 +20444,23 @@ def apply_artifact(artifact): include_nginx_html = component == "engine" and component_publish_dist(component, entries) create_backup(root, backup_dir, entries, include_nginx_html) + if component == "module-foundry": + if module_foundry_runtime_before is None: + die("Module Foundry pre-apply runtime inventory is missing") + module_foundry_runtime_path = ( + backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE + ) + module_foundry_runtime_path.write_text( + json.dumps( + module_foundry_runtime_before, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + module_foundry_runtime_path.chmod(0o600) if component == "device-plane": if device_plane_runtime_before is None: die("Device Plane pre-apply runtime inventory is missing") @@ -20043,6 +20566,12 @@ def apply_artifact(artifact): ): die("Engine L2 closed-loop app generation was not recreated") run_healthchecks(component, entries, services) + if component == "module-foundry": + if module_foundry_runtime_before is None: + die("Module Foundry candidate predecessor is missing") + validate_module_foundry_candidate_runtime( + module_foundry_runtime_before + ) if is_device_plane_b2_discovery_ingress_slice( component, entries, @@ -20249,6 +20778,35 @@ def apply_artifact(artifact): except Exception as rollback_exc: rollback_status = f"failed:{type(rollback_exc).__name__}" print("engine-automatic-rollback=failed", file=sys.stderr) + elif ( + component == "module-foundry" + and entries is not None + and services is not None + ): + try: + restored_state = rollback_module_foundry_apply( + root, + backup_dir, + entries, + current_stamp, + services, + ) + rollback_status = ( + f"ok:module-foundry-overlay:{restored_state}" + ) + print( + "module-foundry-automatic-rollback=" + f"{rollback_status}", + file=sys.stderr, + ) + except Exception as rollback_exc: + rollback_status = ( + f"failed:{type(rollback_exc).__name__}" + ) + print( + "module-foundry-automatic-rollback=failed", + file=sys.stderr, + ) elif ( component == "platform" and entries is not None diff --git a/infra/deploy-runner/test_module_foundry_runtime_recovery.py b/infra/deploy-runner/test_module_foundry_runtime_recovery.py new file mode 100644 index 0000000..21341d7 --- /dev/null +++ b/infra/deploy-runner/test_module_foundry_runtime_recovery.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 + +import hashlib +import importlib.machinery +import importlib.util +import json +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_module_foundry_runtime_recovery", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +def runtime(container_char="a", image_char="b"): + return { + "schemaVersion": "nodedc.module-foundry.runtime-inventory.v1", + "composeProject": "nodedc-module-foundry", + "service": RUNNER.MODULE_FOUNDRY_SERVICE, + "containerId": container_char * 64, + "imageId": f"sha256:{image_char * 64}", + "imageRef": "nodedc-module-foundry-nodedc-module-foundry:latest", + "health": "healthy", + } + + +class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): + def test_build_retries_before_any_runtime_recreate(self): + first_failure = subprocess.CalledProcessError(17, ["docker", "compose"]) + completed = subprocess.CompletedProcess([], 0) + with ( + mock.patch.object( + RUNNER, + "component_compose_root", + return_value=Path("/compose"), + ), + mock.patch.object( + RUNNER, + "compose_base_cmd", + return_value=["docker", "compose"], + ), + mock.patch.object( + RUNNER.subprocess, + "run", + side_effect=[first_failure, completed], + ) as run, + mock.patch.object(RUNNER.time, "sleep") as sleep, + mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up, + ): + RUNNER.run_module_foundry_compose( + (RUNNER.MODULE_FOUNDRY_SERVICE,) + ) + + expected = [ + "docker", + "compose", + "build", + RUNNER.MODULE_FOUNDRY_SERVICE, + ] + self.assertEqual(run.call_count, 2) + self.assertEqual(run.call_args_list[0].args[0], expected) + self.assertEqual(run.call_args_list[1].args[0], expected) + sleep.assert_called_once_with(2) + up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,)) + + def test_runtime_recreate_uses_built_image_without_build_or_pull(self): + completed = subprocess.CompletedProcess([], 0) + with ( + mock.patch.object( + RUNNER, + "component_compose_root", + return_value=Path("/compose"), + ), + mock.patch.object( + RUNNER, + "compose_base_cmd", + return_value=["docker", "compose"], + ), + mock.patch.object( + RUNNER.subprocess, + "run", + return_value=completed, + ) as run, + ): + RUNNER.run_module_foundry_compose_up( + (RUNNER.MODULE_FOUNDRY_SERVICE,) + ) + + up = run.call_args_list[0].args[0] + self.assertEqual( + up, + [ + "docker", + "compose", + "up", + "-d", + "--force-recreate", + "--pull", + "never", + "--no-deps", + RUNNER.MODULE_FOUNDRY_SERVICE, + ], + ) + self.assertNotIn("--build", up) + self.assertEqual(run.call_args_list[1].args[0], ["docker", "compose", "ps"]) + + def test_rollback_restores_source_and_keeps_unchanged_runtime(self): + with tempfile.TemporaryDirectory( + prefix="module-foundry-rollback-unchanged-" + ) as directory: + work = Path(directory) + root = work / "source" + backup = work / "backup" + tmp = work / "tmp" + root.mkdir() + backup.mkdir() + tmp.mkdir() + entries = ("old.txt", "new.txt") + (root / "old.txt").write_text("old\n", encoding="utf-8") + RUNNER.create_backup(root, backup, entries, False) + (root / "old.txt").write_text("candidate\n", encoding="utf-8") + (root / "new.txt").write_text("candidate\n", encoding="utf-8") + before = runtime() + (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text( + json.dumps(before), + encoding="utf-8", + ) + + with ( + mock.patch.object(RUNNER, "TMP_DIR", tmp), + mock.patch.object( + RUNNER, + "module_foundry_runtime_inventory", + return_value=before, + ), + mock.patch.object(RUNNER, "run_healthchecks") as health, + mock.patch.object(RUNNER.subprocess, "run") as process, + ): + result = RUNNER.rollback_module_foundry_apply( + root, + backup, + entries, + "20260809-000000", + (RUNNER.MODULE_FOUNDRY_SERVICE,), + ) + + self.assertEqual( + result, + "source-restored-runtime-unchanged:2", + ) + self.assertEqual( + (root / "old.txt").read_text(encoding="utf-8"), + "old\n", + ) + self.assertFalse((root / "new.txt").exists()) + process.assert_not_called() + health.assert_called_once() + + def test_rollback_retags_and_recreates_changed_runtime(self): + with tempfile.TemporaryDirectory( + prefix="module-foundry-rollback-changed-" + ) as directory: + work = Path(directory) + root = work / "source" + backup = work / "backup" + tmp = work / "tmp" + root.mkdir() + backup.mkdir() + tmp.mkdir() + entries = ("old.txt",) + (root / "old.txt").write_text("old\n", encoding="utf-8") + RUNNER.create_backup(root, backup, entries, False) + (root / "old.txt").write_text("candidate\n", encoding="utf-8") + before = runtime() + candidate = runtime("c", "d") + (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text( + json.dumps(before), + encoding="utf-8", + ) + + with ( + mock.patch.object(RUNNER, "TMP_DIR", tmp), + mock.patch.object( + RUNNER, + "module_foundry_runtime_inventory", + side_effect=[candidate, before], + ), + mock.patch.object( + RUNNER, + "inspect_local_image", + return_value=before["imageId"], + ), + mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up, + mock.patch.object(RUNNER, "run_healthchecks") as health, + mock.patch.object(RUNNER.subprocess, "run") as process, + ): + result = RUNNER.rollback_module_foundry_apply( + root, + backup, + entries, + "20260809-000000", + (RUNNER.MODULE_FOUNDRY_SERVICE,), + ) + + self.assertEqual(result, "source+runtime-restored:1") + process.assert_called_once_with( + [ + str(RUNNER.DOCKER), + "image", + "tag", + before["imageId"], + before["imageRef"], + ], + check=True, + ) + up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,)) + health.assert_called_once() + + def test_candidate_must_replace_both_container_and_image(self): + before = runtime() + for candidate in ( + runtime("a", "c"), + runtime("c", "b"), + ): + with mock.patch.object( + RUNNER, + "module_foundry_runtime_inventory", + return_value=candidate, + ): + with self.assertRaises(RUNNER.DeployError): + RUNNER.validate_module_foundry_candidate_runtime(before) + + def test_preapply_runtime_rejects_digest_only_image_reference(self): + with tempfile.TemporaryDirectory( + prefix="module-foundry-runtime-reference-" + ) as directory: + backup = Path(directory) + before = runtime() + before["imageRef"] = before["imageId"] + (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text( + json.dumps(before), + encoding="utf-8", + ) + with self.assertRaises(RUNNER.DeployError): + RUNNER.read_module_foundry_runtime_before(backup) + + def test_recovery_accepts_only_exact_failed_partial_predecessor(self): + with tempfile.TemporaryDirectory( + prefix="module-foundry-sector-evidence-" + ) as directory: + work = Path(directory) + backups = work / "backups" + failed = work / "failed" + state = work / "state" + tmp = work / "tmp" + installed = work / "installed" + candidate = work / "candidate" + for path in (backups, failed, state, tmp, installed, candidate): + path.mkdir() + + entries = RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES + for index, rel in enumerate(entries): + for root in (installed, candidate): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"payload-{index}\n", encoding="utf-8") + + backup = backups / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID + backup.mkdir() + for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256: + (backup / name).write_text(f"{name}\n", encoding="utf-8") + backup_hashes = { + name: hashlib.sha256((backup / name).read_bytes()).hexdigest() + for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 + } + + archive_source = work / "archive-source" + (archive_source / "payload").mkdir(parents=True) + (archive_source / "manifest.env").write_text( + "id=" + f"{RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID}\n" + "component=module-foundry\n" + "type=app-overlay\n", + encoding="utf-8", + ) + (archive_source / "files.txt").write_text( + "\n".join(entries) + "\n", + encoding="utf-8", + ) + for rel in entries: + source = candidate / rel + target = archive_source / "payload" / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(source.read_bytes()) + failed_artifact = ( + failed / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT + ) + with tarfile.open(failed_artifact, "w:gz") as archive: + archive.add( + archive_source / "manifest.env", + arcname="manifest.env", + ) + archive.add( + archive_source / "files.txt", + arcname="files.txt", + ) + archive.add(archive_source / "payload", arcname="payload") + failed_sha256 = hashlib.sha256( + failed_artifact.read_bytes() + ).hexdigest() + + journal = state / "failed.jsonl" + journal.write_text( + json.dumps( + { + "artifact": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT, + "backup_id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID, + "component": "module-foundry", + "id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID, + "message": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE, + "rollback_status": "not-required", + "sha256": failed_sha256, + "started_apply": True, + "status": "failed", + } + ) + + "\n", + encoding="utf-8", + ) + manifest = { + "id": RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID, + "component": "module-foundry", + "type": "app-overlay", + } + + patches = ( + mock.patch.object(RUNNER, "BACKUPS_DIR", backups), + mock.patch.object(RUNNER, "FAILED_DIR", failed), + mock.patch.object(RUNNER, "FAILED_STATE_FILE", journal), + mock.patch.object(RUNNER, "TMP_DIR", tmp), + mock.patch.object( + RUNNER, + "MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256", + failed_sha256, + ), + mock.patch.object( + RUNNER, + "MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256", + backup_hashes, + ), + mock.patch.object( + RUNNER, + "component_root", + return_value=installed, + ), + mock.patch.object( + RUNNER, + "module_foundry_runtime_inventory", + return_value=runtime(), + ), + ) + with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6], patches[7]: + result = RUNNER.validate_module_foundry_sector_recovery_evidence( + manifest, + entries, + candidate, + ) + self.assertEqual( + result["mode"], + "failed-overlay-runtime-reconciliation", + ) + (installed / entries[0]).write_text( + "drift\n", + encoding="utf-8", + ) + with self.assertRaises(RUNNER.DeployError): + RUNNER.validate_module_foundry_sector_recovery_evidence( + manifest, + entries, + candidate, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2)