FIX - NAS DEPLOY: recover Module Foundry runtime safely

infra/deploy-runner/nodedc-deploy

@@ -35,6 +35,62 @@ STATE_FILE = STATE_DIR / "applied.jsonl"
 FAILED_STATE_FILE = STATE_DIR / "failed.jsonl"
 LOCK_DIR = STATE_DIR / "deploy.lock"
 DOCKER = Path("/usr/local/bin/docker")
+MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
+MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
+MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
+MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = (
+    "module-foundry-map-sector-workspace-20260809-001"
+)
+MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
+    "apps/catalog/src/MapFixturePreview.tsx",
+    "apps/catalog/src/mapPresentationProfile.ts",
+    "apps/catalog/src/mapSectorGrid.d.mts",
+    "apps/catalog/src/mapSectorGrid.mjs",
+    "apps/catalog/src/styles.css",
+    "scripts/map-object-layers.test.mjs",
+    "scripts/map-presentation-filters.test.mjs",
+    "scripts/map-sector-grid.test.mjs",
+)
+MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = (
+    "module-foundry-map-sector-workspace-20260808-001"
+)
+MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = (
+    "nodedc-module-foundry-map-sector-workspace-20260808-001.tgz."
+    "20260809-014720"
+)
+MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = (
+    "0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740"
+)
+MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = (
+    "module-foundry-module-foundry-map-sector-workspace-20260808-001-"
+    "20260809-014720"
+)
+MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
+    "Command '['/usr/local/bin/docker', 'compose', '-p', "
+    "'nodedc-module-foundry', '--env-file', "
+    "'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
+    "'/volume1/docker/nodedc-platform/module-foundry/source/infra/"
+    "docker-compose.module-foundry.yml', 'up', '-d', '--force-recreate', "
+    "'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
+    "exit status 17."
+)
+MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = {
+    "existing-files.txt": (
+        "57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c"
+    ),
+    "files.txt": (
+        "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
+    ),
+    "manifest.env": (
+        "cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2"
+    ),
+    "missing-files.txt": (
+        "1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f"
+    ),
+    "source-before.tgz": (
+        "bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562"
+    ),
+}
 MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
 MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
 MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
@@ -14961,11 +15017,19 @@ def plan_artifact(artifact):
     provider_catalog_preflight = None
     device_plane_postgres_preflight = None
     device_plane_foundation_recovery_preflight = None
+    module_foundry_sector_recovery_preflight = None
     with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
         manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
         reject_terminal_engine_l2_failed_artifact(manifest, sha)
         reject_terminal_device_plane_foundation_artifact(manifest, sha)
         reject_terminal_device_plane_backhaul_artifact(manifest, sha)
+        module_foundry_sector_recovery_preflight = (
+            validate_module_foundry_sector_recovery_evidence(
+                manifest,
+                entries,
+                payload_dir,
+            )
+        )
         transition_descriptor = None
         transition_preflight = None
         if is_engine_n8n_transition(manifest["component"], entries):
@@ -16520,6 +16584,37 @@ def plan_artifact(artifact):
         print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
         print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
         print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
+        print(
+            "module_foundry_build="
+            f"separate-before-recreate:attempts={MODULE_FOUNDRY_BUILD_ATTEMPTS}"
+        )
+        print("module_foundry_runtime_pull=never")
+        print("module_foundry_rollback=source+image+runtime")
+        if module_foundry_sector_recovery_preflight is not None:
+            print(
+                "module_foundry_transition="
+                f"{module_foundry_sector_recovery_preflight['mode']}"
+            )
+            print(
+                "module_foundry_failed_predecessor="
+                f"{module_foundry_sector_recovery_preflight['failed_patch_id']}"
+            )
+            print(
+                "module_foundry_failed_predecessor_sha256="
+                f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}"
+            )
+            print(
+                "module_foundry_failed_backup="
+                f"{module_foundry_sector_recovery_preflight['failed_backup_id']}"
+            )
+            print(
+                "module_foundry_predecessor_container="
+                f"{module_foundry_sector_recovery_preflight['runtime_container_id']}"
+            )
+            print(
+                "module_foundry_predecessor_image="
+                f"{module_foundry_sector_recovery_preflight['runtime_image_id']}"
+            )
     if component == "device-plane":
         print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
         print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}")
@@ -17241,6 +17336,236 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
     return f"source+runtime-restored:{restored_count}"
 
 
+def is_module_foundry_sector_recovery_slice(manifest, entries):
+    return (
+        manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID
+        and manifest.get("component") == "module-foundry"
+        and manifest.get("type") == "app-overlay"
+        and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
+    )
+
+
+def validate_module_foundry_sector_recovery_evidence(
+    manifest,
+    entries,
+    payload_dir,
+):
+    if not is_module_foundry_sector_recovery_slice(manifest, entries):
+        return None
+
+    backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
+    try:
+        backup_stat = backup_dir.lstat()
+    except FileNotFoundError:
+        die("Module Foundry sector recovery backup is missing")
+    if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
+        backup_stat.st_mode
+    ):
+        die("Module Foundry sector recovery backup is unsafe")
+    if {child.name for child in backup_dir.iterdir()} != set(
+        MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
+    ):
+        die("Module Foundry sector recovery backup file set mismatch")
+    for name, expected_sha256 in (
+        MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items()
+    ):
+        path = backup_dir / name
+        path_stat = path.lstat()
+        if (
+            stat.S_ISLNK(path_stat.st_mode)
+            or not stat.S_ISREG(path_stat.st_mode)
+            or sha256_file(path) != expected_sha256
+        ):
+            die(
+                "Module Foundry sector recovery backup drift detected: "
+                f"{name}"
+            )
+
+    failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
+    try:
+        failed_stat = failed_artifact.lstat()
+    except FileNotFoundError:
+        die("Module Foundry sector failed artifact is missing")
+    if (
+        stat.S_ISLNK(failed_stat.st_mode)
+        or not stat.S_ISREG(failed_stat.st_mode)
+        or sha256_file(failed_artifact)
+        != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
+    ):
+        die("Module Foundry sector failed artifact evidence mismatch")
+
+    try:
+        state_stat = FAILED_STATE_FILE.lstat()
+        state_lines = FAILED_STATE_FILE.read_text(
+            encoding="utf-8"
+        ).splitlines()
+    except (FileNotFoundError, OSError, UnicodeDecodeError):
+        die("Module Foundry sector failed journal is unreadable")
+    if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
+        state_stat.st_mode
+    ):
+        die("Module Foundry sector failed journal is unsafe")
+    records = []
+    for line in state_lines:
+        try:
+            value = json.loads(line)
+        except json.JSONDecodeError:
+            die("Module Foundry sector failed journal contains invalid JSON")
+        if (
+            isinstance(value, dict)
+            and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
+        ):
+            records.append(value)
+    if len(records) != 1:
+        die("Module Foundry sector failed journal evidence count mismatch")
+    record = records[0]
+    if (
+        record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
+        or record.get("backup_id")
+        != MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
+        or record.get("component") != "module-foundry"
+        or record.get("sha256")
+        != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
+        or record.get("started_apply") is not True
+        or record.get("rollback_status") != "not-required"
+        or record.get("status") != "failed"
+        or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE
+    ):
+        die("Module Foundry sector failed journal evidence mismatch")
+
+    with tempfile.TemporaryDirectory(
+        prefix="module-foundry-sector-failed-",
+        dir=TMP_DIR,
+    ) as directory:
+        failed_work = Path(directory)
+        safe_extract(failed_artifact, failed_work)
+        failed_manifest = parse_manifest(failed_work / "manifest.env")
+        failed_entries = parse_files_list(failed_work / "files.txt")
+        if (
+            failed_manifest.get("id")
+            != MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
+            or failed_manifest.get("component") != "module-foundry"
+            or failed_manifest.get("type") != "app-overlay"
+            or tuple(failed_entries)
+            != MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
+        ):
+            die("Module Foundry sector failed artifact contract mismatch")
+        failed_payload_sha256 = collect_exact_files(
+            failed_work / "payload",
+            failed_entries,
+            "Module Foundry failed sector payload",
+        )
+
+    candidate_payload_sha256 = collect_exact_files(
+        payload_dir,
+        entries,
+        "Module Foundry sector recovery payload",
+    )
+    if candidate_payload_sha256 != failed_payload_sha256:
+        die("Module Foundry sector recovery payload changed unexpectedly")
+    installed_payload_sha256 = collect_exact_files(
+        component_root("module-foundry"),
+        entries,
+        "Module Foundry partial sector predecessor",
+    )
+    if installed_payload_sha256 != candidate_payload_sha256:
+        die("Module Foundry partial sector predecessor drift detected")
+
+    runtime = module_foundry_runtime_inventory()
+    return {
+        "mode": "failed-overlay-runtime-reconciliation",
+        "failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
+        "failed_artifact_sha256": (
+            MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
+        ),
+        "failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
+        "runtime_container_id": runtime["containerId"],
+        "runtime_image_id": runtime["imageId"],
+    }
+
+
+def read_module_foundry_runtime_before(backup_dir):
+    runtime = read_strict_json(
+        backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE,
+        "Module Foundry pre-apply runtime inventory",
+        max_bytes=16 * 1024,
+    )
+    if (
+        runtime.get("schemaVersion")
+        != "nodedc.module-foundry.runtime-inventory.v1"
+        or runtime.get("composeProject") != "nodedc-module-foundry"
+        or runtime.get("service") != MODULE_FOUNDRY_SERVICE
+        or not re.fullmatch(
+            r"[a-f0-9]{12,64}",
+            str(runtime.get("containerId", "")),
+        )
+        or not re.fullmatch(
+            r"sha256:[a-f0-9]{64}",
+            str(runtime.get("imageId", "")),
+        )
+        or not isinstance(runtime.get("imageRef"), str)
+        or not re.fullmatch(
+            r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
+            runtime["imageRef"],
+        )
+        or runtime["imageRef"].startswith("sha256:")
+        or "@" in runtime["imageRef"]
+    ):
+        die("Module Foundry pre-apply runtime inventory mismatch")
+    return runtime
+
+
+def rollback_module_foundry_apply(
+    root,
+    backup_dir,
+    entries,
+    current_stamp,
+    applied_services,
+):
+    if tuple(applied_services or ()) != (MODULE_FOUNDRY_SERVICE,):
+        die("Module Foundry rollback service set mismatch")
+    runtime_before = read_module_foundry_runtime_before(backup_dir)
+    restored_count = restore_platform_overlay(
+        root,
+        backup_dir,
+        entries,
+        current_stamp,
+    )
+    current = module_foundry_runtime_inventory(required=False)
+    if (
+        current is not None
+        and current["containerId"] == runtime_before["containerId"]
+        and current["imageId"] == runtime_before["imageId"]
+    ):
+        run_healthchecks("module-foundry", entries, applied_services)
+        return f"source-restored-runtime-unchanged:{restored_count}"
+
+    subprocess.run(
+        [
+            str(DOCKER),
+            "image",
+            "tag",
+            runtime_before["imageId"],
+            runtime_before["imageRef"],
+        ],
+        check=True,
+    )
+    if (
+        inspect_local_image(
+            runtime_before["imageRef"],
+            "Module Foundry rollback image",
+        )
+        != runtime_before["imageId"]
+    ):
+        die("Module Foundry rollback image tag mismatch")
+    run_module_foundry_compose_up(applied_services)
+    run_healthchecks("module-foundry", entries, applied_services)
+    restored = module_foundry_runtime_inventory()
+    if restored["imageId"] != runtime_before["imageId"]:
+        die("Module Foundry rollback runtime image mismatch")
+    return f"source+runtime-restored:{restored_count}"
+
+
 def validate_engine_l2_closed_loop_stable_source(root):
     stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256)
     actual = collect_exact_files(
@@ -17728,6 +18053,67 @@ def run_compose(component, services, entries=None):
     )
 
 
+def run_module_foundry_compose_up(services):
+    if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
+        die("Module Foundry Compose service set mismatch")
+    compose_root = component_compose_root("module-foundry")
+    cmd = [
+        *compose_base_cmd("module-foundry"),
+        "up",
+        "-d",
+        "--force-recreate",
+        "--pull",
+        "never",
+        "--no-deps",
+        *services,
+    ]
+    try:
+        subprocess.run(cmd, cwd=str(compose_root), check=True)
+    except subprocess.CalledProcessError:
+        subprocess.run(
+            [
+                *compose_base_cmd("module-foundry"),
+                "logs",
+                "--no-color",
+                "--tail=180",
+                *services,
+            ],
+            cwd=str(compose_root),
+            check=False,
+        )
+        raise
+    subprocess.run(
+        [*compose_base_cmd("module-foundry"), "ps"],
+        cwd=str(compose_root),
+        check=True,
+    )
+
+
+def run_module_foundry_compose(services):
+    if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
+        die("Module Foundry build service set mismatch")
+    compose_root = component_compose_root("module-foundry")
+    build_cmd = [
+        *compose_base_cmd("module-foundry"),
+        "build",
+        *services,
+    ]
+    for attempt in range(1, MODULE_FOUNDRY_BUILD_ATTEMPTS + 1):
+        try:
+            subprocess.run(build_cmd, cwd=str(compose_root), check=True)
+            break
+        except subprocess.CalledProcessError:
+            if attempt >= MODULE_FOUNDRY_BUILD_ATTEMPTS:
+                raise
+            print(
+                "module-foundry-build-retry="
+                f"{attempt}/{MODULE_FOUNDRY_BUILD_ATTEMPTS - 1}",
+                file=sys.stderr,
+            )
+            time.sleep(2)
+    run_module_foundry_compose_up(services)
+
+
 def run_engine_node_intelligence_compose(services, entries):
     if not is_engine_node_intelligence_transition("engine", entries):
         die("Engine node-intelligence Compose called for unrelated artifact")
@@ -18024,6 +18410,8 @@ def run_component_runtime(component, entries, services):
     prepare_component_runtime(component, entries)
     if is_engine_node_intelligence_transition(component, entries):
         run_engine_node_intelligence_compose(services, entries)
+    elif component == "module-foundry":
+        run_module_foundry_compose(services)
     else:
         run_compose(component, services, entries)
 
@@ -18423,6 +18811,94 @@ def compose_service_container_id(component, service):
     return container_ids[0]
 
 
+def module_foundry_runtime_inventory(required=True):
+    result = subprocess.run(
+        [
+            *compose_base_cmd("module-foundry"),
+            "ps",
+            "-q",
+            MODULE_FOUNDRY_SERVICE,
+        ],
+        cwd=str(component_compose_root("module-foundry")),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    container_ids = [
+        line.strip()
+        for line in result.stdout.splitlines()
+        if line.strip()
+    ]
+    if (
+        result.returncode != 0
+        or len(container_ids) > 1
+        or any(
+            not re.fullmatch(r"[a-f0-9]{12,64}", value)
+            for value in container_ids
+        )
+    ):
+        die("Module Foundry runtime topology is unproven")
+    if not container_ids:
+        if required:
+            die("Module Foundry runtime is missing")
+        return None
+    container_id = container_ids[0]
+    containers = docker_json(
+        ["container", "inspect", container_id],
+        "Module Foundry runtime container inspect",
+    )
+    if (
+        not isinstance(containers, list)
+        or len(containers) != 1
+        or not isinstance(containers[0], dict)
+    ):
+        die("Module Foundry runtime container inspect shape mismatch")
+    container = containers[0]
+    config = container.get("Config") or {}
+    labels = config.get("Labels") or {}
+    state = container.get("State") or {}
+    image_id = container.get("Image")
+    image_ref = config.get("Image")
+    health = (state.get("Health") or {}).get("Status")
+    if (
+        labels.get("com.docker.compose.project")
+        != "nodedc-module-foundry"
+        or labels.get("com.docker.compose.service")
+        != MODULE_FOUNDRY_SERVICE
+        or not re.fullmatch(r"sha256:[a-f0-9]{64}", str(image_id or ""))
+        or not isinstance(image_ref, str)
+        or not re.fullmatch(
+            r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
+            image_ref,
+        )
+        or image_ref.startswith("sha256:")
+        or "@" in image_ref
+        or state.get("Running") is not True
+        or state.get("Status") != "running"
+        or health not in (None, "healthy")
+    ):
+        die("Module Foundry runtime inventory mismatch")
+    return {
+        "schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
+        "composeProject": "nodedc-module-foundry",
+        "service": MODULE_FOUNDRY_SERVICE,
+        "containerId": container_id,
+        "imageId": image_id,
+        "imageRef": image_ref,
+        "health": health or "not-configured",
+    }
+
+
+def validate_module_foundry_candidate_runtime(runtime_before):
+    current = module_foundry_runtime_inventory()
+    if (
+        current["containerId"] == runtime_before["containerId"]
+        or current["imageId"] == runtime_before["imageId"]
+    ):
+        die("Module Foundry candidate generation was not activated")
+    return current
+
+
 def healthcheck_compose_service(component, service):
     healthcheck_container(compose_service_container_id(component, service))
 
@@ -19555,6 +20031,8 @@ def apply_artifact(artifact):
     apply_started = False
     engine_backend_recreated = False
     engine_backend_initial_mode = None
+    module_foundry_runtime_before = None
+    module_foundry_sector_recovery_preflight = None
     runtime_started = False
     current_stamp = stamp()
 
@@ -19633,6 +20111,13 @@ def apply_artifact(artifact):
                         root.mkdir(parents=True, exist_ok=True)
                     else:
                         die(f"component payload root not found: {root}")
+                module_foundry_sector_recovery_preflight = (
+                    validate_module_foundry_sector_recovery_evidence(
+                        manifest,
+                        entries,
+                        payload_dir,
+                    )
+                )
                 if is_engine_data_product_publish_grant_slice(component, entries):
                     preflight_engine_data_product_publish_grant_predecessor(payload_dir)
                     publish_backend_preflight = preflight_engine_credential_backend_runtime()
@@ -19912,6 +20397,27 @@ def apply_artifact(artifact):
                             DEVICE_PLANE_RUNTIME_SERVICES
                         )
                     )
+                if component == "module-foundry":
+                    module_foundry_runtime_before = (
+                        module_foundry_runtime_inventory()
+                    )
+                    if (
+                        module_foundry_sector_recovery_preflight is not None
+                        and (
+                            module_foundry_runtime_before["containerId"]
+                            != module_foundry_sector_recovery_preflight[
+                                "runtime_container_id"
+                            ]
+                            or module_foundry_runtime_before["imageId"]
+                            != module_foundry_sector_recovery_preflight[
+                                "runtime_image_id"
+                            ]
+                        )
+                    ):
+                        die(
+                            "Module Foundry sector predecessor runtime "
+                            "changed during preflight"
+                        )
 
                 if is_engine_n8n_transition(component, entries):
                     transition_descriptor = read_engine_n8n_transition_descriptor(
@@ -19938,6 +20444,23 @@ def apply_artifact(artifact):
 
                 include_nginx_html = component == "engine" and component_publish_dist(component, entries)
                 create_backup(root, backup_dir, entries, include_nginx_html)
+                if component == "module-foundry":
+                    if module_foundry_runtime_before is None:
+                        die("Module Foundry pre-apply runtime inventory is missing")
+                    module_foundry_runtime_path = (
+                        backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE
+                    )
+                    module_foundry_runtime_path.write_text(
+                        json.dumps(
+                            module_foundry_runtime_before,
+                            ensure_ascii=False,
+                            indent=2,
+                            sort_keys=True,
+                        )
+                        + "\n",
+                        encoding="utf-8",
+                    )
+                    module_foundry_runtime_path.chmod(0o600)
                 if component == "device-plane":
                     if device_plane_runtime_before is None:
                         die("Device Plane pre-apply runtime inventory is missing")
@@ -20043,6 +20566,12 @@ def apply_artifact(artifact):
                 ):
                     die("Engine L2 closed-loop app generation was not recreated")
                 run_healthchecks(component, entries, services)
+                if component == "module-foundry":
+                    if module_foundry_runtime_before is None:
+                        die("Module Foundry candidate predecessor is missing")
+                    validate_module_foundry_candidate_runtime(
+                        module_foundry_runtime_before
+                    )
                 if is_device_plane_b2_discovery_ingress_slice(
                     component,
                     entries,
@@ -20249,6 +20778,35 @@ def apply_artifact(artifact):
                     except Exception as rollback_exc:
                         rollback_status = f"failed:{type(rollback_exc).__name__}"
                         print("engine-automatic-rollback=failed", file=sys.stderr)
+                elif (
+                    component == "module-foundry"
+                    and entries is not None
+                    and services is not None
+                ):
+                    try:
+                        restored_state = rollback_module_foundry_apply(
+                            root,
+                            backup_dir,
+                            entries,
+                            current_stamp,
+                            services,
+                        )
+                        rollback_status = (
+                            f"ok:module-foundry-overlay:{restored_state}"
+                        )
+                        print(
+                            "module-foundry-automatic-rollback="
+                            f"{rollback_status}",
+                            file=sys.stderr,
+                        )
+                    except Exception as rollback_exc:
+                        rollback_status = (
+                            f"failed:{type(rollback_exc).__name__}"
+                        )
+                        print(
+                            "module-foundry-automatic-rollback=failed",
+                            file=sys.stderr,
+                        )
                 elif (
                     component == "platform"
                     and entries is not None

infra/deploy-runner/test_module_foundry_runtime_recovery.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+
+import hashlib
+import importlib.machinery
+import importlib.util
+import json
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
+
+
+def load_runner():
+    loader = importlib.machinery.SourceFileLoader(
+        "nodedc_module_foundry_runtime_recovery",
+        str(RUNNER_PATH),
+    )
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    module = importlib.util.module_from_spec(spec)
+    loader.exec_module(module)
+    return module
+
+
+RUNNER = load_runner()
+
+
+def runtime(container_char="a", image_char="b"):
+    return {
+        "schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
+        "composeProject": "nodedc-module-foundry",
+        "service": RUNNER.MODULE_FOUNDRY_SERVICE,
+        "containerId": container_char * 64,
+        "imageId": f"sha256:{image_char * 64}",
+        "imageRef": "nodedc-module-foundry-nodedc-module-foundry:latest",
+        "health": "healthy",
+    }
+
+
+class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
+    def test_build_retries_before_any_runtime_recreate(self):
+        first_failure = subprocess.CalledProcessError(17, ["docker", "compose"])
+        completed = subprocess.CompletedProcess([], 0)
+        with (
+            mock.patch.object(
+                RUNNER,
+                "component_compose_root",
+                return_value=Path("/compose"),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "compose_base_cmd",
+                return_value=["docker", "compose"],
+            ),
+            mock.patch.object(
+                RUNNER.subprocess,
+                "run",
+                side_effect=[first_failure, completed],
+            ) as run,
+            mock.patch.object(RUNNER.time, "sleep") as sleep,
+            mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
+        ):
+            RUNNER.run_module_foundry_compose(
+                (RUNNER.MODULE_FOUNDRY_SERVICE,)
+            )
+
+        expected = [
+            "docker",
+            "compose",
+            "build",
+            RUNNER.MODULE_FOUNDRY_SERVICE,
+        ]
+        self.assertEqual(run.call_count, 2)
+        self.assertEqual(run.call_args_list[0].args[0], expected)
+        self.assertEqual(run.call_args_list[1].args[0], expected)
+        sleep.assert_called_once_with(2)
+        up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
+
+    def test_runtime_recreate_uses_built_image_without_build_or_pull(self):
+        completed = subprocess.CompletedProcess([], 0)
+        with (
+            mock.patch.object(
+                RUNNER,
+                "component_compose_root",
+                return_value=Path("/compose"),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "compose_base_cmd",
+                return_value=["docker", "compose"],
+            ),
+            mock.patch.object(
+                RUNNER.subprocess,
+                "run",
+                return_value=completed,
+            ) as run,
+        ):
+            RUNNER.run_module_foundry_compose_up(
+                (RUNNER.MODULE_FOUNDRY_SERVICE,)
+            )
+
+        up = run.call_args_list[0].args[0]
+        self.assertEqual(
+            up,
+            [
+                "docker",
+                "compose",
+                "up",
+                "-d",
+                "--force-recreate",
+                "--pull",
+                "never",
+                "--no-deps",
+                RUNNER.MODULE_FOUNDRY_SERVICE,
+            ],
+        )
+        self.assertNotIn("--build", up)
+        self.assertEqual(run.call_args_list[1].args[0], ["docker", "compose", "ps"])
+
+    def test_rollback_restores_source_and_keeps_unchanged_runtime(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-rollback-unchanged-"
+        ) as directory:
+            work = Path(directory)
+            root = work / "source"
+            backup = work / "backup"
+            tmp = work / "tmp"
+            root.mkdir()
+            backup.mkdir()
+            tmp.mkdir()
+            entries = ("old.txt", "new.txt")
+            (root / "old.txt").write_text("old\n", encoding="utf-8")
+            RUNNER.create_backup(root, backup, entries, False)
+            (root / "old.txt").write_text("candidate\n", encoding="utf-8")
+            (root / "new.txt").write_text("candidate\n", encoding="utf-8")
+            before = runtime()
+            (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
+                json.dumps(before),
+                encoding="utf-8",
+            )
+
+            with (
+                mock.patch.object(RUNNER, "TMP_DIR", tmp),
+                mock.patch.object(
+                    RUNNER,
+                    "module_foundry_runtime_inventory",
+                    return_value=before,
+                ),
+                mock.patch.object(RUNNER, "run_healthchecks") as health,
+                mock.patch.object(RUNNER.subprocess, "run") as process,
+            ):
+                result = RUNNER.rollback_module_foundry_apply(
+                    root,
+                    backup,
+                    entries,
+                    "20260809-000000",
+                    (RUNNER.MODULE_FOUNDRY_SERVICE,),
+                )
+
+            self.assertEqual(
+                result,
+                "source-restored-runtime-unchanged:2",
+            )
+            self.assertEqual(
+                (root / "old.txt").read_text(encoding="utf-8"),
+                "old\n",
+            )
+            self.assertFalse((root / "new.txt").exists())
+            process.assert_not_called()
+            health.assert_called_once()
+
+    def test_rollback_retags_and_recreates_changed_runtime(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-rollback-changed-"
+        ) as directory:
+            work = Path(directory)
+            root = work / "source"
+            backup = work / "backup"
+            tmp = work / "tmp"
+            root.mkdir()
+            backup.mkdir()
+            tmp.mkdir()
+            entries = ("old.txt",)
+            (root / "old.txt").write_text("old\n", encoding="utf-8")
+            RUNNER.create_backup(root, backup, entries, False)
+            (root / "old.txt").write_text("candidate\n", encoding="utf-8")
+            before = runtime()
+            candidate = runtime("c", "d")
+            (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
+                json.dumps(before),
+                encoding="utf-8",
+            )
+
+            with (
+                mock.patch.object(RUNNER, "TMP_DIR", tmp),
+                mock.patch.object(
+                    RUNNER,
+                    "module_foundry_runtime_inventory",
+                    side_effect=[candidate, before],
+                ),
+                mock.patch.object(
+                    RUNNER,
+                    "inspect_local_image",
+                    return_value=before["imageId"],
+                ),
+                mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
+                mock.patch.object(RUNNER, "run_healthchecks") as health,
+                mock.patch.object(RUNNER.subprocess, "run") as process,
+            ):
+                result = RUNNER.rollback_module_foundry_apply(
+                    root,
+                    backup,
+                    entries,
+                    "20260809-000000",
+                    (RUNNER.MODULE_FOUNDRY_SERVICE,),
+                )
+
+            self.assertEqual(result, "source+runtime-restored:1")
+            process.assert_called_once_with(
+                [
+                    str(RUNNER.DOCKER),
+                    "image",
+                    "tag",
+                    before["imageId"],
+                    before["imageRef"],
+                ],
+                check=True,
+            )
+            up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
+            health.assert_called_once()
+
+    def test_candidate_must_replace_both_container_and_image(self):
+        before = runtime()
+        for candidate in (
+            runtime("a", "c"),
+            runtime("c", "b"),
+        ):
+            with mock.patch.object(
+                RUNNER,
+                "module_foundry_runtime_inventory",
+                return_value=candidate,
+            ):
+                with self.assertRaises(RUNNER.DeployError):
+                    RUNNER.validate_module_foundry_candidate_runtime(before)
+
+    def test_preapply_runtime_rejects_digest_only_image_reference(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-runtime-reference-"
+        ) as directory:
+            backup = Path(directory)
+            before = runtime()
+            before["imageRef"] = before["imageId"]
+            (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
+                json.dumps(before),
+                encoding="utf-8",
+            )
+            with self.assertRaises(RUNNER.DeployError):
+                RUNNER.read_module_foundry_runtime_before(backup)
+
+    def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-sector-evidence-"
+        ) as directory:
+            work = Path(directory)
+            backups = work / "backups"
+            failed = work / "failed"
+            state = work / "state"
+            tmp = work / "tmp"
+            installed = work / "installed"
+            candidate = work / "candidate"
+            for path in (backups, failed, state, tmp, installed, candidate):
+                path.mkdir()
+
+            entries = RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
+            for index, rel in enumerate(entries):
+                for root in (installed, candidate):
+                    path = root / rel
+                    path.parent.mkdir(parents=True, exist_ok=True)
+                    path.write_text(f"payload-{index}\n", encoding="utf-8")
+
+            backup = backups / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
+            backup.mkdir()
+            for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256:
+                (backup / name).write_text(f"{name}\n", encoding="utf-8")
+            backup_hashes = {
+                name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
+                for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
+            }
+
+            archive_source = work / "archive-source"
+            (archive_source / "payload").mkdir(parents=True)
+            (archive_source / "manifest.env").write_text(
+                "id="
+                f"{RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID}\n"
+                "component=module-foundry\n"
+                "type=app-overlay\n",
+                encoding="utf-8",
+            )
+            (archive_source / "files.txt").write_text(
+                "\n".join(entries) + "\n",
+                encoding="utf-8",
+            )
+            for rel in entries:
+                source = candidate / rel
+                target = archive_source / "payload" / rel
+                target.parent.mkdir(parents=True, exist_ok=True)
+                target.write_bytes(source.read_bytes())
+            failed_artifact = (
+                failed / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
+            )
+            with tarfile.open(failed_artifact, "w:gz") as archive:
+                archive.add(
+                    archive_source / "manifest.env",
+                    arcname="manifest.env",
+                )
+                archive.add(
+                    archive_source / "files.txt",
+                    arcname="files.txt",
+                )
+                archive.add(archive_source / "payload", arcname="payload")
+            failed_sha256 = hashlib.sha256(
+                failed_artifact.read_bytes()
+            ).hexdigest()
+
+            journal = state / "failed.jsonl"
+            journal.write_text(
+                json.dumps(
+                    {
+                        "artifact": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT,
+                        "backup_id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
+                        "component": "module-foundry",
+                        "id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
+                        "message": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE,
+                        "rollback_status": "not-required",
+                        "sha256": failed_sha256,
+                        "started_apply": True,
+                        "status": "failed",
+                    }
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            manifest = {
+                "id": RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID,
+                "component": "module-foundry",
+                "type": "app-overlay",
+            }
+
+            patches = (
+                mock.patch.object(RUNNER, "BACKUPS_DIR", backups),
+                mock.patch.object(RUNNER, "FAILED_DIR", failed),
+                mock.patch.object(RUNNER, "FAILED_STATE_FILE", journal),
+                mock.patch.object(RUNNER, "TMP_DIR", tmp),
+                mock.patch.object(
+                    RUNNER,
+                    "MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256",
+                    failed_sha256,
+                ),
+                mock.patch.object(
+                    RUNNER,
+                    "MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256",
+                    backup_hashes,
+                ),
+                mock.patch.object(
+                    RUNNER,
+                    "component_root",
+                    return_value=installed,
+                ),
+                mock.patch.object(
+                    RUNNER,
+                    "module_foundry_runtime_inventory",
+                    return_value=runtime(),
+                ),
+            )
+            with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6], patches[7]:
+                result = RUNNER.validate_module_foundry_sector_recovery_evidence(
+                    manifest,
+                    entries,
+                    candidate,
+                )
+                self.assertEqual(
+                    result["mode"],
+                    "failed-overlay-runtime-reconciliation",
+                )
+                (installed / entries[0]).write_text(
+                    "drift\n",
+                    encoding="utf-8",
+                )
+                with self.assertRaises(RUNNER.DeployError):
+                    RUNNER.validate_module_foundry_sector_recovery_evidence(
+                        manifest,
+                        entries,
+                        candidate,
+                    )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)