FIX - NAS DEPLOY: wait for Foundry runtime health

infra/deploy-runner/README.md

@@ -435,6 +435,11 @@ The check is source-aware: if an apply rolls back to the prior source, rollback
 acceptance uses that prior health contract instead of falsely requiring a
 feature which the restored generation does not contain.
 
+Module Foundry activation and rollback wait for the Compose container health
+barrier before strict image/container inventory acceptance. The HTTP health
+endpoint may become ready while Docker still reports `starting`; that timing
+window is not treated as an application or rollback failure.
+
 The Foundry ↔ Map Gateway signing key is not an application or `.env` setting.
 On the first relevant `platform` or `module-foundry` apply, the root-owned
 runner creates `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret`

infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs

@@ -0,0 +1,304 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import {
+  cp,
+  lstat,
+  mkdir,
+  mkdtemp,
+  readdir,
+  readFile,
+  rm,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const declaredFoundryRoot = process.env.NODEDC_FOUNDRY_SOURCE_ROOT;
+if (!declaredFoundryRoot) {
+  throw new Error("NODEDC_FOUNDRY_SOURCE_ROOT_is_required");
+}
+const foundryRoot = resolve(declaredFoundryRoot);
+const expectedSourceCommit = "58800d957632320fa6717ec3fcff972023528759";
+const artifactDir = resolve(
+  process.env.NODEDC_DEPLOY_ARTIFACT_DIR
+    || resolve(scriptDir, "../deploy-artifacts"),
+);
+const [
+  patchId = "module-foundry-map-runtime-recovery-20260809-003",
+  ...extra
+] = process.argv.slice(2);
+
+if (
+  extra.length
+  || !/^module-foundry-map-runtime-recovery-\d{8}-\d{3}$/.test(patchId)
+) {
+  throw new Error(
+    "usage: build-module-foundry-map-runtime-recovery-artifact.mjs "
+      + "[module-foundry-map-runtime-recovery-YYYYMMDD-NNN]",
+  );
+}
+
+const files = Object.freeze([
+  "package.json",
+  "package-lock.json",
+  "apps/catalog/src",
+  "packages",
+  "registry",
+  "runtime-seed/page-layouts/map.json",
+  "scripts",
+  "server/catalog-server.mjs",
+  "server/foundry-mcp.mjs",
+  "server/map-grid-persistence.test.mjs",
+]);
+const ignoredBasenames = new Set([
+  ".DS_Store",
+  ".git",
+  "node_modules",
+  "runtime-data",
+  "dist",
+]);
+const artifact = join(artifactDir, `nodedc-${patchId}.tgz`);
+const checksum = `${artifact}.sha256`;
+const stage = await mkdtemp(
+  join(tmpdir(), "nodedc-foundry-map-runtime-recovery-"),
+);
+
+await assertFresh(artifact);
+await assertExactSource();
+await assertRecoveryBoundary();
+
+try {
+  for (const sourceRelative of files) {
+    await copySafe(
+      resolve(foundryRoot, sourceRelative),
+      join(stage, "payload", sourceRelative),
+    );
+  }
+  await writeFile(
+    join(stage, "manifest.env"),
+    `id=${patchId}\ncomponent=module-foundry\ntype=app-overlay\n`,
+    "utf8",
+  );
+  await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
+  await mkdir(artifactDir, { recursive: true });
+  run("python3", ["-c", canonicalTarScript(), artifact, stage]);
+  const sha256 = digest(await readFile(artifact));
+  await writeFile(
+    checksum,
+    `${sha256}  ${artifact.split("/").at(-1)}\n`,
+    "utf8",
+  );
+  console.log(JSON.stringify({
+    ok: true,
+    patchId,
+    artifact,
+    checksum,
+    sha256,
+    sourceCommit: expectedSourceCommit,
+    services: ["nodedc-module-foundry"],
+    transition: "reconcile-partial-map-source-and-activate-runtime",
+    files,
+  }, null, 2));
+} finally {
+  await rm(stage, { recursive: true, force: true });
+}
+
+async function assertExactSource() {
+  const declared = process.env.NODEDC_FOUNDRY_SOURCE_COMMIT;
+  if (declared !== expectedSourceCommit) {
+    throw new Error("foundry_source_commit_mismatch");
+  }
+  const dotGit = join(foundryRoot, ".git");
+  try {
+    await lstat(dotGit);
+  } catch (error) {
+    if (error?.code === "ENOENT") return;
+    throw error;
+  }
+  const tracked = spawnSync(
+    "git",
+    ["-C", foundryRoot, "diff", "--quiet", "HEAD", "--", ...files],
+    { encoding: "utf8" },
+  );
+  if (tracked.status !== 0) {
+    throw new Error("foundry_selected_source_is_dirty");
+  }
+  const untracked = git(foundryRoot, [
+    "ls-files",
+    "--others",
+    "--exclude-standard",
+    "--",
+    ...files,
+  ]);
+  if (untracked) {
+    throw new Error("foundry_selected_source_has_untracked_files");
+  }
+}
+
+async function assertRecoveryBoundary() {
+  const packageJson = await readFile(join(foundryRoot, "package.json"), "utf8");
+  const packageLock = await readFile(
+    join(foundryRoot, "package-lock.json"),
+    "utf8",
+  );
+  const preview = await readFile(
+    join(foundryRoot, "apps/catalog/src/MapFixturePreview.tsx"),
+    "utf8",
+  );
+  const renderer = await readFile(
+    join(foundryRoot, "apps/catalog/src/CesiumMapRenderer.tsx"),
+    "utf8",
+  );
+  const policy = await readFile(
+    join(foundryRoot, "apps/catalog/src/mapGridPolicy.mjs"),
+    "utf8",
+  );
+  const sector = await readFile(
+    join(foundryRoot, "apps/catalog/src/mapSectorGrid.mjs"),
+    "utf8",
+  );
+  const server = await readFile(
+    join(foundryRoot, "server/catalog-server.mjs"),
+    "utf8",
+  );
+  const registry = await readFile(
+    join(foundryRoot, "registry/registry.json"),
+    "utf8",
+  );
+
+  for (const marker of [
+    "@nodedc/map-cesium-react",
+    "npm run build --workspace @nodedc/map-cesium-react",
+  ]) {
+    if (!packageJson.includes(marker)) {
+      throw new Error(`map_package_boundary_missing:${marker}`);
+    }
+  }
+  if (!packageLock.includes('"packages/map-cesium-react"')) {
+    throw new Error("map_package_lock_boundary_missing");
+  }
+  for (const marker of [
+    "const sectorSpatialEntities = useMemo",
+    "runtimeBindings={[...sectorScopedPrimaryRuntimeBindings, ...referenceRuntimeBindings]}",
+    'label="Скрыть объекты за сектором"',
+  ]) {
+    if (!preview.includes(marker)) {
+      throw new Error(`map_workspace_boundary_missing:${marker}`);
+    }
+  }
+  for (const marker of [
+    "class GridLayerController",
+    "gridLegacyMode",
+    "viewer.flyTo(entity",
+  ]) {
+    if (!renderer.includes(marker)) {
+      throw new Error(`map_renderer_boundary_missing:${marker}`);
+    }
+  }
+  for (const marker of [
+    "gridLodProfiles",
+    "GRID_LOD_HYSTERESIS_RATIO",
+  ]) {
+    if (!policy.includes(marker)) {
+      throw new Error(`map_policy_boundary_missing:${marker}`);
+    }
+  }
+  for (const marker of [
+    "export function geodeticToLocalGridPlane",
+    "export function localSectorAtGeodetic",
+  ]) {
+    if (!sector.includes(marker)) {
+      throw new Error(`map_sector_boundary_missing:${marker}`);
+    }
+  }
+  for (const marker of [
+    "gridLodProfiles",
+    "validateGridLodProfiles",
+  ]) {
+    if (!server.includes(marker)) {
+      throw new Error(`map_server_boundary_missing:${marker}`);
+    }
+  }
+  if (!registry.includes('"@nodedc/map-cesium-react"')) {
+    throw new Error("map_registry_boundary_missing");
+  }
+}
+
+async function copySafe(source, destination) {
+  const sourceStat = await lstat(source);
+  if (sourceStat.isSymbolicLink()) {
+    throw new Error(
+      `source_symlink_rejected:${relative(foundryRoot, source)}`,
+    );
+  }
+  if (sourceStat.isFile()) {
+    await mkdir(dirname(destination), { recursive: true });
+    await cp(source, destination, { force: true, verbatimSymlinks: true });
+    return;
+  }
+  if (!sourceStat.isDirectory()) {
+    throw new Error(`source_type_rejected:${relative(foundryRoot, source)}`);
+  }
+  await mkdir(destination, { recursive: true });
+  for (const entry of await readdir(source, { withFileTypes: true })) {
+    if (
+      ignoredBasenames.has(entry.name)
+      || entry.name.startsWith(".env")
+      || entry.name.endsWith(".tsbuildinfo")
+    ) {
+      continue;
+    }
+    const childSource = join(source, entry.name);
+    const childDestination = join(destination, entry.name);
+    if (entry.isSymbolicLink()) {
+      throw new Error(
+        `source_symlink_rejected:${relative(foundryRoot, childSource)}`,
+      );
+    }
+    await copySafe(childSource, childDestination);
+  }
+}
+
+async function assertFresh(path) {
+  try {
+    await lstat(path);
+  } catch (error) {
+    if (error?.code === "ENOENT") return;
+    throw error;
+  }
+  throw new Error("artifact_already_exists");
+}
+
+function canonicalTarScript() {
+  return [
+    "import gzip,io,pathlib,sys,tarfile",
+    "root=pathlib.Path(sys.argv[2])",
+    "with open(sys.argv[1],'wb') as out:",
+    " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
+    "  with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
+    "   for top in ('manifest.env','files.txt','payload'):",
+    "    p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
+    "    for x in paths:",
+    "     info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
+    "     info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
+    "     with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
+  ].join("\n");
+}
+
+function digest(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function run(command, args) {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    maxBuffer: 128 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    throw new Error(`${command}_failed:${result.stderr || result.stdout}`);
+  }
+}

infra/deploy-runner/nodedc-deploy

@@ -35,6 +35,80 @@ STATE_FILE = STATE_DIR / "applied.jsonl"
 FAILED_STATE_FILE = STATE_DIR / "failed.jsonl"
 LOCK_DIR = STATE_DIR / "deploy.lock"
 DOCKER = Path("/usr/local/bin/docker")
+MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
+MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
+MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
+MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = (
+    "module-foundry-map-runtime-recovery-20260809-003"
+)
+MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = (
+    "package.json",
+    "package-lock.json",
+    "apps/catalog/src",
+    "packages",
+    "registry",
+    "runtime-seed/page-layouts/map.json",
+    "scripts",
+    "server/catalog-server.mjs",
+    "server/foundry-mcp.mjs",
+    "server/map-grid-persistence.test.mjs",
+)
+MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = (
+    "server/map-grid-persistence.test.mjs",
+)
+MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES = (
+    "apps/catalog/src/MapFixturePreview.tsx",
+    "apps/catalog/src/mapPresentationProfile.ts",
+    "apps/catalog/src/mapSectorGrid.d.mts",
+    "apps/catalog/src/mapSectorGrid.mjs",
+    "apps/catalog/src/styles.css",
+    "scripts/map-object-layers.test.mjs",
+    "scripts/map-presentation-filters.test.mjs",
+    "scripts/map-sector-grid.test.mjs",
+)
+MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = (
+    "8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b"
+)
+MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = (
+    "b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3"
+)
+MODULE_FOUNDRY_MAP_FAILED_ENTRIES = MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
+MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = (
+    "module-foundry-map-runtime-recovery-20260809-002"
+)
+MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = (
+    "nodedc-module-foundry-map-runtime-recovery-20260809-002.tgz."
+    "20260809-092439"
+)
+MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = (
+    "bcaffc5dc6098a6c16f8f9362e6e6a89f8eb2745b898bf1964db8ba1f921f603"
+)
+MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = (
+    "module-foundry-module-foundry-map-runtime-recovery-20260809-002-"
+    "20260809-092439"
+)
+MODULE_FOUNDRY_MAP_FAILED_MESSAGE = "Module Foundry runtime inventory mismatch"
+MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS = "failed:DeployError"
+MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = {
+    "existing-files.txt": (
+        "eae18c8687a8b163f5b7bcf362c44c0be53a107c08f7c8799ee4dc6af9fb4e16"
+    ),
+    "files.txt": (
+        "cde91865f65a8d79fee66b3872a8067caa0c988a4a61f75e3dac2904ff814b10"
+    ),
+    "manifest.env": (
+        "cefce93cb613d771a4814af100f9cda7c2c020143268913905a233324edd96d9"
+    ),
+    "missing-files.txt": (
+        "bec2dcad43b34b2f040ad08414117e0654238a1bc051b37257f1b52f288ce689"
+    ),
+    "module-foundry-runtime-before.json": (
+        "bf2ec7678af00349c15d6b8dd9d4b0a43e16f427ed92c5adc842c59fab07e4fe"
+    ),
+    "source-before.tgz": (
+        "4d34654d32fcd8b328bf83d0f9f5f121fed094fe7bc2e9b1e4b322ccd77da8aa"
+    ),
+}
 MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
 MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
 MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
@@ -14961,11 +15035,19 @@ def plan_artifact(artifact):
     provider_catalog_preflight = None
     device_plane_postgres_preflight = None
     device_plane_foundation_recovery_preflight = None
+    module_foundry_map_recovery_preflight = None
     with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
         manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
         reject_terminal_engine_l2_failed_artifact(manifest, sha)
         reject_terminal_device_plane_foundation_artifact(manifest, sha)
         reject_terminal_device_plane_backhaul_artifact(manifest, sha)
+        module_foundry_map_recovery_preflight = (
+            validate_module_foundry_map_recovery_evidence(
+                manifest,
+                entries,
+                payload_dir,
+            )
+        )
         transition_descriptor = None
         transition_preflight = None
         if is_engine_n8n_transition(manifest["component"], entries):
@@ -16520,6 +16602,45 @@ def plan_artifact(artifact):
         print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
         print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
         print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
+        print(
+            "module_foundry_build="
+            f"separate-before-recreate:attempts={MODULE_FOUNDRY_BUILD_ATTEMPTS}"
+        )
+        print("module_foundry_runtime_pull=never")
+        print("module_foundry_rollback=source+image+runtime")
+        if module_foundry_map_recovery_preflight is not None:
+            print(
+                "module_foundry_transition="
+                f"{module_foundry_map_recovery_preflight['mode']}"
+            )
+            print(
+                "module_foundry_failed_predecessor="
+                f"{module_foundry_map_recovery_preflight['failed_patch_id']}"
+            )
+            print(
+                "module_foundry_failed_predecessor_sha256="
+                f"{module_foundry_map_recovery_preflight['failed_artifact_sha256']}"
+            )
+            print(
+                "module_foundry_failed_backup="
+                f"{module_foundry_map_recovery_preflight['failed_backup_id']}"
+            )
+            print(
+                "module_foundry_installed_tree_sha256="
+                f"{module_foundry_map_recovery_preflight['installed_tree_sha256']}"
+            )
+            print(
+                "module_foundry_candidate_tree_sha256="
+                f"{module_foundry_map_recovery_preflight['candidate_tree_sha256']}"
+            )
+            print(
+                "module_foundry_predecessor_container="
+                f"{module_foundry_map_recovery_preflight['runtime_container_id']}"
+            )
+            print(
+                "module_foundry_predecessor_image="
+                f"{module_foundry_map_recovery_preflight['runtime_image_id']}"
+            )
     if component == "device-plane":
         print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
         print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}")
@@ -17241,6 +17362,303 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
     return f"source+runtime-restored:{restored_count}"
 
 
+def is_module_foundry_map_recovery_slice(manifest, entries):
+    return (
+        manifest.get("id") == MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID
+        and manifest.get("component") == "module-foundry"
+        and manifest.get("type") == "app-overlay"
+        and tuple(entries or ()) == MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
+    )
+
+
+def exact_file_map_sha256(files):
+    return hashlib.sha256(
+        json.dumps(
+            files,
+            ensure_ascii=True,
+            separators=(",", ":"),
+            sort_keys=True,
+        ).encode("utf-8")
+    ).hexdigest()
+
+
+def validate_module_foundry_map_recovery_evidence(
+    manifest,
+    entries,
+    payload_dir,
+):
+    if not is_module_foundry_map_recovery_slice(manifest, entries):
+        return None
+
+    backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
+    try:
+        backup_stat = backup_dir.lstat()
+    except FileNotFoundError:
+        die("Module Foundry map recovery backup is missing")
+    if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
+        backup_stat.st_mode
+    ):
+        die("Module Foundry map recovery backup is unsafe")
+    if {child.name for child in backup_dir.iterdir()} != set(
+        MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256
+    ):
+        die("Module Foundry map recovery backup file set mismatch")
+    for name, expected_sha256 in (
+        MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256.items()
+    ):
+        path = backup_dir / name
+        path_stat = path.lstat()
+        if (
+            stat.S_ISLNK(path_stat.st_mode)
+            or not stat.S_ISREG(path_stat.st_mode)
+            or sha256_file(path) != expected_sha256
+        ):
+            die(
+                "Module Foundry map recovery backup drift detected: "
+                f"{name}"
+            )
+
+    failed_artifact = FAILED_DIR / MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
+    try:
+        failed_stat = failed_artifact.lstat()
+    except FileNotFoundError:
+        die("Module Foundry map failed artifact is missing")
+    if (
+        stat.S_ISLNK(failed_stat.st_mode)
+        or not stat.S_ISREG(failed_stat.st_mode)
+        or sha256_file(failed_artifact)
+        != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
+    ):
+        die("Module Foundry map failed artifact evidence mismatch")
+
+    try:
+        state_stat = FAILED_STATE_FILE.lstat()
+        state_lines = FAILED_STATE_FILE.read_text(
+            encoding="utf-8"
+        ).splitlines()
+    except (FileNotFoundError, OSError, UnicodeDecodeError):
+        die("Module Foundry map failed journal is unreadable")
+    if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
+        state_stat.st_mode
+    ):
+        die("Module Foundry map failed journal is unsafe")
+    records = []
+    for line in state_lines:
+        try:
+            value = json.loads(line)
+        except json.JSONDecodeError:
+            die("Module Foundry map failed journal contains invalid JSON")
+        if (
+            isinstance(value, dict)
+            and value.get("id") == MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
+        ):
+            records.append(value)
+    if len(records) != 1:
+        die("Module Foundry map failed journal evidence count mismatch")
+    record = records[0]
+    if (
+        record.get("artifact") != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
+        or record.get("backup_id")
+        != MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
+        or record.get("component") != "module-foundry"
+        or record.get("sha256")
+        != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
+        or record.get("started_apply") is not True
+        or record.get("rollback_status")
+        != MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS
+        or record.get("status") != "failed"
+        or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE
+    ):
+        die("Module Foundry map failed journal evidence mismatch")
+
+    with tempfile.TemporaryDirectory(
+        prefix="module-foundry-map-failed-",
+        dir=TMP_DIR,
+    ) as directory:
+        failed_work = Path(directory)
+        safe_extract(failed_artifact, failed_work)
+        failed_manifest = parse_manifest(failed_work / "manifest.env")
+        failed_entries = parse_files_list(failed_work / "files.txt")
+        if (
+            failed_manifest.get("id")
+            != MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
+            or failed_manifest.get("component") != "module-foundry"
+            or failed_manifest.get("type") != "app-overlay"
+            or tuple(failed_entries)
+            != MODULE_FOUNDRY_MAP_FAILED_ENTRIES
+        ):
+            die("Module Foundry map failed artifact contract mismatch")
+        failed_payload_sha256 = collect_exact_files(
+            failed_work / "payload",
+            failed_entries,
+            "Module Foundry failed map payload",
+        )
+
+    candidate_payload_sha256 = collect_exact_files(
+        payload_dir,
+        entries,
+        "Module Foundry map recovery payload",
+    )
+    if (
+        exact_file_map_sha256(candidate_payload_sha256)
+        != MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
+    ):
+        die("Module Foundry map recovery candidate drift detected")
+    if {
+        rel: candidate_payload_sha256.get(rel)
+        for rel in failed_payload_sha256
+    } != failed_payload_sha256:
+        die("Module Foundry failed map payload changed unexpectedly")
+
+    installed_root = component_root("module-foundry")
+    for rel in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES:
+        path = installed_root / rel
+        if path.exists() or path.is_symlink():
+            die(
+                "Module Foundry map recovery missing predecessor drift: "
+                f"{rel}"
+            )
+    installed_entries = tuple(
+        rel
+        for rel in entries
+        if rel not in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES
+    )
+    installed_payload_sha256 = collect_exact_files(
+        installed_root,
+        installed_entries,
+        "Module Foundry partial map predecessor",
+    )
+    if (
+        exact_file_map_sha256(installed_payload_sha256)
+        != MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
+    ):
+        die("Module Foundry partial map predecessor drift detected")
+    installed_failed_payload_sha256 = {
+        rel: digest
+        for rel, digest in failed_payload_sha256.items()
+        if any(
+            rel == installed or rel.startswith(f"{installed}/")
+            for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES
+        )
+    }
+    if any(
+        not any(
+            rel == installed or rel.startswith(f"{installed}/")
+            for rel in installed_failed_payload_sha256
+        )
+        for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES
+    ):
+        die("Module Foundry failed installed overlap evidence mismatch")
+    if {
+        rel: installed_payload_sha256.get(rel)
+        for rel in installed_failed_payload_sha256
+    } != installed_failed_payload_sha256:
+        die("Module Foundry installed failed map payload drift detected")
+
+    runtime = module_foundry_runtime_inventory()
+    return {
+        "mode": "failed-map-overlay-source+runtime-reconciliation",
+        "failed_patch_id": MODULE_FOUNDRY_MAP_FAILED_PATCH_ID,
+        "failed_artifact_sha256": (
+            MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
+        ),
+        "failed_backup_id": MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID,
+        "installed_tree_sha256": (
+            MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
+        ),
+        "candidate_tree_sha256": (
+            MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
+        ),
+        "runtime_container_id": runtime["containerId"],
+        "runtime_image_id": runtime["imageId"],
+    }
+
+
+def read_module_foundry_runtime_before(backup_dir):
+    runtime = read_strict_json(
+        backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE,
+        "Module Foundry pre-apply runtime inventory",
+        max_bytes=16 * 1024,
+    )
+    if (
+        runtime.get("schemaVersion")
+        != "nodedc.module-foundry.runtime-inventory.v1"
+        or runtime.get("composeProject") != "nodedc-module-foundry"
+        or runtime.get("service") != MODULE_FOUNDRY_SERVICE
+        or not re.fullmatch(
+            r"[a-f0-9]{12,64}",
+            str(runtime.get("containerId", "")),
+        )
+        or not re.fullmatch(
+            r"sha256:[a-f0-9]{64}",
+            str(runtime.get("imageId", "")),
+        )
+        or not isinstance(runtime.get("imageRef"), str)
+        or not re.fullmatch(
+            r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
+            runtime["imageRef"],
+        )
+        or runtime["imageRef"].startswith("sha256:")
+        or "@" in runtime["imageRef"]
+    ):
+        die("Module Foundry pre-apply runtime inventory mismatch")
+    return runtime
+
+
+def rollback_module_foundry_apply(
+    root,
+    backup_dir,
+    entries,
+    current_stamp,
+    applied_services,
+):
+    if tuple(applied_services or ()) != (MODULE_FOUNDRY_SERVICE,):
+        die("Module Foundry rollback service set mismatch")
+    runtime_before = read_module_foundry_runtime_before(backup_dir)
+    restored_count = restore_platform_overlay(
+        root,
+        backup_dir,
+        entries,
+        current_stamp,
+    )
+    current = module_foundry_runtime_inventory(
+        required=False,
+        require_healthy=False,
+    )
+    if (
+        current is not None
+        and current["containerId"] == runtime_before["containerId"]
+        and current["imageId"] == runtime_before["imageId"]
+    ):
+        run_healthchecks("module-foundry", entries, applied_services)
+        return f"source-restored-runtime-unchanged:{restored_count}"
+
+    subprocess.run(
+        [
+            str(DOCKER),
+            "image",
+            "tag",
+            runtime_before["imageId"],
+            runtime_before["imageRef"],
+        ],
+        check=True,
+    )
+    if (
+        inspect_local_image(
+            runtime_before["imageRef"],
+            "Module Foundry rollback image",
+        )
+        != runtime_before["imageId"]
+    ):
+        die("Module Foundry rollback image tag mismatch")
+    run_module_foundry_compose_up(applied_services)
+    run_healthchecks("module-foundry", entries, applied_services)
+    restored = module_foundry_runtime_inventory()
+    if restored["imageId"] != runtime_before["imageId"]:
+        die("Module Foundry rollback runtime image mismatch")
+    return f"source+runtime-restored:{restored_count}"
+
+
 def validate_engine_l2_closed_loop_stable_source(root):
     stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256)
     actual = collect_exact_files(
@@ -17728,6 +18146,67 @@ def run_compose(component, services, entries=None):
     )
 
 
+def run_module_foundry_compose_up(services):
+    if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
+        die("Module Foundry Compose service set mismatch")
+    compose_root = component_compose_root("module-foundry")
+    cmd = [
+        *compose_base_cmd("module-foundry"),
+        "up",
+        "-d",
+        "--force-recreate",
+        "--pull",
+        "never",
+        "--no-deps",
+        *services,
+    ]
+    try:
+        subprocess.run(cmd, cwd=str(compose_root), check=True)
+    except subprocess.CalledProcessError:
+        subprocess.run(
+            [
+                *compose_base_cmd("module-foundry"),
+                "logs",
+                "--no-color",
+                "--tail=180",
+                *services,
+            ],
+            cwd=str(compose_root),
+            check=False,
+        )
+        raise
+    subprocess.run(
+        [*compose_base_cmd("module-foundry"), "ps"],
+        cwd=str(compose_root),
+        check=True,
+    )
+
+
+def run_module_foundry_compose(services):
+    if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
+        die("Module Foundry build service set mismatch")
+    compose_root = component_compose_root("module-foundry")
+    build_cmd = [
+        *compose_base_cmd("module-foundry"),
+        "build",
+        *services,
+    ]
+    for attempt in range(1, MODULE_FOUNDRY_BUILD_ATTEMPTS + 1):
+        try:
+            subprocess.run(build_cmd, cwd=str(compose_root), check=True)
+            break
+        except subprocess.CalledProcessError:
+            if attempt >= MODULE_FOUNDRY_BUILD_ATTEMPTS:
+                raise
+            print(
+                "module-foundry-build-retry="
+                f"{attempt}/{MODULE_FOUNDRY_BUILD_ATTEMPTS - 1}",
+                file=sys.stderr,
+            )
+            time.sleep(2)
+    run_module_foundry_compose_up(services)
+
+
 def run_engine_node_intelligence_compose(services, entries):
     if not is_engine_node_intelligence_transition("engine", entries):
         die("Engine node-intelligence Compose called for unrelated artifact")
@@ -18024,6 +18503,8 @@ def run_component_runtime(component, entries, services):
     prepare_component_runtime(component, entries)
     if is_engine_node_intelligence_transition(component, entries):
         run_engine_node_intelligence_compose(services, entries)
+    elif component == "module-foundry":
+        run_module_foundry_compose(services)
     else:
         run_compose(component, services, entries)
 
@@ -18423,6 +18904,99 @@ def compose_service_container_id(component, service):
     return container_ids[0]
 
 
+def module_foundry_runtime_inventory(required=True, require_healthy=True):
+    result = subprocess.run(
+        [
+            *compose_base_cmd("module-foundry"),
+            "ps",
+            "-q",
+            MODULE_FOUNDRY_SERVICE,
+        ],
+        cwd=str(component_compose_root("module-foundry")),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    container_ids = [
+        line.strip()
+        for line in result.stdout.splitlines()
+        if line.strip()
+    ]
+    if (
+        result.returncode != 0
+        or len(container_ids) > 1
+        or any(
+            not re.fullmatch(r"[a-f0-9]{12,64}", value)
+            for value in container_ids
+        )
+    ):
+        die("Module Foundry runtime topology is unproven")
+    if not container_ids:
+        if required:
+            die("Module Foundry runtime is missing")
+        return None
+    container_id = container_ids[0]
+    containers = docker_json(
+        ["container", "inspect", container_id],
+        "Module Foundry runtime container inspect",
+    )
+    if (
+        not isinstance(containers, list)
+        or len(containers) != 1
+        or not isinstance(containers[0], dict)
+    ):
+        die("Module Foundry runtime container inspect shape mismatch")
+    container = containers[0]
+    config = container.get("Config") or {}
+    labels = config.get("Labels") or {}
+    state = container.get("State") or {}
+    image_id = container.get("Image")
+    image_ref = config.get("Image")
+    health = (state.get("Health") or {}).get("Status")
+    if (
+        labels.get("com.docker.compose.project")
+        != "nodedc-module-foundry"
+        or labels.get("com.docker.compose.service")
+        != MODULE_FOUNDRY_SERVICE
+        or not re.fullmatch(r"sha256:[a-f0-9]{64}", str(image_id or ""))
+        or not isinstance(image_ref, str)
+        or not re.fullmatch(
+            r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
+            image_ref,
+        )
+        or image_ref.startswith("sha256:")
+        or "@" in image_ref
+        or (
+            require_healthy
+            and (
+                state.get("Running") is not True
+                or state.get("Status") != "running"
+                or health not in (None, "healthy")
+            )
+        )
+    ):
+        die("Module Foundry runtime inventory mismatch")
+    return {
+        "schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
+        "composeProject": "nodedc-module-foundry",
+        "service": MODULE_FOUNDRY_SERVICE,
+        "containerId": container_id,
+        "imageId": image_id,
+        "imageRef": image_ref,
+        "health": health or "not-configured",
+    }
+
+
+def validate_module_foundry_candidate_runtime(runtime_before):
+    current = module_foundry_runtime_inventory()
+    if (
+        current["containerId"] == runtime_before["containerId"]
+        or current["imageId"] == runtime_before["imageId"]
+    ):
+        die("Module Foundry candidate generation was not activated")
+    return current
+
+
 def healthcheck_compose_service(component, service):
     healthcheck_container(compose_service_container_id(component, service))
 
@@ -18932,6 +19506,17 @@ def run_healthchecks(component, entries=None, services=None):
         # successful health probe. Wait for the Compose health barrier before
         # the strict immutable-runtime preflight, both on apply and retry.
         healthcheck_compose_service("engine", "nodedc-backend")
+    if component == "module-foundry":
+        if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
+            die("Module Foundry healthcheck service set mismatch")
+        # The HTTP endpoint can answer before Docker publishes the first
+        # successful health probe. Inventory acceptance is intentionally
+        # strict, so wait for the Compose health barrier on both activation
+        # and rollback before inspecting the immutable runtime identity.
+        healthcheck_compose_service(
+            "module-foundry",
+            MODULE_FOUNDRY_SERVICE,
+        )
     for url in component_healthchecks(component, entries, services):
         healthcheck_url(url)
     if (
@@ -19555,6 +20140,8 @@ def apply_artifact(artifact):
     apply_started = False
     engine_backend_recreated = False
     engine_backend_initial_mode = None
+    module_foundry_runtime_before = None
+    module_foundry_map_recovery_preflight = None
     runtime_started = False
     current_stamp = stamp()
 
@@ -19633,6 +20220,13 @@ def apply_artifact(artifact):
                         root.mkdir(parents=True, exist_ok=True)
                     else:
                         die(f"component payload root not found: {root}")
+                module_foundry_map_recovery_preflight = (
+                    validate_module_foundry_map_recovery_evidence(
+                        manifest,
+                        entries,
+                        payload_dir,
+                    )
+                )
                 if is_engine_data_product_publish_grant_slice(component, entries):
                     preflight_engine_data_product_publish_grant_predecessor(payload_dir)
                     publish_backend_preflight = preflight_engine_credential_backend_runtime()
@@ -19912,6 +20506,27 @@ def apply_artifact(artifact):
                             DEVICE_PLANE_RUNTIME_SERVICES
                         )
                     )
+                if component == "module-foundry":
+                    module_foundry_runtime_before = (
+                        module_foundry_runtime_inventory()
+                    )
+                    if (
+                        module_foundry_map_recovery_preflight is not None
+                        and (
+                            module_foundry_runtime_before["containerId"]
+                            != module_foundry_map_recovery_preflight[
+                                "runtime_container_id"
+                            ]
+                            or module_foundry_runtime_before["imageId"]
+                            != module_foundry_map_recovery_preflight[
+                                "runtime_image_id"
+                            ]
+                        )
+                    ):
+                        die(
+                            "Module Foundry map predecessor runtime "
+                            "changed during preflight"
+                        )
 
                 if is_engine_n8n_transition(component, entries):
                     transition_descriptor = read_engine_n8n_transition_descriptor(
@@ -19938,6 +20553,23 @@ def apply_artifact(artifact):
 
                 include_nginx_html = component == "engine" and component_publish_dist(component, entries)
                 create_backup(root, backup_dir, entries, include_nginx_html)
+                if component == "module-foundry":
+                    if module_foundry_runtime_before is None:
+                        die("Module Foundry pre-apply runtime inventory is missing")
+                    module_foundry_runtime_path = (
+                        backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE
+                    )
+                    module_foundry_runtime_path.write_text(
+                        json.dumps(
+                            module_foundry_runtime_before,
+                            ensure_ascii=False,
+                            indent=2,
+                            sort_keys=True,
+                        )
+                        + "\n",
+                        encoding="utf-8",
+                    )
+                    module_foundry_runtime_path.chmod(0o600)
                 if component == "device-plane":
                     if device_plane_runtime_before is None:
                         die("Device Plane pre-apply runtime inventory is missing")
@@ -20043,6 +20675,12 @@ def apply_artifact(artifact):
                 ):
                     die("Engine L2 closed-loop app generation was not recreated")
                 run_healthchecks(component, entries, services)
+                if component == "module-foundry":
+                    if module_foundry_runtime_before is None:
+                        die("Module Foundry candidate predecessor is missing")
+                    validate_module_foundry_candidate_runtime(
+                        module_foundry_runtime_before
+                    )
                 if is_device_plane_b2_discovery_ingress_slice(
                     component,
                     entries,
@@ -20249,6 +20887,35 @@ def apply_artifact(artifact):
                     except Exception as rollback_exc:
                         rollback_status = f"failed:{type(rollback_exc).__name__}"
                         print("engine-automatic-rollback=failed", file=sys.stderr)
+                elif (
+                    component == "module-foundry"
+                    and entries is not None
+                    and services is not None
+                ):
+                    try:
+                        restored_state = rollback_module_foundry_apply(
+                            root,
+                            backup_dir,
+                            entries,
+                            current_stamp,
+                            services,
+                        )
+                        rollback_status = (
+                            f"ok:module-foundry-overlay:{restored_state}"
+                        )
+                        print(
+                            "module-foundry-automatic-rollback="
+                            f"{rollback_status}",
+                            file=sys.stderr,
+                        )
+                    except Exception as rollback_exc:
+                        rollback_status = (
+                            f"failed:{type(rollback_exc).__name__}"
+                        )
+                        print(
+                            "module-foundry-automatic-rollback=failed",
+                            file=sys.stderr,
+                        )
                 elif (
                     component == "platform"
                     and entries is not None

infra/deploy-runner/test_module_foundry_map_runtime_recovery_artifact.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+
+import hashlib
+import importlib.machinery
+import importlib.util
+import json
+import os
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path, PurePosixPath
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+PLATFORM_ROOT = SCRIPT_DIR.parent.parent
+WORKSPACE_ROOT = PLATFORM_ROOT.parent.parent
+FOUNDRY_ROOT = WORKSPACE_ROOT / "NODEDC_DESIGN_GUIDELINE"
+BUILDER = (
+    SCRIPT_DIR / "build-module-foundry-map-runtime-recovery-artifact.mjs"
+)
+RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
+SOURCE_COMMIT = "58800d957632320fa6717ec3fcff972023528759"
+PATCH_ID = "module-foundry-map-runtime-recovery-20260809-999"
+EXPECTED_FILES = (
+    "package.json",
+    "package-lock.json",
+    "apps/catalog/src",
+    "packages",
+    "registry",
+    "runtime-seed/page-layouts/map.json",
+    "scripts",
+    "server/catalog-server.mjs",
+    "server/foundry-mcp.mjs",
+    "server/map-grid-persistence.test.mjs",
+)
+
+
+def load_runner():
+    loader = importlib.machinery.SourceFileLoader(
+        "nodedc_module_foundry_map_runtime_recovery_artifact",
+        str(RUNNER_PATH),
+    )
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    module = importlib.util.module_from_spec(spec)
+    loader.exec_module(module)
+    return module
+
+
+RUNNER = load_runner()
+
+
+class ModuleFoundryMapRuntimeRecoveryArtifactTest(unittest.TestCase):
+    def export_source(self, root):
+        archive_path = root / "source.tar"
+        source_root = root / "source"
+        source_root.mkdir()
+        subprocess.run(
+            [
+                "git",
+                "-C",
+                str(FOUNDRY_ROOT),
+                "archive",
+                "--format=tar",
+                f"--output={archive_path}",
+                SOURCE_COMMIT,
+            ],
+            check=True,
+        )
+        with tarfile.open(archive_path, "r:") as archive:
+            for member in archive.getmembers():
+                path = PurePosixPath(member.name)
+                self.assertFalse(path.is_absolute())
+                self.assertNotIn("..", path.parts)
+            archive.extractall(source_root, filter="data")
+        return source_root
+
+    def build(self, source_root, artifact_dir):
+        environment = os.environ.copy()
+        environment["NODEDC_FOUNDRY_SOURCE_ROOT"] = str(source_root)
+        environment["NODEDC_FOUNDRY_SOURCE_COMMIT"] = SOURCE_COMMIT
+        environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
+        completed = subprocess.run(
+            ["node", str(BUILDER), PATCH_ID],
+            cwd=PLATFORM_ROOT,
+            env=environment,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return json.loads(completed.stdout)
+
+    def test_builder_is_deterministic_coherent_and_runner_compatible(self):
+        with tempfile.TemporaryDirectory(
+            prefix="nodedc-foundry-map-runtime-recovery-"
+        ) as directory:
+            root = Path(directory)
+            source_root = self.export_source(root)
+            first = self.build(source_root, root / "first")
+            second = self.build(source_root, root / "second")
+            first_artifact = Path(first["artifact"])
+            second_artifact = Path(second["artifact"])
+
+            self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes())
+            self.assertEqual(
+                first["sha256"],
+                hashlib.sha256(first_artifact.read_bytes()).hexdigest(),
+            )
+            self.assertEqual(first["sourceCommit"], SOURCE_COMMIT)
+            self.assertEqual(tuple(first["files"]), EXPECTED_FILES)
+            self.assertEqual(first["services"], ["nodedc-module-foundry"])
+
+            extracted = root / "loaded"
+            extracted.mkdir()
+            manifest, entries, payload = RUNNER.load_artifact(
+                first_artifact,
+                extracted,
+            )
+            self.assertEqual(manifest["id"], PATCH_ID)
+            self.assertEqual(manifest["component"], "module-foundry")
+            self.assertEqual(tuple(entries), EXPECTED_FILES)
+            self.assertEqual(
+                RUNNER.component_services("module-foundry", entries),
+                ("nodedc-module-foundry",),
+            )
+
+            preview = (
+                payload / "apps/catalog/src/MapFixturePreview.tsx"
+            ).read_text(encoding="utf-8")
+            renderer = (
+                payload / "apps/catalog/src/CesiumMapRenderer.tsx"
+            ).read_text(encoding="utf-8")
+            package_lock = (payload / "package-lock.json").read_text(
+                encoding="utf-8"
+            )
+            self.assertIn("const sectorSpatialEntities = useMemo", preview)
+            self.assertIn("class GridLayerController", renderer)
+            self.assertIn('"packages/map-cesium-react"', package_lock)
+            self.assertTrue(
+                (payload / "packages/map-cesium-react/src/index.ts").is_file()
+            )
+
+            with tarfile.open(first_artifact, "r:gz") as archive:
+                members = archive.getmembers()
+                names = [member.name for member in members]
+                regular_payloads = [
+                    archive.extractfile(member).read()
+                    for member in members
+                    if member.isfile()
+                ]
+            self.assertFalse(
+                any(Path(name).name.startswith("._") for name in names)
+            )
+            self.assertFalse(
+                any(
+                    "/.git/" in name
+                    or "/node_modules/" in name
+                    or "/runtime-data/" in name
+                    or "/dist/" in name
+                    for name in names
+                )
+            )
+            self.assertNotIn(
+                "payload/scripts/spark-governance-contract.test.mjs",
+                names,
+            )
+            self.assertNotIn(
+                b"-----BEGIN PRIVATE KEY-----",
+                b"\n".join(regular_payloads),
+            )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

infra/deploy-runner/test_module_foundry_runtime_recovery.py

@@ -0,0 +1,519 @@
+#!/usr/bin/env python3
+
+import hashlib
+import importlib.machinery
+import importlib.util
+import json
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from contextlib import ExitStack
+from pathlib import Path
+from unittest import mock
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
+
+
+def load_runner():
+    loader = importlib.machinery.SourceFileLoader(
+        "nodedc_module_foundry_runtime_recovery",
+        str(RUNNER_PATH),
+    )
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    module = importlib.util.module_from_spec(spec)
+    loader.exec_module(module)
+    return module
+
+
+RUNNER = load_runner()
+
+
+def runtime(container_char="a", image_char="b"):
+    return {
+        "schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
+        "composeProject": "nodedc-module-foundry",
+        "service": RUNNER.MODULE_FOUNDRY_SERVICE,
+        "containerId": container_char * 64,
+        "imageId": f"sha256:{image_char * 64}",
+        "imageRef": "nodedc-module-foundry-nodedc-module-foundry:latest",
+        "health": "healthy",
+    }
+
+
+class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
+    def test_healthchecks_wait_for_compose_health_before_http(self):
+        events = []
+        with (
+            mock.patch.object(
+                RUNNER,
+                "healthcheck_compose_service",
+                side_effect=lambda component, service: events.append(
+                    ("compose", component, service)
+                ),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "component_healthchecks",
+                return_value=("http://foundry/healthz",),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "healthcheck_url",
+                side_effect=lambda check: events.append(("http", check)),
+            ),
+        ):
+            RUNNER.run_healthchecks(
+                "module-foundry",
+                ("package.json",),
+                (RUNNER.MODULE_FOUNDRY_SERVICE,),
+            )
+
+        self.assertEqual(
+            events,
+            [
+                (
+                    "compose",
+                    "module-foundry",
+                    RUNNER.MODULE_FOUNDRY_SERVICE,
+                ),
+                ("http", "http://foundry/healthz"),
+            ],
+        )
+
+    def test_build_retries_before_any_runtime_recreate(self):
+        first_failure = subprocess.CalledProcessError(17, ["docker", "compose"])
+        completed = subprocess.CompletedProcess([], 0)
+        with (
+            mock.patch.object(
+                RUNNER,
+                "component_compose_root",
+                return_value=Path("/compose"),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "compose_base_cmd",
+                return_value=["docker", "compose"],
+            ),
+            mock.patch.object(
+                RUNNER.subprocess,
+                "run",
+                side_effect=[first_failure, completed],
+            ) as run,
+            mock.patch.object(RUNNER.time, "sleep") as sleep,
+            mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
+        ):
+            RUNNER.run_module_foundry_compose(
+                (RUNNER.MODULE_FOUNDRY_SERVICE,)
+            )
+
+        expected = [
+            "docker",
+            "compose",
+            "build",
+            RUNNER.MODULE_FOUNDRY_SERVICE,
+        ]
+        self.assertEqual(run.call_count, 2)
+        self.assertEqual(run.call_args_list[0].args[0], expected)
+        self.assertEqual(run.call_args_list[1].args[0], expected)
+        sleep.assert_called_once_with(2)
+        up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
+
+    def test_runtime_recreate_uses_built_image_without_build_or_pull(self):
+        completed = subprocess.CompletedProcess([], 0)
+        with (
+            mock.patch.object(
+                RUNNER,
+                "component_compose_root",
+                return_value=Path("/compose"),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "compose_base_cmd",
+                return_value=["docker", "compose"],
+            ),
+            mock.patch.object(
+                RUNNER.subprocess,
+                "run",
+                return_value=completed,
+            ) as run,
+        ):
+            RUNNER.run_module_foundry_compose_up(
+                (RUNNER.MODULE_FOUNDRY_SERVICE,)
+            )
+
+        up = run.call_args_list[0].args[0]
+        self.assertEqual(
+            up,
+            [
+                "docker",
+                "compose",
+                "up",
+                "-d",
+                "--force-recreate",
+                "--pull",
+                "never",
+                "--no-deps",
+                RUNNER.MODULE_FOUNDRY_SERVICE,
+            ],
+        )
+        self.assertNotIn("--build", up)
+        self.assertEqual(run.call_args_list[1].args[0], ["docker", "compose", "ps"])
+
+    def test_rollback_restores_source_and_keeps_unchanged_runtime(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-rollback-unchanged-"
+        ) as directory:
+            work = Path(directory)
+            root = work / "source"
+            backup = work / "backup"
+            tmp = work / "tmp"
+            root.mkdir()
+            backup.mkdir()
+            tmp.mkdir()
+            entries = ("old.txt", "new.txt")
+            (root / "old.txt").write_text("old\n", encoding="utf-8")
+            RUNNER.create_backup(root, backup, entries, False)
+            (root / "old.txt").write_text("candidate\n", encoding="utf-8")
+            (root / "new.txt").write_text("candidate\n", encoding="utf-8")
+            before = runtime()
+            (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
+                json.dumps(before),
+                encoding="utf-8",
+            )
+
+            with (
+                mock.patch.object(RUNNER, "TMP_DIR", tmp),
+                mock.patch.object(
+                    RUNNER,
+                    "module_foundry_runtime_inventory",
+                    return_value=before,
+                ),
+                mock.patch.object(RUNNER, "run_healthchecks") as health,
+                mock.patch.object(RUNNER.subprocess, "run") as process,
+            ):
+                result = RUNNER.rollback_module_foundry_apply(
+                    root,
+                    backup,
+                    entries,
+                    "20260809-000000",
+                    (RUNNER.MODULE_FOUNDRY_SERVICE,),
+                )
+
+            self.assertEqual(
+                result,
+                "source-restored-runtime-unchanged:2",
+            )
+            self.assertEqual(
+                (root / "old.txt").read_text(encoding="utf-8"),
+                "old\n",
+            )
+            self.assertFalse((root / "new.txt").exists())
+            process.assert_not_called()
+            health.assert_called_once()
+
+    def test_rollback_retags_and_recreates_changed_runtime(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-rollback-changed-"
+        ) as directory:
+            work = Path(directory)
+            root = work / "source"
+            backup = work / "backup"
+            tmp = work / "tmp"
+            root.mkdir()
+            backup.mkdir()
+            tmp.mkdir()
+            entries = ("old.txt",)
+            (root / "old.txt").write_text("old\n", encoding="utf-8")
+            RUNNER.create_backup(root, backup, entries, False)
+            (root / "old.txt").write_text("candidate\n", encoding="utf-8")
+            before = runtime()
+            candidate = runtime("c", "d")
+            (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
+                json.dumps(before),
+                encoding="utf-8",
+            )
+
+            with (
+                mock.patch.object(RUNNER, "TMP_DIR", tmp),
+                mock.patch.object(
+                    RUNNER,
+                    "module_foundry_runtime_inventory",
+                    side_effect=[candidate, before],
+                ) as inventory,
+                mock.patch.object(
+                    RUNNER,
+                    "inspect_local_image",
+                    return_value=before["imageId"],
+                ),
+                mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
+                mock.patch.object(RUNNER, "run_healthchecks") as health,
+                mock.patch.object(RUNNER.subprocess, "run") as process,
+            ):
+                result = RUNNER.rollback_module_foundry_apply(
+                    root,
+                    backup,
+                    entries,
+                    "20260809-000000",
+                    (RUNNER.MODULE_FOUNDRY_SERVICE,),
+                )
+
+            self.assertEqual(result, "source+runtime-restored:1")
+            self.assertEqual(
+                inventory.call_args_list[0].kwargs,
+                {"required": False, "require_healthy": False},
+            )
+            process.assert_called_once_with(
+                [
+                    str(RUNNER.DOCKER),
+                    "image",
+                    "tag",
+                    before["imageId"],
+                    before["imageRef"],
+                ],
+                check=True,
+            )
+            up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
+            health.assert_called_once()
+
+    def test_candidate_must_replace_both_container_and_image(self):
+        before = runtime()
+        for candidate in (
+            runtime("a", "c"),
+            runtime("c", "b"),
+        ):
+            with mock.patch.object(
+                RUNNER,
+                "module_foundry_runtime_inventory",
+                return_value=candidate,
+            ):
+                with self.assertRaises(RUNNER.DeployError):
+                    RUNNER.validate_module_foundry_candidate_runtime(before)
+
+    def test_preapply_runtime_rejects_digest_only_image_reference(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-runtime-reference-"
+        ) as directory:
+            backup = Path(directory)
+            before = runtime()
+            before["imageRef"] = before["imageId"]
+            (backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
+                json.dumps(before),
+                encoding="utf-8",
+            )
+            with self.assertRaises(RUNNER.DeployError):
+                RUNNER.read_module_foundry_runtime_before(backup)
+
+    def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
+        with tempfile.TemporaryDirectory(
+            prefix="module-foundry-map-evidence-"
+        ) as directory:
+            work = Path(directory)
+            backups = work / "backups"
+            failed = work / "failed"
+            state = work / "state"
+            tmp = work / "tmp"
+            installed = work / "installed"
+            candidate = work / "candidate"
+            for path in (backups, failed, state, tmp, installed, candidate):
+                path.mkdir()
+
+            recovery_entries = (
+                "apps/catalog/src",
+                "scripts",
+                "server/map-grid-persistence.test.mjs",
+            )
+            missing_entries = ("server/map-grid-persistence.test.mjs",)
+            failed_entries = (
+                "apps/catalog/src/MapFixturePreview.tsx",
+                "scripts/map-sector-grid.test.mjs",
+                "server/map-grid-persistence.test.mjs",
+            )
+            candidate_files = {
+                "apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
+                "apps/catalog/src/CesiumMapRenderer.tsx": "renderer-next\n",
+                "scripts/map-sector-grid.test.mjs": "sector-test-current\n",
+                "scripts/map-grid-lod.test.mjs": "grid-test-next\n",
+                "server/map-grid-persistence.test.mjs": "server-test-next\n",
+            }
+            installed_files = {
+                "apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
+                "apps/catalog/src/CesiumMapRenderer.tsx": "renderer-old\n",
+                "scripts/map-sector-grid.test.mjs": "sector-test-current\n",
+                "scripts/map-grid-lod.test.mjs": "grid-test-old\n",
+            }
+            for root, values in (
+                (candidate, candidate_files),
+                (installed, installed_files),
+            ):
+                for rel, value in values.items():
+                    path = root / rel
+                    path.parent.mkdir(parents=True, exist_ok=True)
+                    path.write_text(value, encoding="utf-8")
+
+            failed_patch_id = "module-foundry-map-failed-test"
+            failed_artifact_name = "module-foundry-map-failed-test.tgz.1"
+            failed_backup_id = "module-foundry-map-failed-test-backup"
+            recovery_patch_id = "module-foundry-map-recovery-test"
+            failed_message = "test build failed"
+
+            backup = backups / failed_backup_id
+            backup.mkdir()
+            backup_names = {
+                "existing-files.txt",
+                "files.txt",
+                "manifest.env",
+                "missing-files.txt",
+                "module-foundry-runtime-before.json",
+                "source-before.tgz",
+            }
+            for name in backup_names:
+                (backup / name).write_text(f"{name}\n", encoding="utf-8")
+            backup_hashes = {
+                name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
+                for name in backup_names
+            }
+
+            archive_source = work / "archive-source"
+            (archive_source / "payload").mkdir(parents=True)
+            (archive_source / "manifest.env").write_text(
+                "id="
+                f"{failed_patch_id}\n"
+                "component=module-foundry\n"
+                "type=app-overlay\n",
+                encoding="utf-8",
+            )
+            (archive_source / "files.txt").write_text(
+                "\n".join(failed_entries) + "\n",
+                encoding="utf-8",
+            )
+            for rel in failed_entries:
+                source = candidate / rel
+                target = archive_source / "payload" / rel
+                target.parent.mkdir(parents=True, exist_ok=True)
+                target.write_bytes(source.read_bytes())
+            failed_artifact = failed / failed_artifact_name
+            with tarfile.open(failed_artifact, "w:gz") as archive:
+                archive.add(
+                    archive_source / "manifest.env",
+                    arcname="manifest.env",
+                )
+                archive.add(
+                    archive_source / "files.txt",
+                    arcname="files.txt",
+                )
+                archive.add(archive_source / "payload", arcname="payload")
+            failed_sha256 = hashlib.sha256(
+                failed_artifact.read_bytes()
+            ).hexdigest()
+
+            journal = state / "failed.jsonl"
+            journal.write_text(
+                json.dumps(
+                    {
+                        "artifact": failed_artifact_name,
+                        "backup_id": failed_backup_id,
+                        "component": "module-foundry",
+                        "id": failed_patch_id,
+                        "message": failed_message,
+                        "rollback_status": "failed:DeployError",
+                        "sha256": failed_sha256,
+                        "started_apply": True,
+                        "status": "failed",
+                    }
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            manifest = {
+                "id": recovery_patch_id,
+                "component": "module-foundry",
+                "type": "app-overlay",
+            }
+            candidate_tree = RUNNER.exact_file_map_sha256(
+                RUNNER.collect_exact_files(
+                    candidate,
+                    recovery_entries,
+                    "candidate",
+                )
+            )
+            installed_tree = RUNNER.exact_file_map_sha256(
+                RUNNER.collect_exact_files(
+                    installed,
+                    recovery_entries[:-1],
+                    "installed",
+                )
+            )
+
+            patches = {
+                "BACKUPS_DIR": backups,
+                "FAILED_DIR": failed,
+                "FAILED_STATE_FILE": journal,
+                "TMP_DIR": tmp,
+                "MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID": recovery_patch_id,
+                "MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES": recovery_entries,
+                "MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES": (
+                    missing_entries
+                ),
+                "MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES": (
+                    failed_entries[:-1]
+                ),
+                "MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256": (
+                    installed_tree
+                ),
+                "MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256": (
+                    candidate_tree
+                ),
+                "MODULE_FOUNDRY_MAP_FAILED_ENTRIES": failed_entries,
+                "MODULE_FOUNDRY_MAP_FAILED_PATCH_ID": failed_patch_id,
+                "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT": failed_artifact_name,
+                "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256": failed_sha256,
+                "MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID": failed_backup_id,
+                "MODULE_FOUNDRY_MAP_FAILED_MESSAGE": failed_message,
+                "MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS": (
+                    "failed:DeployError"
+                ),
+                "MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256": backup_hashes,
+            }
+            with ExitStack() as stack:
+                for name, value in patches.items():
+                    stack.enter_context(mock.patch.object(RUNNER, name, value))
+                stack.enter_context(
+                    mock.patch.object(
+                        RUNNER,
+                        "component_root",
+                        return_value=installed,
+                    )
+                )
+                stack.enter_context(
+                    mock.patch.object(
+                        RUNNER,
+                        "module_foundry_runtime_inventory",
+                        return_value=runtime(),
+                    )
+                )
+                result = RUNNER.validate_module_foundry_map_recovery_evidence(
+                    manifest,
+                    recovery_entries,
+                    candidate,
+                )
+                self.assertEqual(
+                    result["mode"],
+                    "failed-map-overlay-source+runtime-reconciliation",
+                )
+                (installed / failed_entries[0]).write_text(
+                    "drift\n",
+                    encoding="utf-8",
+                )
+                with self.assertRaises(RUNNER.DeployError):
+                    RUNNER.validate_module_foundry_map_recovery_evidence(
+                        manifest,
+                        recovery_entries,
+                        candidate,
+                    )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)