SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

FIX - NAS DEPLOY: recover Module Foundry runtime safely

Browse Source
This commit is contained in:
Codex
2026-08-09 02:18:07 +03:00
parent 07224c6f0d
commit 0565cd2642
2 changed files with 961 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+558
View File
@@ -35,6 +35,62 @@ STATE_FILE = STATE_DIR / "applied.jsonl"
FAILED_STATE_FILE = STATE_DIR / "failed.jsonl"
LOCK_DIR = STATE_DIR / "deploy.lock"
DOCKER = Path("/usr/local/bin/docker")
MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = (
"module-foundry-map-sector-workspace-20260809-001"
)
MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
"apps/catalog/src/MapFixturePreview.tsx",
"apps/catalog/src/mapPresentationProfile.ts",
"apps/catalog/src/mapSectorGrid.d.mts",
"apps/catalog/src/mapSectorGrid.mjs",
"apps/catalog/src/styles.css",
"scripts/map-object-layers.test.mjs",
"scripts/map-presentation-filters.test.mjs",
"scripts/map-sector-grid.test.mjs",
)
MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = (
"module-foundry-map-sector-workspace-20260808-001"
)
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = (
"nodedc-module-foundry-map-sector-workspace-20260808-001.tgz."
"20260809-014720"
)
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = (
"0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740"
)
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = (
"module-foundry-module-foundry-map-sector-workspace-20260808-001-"
"20260809-014720"
)
MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
"Command '['/usr/local/bin/docker', 'compose', '-p', "
"'nodedc-module-foundry', '--env-file', "
"'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
"'/volume1/docker/nodedc-platform/module-foundry/source/infra/"
"docker-compose.module-foundry.yml', 'up', '-d', '--force-recreate', "
"'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
"exit status 17."
)
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = {
"existing-files.txt": (
"57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c"
),
"files.txt": (
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
),
"manifest.env": (
"cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2"
),
"missing-files.txt": (
"1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f"
),
"source-before.tgz": (
"bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562"
),
}
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
@@ -14961,11 +15017,19 @@ def plan_artifact(artifact):
provider_catalog_preflight = None
device_plane_postgres_preflight = None
device_plane_foundation_recovery_preflight = None
module_foundry_sector_recovery_preflight = None
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
reject_terminal_engine_l2_failed_artifact(manifest, sha)
reject_terminal_device_plane_foundation_artifact(manifest, sha)
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
module_foundry_sector_recovery_preflight = (
validate_module_foundry_sector_recovery_evidence(
manifest,
entries,
payload_dir,
)
)
transition_descriptor = None
transition_preflight = None
if is_engine_n8n_transition(manifest["component"], entries):
@@ -16520,6 +16584,37 @@ def plan_artifact(artifact):
print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
print(
"module_foundry_build="
f"separate-before-recreate:attempts={MODULE_FOUNDRY_BUILD_ATTEMPTS}"
)
print("module_foundry_runtime_pull=never")
print("module_foundry_rollback=source+image+runtime")
if module_foundry_sector_recovery_preflight is not None:
print(
"module_foundry_transition="
f"{module_foundry_sector_recovery_preflight['mode']}"
)
print(
"module_foundry_failed_predecessor="
f"{module_foundry_sector_recovery_preflight['failed_patch_id']}"
)
print(
"module_foundry_failed_predecessor_sha256="
f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}"
)
print(
"module_foundry_failed_backup="
f"{module_foundry_sector_recovery_preflight['failed_backup_id']}"
)
print(
"module_foundry_predecessor_container="
f"{module_foundry_sector_recovery_preflight['runtime_container_id']}"
)
print(
"module_foundry_predecessor_image="
f"{module_foundry_sector_recovery_preflight['runtime_image_id']}"
)
if component == "device-plane":
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}")
@@ -17241,6 +17336,236 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
return f"source+runtime-restored:{restored_count}"
def is_module_foundry_sector_recovery_slice(manifest, entries):
return (
manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID
and manifest.get("component") == "module-foundry"
and manifest.get("type") == "app-overlay"
and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
)
def validate_module_foundry_sector_recovery_evidence(
manifest,
entries,
payload_dir,
):
if not is_module_foundry_sector_recovery_slice(manifest, entries):
return None
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Module Foundry sector recovery backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Module Foundry sector recovery backup is unsafe")
if {child.name for child in backup_dir.iterdir()} != set(
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
):
die("Module Foundry sector recovery backup file set mismatch")
for name, expected_sha256 in (
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items()
):
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Module Foundry sector recovery backup drift detected: "
f"{name}"
)
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Module Foundry sector failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
):
die("Module Foundry sector failed artifact evidence mismatch")
try:
state_stat = FAILED_STATE_FILE.lstat()
state_lines = FAILED_STATE_FILE.read_text(
encoding="utf-8"
).splitlines()
except (FileNotFoundError, OSError, UnicodeDecodeError):
die("Module Foundry sector failed journal is unreadable")
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
state_stat.st_mode
):
die("Module Foundry sector failed journal is unsafe")
records = []
for line in state_lines:
try:
value = json.loads(line)
except json.JSONDecodeError:
die("Module Foundry sector failed journal contains invalid JSON")
if (
isinstance(value, dict)
and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
):
records.append(value)
if len(records) != 1:
die("Module Foundry sector failed journal evidence count mismatch")
record = records[0]
if (
record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
or record.get("backup_id")
!= MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
or record.get("component") != "module-foundry"
or record.get("sha256")
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "not-required"
or record.get("status") != "failed"
or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE
):
die("Module Foundry sector failed journal evidence mismatch")
with tempfile.TemporaryDirectory(
prefix="module-foundry-sector-failed-",
dir=TMP_DIR,
) as directory:
failed_work = Path(directory)
safe_extract(failed_artifact, failed_work)
failed_manifest = parse_manifest(failed_work / "manifest.env")
failed_entries = parse_files_list(failed_work / "files.txt")
if (
failed_manifest.get("id")
!= MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
or failed_manifest.get("component") != "module-foundry"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
):
die("Module Foundry sector failed artifact contract mismatch")
failed_payload_sha256 = collect_exact_files(
failed_work / "payload",
failed_entries,
"Module Foundry failed sector payload",
)
candidate_payload_sha256 = collect_exact_files(
payload_dir,
entries,
"Module Foundry sector recovery payload",
)
if candidate_payload_sha256 != failed_payload_sha256:
die("Module Foundry sector recovery payload changed unexpectedly")
installed_payload_sha256 = collect_exact_files(
component_root("module-foundry"),
entries,
"Module Foundry partial sector predecessor",
)
if installed_payload_sha256 != candidate_payload_sha256:
die("Module Foundry partial sector predecessor drift detected")
runtime = module_foundry_runtime_inventory()
return {
"mode": "failed-overlay-runtime-reconciliation",
"failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
"failed_artifact_sha256": (
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
),
"failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
"runtime_container_id": runtime["containerId"],
"runtime_image_id": runtime["imageId"],
}
def read_module_foundry_runtime_before(backup_dir):
runtime = read_strict_json(
backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE,
"Module Foundry pre-apply runtime inventory",
max_bytes=16 * 1024,
)
if (
runtime.get("schemaVersion")
!= "nodedc.module-foundry.runtime-inventory.v1"
or runtime.get("composeProject") != "nodedc-module-foundry"
or runtime.get("service") != MODULE_FOUNDRY_SERVICE
or not re.fullmatch(
r"[a-f0-9]{12,64}",
str(runtime.get("containerId", "")),
)
or not re.fullmatch(
r"sha256:[a-f0-9]{64}",
str(runtime.get("imageId", "")),
)
or not isinstance(runtime.get("imageRef"), str)
or not re.fullmatch(
r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
runtime["imageRef"],
)
or runtime["imageRef"].startswith("sha256:")
or "@" in runtime["imageRef"]
):
die("Module Foundry pre-apply runtime inventory mismatch")
return runtime
def rollback_module_foundry_apply(
root,
backup_dir,
entries,
current_stamp,
applied_services,
):
if tuple(applied_services or ()) != (MODULE_FOUNDRY_SERVICE,):
die("Module Foundry rollback service set mismatch")
runtime_before = read_module_foundry_runtime_before(backup_dir)
restored_count = restore_platform_overlay(
root,
backup_dir,
entries,
current_stamp,
)
current = module_foundry_runtime_inventory(required=False)
if (
current is not None
and current["containerId"] == runtime_before["containerId"]
and current["imageId"] == runtime_before["imageId"]
):
run_healthchecks("module-foundry", entries, applied_services)
return f"source-restored-runtime-unchanged:{restored_count}"
subprocess.run(
[
str(DOCKER),
"image",
"tag",
runtime_before["imageId"],
runtime_before["imageRef"],
],
check=True,
)
if (
inspect_local_image(
runtime_before["imageRef"],
"Module Foundry rollback image",
)
!= runtime_before["imageId"]
):
die("Module Foundry rollback image tag mismatch")
run_module_foundry_compose_up(applied_services)
run_healthchecks("module-foundry", entries, applied_services)
restored = module_foundry_runtime_inventory()
if restored["imageId"] != runtime_before["imageId"]:
die("Module Foundry rollback runtime image mismatch")
return f"source+runtime-restored:{restored_count}"
def validate_engine_l2_closed_loop_stable_source(root):
stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256)
actual = collect_exact_files(
@@ -17728,6 +18053,67 @@ def run_compose(component, services, entries=None):
)
def run_module_foundry_compose_up(services):
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
die("Module Foundry Compose service set mismatch")
compose_root = component_compose_root("module-foundry")
cmd = [
*compose_base_cmd("module-foundry"),
"up",
"-d",
"--force-recreate",
"--pull",
"never",
"--no-deps",
*services,
]
try:
subprocess.run(cmd, cwd=str(compose_root), check=True)
except subprocess.CalledProcessError:
subprocess.run(
[
*compose_base_cmd("module-foundry"),
"logs",
"--no-color",
"--tail=180",
*services,
],
cwd=str(compose_root),
check=False,
)
raise
subprocess.run(
[*compose_base_cmd("module-foundry"), "ps"],
cwd=str(compose_root),
check=True,
)
def run_module_foundry_compose(services):
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
die("Module Foundry build service set mismatch")
compose_root = component_compose_root("module-foundry")
build_cmd = [
*compose_base_cmd("module-foundry"),
"build",
*services,
]
for attempt in range(1, MODULE_FOUNDRY_BUILD_ATTEMPTS + 1):
try:
subprocess.run(build_cmd, cwd=str(compose_root), check=True)
break
except subprocess.CalledProcessError:
if attempt >= MODULE_FOUNDRY_BUILD_ATTEMPTS:
raise
print(
"module-foundry-build-retry="
f"{attempt}/{MODULE_FOUNDRY_BUILD_ATTEMPTS - 1}",
file=sys.stderr,
)
time.sleep(2)
run_module_foundry_compose_up(services)
def run_engine_node_intelligence_compose(services, entries):
if not is_engine_node_intelligence_transition("engine", entries):
die("Engine node-intelligence Compose called for unrelated artifact")
@@ -18024,6 +18410,8 @@ def run_component_runtime(component, entries, services):
prepare_component_runtime(component, entries)
if is_engine_node_intelligence_transition(component, entries):
run_engine_node_intelligence_compose(services, entries)
elif component == "module-foundry":
run_module_foundry_compose(services)
else:
run_compose(component, services, entries)
@@ -18423,6 +18811,94 @@ def compose_service_container_id(component, service):
return container_ids[0]
def module_foundry_runtime_inventory(required=True):
result = subprocess.run(
[
*compose_base_cmd("module-foundry"),
"ps",
"-q",
MODULE_FOUNDRY_SERVICE,
],
cwd=str(component_compose_root("module-foundry")),
check=False,
capture_output=True,
text=True,
)
container_ids = [
line.strip()
for line in result.stdout.splitlines()
if line.strip()
]
if (
result.returncode != 0
or len(container_ids) > 1
or any(
not re.fullmatch(r"[a-f0-9]{12,64}", value)
for value in container_ids
)
):
die("Module Foundry runtime topology is unproven")
if not container_ids:
if required:
die("Module Foundry runtime is missing")
return None
container_id = container_ids[0]
containers = docker_json(
["container", "inspect", container_id],
"Module Foundry runtime container inspect",
)
if (
not isinstance(containers, list)
or len(containers) != 1
or not isinstance(containers[0], dict)
):
die("Module Foundry runtime container inspect shape mismatch")
container = containers[0]
config = container.get("Config") or {}
labels = config.get("Labels") or {}
state = container.get("State") or {}
image_id = container.get("Image")
image_ref = config.get("Image")
health = (state.get("Health") or {}).get("Status")
if (
labels.get("com.docker.compose.project")
!= "nodedc-module-foundry"
or labels.get("com.docker.compose.service")
!= MODULE_FOUNDRY_SERVICE
or not re.fullmatch(r"sha256:[a-f0-9]{64}", str(image_id or ""))
or not isinstance(image_ref, str)
or not re.fullmatch(
r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
image_ref,
)
or image_ref.startswith("sha256:")
or "@" in image_ref
or state.get("Running") is not True
or state.get("Status") != "running"
or health not in (None, "healthy")
):
die("Module Foundry runtime inventory mismatch")
return {
"schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
"composeProject": "nodedc-module-foundry",
"service": MODULE_FOUNDRY_SERVICE,
"containerId": container_id,
"imageId": image_id,
"imageRef": image_ref,
"health": health or "not-configured",
}
def validate_module_foundry_candidate_runtime(runtime_before):
current = module_foundry_runtime_inventory()
if (
current["containerId"] == runtime_before["containerId"]
or current["imageId"] == runtime_before["imageId"]
):
die("Module Foundry candidate generation was not activated")
return current
def healthcheck_compose_service(component, service):
healthcheck_container(compose_service_container_id(component, service))
@@ -19555,6 +20031,8 @@ def apply_artifact(artifact):
apply_started = False
engine_backend_recreated = False
engine_backend_initial_mode = None
module_foundry_runtime_before = None
module_foundry_sector_recovery_preflight = None
runtime_started = False
current_stamp = stamp()
@@ -19633,6 +20111,13 @@ def apply_artifact(artifact):
root.mkdir(parents=True, exist_ok=True)
else:
die(f"component payload root not found: {root}")
module_foundry_sector_recovery_preflight = (
validate_module_foundry_sector_recovery_evidence(
manifest,
entries,
payload_dir,
)
)
if is_engine_data_product_publish_grant_slice(component, entries):
preflight_engine_data_product_publish_grant_predecessor(payload_dir)
publish_backend_preflight = preflight_engine_credential_backend_runtime()
@@ -19912,6 +20397,27 @@ def apply_artifact(artifact):
DEVICE_PLANE_RUNTIME_SERVICES
)
)
if component == "module-foundry":
module_foundry_runtime_before = (
module_foundry_runtime_inventory()
)
if (
module_foundry_sector_recovery_preflight is not None
and (
module_foundry_runtime_before["containerId"]
!= module_foundry_sector_recovery_preflight[
"runtime_container_id"
]
or module_foundry_runtime_before["imageId"]
!= module_foundry_sector_recovery_preflight[
"runtime_image_id"
]
)
):
die(
"Module Foundry sector predecessor runtime "
"changed during preflight"
)
if is_engine_n8n_transition(component, entries):
transition_descriptor = read_engine_n8n_transition_descriptor(
@@ -19938,6 +20444,23 @@ def apply_artifact(artifact):
include_nginx_html = component == "engine" and component_publish_dist(component, entries)
create_backup(root, backup_dir, entries, include_nginx_html)
if component == "module-foundry":
if module_foundry_runtime_before is None:
die("Module Foundry pre-apply runtime inventory is missing")
module_foundry_runtime_path = (
backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE
)
module_foundry_runtime_path.write_text(
json.dumps(
module_foundry_runtime_before,
ensure_ascii=False,
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)
module_foundry_runtime_path.chmod(0o600)
if component == "device-plane":
if device_plane_runtime_before is None:
die("Device Plane pre-apply runtime inventory is missing")
@@ -20043,6 +20566,12 @@ def apply_artifact(artifact):
):
die("Engine L2 closed-loop app generation was not recreated")
run_healthchecks(component, entries, services)
if component == "module-foundry":
if module_foundry_runtime_before is None:
die("Module Foundry candidate predecessor is missing")
validate_module_foundry_candidate_runtime(
module_foundry_runtime_before
)
if is_device_plane_b2_discovery_ingress_slice(
component,
entries,
@@ -20249,6 +20778,35 @@ def apply_artifact(artifact):
except Exception as rollback_exc:
rollback_status = f"failed:{type(rollback_exc).__name__}"
print("engine-automatic-rollback=failed", file=sys.stderr)
elif (
component == "module-foundry"
and entries is not None
and services is not None
):
try:
restored_state = rollback_module_foundry_apply(
root,
backup_dir,
entries,
current_stamp,
services,
)
rollback_status = (
f"ok:module-foundry-overlay:{restored_state}"
)
print(
"module-foundry-automatic-rollback="
f"{rollback_status}",
file=sys.stderr,
)
except Exception as rollback_exc:
rollback_status = (
f"failed:{type(rollback_exc).__name__}"
)
print(
"module-foundry-automatic-rollback=failed",
file=sys.stderr,
)
elif (
component == "platform"
and entries is not None