FIX - NAS DEPLOY: recover Module Foundry runtime safely
This commit is contained in:
Codex
@@ -35,6 +35,62 @@ STATE_FILE = STATE_DIR / "applied.jsonl"
|
||||
FAILED_STATE_FILE = STATE_DIR / "failed.jsonl"
|
||||
LOCK_DIR = STATE_DIR / "deploy.lock"
|
||||
DOCKER = Path("/usr/local/bin/docker")
|
||||
MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
|
||||
MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
|
||||
MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
|
||||
MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = (
|
||||
"module-foundry-map-sector-workspace-20260809-001"
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
|
||||
"apps/catalog/src/MapFixturePreview.tsx",
|
||||
"apps/catalog/src/mapPresentationProfile.ts",
|
||||
"apps/catalog/src/mapSectorGrid.d.mts",
|
||||
"apps/catalog/src/mapSectorGrid.mjs",
|
||||
"apps/catalog/src/styles.css",
|
||||
"scripts/map-object-layers.test.mjs",
|
||||
"scripts/map-presentation-filters.test.mjs",
|
||||
"scripts/map-sector-grid.test.mjs",
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = (
|
||||
"module-foundry-map-sector-workspace-20260808-001"
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = (
|
||||
"nodedc-module-foundry-map-sector-workspace-20260808-001.tgz."
|
||||
"20260809-014720"
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = (
|
||||
"0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740"
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = (
|
||||
"module-foundry-module-foundry-map-sector-workspace-20260808-001-"
|
||||
"20260809-014720"
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
|
||||
"Command '['/usr/local/bin/docker', 'compose', '-p', "
|
||||
"'nodedc-module-foundry', '--env-file', "
|
||||
"'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
|
||||
"'/volume1/docker/nodedc-platform/module-foundry/source/infra/"
|
||||
"docker-compose.module-foundry.yml', 'up', '-d', '--force-recreate', "
|
||||
"'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
|
||||
"exit status 17."
|
||||
)
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = {
|
||||
"existing-files.txt": (
|
||||
"57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c"
|
||||
),
|
||||
"files.txt": (
|
||||
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
|
||||
),
|
||||
"manifest.env": (
|
||||
"cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2"
|
||||
),
|
||||
"missing-files.txt": (
|
||||
"1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f"
|
||||
),
|
||||
"source-before.tgz": (
|
||||
"bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562"
|
||||
),
|
||||
}
|
||||
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
|
||||
MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
|
||||
MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
|
||||
@@ -14961,11 +15017,19 @@ def plan_artifact(artifact):
|
||||
provider_catalog_preflight = None
|
||||
device_plane_postgres_preflight = None
|
||||
device_plane_foundation_recovery_preflight = None
|
||||
module_foundry_sector_recovery_preflight = None
|
||||
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
|
||||
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
|
||||
reject_terminal_engine_l2_failed_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_foundation_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
|
||||
module_foundry_sector_recovery_preflight = (
|
||||
validate_module_foundry_sector_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
payload_dir,
|
||||
)
|
||||
)
|
||||
transition_descriptor = None
|
||||
transition_preflight = None
|
||||
if is_engine_n8n_transition(manifest["component"], entries):
|
||||
@@ -16520,6 +16584,37 @@ def plan_artifact(artifact):
|
||||
print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
|
||||
print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
|
||||
print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
|
||||
print(
|
||||
"module_foundry_build="
|
||||
f"separate-before-recreate:attempts={MODULE_FOUNDRY_BUILD_ATTEMPTS}"
|
||||
)
|
||||
print("module_foundry_runtime_pull=never")
|
||||
print("module_foundry_rollback=source+image+runtime")
|
||||
if module_foundry_sector_recovery_preflight is not None:
|
||||
print(
|
||||
"module_foundry_transition="
|
||||
f"{module_foundry_sector_recovery_preflight['mode']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_failed_predecessor="
|
||||
f"{module_foundry_sector_recovery_preflight['failed_patch_id']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_failed_predecessor_sha256="
|
||||
f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_failed_backup="
|
||||
f"{module_foundry_sector_recovery_preflight['failed_backup_id']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_predecessor_container="
|
||||
f"{module_foundry_sector_recovery_preflight['runtime_container_id']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_predecessor_image="
|
||||
f"{module_foundry_sector_recovery_preflight['runtime_image_id']}"
|
||||
)
|
||||
if component == "device-plane":
|
||||
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
|
||||
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}")
|
||||
@@ -17241,6 +17336,236 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
|
||||
return f"source+runtime-restored:{restored_count}"
|
||||
|
||||
|
||||
def is_module_foundry_sector_recovery_slice(manifest, entries):
|
||||
return (
|
||||
manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID
|
||||
and manifest.get("component") == "module-foundry"
|
||||
and manifest.get("type") == "app-overlay"
|
||||
and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
||||
)
|
||||
|
||||
|
||||
def validate_module_foundry_sector_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
payload_dir,
|
||||
):
|
||||
if not is_module_foundry_sector_recovery_slice(manifest, entries):
|
||||
return None
|
||||
|
||||
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
||||
try:
|
||||
backup_stat = backup_dir.lstat()
|
||||
except FileNotFoundError:
|
||||
die("Module Foundry sector recovery backup is missing")
|
||||
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
|
||||
backup_stat.st_mode
|
||||
):
|
||||
die("Module Foundry sector recovery backup is unsafe")
|
||||
if {child.name for child in backup_dir.iterdir()} != set(
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
|
||||
):
|
||||
die("Module Foundry sector recovery backup file set mismatch")
|
||||
for name, expected_sha256 in (
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items()
|
||||
):
|
||||
path = backup_dir / name
|
||||
path_stat = path.lstat()
|
||||
if (
|
||||
stat.S_ISLNK(path_stat.st_mode)
|
||||
or not stat.S_ISREG(path_stat.st_mode)
|
||||
or sha256_file(path) != expected_sha256
|
||||
):
|
||||
die(
|
||||
"Module Foundry sector recovery backup drift detected: "
|
||||
f"{name}"
|
||||
)
|
||||
|
||||
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
||||
try:
|
||||
failed_stat = failed_artifact.lstat()
|
||||
except FileNotFoundError:
|
||||
die("Module Foundry sector failed artifact is missing")
|
||||
if (
|
||||
stat.S_ISLNK(failed_stat.st_mode)
|
||||
or not stat.S_ISREG(failed_stat.st_mode)
|
||||
or sha256_file(failed_artifact)
|
||||
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
|
||||
):
|
||||
die("Module Foundry sector failed artifact evidence mismatch")
|
||||
|
||||
try:
|
||||
state_stat = FAILED_STATE_FILE.lstat()
|
||||
state_lines = FAILED_STATE_FILE.read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
except (FileNotFoundError, OSError, UnicodeDecodeError):
|
||||
die("Module Foundry sector failed journal is unreadable")
|
||||
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
|
||||
state_stat.st_mode
|
||||
):
|
||||
die("Module Foundry sector failed journal is unsafe")
|
||||
records = []
|
||||
for line in state_lines:
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
die("Module Foundry sector failed journal contains invalid JSON")
|
||||
if (
|
||||
isinstance(value, dict)
|
||||
and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
|
||||
):
|
||||
records.append(value)
|
||||
if len(records) != 1:
|
||||
die("Module Foundry sector failed journal evidence count mismatch")
|
||||
record = records[0]
|
||||
if (
|
||||
record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
||||
or record.get("backup_id")
|
||||
!= MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
||||
or record.get("component") != "module-foundry"
|
||||
or record.get("sha256")
|
||||
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
|
||||
or record.get("started_apply") is not True
|
||||
or record.get("rollback_status") != "not-required"
|
||||
or record.get("status") != "failed"
|
||||
or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE
|
||||
):
|
||||
die("Module Foundry sector failed journal evidence mismatch")
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-sector-failed-",
|
||||
dir=TMP_DIR,
|
||||
) as directory:
|
||||
failed_work = Path(directory)
|
||||
safe_extract(failed_artifact, failed_work)
|
||||
failed_manifest = parse_manifest(failed_work / "manifest.env")
|
||||
failed_entries = parse_files_list(failed_work / "files.txt")
|
||||
if (
|
||||
failed_manifest.get("id")
|
||||
!= MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
|
||||
or failed_manifest.get("component") != "module-foundry"
|
||||
or failed_manifest.get("type") != "app-overlay"
|
||||
or tuple(failed_entries)
|
||||
!= MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
||||
):
|
||||
die("Module Foundry sector failed artifact contract mismatch")
|
||||
failed_payload_sha256 = collect_exact_files(
|
||||
failed_work / "payload",
|
||||
failed_entries,
|
||||
"Module Foundry failed sector payload",
|
||||
)
|
||||
|
||||
candidate_payload_sha256 = collect_exact_files(
|
||||
payload_dir,
|
||||
entries,
|
||||
"Module Foundry sector recovery payload",
|
||||
)
|
||||
if candidate_payload_sha256 != failed_payload_sha256:
|
||||
die("Module Foundry sector recovery payload changed unexpectedly")
|
||||
installed_payload_sha256 = collect_exact_files(
|
||||
component_root("module-foundry"),
|
||||
entries,
|
||||
"Module Foundry partial sector predecessor",
|
||||
)
|
||||
if installed_payload_sha256 != candidate_payload_sha256:
|
||||
die("Module Foundry partial sector predecessor drift detected")
|
||||
|
||||
runtime = module_foundry_runtime_inventory()
|
||||
return {
|
||||
"mode": "failed-overlay-runtime-reconciliation",
|
||||
"failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
|
||||
"failed_artifact_sha256": (
|
||||
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
|
||||
),
|
||||
"failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
|
||||
"runtime_container_id": runtime["containerId"],
|
||||
"runtime_image_id": runtime["imageId"],
|
||||
}
|
||||
|
||||
|
||||
def read_module_foundry_runtime_before(backup_dir):
|
||||
runtime = read_strict_json(
|
||||
backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE,
|
||||
"Module Foundry pre-apply runtime inventory",
|
||||
max_bytes=16 * 1024,
|
||||
)
|
||||
if (
|
||||
runtime.get("schemaVersion")
|
||||
!= "nodedc.module-foundry.runtime-inventory.v1"
|
||||
or runtime.get("composeProject") != "nodedc-module-foundry"
|
||||
or runtime.get("service") != MODULE_FOUNDRY_SERVICE
|
||||
or not re.fullmatch(
|
||||
r"[a-f0-9]{12,64}",
|
||||
str(runtime.get("containerId", "")),
|
||||
)
|
||||
or not re.fullmatch(
|
||||
r"sha256:[a-f0-9]{64}",
|
||||
str(runtime.get("imageId", "")),
|
||||
)
|
||||
or not isinstance(runtime.get("imageRef"), str)
|
||||
or not re.fullmatch(
|
||||
r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
|
||||
runtime["imageRef"],
|
||||
)
|
||||
or runtime["imageRef"].startswith("sha256:")
|
||||
or "@" in runtime["imageRef"]
|
||||
):
|
||||
die("Module Foundry pre-apply runtime inventory mismatch")
|
||||
return runtime
|
||||
|
||||
|
||||
def rollback_module_foundry_apply(
|
||||
root,
|
||||
backup_dir,
|
||||
entries,
|
||||
current_stamp,
|
||||
applied_services,
|
||||
):
|
||||
if tuple(applied_services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry rollback service set mismatch")
|
||||
runtime_before = read_module_foundry_runtime_before(backup_dir)
|
||||
restored_count = restore_platform_overlay(
|
||||
root,
|
||||
backup_dir,
|
||||
entries,
|
||||
current_stamp,
|
||||
)
|
||||
current = module_foundry_runtime_inventory(required=False)
|
||||
if (
|
||||
current is not None
|
||||
and current["containerId"] == runtime_before["containerId"]
|
||||
and current["imageId"] == runtime_before["imageId"]
|
||||
):
|
||||
run_healthchecks("module-foundry", entries, applied_services)
|
||||
return f"source-restored-runtime-unchanged:{restored_count}"
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
str(DOCKER),
|
||||
"image",
|
||||
"tag",
|
||||
runtime_before["imageId"],
|
||||
runtime_before["imageRef"],
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
if (
|
||||
inspect_local_image(
|
||||
runtime_before["imageRef"],
|
||||
"Module Foundry rollback image",
|
||||
)
|
||||
!= runtime_before["imageId"]
|
||||
):
|
||||
die("Module Foundry rollback image tag mismatch")
|
||||
run_module_foundry_compose_up(applied_services)
|
||||
run_healthchecks("module-foundry", entries, applied_services)
|
||||
restored = module_foundry_runtime_inventory()
|
||||
if restored["imageId"] != runtime_before["imageId"]:
|
||||
die("Module Foundry rollback runtime image mismatch")
|
||||
return f"source+runtime-restored:{restored_count}"
|
||||
|
||||
|
||||
def validate_engine_l2_closed_loop_stable_source(root):
|
||||
stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256)
|
||||
actual = collect_exact_files(
|
||||
@@ -17728,6 +18053,67 @@ def run_compose(component, services, entries=None):
|
||||
)
|
||||
|
||||
|
||||
def run_module_foundry_compose_up(services):
|
||||
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry Compose service set mismatch")
|
||||
compose_root = component_compose_root("module-foundry")
|
||||
cmd = [
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"up",
|
||||
"-d",
|
||||
"--force-recreate",
|
||||
"--pull",
|
||||
"never",
|
||||
"--no-deps",
|
||||
*services,
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd, cwd=str(compose_root), check=True)
|
||||
except subprocess.CalledProcessError:
|
||||
subprocess.run(
|
||||
[
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"logs",
|
||||
"--no-color",
|
||||
"--tail=180",
|
||||
*services,
|
||||
],
|
||||
cwd=str(compose_root),
|
||||
check=False,
|
||||
)
|
||||
raise
|
||||
subprocess.run(
|
||||
[*compose_base_cmd("module-foundry"), "ps"],
|
||||
cwd=str(compose_root),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def run_module_foundry_compose(services):
|
||||
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry build service set mismatch")
|
||||
compose_root = component_compose_root("module-foundry")
|
||||
build_cmd = [
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"build",
|
||||
*services,
|
||||
]
|
||||
for attempt in range(1, MODULE_FOUNDRY_BUILD_ATTEMPTS + 1):
|
||||
try:
|
||||
subprocess.run(build_cmd, cwd=str(compose_root), check=True)
|
||||
break
|
||||
except subprocess.CalledProcessError:
|
||||
if attempt >= MODULE_FOUNDRY_BUILD_ATTEMPTS:
|
||||
raise
|
||||
print(
|
||||
"module-foundry-build-retry="
|
||||
f"{attempt}/{MODULE_FOUNDRY_BUILD_ATTEMPTS - 1}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
time.sleep(2)
|
||||
run_module_foundry_compose_up(services)
|
||||
|
||||
|
||||
def run_engine_node_intelligence_compose(services, entries):
|
||||
if not is_engine_node_intelligence_transition("engine", entries):
|
||||
die("Engine node-intelligence Compose called for unrelated artifact")
|
||||
@@ -18024,6 +18410,8 @@ def run_component_runtime(component, entries, services):
|
||||
prepare_component_runtime(component, entries)
|
||||
if is_engine_node_intelligence_transition(component, entries):
|
||||
run_engine_node_intelligence_compose(services, entries)
|
||||
elif component == "module-foundry":
|
||||
run_module_foundry_compose(services)
|
||||
else:
|
||||
run_compose(component, services, entries)
|
||||
|
||||
@@ -18423,6 +18811,94 @@ def compose_service_container_id(component, service):
|
||||
return container_ids[0]
|
||||
|
||||
|
||||
def module_foundry_runtime_inventory(required=True):
|
||||
result = subprocess.run(
|
||||
[
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"ps",
|
||||
"-q",
|
||||
MODULE_FOUNDRY_SERVICE,
|
||||
],
|
||||
cwd=str(component_compose_root("module-foundry")),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
container_ids = [
|
||||
line.strip()
|
||||
for line in result.stdout.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if (
|
||||
result.returncode != 0
|
||||
or len(container_ids) > 1
|
||||
or any(
|
||||
not re.fullmatch(r"[a-f0-9]{12,64}", value)
|
||||
for value in container_ids
|
||||
)
|
||||
):
|
||||
die("Module Foundry runtime topology is unproven")
|
||||
if not container_ids:
|
||||
if required:
|
||||
die("Module Foundry runtime is missing")
|
||||
return None
|
||||
container_id = container_ids[0]
|
||||
containers = docker_json(
|
||||
["container", "inspect", container_id],
|
||||
"Module Foundry runtime container inspect",
|
||||
)
|
||||
if (
|
||||
not isinstance(containers, list)
|
||||
or len(containers) != 1
|
||||
or not isinstance(containers[0], dict)
|
||||
):
|
||||
die("Module Foundry runtime container inspect shape mismatch")
|
||||
container = containers[0]
|
||||
config = container.get("Config") or {}
|
||||
labels = config.get("Labels") or {}
|
||||
state = container.get("State") or {}
|
||||
image_id = container.get("Image")
|
||||
image_ref = config.get("Image")
|
||||
health = (state.get("Health") or {}).get("Status")
|
||||
if (
|
||||
labels.get("com.docker.compose.project")
|
||||
!= "nodedc-module-foundry"
|
||||
or labels.get("com.docker.compose.service")
|
||||
!= MODULE_FOUNDRY_SERVICE
|
||||
or not re.fullmatch(r"sha256:[a-f0-9]{64}", str(image_id or ""))
|
||||
or not isinstance(image_ref, str)
|
||||
or not re.fullmatch(
|
||||
r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
|
||||
image_ref,
|
||||
)
|
||||
or image_ref.startswith("sha256:")
|
||||
or "@" in image_ref
|
||||
or state.get("Running") is not True
|
||||
or state.get("Status") != "running"
|
||||
or health not in (None, "healthy")
|
||||
):
|
||||
die("Module Foundry runtime inventory mismatch")
|
||||
return {
|
||||
"schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
|
||||
"composeProject": "nodedc-module-foundry",
|
||||
"service": MODULE_FOUNDRY_SERVICE,
|
||||
"containerId": container_id,
|
||||
"imageId": image_id,
|
||||
"imageRef": image_ref,
|
||||
"health": health or "not-configured",
|
||||
}
|
||||
|
||||
|
||||
def validate_module_foundry_candidate_runtime(runtime_before):
|
||||
current = module_foundry_runtime_inventory()
|
||||
if (
|
||||
current["containerId"] == runtime_before["containerId"]
|
||||
or current["imageId"] == runtime_before["imageId"]
|
||||
):
|
||||
die("Module Foundry candidate generation was not activated")
|
||||
return current
|
||||
|
||||
|
||||
def healthcheck_compose_service(component, service):
|
||||
healthcheck_container(compose_service_container_id(component, service))
|
||||
|
||||
@@ -19555,6 +20031,8 @@ def apply_artifact(artifact):
|
||||
apply_started = False
|
||||
engine_backend_recreated = False
|
||||
engine_backend_initial_mode = None
|
||||
module_foundry_runtime_before = None
|
||||
module_foundry_sector_recovery_preflight = None
|
||||
runtime_started = False
|
||||
current_stamp = stamp()
|
||||
|
||||
@@ -19633,6 +20111,13 @@ def apply_artifact(artifact):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
die(f"component payload root not found: {root}")
|
||||
module_foundry_sector_recovery_preflight = (
|
||||
validate_module_foundry_sector_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
payload_dir,
|
||||
)
|
||||
)
|
||||
if is_engine_data_product_publish_grant_slice(component, entries):
|
||||
preflight_engine_data_product_publish_grant_predecessor(payload_dir)
|
||||
publish_backend_preflight = preflight_engine_credential_backend_runtime()
|
||||
@@ -19912,6 +20397,27 @@ def apply_artifact(artifact):
|
||||
DEVICE_PLANE_RUNTIME_SERVICES
|
||||
)
|
||||
)
|
||||
if component == "module-foundry":
|
||||
module_foundry_runtime_before = (
|
||||
module_foundry_runtime_inventory()
|
||||
)
|
||||
if (
|
||||
module_foundry_sector_recovery_preflight is not None
|
||||
and (
|
||||
module_foundry_runtime_before["containerId"]
|
||||
!= module_foundry_sector_recovery_preflight[
|
||||
"runtime_container_id"
|
||||
]
|
||||
or module_foundry_runtime_before["imageId"]
|
||||
!= module_foundry_sector_recovery_preflight[
|
||||
"runtime_image_id"
|
||||
]
|
||||
)
|
||||
):
|
||||
die(
|
||||
"Module Foundry sector predecessor runtime "
|
||||
"changed during preflight"
|
||||
)
|
||||
|
||||
if is_engine_n8n_transition(component, entries):
|
||||
transition_descriptor = read_engine_n8n_transition_descriptor(
|
||||
@@ -19938,6 +20444,23 @@ def apply_artifact(artifact):
|
||||
|
||||
include_nginx_html = component == "engine" and component_publish_dist(component, entries)
|
||||
create_backup(root, backup_dir, entries, include_nginx_html)
|
||||
if component == "module-foundry":
|
||||
if module_foundry_runtime_before is None:
|
||||
die("Module Foundry pre-apply runtime inventory is missing")
|
||||
module_foundry_runtime_path = (
|
||||
backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE
|
||||
)
|
||||
module_foundry_runtime_path.write_text(
|
||||
json.dumps(
|
||||
module_foundry_runtime_before,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
module_foundry_runtime_path.chmod(0o600)
|
||||
if component == "device-plane":
|
||||
if device_plane_runtime_before is None:
|
||||
die("Device Plane pre-apply runtime inventory is missing")
|
||||
@@ -20043,6 +20566,12 @@ def apply_artifact(artifact):
|
||||
):
|
||||
die("Engine L2 closed-loop app generation was not recreated")
|
||||
run_healthchecks(component, entries, services)
|
||||
if component == "module-foundry":
|
||||
if module_foundry_runtime_before is None:
|
||||
die("Module Foundry candidate predecessor is missing")
|
||||
validate_module_foundry_candidate_runtime(
|
||||
module_foundry_runtime_before
|
||||
)
|
||||
if is_device_plane_b2_discovery_ingress_slice(
|
||||
component,
|
||||
entries,
|
||||
@@ -20249,6 +20778,35 @@ def apply_artifact(artifact):
|
||||
except Exception as rollback_exc:
|
||||
rollback_status = f"failed:{type(rollback_exc).__name__}"
|
||||
print("engine-automatic-rollback=failed", file=sys.stderr)
|
||||
elif (
|
||||
component == "module-foundry"
|
||||
and entries is not None
|
||||
and services is not None
|
||||
):
|
||||
try:
|
||||
restored_state = rollback_module_foundry_apply(
|
||||
root,
|
||||
backup_dir,
|
||||
entries,
|
||||
current_stamp,
|
||||
services,
|
||||
)
|
||||
rollback_status = (
|
||||
f"ok:module-foundry-overlay:{restored_state}"
|
||||
)
|
||||
print(
|
||||
"module-foundry-automatic-rollback="
|
||||
f"{rollback_status}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception as rollback_exc:
|
||||
rollback_status = (
|
||||
f"failed:{type(rollback_exc).__name__}"
|
||||
)
|
||||
print(
|
||||
"module-foundry-automatic-rollback=failed",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif (
|
||||
component == "platform"
|
||||
and entries is not None
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
||||
|
||||
|
||||
def load_runner():
|
||||
loader = importlib.machinery.SourceFileLoader(
|
||||
"nodedc_module_foundry_runtime_recovery",
|
||||
str(RUNNER_PATH),
|
||||
)
|
||||
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
RUNNER = load_runner()
|
||||
|
||||
|
||||
def runtime(container_char="a", image_char="b"):
|
||||
return {
|
||||
"schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
|
||||
"composeProject": "nodedc-module-foundry",
|
||||
"service": RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
"containerId": container_char * 64,
|
||||
"imageId": f"sha256:{image_char * 64}",
|
||||
"imageRef": "nodedc-module-foundry-nodedc-module-foundry:latest",
|
||||
"health": "healthy",
|
||||
}
|
||||
|
||||
|
||||
class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
||||
def test_build_retries_before_any_runtime_recreate(self):
|
||||
first_failure = subprocess.CalledProcessError(17, ["docker", "compose"])
|
||||
completed = subprocess.CompletedProcess([], 0)
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_compose_root",
|
||||
return_value=Path("/compose"),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"compose_base_cmd",
|
||||
return_value=["docker", "compose"],
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER.subprocess,
|
||||
"run",
|
||||
side_effect=[first_failure, completed],
|
||||
) as run,
|
||||
mock.patch.object(RUNNER.time, "sleep") as sleep,
|
||||
mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
|
||||
):
|
||||
RUNNER.run_module_foundry_compose(
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,)
|
||||
)
|
||||
|
||||
expected = [
|
||||
"docker",
|
||||
"compose",
|
||||
"build",
|
||||
RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
]
|
||||
self.assertEqual(run.call_count, 2)
|
||||
self.assertEqual(run.call_args_list[0].args[0], expected)
|
||||
self.assertEqual(run.call_args_list[1].args[0], expected)
|
||||
sleep.assert_called_once_with(2)
|
||||
up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
|
||||
|
||||
def test_runtime_recreate_uses_built_image_without_build_or_pull(self):
|
||||
completed = subprocess.CompletedProcess([], 0)
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_compose_root",
|
||||
return_value=Path("/compose"),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"compose_base_cmd",
|
||||
return_value=["docker", "compose"],
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER.subprocess,
|
||||
"run",
|
||||
return_value=completed,
|
||||
) as run,
|
||||
):
|
||||
RUNNER.run_module_foundry_compose_up(
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,)
|
||||
)
|
||||
|
||||
up = run.call_args_list[0].args[0]
|
||||
self.assertEqual(
|
||||
up,
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--force-recreate",
|
||||
"--pull",
|
||||
"never",
|
||||
"--no-deps",
|
||||
RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
],
|
||||
)
|
||||
self.assertNotIn("--build", up)
|
||||
self.assertEqual(run.call_args_list[1].args[0], ["docker", "compose", "ps"])
|
||||
|
||||
def test_rollback_restores_source_and_keeps_unchanged_runtime(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-rollback-unchanged-"
|
||||
) as directory:
|
||||
work = Path(directory)
|
||||
root = work / "source"
|
||||
backup = work / "backup"
|
||||
tmp = work / "tmp"
|
||||
root.mkdir()
|
||||
backup.mkdir()
|
||||
tmp.mkdir()
|
||||
entries = ("old.txt", "new.txt")
|
||||
(root / "old.txt").write_text("old\n", encoding="utf-8")
|
||||
RUNNER.create_backup(root, backup, entries, False)
|
||||
(root / "old.txt").write_text("candidate\n", encoding="utf-8")
|
||||
(root / "new.txt").write_text("candidate\n", encoding="utf-8")
|
||||
before = runtime()
|
||||
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
||||
json.dumps(before),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
return_value=before,
|
||||
),
|
||||
mock.patch.object(RUNNER, "run_healthchecks") as health,
|
||||
mock.patch.object(RUNNER.subprocess, "run") as process,
|
||||
):
|
||||
result = RUNNER.rollback_module_foundry_apply(
|
||||
root,
|
||||
backup,
|
||||
entries,
|
||||
"20260809-000000",
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
result,
|
||||
"source-restored-runtime-unchanged:2",
|
||||
)
|
||||
self.assertEqual(
|
||||
(root / "old.txt").read_text(encoding="utf-8"),
|
||||
"old\n",
|
||||
)
|
||||
self.assertFalse((root / "new.txt").exists())
|
||||
process.assert_not_called()
|
||||
health.assert_called_once()
|
||||
|
||||
def test_rollback_retags_and_recreates_changed_runtime(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-rollback-changed-"
|
||||
) as directory:
|
||||
work = Path(directory)
|
||||
root = work / "source"
|
||||
backup = work / "backup"
|
||||
tmp = work / "tmp"
|
||||
root.mkdir()
|
||||
backup.mkdir()
|
||||
tmp.mkdir()
|
||||
entries = ("old.txt",)
|
||||
(root / "old.txt").write_text("old\n", encoding="utf-8")
|
||||
RUNNER.create_backup(root, backup, entries, False)
|
||||
(root / "old.txt").write_text("candidate\n", encoding="utf-8")
|
||||
before = runtime()
|
||||
candidate = runtime("c", "d")
|
||||
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
||||
json.dumps(before),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
side_effect=[candidate, before],
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"inspect_local_image",
|
||||
return_value=before["imageId"],
|
||||
),
|
||||
mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
|
||||
mock.patch.object(RUNNER, "run_healthchecks") as health,
|
||||
mock.patch.object(RUNNER.subprocess, "run") as process,
|
||||
):
|
||||
result = RUNNER.rollback_module_foundry_apply(
|
||||
root,
|
||||
backup,
|
||||
entries,
|
||||
"20260809-000000",
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
||||
)
|
||||
|
||||
self.assertEqual(result, "source+runtime-restored:1")
|
||||
process.assert_called_once_with(
|
||||
[
|
||||
str(RUNNER.DOCKER),
|
||||
"image",
|
||||
"tag",
|
||||
before["imageId"],
|
||||
before["imageRef"],
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
|
||||
health.assert_called_once()
|
||||
|
||||
def test_candidate_must_replace_both_container_and_image(self):
|
||||
before = runtime()
|
||||
for candidate in (
|
||||
runtime("a", "c"),
|
||||
runtime("c", "b"),
|
||||
):
|
||||
with mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
return_value=candidate,
|
||||
):
|
||||
with self.assertRaises(RUNNER.DeployError):
|
||||
RUNNER.validate_module_foundry_candidate_runtime(before)
|
||||
|
||||
def test_preapply_runtime_rejects_digest_only_image_reference(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-runtime-reference-"
|
||||
) as directory:
|
||||
backup = Path(directory)
|
||||
before = runtime()
|
||||
before["imageRef"] = before["imageId"]
|
||||
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
||||
json.dumps(before),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(RUNNER.DeployError):
|
||||
RUNNER.read_module_foundry_runtime_before(backup)
|
||||
|
||||
def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-sector-evidence-"
|
||||
) as directory:
|
||||
work = Path(directory)
|
||||
backups = work / "backups"
|
||||
failed = work / "failed"
|
||||
state = work / "state"
|
||||
tmp = work / "tmp"
|
||||
installed = work / "installed"
|
||||
candidate = work / "candidate"
|
||||
for path in (backups, failed, state, tmp, installed, candidate):
|
||||
path.mkdir()
|
||||
|
||||
entries = RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
||||
for index, rel in enumerate(entries):
|
||||
for root in (installed, candidate):
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"payload-{index}\n", encoding="utf-8")
|
||||
|
||||
backup = backups / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
||||
backup.mkdir()
|
||||
for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256:
|
||||
(backup / name).write_text(f"{name}\n", encoding="utf-8")
|
||||
backup_hashes = {
|
||||
name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
|
||||
for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
|
||||
}
|
||||
|
||||
archive_source = work / "archive-source"
|
||||
(archive_source / "payload").mkdir(parents=True)
|
||||
(archive_source / "manifest.env").write_text(
|
||||
"id="
|
||||
f"{RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID}\n"
|
||||
"component=module-foundry\n"
|
||||
"type=app-overlay\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(archive_source / "files.txt").write_text(
|
||||
"\n".join(entries) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for rel in entries:
|
||||
source = candidate / rel
|
||||
target = archive_source / "payload" / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(source.read_bytes())
|
||||
failed_artifact = (
|
||||
failed / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
||||
)
|
||||
with tarfile.open(failed_artifact, "w:gz") as archive:
|
||||
archive.add(
|
||||
archive_source / "manifest.env",
|
||||
arcname="manifest.env",
|
||||
)
|
||||
archive.add(
|
||||
archive_source / "files.txt",
|
||||
arcname="files.txt",
|
||||
)
|
||||
archive.add(archive_source / "payload", arcname="payload")
|
||||
failed_sha256 = hashlib.sha256(
|
||||
failed_artifact.read_bytes()
|
||||
).hexdigest()
|
||||
|
||||
journal = state / "failed.jsonl"
|
||||
journal.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"artifact": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT,
|
||||
"backup_id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
|
||||
"component": "module-foundry",
|
||||
"id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
|
||||
"message": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE,
|
||||
"rollback_status": "not-required",
|
||||
"sha256": failed_sha256,
|
||||
"started_apply": True,
|
||||
"status": "failed",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {
|
||||
"id": RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID,
|
||||
"component": "module-foundry",
|
||||
"type": "app-overlay",
|
||||
}
|
||||
|
||||
patches = (
|
||||
mock.patch.object(RUNNER, "BACKUPS_DIR", backups),
|
||||
mock.patch.object(RUNNER, "FAILED_DIR", failed),
|
||||
mock.patch.object(RUNNER, "FAILED_STATE_FILE", journal),
|
||||
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256",
|
||||
failed_sha256,
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256",
|
||||
backup_hashes,
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_root",
|
||||
return_value=installed,
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
return_value=runtime(),
|
||||
),
|
||||
)
|
||||
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6], patches[7]:
|
||||
result = RUNNER.validate_module_foundry_sector_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
candidate,
|
||||
)
|
||||
self.assertEqual(
|
||||
result["mode"],
|
||||
"failed-overlay-runtime-reconciliation",
|
||||
)
|
||||
(installed / entries[0]).write_text(
|
||||
"drift\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(RUNNER.DeployError):
|
||||
RUNNER.validate_module_foundry_sector_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
candidate,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user