404 lines
15 KiB
Python
404 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import hashlib
|
|
import importlib.machinery
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
|
|
|
|
|
def load_runner():
|
|
loader = importlib.machinery.SourceFileLoader(
|
|
"nodedc_module_foundry_runtime_recovery",
|
|
str(RUNNER_PATH),
|
|
)
|
|
spec = importlib.util.spec_from_loader(loader.name, loader)
|
|
module = importlib.util.module_from_spec(spec)
|
|
loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
RUNNER = load_runner()
|
|
|
|
|
|
def runtime(container_char="a", image_char="b"):
|
|
return {
|
|
"schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
|
|
"composeProject": "nodedc-module-foundry",
|
|
"service": RUNNER.MODULE_FOUNDRY_SERVICE,
|
|
"containerId": container_char * 64,
|
|
"imageId": f"sha256:{image_char * 64}",
|
|
"imageRef": "nodedc-module-foundry-nodedc-module-foundry:latest",
|
|
"health": "healthy",
|
|
}
|
|
|
|
|
|
class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|
def test_build_retries_before_any_runtime_recreate(self):
|
|
first_failure = subprocess.CalledProcessError(17, ["docker", "compose"])
|
|
completed = subprocess.CompletedProcess([], 0)
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"component_compose_root",
|
|
return_value=Path("/compose"),
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"compose_base_cmd",
|
|
return_value=["docker", "compose"],
|
|
),
|
|
mock.patch.object(
|
|
RUNNER.subprocess,
|
|
"run",
|
|
side_effect=[first_failure, completed],
|
|
) as run,
|
|
mock.patch.object(RUNNER.time, "sleep") as sleep,
|
|
mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
|
|
):
|
|
RUNNER.run_module_foundry_compose(
|
|
(RUNNER.MODULE_FOUNDRY_SERVICE,)
|
|
)
|
|
|
|
expected = [
|
|
"docker",
|
|
"compose",
|
|
"build",
|
|
RUNNER.MODULE_FOUNDRY_SERVICE,
|
|
]
|
|
self.assertEqual(run.call_count, 2)
|
|
self.assertEqual(run.call_args_list[0].args[0], expected)
|
|
self.assertEqual(run.call_args_list[1].args[0], expected)
|
|
sleep.assert_called_once_with(2)
|
|
up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
|
|
|
|
def test_runtime_recreate_uses_built_image_without_build_or_pull(self):
|
|
completed = subprocess.CompletedProcess([], 0)
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"component_compose_root",
|
|
return_value=Path("/compose"),
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"compose_base_cmd",
|
|
return_value=["docker", "compose"],
|
|
),
|
|
mock.patch.object(
|
|
RUNNER.subprocess,
|
|
"run",
|
|
return_value=completed,
|
|
) as run,
|
|
):
|
|
RUNNER.run_module_foundry_compose_up(
|
|
(RUNNER.MODULE_FOUNDRY_SERVICE,)
|
|
)
|
|
|
|
up = run.call_args_list[0].args[0]
|
|
self.assertEqual(
|
|
up,
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"up",
|
|
"-d",
|
|
"--force-recreate",
|
|
"--pull",
|
|
"never",
|
|
"--no-deps",
|
|
RUNNER.MODULE_FOUNDRY_SERVICE,
|
|
],
|
|
)
|
|
self.assertNotIn("--build", up)
|
|
self.assertEqual(run.call_args_list[1].args[0], ["docker", "compose", "ps"])
|
|
|
|
def test_rollback_restores_source_and_keeps_unchanged_runtime(self):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="module-foundry-rollback-unchanged-"
|
|
) as directory:
|
|
work = Path(directory)
|
|
root = work / "source"
|
|
backup = work / "backup"
|
|
tmp = work / "tmp"
|
|
root.mkdir()
|
|
backup.mkdir()
|
|
tmp.mkdir()
|
|
entries = ("old.txt", "new.txt")
|
|
(root / "old.txt").write_text("old\n", encoding="utf-8")
|
|
RUNNER.create_backup(root, backup, entries, False)
|
|
(root / "old.txt").write_text("candidate\n", encoding="utf-8")
|
|
(root / "new.txt").write_text("candidate\n", encoding="utf-8")
|
|
before = runtime()
|
|
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
|
json.dumps(before),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with (
|
|
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"module_foundry_runtime_inventory",
|
|
return_value=before,
|
|
),
|
|
mock.patch.object(RUNNER, "run_healthchecks") as health,
|
|
mock.patch.object(RUNNER.subprocess, "run") as process,
|
|
):
|
|
result = RUNNER.rollback_module_foundry_apply(
|
|
root,
|
|
backup,
|
|
entries,
|
|
"20260809-000000",
|
|
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
|
)
|
|
|
|
self.assertEqual(
|
|
result,
|
|
"source-restored-runtime-unchanged:2",
|
|
)
|
|
self.assertEqual(
|
|
(root / "old.txt").read_text(encoding="utf-8"),
|
|
"old\n",
|
|
)
|
|
self.assertFalse((root / "new.txt").exists())
|
|
process.assert_not_called()
|
|
health.assert_called_once()
|
|
|
|
def test_rollback_retags_and_recreates_changed_runtime(self):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="module-foundry-rollback-changed-"
|
|
) as directory:
|
|
work = Path(directory)
|
|
root = work / "source"
|
|
backup = work / "backup"
|
|
tmp = work / "tmp"
|
|
root.mkdir()
|
|
backup.mkdir()
|
|
tmp.mkdir()
|
|
entries = ("old.txt",)
|
|
(root / "old.txt").write_text("old\n", encoding="utf-8")
|
|
RUNNER.create_backup(root, backup, entries, False)
|
|
(root / "old.txt").write_text("candidate\n", encoding="utf-8")
|
|
before = runtime()
|
|
candidate = runtime("c", "d")
|
|
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
|
json.dumps(before),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with (
|
|
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"module_foundry_runtime_inventory",
|
|
side_effect=[candidate, before],
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"inspect_local_image",
|
|
return_value=before["imageId"],
|
|
),
|
|
mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
|
|
mock.patch.object(RUNNER, "run_healthchecks") as health,
|
|
mock.patch.object(RUNNER.subprocess, "run") as process,
|
|
):
|
|
result = RUNNER.rollback_module_foundry_apply(
|
|
root,
|
|
backup,
|
|
entries,
|
|
"20260809-000000",
|
|
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
|
)
|
|
|
|
self.assertEqual(result, "source+runtime-restored:1")
|
|
process.assert_called_once_with(
|
|
[
|
|
str(RUNNER.DOCKER),
|
|
"image",
|
|
"tag",
|
|
before["imageId"],
|
|
before["imageRef"],
|
|
],
|
|
check=True,
|
|
)
|
|
up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
|
|
health.assert_called_once()
|
|
|
|
def test_candidate_must_replace_both_container_and_image(self):
|
|
before = runtime()
|
|
for candidate in (
|
|
runtime("a", "c"),
|
|
runtime("c", "b"),
|
|
):
|
|
with mock.patch.object(
|
|
RUNNER,
|
|
"module_foundry_runtime_inventory",
|
|
return_value=candidate,
|
|
):
|
|
with self.assertRaises(RUNNER.DeployError):
|
|
RUNNER.validate_module_foundry_candidate_runtime(before)
|
|
|
|
def test_preapply_runtime_rejects_digest_only_image_reference(self):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="module-foundry-runtime-reference-"
|
|
) as directory:
|
|
backup = Path(directory)
|
|
before = runtime()
|
|
before["imageRef"] = before["imageId"]
|
|
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
|
json.dumps(before),
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaises(RUNNER.DeployError):
|
|
RUNNER.read_module_foundry_runtime_before(backup)
|
|
|
|
def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="module-foundry-sector-evidence-"
|
|
) as directory:
|
|
work = Path(directory)
|
|
backups = work / "backups"
|
|
failed = work / "failed"
|
|
state = work / "state"
|
|
tmp = work / "tmp"
|
|
installed = work / "installed"
|
|
candidate = work / "candidate"
|
|
for path in (backups, failed, state, tmp, installed, candidate):
|
|
path.mkdir()
|
|
|
|
entries = RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
|
for index, rel in enumerate(entries):
|
|
for root in (installed, candidate):
|
|
path = root / rel
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(f"payload-{index}\n", encoding="utf-8")
|
|
|
|
backup = backups / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
|
backup.mkdir()
|
|
for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256:
|
|
(backup / name).write_text(f"{name}\n", encoding="utf-8")
|
|
backup_hashes = {
|
|
name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
|
|
for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
|
|
}
|
|
|
|
archive_source = work / "archive-source"
|
|
(archive_source / "payload").mkdir(parents=True)
|
|
(archive_source / "manifest.env").write_text(
|
|
"id="
|
|
f"{RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID}\n"
|
|
"component=module-foundry\n"
|
|
"type=app-overlay\n",
|
|
encoding="utf-8",
|
|
)
|
|
(archive_source / "files.txt").write_text(
|
|
"\n".join(entries) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
for rel in entries:
|
|
source = candidate / rel
|
|
target = archive_source / "payload" / rel
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(source.read_bytes())
|
|
failed_artifact = (
|
|
failed / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
|
)
|
|
with tarfile.open(failed_artifact, "w:gz") as archive:
|
|
archive.add(
|
|
archive_source / "manifest.env",
|
|
arcname="manifest.env",
|
|
)
|
|
archive.add(
|
|
archive_source / "files.txt",
|
|
arcname="files.txt",
|
|
)
|
|
archive.add(archive_source / "payload", arcname="payload")
|
|
failed_sha256 = hashlib.sha256(
|
|
failed_artifact.read_bytes()
|
|
).hexdigest()
|
|
|
|
journal = state / "failed.jsonl"
|
|
journal.write_text(
|
|
json.dumps(
|
|
{
|
|
"artifact": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT,
|
|
"backup_id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
|
|
"component": "module-foundry",
|
|
"id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
|
|
"message": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE,
|
|
"rollback_status": "not-required",
|
|
"sha256": failed_sha256,
|
|
"started_apply": True,
|
|
"status": "failed",
|
|
}
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
manifest = {
|
|
"id": RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID,
|
|
"component": "module-foundry",
|
|
"type": "app-overlay",
|
|
}
|
|
|
|
patches = (
|
|
mock.patch.object(RUNNER, "BACKUPS_DIR", backups),
|
|
mock.patch.object(RUNNER, "FAILED_DIR", failed),
|
|
mock.patch.object(RUNNER, "FAILED_STATE_FILE", journal),
|
|
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256",
|
|
failed_sha256,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256",
|
|
backup_hashes,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"component_root",
|
|
return_value=installed,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"module_foundry_runtime_inventory",
|
|
return_value=runtime(),
|
|
),
|
|
)
|
|
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6], patches[7]:
|
|
result = RUNNER.validate_module_foundry_sector_recovery_evidence(
|
|
manifest,
|
|
entries,
|
|
candidate,
|
|
)
|
|
self.assertEqual(
|
|
result["mode"],
|
|
"failed-overlay-runtime-reconciliation",
|
|
)
|
|
(installed / entries[0]).write_text(
|
|
"drift\n",
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaises(RUNNER.DeployError):
|
|
RUNNER.validate_module_foundry_sector_recovery_evidence(
|
|
manifest,
|
|
entries,
|
|
candidate,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|