From 6fa91649337d1702f214dd9bb1a1bc135d291023 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 12 Aug 2026 01:18:40 +0300 Subject: [PATCH] fix(device-edge): reconcile runtime modes safely --- ...ce-edge-vps-runtime-reconciliation-v1.json | 17 +++ .../build-device-edge-vps-artifact.mjs | 25 +++- infra/deploy-runner/nodedc-b2-vps-deploy | 125 ++++++++++++++++-- .../test_device_edge_vps_artifact.py | 79 ++++++++++- 4 files changed, 234 insertions(+), 12 deletions(-) create mode 100644 device-plane/deployment/device-edge-vps-runtime-reconciliation-v1.json diff --git a/device-plane/deployment/device-edge-vps-runtime-reconciliation-v1.json b/device-plane/deployment/device-edge-vps-runtime-reconciliation-v1.json new file mode 100644 index 0000000..dbea64b --- /dev/null +++ b/device-plane/deployment/device-edge-vps-runtime-reconciliation-v1.json @@ -0,0 +1,17 @@ +{ + "schemaVersion": "nodedc.device-edge-vps.runtime-reconciliation.v1", + "component": "device-edge-vps", + "runtimeHost": "koffyvngij", + "publicIpv4": "155.212.211.15", + "transition": "recover-exact-runtime-executable-modes-after-failed-core-channel-publish", + "acceptedFoundationPatch": "device-edge-vps-foundation-20260806-003", + "acceptedFoundationArtifactSha256": "1be852f144e9f0fea32af70bebd07a2607b6a1818825094bd4c1b4062064716a", + "failedPatch": "device-edge-vps-core-channel-20260812-001", + "failedArtifactSha256": "c199980e5754cf3e874a09f42e408fc88885cdbf7c872eac36c0ab768a7bab00", + "runtimeMutation": "restore-root-owned-executable-mode-0755-for-exact-known-binaries", + "publicCoreChannel": "disabled", + "trackerIngress": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "restore-exact-runtime-files-modes-and-reconciliation-marker" +} diff --git a/infra/deploy-runner/build-device-edge-vps-artifact.mjs b/infra/deploy-runner/build-device-edge-vps-artifact.mjs index 1bb197a..7a0009a 100644 --- a/infra/deploy-runner/build-device-edge-vps-artifact.mjs +++ b/infra/deploy-runner/build-device-edge-vps-artifact.mjs @@ -29,11 +29,17 @@ const runtimeCache = resolve( const [phase, patchId, ...extra] = process.argv.slice(2); if ( extra.length - || !["foundation", "backhaul", "relay", "core-channel"].includes(phase) + || ![ + "foundation", + "runtime-reconciliation", + "backhaul", + "relay", + "core-channel", + ].includes(phase) || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId || "") ) { throw new Error( - "usage: build-device-edge-vps-artifact.mjs ", + "usage: build-device-edge-vps-artifact.mjs ", ); } @@ -61,6 +67,9 @@ const entriesByPhase = { `vendor/${nodeArchive}`, `vendor/${tailscaleArchive}`, ], + "runtime-reconciliation": [ + "deployment/device-edge-vps-runtime-reconciliation-v1.json", + ], backhaul: [ "vps/config/backhaul_ssh_config", "vps/systemd/nodedc-b2-backhaul.service", @@ -205,6 +214,18 @@ async function assertBoundary() { throw new Error("foundation_must_not_open_9921"); } } + if (phase === "runtime-reconciliation") { + for (const required of [ + "recover-exact-runtime-executable-modes-after-failed-core-channel-publish", + "restore-root-owned-executable-mode-0755-for-exact-known-binaries", + '"publicCoreChannel": "disabled"', + '"trackerIngress": "disabled"', + ]) { + if (!combined.includes(required)) { + throw new Error(`runtime_reconciliation_boundary_missing:${required}`); + } + } + } if (phase === "backhaul") { for (const required of [ "\"runtimeUser\": \"nodedc-backhaul\"", diff --git a/infra/deploy-runner/nodedc-b2-vps-deploy b/infra/deploy-runner/nodedc-b2-vps-deploy index afa79f3..df65bfc 100755 --- a/infra/deploy-runner/nodedc-b2-vps-deploy +++ b/infra/deploy-runner/nodedc-b2-vps-deploy @@ -64,6 +64,9 @@ TAILSCALE_ARCHIVE = "tailscale_1.102.2_amd64.tgz" TAILSCALE_ARCHIVE_SHA256 = ( "ad2cde12f8de95f7b93a1e0401e652291c603d42b9d60a33fb1741eb38ab04d8" ) +NODE_BIN_SHA256 = "3517c2df0b2f8cd7f422b4b8450ef81c6889f08eb03e281d6de9079b15e6a327" +TAILSCALE_BIN_SHA256 = "58b0fa0907677ea6afe0d3022cc3e99b1a03f39a7ed60144843ed38252e00c80" +TAILSCALED_BIN_SHA256 = "5f17b092bac92326325f6c4ffd9991fad3c073975abe412d02ee68721a500394" NODE_BIN = LIVE_ROOT / "runtime/node/bin/node" TAILSCALE_BIN = LIVE_ROOT / "runtime/tailscale/tailscale" @@ -108,6 +111,9 @@ FOUNDATION_ENTRIES = ( f"vendor/{NODE_ARCHIVE}", f"vendor/{TAILSCALE_ARCHIVE}", ) +RUNTIME_RECONCILIATION_ENTRIES = ( + "deployment/device-edge-vps-runtime-reconciliation-v1.json", +) BACKHAUL_ENTRIES = ( "vps/config/backhaul_ssh_config", "vps/systemd/nodedc-b2-backhaul.service", @@ -133,6 +139,7 @@ CORE_CHANNEL_ENTRIES = ( PHASE_ENTRIES = { "foundation": FOUNDATION_ENTRIES, + "runtime-reconciliation": RUNTIME_RECONCILIATION_ENTRIES, "backhaul": BACKHAUL_ENTRIES, "relay": RELAY_ENTRIES, "core-channel": CORE_CHANNEL_ENTRIES, @@ -153,6 +160,10 @@ PHASE_FILE_SHA256 = { f"vendor/{NODE_ARCHIVE}": NODE_ARCHIVE_SHA256, f"vendor/{TAILSCALE_ARCHIVE}": TAILSCALE_ARCHIVE_SHA256, }, + "runtime-reconciliation": { + "deployment/device-edge-vps-runtime-reconciliation-v1.json": + "edf7e05918efa9c9ab6d759ecdcc493ec6a2c92ed5b486772b3629cd5fa981ab", + }, "backhaul": { "vps/config/backhaul_ssh_config": "d0df8b70dda025b7c1c3fecd2dafffe60a5bb753650d3bc37db65db626cfc1af", @@ -502,6 +513,58 @@ def source_file_state(phase: str): return actual +def assert_runtime_reconciliation_predecessor(foundation_record): + if ( + foundation_record.get("patch") != "device-edge-vps-foundation-20260806-003" + or foundation_record.get("sha256") + != "1be852f144e9f0fea32af70bebd07a2607b6a1818825094bd4c1b4062064716a" + ): + die("runtime reconciliation foundation predecessor mismatch") + failed = [ + record + for record in journal_records(FAILED_JOURNAL) + if ( + record.get("patch") == "device-edge-vps-core-channel-20260812-001" + and record.get("phase") == "core-channel" + and record.get("sha256") + == "c199980e5754cf3e874a09f42e408fc88885cdbf7c872eac36c0ab768a7bab00" + and record.get("status") == "failed" + and record.get("rollback") == "ok" + and record.get("error") == "PermissionError" + ) + ] + if len(failed) != 1: + die("runtime reconciliation failed Core channel predecessor mismatch") + for path, digest in ( + (NODE_BIN, NODE_BIN_SHA256), + (TAILSCALE_BIN, TAILSCALE_BIN_SHA256), + (TAILSCALED_BIN, TAILSCALED_BIN_SHA256), + ): + state = assert_regular_nonsymlink(path, f"runtime reconciliation {path.name}") + if ( + state.st_uid != 0 + or state.st_gid != 0 + or (state.st_mode & 0o777) != 0o644 + or sha256_file(path) != digest + ): + die(f"runtime reconciliation binary predecessor mismatch: {path.name}") + if not service_active("nodedc-b2-tailscaled.service"): + die("runtime reconciliation Tailscale predecessor is inactive") + if service_active("nodedc-device-edge-channel.service") or user_exists(CHANNEL_USER): + die("runtime reconciliation Core channel predecessor is not absent") + assert_channel_trust(require_runtime_owner=False) + for port in (CHANNEL_HEALTH_PORT, CHANNEL_PUBLIC_PORT, 9921): + assert_port_closed(port) + nft = run(["/usr/sbin/nft", "list", "table", "inet", "nodedc_b2_vps"]).stdout + if ( + "policy drop" not in nft + or "tcp dport 22" not in nft + or "tcp dport 8443" in nft + or "tcp dport 9921" in nft + ): + die("runtime reconciliation firewall predecessor mismatch") + + def systemctl(*args, check=True): return run(["/usr/bin/systemctl", *args], check=check) @@ -535,8 +598,13 @@ def current_phase_preflight(phase: str): assert_port_closed(port) return {"predecessor": "clean-ubuntu-24.04.4"} - applied_phase_record("foundation") + foundation_record = applied_phase_record("foundation") source_file_state("foundation") + if phase == "runtime-reconciliation": + assert_runtime_reconciliation_predecessor(foundation_record) + return { + "predecessor": "failed-core-channel-001-rollback-runtime-mode-drift", + } validate_foundation_runtime( require_running_tailnet=phase in {"backhaul", "relay"}, expected_key_user=BACKHAUL_USER if phase == "relay" else SERVICE_USER, @@ -619,6 +687,8 @@ def backup_targets_for_phase(phase: str): common = [LIVE_ROOT / entry for entry in PHASE_ENTRIES[phase]] if phase == "foundation": return common + [SSHD_DROPIN, NFTABLES_CONFIG, TAILSCALE_UNIT] + if phase == "runtime-reconciliation": + return common + [NODE_BIN, TAILSCALE_BIN, TAILSCALED_BIN] if phase == "backhaul": return common + [BACKHAUL_UNIT, BACKHAUL_KNOWN_HOSTS] if phase == "core-channel": @@ -703,13 +773,16 @@ def publish_payload(payload: Path, entries): shutil.copytree(source, target, symlinks=False) else: shutil.copy2(source, target, follow_symlinks=False) - for path in LIVE_ROOT.rglob("*"): - if path.is_symlink(): - die(f"published live source contains symlink: {path}") - if path.is_dir(): - os.chmod(path, 0o755) - else: - os.chmod(path, 0o644) + published = [target] + if target.is_dir(): + published.extend(target.rglob("*")) + for path in published: + if path.is_symlink(): + die(f"published live source contains symlink: {path}") + if path.is_dir(): + os.chmod(path, 0o755) + else: + os.chmod(path, 0o644) os.chown(LIVE_ROOT, 0, 0) @@ -1013,6 +1086,13 @@ def apply_foundation(payload: Path): validate_foundation_runtime(require_running_tailnet=False) +def apply_runtime_reconciliation(_payload: Path): + for path in (NODE_BIN, TAILSCALE_BIN, TAILSCALED_BIN): + os.chown(path, 0, 0) + os.chmod(path, 0o755) + validate_runtime_reconciliation() + + def materialize_known_hosts(account): line = ( f"[{BACKHAUL_TARGET_IP}]:{BACKHAUL_TARGET_PORT} " @@ -1131,6 +1211,26 @@ def validate_foundation_runtime(*, require_running_tailnet: bool, expected_key_u return status +def validate_runtime_reconciliation(): + source_file_state("runtime-reconciliation") + for path, digest in ( + (NODE_BIN, NODE_BIN_SHA256), + (TAILSCALE_BIN, TAILSCALE_BIN_SHA256), + (TAILSCALED_BIN, TAILSCALED_BIN_SHA256), + ): + state = assert_regular_nonsymlink(path, f"reconciled runtime {path.name}") + if ( + state.st_uid != 0 + or state.st_gid != 0 + or (state.st_mode & 0o777) != 0o755 + or sha256_file(path) != digest + ): + die(f"reconciled runtime binary mismatch: {path.name}") + validate_foundation_runtime(require_running_tailnet=False) + for port in (CHANNEL_HEALTH_PORT, CHANNEL_PUBLIC_PORT, 9921): + assert_port_closed(port) + + def validate_backhaul_runtime(): validate_foundation_runtime( require_running_tailnet=True, @@ -1375,6 +1475,13 @@ def plan_artifact(artifact_argument: str): print("public_b2_ingress=disabled") print("services=nodedc-b2-tailscaled") print(f"tailscale_runtime_identity={SERVICE_USER}") + elif phase == "runtime-reconciliation": + print("runtime_reconciliation=exact-known-binaries:0644=>0755") + print("failed_patch=device-edge-vps-core-channel-20260812-001") + print("public_core_channel=disabled") + print("public_b2_ingress=disabled") + print("tracker_tcp_9921=closed") + print("services=preserved:nodedc-b2-tailscaled") elif phase == "backhaul": print(f"target={BACKHAUL_TARGET_IP}:{BACKHAUL_TARGET_PORT}") print(f"target_host_key_fingerprint={BACKHAUL_TARGET_FINGERPRINT}") @@ -1425,6 +1532,8 @@ def apply_artifact(artifact_argument: str): publish_payload(loaded["payload"], loaded["entries"]) if loaded["phase"] == "foundation": apply_foundation(loaded["payload"]) + elif loaded["phase"] == "runtime-reconciliation": + apply_runtime_reconciliation(loaded["payload"]) elif loaded["phase"] == "backhaul": apply_backhaul(loaded["payload"]) elif loaded["phase"] == "relay": diff --git a/infra/deploy-runner/test_device_edge_vps_artifact.py b/infra/deploy-runner/test_device_edge_vps_artifact.py index db399d4..dc80a71 100644 --- a/infra/deploy-runner/test_device_edge_vps_artifact.py +++ b/infra/deploy-runner/test_device_edge_vps_artifact.py @@ -92,7 +92,13 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase): def test_builders_are_deterministic_narrow_and_secret_free(self): self.require_runtime_cache() - for phase in ("foundation", "backhaul", "relay", "core-channel"): + for phase in ( + "foundation", + "runtime-reconciliation", + "backhaul", + "relay", + "core-channel", + ): with self.subTest(phase=phase), tempfile.TemporaryDirectory( prefix=f"nodedc-vps-{phase}-" ) as directory: @@ -164,7 +170,13 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase): old_inbox = RUNNER.INBOX_ROOT RUNNER.INBOX_ROOT = inbox try: - for phase in ("foundation", "backhaul", "relay", "core-channel"): + for phase in ( + "foundation", + "runtime-reconciliation", + "backhaul", + "relay", + "core-channel", + ): result = self.build( inbox, phase, @@ -261,6 +273,69 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase): self.assertIn("command_transport=disabled", rendered) self.assertIn("gelios=untouched", rendered) + def test_runtime_reconciliation_plan_is_exact_and_opens_no_port(self): + with tempfile.TemporaryDirectory(prefix="nodedc-vps-reconcile-plan-") as directory: + inbox = Path(directory) / "inbox" + inbox.mkdir() + result = self.build( + inbox, + "runtime-reconciliation", + "device-edge-vps-runtime-reconciliation-plan-001", + ) + self.assertEqual(result.returncode, 0, result.stderr) + artifact = Path(json.loads(result.stdout)["artifact"]) + old_inbox = RUNNER.INBOX_ROOT + RUNNER.INBOX_ROOT = inbox + try: + with patch.object(RUNNER, "assert_root"), patch.object( + RUNNER, + "preflight", + return_value={ + "predecessor": ( + "failed-core-channel-001-rollback-runtime-mode-drift" + ), + }, + ), patch("builtins.print") as output: + RUNNER.plan_artifact(str(artifact)) + finally: + RUNNER.INBOX_ROOT = old_inbox + rendered = "\n".join( + " ".join(str(arg) for arg in call.args) + for call in output.call_args_list + ) + self.assertIn("phase=runtime-reconciliation", rendered) + self.assertIn( + "runtime_reconciliation=exact-known-binaries:0644=>0755", + rendered, + ) + self.assertIn("public_core_channel=disabled", rendered) + self.assertIn("tracker_tcp_9921=closed", rendered) + + def test_publish_payload_preserves_unselected_executable_modes(self): + with tempfile.TemporaryDirectory(prefix="nodedc-vps-publish-scope-") as directory: + root = Path(directory) + live = root / "live" + payload = root / "payload" + runtime = live / "runtime/node/bin/node" + marker = payload / "deployment/reconciliation.json" + runtime.parent.mkdir(parents=True) + marker.parent.mkdir(parents=True) + runtime.write_bytes(b"runtime-binary") + runtime.chmod(0o755) + marker.write_text("{}\n", encoding="utf-8") + old_live = RUNNER.LIVE_ROOT + RUNNER.LIVE_ROOT = live + try: + with patch.object(RUNNER.os, "chown"): + RUNNER.publish_payload(payload, ("deployment/reconciliation.json",)) + finally: + RUNNER.LIVE_ROOT = old_live + self.assertEqual(runtime.stat().st_mode & 0o777, 0o755) + self.assertEqual( + (live / "deployment/reconciliation.json").stat().st_mode & 0o777, + 0o644, + ) + def test_source_baseline_is_pinned_to_the_exact_accepted_predecessor(self): with tempfile.TemporaryDirectory(prefix="nodedc-vps-baseline-") as directory: journal = Path(directory) / "applied.jsonl"