SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(device-edge): reconcile runtime modes safely

Browse Source
This commit is contained in:
Codex
2026-08-12 01:18:40 +03:00
parent e85bb1448e
commit 6fa9164933
4 changed files with 234 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/build-device-edge-vps-artifact.mjs
+23 -2
View File
@@ -29,11 +29,17 @@ const runtimeCache = resolve(
const [phase, patchId, ...extra] = process.argv.slice(2);
if (
extra.length
|| !["foundation", "backhaul", "relay", "core-channel"].includes(phase)
|| ![
"foundation",
"runtime-reconciliation",
"backhaul",
"relay",
"core-channel",
].includes(phase)
|| !/^[A-Za-z0-9._-]{1,96}$/.test(patchId || "")
) {
throw new Error(
"usage: build-device-edge-vps-artifact.mjs <foundation|backhaul|relay|core-channel> <patch-id>",
"usage: build-device-edge-vps-artifact.mjs <foundation|runtime-reconciliation|backhaul|relay|core-channel> <patch-id>",
);
}
@@ -61,6 +67,9 @@ const entriesByPhase = {
`vendor/${nodeArchive}`,
`vendor/${tailscaleArchive}`,
],
"runtime-reconciliation": [
"deployment/device-edge-vps-runtime-reconciliation-v1.json",
],
backhaul: [
"vps/config/backhaul_ssh_config",
"vps/systemd/nodedc-b2-backhaul.service",
@@ -205,6 +214,18 @@ async function assertBoundary() {
throw new Error("foundation_must_not_open_9921");
}
}
if (phase === "runtime-reconciliation") {
for (const required of [
"recover-exact-runtime-executable-modes-after-failed-core-channel-publish",
"restore-root-owned-executable-mode-0755-for-exact-known-binaries",
'"publicCoreChannel": "disabled"',
'"trackerIngress": "disabled"',
]) {
if (!combined.includes(required)) {
throw new Error(`runtime_reconciliation_boundary_missing:${required}`);
}
}
}
if (phase === "backhaul") {
for (const required of [
"\"runtimeUser\": \"nodedc-backhaul\"",
infra/deploy-runner/nodedc-b2-vps-deploy
+117 -8
View File
@@ -64,6 +64,9 @@ TAILSCALE_ARCHIVE = "tailscale_1.102.2_amd64.tgz"
TAILSCALE_ARCHIVE_SHA256 = (
"ad2cde12f8de95f7b93a1e0401e652291c603d42b9d60a33fb1741eb38ab04d8"
)
NODE_BIN_SHA256 = "3517c2df0b2f8cd7f422b4b8450ef81c6889f08eb03e281d6de9079b15e6a327"
TAILSCALE_BIN_SHA256 = "58b0fa0907677ea6afe0d3022cc3e99b1a03f39a7ed60144843ed38252e00c80"
TAILSCALED_BIN_SHA256 = "5f17b092bac92326325f6c4ffd9991fad3c073975abe412d02ee68721a500394"
NODE_BIN = LIVE_ROOT / "runtime/node/bin/node"
TAILSCALE_BIN = LIVE_ROOT / "runtime/tailscale/tailscale"
@@ -108,6 +111,9 @@ FOUNDATION_ENTRIES = (
f"vendor/{NODE_ARCHIVE}",
f"vendor/{TAILSCALE_ARCHIVE}",
)
RUNTIME_RECONCILIATION_ENTRIES = (
"deployment/device-edge-vps-runtime-reconciliation-v1.json",
)
BACKHAUL_ENTRIES = (
"vps/config/backhaul_ssh_config",
"vps/systemd/nodedc-b2-backhaul.service",
@@ -133,6 +139,7 @@ CORE_CHANNEL_ENTRIES = (
PHASE_ENTRIES = {
"foundation": FOUNDATION_ENTRIES,
"runtime-reconciliation": RUNTIME_RECONCILIATION_ENTRIES,
"backhaul": BACKHAUL_ENTRIES,
"relay": RELAY_ENTRIES,
"core-channel": CORE_CHANNEL_ENTRIES,
@@ -153,6 +160,10 @@ PHASE_FILE_SHA256 = {
f"vendor/{NODE_ARCHIVE}": NODE_ARCHIVE_SHA256,
f"vendor/{TAILSCALE_ARCHIVE}": TAILSCALE_ARCHIVE_SHA256,
},
"runtime-reconciliation": {
"deployment/device-edge-vps-runtime-reconciliation-v1.json":
"edf7e05918efa9c9ab6d759ecdcc493ec6a2c92ed5b486772b3629cd5fa981ab",
},
"backhaul": {
"vps/config/backhaul_ssh_config":
"d0df8b70dda025b7c1c3fecd2dafffe60a5bb753650d3bc37db65db626cfc1af",
@@ -502,6 +513,58 @@ def source_file_state(phase: str):
return actual
def assert_runtime_reconciliation_predecessor(foundation_record):
if (
foundation_record.get("patch") != "device-edge-vps-foundation-20260806-003"
or foundation_record.get("sha256")
!= "1be852f144e9f0fea32af70bebd07a2607b6a1818825094bd4c1b4062064716a"
):
die("runtime reconciliation foundation predecessor mismatch")
failed = [
record
for record in journal_records(FAILED_JOURNAL)
if (
record.get("patch") == "device-edge-vps-core-channel-20260812-001"
and record.get("phase") == "core-channel"
and record.get("sha256")
== "c199980e5754cf3e874a09f42e408fc88885cdbf7c872eac36c0ab768a7bab00"
and record.get("status") == "failed"
and record.get("rollback") == "ok"
and record.get("error") == "PermissionError"
)
]
if len(failed) != 1:
die("runtime reconciliation failed Core channel predecessor mismatch")
for path, digest in (
(NODE_BIN, NODE_BIN_SHA256),
(TAILSCALE_BIN, TAILSCALE_BIN_SHA256),
(TAILSCALED_BIN, TAILSCALED_BIN_SHA256),
):
state = assert_regular_nonsymlink(path, f"runtime reconciliation {path.name}")
if (
state.st_uid != 0
or state.st_gid != 0
or (state.st_mode & 0o777) != 0o644
or sha256_file(path) != digest
):
die(f"runtime reconciliation binary predecessor mismatch: {path.name}")
if not service_active("nodedc-b2-tailscaled.service"):
die("runtime reconciliation Tailscale predecessor is inactive")
if service_active("nodedc-device-edge-channel.service") or user_exists(CHANNEL_USER):
die("runtime reconciliation Core channel predecessor is not absent")
assert_channel_trust(require_runtime_owner=False)
for port in (CHANNEL_HEALTH_PORT, CHANNEL_PUBLIC_PORT, 9921):
assert_port_closed(port)
nft = run(["/usr/sbin/nft", "list", "table", "inet", "nodedc_b2_vps"]).stdout
if (
"policy drop" not in nft
or "tcp dport 22" not in nft
or "tcp dport 8443" in nft
or "tcp dport 9921" in nft
):
die("runtime reconciliation firewall predecessor mismatch")
def systemctl(*args, check=True):
return run(["/usr/bin/systemctl", *args], check=check)
@@ -535,8 +598,13 @@ def current_phase_preflight(phase: str):
assert_port_closed(port)
return {"predecessor": "clean-ubuntu-24.04.4"}
applied_phase_record("foundation")
foundation_record = applied_phase_record("foundation")
source_file_state("foundation")
if phase == "runtime-reconciliation":
assert_runtime_reconciliation_predecessor(foundation_record)
return {
"predecessor": "failed-core-channel-001-rollback-runtime-mode-drift",
}
validate_foundation_runtime(
require_running_tailnet=phase in {"backhaul", "relay"},
expected_key_user=BACKHAUL_USER if phase == "relay" else SERVICE_USER,
@@ -619,6 +687,8 @@ def backup_targets_for_phase(phase: str):
common = [LIVE_ROOT / entry for entry in PHASE_ENTRIES[phase]]
if phase == "foundation":
return common + [SSHD_DROPIN, NFTABLES_CONFIG, TAILSCALE_UNIT]
if phase == "runtime-reconciliation":
return common + [NODE_BIN, TAILSCALE_BIN, TAILSCALED_BIN]
if phase == "backhaul":
return common + [BACKHAUL_UNIT, BACKHAUL_KNOWN_HOSTS]
if phase == "core-channel":
@@ -703,13 +773,16 @@ def publish_payload(payload: Path, entries):
shutil.copytree(source, target, symlinks=False)
else:
shutil.copy2(source, target, follow_symlinks=False)
for path in LIVE_ROOT.rglob("*"):
if path.is_symlink():
die(f"published live source contains symlink: {path}")
if path.is_dir():
os.chmod(path, 0o755)
else:
os.chmod(path, 0o644)
published = [target]
if target.is_dir():
published.extend(target.rglob("*"))
for path in published:
if path.is_symlink():
die(f"published live source contains symlink: {path}")
if path.is_dir():
os.chmod(path, 0o755)
else:
os.chmod(path, 0o644)
os.chown(LIVE_ROOT, 0, 0)
@@ -1013,6 +1086,13 @@ def apply_foundation(payload: Path):
validate_foundation_runtime(require_running_tailnet=False)
def apply_runtime_reconciliation(_payload: Path):
for path in (NODE_BIN, TAILSCALE_BIN, TAILSCALED_BIN):
os.chown(path, 0, 0)
os.chmod(path, 0o755)
validate_runtime_reconciliation()
def materialize_known_hosts(account):
line = (
f"[{BACKHAUL_TARGET_IP}]:{BACKHAUL_TARGET_PORT} "
@@ -1131,6 +1211,26 @@ def validate_foundation_runtime(*, require_running_tailnet: bool, expected_key_u
return status
def validate_runtime_reconciliation():
source_file_state("runtime-reconciliation")
for path, digest in (
(NODE_BIN, NODE_BIN_SHA256),
(TAILSCALE_BIN, TAILSCALE_BIN_SHA256),
(TAILSCALED_BIN, TAILSCALED_BIN_SHA256),
):
state = assert_regular_nonsymlink(path, f"reconciled runtime {path.name}")
if (
state.st_uid != 0
or state.st_gid != 0
or (state.st_mode & 0o777) != 0o755
or sha256_file(path) != digest
):
die(f"reconciled runtime binary mismatch: {path.name}")
validate_foundation_runtime(require_running_tailnet=False)
for port in (CHANNEL_HEALTH_PORT, CHANNEL_PUBLIC_PORT, 9921):
assert_port_closed(port)
def validate_backhaul_runtime():
validate_foundation_runtime(
require_running_tailnet=True,
@@ -1375,6 +1475,13 @@ def plan_artifact(artifact_argument: str):
print("public_b2_ingress=disabled")
print("services=nodedc-b2-tailscaled")
print(f"tailscale_runtime_identity={SERVICE_USER}")
elif phase == "runtime-reconciliation":
print("runtime_reconciliation=exact-known-binaries:0644=>0755")
print("failed_patch=device-edge-vps-core-channel-20260812-001")
print("public_core_channel=disabled")
print("public_b2_ingress=disabled")
print("tracker_tcp_9921=closed")
print("services=preserved:nodedc-b2-tailscaled")
elif phase == "backhaul":
print(f"target={BACKHAUL_TARGET_IP}:{BACKHAUL_TARGET_PORT}")
print(f"target_host_key_fingerprint={BACKHAUL_TARGET_FINGERPRINT}")
@@ -1425,6 +1532,8 @@ def apply_artifact(artifact_argument: str):
publish_payload(loaded["payload"], loaded["entries"])
if loaded["phase"] == "foundation":
apply_foundation(loaded["payload"])
elif loaded["phase"] == "runtime-reconciliation":
apply_runtime_reconciliation(loaded["payload"])
elif loaded["phase"] == "backhaul":
apply_backhaul(loaded["payload"])
elif loaded["phase"] == "relay":
infra/deploy-runner/test_device_edge_vps_artifact.py
+77 -2
View File
@@ -92,7 +92,13 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
def test_builders_are_deterministic_narrow_and_secret_free(self):
self.require_runtime_cache()
for phase in ("foundation", "backhaul", "relay", "core-channel"):
for phase in (
"foundation",
"runtime-reconciliation",
"backhaul",
"relay",
"core-channel",
):
with self.subTest(phase=phase), tempfile.TemporaryDirectory(
prefix=f"nodedc-vps-{phase}-"
) as directory:
@@ -164,7 +170,13 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
old_inbox = RUNNER.INBOX_ROOT
RUNNER.INBOX_ROOT = inbox
try:
for phase in ("foundation", "backhaul", "relay", "core-channel"):
for phase in (
"foundation",
"runtime-reconciliation",
"backhaul",
"relay",
"core-channel",
):
result = self.build(
inbox,
phase,
@@ -261,6 +273,69 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
self.assertIn("command_transport=disabled", rendered)
self.assertIn("gelios=untouched", rendered)
def test_runtime_reconciliation_plan_is_exact_and_opens_no_port(self):
with tempfile.TemporaryDirectory(prefix="nodedc-vps-reconcile-plan-") as directory:
inbox = Path(directory) / "inbox"
inbox.mkdir()
result = self.build(
inbox,
"runtime-reconciliation",
"device-edge-vps-runtime-reconciliation-plan-001",
)
self.assertEqual(result.returncode, 0, result.stderr)
artifact = Path(json.loads(result.stdout)["artifact"])
old_inbox = RUNNER.INBOX_ROOT
RUNNER.INBOX_ROOT = inbox
try:
with patch.object(RUNNER, "assert_root"), patch.object(
RUNNER,
"preflight",
return_value={
"predecessor": (
"failed-core-channel-001-rollback-runtime-mode-drift"
),
},
), patch("builtins.print") as output:
RUNNER.plan_artifact(str(artifact))
finally:
RUNNER.INBOX_ROOT = old_inbox
rendered = "\n".join(
" ".join(str(arg) for arg in call.args)
for call in output.call_args_list
)
self.assertIn("phase=runtime-reconciliation", rendered)
self.assertIn(
"runtime_reconciliation=exact-known-binaries:0644=>0755",
rendered,
)
self.assertIn("public_core_channel=disabled", rendered)
self.assertIn("tracker_tcp_9921=closed", rendered)
def test_publish_payload_preserves_unselected_executable_modes(self):
with tempfile.TemporaryDirectory(prefix="nodedc-vps-publish-scope-") as directory:
root = Path(directory)
live = root / "live"
payload = root / "payload"
runtime = live / "runtime/node/bin/node"
marker = payload / "deployment/reconciliation.json"
runtime.parent.mkdir(parents=True)
marker.parent.mkdir(parents=True)
runtime.write_bytes(b"runtime-binary")
runtime.chmod(0o755)
marker.write_text("{}\n", encoding="utf-8")
old_live = RUNNER.LIVE_ROOT
RUNNER.LIVE_ROOT = live
try:
with patch.object(RUNNER.os, "chown"):
RUNNER.publish_payload(payload, ("deployment/reconciliation.json",))
finally:
RUNNER.LIVE_ROOT = old_live
self.assertEqual(runtime.stat().st_mode & 0o777, 0o755)
self.assertEqual(
(live / "deployment/reconciliation.json").stat().st_mode & 0o777,
0o644,
)
def test_source_baseline_is_pinned_to_the_exact_accepted_predecessor(self):
with tempfile.TemporaryDirectory(prefix="nodedc-vps-baseline-") as directory:
journal = Path(directory) / "applied.jsonl"