SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(device-edge): reconcile runtime modes safely

Browse Source
This commit is contained in:
Codex
2026-08-12 01:18:40 +03:00
parent e85bb1448e
commit 6fa9164933
4 changed files with 234 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-b2-vps-deploy
+117 -8
View File
@@ -64,6 +64,9 @@ TAILSCALE_ARCHIVE = "tailscale_1.102.2_amd64.tgz"
TAILSCALE_ARCHIVE_SHA256 = (
"ad2cde12f8de95f7b93a1e0401e652291c603d42b9d60a33fb1741eb38ab04d8"
)
NODE_BIN_SHA256 = "3517c2df0b2f8cd7f422b4b8450ef81c6889f08eb03e281d6de9079b15e6a327"
TAILSCALE_BIN_SHA256 = "58b0fa0907677ea6afe0d3022cc3e99b1a03f39a7ed60144843ed38252e00c80"
TAILSCALED_BIN_SHA256 = "5f17b092bac92326325f6c4ffd9991fad3c073975abe412d02ee68721a500394"
NODE_BIN = LIVE_ROOT / "runtime/node/bin/node"
TAILSCALE_BIN = LIVE_ROOT / "runtime/tailscale/tailscale"
@@ -108,6 +111,9 @@ FOUNDATION_ENTRIES = (
f"vendor/{NODE_ARCHIVE}",
f"vendor/{TAILSCALE_ARCHIVE}",
)
RUNTIME_RECONCILIATION_ENTRIES = (
"deployment/device-edge-vps-runtime-reconciliation-v1.json",
)
BACKHAUL_ENTRIES = (
"vps/config/backhaul_ssh_config",
"vps/systemd/nodedc-b2-backhaul.service",
@@ -133,6 +139,7 @@ CORE_CHANNEL_ENTRIES = (
PHASE_ENTRIES = {
"foundation": FOUNDATION_ENTRIES,
"runtime-reconciliation": RUNTIME_RECONCILIATION_ENTRIES,
"backhaul": BACKHAUL_ENTRIES,
"relay": RELAY_ENTRIES,
"core-channel": CORE_CHANNEL_ENTRIES,
@@ -153,6 +160,10 @@ PHASE_FILE_SHA256 = {
f"vendor/{NODE_ARCHIVE}": NODE_ARCHIVE_SHA256,
f"vendor/{TAILSCALE_ARCHIVE}": TAILSCALE_ARCHIVE_SHA256,
},
"runtime-reconciliation": {
"deployment/device-edge-vps-runtime-reconciliation-v1.json":
"edf7e05918efa9c9ab6d759ecdcc493ec6a2c92ed5b486772b3629cd5fa981ab",
},
"backhaul": {
"vps/config/backhaul_ssh_config":
"d0df8b70dda025b7c1c3fecd2dafffe60a5bb753650d3bc37db65db626cfc1af",
@@ -502,6 +513,58 @@ def source_file_state(phase: str):
return actual
def assert_runtime_reconciliation_predecessor(foundation_record):
if (
foundation_record.get("patch") != "device-edge-vps-foundation-20260806-003"
or foundation_record.get("sha256")
!= "1be852f144e9f0fea32af70bebd07a2607b6a1818825094bd4c1b4062064716a"
):
die("runtime reconciliation foundation predecessor mismatch")
failed = [
record
for record in journal_records(FAILED_JOURNAL)
if (
record.get("patch") == "device-edge-vps-core-channel-20260812-001"
and record.get("phase") == "core-channel"
and record.get("sha256")
== "c199980e5754cf3e874a09f42e408fc88885cdbf7c872eac36c0ab768a7bab00"
and record.get("status") == "failed"
and record.get("rollback") == "ok"
and record.get("error") == "PermissionError"
)
]
if len(failed) != 1:
die("runtime reconciliation failed Core channel predecessor mismatch")
for path, digest in (
(NODE_BIN, NODE_BIN_SHA256),
(TAILSCALE_BIN, TAILSCALE_BIN_SHA256),
(TAILSCALED_BIN, TAILSCALED_BIN_SHA256),
):
state = assert_regular_nonsymlink(path, f"runtime reconciliation {path.name}")
if (
state.st_uid != 0
or state.st_gid != 0
or (state.st_mode & 0o777) != 0o644
or sha256_file(path) != digest
):
die(f"runtime reconciliation binary predecessor mismatch: {path.name}")
if not service_active("nodedc-b2-tailscaled.service"):
die("runtime reconciliation Tailscale predecessor is inactive")
if service_active("nodedc-device-edge-channel.service") or user_exists(CHANNEL_USER):
die("runtime reconciliation Core channel predecessor is not absent")
assert_channel_trust(require_runtime_owner=False)
for port in (CHANNEL_HEALTH_PORT, CHANNEL_PUBLIC_PORT, 9921):
assert_port_closed(port)
nft = run(["/usr/sbin/nft", "list", "table", "inet", "nodedc_b2_vps"]).stdout
if (
"policy drop" not in nft
or "tcp dport 22" not in nft
or "tcp dport 8443" in nft
or "tcp dport 9921" in nft
):
die("runtime reconciliation firewall predecessor mismatch")
def systemctl(*args, check=True):
return run(["/usr/bin/systemctl", *args], check=check)
@@ -535,8 +598,13 @@ def current_phase_preflight(phase: str):
assert_port_closed(port)
return {"predecessor": "clean-ubuntu-24.04.4"}
applied_phase_record("foundation")
foundation_record = applied_phase_record("foundation")
source_file_state("foundation")
if phase == "runtime-reconciliation":
assert_runtime_reconciliation_predecessor(foundation_record)
return {
"predecessor": "failed-core-channel-001-rollback-runtime-mode-drift",
}
validate_foundation_runtime(
require_running_tailnet=phase in {"backhaul", "relay"},
expected_key_user=BACKHAUL_USER if phase == "relay" else SERVICE_USER,
@@ -619,6 +687,8 @@ def backup_targets_for_phase(phase: str):
common = [LIVE_ROOT / entry for entry in PHASE_ENTRIES[phase]]
if phase == "foundation":
return common + [SSHD_DROPIN, NFTABLES_CONFIG, TAILSCALE_UNIT]
if phase == "runtime-reconciliation":
return common + [NODE_BIN, TAILSCALE_BIN, TAILSCALED_BIN]
if phase == "backhaul":
return common + [BACKHAUL_UNIT, BACKHAUL_KNOWN_HOSTS]
if phase == "core-channel":
@@ -703,13 +773,16 @@ def publish_payload(payload: Path, entries):
shutil.copytree(source, target, symlinks=False)
else:
shutil.copy2(source, target, follow_symlinks=False)
for path in LIVE_ROOT.rglob("*"):
if path.is_symlink():
die(f"published live source contains symlink: {path}")
if path.is_dir():
os.chmod(path, 0o755)
else:
os.chmod(path, 0o644)
published = [target]
if target.is_dir():
published.extend(target.rglob("*"))
for path in published:
if path.is_symlink():
die(f"published live source contains symlink: {path}")
if path.is_dir():
os.chmod(path, 0o755)
else:
os.chmod(path, 0o644)
os.chown(LIVE_ROOT, 0, 0)
@@ -1013,6 +1086,13 @@ def apply_foundation(payload: Path):
validate_foundation_runtime(require_running_tailnet=False)
def apply_runtime_reconciliation(_payload: Path):
for path in (NODE_BIN, TAILSCALE_BIN, TAILSCALED_BIN):
os.chown(path, 0, 0)
os.chmod(path, 0o755)
validate_runtime_reconciliation()
def materialize_known_hosts(account):
line = (
f"[{BACKHAUL_TARGET_IP}]:{BACKHAUL_TARGET_PORT} "
@@ -1131,6 +1211,26 @@ def validate_foundation_runtime(*, require_running_tailnet: bool, expected_key_u
return status
def validate_runtime_reconciliation():
source_file_state("runtime-reconciliation")
for path, digest in (
(NODE_BIN, NODE_BIN_SHA256),
(TAILSCALE_BIN, TAILSCALE_BIN_SHA256),
(TAILSCALED_BIN, TAILSCALED_BIN_SHA256),
):
state = assert_regular_nonsymlink(path, f"reconciled runtime {path.name}")
if (
state.st_uid != 0
or state.st_gid != 0
or (state.st_mode & 0o777) != 0o755
or sha256_file(path) != digest
):
die(f"reconciled runtime binary mismatch: {path.name}")
validate_foundation_runtime(require_running_tailnet=False)
for port in (CHANNEL_HEALTH_PORT, CHANNEL_PUBLIC_PORT, 9921):
assert_port_closed(port)
def validate_backhaul_runtime():
validate_foundation_runtime(
require_running_tailnet=True,
@@ -1375,6 +1475,13 @@ def plan_artifact(artifact_argument: str):
print("public_b2_ingress=disabled")
print("services=nodedc-b2-tailscaled")
print(f"tailscale_runtime_identity={SERVICE_USER}")
elif phase == "runtime-reconciliation":
print("runtime_reconciliation=exact-known-binaries:0644=>0755")
print("failed_patch=device-edge-vps-core-channel-20260812-001")
print("public_core_channel=disabled")
print("public_b2_ingress=disabled")
print("tracker_tcp_9921=closed")
print("services=preserved:nodedc-b2-tailscaled")
elif phase == "backhaul":
print(f"target={BACKHAUL_TARGET_IP}:{BACKHAUL_TARGET_PORT}")
print(f"target_host_key_fingerprint={BACKHAUL_TARGET_FINGERPRINT}")
@@ -1425,6 +1532,8 @@ def apply_artifact(artifact_argument: str):
publish_payload(loaded["payload"], loaded["entries"])
if loaded["phase"] == "foundation":
apply_foundation(loaded["payload"])
elif loaded["phase"] == "runtime-reconciliation":
apply_runtime_reconciliation(loaded["payload"])
elif loaded["phase"] == "backhaul":
apply_backhaul(loaded["payload"])
elif loaded["phase"] == "relay":