diff --git a/device-plane/.dockerignore b/device-plane/.dockerignore index f522c9b..e478899 100644 --- a/device-plane/.dockerignore +++ b/device-plane/.dockerignore @@ -6,5 +6,7 @@ docs node_modules **/test **/*.log +**/*.prev-* +**/*.next-* runtime secrets diff --git a/device-plane/README.md b/device-plane/README.md index a5a48da..26f887c 100644 --- a/device-plane/README.md +++ b/device-plane/README.md @@ -28,6 +28,18 @@ The planned runtime services are: - `device-gateway`: raw TCP sessions, bounded codecs and presence; - `device-postgres`: private persistent state. +`services/device-edge-relay` is a separate, disabled-by-default runtime for the +dedicated Debian ingress host. It is an L4 byte relay: it neither decodes +ARUSNAVI B2, acknowledges a device nor holds a Core token, database credential +or command capability. The implemented backhaul forwards only through a +restricted outbound SSH path to the Synology Gateway loopback listener. The +reviewed ingress target gives only this container a fixed IPvlan L2 address on +the Mini's single Ethernet link, so the host Amnezia full tunnel stays intact +and Docker publishes no host port. The fixed relay address is +`192.168.71.253`, explicitly outside the documented Deco DHCP pool +`192.168.68.50`–`192.168.71.250`. See `docs/IMPLEMENTATION_BASELINE.md` for +the remaining activation gates. + The Foundry `Device Manager` is a canonical page template using a server-owned `device-plane-control` binding. It is not a service in this directory. diff --git a/device-plane/deployment/device-edge-backhaul-v1.json b/device-plane/deployment/device-edge-backhaul-v1.json new file mode 100644 index 0000000..7826b70 --- /dev/null +++ b/device-plane/deployment/device-edge-backhaul-v1.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": "nodedc.device-edge.backhaul.v1", + "mode": "private-tailnet-ssh-local-forward", + "runtimeHost": "ndcmini12", + "selectedServices": [ + "device-edge-backhaul" + ], + "preservedServices": [ + "device-edge-relay", + "tailnet" + ], + "tailnetSocksTarget": "nodedc-device-edge-tailnet-1:1055", + "sshTarget": "100.109.216.21:2222", + "sshUser": "device-backhaul", + "localForward": "0.0.0.0:19921", + "permittedRemoteTarget": "127.0.0.1:9921", + "hostPortPublication": "disabled", + "deviceIngress": "disabled", + "protocolInspection": "disabled", + "commandTransport": "disabled", + "privateKey": "runtime-only-read-only", + "knownHosts": "runner-prepared-exact-ed25519", + "routerNatFirewall": "unchanged", + "gelios": "untouched" +} diff --git a/device-plane/deployment/device-edge-ingress-ipvlan-v1.json b/device-plane/deployment/device-edge-ingress-ipvlan-v1.json new file mode 100644 index 0000000..b453b99 --- /dev/null +++ b/device-plane/deployment/device-edge-ingress-ipvlan-v1.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": "nodedc.device-edge.ingress-ipvlan.v1", + "mode": "single-nic-ipvlan-b2-relay-only", + "runtimeHost": "ndcmini12", + "component": "device-edge", + "selectedServices": [ + "device-edge-relay" + ], + "preservedServices": [ + "device-edge-backhaul", + "tailnet" + ], + "composeProject": "nodedc-device-edge", + "composeFiles": [ + "docker-compose.device-edge.yml", + "docker-compose.device-edge.ingress.yml" + ], + "parentInterface": "enp1s0f0", + "lanSubnet": "192.168.68.0/22", + "lanGateway": "192.168.68.1", + "ingressIpv4": "192.168.71.253", + "ingressIpv4Approval": "approved-outside-dhcp-pool", + "ingressNetwork": "nodedc-device-edge-ingress", + "deviceTcpListen": "192.168.71.253:9921", + "hostPortPublication": "disabled", + "healthPublication": "disabled", + "privateUpstream": "device-edge-backhaul:19921", + "protocolInspection": "gateway-owned", + "identityTrust": "claimed-not-ownership-proof", + "discoveryLifecycle": "quarantine", + "commandTransport": "disabled", + "gelios": "untouched", + "amneziaHostFullTunnel": "preserved", + "routerNatFirewall": "separate-manual-gate", + "rollback": "restore-predecessor-relay-remove-unused-ingress-network" +} diff --git a/device-plane/deployment/device-plane-b2-discovery-loopback-recovery-v1.json b/device-plane/deployment/device-plane-b2-discovery-loopback-recovery-v1.json new file mode 100644 index 0000000..bd74451 --- /dev/null +++ b/device-plane/deployment/device-plane-b2-discovery-loopback-recovery-v1.json @@ -0,0 +1,23 @@ +{ + "schemaVersion": "nodedc.device-plane.b2-discovery-loopback-recovery.v1", + "mode": "failed-b2-loopback-build-reconciliation", + "failedPatchId": "device-plane-b2-discovery-loopback-20260801-003", + "failedArtifactSha256": "7273c5bf67fe6bc1f1da66ad726009240d39ee3aee58201b96c23d6f707a3d84", + "failedBackupId": "device-plane-device-plane-b2-discovery-loopback-20260801-003-20260802-154311", + "sourceAction": "publish-reconciliation-marker-only", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres" + ], + "expectedLoopbackPorts": [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121" + ], + "closedPort": "127.0.0.1:9921/tcp", + "databaseVolume": "nodedc-device-plane-postgres-data", + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "marker-only-runtime-unchanged" +} diff --git a/device-plane/deployment/device-plane-backhaul-target-tailnet-serve-v1.json b/device-plane/deployment/device-plane-backhaul-target-tailnet-serve-v1.json new file mode 100644 index 0000000..9f8943a --- /dev/null +++ b/device-plane/deployment/device-plane-backhaul-target-tailnet-serve-v1.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": "nodedc.device-plane.backhaul-target-tailnet-serve.v1", + "mode": "failed-backhaul-target-to-loopback-tailnet-serve", + "failedPatchId": "device-plane-backhaul-target-20260803-001", + "failedArtifactSha256": "ed0bda4110a756c32be68990e2e0f647409d5a77eec7e26c18502bafbdc1bb76", + "failedBackupId": "device-plane-device-plane-backhaul-target-20260803-001-20260804-035519", + "predecessorPatchId": "device-plane-b2-discovery-loopback-20260803-006", + "predecessorArtifactSha256": "25f9e9e55e283e9b7bb5e128ff14a244f848b1c063acca9724a23206131c9adf", + "sourceAction": "publish-loopback-backhaul-target-source", + "runtimeAction": "build-create-target-and-register-private-tailnet-serve", + "composeOverlay": "docker-compose.device-plane.backhaul-target.yml", + "selectedServices": [ + "device-backhaul-target" + ], + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres" + ], + "loopbackListenAddress": "127.0.0.1", + "listenPort": 2222, + "tailnetAddress": "100.109.216.21", + "tailnetExposure": "tailscale-serve-private", + "tailscaleServeTarget": "tcp://127.0.0.1:2222", + "permittedTarget": "127.0.0.1:9921", + "networkMode": "host", + "dockerPortPublication": "disabled", + "routerNatFirewall": "unchanged", + "edgePublicIngress": "disabled", + "funnel": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "databaseVolume": "nodedc-device-plane-postgres-data", + "runtimeTrust": "runner-managed", + "rollback": "remove-tailnet-serve-target-and-restore-source" +} diff --git a/device-plane/docker-compose.device-edge.backhaul.yml b/device-plane/docker-compose.device-edge.backhaul.yml new file mode 100644 index 0000000..6bee823 --- /dev/null +++ b/device-plane/docker-compose.device-edge.backhaul.yml @@ -0,0 +1,82 @@ +services: + device-edge-backhaul: + build: + context: . + dockerfile: services/device-edge-backhaul/Dockerfile + image: nodedc/device-edge-backhaul:local + pull_policy: never + restart: unless-stopped + user: "1000:1000" + read_only: true + tmpfs: + - /tmp:size=8m,mode=1777 + volumes: + - ../secrets/backhaul/id_ed25519:/run/keys/edge-to-synology:ro + - ../trust/synology-backhaul-known_hosts:/run/trust/known_hosts:ro + command: + - -N + - -T + - -p + - "2222" + - -i + - /run/keys/edge-to-synology + - -L + - 0.0.0.0:19921:127.0.0.1:9921 + - -o + - BatchMode=yes + - -o + - PasswordAuthentication=no + - -o + - KbdInteractiveAuthentication=no + - -o + - PubkeyAuthentication=yes + - -o + - IdentitiesOnly=yes + - -o + - StrictHostKeyChecking=yes + - -o + - UserKnownHostsFile=/run/trust/known_hosts + - -o + - UpdateHostKeys=no + - -o + - ExitOnForwardFailure=yes + - -o + - ServerAliveInterval=15 + - -o + - ServerAliveCountMax=3 + - -o + - TCPKeepAlive=yes + - -o + - LogLevel=VERBOSE + - -o + - ProxyCommand=nc -X 5 -x nodedc-device-edge-tailnet-1:1055 %h %p + - device-backhaul@100.109.216.21 + networks: + - device-edge-private + - device-edge-tailnet + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + pids_limit: 32 + healthcheck: + test: + - CMD + - nc + - -z + - -w + - "3" + - 127.0.0.1 + - "19921" + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + +networks: + device-edge-private: + name: nodedc-device-edge-private + external: true + device-edge-tailnet: + name: nodedc-device-edge-tailnet + external: true diff --git a/device-plane/docker-compose.device-edge.ingress.yml b/device-plane/docker-compose.device-edge.ingress.yml new file mode 100644 index 0000000..39512a7 --- /dev/null +++ b/device-plane/docker-compose.device-edge.ingress.yml @@ -0,0 +1,32 @@ +services: + device-edge-relay: + environment: + DEVICE_EDGE_RELAY_INGRESS_ENABLED: "true" + DEVICE_EDGE_RELAY_TCP_HOST: 0.0.0.0 + DEVICE_EDGE_RELAY_TCP_PORT: "9921" + DEVICE_EDGE_RELAY_UPSTREAM_HOST: device-edge-backhaul + DEVICE_EDGE_RELAY_UPSTREAM_PORT: "19921" + DEVICE_EDGE_RELAY_MAX_SESSIONS: "100" + DEVICE_EDGE_RELAY_MAX_SESSIONS_PER_ADDRESS: "10" + DEVICE_EDGE_RELAY_MAX_CONNECTIONS_PER_MINUTE_PER_ADDRESS: "30" + DEVICE_EDGE_RELAY_SESSION_TIMEOUT_MS: "10000" + networks: + device-edge-private: + gw_priority: 0 + device-edge-ingress: + ipv4_address: 192.168.71.253 + gw_priority: 100 + +networks: + device-edge-private: + name: nodedc-device-edge-private + device-edge-ingress: + name: nodedc-device-edge-ingress + driver: ipvlan + driver_opts: + parent: enp1s0f0 + ipvlan_mode: l2 + ipam: + config: + - subnet: 192.168.68.0/22 + gateway: 192.168.68.1 diff --git a/device-plane/docker-compose.device-edge.yml b/device-plane/docker-compose.device-edge.yml new file mode 100644 index 0000000..a5875cf --- /dev/null +++ b/device-plane/docker-compose.device-edge.yml @@ -0,0 +1,42 @@ +services: + device-edge-relay: + build: + context: . + dockerfile: services/device-edge-relay/Dockerfile + image: nodedc/device-edge-relay:local + pull_policy: never + restart: unless-stopped + user: "1000:1000" + read_only: true + tmpfs: + - /tmp:size=16m,mode=1777 + environment: + DEVICE_EDGE_RELAY_HEALTH_HOST: 127.0.0.1 + DEVICE_EDGE_RELAY_HEALTH_PORT: "18221" + DEVICE_EDGE_RELAY_INGRESS_ENABLED: "false" + DEVICE_EDGE_RELAY_TCP_PORT: "9921" + DEVICE_EDGE_RELAY_MAX_SESSIONS: "100" + DEVICE_EDGE_RELAY_MAX_SESSIONS_PER_ADDRESS: "10" + DEVICE_EDGE_RELAY_MAX_CONNECTIONS_PER_MINUTE_PER_ADDRESS: "30" + DEVICE_EDGE_RELAY_SESSION_TIMEOUT_MS: "10000" + networks: + device-edge-private: {} + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: + - CMD + - node + - -e + - fetch('http://127.0.0.1:18221/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1)) + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + +networks: + device-edge-private: + name: nodedc-device-edge-private + internal: true diff --git a/device-plane/docker-compose.device-plane.backhaul-target.yml b/device-plane/docker-compose.device-plane.backhaul-target.yml new file mode 100644 index 0000000..6f23580 --- /dev/null +++ b/device-plane/docker-compose.device-plane.backhaul-target.yml @@ -0,0 +1,39 @@ +services: + device-backhaul-target: + image: nodedc/device-backhaul-target:local + pull_policy: never + restart: unless-stopped + network_mode: host + read_only: true + tmpfs: + - /run:size=8m,mode=0755 + - /tmp:size=8m,mode=1777 + volumes: + - type: bind + source: /volume1/docker/nodedc-device-plane/secrets/backhaul-target/ssh_host_ed25519_key + target: /run/nodedc-secrets/ssh_host_ed25519_key + read_only: true + bind: + create_host_path: false + - type: bind + source: /volume1/docker/nodedc-device-plane/secrets/backhaul-target/authorized_keys + target: /run/nodedc-secrets/authorized_keys + read_only: true + bind: + create_host_path: false + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID + - SYS_CHROOT + healthcheck: + test: ["CMD", "nc", "-z", "-w", "3", "127.0.0.1", "2222"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s diff --git a/device-plane/docs/IMPLEMENTATION_BASELINE.md b/device-plane/docs/IMPLEMENTATION_BASELINE.md index b619b1f..a08e9ad 100644 --- a/device-plane/docs/IMPLEMENTATION_BASELINE.md +++ b/device-plane/docs/IMPLEMENTATION_BASELINE.md @@ -27,6 +27,26 @@ Device Control Core <-> Device PostgreSQL Device Gateway <-> physical devices ``` +The isolated ingress placement replaces the direct physical-device arrow when +the raw route must not terminate on the multi-service Synology: + +```text +ARUSNAVI B2 device + | + | raw TCP 9921 (future, separately approved) + v +Device Edge Relay on dedicated mini + | + | outbound restricted SSH local-forward; opaque byte stream only + v +Synology loopback 127.0.0.1:9921 -> Device Gateway -> Device Control Core +``` + +The Edge Relay owns neither protocol acknowledgement nor device identity. It +does not receive the Gateway/Core token, PostgreSQL credentials, Foundry +bindings or any command capability. The Synology Gateway remains the sole B2 +codec and acknowledgement owner. + Engine L2 may consume safe decoded observations and build workflows/Data Products. It does not own TCP sessions, secrets or the command transport. @@ -50,7 +70,9 @@ platform/device-plane/ packages/arusnavi-b2-adapter/ services/device-control-core/ services/device-gateway/ + services/device-edge-relay/ docker-compose.device-plane.yml + docker-compose.device-edge.yml ``` Planned Synology runtime: @@ -99,6 +121,41 @@ as `443 -> 9921`. The artifact never changes DSM firewall, DSM Router Configuration, DNS or a physical router. +### Dedicated mini Device Edge + +The Debian mini is the isolated raw-TCP edge. Its accepted predecessor keeps the +relay disabled and publishes health only on `127.0.0.1:18221`. The reviewed +target removes even that host publication: health remains container-internal, +the relay stays on the `internal: true` private bridge for backhaul, and a +second IPvlan L2 attachment gives only the relay a LAN-routable address for +`9921/TCP`. The relay has bounded global/per-address sessions and connection +rate, emits no bytes of its own and does not inspect device payloads. + +IPvlan deliberately reuses the Mini's one physical parent `enp1s0f0`; a second +Ethernet adapter is not required. The host keeps `192.168.68.54/22` and the +Amnezia `0.0.0.0/1` plus `128.0.0.0/1` routes. The relay has its own fixed LAN +IPv4 and default route through `192.168.68.1`, while its private connected route +continues to reach `device-edge-backhaul:19921`. No Docker host `ports:` entry, +host-network mode, privileged container or VPN teardown is allowed. + +Enabling ingress is a separate reviewed operation and requires all of the +following evidence: + +1. A distinct, no-shell Synology SSH account and key whose sole permitted open + target is `127.0.0.1:9921`; host-key pinning and a persistent, monitored + tunnel are required. +2. A private backhaul sidecar/network; the raw listener may forward only to + that tunnel. The Core token and all Core/Database secrets remain on + Synology. +3. Router evidence proving the fixed relay IPv4 is outside DHCP, followed by a + manual router/NAT rule mapping only TCP `9921` to that IPv4. The artifact + cannot choose an address and never changes router, firewall or DHCP state. +4. The host full-tunnel VPN remains active. Before production activation, the + exact single-NIC IPvlan design must pass duplicate-address detection, + gateway reachability, external return-path and private-backhaul checks. +5. One pre-authorized B2 pilot route, quarantine-only Gateway/Core ingest and + disabled command transport. + ## Identity and onboarding An IMEI is a claimed protocol identifier, not proof of tenant ownership. @@ -185,20 +242,21 @@ raw TCP remain forbidden until separate reviewed acceptance slices. absence preflight: an existing database container or volume fails closed, and rollback never removes the volume. -## Next source slice +## Next activation slice -1. Add the versioned B2 discovery-ingress descriptor and runner acceptance - contract without weakening foundation/recovery acceptance. -2. Build a deterministic data-only app-overlay and promote the runner extension - through the standalone root administrative gate. -3. Plan/apply the exact discovery-ingress artifact and independently prove: - Core/Gateway/PostgreSQL health, PostgreSQL generation preservation, - `127.0.0.1:9921` loopback-only publication, verified framing, - quarantine-only Core ingest and disabled command transport. -4. Do not publish raw device TCP from the multi-service Synology. Design and - independently review an isolated edge/DMZ/VPS ingress with no lateral NAS - access, explicit firewall/NAT evidence, acceptance and rollback. -5. Only after that separate edge gate, add the NDC route to one approved B2 - free server slot while preserving Gelios, then prove - HEADER/discovery/PACKAGE acknowledgement. Claim and tenant assignment - remain a later explicit platform-admin operation. +1. The Deco DHCP range has been recorded as `192.168.68.50` through + `192.168.71.250`; the fixed Relay IPv4 is `192.168.71.253`, outside that + pool and independently DAD-tested. It is pinned in Compose, descriptor, + builder and the separate Edge runner. +2. Build the deterministic `component=device-edge` artifact, promote the + root-owned Edge runner and review its `plan`. The Synology runner and inbox + are not used for this host. +3. Apply only `device-edge-relay`; prove exact IPvlan runtime, no host ports, + internal health, private backhaul reachability, unchanged backhaul/tailnet + identities and preserved Amnezia routes. Automatic rollback restores the + predecessor relay and removes the unused ingress network. +4. Independently add the single router/NAT rule for TCP `9921` only and verify + that Synology still exposes no public device port. +5. Add the NDC route to one approved B2 free server slot while preserving + Gelios, then prove HEADER/discovery/PACKAGE acknowledgement. Claim and tenant + assignment remain a later explicit platform-admin operation. diff --git a/device-plane/package-lock.json b/device-plane/package-lock.json index 87b6bd1..9f5d91e 100644 --- a/device-plane/package-lock.json +++ b/device-plane/package-lock.json @@ -23,6 +23,10 @@ "resolved": "services/device-control-core", "link": true }, + "node_modules/@nodedc/device-edge-relay": { + "resolved": "services/device-edge-relay", + "link": true + }, "node_modules/@nodedc/device-gateway": { "resolved": "services/device-gateway", "link": true @@ -201,6 +205,13 @@ "node": ">=20" } }, + "services/device-edge-relay": { + "name": "@nodedc/device-edge-relay", + "version": "0.1.0", + "engines": { + "node": ">=20" + } + }, "services/device-gateway": { "name": "@nodedc/device-gateway", "version": "0.1.0", diff --git a/device-plane/services/device-backhaul-target/Dockerfile b/device-plane/services/device-backhaul-target/Dockerfile new file mode 100644 index 0000000..799a4bc --- /dev/null +++ b/device-plane/services/device-backhaul-target/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine:3.22 + +RUN apk add --no-cache openssh-server \ + && addgroup -g 11021 device-backhaul \ + && adduser -D -H -u 11021 -G device-backhaul -s /sbin/nologin device-backhaul \ + && passwd -u device-backhaul + +COPY services/device-backhaul-target/sshd_config /etc/ssh/sshd_config + +EXPOSE 2222/tcp + +CMD ["sh", "-c", "mkdir -p /run/sshd && exec /usr/sbin/sshd -D -e -f /etc/ssh/sshd_config"] diff --git a/device-plane/services/device-backhaul-target/sshd_config b/device-plane/services/device-backhaul-target/sshd_config new file mode 100644 index 0000000..f3907ed --- /dev/null +++ b/device-plane/services/device-backhaul-target/sshd_config @@ -0,0 +1,35 @@ +Port 2222 +ListenAddress 127.0.0.1 +AddressFamily inet +Protocol 2 + +HostKey /run/nodedc-secrets/ssh_host_ed25519_key +AuthorizedKeysFile /run/nodedc-secrets/authorized_keys +StrictModes no + +PermitRootLogin no +PasswordAuthentication no +KbdInteractiveAuthentication no +PubkeyAuthentication yes +AuthenticationMethods publickey +AllowUsers device-backhaul + +AllowTcpForwarding local +PermitOpen 127.0.0.1:9921 +GatewayPorts no +PermitTunnel no +AllowAgentForwarding no +X11Forwarding no +PermitTTY no +PermitUserEnvironment no +ForceCommand /bin/false + +ClientAliveInterval 30 +ClientAliveCountMax 3 +LoginGraceTime 20 +MaxAuthTries 3 +MaxSessions 4 +MaxStartups 4:30:8 +UseDNS no +PrintMotd no +LogLevel VERBOSE diff --git a/device-plane/services/device-control-core/Dockerfile b/device-plane/services/device-control-core/Dockerfile index 6360bb1..f8f9a78 100644 --- a/device-plane/services/device-control-core/Dockerfile +++ b/device-plane/services/device-control-core/Dockerfile @@ -3,9 +3,11 @@ FROM node:22-alpine WORKDIR /app COPY package.json package-lock.json ./ -COPY packages ./packages +COPY packages/device-protocol-contract ./packages/device-protocol-contract +COPY packages/arusnavi-b2-adapter ./packages/arusnavi-b2-adapter COPY services/device-control-core ./services/device-control-core COPY services/device-gateway/package.json ./services/device-gateway/package.json +COPY services/device-edge-relay/package.json ./services/device-edge-relay/package.json RUN npm ci --omit=dev --ignore-scripts diff --git a/device-plane/services/device-edge-backhaul/Dockerfile b/device-plane/services/device-edge-backhaul/Dockerfile new file mode 100644 index 0000000..6873a46 --- /dev/null +++ b/device-plane/services/device-edge-backhaul/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:3.22 + +RUN apk add --no-cache openssh-client-default netcat-openbsd \ + && addgroup -g 1000 edge-backhaul \ + && adduser -D -H -u 1000 -G edge-backhaul -s /sbin/nologin edge-backhaul + +USER 1000:1000 + +ENTRYPOINT ["/usr/bin/ssh"] diff --git a/device-plane/services/device-edge-backhaul/test/source.test.mjs b/device-plane/services/device-edge-backhaul/test/source.test.mjs new file mode 100644 index 0000000..e4e2583 --- /dev/null +++ b/device-plane/services/device-edge-backhaul/test/source.test.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; + +const here = dirname(fileURLToPath(import.meta.url)); +const devicePlane = resolve(here, "../../.."); + +test("edge backhaul is key-only, pinned and never host-published", async () => { + const compose = await readFile( + resolve(devicePlane, "docker-compose.device-edge.backhaul.yml"), + "utf8", + ); + const descriptor = JSON.parse(await readFile( + resolve(devicePlane, "deployment/device-edge-backhaul-v1.json"), + "utf8", + )); + + assert.match(compose, /StrictHostKeyChecking=yes/); + assert.match(compose, /UserKnownHostsFile=\/run\/trust\/known_hosts/); + assert.match(compose, /PasswordAuthentication=no/); + assert.match(compose, /KbdInteractiveAuthentication=no/); + assert.match(compose, /ExitOnForwardFailure=yes/); + assert.match(compose, /ProxyCommand=nc -X 5 -x nodedc-device-edge-tailnet-1:1055/); + assert.match(compose, /\.\.\/secrets\/backhaul\/id_ed25519/); + assert.doesNotMatch(compose, /\.\.\/keys\/edge-to-synology/); + assert.match(compose, /0\.0\.0\.0:19921:127\.0\.0\.1:9921/); + assert.doesNotMatch(compose, /^\s+ports:/m); + assert.doesNotMatch(compose, /0\.0\.0\.0:9921/); + assert.equal(descriptor.hostPortPublication, "disabled"); + assert.equal(descriptor.deviceIngress, "disabled"); + assert.equal(descriptor.commandTransport, "disabled"); +}); diff --git a/device-plane/services/device-edge-relay/Dockerfile b/device-plane/services/device-edge-relay/Dockerfile new file mode 100644 index 0000000..64ad838 --- /dev/null +++ b/device-plane/services/device-edge-relay/Dockerfile @@ -0,0 +1,9 @@ +FROM node:22-alpine + +WORKDIR /app + +COPY services/device-edge-relay/src ./src + +USER node + +CMD ["node", "src/server.mjs"] diff --git a/device-plane/services/device-edge-relay/package.json b/device-plane/services/device-edge-relay/package.json new file mode 100644 index 0000000..1afe259 --- /dev/null +++ b/device-plane/services/device-edge-relay/package.json @@ -0,0 +1,13 @@ +{ + "name": "@nodedc/device-edge-relay", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/server.mjs", + "test": "node --test test/*.test.mjs" + }, + "engines": { + "node": ">=20" + } +} diff --git a/device-plane/services/device-edge-relay/src/runtime.mjs b/device-plane/services/device-edge-relay/src/runtime.mjs new file mode 100644 index 0000000..bd33dae --- /dev/null +++ b/device-plane/services/device-edge-relay/src/runtime.mjs @@ -0,0 +1,294 @@ +import { createServer as createHttpServer } from "node:http"; +import { connect, createServer as createTcpServer } from "node:net"; + +export function createDeviceEdgeRelayRuntime(options = {}) { + const config = normalizeConfig(options); + const sessions = new Map(); + const sessionsByAddress = new Map(); + const connectionWindows = new Map(); + let totalAccepted = 0; + let totalRejected = 0; + let totalForwarded = 0; + + const tcpServer = createTcpServer({ allowHalfOpen: true }, (socket) => { + const remoteAddress = normalizeRemoteAddress(socket.remoteAddress); + if ( + sessions.size >= config.maxConcurrentSessions + || currentAddressSessions(remoteAddress) >= config.maxSessionsPerAddress + || !consumeConnectionPermit(remoteAddress) + ) { + totalRejected += 1; + socket.destroy(); + return; + } + + const session = { + remoteAddress, + socket, + upstream: null, + closed: false, + forwarded: false, + }; + sessions.set(socket, session); + incrementAddressSessions(remoteAddress); + totalAccepted += 1; + + socket.setNoDelay(true); + socket.setTimeout(config.sessionTimeoutMs); + socket.pause(); + socket.on("timeout", () => rejectSession(session)); + socket.on("close", () => closeSession(session)); + socket.on("error", () => rejectSession(session)); + + const upstream = connect({ + host: config.upstreamHost, + port: config.upstreamPort, + }); + session.upstream = upstream; + upstream.setNoDelay(true); + upstream.setTimeout(config.sessionTimeoutMs); + upstream.on("connect", () => { + if (session.closed) { + upstream.destroy(); + return; + } + session.forwarded = true; + totalForwarded += 1; + socket.pipe(upstream); + upstream.pipe(socket); + socket.resume(); + }); + upstream.on("timeout", () => rejectSession(session)); + upstream.on("error", () => rejectSession(session)); + upstream.on("close", () => closeSession(session)); + }); + + const healthServer = createHttpServer((request, response) => { + response.setHeader("Content-Type", "application/json; charset=utf-8"); + response.setHeader("Cache-Control", "no-store"); + response.setHeader("X-Content-Type-Options", "nosniff"); + if (request.method !== "GET" || request.url !== "/healthz") { + response.statusCode = 404; + response.end('{"ok":false,"error":"device_edge_relay_route_not_found"}\n'); + return; + } + response.statusCode = 200; + response.end(`${JSON.stringify({ + ok: true, + service: "nodedc-device-edge-relay", + ingress: config.ingressEnabled ? "relay-only" : "disabled", + protocolInspection: "disabled", + commandTransport: "disabled", + sessions: { + active: sessions.size, + accepted: totalAccepted, + rejected: totalRejected, + forwarded: totalForwarded, + }, + })}\n`); + }); + + return { + async start() { + await listen(healthServer, config.healthPort, config.healthHost); + if (config.ingressEnabled) { + await listen(tcpServer, config.tcpPort, config.tcpHost); + } + return { + healthAddress: healthServer.address(), + tcpAddress: config.ingressEnabled ? tcpServer.address() : null, + }; + }, + async stop() { + for (const session of sessions.values()) rejectSession(session); + await Promise.all([ + closeServer(healthServer), + config.ingressEnabled ? closeServer(tcpServer) : Promise.resolve(), + ]); + }, + status() { + return { + activeSessions: sessions.size, + totalAccepted, + totalRejected, + totalForwarded, + ingress: config.ingressEnabled ? "relay-only" : "disabled", + protocolInspection: "disabled", + commandTransport: "disabled", + }; + }, + }; + + function currentAddressSessions(remoteAddress) { + return sessionsByAddress.get(remoteAddress) || 0; + } + + function incrementAddressSessions(remoteAddress) { + sessionsByAddress.set( + remoteAddress, + currentAddressSessions(remoteAddress) + 1, + ); + } + + function decrementAddressSessions(remoteAddress) { + const current = currentAddressSessions(remoteAddress); + if (current <= 1) { + sessionsByAddress.delete(remoteAddress); + } else { + sessionsByAddress.set(remoteAddress, current - 1); + } + } + + function consumeConnectionPermit(remoteAddress) { + const nowMs = config.now().getTime(); + const current = connectionWindows.get(remoteAddress); + if (!current || nowMs - current.startedAt >= 60_000) { + connectionWindows.set(remoteAddress, { startedAt: nowMs, count: 1 }); + return true; + } + if (current.count >= config.maxConnectionsPerMinutePerAddress) return false; + current.count += 1; + return true; + } + + function rejectSession(session) { + if (!session.closed) totalRejected += 1; + session.socket.destroy(); + session.upstream?.destroy(); + closeSession(session); + } + + function closeSession(session) { + if (session.closed) return; + session.closed = true; + sessions.delete(session.socket); + decrementAddressSessions(session.remoteAddress); + } +} + +function normalizeConfig(input) { + const ingressEnabled = input.ingressEnabled === true; + return { + ingressEnabled, + healthHost: normalizeHost(input.healthHost, "127.0.0.1"), + healthPort: parseInteger( + input.healthPort, + 18221, + 0, + 65535, + "device_edge_relay_health_port_invalid", + ), + tcpHost: normalizeTcpHost(input.tcpHost, ingressEnabled), + tcpPort: parseInteger( + input.tcpPort, + 9921, + 0, + 65535, + "device_edge_relay_tcp_port_invalid", + ), + upstreamHost: ingressEnabled + ? normalizeUpstreamHost(input.upstreamHost) + : "disabled", + upstreamPort: ingressEnabled + ? parseInteger( + input.upstreamPort, + undefined, + 1, + 65535, + "device_edge_relay_upstream_port_invalid", + ) + : 0, + maxConcurrentSessions: parseInteger( + input.maxConcurrentSessions, + 100, + 1, + 10000, + "device_edge_relay_session_limit_invalid", + ), + maxSessionsPerAddress: parseInteger( + input.maxSessionsPerAddress, + 10, + 1, + 1000, + "device_edge_relay_address_session_limit_invalid", + ), + maxConnectionsPerMinutePerAddress: parseInteger( + input.maxConnectionsPerMinutePerAddress, + 30, + 1, + 10000, + "device_edge_relay_connection_rate_invalid", + ), + sessionTimeoutMs: parseInteger( + input.sessionTimeoutMs, + 10000, + 100, + 60000, + "device_edge_relay_session_timeout_invalid", + ), + now: typeof input.now === "function" ? input.now : () => new Date(), + }; +} + +function normalizeHost(value, fallback) { + const normalized = String(value || fallback).trim(); + if (!["127.0.0.1", "::1", "0.0.0.0", "::"].includes(normalized)) { + throw new TypeError("device_edge_relay_health_host_invalid"); + } + return normalized; +} + +function normalizeTcpHost(value, ingressEnabled) { + const fallback = ingressEnabled ? "0.0.0.0" : "127.0.0.1"; + const normalized = String(value || fallback).trim(); + const allowed = ingressEnabled ? ["0.0.0.0", "::"] : ["127.0.0.1", "::1"]; + if (!allowed.includes(normalized)) { + throw new TypeError( + ingressEnabled + ? "device_edge_relay_public_ingress_host_invalid" + : "device_edge_relay_baseline_loopback_only", + ); + } + return normalized; +} + +function normalizeUpstreamHost(value) { + const normalized = String(value || "").trim(); + if ( + normalized.length === 0 + || normalized.length > 253 + || /[/:\\s]/.test(normalized) + ) { + throw new TypeError("device_edge_relay_upstream_host_invalid"); + } + return normalized; +} + +function normalizeRemoteAddress(value) { + const normalized = String(value || "unknown").trim(); + return normalized.slice(0, 64) || "unknown"; +} + +function parseInteger(value, fallback, minimum, maximum, errorCode) { + const parsed = Number(value ?? fallback); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new TypeError(errorCode); + } + return parsed; +} + +function listen(server, port, host) { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.off("error", reject); + resolve(); + }); + }); +} + +function closeServer(server) { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} diff --git a/device-plane/services/device-edge-relay/src/server.mjs b/device-plane/services/device-edge-relay/src/server.mjs new file mode 100644 index 0000000..3a00a0a --- /dev/null +++ b/device-plane/services/device-edge-relay/src/server.mjs @@ -0,0 +1,76 @@ +import { createDeviceEdgeRelayRuntime } from "./runtime.mjs"; + +const runtime = createDeviceEdgeRelayRuntime({ + ingressEnabled: parseBoolean( + process.env.DEVICE_EDGE_RELAY_INGRESS_ENABLED, + false, + ), + healthHost: process.env.DEVICE_EDGE_RELAY_HEALTH_HOST || "127.0.0.1", + healthPort: parsePort(process.env.DEVICE_EDGE_RELAY_HEALTH_PORT, 18221), + tcpHost: process.env.DEVICE_EDGE_RELAY_TCP_HOST, + tcpPort: parsePort(process.env.DEVICE_EDGE_RELAY_TCP_PORT, 9921), + upstreamHost: process.env.DEVICE_EDGE_RELAY_UPSTREAM_HOST, + upstreamPort: parsePort( + process.env.DEVICE_EDGE_RELAY_UPSTREAM_PORT, + undefined, + ), + maxConcurrentSessions: parsePositiveInt( + process.env.DEVICE_EDGE_RELAY_MAX_SESSIONS, + 100, + ), + maxSessionsPerAddress: parsePositiveInt( + process.env.DEVICE_EDGE_RELAY_MAX_SESSIONS_PER_ADDRESS, + 10, + ), + maxConnectionsPerMinutePerAddress: parsePositiveInt( + process.env.DEVICE_EDGE_RELAY_MAX_CONNECTIONS_PER_MINUTE_PER_ADDRESS, + 30, + ), + sessionTimeoutMs: parsePositiveInt( + process.env.DEVICE_EDGE_RELAY_SESSION_TIMEOUT_MS, + 10000, + ), +}); + +const addresses = await runtime.start(); +console.log(JSON.stringify({ + event: "device_edge_relay_started", + health: addresses.healthAddress, + tcp: addresses.tcpAddress, + ingress: runtime.status().ingress, + protocolInspection: "disabled", + commandTransport: "disabled", +})); + +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); + +async function shutdown() { + await runtime.stop(); + process.exit(0); +} + +function parsePort(value, fallback) { + if (value === undefined && fallback === undefined) return undefined; + const parsed = Number(value ?? fallback); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 65535) { + throw new Error("device_edge_relay_port_invalid"); + } + return parsed; +} + +function parsePositiveInt(value, fallback) { + const parsed = Number(value || fallback); + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new Error("device_edge_relay_positive_integer_invalid"); + } + return parsed; +} + +function parseBoolean(value, fallback) { + if (value === undefined || value === null || value === "") return fallback; + const normalized = String(value).trim().toLowerCase(); + if (["1", "true", "yes", "on"].includes(normalized)) return true; + if (["0", "false", "no", "off"].includes(normalized)) return false; + throw new Error("device_edge_relay_boolean_invalid"); +} diff --git a/device-plane/services/device-edge-relay/test/ingress-source.test.mjs b/device-plane/services/device-edge-relay/test/ingress-source.test.mjs new file mode 100644 index 0000000..e5e68b7 --- /dev/null +++ b/device-plane/services/device-edge-relay/test/ingress-source.test.mjs @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const devicePlaneRoot = resolve( + dirname(fileURLToPath(import.meta.url)), + "../../..", +); + +test("single-NIC ingress source has no host publication and a fixed ipvlan", async () => { + const baseline = await readFile( + resolve(devicePlaneRoot, "docker-compose.device-edge.yml"), + "utf8", + ); + const ingress = await readFile( + resolve(devicePlaneRoot, "docker-compose.device-edge.ingress.yml"), + "utf8", + ); + + assert.match( + baseline, + /DEVICE_EDGE_RELAY_HEALTH_HOST: 127\.0\.0\.1/, + ); + assert.doesNotMatch(baseline, /^\s+ports:/m); + assert.doesNotMatch(baseline, /device-edge-control/); + + for (const required of [ + 'DEVICE_EDGE_RELAY_INGRESS_ENABLED: "true"', + "DEVICE_EDGE_RELAY_UPSTREAM_HOST: device-edge-backhaul", + 'DEVICE_EDGE_RELAY_UPSTREAM_PORT: "19921"', + "name: nodedc-device-edge-ingress", + "driver: ipvlan", + "parent: enp1s0f0", + "ipvlan_mode: l2", + "ipv4_address: 192.168.71.253", + "subnet: 192.168.68.0/22", + "gateway: 192.168.68.1", + "gw_priority: 100", + ]) { + assert.ok(ingress.includes(required), `missing ingress boundary: ${required}`); + } + + for (const forbidden of [ + "ports:", + "network_mode: host", + "privileged: true", + "DEVICE_EDGE_RELAY_COMMAND", + "0.0.0.0:9921:9921", + ]) { + assert.ok( + !ingress.includes(forbidden), + `forbidden ingress boundary: ${forbidden}`, + ); + } +}); + +test("ingress descriptor keeps address approval and router exposure gated", async () => { + const descriptor = JSON.parse(await readFile( + resolve( + devicePlaneRoot, + "deployment/device-edge-ingress-ipvlan-v1.json", + ), + "utf8", + )); + + assert.equal(descriptor.component, "device-edge"); + assert.deepEqual(descriptor.selectedServices, ["device-edge-relay"]); + assert.deepEqual( + descriptor.preservedServices, + ["device-edge-backhaul", "tailnet"], + ); + assert.equal(descriptor.ingressIpv4Approval, "approved-outside-dhcp-pool"); + assert.equal(descriptor.hostPortPublication, "disabled"); + assert.equal(descriptor.healthPublication, "disabled"); + assert.equal(descriptor.privateUpstream, "device-edge-backhaul:19921"); + assert.equal(descriptor.protocolInspection, "gateway-owned"); + assert.equal(descriptor.discoveryLifecycle, "quarantine"); + assert.equal(descriptor.commandTransport, "disabled"); + assert.equal(descriptor.gelios, "untouched"); + assert.equal(descriptor.amneziaHostFullTunnel, "preserved"); + assert.equal(descriptor.routerNatFirewall, "separate-manual-gate"); +}); diff --git a/device-plane/services/device-edge-relay/test/runtime.test.mjs b/device-plane/services/device-edge-relay/test/runtime.test.mjs new file mode 100644 index 0000000..0b36acc --- /dev/null +++ b/device-plane/services/device-edge-relay/test/runtime.test.mjs @@ -0,0 +1,95 @@ +import assert from "node:assert/strict"; +import { createServer, connect } from "node:net"; +import test from "node:test"; + +import { createDeviceEdgeRelayRuntime } from "../src/runtime.mjs"; + +test("baseline starts only loopback health and no device TCP listener", async () => { + const runtime = createDeviceEdgeRelayRuntime({ healthPort: 0 }); + const addresses = await runtime.start(); + try { + assert.equal(addresses.tcpAddress, null); + const response = await fetch( + `http://127.0.0.1:${addresses.healthAddress.port}/healthz`, + ); + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.ingress, "disabled"); + assert.equal(body.protocolInspection, "disabled"); + assert.equal(body.commandTransport, "disabled"); + } finally { + await runtime.stop(); + } +}); + +test("relay is transparent and never emits its own protocol bytes", async () => { + const upstream = await startEchoServer(); + const runtime = createDeviceEdgeRelayRuntime({ + healthPort: 0, + ingressEnabled: true, + tcpHost: "0.0.0.0", + tcpPort: 0, + upstreamHost: "127.0.0.1", + upstreamPort: upstream.port, + }); + const addresses = await runtime.start(); + try { + const response = await sendAndCollect( + addresses.tcpAddress.port, + Buffer.from("ff230102030405060708", "hex"), + ); + assert.equal(response.toString("hex"), "ff230102030405060708"); + assert.equal(runtime.status().totalForwarded, 1); + assert.equal(runtime.status().commandTransport, "disabled"); + } finally { + await runtime.stop(); + await closeServer(upstream.server); + } +}); + +test("enabled relay requires a concrete private upstream", () => { + assert.throws( + () => createDeviceEdgeRelayRuntime({ + ingressEnabled: true, + tcpHost: "0.0.0.0", + upstreamPort: 19921, + }), + /device_edge_relay_upstream_host_invalid/, + ); +}); + +test("baseline rejects a non-loopback device binding", () => { + assert.throws( + () => createDeviceEdgeRelayRuntime({ tcpHost: "0.0.0.0" }), + /device_edge_relay_baseline_loopback_only/, + ); +}); + +function startEchoServer() { + const server = createServer((socket) => socket.pipe(socket)); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve({ server, port: server.address().port }); + }); + }); +} + +function sendAndCollect(port, payload) { + return new Promise((resolve, reject) => { + const chunks = []; + const socket = connect({ host: "127.0.0.1", port }, () => { + socket.end(payload); + }); + socket.on("data", (chunk) => chunks.push(chunk)); + socket.on("close", () => resolve(Buffer.concat(chunks))); + socket.on("error", reject); + }); +} + +function closeServer(server) { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} diff --git a/device-plane/services/device-gateway/Dockerfile b/device-plane/services/device-gateway/Dockerfile index 80a15ab..3b4cdac 100644 --- a/device-plane/services/device-gateway/Dockerfile +++ b/device-plane/services/device-gateway/Dockerfile @@ -3,9 +3,11 @@ FROM node:22-alpine WORKDIR /app COPY package.json package-lock.json ./ -COPY packages ./packages +COPY packages/device-protocol-contract ./packages/device-protocol-contract +COPY packages/arusnavi-b2-adapter ./packages/arusnavi-b2-adapter COPY services/device-gateway ./services/device-gateway COPY services/device-control-core/package.json ./services/device-control-core/package.json +COPY services/device-edge-relay/package.json ./services/device-edge-relay/package.json RUN npm ci --omit=dev --ignore-scripts diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index a3aa99c..7ccf280 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -20,6 +20,56 @@ The runner accepts data-only app-overlay artifacts from: /volume1/docker/nodedc-deploy/inbox ``` +## Dedicated Device Edge runner + +The Debian Device Edge is a separate root-owned deployment domain. It does not +use the Synology runner, inbox, state or backup tree. Its live runner and fixed +roots are: + +```text +/usr/local/sbin/nodedc-edge-deploy +/home/ndcsudo/nodedc-device-edge/deploy/inbox +/home/ndcsudo/nodedc-device-edge/source +/var/lib/nodedc-edge-deploy +``` + +`nodedc-edge-deploy` accepts only `component=device-edge`, validates an exact +five-entry payload and can build/recreate only `device-edge-relay`. The existing +`device-edge-backhaul` and `tailnet` containers are identity-snapshotted before +the transition and must remain byte-for-byte runtime-equivalent through apply +or automatic rollback. It has no registry entry in the Synology runner. + +The first ingress transition uses a Docker IPvlan L2 address on the Mini's +single Ethernet parent. It publishes no Docker host port and preserves the +Amnezia host full tunnel; only the relay container receives a LAN-routable +address. The fixed IPv4 is a runner/Compose/descriptor constant: +`192.168.71.253`. Router evidence on 2026-08-04 proves the Deco DHCP pool is +`192.168.68.50` through `192.168.71.250`, so the address is explicitly outside +the pool. Router port-forwarding/firewall remains a separate manual gate. + +Build and test the transition source: + +```bash +PYTHONDONTWRITEBYTECODE=1 \ + python3 infra/deploy-runner/test_device_edge_ingress_artifact.py +npm test --prefix device-plane +``` + +After the fixed address is approved, build the production artifact with a fresh +transition id, stage it into the Edge inbox, then use only the canonical pair: + +```bash +sudo /usr/local/sbin/nodedc-edge-deploy plan \ + /home/ndcsudo/nodedc-device-edge/deploy/inbox/.tgz +sudo /usr/local/sbin/nodedc-edge-deploy apply \ + /home/ndcsudo/nodedc-device-edge/deploy/inbox/.tgz +``` + +The apply acceptance checks the exact IPvlan parent/subnet/gateway/address, +absence of host port publication, internal relay health, private backhaul +reachability, preserved VPN routes and unchanged backhaul/tailnet container +identity. Gelios and Device Plane command transport are outside this domain. + Supported components in this source: - `engine` diff --git a/infra/deploy-runner/build-device-edge-ingress-artifact.mjs b/infra/deploy-runner/build-device-edge-ingress-artifact.mjs new file mode 100644 index 0000000..34da0ae --- /dev/null +++ b/infra/deploy-runner/build-device-edge-ingress-artifact.mjs @@ -0,0 +1,256 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-edge-ingress-ipvlan-20260804-001", + ...extra +] = process.argv.slice(2); + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-edge-ingress-artifact.mjs [patch-id]", + ); +} + +const files = [ + "docker-compose.device-edge.yml", + "docker-compose.device-edge.ingress.yml", + "services/device-edge-relay/Dockerfile", + "services/device-edge-relay/src", + "deployment/device-edge-ingress-ipvlan-v1.json", +]; +const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); +const descriptor = await assertBoundary(); +if (descriptor.ingressIpv4Approval !== "approved-outside-dhcp-pool") { + throw new Error("device_edge_ingress_ipv4_approval_pending"); +} + +const stage = await mkdtemp(join(tmpdir(), "nodedc-device-edge-ingress-")); +const payload = join(stage, "payload"); +const target = join(artifactDir, `nodedc-device-edge-${patchId}.tgz`); + +try { + await mkdir(payload, { recursive: true }); + for (const sourceRelative of files) { + await copySafe( + resolve(sourceRoot, sourceRelative), + join(payload, sourceRelative), + ); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-edge\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 128 * 1024 * 1024 }, + ); + if (tar.status !== 0) { + throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + } + + const digest = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact: target, + sha256: digest, + component: "device-edge", + transition: "single-nic-ipvlan-b2-relay-only", + entries: files, + services: ["device-edge-relay"], + preservedRuntime: ["device-edge-backhaul", "tailnet", "Gelios"], + ingress: { + parent: descriptor.parentInterface, + subnet: descriptor.lanSubnet, + gateway: descriptor.lanGateway, + ipv4: descriptor.ingressIpv4, + ipv4Approval: descriptor.ingressIpv4Approval, + tcp: 9921, + hostPortPublication: "disabled", + lifecycle: "quarantine", + commandTransport: "disabled", + }, + rollback: descriptor.rollback, + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertBoundary() { + const baseline = await readFile( + resolve(sourceRoot, "docker-compose.device-edge.yml"), + "utf8", + ); + const ingress = await readFile( + resolve(sourceRoot, "docker-compose.device-edge.ingress.yml"), + "utf8", + ); + const descriptor = JSON.parse(await readFile( + resolve( + sourceRoot, + "deployment/device-edge-ingress-ipvlan-v1.json", + ), + "utf8", + )); + + for (const fragment of [ + "DEVICE_EDGE_RELAY_HEALTH_HOST: 127.0.0.1", + 'DEVICE_EDGE_RELAY_INGRESS_ENABLED: "false"', + "read_only: true", + 'user: "1000:1000"', + "no-new-privileges:true", + "cap_drop:", + "- ALL", + ]) { + if (!baseline.includes(fragment)) { + throw new Error(`device_edge_baseline_boundary_missing:${fragment}`); + } + } + for (const forbidden of ["ports:", "device-edge-control"]){ + if (baseline.includes(forbidden)) { + throw new Error(`device_edge_baseline_boundary_violation:${forbidden}`); + } + } + + for (const fragment of [ + 'DEVICE_EDGE_RELAY_INGRESS_ENABLED: "true"', + "DEVICE_EDGE_RELAY_UPSTREAM_HOST: device-edge-backhaul", + 'DEVICE_EDGE_RELAY_UPSTREAM_PORT: "19921"', + "name: nodedc-device-edge-ingress", + "driver: ipvlan", + "parent: enp1s0f0", + "ipvlan_mode: l2", + "ipv4_address: 192.168.71.253", + "gw_priority: 100", + "subnet: 192.168.68.0/22", + "gateway: 192.168.68.1", + ]) { + if (!ingress.includes(fragment)) { + throw new Error(`device_edge_ingress_boundary_missing:${fragment}`); + } + } + for (const forbidden of [ + "ports:", + "network_mode: host", + "privileged: true", + "DEVICE_EDGE_RELAY_COMMAND", + "0.0.0.0:9921:9921", + ]) { + if (ingress.includes(forbidden)) { + throw new Error(`device_edge_ingress_boundary_violation:${forbidden}`); + } + } + + const expected = { + schemaVersion: "nodedc.device-edge.ingress-ipvlan.v1", + mode: "single-nic-ipvlan-b2-relay-only", + runtimeHost: "ndcmini12", + component: "device-edge", + selectedServices: ["device-edge-relay"], + preservedServices: ["device-edge-backhaul", "tailnet"], + composeProject: "nodedc-device-edge", + composeFiles: [ + "docker-compose.device-edge.yml", + "docker-compose.device-edge.ingress.yml", + ], + parentInterface: "enp1s0f0", + lanSubnet: "192.168.68.0/22", + lanGateway: "192.168.68.1", + ingressIpv4: "192.168.71.253", + ingressIpv4Approval: "approved-outside-dhcp-pool", + ingressNetwork: "nodedc-device-edge-ingress", + deviceTcpListen: "192.168.71.253:9921", + hostPortPublication: "disabled", + healthPublication: "disabled", + privateUpstream: "device-edge-backhaul:19921", + protocolInspection: "gateway-owned", + identityTrust: "claimed-not-ownership-proof", + discoveryLifecycle: "quarantine", + commandTransport: "disabled", + gelios: "untouched", + amneziaHostFullTunnel: "preserved", + routerNatFirewall: "separate-manual-gate", + rollback: "restore-predecessor-relay-remove-unused-ingress-network", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_edge_ingress_descriptor_mismatch"); + } + return descriptor; +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) { + throw new Error(`source_symlink_rejected:${relative(sourceRoot, source)}`); + } + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + return; + } + if (!sourceStat.isDirectory()) { + throw new Error(`source_type_rejected:${source}`); + } + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if ( + ignoredBasenames.has(entry.name) + || entry.name.startsWith(".env") + ) { + continue; + } + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(sourceRoot, childSource)}`, + ); + } + await copySafe(childSource, childDestination); + } +} diff --git a/infra/deploy-runner/build-device-plane-b2-discovery-ingress-artifact.mjs b/infra/deploy-runner/build-device-plane-b2-discovery-ingress-artifact.mjs index 79348e9..129917d 100644 --- a/infra/deploy-runner/build-device-plane-b2-discovery-ingress-artifact.mjs +++ b/infra/deploy-runner/build-device-plane-b2-discovery-ingress-artifact.mjs @@ -43,6 +43,7 @@ const files = [ "packages/arusnavi-b2-adapter", "services/device-control-core", "services/device-gateway", + "services/device-edge-relay/package.json", "deployment/device-plane-b2-discovery-ingress-v1.json", ]; const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); diff --git a/infra/deploy-runner/build-device-plane-b2-discovery-loopback-recovery-artifact.mjs b/infra/deploy-runner/build-device-plane-b2-discovery-loopback-recovery-artifact.mjs new file mode 100644 index 0000000..dfe1102 --- /dev/null +++ b/infra/deploy-runner/build-device-plane-b2-discovery-loopback-recovery-artifact.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const descriptorRelative = + "deployment/device-plane-b2-discovery-loopback-recovery-v1.json"; +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-plane-b2-discovery-loopback-recovery-20260802-004", + ...extra +] = process.argv.slice(2); + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: " + + "build-device-plane-b2-discovery-loopback-recovery-artifact.mjs " + + "[patch-id]", + ); +} + +const files = [descriptorRelative]; +const stage = await mkdtemp( + join(tmpdir(), "nodedc-device-plane-b2-loopback-recovery-"), +); +const payload = join(stage, "payload"); +const target = join( + artifactDir, + `nodedc-device-plane-${patchId}.tgz`, +); + +await assertRecoveryDescriptor(); + +try { + const source = resolve(sourceRoot, descriptorRelative); + const sourceStat = await lstat(source); + if (!sourceStat.isFile() || sourceStat.isSymbolicLink()) { + throw new Error("device_plane_b2_recovery_descriptor_unsafe"); + } + await mkdir(dirname(join(payload, descriptorRelative)), { + recursive: true, + }); + await cp(source, join(payload, descriptorRelative), { + force: true, + verbatimSymlinks: true, + }); + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile( + join(stage, "files.txt"), + `${files.join("\n")}\n`, + "utf8", + ); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + }, + ); + if (tar.status !== 0) { + throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + } + + const digest = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact: target, + sha256: digest, + component: "device-plane", + transition: "failed-b2-loopback-build-reconciliation", + entries: files, + build: [], + services: [], + preservedRuntime: [ + "device-control-core", + "device-gateway", + "device-postgres", + "nodedc-device-plane-postgres-data", + "Gelios", + ], + sourceAction: "publish-reconciliation-marker-only", + runtimeAction: "read-only-acceptance", + ingress: "disabled:127.0.0.1:9921/tcp:closed", + rollback: "marker-only-runtime-unchanged", + excluded: [ + "application-source", + "compose", + "Dockerfile", + "secrets", + "runtime", + "database", + "Gelios", + ], + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertRecoveryDescriptor() { + const descriptor = JSON.parse(await readFile( + resolve(sourceRoot, descriptorRelative), + "utf8", + )); + const expected = { + schemaVersion: "nodedc.device-plane.b2-discovery-loopback-recovery.v1", + mode: "failed-b2-loopback-build-reconciliation", + failedPatchId: "device-plane-b2-discovery-loopback-20260801-003", + failedArtifactSha256: + "7273c5bf67fe6bc1f1da66ad726009240d39ee3aee58201b96c23d6f707a3d84", + failedBackupId: + "device-plane-device-plane-b2-discovery-loopback-20260801-003-20260802-154311", + sourceAction: "publish-reconciliation-marker-only", + runtimeAction: "read-only-acceptance", + preservedServices: [ + "device-control-core", + "device-gateway", + "device-postgres", + ], + expectedLoopbackPorts: [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121", + ], + closedPort: "127.0.0.1:9921/tcp", + databaseVolume: "nodedc-device-plane-postgres-data", + commandTransport: "disabled", + gelios: "untouched", + rollback: "marker-only-runtime-unchanged", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_plane_b2_recovery_descriptor_mismatch"); + } +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} diff --git a/infra/deploy-runner/build-device-plane-backhaul-target-artifact.mjs b/infra/deploy-runner/build-device-plane-backhaul-target-artifact.mjs new file mode 100644 index 0000000..1bf7cb2 --- /dev/null +++ b/infra/deploy-runner/build-device-plane-backhaul-target-artifact.mjs @@ -0,0 +1,256 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-plane-backhaul-target-tailnet-serve-20260804-002", + ...extra +] = process.argv.slice(2); + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-plane-backhaul-target-artifact.mjs [patch-id]", + ); +} + +const files = [ + "docker-compose.device-plane.backhaul-target.yml", + "services/device-backhaul-target", + "deployment/device-plane-backhaul-target-tailnet-serve-v1.json", +]; +const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); +const stage = await mkdtemp( + join(tmpdir(), "nodedc-device-plane-backhaul-target-"), +); +const payload = join(stage, "payload"); +const target = join( + artifactDir, + `nodedc-device-plane-${patchId}.tgz`, +); + +await assertBoundary(); + +try { + await mkdir(payload, { recursive: true }); + for (const sourceRelative of files) { + await copySafe( + resolve(sourceRoot, sourceRelative), + join(payload, sourceRelative), + ); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 128 * 1024 * 1024 }, + ); + if (tar.status !== 0) { + throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + } + + const digest = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact: target, + sha256: digest, + component: "device-plane", + transition: "failed-backhaul-target-to-loopback-tailnet-serve", + entries: files, + services: ["device-backhaul-target"], + preservedRuntime: [ + "device-control-core", + "device-gateway", + "device-postgres", + "nodedc-device-plane-postgres-data", + "Gelios", + ], + ingress: { + loopbackListen: "127.0.0.1:2222/tcp", + tailnetListen: "100.109.216.21:2222/tcp", + transport: "tailscale-serve-private-ssh", + serveTarget: "tcp://127.0.0.1:2222", + permittedTarget: "127.0.0.1:9921", + dockerPortPublication: "disabled", + routerNatFirewall: "unchanged", + edgePublicIngress: "disabled", + funnel: "disabled", + commandTransport: "disabled", + }, + runtimeTrust: "runner-managed-not-in-artifact", + rollback: "remove-tailnet-serve-target-and-restore-source", + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertBoundary() { + const compose = await readFile( + resolve(sourceRoot, "docker-compose.device-plane.backhaul-target.yml"), + "utf8", + ); + for (const fragment of [ + "device-backhaul-target:", + "image: nodedc/device-backhaul-target:local", + "network_mode: host", + '"127.0.0.1", "2222"', + "/secrets/backhaul-target/ssh_host_ed25519_key", + "/secrets/backhaul-target/authorized_keys", + "no-new-privileges:true", + ]) { + if (!compose.includes(fragment)) { + throw new Error(`device_plane_backhaul_boundary_missing:${fragment}`); + } + } + for (const forbidden of [ + "PasswordAuthentication yes", + "0.0.0.0:2222", + "9921:9921/udp", + "DEVICE_GATEWAY_COMMAND", + ]) { + if (compose.includes(forbidden)) { + throw new Error(`device_plane_backhaul_boundary_violation:${forbidden}`); + } + } + + const sshd = await readFile( + resolve(sourceRoot, "services/device-backhaul-target/sshd_config"), + "utf8", + ); + for (const fragment of [ + "ListenAddress 127.0.0.1", + "PasswordAuthentication no", + "KbdInteractiveAuthentication no", + "AllowTcpForwarding local", + "PermitOpen 127.0.0.1:9921", + "GatewayPorts no", + "PermitTunnel no", + "AllowAgentForwarding no", + "PermitTTY no", + "ForceCommand /bin/false", + ]) { + if (!sshd.includes(fragment)) { + throw new Error(`device_plane_backhaul_sshd_boundary_missing:${fragment}`); + } + } + + const descriptor = JSON.parse(await readFile( + resolve( + sourceRoot, + "deployment/device-plane-backhaul-target-tailnet-serve-v1.json", + ), + "utf8", + )); + const expected = { + schemaVersion: "nodedc.device-plane.backhaul-target-tailnet-serve.v1", + mode: "failed-backhaul-target-to-loopback-tailnet-serve", + failedPatchId: "device-plane-backhaul-target-20260803-001", + failedArtifactSha256: + "ed0bda4110a756c32be68990e2e0f647409d5a77eec7e26c18502bafbdc1bb76", + failedBackupId: + "device-plane-device-plane-backhaul-target-20260803-001-20260804-035519", + predecessorPatchId: + "device-plane-b2-discovery-loopback-20260803-006", + predecessorArtifactSha256: + "25f9e9e55e283e9b7bb5e128ff14a244f848b1c063acca9724a23206131c9adf", + sourceAction: "publish-loopback-backhaul-target-source", + runtimeAction: "build-create-target-and-register-private-tailnet-serve", + composeOverlay: "docker-compose.device-plane.backhaul-target.yml", + selectedServices: ["device-backhaul-target"], + preservedServices: [ + "device-control-core", + "device-gateway", + "device-postgres", + ], + loopbackListenAddress: "127.0.0.1", + listenPort: 2222, + tailnetAddress: "100.109.216.21", + tailnetExposure: "tailscale-serve-private", + tailscaleServeTarget: "tcp://127.0.0.1:2222", + permittedTarget: "127.0.0.1:9921", + networkMode: "host", + dockerPortPublication: "disabled", + routerNatFirewall: "unchanged", + edgePublicIngress: "disabled", + funnel: "disabled", + commandTransport: "disabled", + gelios: "untouched", + databaseVolume: "nodedc-device-plane-postgres-data", + runtimeTrust: "runner-managed", + rollback: "remove-tailnet-serve-target-and-restore-source", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_plane_backhaul_descriptor_mismatch"); + } +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) { + throw new Error(`source_symlink_rejected:${relative(sourceRoot, source)}`); + } + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + return; + } + if (!sourceStat.isDirectory()) { + throw new Error(`source_type_rejected:${source}`); + } + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if ( + ignoredBasenames.has(entry.name) + || entry.name.startsWith(".env") + || entry.name.endsWith("~") + ) { + continue; + } + await copySafe(join(source, entry.name), join(destination, entry.name)); + } +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index a59501a..df125a9 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -1,9 +1,11 @@ #!/usr/bin/env python3 import argparse import base64 +import grp import hashlib import json import os +import pwd import re import secrets import shutil @@ -45,6 +47,102 @@ DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE = DEVICE_PLANE_SECRET_DIR / "gateway-core-t DEVICE_PLANE_IDENTIFIER_PEPPER_FILE = DEVICE_PLANE_SECRET_DIR / "identifier-pepper" DEVICE_PLANE_CONTROL_CORE_IMAGE = "nodedc/device-control-core:local" DEVICE_PLANE_GATEWAY_IMAGE = "nodedc/device-gateway:local" +DEVICE_PLANE_BACKHAUL_TARGET_IMAGE = "nodedc/device-backhaul-target:local" +DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL = ( + "deployment/device-plane-backhaul-target-v1.json" +) +DEVICE_PLANE_BACKHAUL_TARGET_REL = ( + "deployment/device-plane-backhaul-target-tailnet-serve-v1.json" +) +DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL = ( + "docker-compose.device-plane.backhaul-target.yml" +) +DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES = ( + DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "services/device-backhaul-target", + DEVICE_PLANE_BACKHAUL_TARGET_REL, +) +DEVICE_PLANE_BACKHAUL_FAILED_TARGET_ENTRIES = ( + DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "services/device-backhaul-target", + DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL, +) +DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_PATCH_ID = ( + "device-plane-b2-discovery-loopback-20260803-006" +) +DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_ARTIFACT_SHA256 = ( + "25f9e9e55e283e9b7bb5e128ff14a244f848b1c063acca9724a23206131c9adf" +) +DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_SHA256 = ( + "e10807cb3bab4c1d7ae2ac8ce4af1936717036930bb0b1c6f8f2b89c3f82b7c0" +) +DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID = ( + "device-plane-backhaul-target-20260803-001" +) +DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256 = ( + "ed0bda4110a756c32be68990e2e0f647409d5a77eec7e26c18502bafbdc1bb76" +) +DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-plane-backhaul-target-20260803-001.tgz." + "20260804-035519" +) +DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID = ( + "device-plane-device-plane-backhaul-target-20260803-001-" + "20260804-035519" +) +DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_SHA256 = { + "manifest.env": ( + "7b26cf56c52723d9ce559d9f9216757038e4d3aa812821b5f05046bfcf8c01eb" + ), + "files.txt": ( + "761b9673fddb27558873a72f182d50ca46b2fa6808cbbeb03b40dba197a7586a" + ), + "source-before.tgz": ( + "b008e5d971527a7e4fccda314d677b6a6583e1a0a5e293cdc937fa2b8f19e10e" + ), + "existing-files.txt": ( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ), + "missing-files.txt": ( + "761b9673fddb27558873a72f182d50ca46b2fa6808cbbeb03b40dba197a7586a" + ), + "runtime-before.json": ( + "d45c54ebb286af8850b478b07410dd4ece1135aa9717cd8f0d4aa9c94c56e3d7" + ), +} +DEVICE_PLANE_BACKHAUL_FAILED_MESSAGE = ( + "container healthcheck failed for " + "2b6b1b324368421acee9b0945b18371bc500ece4ce99b2ff12da7da4ab157e49: " + "unhealthy" +) +DEVICE_PLANE_BACKHAUL_LOOPBACK_ADDRESS = "127.0.0.1" +DEVICE_PLANE_BACKHAUL_LISTEN_PORT = 2222 +DEVICE_PLANE_BACKHAUL_TAILNET_ADDRESS = "100.109.216.21" +DEVICE_PLANE_BACKHAUL_TAILSCALE_SERVE_TARGET = "tcp://127.0.0.1:2222" +DEVICE_PLANE_TAILSCALE = Path( + "/var/packages/Tailscale/target/bin/tailscale" +) +DEVICE_PLANE_TAILSCALE_PRIVILEGE = Path( + "/var/packages/Tailscale/conf/privilege" +) +DEVICE_PLANE_TAILSCALE_USER = "tailscale" +DEVICE_PLANE_TAILSCALE_GROUP = "tailscale" +DEVICE_PLANE_BACKHAUL_PERMITTED_TARGET = "127.0.0.1:9921" +DEVICE_PLANE_BACKHAUL_ENROLLMENT_DIR = DEVICE_PLANE_ROOT / "enrollment" +DEVICE_PLANE_BACKHAUL_ENROLLMENT_PUBLIC_KEY_FILE = ( + DEVICE_PLANE_BACKHAUL_ENROLLMENT_DIR / "device-edge-backhaul.pub" +) +DEVICE_PLANE_BACKHAUL_SECRET_DIR = DEVICE_PLANE_SECRET_DIR / "backhaul-target" +DEVICE_PLANE_BACKHAUL_HOST_KEY_FILE = ( + DEVICE_PLANE_BACKHAUL_SECRET_DIR / "ssh_host_ed25519_key" +) +DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE = ( + DEVICE_PLANE_BACKHAUL_SECRET_DIR / "authorized_keys" +) +DEVICE_PLANE_BACKHAUL_TRUST_DIR = DEVICE_PLANE_ROOT / "trust" / "backhaul-target" +DEVICE_PLANE_BACKHAUL_HOST_PUBLIC_KEY_FILE = ( + DEVICE_PLANE_BACKHAUL_TRUST_DIR / "ssh_host_ed25519_key.pub" +) DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL = ( "deployment/device-postgres-bootstrap-v1.json" ) @@ -82,6 +180,7 @@ DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL = ( ) DEVICE_PLANE_B2_DISCOVERY_INGRESS_ENTRIES = ( *DEVICE_PLANE_FOUNDATION_ENTRIES, + "services/device-edge-relay/package.json", DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL, ) DEVICE_PLANE_B2_DISCOVERY_INGRESS_PREDECESSOR_PATCH_ID = ( @@ -93,6 +192,56 @@ DEVICE_PLANE_B2_DISCOVERY_INGRESS_PREDECESSOR_ARTIFACT_SHA256 = ( DEVICE_PLANE_B2_DISCOVERY_INGRESS_COMPOSE_SHA256 = ( "eb1018cc0ffeaa0c019944d8810e2daa01eeca4c06289efff175785fb16457e7" ) +DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL = ( + "deployment/device-plane-b2-discovery-loopback-recovery-v1.json" +) +DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ENTRIES = ( + DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, +) +DEVICE_PLANE_B2_DISCOVERY_FAILED_ENTRIES = ( + *DEVICE_PLANE_FOUNDATION_ENTRIES, + DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL, +) +DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID = ( + "device-plane-b2-discovery-loopback-20260801-003" +) +DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-plane-b2-discovery-loopback-" + "20260801-003.tgz.20260802-154311" +) +DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT_SHA256 = ( + "7273c5bf67fe6bc1f1da66ad726009240d39ee3aee58201b96c23d6f707a3d84" +) +DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_ID = ( + "device-plane-device-plane-b2-discovery-loopback-20260801-003-" + "20260802-154311" +) +DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_SHA256 = { + "manifest.env": ( + "2d93219283cb2f7ca65a8349c84a1d6ff0568bea33d564a9acb5286df0061c93" + ), + "files.txt": ( + "596a364ea56b92789b4ba9f9e6d2628e5a7a56a1380abfc3124c08e08ff9d6ae" + ), + "existing-files.txt": ( + "3b7dda5cc42ce460e4a74f6df986d4a4fa8aa28511e9d9b123deb445a0382083" + ), + "missing-files.txt": ( + "fc252f30b3ea51aafcf8002d64de077b33dcfd20e77fa05cb88457876780b620" + ), + "runtime-before.json": ( + "4171d00e5c8a8db9d26042241ff50c70a0aa96cfdbdd2eeb596f0e860349e6a2" + ), + "source-before.tgz": ( + "1cbe3b31be02ce733afe17f3dd391708787f0b3ffcea038d0619316e831f7e34" + ), +} +DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_PATCH_ID = ( + "device-plane-b2-discovery-loopback-recovery-20260802-004" +) +DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ARTIFACT_SHA256 = ( + "0888a6ff3404772179d74b46919fd3f193adacedc40944d20349a37d262974aa" +) DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = ( "device-plane-foundation-20260725-001" ) @@ -195,6 +344,7 @@ DEVICE_PLANE_RUNTIME_SERVICES = ( "device-gateway", "device-postgres", ) +DEVICE_PLANE_BACKHAUL_TARGET_SERVICE = "device-backhaul-target" EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner" EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token" ENGINE_CREDENTIAL_PROVISIONER_PRIVATE_KEY_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "engine-credential-provisioner-ed25519.pem" @@ -2470,8 +2620,10 @@ def denied_payload_path(component, rel): pass elif component == "device-plane" and rel in ( "docker-compose.device-plane.yml", + DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, "services/device-control-core/Dockerfile", "services/device-gateway/Dockerfile", + "services/device-backhaul-target/Dockerfile", ): pass elif component == "proxy-contur" and rel in ( @@ -2878,14 +3030,19 @@ def allowed_payload_path(component, rel): "package.json", "package-lock.json", "docker-compose.device-plane.yml", + DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, DEVICE_PLANE_FOUNDATION_RECOVERY_REL, DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL, DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL, + DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, + DEVICE_PLANE_BACKHAUL_TARGET_REL, "packages/device-protocol-contract", "packages/arusnavi-b2-adapter", "services/device-control-core", "services/device-gateway", + "services/device-edge-relay/package.json", + "services/device-backhaul-target", ): return True if rel.startswith(( @@ -2893,6 +3050,7 @@ def allowed_payload_path(component, rel): "packages/arusnavi-b2-adapter/", "services/device-control-core/", "services/device-gateway/", + "services/device-backhaul-target/", )): return True @@ -7923,6 +8081,18 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_b2_discovery_ingress_payload(payload_dir) + if is_device_plane_b2_discovery_rollback_recovery_slice( + manifest["component"], + entries, + ): + validate_device_plane_b2_discovery_rollback_recovery_payload( + payload_dir + ) + if is_device_plane_backhaul_target_slice( + manifest["component"], + entries, + ): + validate_device_plane_backhaul_target_payload(payload_dir) if manifest["component"] == "n8n-private-extension": validate_n8n_private_extension_release(payload_dir, entries) if manifest["component"] == "engine": @@ -8218,6 +8388,18 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256): ) +def reject_terminal_device_plane_backhaul_artifact(manifest, sha256): + if ( + manifest.get("id") == DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID + or sha256 == DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256 + ): + die( + "Device Plane backhaul target 001 is terminal failed; " + "use the exact registered loopback plus private Tailscale Serve " + "successor" + ) + + class DeployLock: def __enter__(self): try: @@ -8268,6 +8450,692 @@ def is_device_plane_b2_discovery_ingress_slice(component, entries): ) +def is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, +): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ENTRIES + ) + + +def is_device_plane_backhaul_target_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) == DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES + ) + + +def expected_device_plane_backhaul_target_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.backhaul-target-tailnet-serve.v1" + ), + "mode": "failed-backhaul-target-to-loopback-tailnet-serve", + "failedPatchId": DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID, + "failedArtifactSha256": ( + DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256 + ), + "failedBackupId": DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID, + "predecessorPatchId": ( + DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_PATCH_ID + ), + "predecessorArtifactSha256": ( + DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_ARTIFACT_SHA256 + ), + "sourceAction": "publish-loopback-backhaul-target-source", + "runtimeAction": ( + "build-create-target-and-register-private-tailnet-serve" + ), + "composeOverlay": DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "selectedServices": [DEVICE_PLANE_BACKHAUL_TARGET_SERVICE], + "preservedServices": list(DEVICE_PLANE_RUNTIME_SERVICES), + "loopbackListenAddress": DEVICE_PLANE_BACKHAUL_LOOPBACK_ADDRESS, + "listenPort": DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + "tailnetAddress": DEVICE_PLANE_BACKHAUL_TAILNET_ADDRESS, + "tailnetExposure": "tailscale-serve-private", + "tailscaleServeTarget": ( + DEVICE_PLANE_BACKHAUL_TAILSCALE_SERVE_TARGET + ), + "permittedTarget": DEVICE_PLANE_BACKHAUL_PERMITTED_TARGET, + "networkMode": "host", + "dockerPortPublication": "disabled", + "routerNatFirewall": "unchanged", + "edgePublicIngress": "disabled", + "funnel": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "runtimeTrust": "runner-managed", + "rollback": "remove-tailnet-serve-target-and-restore-source", + } + + +def validate_device_plane_backhaul_target_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_BACKHAUL_TARGET_REL, + "Device Plane backhaul target descriptor", + max_bytes=16 * 1024, + ) + if descriptor != expected_device_plane_backhaul_target_descriptor(): + die("Device Plane backhaul target descriptor mismatch") + compose = payload_dir / DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL + if ( + compose.is_symlink() + or not compose.is_file() + or sha256_file(compose) + != DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_SHA256 + ): + die("Device Plane backhaul target Compose mismatch") + return descriptor + + +def read_device_plane_backhaul_enrollment_public_key(): + path = DEVICE_PLANE_BACKHAUL_ENROLLMENT_PUBLIC_KEY_FILE + try: + path_stat = path.lstat() + text = path.read_text(encoding="ascii") + except (FileNotFoundError, OSError, UnicodeDecodeError): + die( + "Device Plane Edge enrollment public key is missing or unreadable: " + f"{path}" + ) + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or path_stat.st_size > 1024 + ): + die("Device Plane Edge enrollment public key is unsafe") + if text != text.strip() + "\n" or "\n" in text.strip(): + die("Device Plane Edge enrollment public key must be one line") + parts = text.strip().split() + if len(parts) not in (2, 3) or parts[0] != "ssh-ed25519": + die("Device Plane Edge enrollment public key type mismatch") + try: + blob = base64.b64decode(parts[1], validate=True) + except Exception: + die("Device Plane Edge enrollment public key encoding mismatch") + expected_prefix = b"\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20" + if len(blob) != len(expected_prefix) + 32 or not blob.startswith( + expected_prefix + ): + die("Device Plane Edge enrollment public key shape mismatch") + normalized = f"ssh-ed25519 {parts[1]} nodedc-device-edge-backhaul" + return { + "line": normalized, + "sha256": hashlib.sha256((normalized + "\n").encode("ascii")).hexdigest(), + } + + +def device_plane_tailscale_drop_privileges(uid, gid): + def demote(): + os.setgroups([]) + os.setgid(gid) + os.setuid(uid) + + return demote + + +def validate_device_plane_tailscale_cli(): + privilege_path = DEVICE_PLANE_TAILSCALE_PRIVILEGE + try: + privilege_stat = privilege_path.lstat() + except FileNotFoundError: + die( + "Synology Tailscale package privilege contract is missing: " + f"{privilege_path}" + ) + if ( + not stat.S_ISREG(privilege_stat.st_mode) + or privilege_stat.st_uid != 0 + or privilege_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ): + die("Synology Tailscale package privilege boundary mismatch") + privilege = read_strict_json( + privilege_path, + "Synology Tailscale package privilege contract", + max_bytes=16 * 1024, + ) + defaults = privilege.get("defaults") + if ( + not isinstance(defaults, dict) + or defaults.get("run-as") != "package" + or privilege.get("username") != DEVICE_PLANE_TAILSCALE_USER + or privilege.get("groupname") != DEVICE_PLANE_TAILSCALE_GROUP + ): + die("Synology Tailscale package identity contract mismatch") + try: + account = pwd.getpwnam(DEVICE_PLANE_TAILSCALE_USER) + group = grp.getgrnam(DEVICE_PLANE_TAILSCALE_GROUP) + except KeyError: + die("Synology Tailscale package account is missing") + if ( + account.pw_uid <= 0 + or group.gr_gid <= 0 + or account.pw_gid != group.gr_gid + ): + die("Synology Tailscale package account boundary mismatch") + + try: + binary_lstat = DEVICE_PLANE_TAILSCALE.lstat() + except FileNotFoundError: + die( + "Synology Tailscale CLI is missing: " + f"{DEVICE_PLANE_TAILSCALE}" + ) + if ( + not stat.S_ISREG(binary_lstat.st_mode) + or binary_lstat.st_uid not in (0, account.pw_uid) + or binary_lstat.st_mode + & ( + stat.S_IWGRP + | stat.S_IWOTH + | stat.S_ISUID + | stat.S_ISGID + ) + or binary_lstat.st_size < 1024 * 1024 + or binary_lstat.st_size > 256 * 1024 * 1024 + ): + die( + "Synology Tailscale CLI file boundary mismatch: " + f"uid={binary_lstat.st_uid} " + f"mode={stat.S_IMODE(binary_lstat.st_mode):04o} " + f"size={binary_lstat.st_size}" + ) + if binary_lstat.st_uid == account.pw_uid: + executable = bool(binary_lstat.st_mode & stat.S_IXUSR) + elif binary_lstat.st_gid == group.gr_gid: + executable = bool(binary_lstat.st_mode & stat.S_IXGRP) + else: + executable = bool(binary_lstat.st_mode & stat.S_IXOTH) + if not executable: + die("Synology Tailscale CLI is not executable by package account") + + context = { + "binary": str(DEVICE_PLANE_TAILSCALE), + "uid": account.pw_uid, + "gid": group.gr_gid, + "binarySha256": sha256_file(DEVICE_PLANE_TAILSCALE), + } + try: + help_result = subprocess.run( + [context["binary"], "serve", "--help"], + check=True, + capture_output=True, + text=True, + timeout=15, + preexec_fn=device_plane_tailscale_drop_privileges( + context["uid"], + context["gid"], + ), + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + die("Synology Tailscale Serve capability is unavailable") + help_text = help_result.stdout + help_result.stderr + for required in ("--tcp", "--bg", "--yes"): + if required not in help_text: + die( + "Synology Tailscale Serve capability mismatch: " + f"{required}" + ) + return context + + +def run_device_plane_tailscale(args, **kwargs): + context = validate_device_plane_tailscale_cli() + return subprocess.run( + [context["binary"], *args], + preexec_fn=device_plane_tailscale_drop_privileges( + context["uid"], + context["gid"], + ), + **kwargs, + ) + + +def read_device_plane_tailscale_json(*args, allow_no_config=False): + try: + result = run_device_plane_tailscale( + args, + check=False, + capture_output=True, + text=True, + timeout=20, + ) + except (OSError, subprocess.TimeoutExpired): + die("Synology Tailscale state query failed") + combined = (result.stdout + "\n" + result.stderr).strip() + if result.returncode != 0: + if allow_no_config and "no serve config" in combined.lower(): + return {} + die( + "Synology Tailscale state query failed: " + f"exit={result.returncode}" + ) + if len(result.stdout.encode("utf-8")) > 4 * 1024 * 1024: + die("Synology Tailscale state response is too large") + try: + value = json.loads(result.stdout or "{}") + except json.JSONDecodeError: + die("Synology Tailscale state response is invalid JSON") + if not isinstance(value, dict): + die("Synology Tailscale state response must be an object") + return value + + +def device_plane_tailscale_handlers_for_port(config, port): + matches = [] + + def walk(value, path): + if not isinstance(value, dict): + return + tcp = value.get("TCP") + if tcp is not None: + if not isinstance(tcp, dict): + die("Synology Tailscale TCP Serve state is invalid") + handler = tcp.get(str(port)) + if handler is not None: + matches.append((tuple(path), handler)) + for key in ("Foreground", "Services"): + nested = value.get(key) + if nested is None: + continue + if not isinstance(nested, dict): + die("Synology Tailscale nested Serve state is invalid") + for name, child in nested.items(): + walk(child, (*path, key, str(name))) + + walk(config, ()) + return matches + + +def device_plane_tailscale_funnel_uses_port(config, port): + allow_funnel = config.get("AllowFunnel") or {} + if not isinstance(allow_funnel, dict): + die("Synology Tailscale Funnel state is invalid") + suffix = f":{port}" + return any( + enabled is True + and (str(host_port) == str(port) or str(host_port).endswith(suffix)) + for host_port, enabled in allow_funnel.items() + ) + + +def remove_device_plane_tailscale_port(config, port): + normalized = json.loads(json.dumps(config)) + tcp = normalized.get("TCP") + if isinstance(tcp, dict): + tcp.pop(str(port), None) + if not tcp: + normalized.pop("TCP", None) + return normalized + + +def validate_device_plane_tailscale_runtime(require_target): + status = read_device_plane_tailscale_json("status", "--json") + self_state = status.get("Self") + if not isinstance(self_state, dict): + die("Synology Tailscale self state is missing") + addresses = self_state.get("TailscaleIPs") or [] + if ( + not isinstance(addresses, list) + or DEVICE_PLANE_BACKHAUL_TAILNET_ADDRESS not in addresses + or self_state.get("Online") is not True + or status.get("BackendState") != "Running" + ): + die("Synology Tailscale tailnet state mismatch") + + serve = read_device_plane_tailscale_json( + "serve", + "status", + "--json", + allow_no_config=True, + ) + handlers = device_plane_tailscale_handlers_for_port( + serve, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ) + if device_plane_tailscale_funnel_uses_port( + serve, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ): + die("Synology Tailscale Funnel exposure is forbidden") + if require_target: + expected_handler = { + "TCPForward": ( + f"{DEVICE_PLANE_BACKHAUL_LOOPBACK_ADDRESS}:" + f"{DEVICE_PLANE_BACKHAUL_LISTEN_PORT}" + ) + } + if handlers != [((), expected_handler)]: + die("Synology Tailscale private Serve target mismatch") + elif handlers: + die("Synology Tailscale port 2222 is already configured") + return { + "self": self_state, + "serve": serve, + } + + +def enable_device_plane_tailscale_serve(expected_before): + current = validate_device_plane_tailscale_runtime( + require_target=False + )["serve"] + if current != expected_before: + die("Synology Tailscale Serve state changed before activation") + run_device_plane_tailscale( + [ + "serve", + "--bg", + "--yes", + f"--tcp={DEVICE_PLANE_BACKHAUL_LISTEN_PORT}", + DEVICE_PLANE_BACKHAUL_TAILSCALE_SERVE_TARGET, + ], + check=True, + timeout=30, + ) + after = validate_device_plane_tailscale_runtime( + require_target=True + )["serve"] + if remove_device_plane_tailscale_port( + after, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ) != expected_before: + die("Synology Tailscale Serve activation changed unrelated state") + return after + + +def disable_device_plane_tailscale_serve(expected_after): + current = read_device_plane_tailscale_json( + "serve", + "status", + "--json", + allow_no_config=True, + ) + handlers = device_plane_tailscale_handlers_for_port( + current, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ) + if not handlers: + if current != expected_after: + die("Synology Tailscale Serve rollback state mismatch") + return False + expected_handler = { + "TCPForward": ( + f"{DEVICE_PLANE_BACKHAUL_LOOPBACK_ADDRESS}:" + f"{DEVICE_PLANE_BACKHAUL_LISTEN_PORT}" + ) + } + if ( + handlers != [((), expected_handler)] + or device_plane_tailscale_funnel_uses_port( + current, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ) + or remove_device_plane_tailscale_port( + current, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ) != expected_after + ): + die("Synology Tailscale Serve rollback refuses unexpected state") + run_device_plane_tailscale( + [ + "serve", + f"--tcp={DEVICE_PLANE_BACKHAUL_LISTEN_PORT}", + "off", + ], + check=True, + timeout=30, + ) + final = validate_device_plane_tailscale_runtime( + require_target=False + )["serve"] + if final != expected_after: + die("Synology Tailscale Serve rollback changed unrelated state") + return True + + +def expected_failed_device_plane_backhaul_target_descriptor(): + return { + "schemaVersion": "nodedc.device-plane.backhaul-target.v1", + "mode": "restricted-edge-backhaul-target", + "predecessorPatchId": ( + DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_PATCH_ID + ), + "predecessorArtifactSha256": ( + DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_ARTIFACT_SHA256 + ), + "sourceAction": "publish-restricted-backhaul-target-source", + "runtimeAction": "build-and-create-device-backhaul-target", + "composeOverlay": DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "selectedServices": [DEVICE_PLANE_BACKHAUL_TARGET_SERVICE], + "preservedServices": list(DEVICE_PLANE_RUNTIME_SERVICES), + "listenAddress": DEVICE_PLANE_BACKHAUL_TAILNET_ADDRESS, + "listenPort": DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + "permittedTarget": DEVICE_PLANE_BACKHAUL_PERMITTED_TARGET, + "networkMode": "host", + "edgeIngress": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "runtimeTrust": "runner-managed", + "rollback": "remove-target-and-restore-source", + } + + +def validate_device_plane_backhaul_failed_evidence(): + backup_dir = BACKUPS_DIR / DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Plane failed backhaul backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Plane failed backhaul backup is unsafe") + if {child.name for child in backup_dir.iterdir()} != set( + DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_SHA256 + ): + die("Device Plane failed backhaul backup file set mismatch") + for name, expected_sha256 in ( + DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_SHA256.items() + ): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Plane failed backhaul backup drift detected: " + f"{name}" + ) + + failed_artifact = FAILED_DIR / DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Plane failed backhaul artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256 + ): + die("Device Plane failed backhaul artifact evidence mismatch") + + try: + state_stat = FAILED_STATE_FILE.lstat() + state_lines = FAILED_STATE_FILE.read_text( + encoding="utf-8" + ).splitlines() + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Device Plane failed backhaul journal is unreadable") + if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG( + state_stat.st_mode + ): + die("Device Plane failed backhaul journal is unsafe") + records = [] + for line in state_lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + die("Device Plane failed backhaul journal contains invalid JSON") + if ( + isinstance(value, dict) + and value.get("id") == DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID + ): + records.append(value) + if len(records) != 1: + die("Device Plane failed backhaul journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") + != ( + "ok:device-plane-overlay:source-restored-target-removed-" + "preserved-runtime-unchanged:3" + ) + or record.get("status") != "failed" + or record.get("message") != DEVICE_PLANE_BACKHAUL_FAILED_MESSAGE + ): + die("Device Plane failed backhaul journal evidence mismatch") + + with tempfile.TemporaryDirectory( + prefix="device-plane-failed-backhaul-", + dir=TMP_DIR, + ) as directory: + failed_work = Path(directory) + safe_extract(failed_artifact, failed_work) + failed_manifest = parse_manifest(failed_work / "manifest.env") + failed_entries = parse_files_list(failed_work / "files.txt") + failed_payload = failed_work / "payload" + expected_payload_paths = { + "deployment", + DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL, + DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "services", + "services/device-backhaul-target", + "services/device-backhaul-target/Dockerfile", + "services/device-backhaul-target/sshd_config", + } + actual_payload_paths = { + path.relative_to(failed_payload).as_posix() + for path in failed_payload.rglob("*") + } + if ( + failed_manifest.get("id") + != DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_BACKHAUL_FAILED_TARGET_ENTRIES + or actual_payload_paths != expected_payload_paths + ): + die("Device Plane failed backhaul artifact contract mismatch") + failed_descriptor = read_strict_json( + failed_payload / DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL, + "failed Device Plane backhaul descriptor", + max_bytes=16 * 1024, + ) + if ( + failed_descriptor + != expected_failed_device_plane_backhaul_target_descriptor() + ): + die("Device Plane failed backhaul descriptor mismatch") + return { + "backup": backup_dir, + "failedArtifact": failed_artifact, + } + + +def validate_device_plane_backhaul_target_evidence(payload_dir): + descriptor = validate_device_plane_backhaul_target_payload(payload_dir) + failed = validate_device_plane_backhaul_failed_evidence() + if not state_has_patch_id( + DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_PATCH_ID + ): + die("Device Plane backhaul predecessor patch is not applied") + if not state_has_sha( + DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_ARTIFACT_SHA256 + ): + die("Device Plane backhaul predecessor artifact is not applied") + + root = component_root("device-plane") + compose = root / "docker-compose.device-plane.yml" + if ( + compose.is_symlink() + or not compose.is_file() + or sha256_file(compose) != DEVICE_PLANE_B2_DISCOVERY_INGRESS_COMPOSE_SHA256 + ): + die("Device Plane backhaul predecessor Compose drift detected") + installed_b2 = read_strict_json( + root / DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL, + "installed Device Plane B2 descriptor", + max_bytes=16 * 1024, + ) + if installed_b2 != expected_device_plane_b2_discovery_ingress_descriptor(): + die("Device Plane backhaul predecessor B2 descriptor mismatch") + for rel in ( + DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL, + DEVICE_PLANE_BACKHAUL_TARGET_REL, + DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "services/device-backhaul-target", + ): + target = root / rel + if target.exists() or target.is_symlink(): + die( + "Device Plane failed backhaul source was not restored: " + f"{rel}" + ) + if device_plane_service_container_ids( + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE + ): + die("Device Plane backhaul target container already exists") + + runtime = device_plane_runtime_inventory(DEVICE_PLANE_RUNTIME_SERVICES) + names = device_plane_inventory_service_names(runtime) + if set(names) != set(DEVICE_PLANE_RUNTIME_SERVICES): + die("Device Plane backhaul predecessor runtime is incomplete") + for item in runtime["services"]: + if ( + item["status"] != "running" + or item["running"] is not True + or item["health"] != "healthy" + or item["restartCount"] != 0 + ): + die( + "Device Plane backhaul predecessor service is unhealthy: " + f"{item['service']}" + ) + assert_loopback_tcp_port_open(9921) + enrollment = read_device_plane_backhaul_enrollment_public_key() + tailscale_cli = validate_device_plane_tailscale_cli() + tailscale = validate_device_plane_tailscale_runtime( + require_target=False + ) + return { + "mode": descriptor["mode"], + "failedBackup": str(failed["backup"]), + "failedArtifact": str(failed["failedArtifact"]), + "runtime": runtime, + "enrollmentPublicKeySha256": enrollment["sha256"], + "tailscaleCli": tailscale_cli, + "tailscaleServeBefore": tailscale["serve"], + } + + def expected_device_plane_foundation_recovery_descriptor(): return { "schemaVersion": "nodedc.device-plane.foundation-recovery.v1", @@ -8396,6 +9264,57 @@ def expected_device_plane_b2_discovery_ingress_descriptor(): } +def expected_device_plane_b2_discovery_rollback_recovery_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.b2-discovery-loopback-recovery.v1" + ), + "mode": "failed-b2-loopback-build-reconciliation", + "failedPatchId": DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID, + "failedArtifactSha256": ( + DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT_SHA256 + ), + "failedBackupId": ( + DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_ID + ), + "sourceAction": "publish-reconciliation-marker-only", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres", + ], + "expectedLoopbackPorts": [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121", + ], + "closedPort": "127.0.0.1:9921/tcp", + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "marker-only-runtime-unchanged", + } + + +def validate_device_plane_b2_discovery_rollback_recovery_payload( + payload_dir, +): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, + "Device Plane B2 discovery rollback recovery descriptor", + max_bytes=16 * 1024, + ) + expected = ( + expected_device_plane_b2_discovery_rollback_recovery_descriptor() + ) + if descriptor != expected: + die( + "Device Plane B2 discovery rollback recovery descriptor " + "mismatch" + ) + return descriptor + + def validate_device_plane_b2_discovery_ingress_payload(payload_dir): descriptor = read_strict_json( payload_dir / DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL, @@ -8427,6 +9346,14 @@ def validate_device_plane_b2_discovery_ingress_evidence(payload_dir): DEVICE_PLANE_B2_DISCOVERY_INGRESS_PREDECESSOR_ARTIFACT_SHA256 ): die("Device Plane B2 discovery ingress predecessor artifact missing") + if not state_has_patch_id( + DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_PATCH_ID + ): + die("Device Plane B2 rollback recovery patch is not applied") + if not state_has_sha( + DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ARTIFACT_SHA256 + ): + die("Device Plane B2 rollback recovery artifact is not applied") validate_device_plane_foundation_network_publication_installed_source() runtime = validate_device_plane_foundation_runtime( network_publication=True @@ -8438,14 +9365,170 @@ def validate_device_plane_b2_discovery_ingress_evidence(payload_dir): ) if target_descriptor.exists() or target_descriptor.is_symlink(): die("Device Plane B2 discovery ingress descriptor already installed") + recovery_descriptor = read_strict_json( + component_root("device-plane") + / DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, + "installed Device Plane B2 rollback recovery descriptor", + max_bytes=16 * 1024, + ) + if ( + recovery_descriptor + != expected_device_plane_b2_discovery_rollback_recovery_descriptor() + ): + die("installed Device Plane B2 rollback recovery mismatch") return { "mode": descriptor["mode"], "runtime": runtime, } +def validate_device_plane_b2_discovery_rollback_recovery_evidence( + payload_dir, +): + descriptor = ( + validate_device_plane_b2_discovery_rollback_recovery_payload( + payload_dir + ) + ) + backup_dir = ( + BACKUPS_DIR + / DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_ID + ) + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Plane B2 failed-apply backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Plane B2 failed-apply backup is unsafe") + expected_backup = ( + DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_SHA256 + ) + if {child.name for child in backup_dir.iterdir()} != set( + expected_backup + ): + die("Device Plane B2 failed-apply backup file set mismatch") + for name, expected_sha256 in expected_backup.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Plane B2 failed-apply backup drift detected: " + f"{name}" + ) + + failed_artifact = ( + FAILED_DIR / DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT + ) + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Plane B2 failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT_SHA256 + ): + die("Device Plane B2 failed artifact evidence mismatch") + + try: + state_stat = FAILED_STATE_FILE.lstat() + state_lines = FAILED_STATE_FILE.read_text( + encoding="utf-8" + ).splitlines() + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Device Plane B2 failed journal is unreadable") + if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG( + state_stat.st_mode + ): + die("Device Plane B2 failed journal is unsafe") + records = [] + for line in state_lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + die("Device Plane B2 failed journal contains invalid JSON") + if ( + isinstance(value, dict) + and value.get("id") == DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID + ): + records.append(value) + if len(records) != 1: + die("Device Plane B2 failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") + != DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "failed:CalledProcessError" + or record.get("status") != "failed" + or record.get("message") + != ( + "Command '['/usr/local/bin/docker', 'build', '--no-cache', " + "'--network=host', '-f', " + "'services/device-control-core/Dockerfile', '-t', " + "'nodedc/device-control-core:local', '.']' returned non-zero " + "exit status 1." + ) + ): + die("Device Plane B2 failed journal evidence mismatch") + + with tempfile.TemporaryDirectory( + prefix="device-plane-failed-b2-loopback-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, _failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_B2_DISCOVERY_FAILED_ENTRIES + ): + die("Device Plane B2 failed artifact contract mismatch") + + validate_device_plane_foundation_network_publication_installed_source() + runtime = validate_device_plane_foundation_runtime( + network_publication=True + ) + assert_loopback_tcp_port_closed(9921) + root = component_root("device-plane") + failed_descriptor = root / DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL + if failed_descriptor.exists() or failed_descriptor.is_symlink(): + die("Device Plane B2 failed descriptor remains installed") + target_descriptor = ( + root / DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL + ) + if target_descriptor.exists() or target_descriptor.is_symlink(): + die("Device Plane B2 rollback recovery already installed") + return { + "mode": descriptor["mode"], + "backup": backup_dir, + "failedArtifact": failed_artifact, + "runtime": runtime, + } + + def device_plane_service_container_ids(service): - if service not in DEVICE_PLANE_RUNTIME_SERVICES: + if service not in ( + *DEVICE_PLANE_RUNTIME_SERVICES, + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE, + ): die(f"Device Plane runtime service is not registered: {service}") result = subprocess.run( [ @@ -9604,6 +10687,209 @@ def validate_device_plane_b2_discovery_ingress_runtime(runtime_before): return accepted +def validate_device_plane_preserved_runtime_unchanged(runtime_before, label): + before_names = device_plane_inventory_service_names(runtime_before) + if set(before_names) != set(DEVICE_PLANE_RUNTIME_SERVICES): + die(f"{label} predecessor inventory mismatch") + before = { + item["service"]: item + for item in runtime_before["services"] + } + current = device_plane_runtime_inventory(DEVICE_PLANE_RUNTIME_SERVICES) + current_names = device_plane_inventory_service_names(current) + if set(current_names) != set(DEVICE_PLANE_RUNTIME_SERVICES): + die(f"{label} preserved runtime is incomplete") + for item in current["services"]: + predecessor = before[item["service"]] + if ( + item["containerId"] != predecessor["containerId"] + or item["imageId"] != predecessor["imageId"] + or item["status"] != "running" + or item["running"] is not True + or item["health"] != "healthy" + or item["restartCount"] != 0 + ): + die(f"{label} changed preserved service: {item['service']}") + return current + + +def validate_device_plane_backhaul_target_runtime(runtime_before): + current = validate_device_plane_preserved_runtime_unchanged( + runtime_before, + "Device Plane backhaul", + ) + root = component_root("device-plane") + installed_descriptor = read_strict_json( + root / DEVICE_PLANE_BACKHAUL_TARGET_REL, + "installed Device Plane backhaul target descriptor", + max_bytes=16 * 1024, + ) + if installed_descriptor != expected_device_plane_backhaul_target_descriptor(): + die("installed Device Plane backhaul target descriptor mismatch") + installed_compose = root / DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL + if ( + installed_compose.is_symlink() + or not installed_compose.is_file() + or sha256_file(installed_compose) + != DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_SHA256 + ): + die("installed Device Plane backhaul target Compose mismatch") + failed_descriptor = root / DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL + if failed_descriptor.exists() or failed_descriptor.is_symlink(): + die("failed Device Plane backhaul descriptor remains installed") + + container_ids = device_plane_service_container_ids( + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE + ) + if len(container_ids) != 1: + die("Device Plane backhaul target service count mismatch") + container = inspect_device_plane_container(container_ids[0]) + state = container.get("State") or {} + config = container.get("Config") or {} + host_config = container.get("HostConfig") or {} + labels = config.get("Labels") or {} + mounts = container.get("Mounts") or [] + expected_mounts = { + "/run/nodedc-secrets/ssh_host_ed25519_key": ( + DEVICE_PLANE_BACKHAUL_HOST_KEY_FILE + ), + "/run/nodedc-secrets/authorized_keys": ( + DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE + ), + } + actual_mounts = { + mount.get("Destination"): mount + for mount in mounts + } + if ( + state.get("Status") != "running" + or state.get("Running") is not True + or state.get("Restarting") is True + or state.get("ExitCode") != 0 + or state.get("Error") not in ("", None) + or (state.get("Health") or {}).get("Status") != "healthy" + or int(container.get("RestartCount") or 0) != 0 + or config.get("Image") != DEVICE_PLANE_BACKHAUL_TARGET_IMAGE + or config.get("User", "") != "" + or host_config.get("NetworkMode") != "host" + or (host_config.get("PortBindings") or {}) != {} + or ((container.get("NetworkSettings") or {}).get("Ports") or {}) + != {} + or host_config.get("ReadonlyRootfs") is not True + or set(host_config.get("CapDrop") or ()) != {"ALL"} + or set(host_config.get("CapAdd") or ()) + != {"CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID", "SYS_CHROOT"} + or "no-new-privileges:true" + not in set(host_config.get("SecurityOpt") or ()) + or (host_config.get("RestartPolicy") or {}).get("Name") + != "unless-stopped" + or labels.get("com.docker.compose.project") + != "nodedc-device-plane" + or labels.get("com.docker.compose.service") + != DEVICE_PLANE_BACKHAUL_TARGET_SERVICE + or set(actual_mounts) != set(expected_mounts) + ): + die("Device Plane backhaul target runtime mismatch") + for destination, source in expected_mounts.items(): + mount = actual_mounts[destination] + if ( + mount.get("Type") != "bind" + or mount.get("Source") != str(source) + or mount.get("RW") is not False + ): + die("Device Plane backhaul target mount mismatch") + environment = container_environment( + container, + "Device Plane backhaul target", + ) + for name in environment: + if any(token in name.upper() for token in ("KEY", "PASS", "TOKEN")): + die("Device Plane backhaul target plaintext environment rejected") + + effective = subprocess.run( + [ + str(DOCKER), + "exec", + container_ids[0], + "/usr/sbin/sshd", + "-T", + "-f", + "/etc/ssh/sshd_config", + ], + check=True, + capture_output=True, + text=True, + ).stdout.lower() + for required in ( + "port 2222", + "listenaddress 127.0.0.1:2222", + "passwordauthentication no", + "kbdinteractiveauthentication no", + "allowtcpforwarding local", + "permitopen 127.0.0.1:9921", + "gatewayports no", + "permittty no", + "allowagentforwarding no", + "x11forwarding no", + "permittunnel no", + "forcecommand /bin/false", + ): + if required not in effective: + die( + "Device Plane backhaul effective sshd contract mismatch: " + f"{required}" + ) + + enrollment = read_device_plane_backhaul_enrollment_public_key() + authorized = ( + 'restrict,port-forwarding,permitopen="127.0.0.1:9921" ' + f"{enrollment['line']}\n" + ) + for path, mode, max_size in ( + (DEVICE_PLANE_BACKHAUL_HOST_KEY_FILE, 0o400, 2048), + (DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE, 0o444, 2048), + (DEVICE_PLANE_BACKHAUL_HOST_PUBLIC_KEY_FILE, 0o444, 1024), + ): + try: + path_stat = path.lstat() + except FileNotFoundError: + die("Device Plane backhaul runtime trust file is missing") + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or path_stat.st_uid != 0 + or stat.S_IMODE(path_stat.st_mode) != mode + or path_stat.st_size > max_size + ): + die("Device Plane backhaul runtime trust boundary mismatch") + if DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE.read_text( + encoding="ascii" + ) != authorized: + die("Device Plane backhaul authorized key mismatch") + + try: + connection = socket.create_connection( + ( + DEVICE_PLANE_BACKHAUL_LOOPBACK_ADDRESS, + DEVICE_PLANE_BACKHAUL_LISTEN_PORT, + ), + timeout=5, + ) + connection.settimeout(5) + banner = connection.recv(256) + connection.close() + except OSError as exc: + die(f"Device Plane backhaul SSH listener is unavailable: {exc}") + if not banner.startswith(b"SSH-2.0-OpenSSH_"): + die("Device Plane backhaul SSH banner mismatch") + validate_device_plane_tailscale_runtime(require_target=True) + assert_loopback_tcp_port_open(9921) + return { + "preserved": current, + "targetContainerId": container.get("Id"), + } + + def validate_device_plane_runtime_secret_metadata(): try: directory_stat = DEVICE_PLANE_SECRET_DIR.lstat() @@ -11271,6 +12557,17 @@ def is_platform_provider_catalog_only(entries): def component_services(component, entries=None): + if is_device_plane_backhaul_target_slice(component, entries): + return (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,) + + if is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, + ): + # Reconciliation writes only the reviewed marker after proving the + # failed archive, journal, backup, restored source and live runtime. + return () + if is_device_plane_postgres_bootstrap_slice(component, entries): # This exact one-time transition is the only Device Plane artifact that # may select durable state. Its preflight requires both container and @@ -11590,6 +12887,17 @@ def component_compose_files( files.append(root / ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL) return tuple(files) files = COMPONENTS[component].get("compose_files", ()) + if component == "device-plane": + overlay = DEVICE_PLANE_ROOT / DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL + if overlay.exists() or overlay.is_symlink(): + if ( + overlay.is_symlink() + or not overlay.is_file() + or sha256_file(overlay) + != DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_SHA256 + ): + die("installed Device Plane backhaul Compose drift detected") + files = (*files, overlay) if component == "platform": # The EDP overlay is optional until its first dedicated artifact lands. # Once installed (or after a candidate has been copied), it participates @@ -11623,6 +12931,27 @@ def component_build_args(component, entries=None): def component_builds(component, entries=None): + if is_device_plane_backhaul_target_slice(component, entries): + return (( + DEVICE_PLANE_ROOT, + ( + "build", + "--no-cache", + "--network=host", + "-f", + "services/device-backhaul-target/Dockerfile", + "-t", + DEVICE_PLANE_BACKHAUL_TARGET_IMAGE, + ".", + ), + ),) + + if is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, + ): + return () + if is_device_plane_postgres_bootstrap_slice(component, entries): return () @@ -13609,6 +14938,8 @@ def plan_artifact(artifact): device_plane_foundation_recovery_preflight = None device_plane_network_publication_preflight = None device_plane_b2_ingress_preflight = None + device_plane_b2_recovery_preflight = None + device_plane_backhaul_preflight = None device_plane_runtime_before = None composite_provider_v4_preflight = None provider_rotating_slot_preflight = None @@ -13634,6 +14965,7 @@ def plan_artifact(artifact): manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_device_plane_foundation_artifact(manifest, sha) + reject_terminal_device_plane_backhaul_artifact(manifest, sha) transition_descriptor = None transition_preflight = None if is_engine_n8n_transition(manifest["component"], entries): @@ -13801,6 +15133,22 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_b2_discovery_rollback_recovery_slice( + manifest["component"], + entries, + ): + device_plane_b2_recovery_preflight = ( + validate_device_plane_b2_discovery_rollback_recovery_evidence( + payload_dir + ) + ) + if is_device_plane_backhaul_target_slice( + manifest["component"], + entries, + ): + device_plane_backhaul_preflight = ( + validate_device_plane_backhaul_target_evidence(payload_dir) + ) component = manifest["component"] root = component_root(component) @@ -15300,6 +16648,109 @@ def plan_artifact(artifact): "device_plane_rollback=" "source+predecessor-stateless-runtime" ) + if device_plane_b2_recovery_preflight is not None: + print( + "device_plane_transition=" + f"{device_plane_b2_recovery_preflight['mode']}" + ) + print( + "failed_patch=" + f"{DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT_SHA256}" + ) + print( + "recovery_backup=" + f"{device_plane_b2_recovery_preflight['backup'].name}" + ) + print("device_plane_build=none") + print("device_plane_runtime_mutation=none") + print( + "device_plane_runtime_services=" + "preserved:device-control-core,device-gateway,device-postgres" + ) + print( + "device_plane_source_action=" + "publish-reconciliation-marker-only" + ) + print("device_gateway_tcp_9921=disabled:unpublished") + print("device_plane_rollback=marker-only-runtime-unchanged") + if device_plane_backhaul_preflight is not None: + print( + "device_plane_transition=" + f"{device_plane_backhaul_preflight['mode']}" + ) + print( + "failed_patch=" + f"{DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256}" + ) + print( + "failed_backup=" + f"{DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID}" + ) + print( + "failed_rollback=" + "ok:source-restored-target-removed-preserved-runtime-unchanged" + ) + print( + "device_plane_predecessor_patch=" + f"{DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_PATCH_ID}" + ) + print( + "device_plane_predecessor_artifact_sha256=" + f"{DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_ARTIFACT_SHA256}" + ) + print( + "device_plane_runtime_mutation=" + "build+create:device-backhaul-target+tailscale-serve" + ) + print( + "device_plane_runtime_services=" + "preserved:device-control-core,device-gateway,device-postgres" + ) + print( + "device_backhaul_loopback_listen=" + f"{DEVICE_PLANE_BACKHAUL_LOOPBACK_ADDRESS}:" + f"{DEVICE_PLANE_BACKHAUL_LISTEN_PORT}/tcp" + ) + print( + "device_backhaul_tailnet_listen=" + f"tailscale-serve:{DEVICE_PLANE_BACKHAUL_TAILNET_ADDRESS}:" + f"{DEVICE_PLANE_BACKHAUL_LISTEN_PORT}/tcp=>" + f"{DEVICE_PLANE_BACKHAUL_TAILSCALE_SERVE_TARGET}" + ) + print( + "device_backhaul_permitopen=" + f"{DEVICE_PLANE_BACKHAUL_PERMITTED_TARGET}" + ) + print( + "device_backhaul_enrollment_public_key_sha256=" + f"{device_plane_backhaul_preflight['enrollmentPublicKeySha256']}" + ) + print( + "device_backhaul_runtime_trust=" + "runner-managed:host-key,authorized-keys,public-trust" + ) + print( + "device_backhaul_tailscale_cli=" + "official-package-account:" + f"{DEVICE_PLANE_TAILSCALE_USER}:sha256:" + f"{device_plane_backhaul_preflight['tailscaleCli']['binarySha256']}" + ) + print("device_backhaul_docker_port_publication=disabled") + print("device_backhaul_tailscale_funnel=disabled") + print("device_backhaul_router_nat_firewall=unchanged") + print("device_edge_public_ingress=disabled") + print( + "device_plane_rollback=" + "remove-tailnet-serve-target-and-restore-source" + ) if device_plane_postgres_preflight is not None: print( "device_postgres_bootstrap=" @@ -15700,6 +17151,14 @@ def rollback_device_plane_apply( for service in applied_services if service not in baseline_services ) + tailscale_serve_before = None + if is_device_plane_backhaul_target_slice("device-plane", entries): + tailscale_serve_before = read_strict_json( + backup_dir / "tailscale-serve-before.json", + "Device Plane pre-apply Tailscale Serve state", + max_bytes=4 * 1024 * 1024, + ) + disable_device_plane_tailscale_serve(tailscale_serve_before) candidate_cleanup_failed = False if runtime_started and candidate_only_services: # Remove only candidate services while the candidate Compose file is @@ -15721,6 +17180,26 @@ def rollback_device_plane_apply( ) if candidate_cleanup_failed: die("Device Plane candidate-only runtime cleanup failed after source restore") + if is_device_plane_backhaul_target_slice("device-plane", entries): + if device_plane_service_container_ids( + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE + ): + die("Device Plane backhaul rollback retained target runtime") + validate_device_plane_preserved_runtime_unchanged( + runtime_inventory, + "Device Plane backhaul rollback", + ) + tailscale = validate_device_plane_tailscale_runtime( + require_target=False + ) + if tailscale["serve"] != tailscale_serve_before: + die("Device Plane backhaul rollback did not restore Tailscale Serve") + assert_loopback_tcp_port_open(9921) + return ( + "tailscale-serve-restored-source-restored-target-removed-" + "preserved-runtime-unchanged:" + f"{restored_count}" + ) if not runtime_started or not baseline_services: return f"source-restored-runtime-unchanged:{restored_count}" @@ -16285,6 +17764,100 @@ def run_engine_node_intelligence_compose(services, entries): ) +def ensure_device_plane_backhaul_target_state(): + enrollment = read_device_plane_backhaul_enrollment_public_key() + DEVICE_PLANE_BACKHAUL_SECRET_DIR.mkdir( + parents=True, + exist_ok=True, + ) + os.chown(DEVICE_PLANE_BACKHAUL_SECRET_DIR, 0, 0) + DEVICE_PLANE_BACKHAUL_SECRET_DIR.chmod(0o700) + + if not DEVICE_PLANE_BACKHAUL_HOST_KEY_FILE.exists(): + subprocess.run( + [ + str(DOCKER), + "run", + "--rm", + "--entrypoint", + "/usr/bin/ssh-keygen", + "-v", + f"{DEVICE_PLANE_BACKHAUL_SECRET_DIR}:/keys", + DEVICE_PLANE_BACKHAUL_TARGET_IMAGE, + "-q", + "-t", + "ed25519", + "-N", + "", + "-C", + "nodedc-device-plane-backhaul-target", + "-f", + "/keys/ssh_host_ed25519_key", + ], + check=True, + ) + + host_public_source = Path( + f"{DEVICE_PLANE_BACKHAUL_HOST_KEY_FILE}.pub" + ) + for path, expected_mode, max_size in ( + (DEVICE_PLANE_BACKHAUL_HOST_KEY_FILE, 0o400, 2048), + (host_public_source, 0o444, 1024), + ): + try: + path_stat = path.lstat() + except FileNotFoundError: + die("Device Plane backhaul host key generation failed") + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or path_stat.st_uid != 0 + or path_stat.st_size > max_size + ): + die("Device Plane backhaul host key boundary mismatch") + os.chown(path, 0, 0) + path.chmod(expected_mode) + + authorized = ( + 'restrict,port-forwarding,permitopen="127.0.0.1:9921" ' + f"{enrollment['line']}\n" + ) + temporary = DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE.with_suffix( + ".installing" + ) + if temporary.exists() or temporary.is_symlink(): + die("Device Plane backhaul authorized_keys staging path exists") + temporary.write_text(authorized, encoding="ascii") + os.chown(temporary, 0, 0) + temporary.chmod(0o444) + os.replace(temporary, DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE) + + DEVICE_PLANE_BACKHAUL_TRUST_DIR.mkdir(parents=True, exist_ok=True) + os.chown(DEVICE_PLANE_BACKHAUL_TRUST_DIR, 0, 0) + DEVICE_PLANE_BACKHAUL_TRUST_DIR.chmod(0o755) + public_value = host_public_source.read_text(encoding="ascii") + public_temporary = DEVICE_PLANE_BACKHAUL_HOST_PUBLIC_KEY_FILE.with_suffix( + ".installing" + ) + if public_temporary.exists() or public_temporary.is_symlink(): + die("Device Plane backhaul public trust staging path exists") + public_temporary.write_text(public_value, encoding="ascii") + os.chown(public_temporary, 0, 0) + public_temporary.chmod(0o444) + os.replace(public_temporary, DEVICE_PLANE_BACKHAUL_HOST_PUBLIC_KEY_FILE) + + expected_authorized_sha256 = hashlib.sha256( + authorized.encode("ascii") + ).hexdigest() + if ( + sha256_file(DEVICE_PLANE_BACKHAUL_AUTHORIZED_KEYS_FILE) + != expected_authorized_sha256 + or sha256_file(DEVICE_PLANE_BACKHAUL_HOST_PUBLIC_KEY_FILE) + != sha256_file(host_public_source) + ): + die("Device Plane backhaul runtime trust verification failed") + + def stop_and_remove_compose_services(component, services): if not services: return @@ -16356,6 +17929,8 @@ def prepare_component_runtime(component, entries=None): MAP_GATEWAY_SECRET_RE, "device plane identifier pepper", ) + if is_device_plane_backhaul_target_slice(component, entries): + ensure_device_plane_backhaul_target_state() return if component == "proxy-contur": @@ -16431,6 +18006,11 @@ def run_component_runtime(component, entries, services): return if component_artifact_only(component): return + if is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, + ): + return if is_device_plane_foundation_recovery_slice(component, entries): return if is_device_plane_foundation_network_publication_slice( @@ -16448,6 +18028,40 @@ def run_component_runtime(component, entries, services): run_compose(component, services, entries) +def run_device_plane_runtime_for_apply( + entries, + services, + mark_runtime_started, + backhaul_serve_before=None, +): + if is_device_plane_b2_discovery_rollback_recovery_slice( + "device-plane", + entries, + ) or is_device_plane_foundation_recovery_slice( + "device-plane", + entries, + ): + return + # Build and runtime preparation are pre-runtime phases. A failure here + # must restore source only; it must never trigger a predecessor rebuild. + run_build("device-plane", entries) + prepare_component_runtime("device-plane", entries) + if services: + mark_runtime_started() + run_compose("device-plane", services, entries) + if is_device_plane_backhaul_target_slice("device-plane", entries): + if backhaul_serve_before is None: + die("Device Plane backhaul Tailscale preflight state is missing") + # Expose nothing to the tailnet until the loopback-only SSH target is + # healthy. Tailscale Serve is the only ingress mutation and remains + # private to the tailnet; Docker publishes no host port. + healthcheck_compose_service( + "device-plane", + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE, + ) + enable_device_plane_tailscale_serve(backhaul_serve_before) + + def healthcheck_url(check): headers = {} expected_json = {} @@ -16563,6 +18177,32 @@ def module_foundry_healthcheck(): def component_healthchecks(component, entries=None, services=None): + if is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, + ): + return ( + { + "url": "http://127.0.0.1:18120/healthz", + "expected_json": { + "ok": True, + "service": "nodedc-device-control-core", + "database": "ready", + "discoveryIngest": "disabled", + "commandTransport": "disabled", + }, + }, + { + "url": "http://127.0.0.1:18121/healthz", + "expected_json": { + "ok": True, + "service": "nodedc-device-gateway", + "tcpListener": "disabled", + "publicIngress": "disabled", + "commandTransport": "disabled", + }, + }, + ) if is_device_plane_postgres_bootstrap_slice(component, entries): return () if is_device_plane_foundation_recovery_slice(component, entries): @@ -17045,6 +18685,44 @@ process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plan def run_healthchecks(component, entries=None, services=None): + if is_device_plane_backhaul_target_slice(component, entries): + if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,): + die("Device Plane backhaul target service set mismatch") + for service in DEVICE_PLANE_RUNTIME_SERVICES: + healthcheck_compose_service("device-plane", service) + healthcheck_compose_service( + "device-plane", + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE, + ) + return + + if is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, + ): + if tuple(services or ()) != (): + die("Device Plane B2 rollback recovery service set mismatch") + for service in DEVICE_PLANE_RUNTIME_SERVICES: + healthcheck_compose_service("device-plane", service) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + validate_device_plane_foundation_network_publication_installed_source() + installed_descriptor = read_strict_json( + component_root("device-plane") + / DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, + "installed Device Plane B2 rollback recovery descriptor", + max_bytes=16 * 1024, + ) + if ( + installed_descriptor + != expected_device_plane_b2_discovery_rollback_recovery_descriptor() + ): + die("installed Device Plane B2 rollback recovery mismatch") + validate_device_plane_foundation_runtime( + network_publication=True + ) + assert_loopback_tcp_port_closed(9921) + return if is_device_plane_postgres_bootstrap_slice(component, entries): if tuple(services or ()) != ("device-postgres",): die("Device Plane PostgreSQL bootstrap service set mismatch") @@ -17872,6 +19550,7 @@ def apply_artifact(artifact): transition_descriptor = None node_intelligence_descriptor = None l2_closed_loop_preflight = None + device_plane_backhaul_preflight = None node_intelligence_service_stopped = False apply_started = False engine_backend_recreated = False @@ -17889,6 +19568,10 @@ def apply_artifact(artifact): manifest, sha, ) + reject_terminal_device_plane_backhaul_artifact( + manifest, + sha, + ) patch_id = manifest["id"] component = manifest["component"] root = component_root(component) @@ -17929,6 +19612,22 @@ def apply_artifact(artifact): validate_device_plane_b2_discovery_ingress_evidence( payload_dir ) + if is_device_plane_b2_discovery_rollback_recovery_slice( + component, + entries, + ): + validate_device_plane_b2_discovery_rollback_recovery_evidence( + payload_dir + ) + if is_device_plane_backhaul_target_slice( + component, + entries, + ): + device_plane_backhaul_preflight = ( + validate_device_plane_backhaul_target_evidence( + payload_dir + ) + ) if not root.is_dir(): if bootstrap_root: root.mkdir(parents=True, exist_ok=True) @@ -18256,6 +19955,23 @@ def apply_artifact(artifact): encoding="utf-8", ) runtime_inventory_path.chmod(0o600) + if device_plane_backhaul_preflight is not None: + tailscale_before_path = ( + backup_dir / "tailscale-serve-before.json" + ) + tailscale_before_path.write_text( + json.dumps( + device_plane_backhaul_preflight[ + "tailscaleServeBefore" + ], + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + tailscale_before_path.chmod(0o600) if component == "platform" and touches_external_data_plane_files(entries): validate_backup_partition( entries, @@ -18292,6 +20008,23 @@ def apply_artifact(artifact): # can be safely reconciled by recreating on that runtime. engine_backend_recreated = True run_compose(component, services, entries) + elif component == "device-plane": + def mark_device_plane_runtime_started(): + nonlocal runtime_started + runtime_started = bool(services) + + run_device_plane_runtime_for_apply( + entries, + services, + mark_device_plane_runtime_started, + backhaul_serve_before=( + device_plane_backhaul_preflight[ + "tailscaleServeBefore" + ] + if device_plane_backhaul_preflight is not None + else None + ), + ) else: # Mark the generic runtime before Compose so a partial # candidate start is always eligible for its domain rollback. @@ -18322,6 +20055,18 @@ def apply_artifact(artifact): validate_device_plane_b2_discovery_ingress_runtime( device_plane_runtime_before ) + if is_device_plane_backhaul_target_slice( + component, + entries, + ): + if device_plane_runtime_before is None: + die( + "Device Plane backhaul predecessor runtime " + "inventory is missing" + ) + validate_device_plane_backhaul_target_runtime( + device_plane_runtime_before + ) applied_path = move_artifact(artifact, APPLIED_DIR) append_jsonl(STATE_FILE, { diff --git a/infra/deploy-runner/nodedc-edge-deploy b/infra/deploy-runner/nodedc-edge-deploy new file mode 100644 index 0000000..4c42e67 --- /dev/null +++ b/infra/deploy-runner/nodedc-edge-deploy @@ -0,0 +1,854 @@ +#!/usr/bin/env python3 +"""Canonical data-only deploy runner for the dedicated NODE.DC Device Edge.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import select +import shutil +import socket +import struct +import subprocess +import sys +import tarfile +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + + +RUNNER_PATH = Path("/usr/local/sbin/nodedc-edge-deploy") +LIVE_ROOT = Path("/home/ndcsudo/nodedc-device-edge/source") +INBOX_ROOT = Path("/home/ndcsudo/nodedc-device-edge/deploy/inbox") +STATE_ROOT = Path("/var/lib/nodedc-edge-deploy") +APPLIED_ROOT = STATE_ROOT / "applied" +FAILED_ROOT = STATE_ROOT / "failed" +BACKUP_ROOT = STATE_ROOT / "backups" +APPLIED_JOURNAL = STATE_ROOT / "state/applied.jsonl" +FAILED_JOURNAL = STATE_ROOT / "state/failed.jsonl" +DEPLOY_LOCK = STATE_ROOT / "state/deploy.lock" + +DOCKER = "/usr/bin/docker" +COMPONENT = "device-edge" +ARTIFACT_TYPE = "app-overlay" +PATCH_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,96}$") +MAX_ARTIFACT_BYTES = 16 * 1024 * 1024 + +COMPOSE_PROJECT = "nodedc-device-edge" +BASE_COMPOSE = LIVE_ROOT / "docker-compose.device-edge.yml" +INGRESS_COMPOSE = LIVE_ROOT / "docker-compose.device-edge.ingress.yml" +RELAY_SERVICE = "device-edge-relay" +RELAY_CONTAINER = "nodedc-device-edge-device-edge-relay-1" +BACKHAUL_CONTAINER = "nodedc-device-edge-device-edge-backhaul-1" +TAILNET_CONTAINER = "nodedc-device-edge-tailnet-1" +RELAY_IMAGE = "nodedc/device-edge-relay:local" + +INGRESS_PARENT = "enp1s0f0" +INGRESS_SUBNET = "192.168.68.0/22" +INGRESS_GATEWAY = "192.168.68.1" +INGRESS_IPV4 = "192.168.71.253" +INGRESS_PORT = 9921 +INGRESS_NETWORK = "nodedc-device-edge-ingress" +INGRESS_IPV4_APPROVED = True +INGRESS_IPV4_APPROVAL = "approved-outside-dhcp-pool" + +ENTRIES = ( + "docker-compose.device-edge.yml", + "docker-compose.device-edge.ingress.yml", + "services/device-edge-relay/Dockerfile", + "services/device-edge-relay/src", + "deployment/device-edge-ingress-ipvlan-v1.json", +) + +PAYLOAD_FILE_SHA256 = { + "docker-compose.device-edge.yml": + "7f13c11d6d4d541964053c0a8cf791e401947d34c42e0f7c26f9f9df26fa00b5", + "docker-compose.device-edge.ingress.yml": + "a4afd04755530fc3b9be64d1a65f0f7282a9539bcc1985bfd880aa904e1c4d8f", + "services/device-edge-relay/Dockerfile": + "f2f15b7618ac2ab3a1d4dd40041d06d1e9a695edcfc168d8835f9560b4897e70", + "services/device-edge-relay/src/runtime.mjs": + "ae8bf8b55603bab266b6fa6e9bc65c9f310a9d94a54db04e2130704e38622ffc", + "services/device-edge-relay/src/server.mjs": + "e4b051b74f934bd37322440e6a013fb6774a76607da08f9cc1e844fc109c83c1", + "deployment/device-edge-ingress-ipvlan-v1.json": + "b9ce402db0c059a76f07a8d4a34297aff2250fd1c0d1aed9970b3a88f4e75d7f", +} + +PREDECESSOR_FILE_SHA256 = { + "docker-compose.device-edge.yml": + "50dc7366d7db451935dd4d67f76ec8b342b3a94be6d22176976db910981445d0", + "services/device-edge-relay/Dockerfile": + "f2f15b7618ac2ab3a1d4dd40041d06d1e9a695edcfc168d8835f9560b4897e70", + "services/device-edge-relay/src/runtime.mjs": + "ae8bf8b55603bab266b6fa6e9bc65c9f310a9d94a54db04e2130704e38622ffc", + "services/device-edge-relay/src/server.mjs": + "e4b051b74f934bd37322440e6a013fb6774a76607da08f9cc1e844fc109c83c1", +} + +PREDECESSOR_ABSENT = { + "docker-compose.device-edge.ingress.yml", + "deployment/device-edge-ingress-ipvlan-v1.json", +} + + +class DeployError(RuntimeError): + pass + + +def die(message: str) -> None: + raise DeployError(message) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run(command, *, check=True, capture=True, cwd=None, timeout=180): + result = subprocess.run( + [str(value) for value in command], + check=False, + capture_output=capture, + text=True, + cwd=str(cwd) if cwd else None, + timeout=timeout, + ) + if check and result.returncode != 0: + detail = (result.stderr or result.stdout or "command failed").strip() + die(f"command failed: {command[0]}: {detail}") + return result + + +def docker_json(*args): + result = run([DOCKER, *args]) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as error: + die(f"Docker JSON response invalid: {error}") + + +def expected_descriptor(): + return { + "schemaVersion": "nodedc.device-edge.ingress-ipvlan.v1", + "mode": "single-nic-ipvlan-b2-relay-only", + "runtimeHost": "ndcmini12", + "component": COMPONENT, + "selectedServices": [RELAY_SERVICE], + "preservedServices": ["device-edge-backhaul", "tailnet"], + "composeProject": COMPOSE_PROJECT, + "composeFiles": [ + "docker-compose.device-edge.yml", + "docker-compose.device-edge.ingress.yml", + ], + "parentInterface": INGRESS_PARENT, + "lanSubnet": INGRESS_SUBNET, + "lanGateway": INGRESS_GATEWAY, + "ingressIpv4": INGRESS_IPV4, + "ingressIpv4Approval": INGRESS_IPV4_APPROVAL, + "ingressNetwork": INGRESS_NETWORK, + "deviceTcpListen": f"{INGRESS_IPV4}:{INGRESS_PORT}", + "hostPortPublication": "disabled", + "healthPublication": "disabled", + "privateUpstream": "device-edge-backhaul:19921", + "protocolInspection": "gateway-owned", + "identityTrust": "claimed-not-ownership-proof", + "discoveryLifecycle": "quarantine", + "commandTransport": "disabled", + "gelios": "untouched", + "amneziaHostFullTunnel": "preserved", + "routerNatFirewall": "separate-manual-gate", + "rollback": "restore-predecessor-relay-remove-unused-ingress-network", + } + + +def assert_root(): + if os.geteuid() != 0: + die("nodedc-edge-deploy must run as root") + + +def assert_regular_nonsymlink(path: Path, label: str): + if not path.exists() or path.is_symlink() or not path.is_file(): + die(f"{label} must be a regular non-symlink file") + + +def parse_manifest(raw: str): + values = {} + for line in raw.splitlines(): + if not line or "=" not in line: + die("artifact manifest is malformed") + key, value = line.split("=", 1) + if key in values or key not in {"id", "component", "type"}: + die("artifact manifest key set is invalid") + values[key] = value + if set(values) != {"id", "component", "type"}: + die("artifact manifest key set is incomplete") + if not PATCH_ID_RE.fullmatch(values["id"]): + die("artifact patch id is invalid") + if values["component"] != COMPONENT or values["type"] != ARTIFACT_TYPE: + die("artifact component/type mismatch") + return values + + +def safe_tar_member(member: tarfile.TarInfo): + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not path.parts: + die("artifact contains an unsafe path") + if not (member.isfile() or member.isdir()): + die("artifact contains a non-file/non-directory member") + lowered = {part.lower() for part in path.parts} + if any( + part.startswith(".env") + or part in { + ".git", + "node_modules", + "secrets", + "keys", + "trust", + "runtime", + "logs", + "uploads", + } + for part in lowered + ): + die("artifact contains a forbidden boundary") + if any(part.startswith("._") for part in path.parts): + die("artifact contains AppleDouble metadata") + + +def load_artifact(artifact: Path, extraction_root: Path): + artifact = artifact.resolve(strict=True) + if artifact.parent != INBOX_ROOT.resolve(strict=True): + die("artifact must be an explicit file in the Device Edge inbox") + assert_regular_nonsymlink(artifact, "artifact") + if artifact.suffix != ".tgz" or artifact.stat().st_size > MAX_ARTIFACT_BYTES: + die("artifact extension/size rejected") + + seen = set() + with tarfile.open(artifact, "r:gz") as archive: + for member in archive.getmembers(): + safe_tar_member(member) + if member.name in seen: + die("artifact contains duplicate members") + seen.add(member.name) + required = {"manifest.env", "files.txt", "payload"} + if not required.issubset(seen): + die("artifact top-level contract is incomplete") + if any(name.split("/", 1)[0] not in required for name in seen): + die("artifact contains an unexpected top-level member") + archive.extractall(extraction_root, filter="data") + + manifest = parse_manifest( + (extraction_root / "manifest.env").read_text(encoding="utf-8") + ) + entries = tuple( + line for line in + (extraction_root / "files.txt").read_text(encoding="utf-8").splitlines() + if line + ) + if entries != ENTRIES or len(entries) != len(set(entries)): + die("Device Edge artifact file selection mismatch") + payload = extraction_root / "payload" + validate_payload(payload) + return manifest, entries, payload, sha256_file(artifact), artifact + + +def validate_payload(payload: Path): + actual_files = { + path.relative_to(payload).as_posix(): sha256_file(path) + for path in payload.rglob("*") + if path.is_file() + } + if actual_files != PAYLOAD_FILE_SHA256: + die("Device Edge artifact payload digest set mismatch") + descriptor = json.loads( + (payload / "deployment/device-edge-ingress-ipvlan-v1.json") + .read_text(encoding="utf-8") + ) + if descriptor != expected_descriptor(): + die("Device Edge ingress descriptor mismatch") + + +def journal_records(path: Path): + if not path.exists(): + return [] + records = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + die(f"journal is malformed: {path}") + return records + + +def assert_new_identity(patch_id: str, artifact_sha256: str): + records = journal_records(APPLIED_JOURNAL) + journal_records(FAILED_JOURNAL) + if any(record.get("patch") == patch_id for record in records): + die("Device Edge patch id is terminally recorded") + if any(record.get("sha256") == artifact_sha256 for record in records): + die("Device Edge artifact digest is terminally recorded") + + +def current_source_state(): + state = {} + for relative, expected in PREDECESSOR_FILE_SHA256.items(): + path = LIVE_ROOT / relative + assert_regular_nonsymlink(path, f"predecessor {relative}") + state[relative] = sha256_file(path) + if state[relative] != expected: + die(f"Device Edge predecessor drift: {relative}") + for relative in PREDECESSOR_ABSENT: + if (LIVE_ROOT / relative).exists(): + die(f"Device Edge predecessor unexpected path: {relative}") + return state + + +def inspect_container(name: str): + response = docker_json("inspect", name) + if len(response) != 1: + die(f"container inspect cardinality mismatch: {name}") + return response[0] + + +def container_health(container): + health = container.get("State", {}).get("Health") + return health.get("Status") if health else None + + +def preserved_runtime_snapshot(): + snapshot = {} + for name in (BACKHAUL_CONTAINER, TAILNET_CONTAINER): + container = inspect_container(name) + if container.get("State", {}).get("Status") != "running": + die(f"preserved Device Edge service is not running: {name}") + if name == BACKHAUL_CONTAINER and container_health(container) != "healthy": + die("Device Edge backhaul is not healthy") + snapshot[name] = { + "Id": container.get("Id"), + "Image": container.get("Image"), + "StartedAt": container.get("State", {}).get("StartedAt"), + "RestartCount": container.get("RestartCount"), + "PortBindings": container.get("HostConfig", {}).get("PortBindings"), + } + return snapshot + + +def assert_preserved_runtime(snapshot): + current_snapshot = preserved_runtime_snapshot() + for name, expected in snapshot.items(): + current = current_snapshot[name] + if current != expected: + die(f"preserved Device Edge runtime changed: {name}") + + +def validate_predecessor_runtime(): + relay = inspect_container(RELAY_CONTAINER) + if relay.get("State", {}).get("Status") != "running": + die("Device Edge predecessor relay is not running") + if container_health(relay) != "healthy": + die("Device Edge predecessor relay is not healthy") + environment = set(relay.get("Config", {}).get("Env") or []) + if "DEVICE_EDGE_RELAY_INGRESS_ENABLED=false" not in environment: + die("Device Edge predecessor ingress is not disabled") + bindings = relay.get("HostConfig", {}).get("PortBindings") or {} + expected = {"18221/tcp": [{"HostIp": "127.0.0.1", "HostPort": "18221"}]} + if bindings != expected: + die("Device Edge predecessor host publication mismatch") + if INGRESS_NETWORK in (relay.get("NetworkSettings", {}).get("Networks") or {}): + die("Device Edge predecessor unexpectedly uses ingress network") + + +def validate_host_network_boundary(): + if socket.gethostname() != "ndcmini12": + die("Device Edge runtime host mismatch") + route = run(["/usr/sbin/ip", "-4", "route", "show"]).stdout + for line in ( + "0.0.0.0/1 dev amn0 metric 1", + "128.0.0.0/1 dev amn0 metric 1", + "default via 192.168.68.1 dev enp1s0f0", + "192.168.68.0/22 dev enp1s0f0", + ): + if line not in route: + die(f"Device Edge host route boundary mismatch: {line}") + if run(["/usr/bin/systemctl", "is-active", "AmneziaVPN.service"]).stdout.strip() != "active": + die("AmneziaVPN must remain active for this transition") + interface = run([ + "/usr/sbin/ip", "-4", "-brief", "address", "show", "dev", INGRESS_PARENT, + ]).stdout + if "192.168.68.54/22" not in interface or "UP" not in interface: + die("Device Edge physical interface boundary mismatch") + + +def arp_duplicate_detected(target_ip: str, interface: str, attempts=3): + protocol = 0x0806 + raw = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(protocol)) + try: + raw.bind((interface, 0)) + source_mac = raw.getsockname()[4] + target = socket.inet_aton(target_ip) + ethernet = b"\xff" * 6 + source_mac + struct.pack("!H", protocol) + arp = struct.pack( + "!HHBBH6s4s6s4s", + 1, + 0x0800, + 6, + 4, + 1, + source_mac, + b"\x00" * 4, + b"\x00" * 6, + target, + ) + raw.setblocking(False) + for _ in range(attempts): + raw.send(ethernet + arp) + deadline = time.monotonic() + 0.7 + while time.monotonic() < deadline: + ready, _, _ = select.select([raw], [], [], deadline - time.monotonic()) + if not ready: + break + packet = raw.recv(2048) + if len(packet) < 42 or packet[12:14] != b"\x08\x06": + continue + if packet[28:32] == target and packet[22:28] != source_mac: + return True + return False + finally: + raw.close() + + +def preflight(manifest, artifact_sha256): + if not INGRESS_IPV4_APPROVED: + die("Device Edge ingress IPv4 approval is not granted") + if INGRESS_IPV4_APPROVAL != "approved-outside-dhcp-pool": + die("Device Edge ingress IPv4 approval contract mismatch") + assert_new_identity(manifest["id"], artifact_sha256) + current_source_state() + validate_predecessor_runtime() + preserved = preserved_runtime_snapshot() + validate_host_network_boundary() + if arp_duplicate_detected(INGRESS_IPV4, INGRESS_PARENT): + die("Device Edge ingress IPv4 duplicate detected") + if run([DOCKER, "network", "inspect", INGRESS_NETWORK], check=False).returncode == 0: + die("Device Edge ingress network already exists") + return preserved + + +def compose_command(*args, baseline=False): + command = [ + DOCKER, + "compose", + "--project-name", + COMPOSE_PROJECT, + "--file", + str(BASE_COMPOSE), + ] + if not baseline: + command.extend(["--file", str(INGRESS_COMPOSE)]) + command.extend(args) + return command + + +def ensure_state_directories(): + for path in ( + APPLIED_ROOT, + FAILED_ROOT, + BACKUP_ROOT, + APPLIED_JOURNAL.parent, + ): + path.mkdir(parents=True, exist_ok=True, mode=0o750) + os.chmod(path, 0o750) + + +def acquire_lock(): + ensure_state_directories() + try: + descriptor = os.open( + DEPLOY_LOCK, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + except FileExistsError: + die("Device Edge deploy lock is present") + os.write(descriptor, f"pid={os.getpid()}\n".encode()) + os.close(descriptor) + + +def release_lock(): + try: + DEPLOY_LOCK.unlink() + except FileNotFoundError: + pass + + +def create_backup(patch_id: str): + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + backup_id = f"{patch_id}-{timestamp}" + backup = BACKUP_ROOT / backup_id + backup.mkdir(parents=False, mode=0o750) + present = [] + absent = [] + for relative in ENTRIES: + source = LIVE_ROOT / relative + target = backup / "payload" / relative + if not source.exists(): + absent.append(relative) + continue + present.append(relative) + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target, symlinks=False) + else: + shutil.copy2(source, target, follow_symlinks=False) + (backup / "backup.json").write_text(json.dumps({ + "schemaVersion": "nodedc.device-edge.backup.v1", + "patch": patch_id, + "present": present, + "absent": absent, + }, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return backup_id, backup + + +def publish_payload(payload: Path): + for relative in ENTRIES: + source = payload / relative + target = LIVE_ROOT / relative + if target.exists(): + if target.is_dir(): + shutil.rmtree(target) + else: + target.unlink() + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target, symlinks=False) + else: + shutil.copy2(source, target, follow_symlinks=False) + + +def restore_backup(backup: Path): + descriptor = json.loads((backup / "backup.json").read_text(encoding="utf-8")) + for relative in ENTRIES: + target = LIVE_ROOT / relative + if target.exists(): + if target.is_dir(): + shutil.rmtree(target) + else: + target.unlink() + for relative in descriptor["present"]: + source = backup / "payload" / relative + target = LIVE_ROOT / relative + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target, symlinks=False) + else: + shutil.copy2(source, target, follow_symlinks=False) + + +def build_relay(): + run([ + DOCKER, + "build", + "--no-cache", + "--network=host", + "--file", + "services/device-edge-relay/Dockerfile", + "--tag", + RELAY_IMAGE, + ".", + ], cwd=LIVE_ROOT, timeout=900, capture=False) + + +def wait_healthy(name: str, timeout_seconds=150): + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + try: + container = inspect_container(name) + except DeployError: + time.sleep(2) + continue + if ( + container.get("State", {}).get("Status") == "running" + and container_health(container) == "healthy" + ): + return container + if container.get("State", {}).get("Status") in {"exited", "dead"}: + die(f"container stopped before health acceptance: {name}") + time.sleep(2) + die(f"container health timeout: {name}") + + +def validate_network_runtime(relay): + networks = relay.get("NetworkSettings", {}).get("Networks") or {} + if set(networks) != {"nodedc-device-edge-private", INGRESS_NETWORK}: + die("Device Edge relay network set mismatch") + if networks[INGRESS_NETWORK].get("IPAddress") != INGRESS_IPV4: + die("Device Edge relay IPvlan address mismatch") + response = docker_json("network", "inspect", INGRESS_NETWORK) + if len(response) != 1: + die("Device Edge ingress network cardinality mismatch") + network = response[0] + if network.get("Driver") != "ipvlan" or network.get("Internal") is True: + die("Device Edge ingress network driver mismatch") + options = network.get("Options") or {} + if options.get("parent") != INGRESS_PARENT or options.get("ipvlan_mode") != "l2": + die("Device Edge ingress network option mismatch") + configs = network.get("IPAM", {}).get("Config") or [] + if len(configs) != 1: + die("Device Edge ingress IPAM cardinality mismatch") + if configs[0].get("Subnet") != INGRESS_SUBNET or configs[0].get("Gateway") != INGRESS_GATEWAY: + die("Device Edge ingress IPAM mismatch") + + +def validate_relay_runtime(preserved): + relay = wait_healthy(RELAY_CONTAINER) + if relay.get("Config", {}).get("User") != "1000:1000": + die("Device Edge relay user mismatch") + host = relay.get("HostConfig", {}) + if host.get("ReadonlyRootfs") is not True or host.get("Privileged") is not False: + die("Device Edge relay filesystem/privilege mismatch") + if set(host.get("CapDrop") or []) != {"ALL"}: + die("Device Edge relay capability mismatch") + if host.get("PortBindings") not in ({}, None): + die("Device Edge relay host port publication detected") + environment = set(relay.get("Config", {}).get("Env") or []) + required = { + "DEVICE_EDGE_RELAY_HEALTH_HOST=127.0.0.1", + "DEVICE_EDGE_RELAY_INGRESS_ENABLED=true", + "DEVICE_EDGE_RELAY_TCP_HOST=0.0.0.0", + "DEVICE_EDGE_RELAY_TCP_PORT=9921", + "DEVICE_EDGE_RELAY_UPSTREAM_HOST=device-edge-backhaul", + "DEVICE_EDGE_RELAY_UPSTREAM_PORT=19921", + } + if not required.issubset(environment): + die("Device Edge relay environment mismatch") + validate_network_runtime(relay) + health_result = run([ + DOCKER, + "exec", + RELAY_CONTAINER, + "node", + "-e", + "fetch('http://127.0.0.1:18221/healthz').then(async r=>{if(!r.ok)process.exit(2);console.log(await r.text())}).catch(()=>process.exit(3))", + ]) + try: + health = json.loads(health_result.stdout) + except json.JSONDecodeError: + die("Device Edge relay health JSON invalid") + expected_health = { + "ok": True, + "service": "nodedc-device-edge-relay", + "ingress": "relay-only", + "protocolInspection": "disabled", + "commandTransport": "disabled", + } + for key, expected in expected_health.items(): + if health.get(key) != expected: + die(f"Device Edge relay health contract mismatch: {key}") + run([ + DOCKER, + "exec", + RELAY_CONTAINER, + "node", + "-e", + "const n=require('node:net');const s=n.connect({host:'device-edge-backhaul',port:19921});s.setTimeout(5000);s.once('connect',()=>{s.destroy();process.exit(0)});s.once('timeout',()=>process.exit(2));s.once('error',()=>process.exit(3))", + ]) + validate_host_network_boundary() + assert_preserved_runtime(preserved) + + +def write_journal(path: Path, record): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def archive_artifact(artifact: Path, destination_root: Path): + destination = destination_root / artifact.name + if destination.exists(): + die("Device Edge artifact archive collision") + os.replace(artifact, destination) + return destination + + +def rollback(backup: Path, preserved): + restore_backup(backup) + run(compose_command( + "up", + "--detach", + "--no-deps", + "--force-recreate", + "--pull", + "never", + RELAY_SERVICE, + baseline=True, + ), cwd=LIVE_ROOT, timeout=300, capture=False) + wait_healthy(RELAY_CONTAINER) + run([DOCKER, "network", "rm", INGRESS_NETWORK], check=False) + current_source_state() + validate_predecessor_runtime() + assert_preserved_runtime(preserved) + + +def plan_artifact(artifact_argument: str): + assert_root() + artifact = Path(artifact_argument) + with tempfile.TemporaryDirectory(prefix="nodedc-edge-plan-") as directory: + manifest, entries, _payload, digest, resolved = load_artifact( + artifact, + Path(directory), + ) + preflight(manifest, digest) + print("== plan ==") + print(f"artifact={resolved.name}") + print(f"sha256={digest}") + print(f"id={manifest['id']}") + print(f"component={COMPONENT}") + print(f"type={ARTIFACT_TYPE}") + print(f"payload_root={LIVE_ROOT}") + print(f"compose_root={LIVE_ROOT}") + print(f"compose_project={COMPOSE_PROJECT}") + print("compose_files=docker-compose.device-edge.yml docker-compose.device-edge.ingress.yml") + print("build=/usr/bin/docker build --no-cache --network=host -f services/device-edge-relay/Dockerfile -t nodedc/device-edge-relay:local .") + print("services=device-edge-relay") + print("preserved_services=device-edge-backhaul tailnet") + print(f"device_edge_ingress=ipvlan:l2:{INGRESS_PARENT}:{INGRESS_IPV4}:{INGRESS_PORT}/tcp") + print(f"device_edge_lan={INGRESS_SUBNET}:gateway:{INGRESS_GATEWAY}") + print(f"device_edge_ingress_ipv4_approval={INGRESS_IPV4_APPROVAL}") + print("device_edge_host_port_publication=disabled") + print("device_edge_health_publication=disabled") + print("device_edge_private_upstream=device-edge-backhaul:19921") + print("device_edge_command_transport=disabled") + print("device_edge_discovery_lifecycle=quarantine") + print("device_edge_gelios=untouched") + print("device_edge_amnezia=preserved:active:host-full-tunnel") + print("device_edge_router_nat_firewall=unchanged") + print("device_edge_rollback=restore-predecessor-relay-remove-unused-ingress-network") + print("state=new") + print("== files ==") + for entry in entries: + print(f" {entry}") + + +def apply_artifact(artifact_argument: str): + assert_root() + artifact = Path(artifact_argument) + acquire_lock() + manifest = None + digest = None + resolved = None + backup_id = None + backup = None + preserved = None + try: + with tempfile.TemporaryDirectory(prefix="nodedc-edge-apply-") as directory: + manifest, _entries, payload, digest, resolved = load_artifact( + artifact, + Path(directory), + ) + preserved = preflight(manifest, digest) + backup_id, backup = create_backup(manifest["id"]) + publish_payload(payload) + build_relay() + run(compose_command( + "up", + "--detach", + "--no-deps", + "--force-recreate", + "--pull", + "never", + RELAY_SERVICE, + ), cwd=LIVE_ROOT, timeout=300, capture=False) + validate_relay_runtime(preserved) + archived = archive_artifact(resolved, APPLIED_ROOT) + write_journal(APPLIED_JOURNAL, { + "status": "ok", + "patch": manifest["id"], + "component": COMPONENT, + "sha256": digest, + "artifact": archived.name, + "backup": backup_id, + "appliedAt": datetime.now(timezone.utc).isoformat(), + }) + print( + f"deploy-ok patch={manifest['id']} component={COMPONENT} " + f"backup={backup_id}" + ) + except Exception as error: + rollback_status = "not-started" + if backup is not None and preserved is not None: + try: + rollback(backup, preserved) + rollback_status = "ok" + except Exception as rollback_error: + rollback_status = f"failed:{type(rollback_error).__name__}" + if resolved is not None and resolved.exists(): + failed_name = ( + FAILED_ROOT + / f"{resolved.name}.{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}" + ) + os.replace(resolved, failed_name) + if manifest is not None and digest is not None: + write_journal(FAILED_JOURNAL, { + "status": "failed", + "patch": manifest["id"], + "component": COMPONENT, + "sha256": digest, + "backup": backup_id, + "rollback": rollback_status, + "error": type(error).__name__, + "failedAt": datetime.now(timezone.utc).isoformat(), + }) + if rollback_status.startswith("failed"): + die(f"apply failed and rollback failed: {error}") + die(f"apply failed; automatic rollback={rollback_status}: {error}") + finally: + release_lock() + + +def verify_install(): + assert_root() + path = RUNNER_PATH if RUNNER_PATH.exists() else Path(__file__).resolve() + assert_regular_nonsymlink(path, "runner") + docker_version = run([DOCKER, "version", "--format", "{{.Server.Version}}"]).stdout.strip() + compose_version = run([DOCKER, "compose", "version", "--short"]).stdout.strip() + print(f"path={path}") + print(f"sha256={sha256_file(path)}") + print(f"python={sys.version.split()[0]}") + print(f"docker={docker_version}") + print(f"compose={compose_version}") + print(f"device_edge_ingress_ipv4={INGRESS_IPV4}") + print(f"device_edge_ingress_ipv4_approval={INGRESS_IPV4_APPROVAL}") + print("verify-install-ok") + + +def main(arguments): + if len(arguments) == 1 and arguments[0] == "verify-install": + verify_install() + return 0 + if len(arguments) == 2 and arguments[0] == "plan": + plan_artifact(arguments[1]) + return 0 + if len(arguments) == 2 and arguments[0] == "apply": + apply_artifact(arguments[1]) + return 0 + print( + "usage: nodedc-edge-deploy verify-install | plan | apply ", + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + try: + raise SystemExit(main(sys.argv[1:])) + except DeployError as error: + print(f"ERROR: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/infra/deploy-runner/test_device_edge_ingress_artifact.py b/infra/deploy-runner/test_device_edge_ingress_artifact.py new file mode 100644 index 0000000..f754487 --- /dev/null +++ b/infra/deploy-runner/test_device_edge_ingress_artifact.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +SCRIPT_DIR = Path(__file__).resolve().parent +BUILDER = SCRIPT_DIR / "build-device-edge-ingress-artifact.mjs" +RUNNER_PATH = SCRIPT_DIR / "nodedc-edge-deploy" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_edge_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DeviceEdgeIngressArtifactTest(unittest.TestCase): + def build(self, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + return subprocess.run( + ["node", str(BUILDER), patch_id], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + def test_builder_is_deterministic_narrow_and_secret_free(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-edge-artifact-", + ) as directory: + artifact_dir = Path(directory) + patch_id = "device-edge-ingress-ipvlan-unit-001" + first_result = self.build(artifact_dir, patch_id) + self.assertEqual(first_result.returncode, 0, first_result.stderr) + first = json.loads(first_result.stdout) + first_bytes = Path(first["artifact"]).read_bytes() + second_result = self.build(artifact_dir, patch_id) + self.assertEqual(second_result.returncode, 0, second_result.stderr) + second = json.loads(second_result.stdout) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual(first["sha256"], second["sha256"]) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual(first["component"], "device-edge") + self.assertEqual(first["entries"], list(RUNNER.ENTRIES)) + self.assertEqual(first["services"], ["device-edge-relay"]) + self.assertEqual( + first["ingress"]["ipv4Approval"], + "approved-outside-dhcp-pool", + ) + + with tarfile.open(first["artifact"], "r:gz") as archive: + members = archive.getmembers() + names = {member.name for member in members} + manifest = archive.extractfile("manifest.env").read().decode() + files = archive.extractfile("files.txt").read().decode().splitlines() + payload_bytes = b"\n".join( + archive.extractfile(member).read() + for member in members + if member.isfile() + ) + + self.assertEqual( + manifest, + f"id={patch_id}\ncomponent=device-edge\ntype=app-overlay\n", + ) + self.assertEqual(files, list(RUNNER.ENTRIES)) + self.assertIn( + "payload/docker-compose.device-edge.ingress.yml", + names, + ) + self.assertNotIn(b"PRIVATE KEY", payload_bytes) + self.assertFalse(any( + "/test/" in name + or "/secrets/" in name + or "/keys/" in name + or "/trust/" in name + or "/node_modules/" in name + or Path(name).name.startswith(".env") + for name in names + )) + + def test_production_builder_accepts_the_explicitly_approved_address(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-edge-address-gate-", + ) as directory: + result = self.build( + Path(directory), + "device-edge-ingress-ipvlan-20260804-001", + ) + self.assertEqual(result.returncode, 0, result.stderr) + built = json.loads(result.stdout) + self.assertEqual( + built["ingress"]["ipv4Approval"], + "approved-outside-dhcp-pool", + ) + self.assertTrue(Path(built["artifact"]).is_file()) + + def test_runner_loads_exact_artifact_and_enters_runtime_preflight(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-edge-runner-load-", + ) as directory: + workspace = Path(directory) + inbox = workspace / "inbox" + inbox.mkdir() + result = self.build( + inbox, + "device-edge-ingress-ipvlan-20260804-002", + ) + self.assertEqual(result.returncode, 0, result.stderr) + artifact = Path(json.loads(result.stdout)["artifact"]) + extracted = workspace / "extracted" + extracted.mkdir() + + old_inbox = RUNNER.INBOX_ROOT + RUNNER.INBOX_ROOT = inbox + try: + manifest, entries, payload, digest, resolved = ( + RUNNER.load_artifact(artifact, extracted) + ) + finally: + RUNNER.INBOX_ROOT = old_inbox + + self.assertEqual(manifest["component"], "device-edge") + self.assertEqual(entries, RUNNER.ENTRIES) + self.assertEqual(resolved, artifact.resolve()) + self.assertEqual(digest, hashlib.sha256(artifact.read_bytes()).hexdigest()) + self.assertEqual( + json.loads( + (payload / "deployment/device-edge-ingress-ipvlan-v1.json") + .read_text(encoding="utf-8") + ), + RUNNER.expected_descriptor(), + ) + preserved = { + RUNNER.BACKHAUL_CONTAINER: {"Id": "backhaul"}, + RUNNER.TAILNET_CONTAINER: {"Id": "tailnet"}, + } + with patch.object(RUNNER, "assert_new_identity"), patch.object( + RUNNER, + "current_source_state", + ), patch.object(RUNNER, "validate_predecessor_runtime"), patch.object( + RUNNER, + "preserved_runtime_snapshot", + return_value=preserved, + ), patch.object(RUNNER, "validate_host_network_boundary"), patch.object( + RUNNER, + "arp_duplicate_detected", + return_value=False, + ), patch.object( + RUNNER, + "run", + return_value=subprocess.CompletedProcess([], 1, "", ""), + ): + self.assertEqual(RUNNER.preflight(manifest, digest), preserved) + + def test_backup_restore_preserves_exact_predecessor_partition(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-edge-backup-", + ) as directory: + workspace = Path(directory) + live = workspace / "live" + backups = workspace / "backups" + live.mkdir() + backups.mkdir() + for relative in RUNNER.ENTRIES: + if relative in RUNNER.PREDECESSOR_ABSENT: + continue + target = live / relative + if relative.endswith("/src"): + target.mkdir(parents=True) + (target / "server.mjs").write_text("old\n", encoding="utf-8") + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"old:{relative}\n", encoding="utf-8") + + old_live = RUNNER.LIVE_ROOT + old_backups = RUNNER.BACKUP_ROOT + RUNNER.LIVE_ROOT = live + RUNNER.BACKUP_ROOT = backups + try: + _backup_id, backup = RUNNER.create_backup("unit-backup") + for relative in RUNNER.ENTRIES: + target = live / relative + if target.exists(): + if target.is_dir(): + import shutil + shutil.rmtree(target) + else: + target.unlink() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("candidate\n", encoding="utf-8") + RUNNER.restore_backup(backup) + finally: + RUNNER.LIVE_ROOT = old_live + RUNNER.BACKUP_ROOT = old_backups + + for relative in RUNNER.PREDECESSOR_ABSENT: + self.assertFalse((live / relative).exists()) + self.assertEqual( + (live / "docker-compose.device-edge.yml").read_text(), + "old:docker-compose.device-edge.yml\n", + ) + self.assertEqual( + (live / "services/device-edge-relay/src/server.mjs").read_text(), + "old\n", + ) + + def test_runner_selection_and_compose_commands_are_exact(self): + self.assertTrue(RUNNER.INGRESS_IPV4_APPROVED) + self.assertEqual( + RUNNER.INGRESS_IPV4_APPROVAL, + "approved-outside-dhcp-pool", + ) + self.assertEqual(RUNNER.RELAY_SERVICE, "device-edge-relay") + self.assertEqual( + RUNNER.expected_descriptor()["preservedServices"], + ["device-edge-backhaul", "tailnet"], + ) + self.assertEqual( + RUNNER.compose_command( + "up", + "--detach", + "--no-deps", + "--force-recreate", + "--pull", + "never", + RUNNER.RELAY_SERVICE, + ), + [ + RUNNER.DOCKER, + "compose", + "--project-name", + RUNNER.COMPOSE_PROJECT, + "--file", + str(RUNNER.BASE_COMPOSE), + "--file", + str(RUNNER.INGRESS_COMPOSE), + "up", + "--detach", + "--no-deps", + "--force-recreate", + "--pull", + "never", + RUNNER.RELAY_SERVICE, + ], + ) + self.assertNotIn("down", RUNNER_PATH.read_text(encoding="utf-8")) + + def test_preserved_runtime_is_compared_from_one_atomic_snapshot(self): + expected = { + RUNNER.BACKHAUL_CONTAINER: {"Id": "backhaul"}, + RUNNER.TAILNET_CONTAINER: {"Id": "tailnet"}, + } + with patch.object( + RUNNER, + "preserved_runtime_snapshot", + return_value=expected, + ) as snapshot: + RUNNER.assert_preserved_runtime(expected) + snapshot.assert_called_once_with() + + def test_network_acceptance_rejects_any_non_ipvlan_substitution(self): + relay = { + "NetworkSettings": { + "Networks": { + "nodedc-device-edge-private": {"IPAddress": "172.18.0.4"}, + RUNNER.INGRESS_NETWORK: {"IPAddress": RUNNER.INGRESS_IPV4}, + }, + }, + } + accepted_network = [{ + "Driver": "ipvlan", + "Internal": False, + "Options": { + "parent": RUNNER.INGRESS_PARENT, + "ipvlan_mode": "l2", + }, + "IPAM": { + "Config": [{ + "Subnet": RUNNER.INGRESS_SUBNET, + "Gateway": RUNNER.INGRESS_GATEWAY, + }], + }, + }] + with patch.object(RUNNER, "docker_json", return_value=accepted_network): + RUNNER.validate_network_runtime(relay) + + rejected_network = json.loads(json.dumps(accepted_network)) + rejected_network[0]["Driver"] = "bridge" + with patch.object(RUNNER, "docker_json", return_value=rejected_network): + with self.assertRaisesRegex( + RUNNER.DeployError, + "ingress network driver mismatch", + ): + RUNNER.validate_network_runtime(relay) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/infra/deploy-runner/test_device_plane_b2_discovery_ingress_artifact.py b/infra/deploy-runner/test_device_plane_b2_discovery_ingress_artifact.py index ff406c5..217b24a 100644 --- a/infra/deploy-runner/test_device_plane_b2_discovery_ingress_artifact.py +++ b/infra/deploy-runner/test_device_plane_b2_discovery_ingress_artifact.py @@ -98,6 +98,22 @@ class DevicePlaneB2DiscoveryIngressArtifactTest(unittest.TestCase): .read() .decode("utf-8") ) + edge_manifest = json.loads( + archive.extractfile( + "payload/services/device-edge-relay/package.json" + ) + .read() + .decode("utf-8") + ) + dockerignore = archive.extractfile( + "payload/.dockerignore" + ).read().decode("utf-8") + core_dockerfile = archive.extractfile( + "payload/services/device-control-core/Dockerfile" + ).read().decode("utf-8") + gateway_dockerfile = archive.extractfile( + "payload/services/device-gateway/Dockerfile" + ).read().decode("utf-8") self.assertEqual(files, first["entries"]) self.assertEqual( @@ -108,6 +124,24 @@ class DevicePlaneB2DiscoveryIngressArtifactTest(unittest.TestCase): descriptor, RUNNER.expected_device_plane_b2_discovery_ingress_descriptor(), ) + self.assertEqual( + edge_manifest["name"], + "@nodedc/device-edge-relay", + ) + self.assertIn("**/*.prev-*", dockerignore.splitlines()) + self.assertIn("**/*.next-*", dockerignore.splitlines()) + for dockerfile in (core_dockerfile, gateway_dockerfile): + self.assertNotIn("COPY packages ./packages", dockerfile) + self.assertIn( + "COPY packages/device-protocol-contract " + "./packages/device-protocol-contract", + dockerfile, + ) + self.assertIn( + "COPY packages/arusnavi-b2-adapter " + "./packages/arusnavi-b2-adapter", + dockerfile, + ) def test_compose_opens_only_discovery_tcp_and_preserves_database(self): compose = COMPOSE.read_text(encoding="utf-8") @@ -153,6 +187,48 @@ class DevicePlaneB2DiscoveryIngressArtifactTest(unittest.TestCase): "disabled", ) + def test_ingress_preflight_requires_terminal_recovery_marker(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-b2-recovery-barrier-", + ) as directory: + workspace = Path(directory) + built = self.build( + workspace, + "device-plane-b2-discovery-ingress-unit-recovery-barrier", + ) + extracted = workspace / "extracted" + extracted.mkdir() + _manifest, _entries, payload = RUNNER.load_artifact( + Path(built["artifact"]), + extracted, + ) + + def has_patch(patch_id): + return ( + patch_id + == RUNNER.DEVICE_PLANE_B2_DISCOVERY_INGRESS_PREDECESSOR_PATCH_ID + ) + + with ( + mock.patch.object( + RUNNER, + "state_has_patch_id", + side_effect=has_patch, + ), + mock.patch.object( + RUNNER, + "state_has_sha", + return_value=True, + ), + ): + with self.assertRaisesRegex( + RUNNER.DeployError, + "rollback recovery patch is not applied", + ): + RUNNER.validate_device_plane_b2_discovery_ingress_evidence( + payload + ) + def test_runtime_acceptance_preserves_postgres_and_replaces_stateless(self): before = { "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", diff --git a/infra/deploy-runner/test_device_plane_b2_discovery_loopback_recovery_artifact.py b/infra/deploy-runner/test_device_plane_b2_discovery_loopback_recovery_artifact.py new file mode 100644 index 0000000..d67a394 --- /dev/null +++ b/infra/deploy-runner/test_device_plane_b2_discovery_loopback_recovery_artifact.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +BUILDER = ( + SCRIPT_DIR + / "build-device-plane-b2-discovery-loopback-recovery-artifact.mjs" +) +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_device_plane_b2_recovery_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DevicePlaneB2DiscoveryLoopbackRecoveryArtifactTest( + unittest.TestCase, +): + def build(self, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + ["node", str(BUILDER), patch_id], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_marker_only_exact_and_deterministic(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-b2-recovery-artifact-", + ) as directory: + artifact_dir = Path(directory) + patch_id = RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_PATCH_ID + first = self.build(artifact_dir, patch_id) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build(artifact_dir, patch_id) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual(first["build"], []) + self.assertEqual(first["services"], []) + self.assertEqual( + first["entries"], + list( + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ENTRIES + ), + ) + + with tarfile.open(first["artifact"], "r:gz") as archive: + names = { + member.name + for member in archive.getmembers() + if member.isfile() + } + descriptor = json.loads( + archive.extractfile( + "payload/" + + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL + ) + .read() + .decode("utf-8") + ) + self.assertEqual( + names, + { + "manifest.env", + "files.txt", + "payload/" + + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, + }, + ) + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_b2_discovery_rollback_recovery_descriptor(), + ) + entries = ( + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ENTRIES + ) + self.assertEqual( + RUNNER.component_services("device-plane", entries), + (), + ) + self.assertEqual( + RUNNER.component_builds("device-plane", entries), + (), + ) + + def test_recovery_preflight_requires_exact_failed_evidence(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-b2-recovery-preflight-", + ) as directory: + workspace = Path(directory) + artifacts = workspace / "artifacts" + failed_root = workspace / "failed" + backups_root = workspace / "backups" + state_root = workspace / "state" + temp_root = workspace / "tmp" + live_root = workspace / "live" + for path in ( + artifacts, + failed_root, + backups_root, + state_root, + temp_root, + live_root, + ): + path.mkdir(parents=True, exist_ok=True) + + recovery = self.build( + artifacts, + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_PATCH_ID, + ) + extracted = workspace / "extracted" + extracted.mkdir() + _manifest, entries, payload = RUNNER.load_artifact( + Path(recovery["artifact"]), + extracted, + ) + self.assertEqual( + tuple(entries), + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_ENTRIES, + ) + + failed_artifact = ( + failed_root / RUNNER.DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT + ) + failed_artifact.write_bytes(b"failed-b2-artifact-fixture\n") + failed_sha = RUNNER.DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT_SHA256 + + backup = ( + backups_root + / RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_ID + ) + backup.mkdir() + backup_names = ( + "manifest.env", + "files.txt", + "existing-files.txt", + "missing-files.txt", + "runtime-before.json", + "source-before.tgz", + ) + for name in backup_names: + (backup / name).write_text( + f"fixture:{name}\n", + encoding="utf-8", + ) + backup_hashes = { + name: hashlib.sha256((backup / name).read_bytes()).hexdigest() + for name in backup_names + } + + (state_root / "failed.jsonl").write_text( + json.dumps({ + "artifact": RUNNER.DEVICE_PLANE_B2_DISCOVERY_FAILED_ARTIFACT, + "backup_id": ( + RUNNER.DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_ID + ), + "component": "device-plane", + "id": RUNNER.DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID, + "message": ( + "Command '['/usr/local/bin/docker', 'build', " + "'--no-cache', '--network=host', '-f', " + "'services/device-control-core/Dockerfile', '-t', " + "'nodedc/device-control-core:local', '.']' returned " + "non-zero exit status 1." + ), + "rollback_status": "failed:CalledProcessError", + "sha256": failed_sha, + "started_apply": True, + "status": "failed", + }) + + "\n", + encoding="utf-8", + ) + + runtime = {"accepted": True} + failed_manifest = { + "id": RUNNER.DEVICE_PLANE_B2_DISCOVERY_FAILED_PATCH_ID, + "component": "device-plane", + "type": "app-overlay", + } + with ( + mock.patch.object(RUNNER, "BACKUPS_DIR", backups_root), + mock.patch.object(RUNNER, "FAILED_DIR", failed_root), + mock.patch.object( + RUNNER, + "FAILED_STATE_FILE", + state_root / "failed.jsonl", + ), + mock.patch.object(RUNNER, "TMP_DIR", temp_root), + mock.patch.object( + RUNNER, + "component_root", + return_value=live_root, + ), + mock.patch.object( + RUNNER, + "DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_BACKUP_SHA256", + backup_hashes, + ), + mock.patch.object( + RUNNER, + "sha256_file", + side_effect=lambda path: ( + failed_sha + if Path(path) == failed_artifact + else hashlib.sha256(Path(path).read_bytes()).hexdigest() + ), + ), + mock.patch.object( + RUNNER, + "load_artifact", + return_value=( + failed_manifest, + list(RUNNER.DEVICE_PLANE_B2_DISCOVERY_FAILED_ENTRIES), + workspace / "unused-payload", + ), + ), + mock.patch.object( + RUNNER, + "validate_device_plane_foundation_network_publication_installed_source", + ), + mock.patch.object( + RUNNER, + "validate_device_plane_foundation_runtime", + return_value=runtime, + ), + mock.patch.object(RUNNER, "assert_loopback_tcp_port_closed"), + ): + evidence = ( + RUNNER.validate_device_plane_b2_discovery_rollback_recovery_evidence( + payload + ) + ) + self.assertEqual( + evidence["mode"], + "failed-b2-loopback-build-reconciliation", + ) + self.assertEqual(evidence["runtime"], runtime) + + def test_build_failure_does_not_mark_runtime_started(self): + marker = mock.Mock() + entries = RUNNER.DEVICE_PLANE_B2_DISCOVERY_INGRESS_ENTRIES + services = ("device-control-core", "device-gateway") + failure = subprocess.CalledProcessError(1, ["docker", "build"]) + with ( + mock.patch.object(RUNNER, "run_build", side_effect=failure), + mock.patch.object(RUNNER, "prepare_component_runtime") as prepare, + mock.patch.object(RUNNER, "run_compose") as compose, + ): + with self.assertRaises(subprocess.CalledProcessError): + RUNNER.run_device_plane_runtime_for_apply( + entries, + services, + marker, + ) + marker.assert_not_called() + prepare.assert_not_called() + compose.assert_not_called() + + def test_compose_failure_is_marked_after_build_and_prepare(self): + events = [] + + def mark(): + events.append("mark") + + with ( + mock.patch.object( + RUNNER, + "run_build", + side_effect=lambda *_args: events.append("build"), + ), + mock.patch.object( + RUNNER, + "prepare_component_runtime", + side_effect=lambda *_args: events.append("prepare"), + ), + mock.patch.object( + RUNNER, + "run_compose", + side_effect=RuntimeError("compose failed"), + ), + ): + with self.assertRaisesRegex(RuntimeError, "compose failed"): + RUNNER.run_device_plane_runtime_for_apply( + RUNNER.DEVICE_PLANE_B2_DISCOVERY_INGRESS_ENTRIES, + ("device-control-core", "device-gateway"), + mark, + ) + self.assertEqual(events, ["build", "prepare", "mark"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/deploy-runner/test_device_plane_backhaul_target_artifact.py b/infra/deploy-runner/test_device_plane_backhaul_target_artifact.py new file mode 100644 index 0000000..d9b0bb2 --- /dev/null +++ b/infra/deploy-runner/test_device_plane_backhaul_target_artifact.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +import base64 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +BUILDER = SCRIPT_DIR / "build-device-plane-backhaul-target-artifact.mjs" +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +COMPOSE = ( + SCRIPT_DIR.parent.parent + / "device-plane/docker-compose.device-plane.backhaul-target.yml" +) +PREDECESSOR_COMPOSE = ( + SCRIPT_DIR.parent.parent / "device-plane/docker-compose.device-plane.yml" +) +SSHD_CONFIG = ( + SCRIPT_DIR.parent.parent + / "device-plane/services/device-backhaul-target/sshd_config" +) + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_device_plane_backhaul_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +def valid_public_key(comment="nodedc-device-edge-backhaul"): + blob = b"\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20" + bytes(range(32)) + return f"ssh-ed25519 {base64.b64encode(blob).decode()} {comment}\n" + + +def healthy_inventory(): + return { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [ + { + "service": service, + "containerId": character * 64, + "imageId": f"sha256:{character * 64}", + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + } + for service, character in ( + ("device-control-core", "a"), + ("device-gateway", "b"), + ("device-postgres", "c"), + ) + ], + } + + +class DevicePlaneBackhaulTargetArtifactTest(unittest.TestCase): + def build(self, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + ["node", str(BUILDER), patch_id], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_exact_deterministic_and_contains_no_keys(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-backhaul-artifact-", + ) as directory: + target = Path(directory) + first = self.build(target, "device-plane-backhaul-target-unit-001") + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build(target, "device-plane-backhaul-target-unit-001") + second_bytes = Path(second["artifact"]).read_bytes() + self.assertEqual(first_bytes, second_bytes) + self.assertEqual(first["sha256"], hashlib.sha256(first_bytes).hexdigest()) + self.assertEqual( + first["entries"], + list(RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES), + ) + self.assertEqual(first["services"], ["device-backhaul-target"]) + + with tarfile.open(first["artifact"], "r:gz") as archive: + names = archive.getnames() + files = archive.extractfile("files.txt").read().decode().splitlines() + descriptor = json.loads( + archive.extractfile( + "payload/deployment/" + "device-plane-backhaul-target-tailnet-serve-v1.json" + ).read() + ) + self.assertEqual(files, first["entries"]) + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_backhaul_target_descriptor(), + ) + self.assertFalse(any( + name.endswith((".key", ".pem", "authorized_keys")) + for name in names + )) + + def test_registry_selects_only_target_and_preserves_red_boundaries(self): + entries = RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES + self.assertEqual( + RUNNER.component_services("device-plane", entries), + ("device-backhaul-target",), + ) + builds = RUNNER.component_builds("device-plane", entries) + self.assertEqual(len(builds), 1) + self.assertIn("services/device-backhaul-target/Dockerfile", builds[0][1]) + self.assertIn(RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_IMAGE, builds[0][1]) + for path in ( + RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL, + "services/device-backhaul-target/Dockerfile", + "services/device-backhaul-target/sshd_config", + RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_REL, + ): + self.assertTrue(RUNNER.allowed_payload_path("device-plane", path)) + + compose = COMPOSE.read_text(encoding="utf-8") + sshd = SSHD_CONFIG.read_text(encoding="utf-8") + self.assertIn("network_mode: host", compose) + self.assertNotIn("0.0.0.0:2222", compose) + self.assertIn('"127.0.0.1", "2222"', compose) + self.assertIn("ListenAddress 127.0.0.1", sshd) + self.assertIn("AllowTcpForwarding local", sshd) + self.assertIn("PermitOpen 127.0.0.1:9921", sshd) + self.assertIn("ForceCommand /bin/false", sshd) + self.assertIn("PasswordAuthentication no", sshd) + + def test_preflight_requires_exact_applied_006_and_enrollment_key(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-backhaul-preflight-", + ) as directory: + work = Path(directory) + built = self.build(work, "device-plane-backhaul-target-unit-002") + extracted = work / "extracted" + extracted.mkdir() + _manifest, _entries, payload = RUNNER.load_artifact( + Path(built["artifact"]), + extracted, + ) + live = work / "live" + live.mkdir() + (live / "docker-compose.device-plane.yml").write_bytes( + PREDECESSOR_COMPOSE.read_bytes() + ) + descriptor = live / RUNNER.DEVICE_PLANE_B2_DISCOVERY_INGRESS_REL + descriptor.parent.mkdir(parents=True) + descriptor.write_text( + json.dumps(RUNNER.expected_device_plane_b2_discovery_ingress_descriptor()), + encoding="utf-8", + ) + enrollment = work / "device-edge-backhaul.pub" + enrollment.write_text(valid_public_key(), encoding="ascii") + + def has_patch(value): + return value == RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_PATCH_ID + + def has_sha(value): + return value == RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_PREDECESSOR_ARTIFACT_SHA256 + + with ( + mock.patch.object(RUNNER, "component_root", return_value=live), + mock.patch.object( + RUNNER, + "DEVICE_PLANE_BACKHAUL_ENROLLMENT_PUBLIC_KEY_FILE", + enrollment, + ), + mock.patch.object(RUNNER, "state_has_patch_id", side_effect=has_patch), + mock.patch.object(RUNNER, "state_has_sha", side_effect=has_sha), + mock.patch.object( + RUNNER, + "device_plane_service_container_ids", + return_value=[], + ), + mock.patch.object( + RUNNER, + "device_plane_runtime_inventory", + return_value=healthy_inventory(), + ), + mock.patch.object(RUNNER, "assert_loopback_tcp_port_open"), + mock.patch.object( + RUNNER, + "validate_device_plane_backhaul_failed_evidence", + return_value={ + "backup": work / "failed-backup", + "failedArtifact": work / "failed-artifact.tgz", + }, + ), + mock.patch.object( + RUNNER, + "validate_device_plane_tailscale_cli", + return_value={ + "binary": str(RUNNER.DEVICE_PLANE_TAILSCALE), + "uid": 1024, + "gid": 1024, + "binarySha256": "d" * 64, + }, + ), + mock.patch.object( + RUNNER, + "validate_device_plane_tailscale_runtime", + return_value={ + "self": { + "Online": True, + "TailscaleIPs": ["100.109.216.21"], + }, + "serve": {}, + }, + ), + ): + accepted = RUNNER.validate_device_plane_backhaul_target_evidence(payload) + self.assertEqual( + accepted["mode"], + "failed-backhaul-target-to-loopback-tailnet-serve", + ) + self.assertRegex(accepted["enrollmentPublicKeySha256"], r"^[a-f0-9]{64}$") + self.assertEqual(accepted["tailscaleServeBefore"], {}) + self.assertEqual(accepted["tailscaleCli"]["uid"], 1024) + + def test_registered_health_gate_checks_preserved_and_target_services(self): + with mock.patch.object(RUNNER, "healthcheck_compose_service") as health: + RUNNER.run_healthchecks( + "device-plane", + RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES, + (RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,), + ) + self.assertEqual( + [call.args[1] for call in health.call_args_list], + [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + ) + + def test_candidate_rollback_removes_only_target_and_preserves_runtime(self): + runtime = healthy_inventory() + with ( + mock.patch.object( + RUNNER, + "read_backup_path_list", + side_effect=[[], list(RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES)], + ), + mock.patch.object( + RUNNER, + "validate_backup_partition", + return_value=(set(), set(RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES)), + ), + mock.patch.object( + RUNNER, + "read_strict_json", + side_effect=[runtime, {}], + ), + mock.patch.object( + RUNNER, + "disable_device_plane_tailscale_serve", + ) as disable_serve, + mock.patch.object(RUNNER, "stop_and_remove_compose_services") as stop, + mock.patch.object(RUNNER, "restore_platform_overlay", return_value=3), + mock.patch.object( + RUNNER, + "device_plane_service_container_ids", + return_value=[], + ), + mock.patch.object( + RUNNER, + "device_plane_runtime_inventory", + return_value=runtime, + ), + mock.patch.object( + RUNNER, + "validate_device_plane_tailscale_runtime", + return_value={"serve": {}}, + ), + mock.patch.object(RUNNER, "assert_loopback_tcp_port_open") as port, + ): + result = RUNNER.rollback_device_plane_apply( + Path("/unused/live"), + Path("/unused/backup"), + RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES, + "20260803-000000", + True, + (RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,), + ) + self.assertEqual( + result, + "tailscale-serve-restored-source-restored-target-removed-" + "preserved-runtime-unchanged:3", + ) + stop.assert_called_once_with( + "device-plane", + (RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,), + ) + disable_serve.assert_called_once_with({}) + port.assert_called_once_with(9921) + + def test_tailscale_serve_port_parser_rejects_funnel_and_nested_collision(self): + clean = {"TCP": {"443": {"HTTPS": True}}} + self.assertEqual( + RUNNER.device_plane_tailscale_handlers_for_port(clean, 2222), + [], + ) + active = { + "TCP": {"2222": {"TCPForward": "127.0.0.1:2222"}}, + "Foreground": { + "session": { + "TCP": {"443": {"HTTPS": True}}, + }, + }, + } + self.assertEqual( + RUNNER.device_plane_tailscale_handlers_for_port(active, 2222), + [((), {"TCPForward": "127.0.0.1:2222"})], + ) + self.assertFalse( + RUNNER.device_plane_tailscale_funnel_uses_port(active, 2222) + ) + active["AllowFunnel"] = {"edge.example.ts.net:2222": True} + self.assertTrue( + RUNNER.device_plane_tailscale_funnel_uses_port(active, 2222) + ) + + def test_runtime_activation_enables_private_tailscale_serve_after_health(self): + calls = [] + with ( + mock.patch.object(RUNNER, "run_build", side_effect=lambda *a: calls.append("build")), + mock.patch.object( + RUNNER, + "prepare_component_runtime", + side_effect=lambda *a: calls.append("prepare"), + ), + mock.patch.object( + RUNNER, + "run_compose", + side_effect=lambda *a: calls.append("compose"), + ), + mock.patch.object( + RUNNER, + "healthcheck_compose_service", + side_effect=lambda *a: calls.append("health"), + ), + mock.patch.object( + RUNNER, + "enable_device_plane_tailscale_serve", + side_effect=lambda *a: calls.append("serve"), + ) as enable, + ): + RUNNER.run_device_plane_runtime_for_apply( + RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_ENTRIES, + (RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,), + lambda: calls.append("started"), + backhaul_serve_before={"TCP": {"443": {"HTTPS": True}}}, + ) + self.assertEqual( + calls, + ["build", "prepare", "started", "compose", "health", "serve"], + ) + enable.assert_called_once_with({"TCP": {"443": {"HTTPS": True}}}) + + def test_tailscale_serve_enable_and_disable_preserve_unrelated_routes(self): + before = {"TCP": {"443": {"HTTPS": True}}} + active = { + "TCP": { + "443": {"HTTPS": True}, + "2222": {"TCPForward": "127.0.0.1:2222"}, + }, + } + with ( + mock.patch.object( + RUNNER, + "validate_device_plane_tailscale_runtime", + side_effect=[{"serve": before}, {"serve": active}], + ), + mock.patch.object(RUNNER, "run_device_plane_tailscale") as run, + ): + result = RUNNER.enable_device_plane_tailscale_serve(before) + self.assertEqual(result, active) + self.assertEqual( + run.call_args.args[0], + [ + "serve", + "--bg", + "--yes", + "--tcp=2222", + "tcp://127.0.0.1:2222", + ], + ) + + with ( + mock.patch.object( + RUNNER, + "read_device_plane_tailscale_json", + return_value=active, + ), + mock.patch.object( + RUNNER, + "validate_device_plane_tailscale_runtime", + return_value={"serve": before}, + ), + mock.patch.object(RUNNER, "run_device_plane_tailscale") as run, + ): + changed = RUNNER.disable_device_plane_tailscale_serve(before) + self.assertTrue(changed) + self.assertEqual( + run.call_args.args[0], + [ + "serve", + "--tcp=2222", + "off", + ], + ) + + def test_tailscale_cli_runs_as_official_package_account(self): + context = { + "binary": "/var/packages/Tailscale/target/bin/tailscale", + "uid": 1051, + "gid": 1051, + "binarySha256": "e" * 64, + } + with ( + mock.patch.object( + RUNNER, + "validate_device_plane_tailscale_cli", + return_value=context, + ), + mock.patch.object(RUNNER.subprocess, "run") as run, + mock.patch.object( + RUNNER, + "device_plane_tailscale_drop_privileges", + return_value="drop-to-package-account", + ) as drop, + ): + RUNNER.run_device_plane_tailscale( + ["status", "--json"], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual( + run.call_args.args[0], + [context["binary"], "status", "--json"], + ) + self.assertEqual( + run.call_args.kwargs["preexec_fn"], + "drop-to-package-account", + ) + drop.assert_called_once_with(1051, 1051) + + def test_tailscale_cli_accepts_package_owned_binary_without_root_execution(self): + privilege_path = mock.MagicMock() + privilege_path.__str__.return_value = ( + "/var/packages/Tailscale/conf/privilege" + ) + privilege_path.lstat.return_value = SimpleNamespace( + st_mode=RUNNER.stat.S_IFREG | 0o644, + st_uid=0, + ) + binary_path = mock.MagicMock() + binary_path.__str__.return_value = ( + "/var/packages/Tailscale/target/bin/tailscale" + ) + binary_path.lstat.return_value = SimpleNamespace( + st_mode=RUNNER.stat.S_IFREG | 0o755, + st_uid=1051, + st_gid=1051, + st_size=32 * 1024 * 1024, + ) + account = SimpleNamespace(pw_uid=1051, pw_gid=1051) + group = SimpleNamespace(gr_gid=1051) + help_result = SimpleNamespace( + stdout="--tcp --bg --yes", + stderr="", + ) + with ( + mock.patch.object( + RUNNER, + "DEVICE_PLANE_TAILSCALE_PRIVILEGE", + privilege_path, + ), + mock.patch.object( + RUNNER, + "DEVICE_PLANE_TAILSCALE", + binary_path, + ), + mock.patch.object( + RUNNER, + "read_strict_json", + return_value={ + "defaults": {"run-as": "package"}, + "username": "tailscale", + "groupname": "tailscale", + }, + ), + mock.patch.object(RUNNER.pwd, "getpwnam", return_value=account), + mock.patch.object(RUNNER.grp, "getgrnam", return_value=group), + mock.patch.object( + RUNNER, + "sha256_file", + return_value="f" * 64, + ), + mock.patch.object( + RUNNER.subprocess, + "run", + return_value=help_result, + ) as run, + ): + context = RUNNER.validate_device_plane_tailscale_cli() + self.assertEqual(context["uid"], 1051) + self.assertEqual(context["gid"], 1051) + self.assertEqual(context["binarySha256"], "f" * 64) + self.assertTrue(callable(run.call_args.kwargs["preexec_fn"])) + + def test_failed_001_evidence_is_exact_and_terminal(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-backhaul-failed-evidence-", + ) as directory: + root = Path(directory) + backups = root / "backups" + failed = root / "failed" + state = root / "state" + tmp = root / "tmp" + for path in (backups, failed, state, tmp): + path.mkdir() + + backup = backups / RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID + backup.mkdir() + backup_hashes = {} + for name in ( + "manifest.env", + "files.txt", + "source-before.tgz", + "existing-files.txt", + "missing-files.txt", + "runtime-before.json", + ): + payload = f"fixture:{name}\n".encode() + (backup / name).write_bytes(payload) + backup_hashes[name] = hashlib.sha256(payload).hexdigest() + + stage = root / "failed-stage" + payload = stage / "payload" + service = payload / "services/device-backhaul-target" + deployment = payload / "deployment" + service.mkdir(parents=True) + deployment.mkdir(parents=True) + (stage / "manifest.env").write_text( + "id=device-plane-backhaul-target-20260803-001\n" + "component=device-plane\n" + "type=app-overlay\n", + encoding="utf-8", + ) + (stage / "files.txt").write_text( + "\n".join(RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_TARGET_ENTRIES) + + "\n", + encoding="utf-8", + ) + (payload / RUNNER.DEVICE_PLANE_BACKHAUL_TARGET_COMPOSE_REL).write_text( + "services: {}\n", + encoding="utf-8", + ) + (service / "Dockerfile").write_text( + "FROM scratch\n", + encoding="utf-8", + ) + (service / "sshd_config").write_text( + "PasswordAuthentication no\n", + encoding="utf-8", + ) + (payload / RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_TARGET_REL).write_text( + json.dumps( + RUNNER.expected_failed_device_plane_backhaul_target_descriptor() + ) + + "\n", + encoding="utf-8", + ) + failed_artifact = failed / RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT + with tarfile.open(failed_artifact, "w:gz") as archive: + for name in ("manifest.env", "files.txt", "payload"): + archive.add(stage / name, arcname=name) + failed_sha = hashlib.sha256(failed_artifact.read_bytes()).hexdigest() + + failed_state = state / "failed.jsonl" + failed_state.write_text( + json.dumps({ + "artifact": RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT, + "backup_id": RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_ID, + "component": "device-plane", + "id": RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID, + "message": RUNNER.DEVICE_PLANE_BACKHAUL_FAILED_MESSAGE, + "rollback_status": ( + "ok:device-plane-overlay:source-restored-target-removed-" + "preserved-runtime-unchanged:3" + ), + "sha256": failed_sha, + "started_apply": True, + "status": "failed", + }) + + "\n", + encoding="utf-8", + ) + + with ( + mock.patch.object(RUNNER, "BACKUPS_DIR", backups), + mock.patch.object(RUNNER, "FAILED_DIR", failed), + mock.patch.object(RUNNER, "FAILED_STATE_FILE", failed_state), + mock.patch.object(RUNNER, "TMP_DIR", tmp), + mock.patch.object( + RUNNER, + "DEVICE_PLANE_BACKHAUL_FAILED_BACKUP_SHA256", + backup_hashes, + ), + mock.patch.object( + RUNNER, + "DEVICE_PLANE_BACKHAUL_FAILED_ARTIFACT_SHA256", + failed_sha, + ), + ): + evidence = RUNNER.validate_device_plane_backhaul_failed_evidence() + self.assertEqual(evidence["backup"], backup) + self.assertEqual(evidence["failedArtifact"], failed_artifact) + + +if __name__ == "__main__": + unittest.main()