Files
NODEDC_MISSION_CORE/tests/fleet/test_pairing.py
T
DCCONSTRUCTIONS a3c15e11e9 Add packaged Insta360 X4 integration and recover paired Node channels
Discover independent camera instances and prepare their versioned runtime
from Node or remote Core. Add isolated SDK workers, camera controls, raw
dual-fisheye WebRTC preview, and shared action/region loading states.

Recover existing Node bindings over known Tailscale addresses after a Core
LAN address change. Preserve identities and trust, pin both peers, migrate
endpoints with revision checks, and require real heartbeats for online status.
Fix the Python client certificate profile for Go X509 verification.

Pin Design Guideline 8c53f73 and retain installer/build/acceptance history.
Node 0.8.19 is installed; X4 0.1.3-3 is bundled but hardware activation is pending.

Validation: qualified DG/Node builds and Go race tests; 31 fleet tests;
Python-to-Go certificate interoperability and live tailnet recovery with five
fresh heartbeats; prior 38 X4 tests and bounded remote WebRTC acceptance.
Clean-OS, replug/power autonomy, local X4 video and long-run stability remain open.
2026-09-10 09:21:24 +03:00

581 lines
20 KiB
Python

import base64
import json
import secrets
import time
from datetime import UTC, datetime, timedelta
import pytest
from cryptography import x509
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from fastapi import FastAPI
from fastapi.testclient import TestClient
from k1link.fleet.registry import FleetRegistry
from k1link.fleet.trust import SCHEMA, PairingError, parse_invitation, pem, public_id
from k1link.web.fleet_api import router
def code(value):
return "MCN1." + base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=")
@pytest.fixture
def setup(tmp_path, monkeypatch):
fleet = FleetRegistry(tmp_path)
key = Ed25519PrivateKey.generate()
node_id = public_id("node_", key.public_key())
invite = dict(
schema=SCHEMA,
node_id=node_id,
id=secrets.token_urlsafe(32),
secret=secrets.token_urlsafe(32),
expires_at=int(time.time()) + 600,
endpoint="https://192.168.20.4:8781",
)
calls = []
def request(invitation, path, payload):
calls.append((path, payload))
if path.endswith("/inspect"):
return (
dict(schema=SCHEMA, node_id=node_id, name="Test board", host={"os": "Linux"}),
key.public_key(),
"192.168.20.5",
)
if path.endswith("/offer"):
return (
dict(node_id=node_id, receipt=secrets.token_urlsafe(32)),
key.public_key(),
"192.168.20.5",
)
return dict(node_id=node_id), key.public_key(), "192.168.20.5"
monkeypatch.setattr("k1link.fleet.registry.node_request", request)
monkeypatch.setattr(fleet, "listen", lambda address: None)
yield fleet, invite, key, calls
fleet.close()
def create(setup):
fleet, invite, _, _ = setup
preview = fleet.preview(code(invite))
row = fleet.add(preview["preview_id"], "Test vehicle", "ugv")
return row, preview
def heartbeat(row):
return dict(
schema=SCHEMA,
node_id=row["node_id"],
binding_id=row["binding"]["binding_id"],
execution_binding={
"node_id": row["node_id"],
"agent_instance_id": "agent_test",
"platform": "linux",
},
host={"hostname": "synthetic", "usb": [], "networks": []},
devices=[],
)
def cert(row):
return x509.load_pem_x509_certificate(row["binding"]["client_pem"].encode()).public_bytes(
serialization.Encoding.DER
)
def test_migrated_heartbeat_preserves_identity_and_old_reply_cannot_reverse(setup):
fleet, _, _, _ = setup
public, _ = create(setup)
fleet.advance(public["id"])
old = fleet.find(public["id"])
payload = {
**heartbeat(old),
"core_endpoint": "https://100.64.20.5:8782",
"endpoint_revision": 1,
}
assert fleet.receive(cert(old), "/v1/node/heartbeat", payload, address="192.168.20.5")[0] == 400
assert fleet.find(public["id"])["binding"] == old["binding"]
assert fleet.receive(cert(old), "/v1/node/heartbeat", payload, address="100.64.20.5")[0] == 200
current = fleet.find(public["id"])
assert current["core_address"] == "100.64.20.5"
assert current["binding"]["binding_id"] == old["binding"]["binding_id"]
assert current["binding"]["client_pem"] == old["binding"]["client_pem"]
assert (
fleet.receive(cert(old), "/v1/node/heartbeat", heartbeat(old), address="192.168.20.5")[0]
== 409
)
assert fleet.find(public["id"])["binding"] == current["binding"]
fleet.revoke(public["id"])
assert fleet.receive(cert(old), "/v1/node/heartbeat", payload, address="100.64.20.5")[0] == 410
def test_recovery_only_uses_previously_reported_tailnet_addresses():
from k1link.fleet.recovery import node_addresses
assert node_addresses(
{
"inventory": {
"networks": [
{
"up": True,
"addresses": ["192.168.20.4/24", "8.8.8.8", "100.64.20.4/32", "::1"],
},
{"up": False, "addresses": ["100.64.1.1/32"]},
]
}
}
) == ["100.64.20.4"]
def test_recovery_client_leaf_is_distinct_from_issuer_with_same_pinned_key(setup):
from cryptography.x509.oid import ExtendedKeyUsageOID
from k1link.fleet.recovery import client_context
fleet, _, _, _ = setup
client_context(fleet.trust)
leaf, root = x509.load_pem_x509_certificates((fleet.root / "recovery-client.pem").read_bytes())
assert leaf.subject != root.subject
assert leaf.issuer == root.subject
assert leaf.public_key().public_bytes_raw() == root.public_key().public_bytes_raw()
assert public_id("core_", leaf.public_key()) == fleet.trust.core_id
assert (
ExtendedKeyUsageOID.CLIENT_AUTH
in leaf.extensions.get_extension_for_class(x509.ExtendedKeyUsage).value
)
leaf.verify_directly_issued_by(root)
def test_recovery_pins_node_before_application_request(setup, monkeypatch):
from k1link.fleet import recovery
fleet, _, _, _ = setup
public, _ = create(setup)
fleet.advance(public["id"])
row = fleet.find(public["id"])
row["inventory"] = {"networks": [{"up": True, "addresses": ["100.64.20.4/32"]}]}
fleet.save(row)
calls = []
class Connection:
sock = None
def connect(self):
self.sock = self
def getpeercert(self, **kwargs):
return b"not the saved Node certificate"
def close(self):
calls.append("closed")
monkeypatch.setattr(recovery.http.client, "HTTPSConnection", lambda *a, **kw: Connection())
monkeypatch.setattr(recovery, "request", lambda *a: calls.append("request"))
with pytest.raises(ValueError):
recovery.recover(fleet, row, "100.64.20.4", None)
assert calls == ["closed"]
def test_recovery_rechecks_revocation_and_waits_for_heartbeat(setup, monkeypatch):
from k1link.fleet import recovery
fleet, _, _, _ = setup
public, _ = create(setup)
fleet.advance(public["id"])
row = fleet.find(public["id"])
row["inventory"] = {"networks": [{"up": True, "addresses": ["100.64.20.4/32"]}]}
fleet.save(row)
calls = []
class Connection:
def connect(self):
self.sock = self
def getpeercert(self, **kwargs):
return b"verified below by test adapter"
def getsockname(self):
return ("100.64.20.5", 30000)
def close(self):
pass
monkeypatch.setattr(recovery.http.client, "HTTPSConnection", lambda *a, **kw: Connection())
monkeypatch.setattr(recovery, "verify_node_certificate", lambda *a: None)
def request(connection, path, body):
calls.append(path)
return {
"schema": recovery.SCHEMA,
"node_id": row["node_id"],
"core_id": fleet.trust.core_id,
"binding_id": row["binding"]["binding_id"],
"endpoint": row["binding"]["endpoint"],
"endpoint_revision": 0,
}
monkeypatch.setattr(recovery, "request", request)
recovery.recover(fleet, row, "100.64.20.4", None)
assert calls == ["/v1/channel/inspect", "/v1/channel/migrate"]
assert fleet.find(public["id"])["binding"] == row["binding"]
assert fleet.public(fleet.find(public["id"]))["connectivity"] == "offline"
calls.clear()
fleet.revoke(public["id"])
recovery.recover(fleet, row, "100.64.20.4", None)
assert calls == ["/v1/channel/inspect"]
def test_preview_add_is_durable_and_idempotent(setup):
fleet, invite, _, calls = setup
public, preview = create(setup)
assert public["enrollment"] == "pending"
assert public["connectivity"] == "offline"
assert fleet.add(preview["preview_id"], "Test vehicle", "ugv")["id"] == public["id"]
fleet.advance(public["id"])
assert fleet.listing()["items"][0]["enrollment"] == "paired"
assert fleet.listing()["items"][0]["connectivity"] == "offline"
row = fleet.find(public["id"])
assert row["invitation"] is None and row["receipt"] is None
assert fleet.receive(cert(row), "/v1/node/heartbeat", heartbeat(row))[0] == 200
listed = fleet.listing()
assert listed["items"][0]["connectivity"] == "online"
assert invite["secret"] not in json.dumps(listed)
assert "client_pem" not in json.dumps(listed)
reopened = FleetRegistry(fleet.root)
try:
assert reopened.trust.core_id == fleet.trust.core_id
assert reopened.listing()["items"][0]["id"] == public["id"]
finally:
reopened.close()
def test_lost_commit_ack_recovers_from_authenticated_heartbeat(setup, monkeypatch):
fleet, _, _, _ = setup
public, _ = create(setup)
original = __import__("k1link.fleet.registry", fromlist=["node_request"]).node_request
def drop_ack(invitation, path, body):
if path.endswith("/commit"):
raise PairingError("Lost acknowledgement")
return original(invitation, path, body)
monkeypatch.setattr("k1link.fleet.registry.node_request", drop_ack)
with pytest.raises(PairingError):
fleet.advance(public["id"])
row = fleet.find(public["id"])
assert row["receipt"] and row["enrollment"] == "pending"
assert fleet.receive(cert(row), "/v1/node/heartbeat", heartbeat(row))[0] == 200
assert fleet.find(public["id"])["enrollment"] == "paired"
def test_revoke_during_network_completion_cannot_resurrect(setup, monkeypatch):
fleet, _, _, _ = setup
public, _ = create(setup)
original = __import__("k1link.fleet.registry", fromlist=["node_request"]).node_request
def revoke_first(invitation, path, body):
value = original(invitation, path, body)
fleet.revoke(public["id"])
return value
monkeypatch.setattr("k1link.fleet.registry.node_request", revoke_first)
fleet.advance(public["id"])
row = fleet.find(public["id"])
assert row["enrollment"] == "revoked"
assert fleet.receive(cert(row), "/v1/node/heartbeat", heartbeat(row))[0] == 410
def test_repair_retains_vehicle_and_rejects_old_binding(setup):
fleet, invite, _, _ = setup
public, _ = create(setup)
old = fleet.find(public["id"])
fleet.revoke(public["id"])
invite["id"] = secrets.token_urlsafe(32)
updated, _ = create(setup)
assert updated["id"] == public["id"] and updated["revision"] == 3
assert fleet.receive(cert(old), "/v1/node/heartbeat", heartbeat(old))[0] == 410
def test_rotation_has_bounded_old_certificate_grace(setup):
fleet, _, key, _ = setup
public, _ = create(setup)
row = fleet.find(public["id"])
original = x509.load_pem_x509_certificate(row["binding"]["client_pem"].encode())
soon = (
x509.CertificateBuilder()
.subject_name(original.subject)
.issuer_name(original.issuer)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(datetime.now(UTC) - timedelta(hours=1))
.not_valid_after(datetime.now(UTC) + timedelta(days=1))
.sign(fleet.trust.key, None)
)
row["binding"]["client_pem"] = pem(soon)
fleet.save(row)
status, reply = fleet.receive(cert(row), "/v1/node/heartbeat", heartbeat(row))
assert status == 200 and reply["client_pem"] != pem(soon)
assert fleet.receive(cert(row), "/v1/node/heartbeat", heartbeat(row))[0] == 200
current = fleet.find(public["id"])
current["certificate_previous"]["until"] = time.time() - 1
fleet.save(current)
assert fleet.receive(cert(row), "/v1/node/heartbeat", heartbeat(row))[0] == 403
@pytest.mark.parametrize(
"address",
[
"https://8.8.8.8:8781",
"https://127.0.0.1:8781",
"https://192.168.1.2:22",
"https://user@192.168.1.2:8781",
"https://192.168.1.2:8781/api",
"http://192.168.1.2:8781",
"https://local.test:8781",
],
)
def test_invitation_never_targets_public_loopback_dns_or_other_ports(setup, address):
_, invite, _, _ = setup
invite["endpoint"] = address
with pytest.raises(PairingError):
parse_invitation(code(invite))
def test_expiry_and_node_inventory_admission(setup):
fleet, invite, _, _ = setup
public, _ = create(setup)
row = fleet.find(public["id"])
row["invitation"]["expires_at"] = 0
fleet.save(row)
fleet.advance(public["id"])
assert fleet.find(public["id"])["enrollment"] == "failed"
invite["expires_at"] = 0
with pytest.raises(PairingError):
parse_invitation(code(invite))
def test_operator_api_rejects_cross_origin_and_remote_peer(setup):
fleet, _, _, _ = setup
app = FastAPI()
app.state.fleet = fleet
app.include_router(router)
with TestClient(app, base_url="http://127.0.0.1:8000", client=("127.0.0.1", 55555)) as client:
assert client.get("/api/v1/fleet").status_code == 200
assert (
client.get("/api/v1/fleet", headers={"Origin": "https://attacker.test"}).status_code
== 403
)
assert (
client.get("/api/v1/fleet", headers={"Host": "attacker.test:8000"}).status_code == 403
)
assert client.post("/api/v1/fleet/preview", json={"code": "bad"}).status_code == 409
with TestClient(
app, base_url="http://127.0.0.1:8000", client=("192.168.20.6", 55555)
) as client:
assert client.get("/api/v1/fleet").status_code == 403
@pytest.mark.parametrize("wrong_identity", [False, True])
def test_bootstrap_pin_is_verified_before_secret_is_sent(setup, monkeypatch, wrong_identity):
from cryptography.x509.oid import NameOID
from k1link.fleet.trust import node_request
_, invite, key, _ = setup
if wrong_identity:
key = Ed25519PrivateKey.generate()
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Synthetic Node")])
certificate = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(datetime.now(UTC) - timedelta(minutes=1))
.not_valid_after(datetime.now(UTC) + timedelta(hours=1))
.sign(key, None)
.public_bytes(serialization.Encoding.DER)
)
sent = []
class Socket:
def getpeercert(self, **_kwargs):
return certificate
def getsockname(self):
return ("192.168.20.5", 41000)
class Response:
status = 200
def read(self, _limit):
return b'{"ok":true}'
class Connection:
sock = Socket()
def __init__(self, *_args, **_kwargs):
pass
def connect(self):
pass
def request(self, *args):
sent.append(args)
def getresponse(self):
return Response()
def close(self):
pass
monkeypatch.setattr("k1link.fleet.trust.http.client.HTTPSConnection", Connection)
if wrong_identity:
with pytest.raises(PairingError):
node_request(invite, "/v1/pair/inspect", {"secret": invite["secret"]})
assert not sent
else:
assert node_request(invite, "/v1/pair/inspect", {"secret": invite["secret"]})[0] == {
"ok": True
}
assert len(sent) == 1
def sensor_inventory(row):
now = datetime.now(UTC).isoformat()
item = {
"id": "rsd455_01234567890123456789012345678901",
"name": "Test camera",
"snapshot": {
"context": {
"session_id": "sensor_test",
"device": {
"device_id": "rsd455_01234567890123456789012345678901",
"model": {
"plugin_id": "missioncore.realsense",
"plugin_version": "0.6.0",
"model_id": "realsense.d455",
},
"stability": "stable",
"basis": "hardware-identifier",
},
"execution": {
"node_id": row["node_id"],
"agent_instance_id": "driver_test",
"platform": "linux",
},
"opened_at": now,
},
"revision": 0,
"enrollment": "empty",
"connectivity": "connected",
"acquisition": "idle",
"observed_at": now,
},
}
value = heartbeat(row)
value["devices"] = [item]
value["sensor_state"] = {"items": [item], "operations": [], "preparation": None}
return value
def test_sensor_commands_require_pairing_and_keep_uncertain_outcome(setup):
from k1link.fleet import sensors
fleet, _, _, _ = setup
public, _ = create(setup)
row = setup[0].find(public["id"])
inventory = sensor_inventory(row)
fleet.receive(cert(row), "/v1/node/heartbeat", inventory)
now = datetime.now(UTC)
command = {
"api_version": "missioncore.nodedc/plugin-sdk/v0alpha2",
"kind": "OperationRequest",
"operation_id": "op_01234567890123456789012345678901",
"idempotency_key": "op_01234567890123456789012345678901",
"session": {"device_id": inventory["devices"][0]["id"], "session_id": "sensor_test"},
"requested_at": now.isoformat(),
"deadline_at": (now + timedelta(seconds=60)).isoformat(),
"action_id": "start",
"parameters": {"record": True},
}
first = sensors.submit(fleet, row["id"], command)
assert first["state"] == "queued"
assert sensors.submit(fleet, row["id"], command) == first
with pytest.raises(PairingError):
sensors.submit(fleet, row["id"], {**command, "action_id": "stop"})
_, response = fleet.receive(cert(row), "/v1/node/heartbeat", inventory)
assert response["sensor_commands"] == [command]
inventory["sensor_results"] = [
{"command": command, "state": "unknown", "error": "test uncertainty"}
]
_, response = fleet.receive(cert(row), "/v1/node/heartbeat", inventory)
assert response["sensor_commands"] == []
assert sensors.operation(fleet, row["id"], command["operation_id"])["state"] == "unknown"
fleet.revoke(row["id"])
with pytest.raises(PairingError):
sensors.submit(fleet, row["id"], command)
def test_sensor_cannot_claim_another_board_identity(setup):
from k1link.fleet.sensors import validate_inventory
public, _ = create(setup)
row = setup[0].find(public["id"])
value = sensor_inventory(row)
value["devices"][0]["snapshot"]["context"]["execution"]["node_id"] = "node_wrong"
with pytest.raises(ValueError):
validate_inventory(value, row["node_id"])
def test_uninitialized_x4_can_be_prepared_remotely_with_instance_progress(setup):
from k1link.fleet import sensors
public, _ = create(setup)
fleet = setup[0]
row = fleet.find(public["id"])
inventory = sensor_inventory(row)
item = inventory["devices"][0]
item.update(id="instax4_" + "a" * 32, kind="insta360.x4", prepared=False, configured=False)
item["snapshot"]["context"]["device"].update(
device_id=item["id"],
model={
"plugin_id": "missioncore.insta360",
"plugin_version": "0.1.0",
"model_id": "insta360.x4",
},
)
assert fleet.receive(cert(row), "/v1/node/heartbeat", inventory)[0] == 200
now = datetime.now(UTC)
command = {
"api_version": "missioncore.nodedc/plugin-sdk/v0alpha2",
"kind": "OperationRequest",
"operation_id": "op_" + "b" * 32,
"idempotency_key": "op_" + "b" * 32,
"session": {"device_id": item["id"], "session_id": "sensor_test"},
"requested_at": now.isoformat(),
"deadline_at": (now + timedelta(seconds=350)).isoformat(),
"action_id": "prepare",
"parameters": {},
}
assert sensors.submit(fleet, row["id"], command)["state"] == "queued"
_, result = fleet.receive(cert(row), "/v1/node/heartbeat", inventory)
assert result["sensor_commands"] == [command]
progress = {
"operation_id": command["operation_id"],
"device_id": item["id"],
"model_id": "insta360.x4",
"phase": "verify",
"state": "running",
"steps": [{"id": "profile", "state": "complete"}, {"id": "verify", "state": "running"}],
}
inventory["sensor_state"]["preparations"] = [progress]
inventory["sensor_results"] = [
{"command": command, "state": "running", "preparation": progress}
]
assert fleet.receive(cert(row), "/v1/node/heartbeat", inventory)[0] == 200
assert sensors.operation(fleet, row["id"], command["operation_id"])["preparation"] == progress
assert fleet.find(row["id"])["sensor_state"]["preparations"] == [progress]