diff --git a/.gitignore b/.gitignore index 227e274..f8e0715 100644 --- a/.gitignore +++ b/.gitignore @@ -37,8 +37,10 @@ artifacts/decoded/ *.pcd *.npz *.raw +*.k1mqtt +mqtt.metadata.jsonl +mqtt.summary.json # Small synthetic/redacted fixtures under tests/fixtures are allowed. !tests/fixtures/**/*.pcap !tests/fixtures/**/*.pcapng - diff --git a/README.md b/README.md index 2d1498d..e35f6b2 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,15 @@ Pre-production research project for connecting an owner-controlled XGRIDS/LixelKity K1 to a Mac without LixelGO, firmware changes, device opening, or speculative writes. -Current status: repository scaffold and implementation plan. No command in the -repository currently writes to the K1, changes the router, or installs system -packages. +Current status: live proof completed on firmware 3.0.2. The Mac provisioned the +K1 onto an existing LAN without LixelGO, connected to its MQTT broker, captured +the scan-correlated point-cloud and pose streams, and decoded both successfully. + +The repository now contains one narrowly gated state-changing command: +`ble wifi-configure`. It accepts only the reviewed firmware-3 provisioning +profile and requires explicit `--confirm-write`; the Wi-Fi password is collected +through a hidden local macOS dialog. MQTT capture and decoding are read-only. +Nothing changes router settings, firmware or global Python packages. ## Available stand @@ -29,9 +35,10 @@ The project has three independent gates: 3. Without LixelGO, K1 can be associated with Wi-Fi and a proprietary data session can be opened. -Only after gate 3 do point-cloud, pose, status, and camera stream decoders become -meaningful implementation work. Local `map.las` and `poses.csv` prove internal -capabilities, not an externally accessible network format. +All three gates are now proven on the tested unit. The external stream is plain +MQTT 3.1.1 on TCP 1883. Firmware-3 `lio_pcl` is protobuf wrapped in a raw LZ4 +block, and `lio_pose` is an uncompressed protobuf. Raw panoramic camera access is +still unproven and is not implied by point-cloud success. ## Local environment @@ -50,19 +57,33 @@ uv run pytest and reports external tools; it does not request Bluetooth permission, scan the LAN, touch the K1, alter Homebrew, or change capture permissions. -The currently implemented laboratory commands are: +The implemented laboratory commands include: ```bash uv run k1link ble scan --duration 30 --out sessions//captures/ble.json uv run k1link ble gatt-dump --device \ --out sessions//captures/gatt.json +uv run k1link ble wifi-configure --device \ + --profile xgrids-k1-fw3-wifi-v1 --write-mode with_response \ + --confirm-write --out sessions//captures/wifi.sensitive.json uv run k1link net snapshot --out sessions//captures/network.json +uv run k1link net mqtt-capture --host \ + --confirm-owned-device --duration 180 \ + --out sessions//captures/mqtt-run +uv run k1link analyze mqtt-streams \ + --capture sessions//captures/mqtt-run/mqtt.raw.k1mqtt \ + --out sessions//analysis/mqtt-streams.summary.json ``` -On macOS the scan is active CoreBluetooth discovery, but it does not connect to -or modify devices. `gatt-dump` connects and performs service discovery only; it -does not read characteristic values, subscribe to notifications or write -configuration. Session output is sensitive and ignored by Git. +On macOS the BLE scan is active CoreBluetooth discovery, but it does not connect +to or modify devices. `gatt-dump` connects and performs service discovery only. +`mqtt-capture` accepts only a literal RFC1918 target, uses a fixed report-topic +allowlist, never publishes and never reconnects. It writes a length-framed raw +file, JSONL metadata and an integrity summary with mode `0600`. + +Session output is sensitive and ignored by Git. It can contain device identity, +trajectory, mapped interiors and local addressing even when no credentials are +present. ## Documentation @@ -70,6 +91,9 @@ configuration. Session output is sensitive and ignored by Git. - [Implementation gates](docs/01_IMPLEMENTATION_PLAN.md) - [First lab runbook](docs/02_FIRST_LAB_RUNBOOK.md) - [Artifact and secret policy](docs/03_ARTIFACT_POLICY.md) +- [Reviewed BLE Wi-Fi profile](docs/04_K1_WIFI_PROVISIONING_PROFILE.md) +- [Verified MQTT stream profile](docs/05_K1_MQTT_STREAM_PROFILE.md) +- [Redacted live lab report](docs/lab/001_K1_LIVE_MQTT_20260715.redacted.md) - [Session manifest schema](schemas/session-manifest.schema.json) - [Reference input provenance](docs/reference/README.md) @@ -84,10 +108,12 @@ controlled notification listening, autonomous button operation, targeted capture of traffic to or from the confirmed K1 address, and offline analysis of owned artifacts. -BLE writes, provisioning, application-session packets, active service probes, -and router configuration changes require evidence and an explicit reviewed -step. Random writes, fuzzing, brute force, firmware operations, destructive file -access, and credential guessing are out of scope. +The reviewed provisioning write requires its named profile and explicit operator +confirmation. Application command publishing remains disabled: physical +double-click is the verified start/stop mechanism. Any future MQTT publisher, +router configuration change or new BLE write requires its own evidence and +reviewed step. Random writes, fuzzing, brute force, firmware operations, +destructive file access and credential guessing remain out of scope. Real captures, projects, router metadata, serials, credentials, maps, images, and logs are ignored by normal Git. Redacted manifests and SHA-256 inventories diff --git a/docs/00_TECHNICAL_AUDIT.md b/docs/00_TECHNICAL_AUDIT.md index 0caac03..a201a94 100644 --- a/docs/00_TECHNICAL_AUDIT.md +++ b/docs/00_TECHNICAL_AUDIT.md @@ -1,14 +1,19 @@ # Technical audit -Status: pre-production planning, 2026-07-15. +Status: live feasibility validated, 2026-07-15. ## Executive finding -The project is feasible as a disciplined black-box investigation, but the -existence of a complete K1-to-Mac realtime link is not yet established. The -first hard problem is not decoding points or images. It is bootstrapping the K1 -from BLE availability into Wi-Fi association and then opening the proprietary -application data session without LixelGO. +The K1-to-Mac realtime link is feasible and has been demonstrated without +LixelGO. The tested Mac provisioned firmware 3.0.2 over BLE, confirmed K1 LAN +association, opened the device's MQTT 3.1.1 service on TCP 1883, captured a +physical-button scan and decoded both point-cloud and pose reports. + +All 1,140 captured `lio_pcl` frames decoded as raw-LZ4 protobuf blocks, yielding +4,165,862 points. All 1,215 `lio_pose` messages decoded, and the resulting +approximately 1.566 m displacement matched the controlled movement. The +remaining feasibility question is the panoramic camera branch, not the core +realtime LiDAR/pose path. The supplied Bible is useful as an OSINT dossier. It is not an executable plan for the actual stand because many experiments assume a phone and LixelGO. The @@ -31,16 +36,16 @@ writes. Each of these remains scoped to the actual firmware/hardware state observed in the lab. The physical unit is authoritative. -### Strong hypotheses +### Resolved hypotheses -- BLE is the bootstrap/control plane and Wi-Fi is the likely high-rate data - plane. -- The user-provided SSID and PSK are transported inside a vendor-defined GATT - protocol. -- The application may need a second token, certificate, handshake, or stream - subscription after ordinary Wi-Fi association. -- The externally exposed point cloud is more likely a processed/downsampled - preview than raw LiDAR packets. +- BLE is the bootstrap/provisioning plane and Wi-Fi/MQTT is the high-rate data + plane on the tested firmware. +- SSID and PSK use a verified fixed 99-byte GATT frame; the hidden credential + path never persists the password. +- The report subscriptions require no second MQTT credential, certificate or + application publish handshake on the direct LAN path. +- The external `lio_pcl` stream is a processed point cloud rather than raw + LiDAR packets. XGRIDS provides stronger product evidence for an external point-cloud path than the local LAS alone: current LixelStudio materials describe K1 remote control @@ -49,18 +54,18 @@ and this does not disclose or guarantee access to the protocol from macOS. It does, however, justify keeping realtime point cloud as the primary stream target. No comparable official evidence proves an exportable raw panorama/camera stream. -### Unsupported until measured +### Still unsupported until measured -- K1 remembers an existing Wi-Fi profile. -- K1 exposes its own access point. -- Provisioning fields are plain UTF-8, JSON, TLV, CBOR, or protobuf. -- A writable GATT characteristic can be used without bonding or an app token. - Local `map.las` layout resembles the network stream. -- Live pose or camera frames leave the device. -- A point stream contains simple float32 XYZ tuples. -- Network payloads are unencrypted. +- Raw or stitched panoramic camera frames leave the device. +- The upper 24 bits of point `rgbi` are usable packed RGB. +- MQTT application commands can be published safely without first reproducing + device/session header state and response handling. -## Major corrections to the source plan +## Initial corrections to the source plan + +These corrections governed the safe experiment. The resolved outcomes above +and the verified profiles now supersede their pre-lab uncertainty. 1. Experiments requiring LixelGO are removed from the active critical path. There will be no app-session capture, provisioning diff, app-start comparison, @@ -142,6 +147,11 @@ device baseline -> point cloud / pose / status / optional camera decoders ``` +The chain through point-cloud/pose decoding is now complete. See the +[MQTT stream profile](05_K1_MQTT_STREAM_PROFILE.md) and +[redacted lab report](lab/001_K1_LIVE_MQTT_20260715.redacted.md) for measured +formats, counts and artifact hashes. + ## Verified primary references - [XGRIDS K1 firmware and release notes](https://www.xgrids.com/intl/support/download?page=K1) diff --git a/docs/01_IMPLEMENTATION_PLAN.md b/docs/01_IMPLEMENTATION_PLAN.md index 6641c04..2d66620 100644 --- a/docs/01_IMPLEMENTATION_PLAN.md +++ b/docs/01_IMPLEMENTATION_PLAN.md @@ -3,6 +3,24 @@ This plan supersedes the app-dependent experiment order in the reference Bible. Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result. +## Live checkpoint — 2026-07-15 + +| Stage | Result | +| --- | --- | +| Stage 0 host/repository | GO — isolated Python 3.12 environment and private Git | +| Gate 1 physical operation | GO — autonomous double-click start/stop verified | +| Stage 1 BLE discovery | GO — repeatable advertisement and GATT profile | +| Stage 2 BLE provisioning | GO — reviewed 99-byte profile, LAN association confirmed | +| Stage 3 application session | GO — MQTT 3.1.1 on confirmed K1 TCP 1883 | +| Stage 4 artifacts/flows | GO — bounded capture, hashes and negative control | +| Stage 5 point cloud | GO — raw-LZ4 protobuf, 1,140 live frames decoded | +| Stage 5 pose | GO — 1,215 live frames decoded and motion-correlated | +| Stage 5 camera | PAUSE — no independent frame/video stream observed | + +USB project copying remains optional ground truth rather than a blocker for the +now-verified network path. MQTT control publishing remains deliberately deferred +because the physical button is a known-safe start/stop mechanism. + ## Stage 0 — repository and host baseline Deliverables: diff --git a/docs/04_K1_WIFI_PROVISIONING_PROFILE.md b/docs/04_K1_WIFI_PROVISIONING_PROFILE.md new file mode 100644 index 0000000..ecf72a3 --- /dev/null +++ b/docs/04_K1_WIFI_PROVISIONING_PROFILE.md @@ -0,0 +1,107 @@ +# Reviewed K1 Wi-Fi provisioning profile + +Profile ID: `xgrids-k1-fw3-wifi-v1` + +Status: reviewed for one controlled write against the owner-controlled K1. This +profile is not a generic XGRIDS protocol claim and must not be used for fuzzing. + +## Evidence and target + +- Observed device firmware: 3.0.2. +- Live metadata-only GATT discovery exposes service `7f00`, read/write + characteristic `7f01`, and read characteristic `7f02`. +- Static analysis of the official Android application's production path maps + its service field to `7f00`, discovered write field to `7f01`, and status/read + field to `7f02`. +- The application sends operator-supplied router credentials; it does not derive + them from the K1 and it does not require the password of the K1's own AP. + +Full UUIDs: + +- service: `00007f00-0000-1000-8000-00805f9b34fb`; +- write: `00007f01-0000-1000-8000-00805f9b34fb`; +- status/read: `00007f02-0000-1000-8000-00805f9b34fb`. + +## Request frame + +The application writes exactly 99 bytes: + +| Offset | Length | Meaning | +| --- | ---: | --- | +| 0 | 1 | SSID UTF-8 byte length, 1 through 32 | +| 1 | 32 | SSID UTF-8 bytes followed by zero padding | +| 33 | 1 | password UTF-8 byte length, 1 through 64 | +| 34 | 64 | password UTF-8 bytes followed by zero padding | +| 98 | 1 | zero | + +There is no checksum, nonce, token, certificate, signature, or separate commit +command in this production call path. The implementation must never print, +persist, or accept the password as a command-line argument. + +The Android application unequivocally requests a write without response and +negotiates MTU 120, making the 99-byte frame one ATT command. A live read-only +CoreBluetooth check reports MTU 256 and a maximum write-without-response size of +253, so the same frame also fits one command on this Mac. + +CoreBluetooth metadata advertises `write`, not `write-without-response`, despite +the official application forcing the latter. The macOS tool exposes that +discrepancy: `auto` follows advertised properties, while the explicit +`without_response` mode mirrors the application only after checking the live +maximum is at least 99. The first controlled experiment uses that explicit +mode. It never fragments or retries the payload automatically. + +## Expected transition and evidence of acceptance + +A completed GATT write only proves transport completion. It does not prove that +the K1 joined Wi-Fi. The application polls `7f02`; the observed response frame +contains a fixed-width mode slot, an address slot, and a status byte at offset +50. The current AP baseline reports mode `WIFI_AP` and address `192.168.56.1`. + +For the controlled experiment, acceptance required at least one of: + +1. `7f02` reports a non-AP IPv4 address; +2. the same address appears as a new router/ARP client after the write; +3. a statically evidenced K1 application endpoint is reached at that confirmed + address. + +Do not infer success from a write callback alone. + +## Safety, recovery and stop conditions + +- Perform one write per explicitly named attempt, using credentials for the LAN + already used by the Mac. Never retry automatically. +- Do not alter Deco settings, scan the subnet, or guess any credential. +- If the status does not change, do not retry automatically. +- If the supplied credentials are wrong, reconnect over BLE and overwrite them + with corrected credentials using the same reviewed profile. +- A single normal power cycle is allowed as a recovery check. It is not claimed + to erase stored credentials. Factory reset and firmware actions are prohibited. +- Stop on repeated disconnects, an unexpected GATT layout, overheating, an + activation prompt, or any device fault indication. + +The exact factory-return command is not known. That limitation is accepted for +the first write because BLE remains the documented configuration transport and +the operator controls the intended LAN credentials. + +## Live transport attempts + +Attempt 01 mirrored the Android application: one 99-byte write without response, +with a live CoreBluetooth maximum of 253 bytes. For 60 seconds afterward, `7f02` +remained at the AP baseline and the existing LAN neighbor table did not change. +There was no automatic retry. + +Attempt 02 was separately reviewed because `7f01` explicitly advertises ordinary +write-with-response to CoreBluetooth. The identical 99-byte payload was sent +once using that advertised transport. This tested the transport mismatch only; +it did not change credential values or application framing. + +Attempt 02 succeeded immediately. `7f02` reported the operator-selected LAN, +status code 1 and a non-AP private IPv4 address. The existing neighbor table then +confirmed the same new address on the Mac's Wi-Fi interface. No third +provisioning write was made. + +Offline analysis of the owned application and a targeted check against only that +confirmed address identified plain MQTT on TCP 1883 as the production +application endpoint. Port 8008 belongs to a separate QuickLink/AP WebSocket +branch and was correctly refused in the bridge/LAN topology. The successful +MQTT result is documented in `05_K1_MQTT_STREAM_PROFILE.md`. diff --git a/docs/05_K1_MQTT_STREAM_PROFILE.md b/docs/05_K1_MQTT_STREAM_PROFILE.md new file mode 100644 index 0000000..6ebe876 --- /dev/null +++ b/docs/05_K1_MQTT_STREAM_PROFILE.md @@ -0,0 +1,164 @@ +# Verified K1 MQTT stream profile + +Status: verified against one owner-controlled LixelKity K1 running firmware +3.0.2. This is a capture and decoder profile, not a claim of vendor support or a +stable public API. + +## Transport + +After BLE provisioning, the K1 joins the operator's LAN and exposes: + +| Property | Verified value | +| --- | --- | +| Transport | TCP | +| Port | 1883 | +| Application protocol | MQTT 3.1.1 | +| TLS | not used by this direct-LAN path | +| Username/password | not used by the observed client setup | +| K1 address | DHCP/private IPv4, retained only in the ignored session | + +The direct route for the confirmed K1 address used the Mac Wi-Fi interface even +while the Mac's default route was a VPN tunnel. Commands must resolve the route +for the confirmed K1 address and must not scan the surrounding subnet. + +The repository's capture client is subscribe-only. It sends the minimum MQTT +CONNECT, SUBSCRIBE, keepalive and DISCONNECT protocol packets; it has no reason +to publish an application command. + +## Verified report topics + +The firmware-3 stream set is: + +```text +lixel/application/report/heartbeat +lixel/application/report/device_status +lixel/application/report/lio_pcl +lixel/application/report/lio_pose +lixel/application/report/modeling +``` + +Additional report topics exist for system errors, modeling status, PGO, +control points and upgrades. The safe discovery subscription is restricted to +`lixel/application/report/#`. It does not subscribe to request topics. + +The application also retains compatibility subscribers for: + +```text +RealtimePointcloud +RealtimePath +DeviceStatus +``` + +No legacy stream was emitted by the tested firmware during the live run. + +## Firmware-3 point cloud + +`lixel/application/report/lio_pcl` contains a protobuf compression envelope: + +```proto +message MqttCompressMsg { + Header header = 1; + CompressionType compression = 2; // 0=LZ4, 1=Zstd + uint32 compressed_size = 3; // expected decoded byte count + bytes compressed_data = 4; +} +``` + +The verified value is compression enum 0. `compressed_data` is a raw LZ4 block, +not an LZ4 frame. The decoder must supply `compressed_size`, enforce the exact +decoded length and reject enum 1 until a bounded Zstd path is reviewed. + +The decoded protobuf is: + +```proto +message LioPclReport { + Header header = 1; + repeated Point points = 2; +} + +message Point { + sint64 x = 1; + sint64 y = 2; + sint64 z = 3; + uint32 rgbi = 4; +} +``` + +Coordinates are ZigZag `sint64` values. Sensor-native metric coordinates are +`(x/header.scaler, y/header.scaler, z/header.scaler)`; a zero scaler is invalid. +The tested frames consistently used scaler 1000. + +The application uses only `rgbi & 0xff` as intensity/alpha. The connector keeps +the complete `uint32`; interpreting the upper 24 bits as RGB remains unverified. +Raw coordinates are preserved. Any Y/Z swap belongs in a derived visualization, +not in capture storage. + +## Firmware-3 pose + +`lixel/application/report/lio_pose` is an uncompressed protobuf: + +```proto +message LioPoseReport { + Header header = 1; + PoseStamped pose = 2; + float distance = 3; + float pose_accuracy = 4; +} + +message PoseStamped { sint64 stamp = 1; Pose pose = 2; } +message Pose { Point position = 1; Quaternion orientation = 2; } +message Point { double x = 1; double y = 2; double z = 3; } +message Quaternion { double x = 1; double y = 2; double z = 3; double w = 4; } +``` + +Storage keeps position `(x,y,z)` and quaternion `(x,y,z,w)` exactly as received. +Non-finite values are rejected. Quaternion normalization, coordinate transforms +and sensor-to-vehicle extrinsics are derived operations and must not overwrite +raw values. + +## Legacy formats + +`RealtimePointcloud` starts with a 12-byte envelope. Little-endian uint32 at +offset 0 is the point stride. Records contain float32 `x,y,z` at offsets 0,4,8, +RGB bytes at 12,13,14 and, when stride is at least 16, intensity at 15. Envelope +bytes 4 through 11 remain uninterpreted and are retained. + +`RealtimePath` contains float32 position `x,y,z` at offsets 0,4,8; offset 12 is +unknown; quaternion wire order is `w,x,y,z` at offsets 16,20,24,28. The decoded +orientation is exposed as `(x,y,z,w)`. Unknown tail fields are retained. + +`PrePathArray` is exactly 16 little-endian float64 values. + +## Application control boundary + +Static analysis identified the application start/stop topic as +`lixel/application/request/modeling`, QoS 2, with a protobuf +`ModelingRequest`. Start action is 1 and stop action is 2. A valid request also +contains a device/session/OpenAPI header and, for start, project/record/scan/mount +settings. + +Publishing is deliberately not implemented. The physical double-click provides +a verified autonomous start/stop path and avoids inventing session headers or +changing scan settings. A future publisher requires a separate reviewed profile, +explicit confirmation, response handling and rollback. + +## Decoder and capture bounds + +Default defensive limits are applied before allocation or iteration: + +- 2 MiB maximum MQTT payload; +- 1 MiB maximum compressed block; +- 8 MiB maximum decoded block; +- 64:1 maximum claimed expansion ratio; +- 250,000 points per frame; +- bounded protobuf field counts and exact LZ4 decoded length. + +Per-frame failures do not justify firmware writes or speculative recovery. +Always preserve the raw MQTT record and report the decoder error separately. + +## Current camera result + +No independent camera, panorama, JPEG, H.264 or RTSP topic was observed in this +run. `lio_pcl` may carry intensity or packed color information, but that does not +prove access to the two raw panoramic camera streams. Camera discovery remains a +separate evidence gate. diff --git a/docs/lab/001_K1_LIVE_MQTT_20260715.manifest.redacted.json b/docs/lab/001_K1_LIVE_MQTT_20260715.manifest.redacted.json new file mode 100644 index 0000000..6ce69ff --- /dev/null +++ b/docs/lab/001_K1_LIVE_MQTT_20260715.manifest.redacted.json @@ -0,0 +1,114 @@ +{ + "schema_version": 1, + "session_id": "20260715T122850Z_live_power_cycle", + "experiment": "BLE provisioning and physical-button MQTT stream capture", + "tool": { + "version": "0.1.0", + "git_commit": "faa442e+working-tree-lab-tooling" + }, + "host": { + "macos_version": "26.5.1", + "machine": "arm64", + "python_version": "3.12.13", + "wifi_interface": "en0", + "vpn_active": true + }, + "device": { + "model": "XGRIDS/LixelKity K1", + "serial_suffix": null, + "activation_state": "active", + "firmware": "3.0.2" + }, + "network": { + "topology": "current_lan", + "k1_address_redacted": "RFC1918 private IPv4" + }, + "timeline": [ + { + "event": "MQTT standby baseline begins", + "monotonic_seconds": 0.0, + "utc": "2026-07-15T14:05:15.186Z", + "note": "heartbeat and device_status only" + }, + { + "event": "first live pose report", + "monotonic_seconds": 57.778, + "utc": "2026-07-15T14:06:12.964Z", + "note": "physical double-click scan start" + }, + { + "event": "first live point-cloud report", + "monotonic_seconds": 57.795, + "utc": "2026-07-15T14:06:12.981Z", + "note": "firmware-3 lio_pcl topic" + }, + { + "event": "full-payload capture window ends", + "monotonic_seconds": 179.166, + "utc": "2026-07-15T14:08:14.352Z", + "note": "180-second subscriber timeout" + }, + { + "event": "post-stop negative control begins", + "monotonic_seconds": 369.254, + "utc": "2026-07-15T14:11:24.440Z", + "note": "solid-green standby; no lio_pcl/lio_pose/modeling reports" + }, + { + "event": "repository-native standby smoke capture", + "monotonic_seconds": 2443.564, + "utc": "2026-07-15T14:45:58.750Z", + "note": "5-second fixed-topic subscriber; 11 heartbeat/status frames" + } + ], + "artifacts": [ + { + "path": "reference/LG_i18n_125_20250820.apk", + "size_bytes": 118234630, + "sha256": "c83736f782981282b933cd4737c4f9fa46cbd7f651660c1a9057580f7312361f", + "classification": "sensitive" + }, + { + "path": "captures/wifi_provisioning_attempt_01.json", + "size_bytes": 1258, + "sha256": "cc73705c185315286f4009b6619f82a7953b38528a919f7fd2b18ecd788c7637", + "classification": "sensitive" + }, + { + "path": "captures/wifi_provisioning_attempt_02.json", + "size_bytes": 1247, + "sha256": "4acf432dd8797802a0033efb7c89f3c11827f681bbf152249c02cccb41d9f873", + "classification": "sensitive" + }, + { + "path": "captures/mqtt_scan_full_payloads_03.tsv", + "size_bytes": 79937918, + "sha256": "6b2a4a66d24a89ac048a2be11c5ab45923ee60d4ef07ad3ed3fa3cc508bf8500", + "classification": "sensitive" + }, + { + "path": "captures/mqtt_post_stop_metadata_04.tsv", + "size_bytes": 1488, + "sha256": "677835db8175ab6001bec6ecc703413c07795062bb9b4e86c187c77c3e6c73eb", + "classification": "sensitive" + }, + { + "path": "captures/native_mqtt_standby_05/mqtt.raw.k1mqtt", + "size_bytes": 1924, + "sha256": "1ff2984d7f305e41f80c13b820405b9b18c8c79c15efe698d51a02fffa22ca82", + "classification": "sensitive" + }, + { + "path": "analysis/native_mqtt_standby_05.summary.json", + "size_bytes": 1481, + "sha256": "da72847c9454bd69915c65d49101cb93261d034ec1c1db329ead0b20a6dcbb0e", + "classification": "sensitive" + } + ], + "decision": { + "status": "GO", + "gate": "Stage 5 point-cloud and pose decoding", + "reason": "All captured firmware-3 point-cloud and pose frames decoded and correlated with controlled physical motion.", + "next_smallest_experiment": "Longer repository-native capture on verified power bank; keep camera discovery separate." + } +} diff --git a/docs/lab/001_K1_LIVE_MQTT_20260715.redacted.md b/docs/lab/001_K1_LIVE_MQTT_20260715.redacted.md new file mode 100644 index 0000000..6f7a4ba --- /dev/null +++ b/docs/lab/001_K1_LIVE_MQTT_20260715.redacted.md @@ -0,0 +1,117 @@ +# K1 Lab Report 001 — redacted + +Date: 2026-07-15 + +Decision: **GO** for BLE provisioning, LAN association, MQTT capture, firmware-3 +point-cloud decoding and live pose decoding. Camera stream remains unproven. + +## Stand + +- owner-controlled XGRIDS/LixelKity K1; +- observed firmware 3.0.2; +- Apple Silicon MacBook; +- existing TP-Link mesh LAN; +- no LixelGO, phone, SDK, router reconfiguration or firmware change; +- device identity, SSID, password, IP and MAC retained only in ignored artifacts. + +## Physical result + +- power-on: slow blue to solid green standby; +- autonomous scan start: physical double-click from standby; +- start transition: fast green, LiDAR rotation, then scanning green; +- initialization: device held still for at least 20 seconds; +- controlled movement: approximately one metre plus a turn; +- stop: physical double-click, fast green while saving, LiDAR stopped, solid green; +- single-click is not the scan-start action; +- a depleted battery caused one clean power loss; the battery was replaced before + the successful run; +- later transfer from battery/USB connection to a verified power bank did not + reset the device or change the standby LED. + +## BLE and Wi-Fi result + +The reviewed 99-byte firmware-3 provisioning frame was tested twice: + +1. write without response, mirroring the Android application: transport + completed but K1 remained in AP mode; +2. write with response, matching the live CoreBluetooth property: immediate + success, status code 1 and a LAN IPv4 address. + +There was no automatic retry and no third provisioning write. The password was +entered in a hidden local macOS dialog and never appeared in arguments, logs, +artifacts or Git. + +## MQTT result + +Targeted connection to the confirmed K1 address accepted plain MQTT 3.1.1 on +TCP 1883 with no username/password. A subscribe-only client received standby +heartbeat and status at approximately 1 Hz. + +The 180-second scan capture contained 2,836 valid records and no framing/declared +length errors: + +| Topic | Messages | MQTT payload bytes | +| --- | ---: | ---: | +| `device_status` | 180 | 31,830 | +| `heartbeat` | 180 | 17,997 | +| `lio_pcl` | 1,140 | 39,617,777 | +| `lio_pose` | 1,215 | 204,370 | +| `modeling` | 121 | 10,867 | + +After physical stop, a separate 12-second negative-control subscription received +only 12 heartbeat and 12 status messages. It received no `lio_pcl`, `lio_pose` or +`modeling` reports. + +After implementation, the repository-native `k1link net mqtt-capture` command +was smoke-tested for 5 seconds in standby. It completed a fixed allowlisted +subscription, saved 11 messages in the length-framed `.k1mqtt` format and exited +on duration. `k1link analyze mqtt-streams` then verified the raw capture hash and +all frame boundaries; as expected for standby, it found no PCL or pose frames. + +## Decoder result + +All 1,140 point-cloud MQTT payloads decoded successfully: + +- compression: raw LZ4 block, enum 0 in every frame; +- decoded protobuf size: 37,083 through 55,111 bytes per frame; +- scaler: 1000 in every frame; +- 2,703 through 4,065 points per frame; +- total decoded points: 4,165,862; +- decode failures: 0. + +All 1,215 pose payloads decoded successfully: + +- protobuf position and quaternion values were finite; +- first-to-last pose displacement was approximately 1.566 m, consistent with + the operator's controlled movement; +- reported pose accuracy was approximately 0.001 throughout; +- decode failures: 0. + +Scene bounds, trajectory coordinates and raw payloads remain private and ignored. + +## Artifact integrity + +The raw files are under the ignored session +`20260715T122850Z_live_power_cycle`. Only redacted facts and hashes are committed. + +| Artifact | SHA-256 | +| --- | --- | +| owned reference APK | `c83736f782981282b933cd4737c4f9fa46cbd7f651660c1a9057580f7312361f` | +| provisioning attempt 01 | `cc73705c185315286f4009b6619f82a7953b38528a919f7fd2b18ecd788c7637` | +| provisioning attempt 02 | `4acf432dd8797802a0033efb7c89f3c11827f681bbf152249c02cccb41d9f873` | +| full MQTT scan capture | `6b2a4a66d24a89ac048a2be11c5ab45923ee60d4ef07ad3ed3fa3cc508bf8500` | +| post-stop metadata control | `677835db8175ab6001bec6ecc703413c07795062bb9b4e86c187c77c3e6c73eb` | +| native standby `.k1mqtt` smoke capture | `1ff2984d7f305e41f80c13b820405b9b18c8c79c15efe698d51a02fffa22ca82` | +| native aggregate analysis | `da72847c9454bd69915c65d49101cb93261d034ec1c1db329ead0b20a6dcbb0e` | + +## Remaining gates + +1. Integrate the bounded stream decoder with the repository-native binary MQTT + capture format. +2. Run a longer power-bank capture and measure packet loss, CPU, disk and thermal + behavior. +3. Establish K1-to-vehicle/airframe extrinsics and time synchronization before + using pose on a robot or UAV. +4. Determine whether usable RGB is packed into `rgbi` or delivered elsewhere. +5. Treat raw panoramic camera access as unproven until a scan-correlated stream + or endpoint is observed. diff --git a/pyproject.toml b/pyproject.toml index fdff046..a6b9326 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ license = { text = "Proprietary" } authors = [{ name = "NODE.DC" }] dependencies = [ "bleak==3.0.2", + "lz4>=4.4,<5", + "paho-mqtt>=2.1,<3", "rich>=13.9,<15", "typer>=0.15,<1", ] @@ -45,3 +47,6 @@ python_version = "3.12" strict = true packages = ["k1link"] +[[tool.mypy.overrides]] +module = ["lz4", "lz4.*"] +ignore_missing_imports = true diff --git a/src/k1link/analyze/__init__.py b/src/k1link/analyze/__init__.py new file mode 100644 index 0000000..32cce79 --- /dev/null +++ b/src/k1link/analyze/__init__.py @@ -0,0 +1,15 @@ +"""Bounded, offline analysis of sensitive K1 evidence artifacts.""" + +from k1link.analyze.stream_summary import ( + DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES, + MAX_STREAM_SUMMARY_PAYLOAD_BYTES, + StreamSummary, + summarize_mqtt_streams, +) + +__all__ = [ + "DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES", + "MAX_STREAM_SUMMARY_PAYLOAD_BYTES", + "StreamSummary", + "summarize_mqtt_streams", +] diff --git a/src/k1link/analyze/stream_summary.py b/src/k1link/analyze/stream_summary.py new file mode 100644 index 0000000..ffa1340 --- /dev/null +++ b/src/k1link/analyze/stream_summary.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import hashlib +import math +import os +import stat +from dataclasses import dataclass, field +from pathlib import Path +from typing import TypedDict + +from k1link.artifacts import utc_now_iso +from k1link.mqtt import DEFAULT_MAX_MESSAGE_BYTES, iter_capture_frames +from k1link.protocol import ( + DecodeLimits, + StreamDecodeError, + decode_lio_pcl, + decode_lio_pose, +) + +LIO_PCL_TOPIC = "lixel/application/report/lio_pcl" +LIO_POSE_TOPIC = "lixel/application/report/lio_pose" +DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES = DecodeLimits().max_mqtt_payload_bytes +MAX_STREAM_SUMMARY_PAYLOAD_BYTES = DEFAULT_MAX_MESSAGE_BYTES +_HASH_CHUNK_BYTES = 1024 * 1024 + + +class IntRangeSummary(TypedDict): + min: int | None + max: int | None + + +class ScalerSummary(TypedDict): + min: int | None + max: int | None + constant: bool | None + + +class ByteSummary(TypedDict): + total: int + per_frame: IntRangeSummary + + +class PointSummary(TypedDict): + total: int + per_frame: IntRangeSummary + + +class SourceSummary(TypedDict): + bytes: int + sha256: str + + +class LimitSummary(TypedDict): + max_payload_bytes: int + max_compressed_bytes: int + max_decompressed_bytes: int + max_compression_ratio: int + max_points_per_frame: int + + +class FrameSummary(TypedDict): + count: int + payload_bytes: int + encoded_frame_bytes: int + other_count: int + other_payload_bytes: int + + +class DecodeSummary(TypedDict): + attempted: int + successes: int + errors: int + + +class PointCloudSummary(TypedDict): + frame_count: int + payload_bytes: int + decode_successes: int + decode_errors: int + points: PointSummary + scalers: ScalerSummary + compressed_bytes: ByteSummary + decompressed_bytes: ByteSummary + + +class PoseSummary(TypedDict): + frame_count: int + payload_bytes: int + decode_successes: int + decode_errors: int + first_to_last_displacement_meters: float | None + + +class StreamSummary(TypedDict): + schema_version: int + created_at_utc: str + sensitivity: str + source: SourceSummary + limits: LimitSummary + frames: FrameSummary + decoding: DecodeSummary + point_cloud: PointCloudSummary + pose: PoseSummary + + +@dataclass(slots=True) +class _IntRange: + minimum: int | None = None + maximum: int | None = None + + def add(self, value: int) -> None: + if self.minimum is None or value < self.minimum: + self.minimum = value + if self.maximum is None or value > self.maximum: + self.maximum = value + + def summary(self) -> IntRangeSummary: + return {"min": self.minimum, "max": self.maximum} + + +@dataclass(slots=True) +class _PointCloudAccumulator: + frame_count: int = 0 + payload_bytes: int = 0 + decode_successes: int = 0 + decode_errors: int = 0 + point_total: int = 0 + compressed_total: int = 0 + decompressed_total: int = 0 + points_per_frame: _IntRange = field(default_factory=_IntRange) + scalers: _IntRange = field(default_factory=_IntRange) + compressed_per_frame: _IntRange = field(default_factory=_IntRange) + decompressed_per_frame: _IntRange = field(default_factory=_IntRange) + first_scaler: int | None = None + scaler_constant: bool = True + + def record_payload(self, payload_bytes: int) -> None: + self.frame_count += 1 + self.payload_bytes += payload_bytes + + def record_error(self) -> None: + self.decode_errors += 1 + + def record_decoded( + self, + *, + point_count: int, + scaler: int, + compressed_bytes: int, + decompressed_bytes: int, + ) -> None: + self.decode_successes += 1 + self.point_total += point_count + self.compressed_total += compressed_bytes + self.decompressed_total += decompressed_bytes + self.points_per_frame.add(point_count) + self.scalers.add(scaler) + self.compressed_per_frame.add(compressed_bytes) + self.decompressed_per_frame.add(decompressed_bytes) + if self.first_scaler is None: + self.first_scaler = scaler + elif scaler != self.first_scaler: + self.scaler_constant = False + + def summary(self) -> PointCloudSummary: + scaler_summary: ScalerSummary = { + "min": self.scalers.minimum, + "max": self.scalers.maximum, + "constant": self.scaler_constant if self.decode_successes else None, + } + return { + "frame_count": self.frame_count, + "payload_bytes": self.payload_bytes, + "decode_successes": self.decode_successes, + "decode_errors": self.decode_errors, + "points": { + "total": self.point_total, + "per_frame": self.points_per_frame.summary(), + }, + "scalers": scaler_summary, + "compressed_bytes": { + "total": self.compressed_total, + "per_frame": self.compressed_per_frame.summary(), + }, + "decompressed_bytes": { + "total": self.decompressed_total, + "per_frame": self.decompressed_per_frame.summary(), + }, + } + + +@dataclass(slots=True) +class _PoseAccumulator: + frame_count: int = 0 + payload_bytes: int = 0 + decode_successes: int = 0 + decode_errors: int = 0 + first_position: tuple[float, float, float] | None = None + last_position: tuple[float, float, float] | None = None + + def record_payload(self, payload_bytes: int) -> None: + self.frame_count += 1 + self.payload_bytes += payload_bytes + + def record_error(self) -> None: + self.decode_errors += 1 + + def record_decoded(self, position: tuple[float, float, float]) -> None: + self.decode_successes += 1 + if self.first_position is None: + self.first_position = position + self.last_position = position + + def summary(self) -> PoseSummary: + displacement: float | None = None + if self.first_position is not None and self.last_position is not None: + candidate = math.dist(self.first_position, self.last_position) + if math.isfinite(candidate): + displacement = candidate + return { + "frame_count": self.frame_count, + "payload_bytes": self.payload_bytes, + "decode_successes": self.decode_successes, + "decode_errors": self.decode_errors, + "first_to_last_displacement_meters": displacement, + } + + +def summarize_mqtt_streams( + capture: Path, + *, + max_payload_bytes: int = DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES, +) -> StreamSummary: + """Stream a raw MQTT capture into an aggregate-only, coordinate-free summary.""" + if not 1 <= max_payload_bytes <= MAX_STREAM_SUMMARY_PAYLOAD_BYTES: + raise ValueError( + "max_payload_bytes must be between 1 and " + f"{MAX_STREAM_SUMMARY_PAYLOAD_BYTES}" + ) + + capture_path = capture.expanduser() + before = capture_path.stat() + if not stat.S_ISREG(before.st_mode): + raise ValueError("capture must be a regular file") + + capture_sha256 = _sha256_file(capture_path) + hashed = capture_path.stat() + if _file_identity(before) != _file_identity(hashed): + raise RuntimeError("capture changed while it was being hashed") + + limits = DecodeLimits(max_mqtt_payload_bytes=max_payload_bytes) + point_cloud = _PointCloudAccumulator() + pose = _PoseAccumulator() + frame_count = 0 + payload_bytes = 0 + encoded_frame_bytes = 0 + other_count = 0 + other_payload_bytes = 0 + + for capture_frame in iter_capture_frames( + capture_path, + max_payload_bytes=max_payload_bytes, + ): + frame_count += 1 + frame_payload_bytes = len(capture_frame.payload) + payload_bytes += frame_payload_bytes + encoded_frame_bytes += capture_frame.raw_frame_bytes + + if capture_frame.topic == LIO_PCL_TOPIC: + point_cloud.record_payload(frame_payload_bytes) + try: + decoded = decode_lio_pcl(capture_frame.payload, limits) + except StreamDecodeError: + point_cloud.record_error() + continue + point_cloud.record_decoded( + point_count=len(decoded.points), + scaler=decoded.header.scaler, + compressed_bytes=decoded.compressed_bytes, + decompressed_bytes=decoded.decompressed_bytes, + ) + continue + + if capture_frame.topic == LIO_POSE_TOPIC: + pose.record_payload(frame_payload_bytes) + try: + decoded_pose = decode_lio_pose(capture_frame.payload, limits) + except StreamDecodeError: + pose.record_error() + continue + pose.record_decoded(decoded_pose.position_xyz) + continue + + # Unknown topic text is intentionally neither retained nor emitted: a malformed + # capture could place an identifier in that field. + other_count += 1 + other_payload_bytes += frame_payload_bytes + + after = capture_path.stat() + if _file_identity(hashed) != _file_identity(after): + raise RuntimeError("capture changed while it was being analyzed") + + decode_successes = point_cloud.decode_successes + pose.decode_successes + decode_errors = point_cloud.decode_errors + pose.decode_errors + return { + "schema_version": 1, + "created_at_utc": utc_now_iso(), + "sensitivity": ( + "sensitive derived K1 stream statistics; keep in ignored storage; " + "identifiers, keys, error text and coordinates are omitted" + ), + "source": { + "bytes": hashed.st_size, + "sha256": capture_sha256, + }, + "limits": { + "max_payload_bytes": limits.max_mqtt_payload_bytes, + "max_compressed_bytes": limits.max_compressed_bytes, + "max_decompressed_bytes": limits.max_decompressed_bytes, + "max_compression_ratio": limits.max_compression_ratio, + "max_points_per_frame": limits.max_points_per_frame, + }, + "frames": { + "count": frame_count, + "payload_bytes": payload_bytes, + "encoded_frame_bytes": encoded_frame_bytes, + "other_count": other_count, + "other_payload_bytes": other_payload_bytes, + }, + "decoding": { + "attempted": point_cloud.frame_count + pose.frame_count, + "successes": decode_successes, + "errors": decode_errors, + }, + "point_cloud": point_cloud.summary(), + "pose": pose.summary(), + } + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(_HASH_CHUNK_BYTES), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _file_identity(file_stat: os.stat_result) -> tuple[int, int, int, int]: + return ( + file_stat.st_dev, + file_stat.st_ino, + file_stat.st_size, + file_stat.st_mtime_ns, + ) diff --git a/src/k1link/ble/reader.py b/src/k1link/ble/reader.py new file mode 100644 index 0000000..e5a229a --- /dev/null +++ b/src/k1link/ble/reader.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import asyncio +from importlib.metadata import version +from typing import TypedDict + +from bleak import BleakClient, BleakScanner +from bleak.exc import BleakDeviceNotFoundError + +from k1link.artifacts import utc_now_iso + + +class CharacteristicReadResult(TypedDict): + schema_version: int + started_at_utc: str + completed_at_utc: str + adapter: str + bleak_version: str + device_macos_uuid: str + device_name: str + characteristic_uuid: str + operation: str + value_length: int + value_hex: str + + +async def read_characteristic_once( + device_macos_uuid: str, + characteristic_uuid: str, + timeout_seconds: float, +) -> CharacteristicReadResult: + """Read one explicitly selected characteristic once without pairing or writes.""" + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + + started_at = utc_now_iso() + async with asyncio.timeout(timeout_seconds): + device = await BleakScanner.find_device_by_address( + device_macos_uuid, + timeout=min(20.0, timeout_seconds), + ) + if device is None: + raise BleakDeviceNotFoundError( + device_macos_uuid, + "Device was not rediscovered; keep the K1 powered and nearby.", + ) + + async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: + characteristic = client.services.get_characteristic(characteristic_uuid) + if characteristic is None: + raise ValueError(f"Characteristic not found: {characteristic_uuid}") + if "read" not in characteristic.properties: + raise ValueError(f"Characteristic is not readable: {characteristic_uuid}") + value = bytes(await client.read_gatt_char(characteristic)) + + return { + "schema_version": 1, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "adapter": "CoreBluetooth", + "bleak_version": version("bleak"), + "device_macos_uuid": device_macos_uuid, + "device_name": client.name, + "characteristic_uuid": characteristic.uuid, + "operation": "single_gatt_read_no_pair_no_write", + "value_length": len(value), + "value_hex": value.hex(), + } diff --git a/src/k1link/ble/wifi_provisioning.py b/src/k1link/ble/wifi_provisioning.py new file mode 100644 index 0000000..833106c --- /dev/null +++ b/src/k1link/ble/wifi_provisioning.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import asyncio +import ipaddress +from importlib.metadata import version +from time import monotonic +from typing import Literal, TypedDict + +from bleak import BleakClient, BleakScanner +from bleak.exc import BleakDeviceNotFoundError, BleakError + +from k1link.artifacts import utc_now_iso + +PROFILE_ID = "xgrids-k1-fw3-wifi-v1" +SERVICE_UUID = "00007f00-0000-1000-8000-00805f9b34fb" +WRITE_CHARACTERISTIC_UUID = "00007f01-0000-1000-8000-00805f9b34fb" +STATUS_CHARACTERISTIC_UUID = "00007f02-0000-1000-8000-00805f9b34fb" +FRAME_LENGTH = 99 +SSID_SLOT_LENGTH = 32 +PASSWORD_SLOT_LENGTH = 64 +AP_FALLBACK_IPV4 = "192.168.56.1" +ProvisioningOutcome = Literal[ + "lan_address_observed", + "status_changed", + "no_status_change_before_timeout", + "ble_disconnected_after_write", +] +WriteMode = Literal["auto", "with_response", "without_response"] +ResolvedWriteMode = Literal["with_response", "without_response"] + + +class WifiStatus(TypedDict): + value_length: int + mode: str | None + ipv4: str | None + status_code: int + reserved: int | None + trailer_hex: str + + +class StatusObservation(TypedDict): + observed_at_utc: str + seconds_after_write: float + status: WifiStatus + + +class WifiProvisioningResult(TypedDict): + schema_version: int + profile_id: str + started_at_utc: str + completed_at_utc: str + adapter: str + bleak_version: str + device_macos_uuid: str + device_name: str + service_uuid: str + write_characteristic_uuid: str + status_characteristic_uuid: str + operation: str + write_mode: ResolvedWriteMode + write_without_response_advertised: bool + max_write_without_response_size: int + frame_length: int + baseline_status: WifiStatus + observations: list[StatusObservation] + outcome: ProvisioningOutcome + + +def build_wifi_provisioning_frame(ssid: str, password: str) -> bytearray: + """Build the deterministic 99-byte frame used by LixelGO for K1 Wi-Fi setup.""" + ssid_bytes = ssid.encode("utf-8") + password_bytes = password.encode("utf-8") + + if not ssid_bytes: + raise ValueError("SSID must not be empty") + if not password_bytes: + raise ValueError("Wi-Fi password must not be empty") + if len(ssid_bytes) > SSID_SLOT_LENGTH: + raise ValueError("SSID must be at most 32 UTF-8 bytes") + if len(password_bytes) > PASSWORD_SLOT_LENGTH: + raise ValueError("Wi-Fi password must be at most 64 UTF-8 bytes") + + frame = bytearray(FRAME_LENGTH) + frame[0] = len(ssid_bytes) + frame[1 : 1 + len(ssid_bytes)] = ssid_bytes + frame[33] = len(password_bytes) + frame[34 : 34 + len(password_bytes)] = password_bytes + frame[98] = 0 + return frame + + +def parse_wifi_status(value: bytes) -> WifiStatus: + """Parse the non-secret status frame returned by the K1 read characteristic.""" + if len(value) < 51: + raise ValueError("K1 Wi-Fi status must contain at least 51 bytes") + + mode_length = value[0] + if mode_length > SSID_SLOT_LENGTH: + raise ValueError("K1 Wi-Fi status mode length is invalid") + try: + mode = value[1 : 1 + mode_length].decode("utf-8") if mode_length else None + except UnicodeDecodeError as exc: + raise ValueError("K1 Wi-Fi status mode is not valid UTF-8") from exc + + address_length = value[33] + address_start = 34 + address_end = address_start + address_length + if address_end > len(value): + raise ValueError("K1 Wi-Fi status address length exceeds the frame") + + ipv4: str | None = None + if address_length: + try: + address = ipaddress.ip_address(value[address_start:address_end]) + except ValueError: + address = None + if isinstance(address, ipaddress.IPv4Address): + ipv4 = str(address) + + return { + "value_length": len(value), + "mode": mode, + "ipv4": ipv4, + "status_code": value[50], + "reserved": value[51] if len(value) > 51 else None, + "trailer_hex": value[52:].hex() if len(value) > 52 else "", + } + + +def _outcome( + baseline: WifiStatus, + observations: list[StatusObservation], + disconnected: bool, +) -> ProvisioningOutcome: + if observations: + final = observations[-1]["status"] + if final["ipv4"] not in (None, AP_FALLBACK_IPV4): + return "lan_address_observed" + if final != baseline: + return "status_changed" + if disconnected: + return "ble_disconnected_after_write" + return "no_status_change_before_timeout" + + +async def provision_wifi_once( + device_macos_uuid: str, + ssid: str, + password: str, + timeout_seconds: float = 45.0, + poll_interval_seconds: float = 1.0, + write_mode: WriteMode = "auto", +) -> WifiProvisioningResult: + """Perform one reviewed provisioning write and poll the K1 status characteristic.""" + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + if poll_interval_seconds <= 0: + raise ValueError("poll_interval_seconds must be positive") + if write_mode not in ("auto", "with_response", "without_response"): + raise ValueError(f"Unsupported write mode: {write_mode}") + + frame = build_wifi_provisioning_frame(ssid, password) + started_at = utc_now_iso() + observations: list[StatusObservation] = [] + disconnected = False + + try: + async with asyncio.timeout(timeout_seconds + 25.0): + device = await BleakScanner.find_device_by_address( + device_macos_uuid, + timeout=min(20.0, timeout_seconds), + ) + if device is None: + raise BleakDeviceNotFoundError( + device_macos_uuid, + "Device was not rediscovered; keep the K1 powered and nearby.", + ) + + async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: + device_name = client.name + service = client.services.get_service(SERVICE_UUID) + write_characteristic = client.services.get_characteristic( + WRITE_CHARACTERISTIC_UUID + ) + status_characteristic = client.services.get_characteristic( + STATUS_CHARACTERISTIC_UUID + ) + if service is None: + raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}") + if write_characteristic is None: + raise ValueError( + "Reviewed K1 write characteristic not found: " + f"{WRITE_CHARACTERISTIC_UUID}" + ) + if status_characteristic is None: + raise ValueError( + "Reviewed K1 status characteristic not found: " + f"{STATUS_CHARACTERISTIC_UUID}" + ) + if write_characteristic.service_uuid != service.uuid: + raise ValueError("K1 write characteristic is attached to an unexpected service") + if status_characteristic.service_uuid != service.uuid: + raise ValueError( + "K1 status characteristic is attached to an unexpected service" + ) + if "read" not in status_characteristic.properties: + raise ValueError("Reviewed K1 status characteristic is not readable") + + properties = set(write_characteristic.properties) + max_without_response = ( + write_characteristic.max_write_without_response_size + ) + resolved_write_mode: ResolvedWriteMode + if write_mode == "auto": + if "write-without-response" in properties: + resolved_write_mode = "without_response" + elif "write" in properties: + resolved_write_mode = "with_response" + else: + raise ValueError("Reviewed K1 characteristic is not writable") + elif write_mode == "with_response": + if "write" not in properties: + raise ValueError( + "Reviewed K1 characteristic does not advertise writes with response" + ) + resolved_write_mode = "with_response" + else: + if len(frame) > max_without_response: + raise ValueError( + "Provisioning frame exceeds the negotiated write-without-response size" + ) + resolved_write_mode = "without_response" + + baseline_value = bytes(await client.read_gatt_char(status_characteristic)) + baseline = parse_wifi_status(baseline_value) + + await client.write_gatt_char( + write_characteristic, + frame, + response=resolved_write_mode == "with_response", + ) + write_completed = monotonic() + deadline = write_completed + timeout_seconds + + while monotonic() < deadline: + try: + value = bytes(await client.read_gatt_char(status_characteristic)) + except BleakError: + if not client.is_connected: + disconnected = True + break + raise + status = parse_wifi_status(value) + observation: StatusObservation = { + "observed_at_utc": utc_now_iso(), + "seconds_after_write": round(monotonic() - write_completed, 3), + "status": status, + } + if not observations or status != observations[-1]["status"]: + observations.append(observation) + if status["ipv4"] not in (None, AP_FALLBACK_IPV4): + break + await asyncio.sleep(poll_interval_seconds) + + return { + "schema_version": 1, + "profile_id": PROFILE_ID, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "adapter": "CoreBluetooth", + "bleak_version": version("bleak"), + "device_macos_uuid": device_macos_uuid, + "device_name": device_name, + "service_uuid": service.uuid, + "write_characteristic_uuid": write_characteristic.uuid, + "status_characteristic_uuid": status_characteristic.uuid, + "operation": "single_reviewed_wifi_provisioning_write", + "write_mode": resolved_write_mode, + "write_without_response_advertised": ( + "write-without-response" in properties + ), + "max_write_without_response_size": max_without_response, + "frame_length": len(frame), + "baseline_status": baseline, + "observations": observations, + "outcome": _outcome(baseline, observations, disconnected), + } + finally: + frame[:] = b"\x00" * len(frame) diff --git a/src/k1link/cli.py b/src/k1link/cli.py index 8adb5e3..7af5270 100644 --- a/src/k1link/cli.py +++ b/src/k1link/cli.py @@ -15,10 +15,28 @@ from rich.console import Console from rich.table import Table from k1link import __version__ +from k1link.analyze import ( + DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES, + MAX_STREAM_SUMMARY_PAYLOAD_BYTES, + summarize_mqtt_streams, +) from k1link.artifacts import write_json_atomic from k1link.ble.gatt import dump_metadata from k1link.ble.scanner import scan +from k1link.ble.wifi_provisioning import ( + PROFILE_ID, + WriteMode, + provision_wifi_once, +) +from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials +from k1link.mqtt import ( + DEFAULT_MAX_MESSAGE_BYTES, + MAX_CONFIGURABLE_MESSAGE_BYTES, + CaptureError, + capture_mqtt, +) from k1link.net.snapshot import snapshot +from k1link.usb.snapshot import snapshot as usb_snapshot app = typer.Typer( name="k1link", @@ -27,9 +45,13 @@ app = typer.Typer( ) console = Console() ble_app = typer.Typer(help="Bluetooth LE discovery and metadata commands.", no_args_is_help=True) -net_app = typer.Typer(help="Passive local network observation commands.", no_args_is_help=True) +net_app = typer.Typer(help="Read-only local network observation commands.", no_args_is_help=True) +usb_app = typer.Typer(help="Read-only macOS USB metadata commands.", no_args_is_help=True) +analyze_app = typer.Typer(help="Bounded offline evidence analysis commands.", no_args_is_help=True) app.add_typer(ble_app, name="ble") app.add_typer(net_app, name="net") +app.add_typer(usb_app, name="usb") +app.add_typer(analyze_app, name="analyze") class ToolStatus(TypedDict): @@ -245,6 +267,86 @@ def ble_gatt_dump( console.print(f"Saved: {out}") +@ble_app.command("wifi-configure") +def ble_wifi_configure( + device: Annotated[str, typer.Option(help="CoreBluetooth/macOS UUID from ble scan.")], + out: Annotated[ + Path, + typer.Option(help="Ignored sensitive session JSON path; parent directories are created."), + ], + profile: Annotated[ + str, + typer.Option(help="Exact reviewed provisioning profile ID."), + ], + confirm_write: Annotated[ + bool, + typer.Option( + "--confirm-write", + help="Confirm one state-changing BLE Wi-Fi provisioning write.", + ), + ] = False, + write_mode: Annotated[ + WriteMode, + typer.Option(help="ATT write mode; auto follows the live characteristic properties."), + ] = "auto", + timeout: Annotated[ + float, + typer.Option(min=10.0, max=120.0, help="Status polling timeout after the write."), + ] = 45.0, +) -> None: + """Send router credentials once using the reviewed K1 firmware-3 profile.""" + if profile != PROFILE_ID: + console.print(f"[red]Unknown or unreviewed profile:[/red] {profile}") + raise typer.Exit(code=2) + if not confirm_write: + console.print( + "[red]Write not confirmed.[/red] " + "Add --confirm-write after reviewing the profile." + ) + raise typer.Exit(code=2) + + console.print( + "Two local macOS dialogs will request the Wi-Fi name and hidden password. " + "The password is never printed, logged, or written to the result file; " + "the K1 may echo the SSID in the ignored sensitive status result." + ) + try: + ssid, password = prompt_wifi_credentials() + except CredentialDialogError as exc: + console.print(f"[red]Credential entry failed:[/red] {exc}") + raise typer.Exit(code=2) from exc + + console.print("Credentials accepted locally. Starting the single reviewed BLE write.") + try: + result = asyncio.run( + provision_wifi_once( + device, + ssid, + password, + timeout_seconds=timeout, + write_mode=write_mode, + ) + ) + except (BleakError, OSError, TimeoutError, ValueError) as exc: + console.print(f"[red]Wi-Fi provisioning failed:[/red] {type(exc).__name__}: {exc}") + console.print("No automatic retry was attempted.") + raise typer.Exit(code=2) from exc + finally: + password = "" + ssid = "" + + write_json_atomic(out, result) + observations = result["observations"] + final_status = observations[-1]["status"] if observations else result["baseline_status"] + console.print(f"Outcome: {result['outcome']}; ATT mode: {result['write_mode']}") + console.print( + f"Final status code: {final_status['status_code']}; " + f"reported IPv4: {final_status['ipv4'] or '-'}" + ) + console.print("Saved sensitive device/network metadata; do not commit the output.") + console.print(f"Saved: {out}") + + @net_app.command("snapshot") def net_snapshot( out: Annotated[ @@ -259,5 +361,155 @@ def net_snapshot( console.print(f"Saved: {out}") +@net_app.command("mqtt-capture") +def net_mqtt_capture( + host: Annotated[ + str, + typer.Option("--host", help="Confirmed K1 RFC1918 IPv4 address; hostnames are rejected."), + ], + out: Annotated[ + Path, + typer.Option( + "--out", + help="Sensitive output directory; use captures/... so Git ignores it.", + ), + ], + confirm_owned_device: Annotated[ + bool, + typer.Option( + "--confirm-owned-device", + help="Confirm the target is an owner-controlled K1 before connecting.", + ), + ] = False, + port: Annotated[ + int, + typer.Option(min=1, max=65535, help="MQTT broker TCP port."), + ] = 1883, + duration: Annotated[ + float, + typer.Option(min=1.0, max=3600.0, help="Capture duration after SUBACK, in seconds."), + ] = 60.0, + max_message_bytes: Annotated[ + int, + typer.Option( + min=1, + max=MAX_CONFIGURABLE_MESSAGE_BYTES, + help="Abort before storing a payload larger than this byte limit.", + ), + ] = DEFAULT_MAX_MESSAGE_BYTES, +) -> None: + """Capture fixed K1 MQTT report topics once; never publish or reconnect.""" + if not confirm_owned_device: + console.print( + "[red]Target ownership not confirmed.[/red] " + "Add --confirm-owned-device for the confirmed K1 IPv4 address." + ) + raise typer.Exit(code=2) + + console.print( + "Starting one read-only MQTT subscription session. " + "No application messages will be published and no reconnect will be attempted." + ) + try: + result = capture_mqtt( + host, + out, + port=port, + duration_seconds=duration, + max_message_bytes=max_message_bytes, + on_ready=lambda: console.print( + "[green]MQTT subscriptions active; capture timer started.[/green]" + ), + ) + except CaptureError as exc: + console.print(f"[red]MQTT capture failed:[/red] {exc}") + if exc.summary is not None: + console.print(f"Partial artifacts preserved in: {out}") + raise typer.Exit(code=2) from exc + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]MQTT capture failed:[/red] {type(exc).__name__}: {exc}") + raise typer.Exit(code=2) from exc + + console.print( + f"Capture stopped: {result['stop_reason']}; messages: {result['message_count']}; " + f"payload bytes: {result['payload_bytes']}" + ) + console.print("Saved sensitive raw MQTT evidence; do not commit the output.") + console.print(f"Saved: {out}") + + +@usb_app.command("snapshot") +def usb_snapshot_command( + out: Annotated[ + Path, + typer.Option(help="Ignored session JSON path; parent directories are created."), + ], +) -> None: + """Save XGRIDS USB/interface/storage metadata without opening device files.""" + result = usb_snapshot() + write_json_atomic(out, result) + console.print( + "Saved sensitive USB metadata only; no sudo, device-file reads or device writes used." + ) + console.print(f"XGRIDS candidates: {result['xgrids_device_count']}") + console.print(f"Saved: {out}") + + +@analyze_app.command("mqtt-streams") +def analyze_mqtt_streams( + capture: Annotated[ + Path, + typer.Option( + "--capture", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Repository-native mqtt.raw.k1mqtt capture to analyze offline.", + ), + ], + out: Annotated[ + Path, + typer.Option( + "--out", + help=( + "Sensitive atomic JSON output; keep under captures/, sessions/, or " + "artifacts/decoded/." + ), + ), + ], + max_payload_bytes: Annotated[ + int, + typer.Option( + "--max-payload-bytes", + min=1, + max=MAX_STREAM_SUMMARY_PAYLOAD_BYTES, + help="Reject a capture frame larger than this bounded payload limit.", + ), + ] = DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES, +) -> None: + """Summarize captured firmware-3 point-cloud and pose streams without coordinates.""" + if capture.expanduser().resolve() == out.expanduser().resolve(): + console.print("[red]Analysis failed:[/red] capture and output must be different files") + raise typer.Exit(code=2) + + try: + result = summarize_mqtt_streams( + capture, + max_payload_bytes=max_payload_bytes, + ) + write_json_atomic(out, result) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Analysis failed:[/red] {type(exc).__name__}: {exc}") + raise typer.Exit(code=2) from exc + + console.print( + f"Frames: {result['frames']['count']}; decode successes: " + f"{result['decoding']['successes']}; errors: {result['decoding']['errors']}" + ) + console.print("Saved sensitive aggregate-only output; keep it in ignored storage.") + console.print(f"Saved: {out}") + + if __name__ == "__main__": app() diff --git a/src/k1link/macos_credentials.py b/src/k1link/macos_credentials.py new file mode 100644 index 0000000..5836fee --- /dev/null +++ b/src/k1link/macos_credentials.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import platform +import shutil +import subprocess + + +class CredentialDialogError(RuntimeError): + """Raised when the local macOS credential dialog cannot return a value.""" + + +def _dialog_text(prompt: str, hidden: bool) -> str: + if platform.system() != "Darwin": + raise CredentialDialogError("Secure credential dialogs are supported only on macOS") + osascript = shutil.which("osascript") + if osascript is None: + raise CredentialDialogError("osascript is unavailable") + + hidden_clause = " with hidden answer" if hidden else "" + script = ( + f'text returned of (display dialog "{prompt}" default answer ""' + f'{hidden_clause} buttons {{"Отмена", "Продолжить"}} ' + 'default button "Продолжить" cancel button "Отмена")' + ) + try: + result = subprocess.run( + [osascript, "-e", script], + check=False, + capture_output=True, + text=True, + timeout=300, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise CredentialDialogError("macOS credential dialog failed") from exc + + if result.returncode != 0: + raise CredentialDialogError("Credential entry was cancelled") + value = result.stdout.rstrip("\r\n") + if not value: + raise CredentialDialogError("Credential value must not be empty") + return value + + +def prompt_wifi_credentials() -> tuple[str, str]: + """Collect Wi-Fi credentials locally without placing them in command arguments.""" + ssid = _dialog_text("Имя Wi-Fi сети, к которой подключён Mac", hidden=False) + password = _dialog_text("Пароль этой Wi-Fi сети", hidden=True) + return ssid, password diff --git a/src/k1link/mqtt/__init__.py b/src/k1link/mqtt/__init__.py new file mode 100644 index 0000000..76c4a78 --- /dev/null +++ b/src/k1link/mqtt/__init__.py @@ -0,0 +1,27 @@ +"""Read-only MQTT evidence capture for an owner-controlled K1.""" + +from k1link.mqtt.capture import ( + DEFAULT_MAX_MESSAGE_BYTES, + MAX_CONFIGURABLE_MESSAGE_BYTES, + REPORT_TOPICS, + CaptureError, + CaptureFormatError, + CaptureFrame, + CaptureSummary, + capture_mqtt, + iter_capture_frames, + validate_private_ipv4, +) + +__all__ = [ + "DEFAULT_MAX_MESSAGE_BYTES", + "MAX_CONFIGURABLE_MESSAGE_BYTES", + "REPORT_TOPICS", + "CaptureError", + "CaptureFormatError", + "CaptureFrame", + "CaptureSummary", + "capture_mqtt", + "iter_capture_frames", + "validate_private_ipv4", +] diff --git a/src/k1link/mqtt/capture.py b/src/k1link/mqtt/capture.py new file mode 100644 index 0000000..2c36ce1 --- /dev/null +++ b/src/k1link/mqtt/capture.py @@ -0,0 +1,648 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import json +import math +import os +import struct +import time +from collections.abc import Callable, Iterator +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import IO, Literal, TypedDict + +import paho.mqtt.client as mqtt +from paho.mqtt.enums import CallbackAPIVersion +from paho.mqtt.properties import Properties +from paho.mqtt.reasoncodes import ReasonCode + +from k1link.artifacts import utc_now_iso + +REPORT_TOPICS: tuple[str, ...] = ( + "lixel/application/report/#", + "RealtimePointcloud", + "RealtimePath", + "DeviceStatus", +) + +DEFAULT_MAX_MESSAGE_BYTES = 64 * 1024 * 1024 +MAX_CONFIGURABLE_MESSAGE_BYTES = 256 * 1024 * 1024 +MAX_TOPIC_BYTES = 65_535 +CONNECT_TIMEOUT_SECONDS = 10.0 +KEEPALIVE_SECONDS = 30 +LOOP_INTERVAL_SECONDS = 0.25 + +# Eight-byte file signature followed by repeated >IQ, topic UTF-8 bytes, payload bytes. +RAW_MAGIC = b"K1MQTT\x00\x01" +FRAME_HEADER = struct.Struct(">IQ") + +_PRIVATE_NETWORKS = tuple( + ipaddress.ip_network(cidr) for cidr in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16") +) + +StopReason = Literal[ + "duration_elapsed", + "keyboard_interrupt", + "message_too_large", + "connection_failed", + "connection_lost", + "subscription_failed", + "capture_error", +] + + +class ArtifactPaths(TypedDict): + raw: str + metadata_jsonl: str + summary: str + + +class ArtifactHashes(TypedDict): + raw_sha256: str + metadata_jsonl_sha256: str + + +class RawFormat(TypedDict): + magic_hex: str + frame_header_struct: str + frame_layout: str + + +class CaptureSummary(TypedDict): + schema_version: int + created_at_utc: str + completed_at_utc: str + sensitivity: str + target_ipv4: str + target_port: int + mqtt_protocol: str + subscription_qos: int + clean_session: bool + reconnect_enabled: bool + publishing_enabled: bool + subscriptions: list[str] + requested_duration_seconds: float + capture_elapsed_seconds: float + operation_elapsed_seconds: float + max_message_bytes: int + connected: bool + subscribed: bool + stop_reason: StopReason + error: str | None + message_count: int + rejected_message_count: int + payload_bytes: int + raw_bytes: int + topic_counts: dict[str, int] + raw_format: RawFormat + artifacts: ArtifactPaths + artifact_hashes: ArtifactHashes + + +class CaptureError(RuntimeError): + """A one-shot capture failed after preserving all artifacts written so far.""" + + def __init__(self, message: str, summary: CaptureSummary | None = None) -> None: + super().__init__(message) + self.summary = summary + + +class MessageTooLargeError(CaptureError): + """An MQTT message exceeded the configured evidence boundary.""" + + +class CaptureFormatError(ValueError): + """A raw capture is corrupt, truncated or outside configured reader bounds.""" + + +@dataclass(frozen=True, slots=True) +class CaptureFrame: + sequence: int + topic: str + payload: bytes + raw_frame_offset: int + raw_payload_offset: int + raw_frame_bytes: int + + +@dataclass +class _CaptureState: + connected: bool = False + subscribed: bool = False + stopping: bool = False + subscription_mid: int | None = None + stop_reason: StopReason = "capture_error" + error: str | None = None + + +class _CaptureWriter: + def __init__(self, out_dir: Path, max_message_bytes: int) -> None: + self.out_dir = out_dir.expanduser().resolve() + self.raw_path = self.out_dir / "mqtt.raw.k1mqtt" + self.metadata_path = self.out_dir / "mqtt.metadata.jsonl" + self.summary_path = self.out_dir / "mqtt.summary.json" + self.max_message_bytes = max_message_bytes + self.message_count = 0 + self.rejected_message_count = 0 + self.payload_bytes = 0 + self.topic_counts: dict[str, int] = {} + self._raw: IO[bytes] | None = None + self._metadata: IO[str] | None = None + + def open(self) -> None: + self.out_dir.mkdir(parents=True, exist_ok=True) + artifact_paths = (self.raw_path, self.metadata_path, self.summary_path) + existing = [path.name for path in artifact_paths if path.exists()] + if existing: + names = ", ".join(existing) + raise FileExistsError(f"refusing to overwrite existing capture artifact(s): {names}") + + try: + self._raw = _open_binary_exclusive(self.raw_path) + self._raw.write(RAW_MAGIC) + self._metadata = _open_text_exclusive(self.metadata_path) + except BaseException: + with suppress(OSError): + self.close() + raise + + def record(self, message: mqtt.MQTTMessage) -> None: + raw = self._require_raw() + metadata = self._require_metadata() + topic = message.topic + topic_bytes = topic.encode("utf-8") + payload = message.payload + received_at_utc = utc_now_iso() + received_monotonic_ns = time.monotonic_ns() + + if not 1 <= len(topic_bytes) <= MAX_TOPIC_BYTES: + raise ValueError( + f"incoming MQTT topic is {len(topic_bytes)} bytes; " + f"expected 1..{MAX_TOPIC_BYTES}" + ) + + if len(payload) > self.max_message_bytes: + self.rejected_message_count += 1 + record = { + "schema_version": 1, + "record_type": "rejected_message", + "sequence": self.message_count + self.rejected_message_count, + "received_at_utc": received_at_utc, + "received_monotonic_ns": received_monotonic_ns, + "topic": topic, + "payload_bytes": len(payload), + "max_message_bytes": self.max_message_bytes, + "reason": "message_too_large", + } + self._write_metadata(metadata, record) + raise MessageTooLargeError( + f"message on {topic!r} is {len(payload)} bytes; " + f"limit is {self.max_message_bytes} bytes" + ) + + offset = raw.tell() + header = FRAME_HEADER.pack(len(topic_bytes), len(payload)) + raw.write(header) + raw.write(topic_bytes) + raw.write(payload) + raw.flush() + + self.message_count += 1 + self.payload_bytes += len(payload) + self.topic_counts[topic] = self.topic_counts.get(topic, 0) + 1 + frame_bytes = len(header) + len(topic_bytes) + len(payload) + record = { + "schema_version": 1, + "record_type": "message", + "sequence": self.message_count, + "received_at_utc": received_at_utc, + "received_monotonic_ns": received_monotonic_ns, + "topic": topic, + "qos": message.qos, + "retain": message.retain, + "dup": message.dup, + "payload_bytes": len(payload), + "payload_sha256": hashlib.sha256(payload).hexdigest(), + "raw_frame_offset": offset, + "raw_payload_offset": offset + len(header) + len(topic_bytes), + "raw_frame_bytes": frame_bytes, + } + self._write_metadata(metadata, record) + + def close(self) -> None: + first_error: OSError | None = None + # Make raw frames durable before making their JSONL references durable. + for stream in (self._raw, self._metadata): + if stream is None or stream.closed: + continue + try: + stream.flush() + os.fsync(stream.fileno()) + except OSError as exc: + if first_error is None: + first_error = exc + finally: + try: + stream.close() + except OSError as exc: + if first_error is None: + first_error = exc + if first_error is not None: + raise first_error + + @property + def raw_bytes(self) -> int: + if self.raw_path.exists(): + return self.raw_path.stat().st_size + return 0 + + def _require_raw(self) -> IO[bytes]: + if self._raw is None or self._raw.closed: + raise RuntimeError("capture writer is not open") + return self._raw + + def _require_metadata(self) -> IO[str]: + if self._metadata is None or self._metadata.closed: + raise RuntimeError("capture writer is not open") + return self._metadata + + @staticmethod + def _write_metadata(stream: IO[str], record: dict[str, object]) -> None: + stream.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") + stream.flush() + + +def validate_private_ipv4(value: str) -> str: + """Require a literal RFC1918 address so capture cannot target arbitrary hosts.""" + try: + address = ipaddress.ip_address(value) + except ValueError as exc: + raise ValueError("host must be a literal private IPv4 address") from exc + if not isinstance(address, ipaddress.IPv4Address) or not any( + address in network for network in _PRIVATE_NETWORKS + ): + raise ValueError("host must be an RFC1918 private IPv4 address") + return str(address) + + +def iter_capture_frames( + path: Path, + *, + max_payload_bytes: int = MAX_CONFIGURABLE_MESSAGE_BYTES, + max_topic_bytes: int = MAX_TOPIC_BYTES, +) -> Iterator[CaptureFrame]: + """Yield validated frames from a K1 MQTT raw capture without decoding payloads.""" + if not 1 <= max_payload_bytes <= MAX_CONFIGURABLE_MESSAGE_BYTES: + raise ValueError( + "max_payload_bytes must be between 1 and " + f"{MAX_CONFIGURABLE_MESSAGE_BYTES}" + ) + if not 1 <= max_topic_bytes <= MAX_TOPIC_BYTES: + raise ValueError(f"max_topic_bytes must be between 1 and {MAX_TOPIC_BYTES}") + + with path.expanduser().open("rb") as stream: + magic = stream.read(len(RAW_MAGIC)) + if magic != RAW_MAGIC: + if len(magic) < len(RAW_MAGIC): + raise CaptureFormatError("raw capture is truncated before the complete magic") + raise CaptureFormatError("raw capture magic/version is not supported") + + sequence = 0 + while True: + frame_offset = stream.tell() + header = stream.read(FRAME_HEADER.size) + if not header: + return + if len(header) != FRAME_HEADER.size: + raise CaptureFormatError( + f"frame at offset {frame_offset} has a truncated length header" + ) + topic_length, payload_length = FRAME_HEADER.unpack(header) + if not 1 <= topic_length <= max_topic_bytes: + raise CaptureFormatError( + f"frame at offset {frame_offset} topic length {topic_length} " + f"is outside 1..{max_topic_bytes}" + ) + if payload_length > max_payload_bytes: + raise CaptureFormatError( + f"frame at offset {frame_offset} payload length {payload_length} " + f"exceeds {max_payload_bytes}" + ) + + topic_raw = _read_exact( + stream, + topic_length, + description=f"topic at frame offset {frame_offset}", + ) + try: + topic = topic_raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise CaptureFormatError( + f"frame at offset {frame_offset} topic is not valid UTF-8" + ) from exc + payload_offset = stream.tell() + payload = _read_exact( + stream, + payload_length, + description=f"payload at frame offset {frame_offset}", + ) + sequence += 1 + yield CaptureFrame( + sequence=sequence, + topic=topic, + payload=payload, + raw_frame_offset=frame_offset, + raw_payload_offset=payload_offset, + raw_frame_bytes=FRAME_HEADER.size + topic_length + payload_length, + ) + + +def capture_mqtt( + host: str, + out_dir: Path, + *, + port: int = 1883, + duration_seconds: float = 60.0, + max_message_bytes: int = DEFAULT_MAX_MESSAGE_BYTES, + on_ready: Callable[[], None] | None = None, + _client_factory: Callable[[], mqtt.Client] | None = None, +) -> CaptureSummary: + """Capture the fixed K1 report subscriptions once, without publishing or reconnecting.""" + target_ipv4 = validate_private_ipv4(host) + if not 1 <= port <= 65535: + raise ValueError("port must be between 1 and 65535") + if not math.isfinite(duration_seconds) or duration_seconds <= 0: + raise ValueError("duration_seconds must be finite and greater than zero") + if not 1 <= max_message_bytes <= MAX_CONFIGURABLE_MESSAGE_BYTES: + raise ValueError( + "max_message_bytes must be between 1 and " + f"{MAX_CONFIGURABLE_MESSAGE_BYTES}" + ) + + client = ( + _client_factory() + if _client_factory is not None + else mqtt.Client( + callback_api_version=CallbackAPIVersion.VERSION2, + clean_session=True, + protocol=mqtt.MQTTv311, + reconnect_on_failure=False, + ) + ) + writer = _CaptureWriter(out_dir, max_message_bytes) + writer.open() + state = _CaptureState() + created_at_utc = utc_now_iso() + operation_started = time.monotonic() + capture_started: float | None = None + failure: CaptureError | None = None + + def fail(reason: StopReason, message: str) -> None: + if state.error is None: + state.stop_reason = reason + state.error = message + + def on_connect( + callback_client: mqtt.Client, + _userdata: object, + _flags: mqtt.ConnectFlags, + reason_code: ReasonCode, + _properties: Properties | None, + ) -> None: + if reason_code.is_failure: + fail("connection_failed", f"broker rejected connection: {reason_code}") + return + state.connected = True + try: + result, mid = callback_client.subscribe([(topic, 0) for topic in REPORT_TOPICS]) + except (OSError, RuntimeError, ValueError) as exc: + fail( + "subscription_failed", + f"subscribe failed: {type(exc).__name__}: {exc}", + ) + return + if result != mqtt.MQTT_ERR_SUCCESS or mid is None: + fail("subscription_failed", f"subscribe failed: {mqtt.error_string(result)}") + return + state.subscription_mid = mid + + def on_subscribe( + _callback_client: mqtt.Client, + _userdata: object, + mid: int, + reason_codes: list[ReasonCode], + _properties: Properties | None, + ) -> None: + if mid != state.subscription_mid: + fail("subscription_failed", f"unexpected SUBACK message id: {mid}") + return + if len(reason_codes) != len(REPORT_TOPICS) or any( + reason_code.is_failure for reason_code in reason_codes + ): + fail("subscription_failed", "broker rejected one or more fixed subscriptions") + return + state.subscribed = True + + def on_message( + _callback_client: mqtt.Client, + _userdata: object, + message: mqtt.MQTTMessage, + ) -> None: + if state.error is not None: + return + try: + writer.record(message) + except MessageTooLargeError as exc: + fail("message_too_large", str(exc)) + except (OSError, RuntimeError, ValueError) as exc: + fail("capture_error", f"artifact write failed: {type(exc).__name__}: {exc}") + + def on_disconnect( + _callback_client: mqtt.Client, + _userdata: object, + _flags: mqtt.DisconnectFlags, + reason_code: ReasonCode, + _properties: Properties | None, + ) -> None: + if not state.stopping: + fail("connection_lost", f"broker connection ended: {reason_code}") + + client.on_connect = on_connect + client.on_subscribe = on_subscribe + client.on_message = on_message + client.on_disconnect = on_disconnect + + connect_attempted = False + try: + connect_attempted = True + connect_result = client.connect( + target_ipv4, + port=port, + keepalive=KEEPALIVE_SECONDS, + ) + if connect_result != mqtt.MQTT_ERR_SUCCESS: + fail("connection_failed", f"connect failed: {mqtt.error_string(connect_result)}") + + while state.error is None: + now = time.monotonic() + if state.subscribed and capture_started is None: + capture_started = now + if on_ready is not None: + on_ready() + if capture_started is not None and now - capture_started >= duration_seconds: + state.stop_reason = "duration_elapsed" + break + if capture_started is None and now - operation_started >= CONNECT_TIMEOUT_SECONDS: + fail("connection_failed", "timed out waiting for CONNACK/SUBACK") + break + + loop_result = client.loop(timeout=LOOP_INTERVAL_SECONDS) + if loop_result != mqtt.MQTT_ERR_SUCCESS and state.error is None: + fail( + "connection_lost", + f"MQTT network loop failed: {mqtt.error_string(loop_result)}", + ) + except KeyboardInterrupt: + state.stop_reason = "keyboard_interrupt" + except (OSError, RuntimeError, ValueError) as exc: + fail("connection_failed", f"MQTT capture failed: {type(exc).__name__}: {exc}") + finally: + state.stopping = True + if connect_attempted: + try: + client.disconnect() + except (OSError, RuntimeError, ValueError) as exc: + if state.error is None: + fail("capture_error", f"disconnect failed: {type(exc).__name__}: {exc}") + try: + writer.close() + except OSError as exc: + if state.error is None: + fail("capture_error", f"artifact close failed: {type(exc).__name__}: {exc}") + + operation_completed = time.monotonic() + capture_elapsed = ( + 0.0 if capture_started is None else operation_completed - capture_started + ) + + summary = _build_summary( + writer=writer, + target_ipv4=target_ipv4, + port=port, + duration_seconds=duration_seconds, + capture_elapsed=capture_elapsed, + operation_elapsed=operation_completed - operation_started, + max_message_bytes=max_message_bytes, + created_at_utc=created_at_utc, + state=state, + ) + try: + _write_summary_exclusive(writer.summary_path, summary) + except OSError as exc: + raise CaptureError(f"could not write capture summary: {type(exc).__name__}: {exc}") from exc + + if state.error is not None: + failure = CaptureError(state.error, summary) + if failure is not None: + raise failure + return summary + + +def _build_summary( + *, + writer: _CaptureWriter, + target_ipv4: str, + port: int, + duration_seconds: float, + capture_elapsed: float, + operation_elapsed: float, + max_message_bytes: int, + created_at_utc: str, + state: _CaptureState, +) -> CaptureSummary: + return { + "schema_version": 1, + "created_at_utc": created_at_utc, + "completed_at_utc": utc_now_iso(), + "sensitivity": "contains raw K1 MQTT payloads and local addressing; do not commit", + "target_ipv4": target_ipv4, + "target_port": port, + "mqtt_protocol": "3.1.1", + "subscription_qos": 0, + "clean_session": True, + "reconnect_enabled": False, + "publishing_enabled": False, + "subscriptions": list(REPORT_TOPICS), + "requested_duration_seconds": duration_seconds, + "capture_elapsed_seconds": round(capture_elapsed, 6), + "operation_elapsed_seconds": round(operation_elapsed, 6), + "max_message_bytes": max_message_bytes, + "connected": state.connected, + "subscribed": state.subscribed, + "stop_reason": state.stop_reason, + "error": state.error, + "message_count": writer.message_count, + "rejected_message_count": writer.rejected_message_count, + "payload_bytes": writer.payload_bytes, + "raw_bytes": writer.raw_bytes, + "topic_counts": dict(sorted(writer.topic_counts.items())), + "raw_format": { + "magic_hex": RAW_MAGIC.hex(), + "frame_header_struct": FRAME_HEADER.format, + "frame_layout": "topic_length:uint32, payload_length:uint64, topic_utf8, payload", + }, + "artifacts": { + "raw": writer.raw_path.name, + "metadata_jsonl": writer.metadata_path.name, + "summary": writer.summary_path.name, + }, + "artifact_hashes": { + "raw_sha256": _sha256_file(writer.raw_path), + "metadata_jsonl_sha256": _sha256_file(writer.metadata_path), + }, + } + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _read_exact(stream: IO[bytes], length: int, *, description: str) -> bytes: + value = stream.read(length) + if len(value) != length: + raise CaptureFormatError( + f"{description} is truncated: expected {length} bytes, got {len(value)}" + ) + return value + + +def _open_binary_exclusive(path: Path) -> IO[bytes]: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + return os.fdopen(descriptor, "wb") + except BaseException: + os.close(descriptor) + raise + + +def _open_text_exclusive(path: Path) -> IO[str]: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + return os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") + except BaseException: + os.close(descriptor) + raise + + +def _write_summary_exclusive(path: Path, summary: CaptureSummary) -> None: + serialized = json.dumps(summary, ensure_ascii=False, indent=2) + "\n" + with _open_text_exclusive(path) as stream: + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) diff --git a/src/k1link/protocol/__init__.py b/src/k1link/protocol/__init__.py new file mode 100644 index 0000000..ea3ba82 --- /dev/null +++ b/src/k1link/protocol/__init__.py @@ -0,0 +1,35 @@ +"""Verified protocol decoders for captured K1 application streams.""" + +from k1link.protocol.streams import ( + DecodeLimits, + LegacyPoint, + LegacyPointCloudFrame, + LegacyPoseFrame, + LioPoint, + LioPointCloudFrame, + LioPoseFrame, + StreamDecodeError, + UnsupportedCompressionError, + decode_legacy_pointcloud, + decode_legacy_pose, + decode_lio_pcl, + decode_lio_pose, + decode_pre_path_array, +) + +__all__ = [ + "DecodeLimits", + "LegacyPoint", + "LegacyPointCloudFrame", + "LegacyPoseFrame", + "LioPoint", + "LioPointCloudFrame", + "LioPoseFrame", + "StreamDecodeError", + "UnsupportedCompressionError", + "decode_legacy_pointcloud", + "decode_legacy_pose", + "decode_lio_pcl", + "decode_lio_pose", + "decode_pre_path_array", +] diff --git a/src/k1link/protocol/protobuf_wire.py b/src/k1link/protocol/protobuf_wire.py new file mode 100644 index 0000000..84911cd --- /dev/null +++ b/src/k1link/protocol/protobuf_wire.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass + + +class ProtobufWireError(ValueError): + """Raised when a bounded protobuf wire parse fails.""" + + +@dataclass(frozen=True, slots=True) +class ProtoField: + number: int + wire_type: int + value: int | bytes + + +def read_varint(data: bytes, offset: int) -> tuple[int, int]: + """Read one protobuf unsigned varint, bounded to 64 bits.""" + value = 0 + for shift in range(0, 70, 7): + if offset >= len(data): + raise ProtobufWireError("truncated varint") + octet = data[offset] + offset += 1 + if shift == 63 and octet > 1: + raise ProtobufWireError("varint exceeds 64 bits") + value |= (octet & 0x7F) << shift + if not octet & 0x80: + return value, offset + raise ProtobufWireError("varint exceeds 10 bytes") + + +def decode_zigzag64(value: int) -> int: + """Decode protobuf sint64 ZigZag representation.""" + if value < 0 or value > 0xFFFFFFFFFFFFFFFF: + raise ProtobufWireError("ZigZag input is outside uint64") + return (value >> 1) ^ -(value & 1) + + +def iter_fields(data: bytes, *, max_fields: int = 1_000_000) -> Iterator[ProtoField]: + """Iterate supported protobuf fields without recursion or unbounded allocation.""" + if max_fields < 1: + raise ValueError("max_fields must be positive") + + offset = 0 + field_count = 0 + while offset < len(data): + field_count += 1 + if field_count > max_fields: + raise ProtobufWireError(f"message exceeds {max_fields} fields") + + key, offset = read_varint(data, offset) + number = key >> 3 + wire_type = key & 0x07 + if number == 0: + raise ProtobufWireError("protobuf field number zero is invalid") + + if wire_type == 0: + value, offset = read_varint(data, offset) + yield ProtoField(number, wire_type, value) + continue + + if wire_type == 1: + end = offset + 8 + if end > len(data): + raise ProtobufWireError("truncated fixed64 field") + yield ProtoField(number, wire_type, data[offset:end]) + offset = end + continue + + if wire_type == 2: + length, offset = read_varint(data, offset) + end = offset + length + if end > len(data): + raise ProtobufWireError("truncated length-delimited field") + yield ProtoField(number, wire_type, data[offset:end]) + offset = end + continue + + if wire_type == 5: + end = offset + 4 + if end > len(data): + raise ProtobufWireError("truncated fixed32 field") + yield ProtoField(number, wire_type, data[offset:end]) + offset = end + continue + + raise ProtobufWireError(f"unsupported protobuf wire type {wire_type}") diff --git a/src/k1link/protocol/streams.py b/src/k1link/protocol/streams.py new file mode 100644 index 0000000..ee1b235 --- /dev/null +++ b/src/k1link/protocol/streams.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import math +import struct +from dataclasses import dataclass +from typing import NamedTuple + +import lz4.block + +from k1link.protocol.protobuf_wire import ( + ProtobufWireError, + ProtoField, + decode_zigzag64, + iter_fields, +) + + +class StreamDecodeError(ValueError): + """Raised when a K1 stream payload violates its verified bounds or schema.""" + + +class UnsupportedCompressionError(StreamDecodeError): + """Raised for a protocol compression type that has not been verified.""" + + +@dataclass(frozen=True, slots=True) +class DecodeLimits: + max_mqtt_payload_bytes: int = 2 * 1024 * 1024 + max_compressed_bytes: int = 1024 * 1024 + max_decompressed_bytes: int = 8 * 1024 * 1024 + max_compression_ratio: int = 64 + max_points_per_frame: int = 250_000 + + def __post_init__(self) -> None: + values = ( + self.max_mqtt_payload_bytes, + self.max_compressed_bytes, + self.max_decompressed_bytes, + self.max_compression_ratio, + self.max_points_per_frame, + ) + if any(value < 1 for value in values): + raise ValueError("all decode limits must be positive") + + +@dataclass(frozen=True, slots=True) +class MqttHeader: + seq: int + stamp: int + scaler: int + device_id: str + session_id: str + openapi_key: str | None + + +class LioPoint(NamedTuple): + x_raw: int + y_raw: int + z_raw: int + rgbi: int + + @property + def intensity(self) -> int: + """Return the only RGBA interpretation verified in the application.""" + return self.rgbi & 0xFF + + def scaled_xyz(self, scaler: int) -> tuple[float, float, float]: + if scaler == 0: + raise StreamDecodeError("point scaler is zero") + return self.x_raw / scaler, self.y_raw / scaler, self.z_raw / scaler + + +@dataclass(frozen=True, slots=True) +class LioPointCloudFrame: + header: MqttHeader + compression: int + compressed_bytes: int + decompressed_bytes: int + points: tuple[LioPoint, ...] + + +@dataclass(frozen=True, slots=True) +class LioPoseFrame: + header: MqttHeader + pose_stamp: int + position_xyz: tuple[float, float, float] + orientation_xyzw: tuple[float, float, float, float] + distance: float + pose_accuracy: float + + +class LegacyPoint(NamedTuple): + x: float + y: float + z: float + r: int + g: int + b: int + intensity: int + + +@dataclass(frozen=True, slots=True) +class LegacyPointCloudFrame: + envelope: bytes + stride: int + points: tuple[LegacyPoint, ...] + + +@dataclass(frozen=True, slots=True) +class LegacyPoseFrame: + position_xyz: tuple[float, float, float] + orientation_xyzw: tuple[float, float, float, float] + skipped_offset_12: bytes + unknown_tail: bytes + + +def _int_value(field: ProtoField, name: str) -> int: + if field.wire_type != 0 or not isinstance(field.value, int): + raise StreamDecodeError(f"{name} has the wrong protobuf wire type") + return field.value + + +def _bytes_value(field: ProtoField, name: str, wire_type: int = 2) -> bytes: + if field.wire_type != wire_type or not isinstance(field.value, bytes): + raise StreamDecodeError(f"{name} has the wrong protobuf wire type") + return field.value + + +def _float32(field: ProtoField, name: str) -> float: + value = float(struct.unpack(" float: + value = float(struct.unpack(" str: + try: + return _bytes_value(field, name).decode("utf-8") + except UnicodeDecodeError as exc: + raise StreamDecodeError(f"{name} is not valid UTF-8") from exc + + +def _decode_header(payload: bytes) -> MqttHeader: + seq = 0 + stamp = 0 + scaler = 0 + device_id = "" + session_id = "" + openapi_key: str | None = None + for field in iter_fields(payload, max_fields=64): + if field.number == 1: + seq = _int_value(field, "header.seq") + elif field.number == 2: + stamp = decode_zigzag64(_int_value(field, "header.stamp")) + elif field.number == 3: + scaler = decode_zigzag64(_int_value(field, "header.scaler")) + elif field.number == 4: + device_id = _text(field, "header.device_id") + elif field.number == 5: + session_id = _text(field, "header.session_id") + elif field.number == 6: + openapi_key = _text(field, "header.openapi_key") + return MqttHeader(seq, stamp, scaler, device_id, session_id, openapi_key) + + +def _decode_lio_point(payload: bytes) -> LioPoint: + x_raw = 0 + y_raw = 0 + z_raw = 0 + rgbi = 0 + for field in iter_fields(payload, max_fields=16): + if field.number == 1: + x_raw = decode_zigzag64(_int_value(field, "point.x")) + elif field.number == 2: + y_raw = decode_zigzag64(_int_value(field, "point.y")) + elif field.number == 3: + z_raw = decode_zigzag64(_int_value(field, "point.z")) + elif field.number == 4: + rgbi = _int_value(field, "point.rgbi") & 0xFFFFFFFF + return LioPoint(x_raw, y_raw, z_raw, rgbi) + + +def _decode_lio_pcl_report( + payload: bytes, + limits: DecodeLimits, +) -> tuple[MqttHeader, tuple[LioPoint, ...]]: + header: MqttHeader | None = None + points: list[LioPoint] = [] + try: + for field in iter_fields(payload, max_fields=limits.max_points_per_frame + 64): + if field.number == 1: + header = _decode_header(_bytes_value(field, "lio_pcl.header")) + elif field.number == 2: + if len(points) >= limits.max_points_per_frame: + raise StreamDecodeError( + f"point frame exceeds {limits.max_points_per_frame} points" + ) + points.append(_decode_lio_point(_bytes_value(field, "lio_pcl.point"))) + except ProtobufWireError as exc: + raise StreamDecodeError(f"invalid LioPclReport: {exc}") from exc + + if header is None: + raise StreamDecodeError("LioPclReport has no header") + if header.scaler == 0: + raise StreamDecodeError("LioPclReport header scaler is zero") + if not points: + raise StreamDecodeError("LioPclReport has no points") + return header, tuple(points) + + +def decode_lio_pcl(payload: bytes, limits: DecodeLimits | None = None) -> LioPointCloudFrame: + """Decode the verified K1 lio_pcl envelope and raw LZ4 protobuf block.""" + bounds = limits or DecodeLimits() + if len(payload) > bounds.max_mqtt_payload_bytes: + raise StreamDecodeError("lio_pcl MQTT payload exceeds configured limit") + + compression = 0 + decompressed_size = 0 + compressed_data: bytes | None = None + try: + for field in iter_fields(payload, max_fields=64): + if field.number == 2: + compression = _int_value(field, "compression") + elif field.number == 3: + decompressed_size = _int_value(field, "compressed_size") + elif field.number == 4: + compressed_data = _bytes_value(field, "compressed_data") + except ProtobufWireError as exc: + raise StreamDecodeError(f"invalid MqttCompressMsg: {exc}") from exc + + if compression != 0: + raise UnsupportedCompressionError( + f"compression enum {compression} is not the verified raw-LZ4 mode" + ) + if compressed_data is None or not compressed_data: + raise StreamDecodeError("MqttCompressMsg has no compressed_data") + if len(compressed_data) > bounds.max_compressed_bytes: + raise StreamDecodeError("compressed_data exceeds configured limit") + if decompressed_size < 1 or decompressed_size > bounds.max_decompressed_bytes: + raise StreamDecodeError("compressed_size is outside configured bounds") + if decompressed_size > len(compressed_data) * bounds.max_compression_ratio: + raise StreamDecodeError("claimed LZ4 expansion ratio exceeds configured limit") + + try: + decompressed = lz4.block.decompress( + compressed_data, + uncompressed_size=decompressed_size, + ) + except lz4.block.LZ4BlockError as exc: + raise StreamDecodeError(f"raw LZ4 decode failed: {exc}") from exc + if len(decompressed) != decompressed_size: + raise StreamDecodeError( + f"raw LZ4 length mismatch: expected {decompressed_size}, got {len(decompressed)}" + ) + + header, points = _decode_lio_pcl_report(decompressed, bounds) + return LioPointCloudFrame( + header=header, + compression=compression, + compressed_bytes=len(compressed_data), + decompressed_bytes=len(decompressed), + points=points, + ) + + +def _decode_position(payload: bytes) -> tuple[float, float, float]: + values = [0.0, 0.0, 0.0] + for field in iter_fields(payload, max_fields=16): + if 1 <= field.number <= 3: + values[field.number - 1] = _float64(field, f"position.{field.number}") + return values[0], values[1], values[2] + + +def _decode_orientation(payload: bytes) -> tuple[float, float, float, float]: + values = [0.0, 0.0, 0.0, 0.0] + for field in iter_fields(payload, max_fields=16): + if 1 <= field.number <= 4: + values[field.number - 1] = _float64(field, f"orientation.{field.number}") + return values[0], values[1], values[2], values[3] + + +def _decode_pose( + payload: bytes, +) -> tuple[tuple[float, float, float], tuple[float, float, float, float]]: + position = (0.0, 0.0, 0.0) + orientation = (0.0, 0.0, 0.0, 0.0) + for field in iter_fields(payload, max_fields=16): + if field.number == 1: + position = _decode_position(_bytes_value(field, "pose.position")) + elif field.number == 2: + orientation = _decode_orientation(_bytes_value(field, "pose.orientation")) + return position, orientation + + +def _decode_pose_stamped( + payload: bytes, +) -> tuple[int, tuple[float, float, float], tuple[float, float, float, float]]: + stamp = 0 + position = (0.0, 0.0, 0.0) + orientation = (0.0, 0.0, 0.0, 0.0) + for field in iter_fields(payload, max_fields=16): + if field.number == 1: + stamp = decode_zigzag64(_int_value(field, "pose_stamp.stamp")) + elif field.number == 2: + position, orientation = _decode_pose(_bytes_value(field, "pose_stamp.pose")) + return stamp, position, orientation + + +def decode_lio_pose(payload: bytes, limits: DecodeLimits | None = None) -> LioPoseFrame: + """Decode a direct lixel/application/report/lio_pose protobuf payload.""" + bounds = limits or DecodeLimits() + if len(payload) > bounds.max_mqtt_payload_bytes: + raise StreamDecodeError("lio_pose MQTT payload exceeds configured limit") + + header: MqttHeader | None = None + pose_stamp = 0 + position = (0.0, 0.0, 0.0) + orientation = (0.0, 0.0, 0.0, 0.0) + distance = 0.0 + pose_accuracy = 0.0 + try: + for field in iter_fields(payload, max_fields=64): + if field.number == 1: + header = _decode_header(_bytes_value(field, "lio_pose.header")) + elif field.number == 2: + pose_stamp, position, orientation = _decode_pose_stamped( + _bytes_value(field, "lio_pose.pose") + ) + elif field.number == 3: + distance = _float32(field, "lio_pose.distance") + elif field.number == 4: + pose_accuracy = _float32(field, "lio_pose.pose_accuracy") + except ProtobufWireError as exc: + raise StreamDecodeError(f"invalid LioPoseReport: {exc}") from exc + + if header is None: + raise StreamDecodeError("LioPoseReport has no header") + return LioPoseFrame( + header=header, + pose_stamp=pose_stamp, + position_xyz=position, + orientation_xyzw=orientation, + distance=distance, + pose_accuracy=pose_accuracy, + ) + + +def decode_legacy_pointcloud( + payload: bytes, + *, + max_points: int = 250_000, +) -> LegacyPointCloudFrame: + """Decode the verified legacy RealtimePointcloud envelope and point records.""" + if max_points < 1: + raise ValueError("max_points must be positive") + if len(payload) < 12: + raise StreamDecodeError("legacy pointcloud payload is shorter than 12-byte envelope") + stride = int.from_bytes(payload[:4], "little") + if stride < 15: + raise StreamDecodeError("legacy point stride is smaller than xyz+rgb") + body = payload[12:] + if len(body) % stride: + raise StreamDecodeError("legacy pointcloud body is not divisible by stride") + point_count = len(body) // stride + if point_count > max_points: + raise StreamDecodeError(f"legacy pointcloud exceeds {max_points} points") + + points: list[LegacyPoint] = [] + for offset in range(0, len(body), stride): + x, y, z = struct.unpack_from("= 16 else 255 + points.append(LegacyPoint(x, y, z, r, g, b, intensity)) + return LegacyPointCloudFrame(payload[:12], stride, tuple(points)) + + +def decode_legacy_pose(payload: bytes) -> LegacyPoseFrame: + """Decode the verified legacy RealtimePath position/quaternion fields.""" + if len(payload) < 32: + raise StreamDecodeError("legacy pose payload is shorter than 32 bytes") + x, y, z = struct.unpack_from(" tuple[float, ...]: + """Decode the exact 16-float64 legacy PrePathArray matrix.""" + if len(payload) != 128: + raise StreamDecodeError("PrePathArray payload must be exactly 128 bytes") + values = struct.unpack("<16d", payload) + if not all(math.isfinite(value) for value in values): + raise StreamDecodeError("PrePathArray contains a non-finite value") + return values diff --git a/src/k1link/usb/__init__.py b/src/k1link/usb/__init__.py new file mode 100644 index 0000000..4a50f4b --- /dev/null +++ b/src/k1link/usb/__init__.py @@ -0,0 +1 @@ +"""Read-only USB discovery helpers for macOS.""" diff --git a/src/k1link/usb/snapshot.py b/src/k1link/usb/snapshot.py new file mode 100644 index 0000000..3dc6ef0 --- /dev/null +++ b/src/k1link/usb/snapshot.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import plistlib +import re +import subprocess +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import TypedDict, cast + +from k1link.artifacts import utc_now_iso + +USB_IOREG_COMMAND = ( + "/usr/sbin/ioreg", + "-a", + "-r", + "-c", + "IOUSBHostDevice", + "-l", + "-w", + "0", +) +SERIAL_IOREG_COMMAND = ( + "/usr/sbin/ioreg", + "-a", + "-r", + "-c", + "IOSerialBSDClient", + "-l", + "-w", + "0", +) +DISKUTIL_COMMAND = ("/usr/sbin/diskutil", "list", "-plist", "external", "physical") + + +@dataclass(frozen=True) +class CommandOutput: + argv: tuple[str, ...] + returncode: int | None + stdout: bytes + stderr: bytes + error: str | None + + +CommandRunner = Callable[[list[str]], CommandOutput] + + +class SourceStatus(TypedDict): + name: str + argv: list[str] + ok: bool + returncode: int | None + error: str | None + + +class UsbInterfaceRecord(TypedDict): + name: str | None + function_hints: list[str] + interface_number: int | None + interface_class: int | None + interface_class_hex: str | None + interface_class_name: str | None + interface_subclass: int | None + interface_protocol: int | None + alternate_setting: int | None + configuration_value: int | None + endpoint_count: int | None + + +class UsbDeviceRecord(TypedDict): + product_name: str | None + vendor_name: str | None + serial_number: str | None + vendor_id: int | None + vendor_id_hex: str | None + product_id: int | None + product_id_hex: str | None + device_class: int | None + device_subclass: int | None + device_protocol: int | None + usb_bcd: int | None + device_bcd: int | None + usb_speed: int | None + link_speed_bits_per_second: int | None + usb_address: int | None + location_id: int | None + registry_entry_id: int | None + bsd_names: list[str] + interface_capabilities: list[str] + interfaces: list[UsbInterfaceRecord] + + +class ExternalStorageEntry(TypedDict): + device_identifier: str + parent_device_identifier: str | None + content: str | None + size_bytes: int | None + volume_name: str | None + mount_point: str | None + os_internal: bool | None + xgrids_related: bool + + +class ExternalStorageRecord(TypedDict): + all_disks: list[str] + whole_disks: list[str] + volumes_from_disks: list[str] + entries: list[ExternalStorageEntry] + + +class UsbModemRecord(TypedDict): + callout_device: str | None + dialin_device: str | None + tty_base_name: str | None + client_type: str | None + + +class SafetyRecord(TypedDict): + metadata_only: bool + sudo_used: bool + device_file_contents_read: bool + device_writes_performed: bool + + +class UsbSnapshot(TypedDict): + schema_version: int + created_at_utc: str + sensitivity: str + safety: SafetyRecord + sources: list[SourceStatus] + xgrids_device_count: int + xgrids_devices: list[UsbDeviceRecord] + external_storage: ExternalStorageRecord + usbmodem_device_names: list[str] + usbmodem_devices: list[UsbModemRecord] + + +def run_command(argv: list[str]) -> CommandOutput: + """Run a fixed read-only macOS metadata command without privilege escalation.""" + try: + result = subprocess.run( + argv, + check=False, + capture_output=True, + timeout=20, + ) + except (OSError, subprocess.SubprocessError) as exc: + return CommandOutput( + argv=tuple(argv), + returncode=None, + stdout=b"", + stderr=b"", + error=f"{type(exc).__name__}: {exc}", + ) + return CommandOutput( + argv=tuple(argv), + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + error=None, + ) + + +def _node(value: object) -> dict[str, object] | None: + if not isinstance(value, dict): + return None + raw = cast("dict[object, object]", value) + return {key: item for key, item in raw.items() if isinstance(key, str)} + + +def _items(value: object) -> list[object]: + if not isinstance(value, list): + return [] + return cast("list[object]", value) + + +def _walk_registry_nodes(value: object) -> Iterator[dict[str, object]]: + for item in _items(value): + yield from _walk_registry_nodes(item) + + node = _node(value) + if node is None: + return + yield node + yield from _walk_registry_nodes(node.get("IORegistryEntryChildren")) + + +def _string(node: dict[str, object], *keys: str) -> str | None: + for key in keys: + value = node.get(key) + if isinstance(value, str): + return value + return None + + +def _integer(node: dict[str, object], key: str) -> int | None: + value = node.get(key) + if isinstance(value, int) and not isinstance(value, bool): + return value + return None + + +def _boolean(node: dict[str, object], key: str) -> bool | None: + value = node.get(key) + return value if isinstance(value, bool) else None + + +def _hex_id(value: int | None) -> str | None: + return None if value is None else f"0x{value:04x}" + + +def _is_xgrids_device(node: dict[str, object]) -> bool: + names = [ + _string(node, "USB Product Name"), + _string(node, "kUSBProductString"), + _string(node, "IORegistryEntryName"), + _string(node, "USB Vendor Name"), + _string(node, "kUSBVendorString"), + ] + identity = " ".join(name.casefold() for name in names if name is not None) + return "xgrids" in identity or "lixel" in identity + + +def _usb_class_name(interface_class: int | None) -> str | None: + if interface_class is None: + return None + return { + 0x02: "communications_and_cdc_control", + 0x08: "mass_storage", + 0x0A: "cdc_data", + 0xE0: "wireless_controller", + 0xFF: "vendor_specific", + }.get(interface_class) + + +def _function_hints( + name: str | None, + interface_class: int | None, + interface_subclass: int | None, +) -> list[str]: + normalized = (name or "").casefold() + hints: set[str] = set() + if "rndis" in normalized: + hints.add("rndis") + if "mass storage" in normalized or interface_class == 0x08: + hints.add("mass_storage") + if "ncm" in normalized or (interface_class == 0x02 and interface_subclass == 0x0D): + hints.add("ncm") + if any(marker in normalized for marker in ("serial", "modem", "acm")) or ( + interface_class == 0x02 and interface_subclass == 0x02 + ): + hints.add("serial") + if interface_class == 0x0A: + hints.add("cdc_data") + return sorted(hints) + + +def _interface_record(node: dict[str, object]) -> UsbInterfaceRecord: + name = _string(node, "kUSBString", "IORegistryEntryName") + interface_class = _integer(node, "bInterfaceClass") + interface_subclass = _integer(node, "bInterfaceSubClass") + return { + "name": name, + "function_hints": _function_hints(name, interface_class, interface_subclass), + "interface_number": _integer(node, "bInterfaceNumber"), + "interface_class": interface_class, + "interface_class_hex": _hex_id(interface_class), + "interface_class_name": _usb_class_name(interface_class), + "interface_subclass": interface_subclass, + "interface_protocol": _integer(node, "bInterfaceProtocol"), + "alternate_setting": _integer(node, "bAlternateSetting"), + "configuration_value": _integer(node, "bConfigurationValue"), + "endpoint_count": _integer(node, "bNumEndpoints"), + } + + +def _interface_sort_key(record: UsbInterfaceRecord) -> tuple[bool, int, str]: + number = record["interface_number"] + return (number is None, number if number is not None else 0, record["name"] or "") + + +def _device_record(node: dict[str, object]) -> UsbDeviceRecord: + interfaces = [ + _interface_record(child) + for child in _walk_registry_nodes(node.get("IORegistryEntryChildren")) + if _string(child, "IOObjectClass") == "IOUSBHostInterface" + ] + interfaces.sort(key=_interface_sort_key) + + bsd_names = sorted( + { + name + for child in _walk_registry_nodes(node.get("IORegistryEntryChildren")) + if (name := _string(child, "BSD Name")) is not None + } + ) + interface_capabilities = sorted( + {hint for interface in interfaces for hint in interface["function_hints"]} + ) + vendor_id = _integer(node, "idVendor") + product_id = _integer(node, "idProduct") + return { + "product_name": _string( + node, "USB Product Name", "kUSBProductString", "IORegistryEntryName" + ), + "vendor_name": _string(node, "USB Vendor Name", "kUSBVendorString"), + "serial_number": _string(node, "USB Serial Number", "kUSBSerialNumberString"), + "vendor_id": vendor_id, + "vendor_id_hex": _hex_id(vendor_id), + "product_id": product_id, + "product_id_hex": _hex_id(product_id), + "device_class": _integer(node, "bDeviceClass"), + "device_subclass": _integer(node, "bDeviceSubClass"), + "device_protocol": _integer(node, "bDeviceProtocol"), + "usb_bcd": _integer(node, "bcdUSB"), + "device_bcd": _integer(node, "bcdDevice"), + "usb_speed": _integer(node, "USBSpeed"), + "link_speed_bits_per_second": _integer(node, "UsbLinkSpeed"), + "usb_address": _integer(node, "USB Address"), + "location_id": _integer(node, "locationID"), + "registry_entry_id": _integer(node, "IORegistryEntryID"), + "bsd_names": bsd_names, + "interface_capabilities": interface_capabilities, + "interfaces": interfaces, + } + + +def parse_xgrids_devices(plist: object) -> list[UsbDeviceRecord]: + devices = [ + _device_record(node) + for node in (_node(item) for item in _items(plist)) + if node is not None and _is_xgrids_device(node) + ] + devices.sort( + key=lambda record: ( + record["product_name"] or "", + record["serial_number"] or "", + record["location_id"] or 0, + ) + ) + return devices + + +def _string_list(node: dict[str, object], key: str) -> list[str]: + return sorted(item for item in _items(node.get(key)) if isinstance(item, str)) + + +def _disk_root(device_identifier: str) -> str | None: + match = re.fullmatch(r"(disk\d+)(?:s\d+)*", device_identifier) + return None if match is None else match.group(1) + + +def _external_storage_entries( + value: object, + parent_device_identifier: str | None, + xgrids_disk_roots: set[str], +) -> list[ExternalStorageEntry]: + entries: list[ExternalStorageEntry] = [] + for item in _items(value): + node = _node(item) + if node is None: + continue + device_identifier = _string(node, "DeviceIdentifier") + next_parent = parent_device_identifier + if device_identifier is not None: + disk_root = _disk_root(device_identifier) + entries.append( + { + "device_identifier": device_identifier, + "parent_device_identifier": parent_device_identifier, + "content": _string(node, "Content"), + "size_bytes": _integer(node, "Size"), + "volume_name": _string(node, "VolumeName"), + "mount_point": _string(node, "MountPoint"), + "os_internal": _boolean(node, "OSInternal"), + "xgrids_related": bool( + disk_root is not None and disk_root in xgrids_disk_roots + ), + } + ) + next_parent = device_identifier + for child_key in ("Partitions", "APFSVolumes"): + entries.extend( + _external_storage_entries( + node.get(child_key), + next_parent, + xgrids_disk_roots, + ) + ) + return entries + + +def parse_external_storage( + plist: object, + xgrids_devices: list[UsbDeviceRecord], +) -> ExternalStorageRecord: + node = _node(plist) or {} + xgrids_disk_roots = { + root + for device in xgrids_devices + for name in device["bsd_names"] + if (root := _disk_root(name)) is not None + } + entries = _external_storage_entries( + node.get("AllDisksAndPartitions"), + None, + xgrids_disk_roots, + ) + entries.sort(key=lambda entry: entry["device_identifier"]) + return { + "all_disks": _string_list(node, "AllDisks"), + "whole_disks": _string_list(node, "WholeDisks"), + "volumes_from_disks": _string_list(node, "VolumesFromDisks"), + "entries": entries, + } + + +def _is_usbmodem_path(path: str | None) -> bool: + return path is not None and PurePosixPath(path).name.startswith(("cu.usbmodem", "tty.usbmodem")) + + +def parse_usbmodem_devices(plist: object) -> tuple[list[str], list[UsbModemRecord]]: + names: set[str] = set() + records: list[UsbModemRecord] = [] + for item in _items(plist): + node = _node(item) + if node is None: + continue + callout = _string(node, "IOCalloutDevice") + dialin = _string(node, "IODialinDevice") + matched_paths: list[str] = [] + for path in (callout, dialin): + if path is not None and _is_usbmodem_path(path): + matched_paths.append(path) + if not matched_paths: + continue + names.update(matched_paths) + records.append( + { + "callout_device": callout, + "dialin_device": dialin, + "tty_base_name": _string(node, "IOTTYBaseName"), + "client_type": _string(node, "IOSerialBSDClientType"), + } + ) + records.sort(key=lambda record: (record["callout_device"] or "", record["dialin_device"] or "")) + return sorted(names), records + + +def _source_status(name: str, output: CommandOutput) -> SourceStatus: + error = output.error + if error is None and output.returncode != 0: + stderr = output.stderr.decode("utf-8", errors="replace").strip() + error = stderr[:500] or f"command exited with status {output.returncode}" + return { + "name": name, + "argv": list(output.argv), + "ok": error is None and output.returncode == 0, + "returncode": output.returncode, + "error": error, + } + + +def _load_plist( + name: str, + output: CommandOutput, + *, + empty_is_list: bool = False, +) -> tuple[object | None, SourceStatus]: + status = _source_status(name, output) + if not status["ok"]: + return None, status + if empty_is_list and not output.stdout.strip(): + return [], status + try: + return plistlib.loads(output.stdout), status + except (plistlib.InvalidFileException, ValueError, TypeError, OverflowError) as exc: + status["ok"] = False + status["error"] = f"invalid plist: {type(exc).__name__}: {exc}" + return None, status + + +def snapshot(runner: CommandRunner = run_command) -> UsbSnapshot: + """Collect USB registry and storage metadata without opening device files.""" + usb_output = runner(list(USB_IOREG_COMMAND)) + serial_output = runner(list(SERIAL_IOREG_COMMAND)) + storage_output = runner(list(DISKUTIL_COMMAND)) + + usb_plist, usb_status = _load_plist("usb_ioreg", usb_output, empty_is_list=True) + serial_plist, serial_status = _load_plist("serial_ioreg", serial_output, empty_is_list=True) + storage_plist, storage_status = _load_plist("external_disks", storage_output) + + devices = parse_xgrids_devices(usb_plist) + usbmodem_names, usbmodem_devices = parse_usbmodem_devices(serial_plist) + external_storage = parse_external_storage(storage_plist, devices) + return { + "schema_version": 1, + "created_at_utc": utc_now_iso(), + "sensitivity": ( + "contains USB serial numbers, BSD device names and external volume metadata; " + "store only in an ignored session path and do not commit" + ), + "safety": { + "metadata_only": True, + "sudo_used": False, + "device_file_contents_read": False, + "device_writes_performed": False, + }, + "sources": [usb_status, serial_status, storage_status], + "xgrids_device_count": len(devices), + "xgrids_devices": devices, + "external_storage": external_storage, + "usbmodem_device_names": usbmodem_names, + "usbmodem_devices": usbmodem_devices, + } diff --git a/tests/test_cli.py b/tests/test_cli.py index f24e208..d05f699 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,6 @@ import json +from pathlib import Path +from typing import Any from typer.testing import CliRunner @@ -21,3 +23,68 @@ def test_doctor_json() -> None: assert isinstance(payload["tools"], list) assert isinstance(payload["network"], dict) assert any(item["name"] == "tcpdump" for item in payload["tools"]) + + +def test_mqtt_capture_requires_owned_device_confirmation(tmp_path: Path) -> None: + result = runner.invoke( + app, + [ + "net", + "mqtt-capture", + "--host", + "192.168.1.20", + "--out", + str(tmp_path / "capture"), + ], + ) + assert result.exit_code == 2 + assert "ownership not confirmed" in result.stdout + assert not (tmp_path / "capture").exists() + + +def test_mqtt_capture_cli_uses_bounded_read_only_capture( + monkeypatch: Any, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + + def fake_capture(host: str, out: Path, **kwargs: object) -> dict[str, object]: + on_ready = kwargs.pop("on_ready") + assert callable(on_ready) + on_ready() + captured.update({"host": host, "out": out, **kwargs}) + return { + "stop_reason": "duration_elapsed", + "message_count": 2, + "payload_bytes": 128, + } + + monkeypatch.setattr("k1link.cli.capture_mqtt", fake_capture) + out = tmp_path / "capture" + result = runner.invoke( + app, + [ + "net", + "mqtt-capture", + "--host", + "10.0.0.42", + "--out", + str(out), + "--duration", + "5", + "--max-message-bytes", + "1024", + "--confirm-owned-device", + ], + ) + + assert result.exit_code == 0 + assert captured == { + "host": "10.0.0.42", + "out": out, + "port": 1883, + "duration_seconds": 5.0, + "max_message_bytes": 1024, + } + assert "messages: 2" in result.stdout + assert "subscriptions active" in result.stdout diff --git a/tests/test_macos_credentials.py b/tests/test_macos_credentials.py new file mode 100644 index 0000000..4c97fbb --- /dev/null +++ b/tests/test_macos_credentials.py @@ -0,0 +1,35 @@ +from unittest.mock import Mock, patch + +import pytest + +from k1link.macos_credentials import CredentialDialogError, _dialog_text + + +@patch("k1link.macos_credentials.platform.system", return_value="Darwin") +@patch("k1link.macos_credentials.shutil.which", return_value="/usr/bin/osascript") +@patch("k1link.macos_credentials.subprocess.run") +def test_dialog_returns_value_without_printing( + run: Mock, + _which: Mock, + _system: Mock, +) -> None: + run.return_value = Mock(returncode=0, stdout="local-value\n", stderr="") + + assert _dialog_text("Prompt", hidden=True) == "local-value" + command = run.call_args.args[0] + assert "local-value" not in command + assert "with hidden answer" in command[-1] + + +@patch("k1link.macos_credentials.platform.system", return_value="Darwin") +@patch("k1link.macos_credentials.shutil.which", return_value="/usr/bin/osascript") +@patch("k1link.macos_credentials.subprocess.run") +def test_dialog_cancel_is_not_echoed( + run: Mock, + _which: Mock, + _system: Mock, +) -> None: + run.return_value = Mock(returncode=1, stdout="", stderr="User canceled.") + + with pytest.raises(CredentialDialogError, match="cancelled"): + _dialog_text("Prompt", hidden=False) diff --git a/tests/test_mqtt_capture.py b/tests/test_mqtt_capture.py new file mode 100644 index 0000000..0832679 --- /dev/null +++ b/tests/test_mqtt_capture.py @@ -0,0 +1,209 @@ +import hashlib +import json +import stat +from collections.abc import Callable +from pathlib import Path +from typing import Any, cast + +import paho.mqtt.client as mqtt +import pytest +from paho.mqtt.packettypes import PacketTypes +from paho.mqtt.reasoncodes import ReasonCode + +from k1link.mqtt.capture import ( + FRAME_HEADER, + RAW_MAGIC, + REPORT_TOPICS, + CaptureError, + CaptureFormatError, + capture_mqtt, + iter_capture_frames, + validate_private_ipv4, +) + + +class FakeClient: + def __init__(self, *, topic: str = "RealtimePath", payload: bytes = b"pose-data") -> None: + self.on_connect: Callable[..., None] | None = None + self.on_subscribe: Callable[..., None] | None = None + self.on_message: Callable[..., None] | None = None + self.on_disconnect: Callable[..., None] | None = None + self.topic = topic + self.payload = payload + self.connect_calls: list[tuple[str, int, int]] = [] + self.subscribe_calls: list[Any] = [] + self.disconnect_count = 0 + self._step = 0 + + def connect(self, host: str, port: int, keepalive: int) -> mqtt.MQTTErrorCode: + self.connect_calls.append((host, port, keepalive)) + return mqtt.MQTT_ERR_SUCCESS + + def subscribe(self, topics: Any) -> tuple[mqtt.MQTTErrorCode, int]: + self.subscribe_calls.append(topics) + return mqtt.MQTT_ERR_SUCCESS, 7 + + def loop(self, timeout: float) -> mqtt.MQTTErrorCode: + assert timeout > 0 + self._step += 1 + if self._step == 1: + assert self.on_connect is not None + self.on_connect( + self, + None, + mqtt.ConnectFlags(session_present=False), + ReasonCode(PacketTypes.CONNACK, "Success"), + None, + ) + elif self._step == 2: + assert self.on_subscribe is not None + self.on_subscribe( + self, + None, + 7, + [ReasonCode(PacketTypes.SUBACK, identifier=0) for _ in REPORT_TOPICS], + None, + ) + elif self._step == 3: + assert self.on_message is not None + message = mqtt.MQTTMessage(topic=self.topic.encode()) + message.payload = self.payload + message.qos = 0 + self.on_message(self, None, message) + else: + raise KeyboardInterrupt + return mqtt.MQTT_ERR_SUCCESS + + def disconnect(self) -> mqtt.MQTTErrorCode: + self.disconnect_count += 1 + return mqtt.MQTT_ERR_SUCCESS + + +@pytest.mark.parametrize("address", ["10.0.0.1", "172.16.0.1", "172.31.255.254", "192.168.4.2"]) +def test_validate_private_ipv4_accepts_only_rfc1918(address: str) -> None: + assert validate_private_ipv4(address) == address + + +@pytest.mark.parametrize( + "address", + ["8.8.8.8", "127.0.0.1", "169.254.1.2", "::1", "k1.local", " 192.168.1.2"], +) +def test_validate_private_ipv4_rejects_other_targets(address: str) -> None: + with pytest.raises(ValueError, match="private IPv4|RFC1918"): + validate_private_ipv4(address) + + +def test_capture_writes_verifiable_frames_metadata_and_summary(tmp_path: Path) -> None: + fake = FakeClient() + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "capture", + duration_seconds=30, + _client_factory=lambda: cast(mqtt.Client, fake), + ) + + assert fake.connect_calls == [("192.168.1.50", 1883, 30)] + assert fake.subscribe_calls == [[(topic, 0) for topic in REPORT_TOPICS]] + assert fake.disconnect_count == 1 + assert summary["stop_reason"] == "keyboard_interrupt" + assert summary["message_count"] == 1 + assert summary["payload_bytes"] == len(fake.payload) + assert summary["subscriptions"] == list(REPORT_TOPICS) + + capture_dir = tmp_path / "capture" + raw = (capture_dir / "mqtt.raw.k1mqtt").read_bytes() + assert raw.startswith(RAW_MAGIC) + topic_length, payload_length = FRAME_HEADER.unpack_from(raw, len(RAW_MAGIC)) + topic_start = len(RAW_MAGIC) + FRAME_HEADER.size + payload_start = topic_start + topic_length + assert raw[topic_start:payload_start].decode() == fake.topic + assert payload_length == len(fake.payload) + assert raw[payload_start:] == fake.payload + + records: list[dict[str, object]] = [ + json.loads(line) + for line in (capture_dir / "mqtt.metadata.jsonl").read_text().splitlines() + ] + assert len(records) == 1 + record = records[0] + assert record["record_type"] == "message" + assert record["topic"] == fake.topic + assert record["payload_bytes"] == len(fake.payload) + assert record["payload_sha256"] == hashlib.sha256(fake.payload).hexdigest() + assert record["raw_frame_offset"] == len(RAW_MAGIC) + assert record["raw_payload_offset"] == payload_start + + frames = list(iter_capture_frames(capture_dir / "mqtt.raw.k1mqtt")) + assert len(frames) == 1 + assert frames[0].sequence == 1 + assert frames[0].topic == fake.topic + assert frames[0].payload == fake.payload + assert frames[0].raw_frame_offset == len(RAW_MAGIC) + assert frames[0].raw_payload_offset == payload_start + + saved_summary = json.loads((capture_dir / "mqtt.summary.json").read_text()) + assert saved_summary == summary + assert saved_summary["artifact_hashes"]["raw_sha256"] == hashlib.sha256(raw).hexdigest() + for artifact_name in ("mqtt.raw.k1mqtt", "mqtt.metadata.jsonl", "mqtt.summary.json"): + assert stat.S_IMODE((capture_dir / artifact_name).stat().st_mode) == 0o600 + + +def test_oversize_payload_is_not_written_to_raw_capture(tmp_path: Path) -> None: + fake = FakeClient(payload=b"oversize") + capture_dir = tmp_path / "capture" + + with pytest.raises(CaptureError, match="limit is 4 bytes") as error: + capture_mqtt( + "10.1.2.3", + capture_dir, + max_message_bytes=4, + _client_factory=lambda: cast(mqtt.Client, fake), + ) + + assert (capture_dir / "mqtt.raw.k1mqtt").read_bytes() == RAW_MAGIC + record = json.loads((capture_dir / "mqtt.metadata.jsonl").read_text()) + assert record["record_type"] == "rejected_message" + assert record["payload_bytes"] == len(fake.payload) + assert error.value.summary is not None + assert error.value.summary["stop_reason"] == "message_too_large" + assert error.value.summary["message_count"] == 0 + assert error.value.summary["rejected_message_count"] == 1 + + +def test_capture_refuses_to_overwrite_existing_artifacts(tmp_path: Path) -> None: + capture_dir = tmp_path / "capture" + capture_dir.mkdir() + raw = capture_dir / "mqtt.raw.k1mqtt" + raw.write_bytes(b"existing evidence") + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + capture_mqtt( + "192.168.1.2", + capture_dir, + _client_factory=lambda: cast(mqtt.Client, FakeClient()), + ) + + assert raw.read_bytes() == b"existing evidence" + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + (b"not-mqtt", "magic/version"), + (RAW_MAGIC + b"\x00", "truncated length header"), + (RAW_MAGIC + FRAME_HEADER.pack(4, 1) + b"ab", "topic .* is truncated"), + (RAW_MAGIC + FRAME_HEADER.pack(1, 4) + b"t" + b"ab", "payload .* is truncated"), + (RAW_MAGIC + FRAME_HEADER.pack(1, 5) + b"t" + b"abcde", "exceeds 4"), + ], +) +def test_capture_reader_rejects_invalid_or_unbounded_frames( + tmp_path: Path, + raw: bytes, + message: str, +) -> None: + path = tmp_path / "capture.raw" + path.write_bytes(raw) + + with pytest.raises(CaptureFormatError, match=message): + list(iter_capture_frames(path, max_payload_bytes=4)) diff --git a/tests/test_stream_decoders.py b/tests/test_stream_decoders.py new file mode 100644 index 0000000..ed8b42e --- /dev/null +++ b/tests/test_stream_decoders.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import math +import struct + +import lz4.block +import pytest + +from k1link.protocol.protobuf_wire import ProtobufWireError, decode_zigzag64, iter_fields +from k1link.protocol.streams import ( + DecodeLimits, + StreamDecodeError, + UnsupportedCompressionError, + decode_legacy_pointcloud, + decode_legacy_pose, + decode_lio_pcl, + decode_lio_pose, + decode_pre_path_array, +) + + +def _varint(value: int) -> bytes: + encoded = bytearray() + while value > 0x7F: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def _key(number: int, wire_type: int) -> bytes: + return _varint((number << 3) | wire_type) + + +def _uint(number: int, value: int) -> bytes: + return _key(number, 0) + _varint(value) + + +def _sint(number: int, value: int) -> bytes: + zigzag = (value << 1) ^ (value >> 63) + return _uint(number, zigzag & 0xFFFFFFFFFFFFFFFF) + + +def _bytes(number: int, value: bytes) -> bytes: + return _key(number, 2) + _varint(len(value)) + value + + +def _fixed32(number: int, value: float) -> bytes: + return _key(number, 5) + struct.pack(" bytes: + return _key(number, 1) + struct.pack(" bytes: + return b"".join( + ( + _uint(1, 7), + _sint(2, 123456), + _sint(3, scaler), + _bytes(4, b"device-redacted"), + _bytes(5, b"session-redacted"), + ) + ) + + +def _pcl_payload(*, compression: int = 0, scaler: int = 1000) -> bytes: + point_1 = _sint(1, 1000) + _sint(2, -2000) + _sint(3, 500) + _uint(4, 0x11223344) + point_2 = _sint(1, -250) + _sint(2, 0) + _sint(3, 4000) + _uint(4, 0xAABBCC09) + report = _bytes(1, _header(scaler=scaler)) + _bytes(2, point_1) + _bytes(2, point_2) + compressed = lz4.block.compress(report, store_size=False) + fields = [] + if compression: + fields.append(_uint(2, compression)) + fields.extend((_uint(3, len(report)), _bytes(4, compressed))) + return b"".join(fields) + + +def test_protobuf_wire_zigzag_and_bounds() -> None: + assert decode_zigzag64(0) == 0 + assert decode_zigzag64(1) == -1 + assert decode_zigzag64(2) == 1 + with pytest.raises(ProtobufWireError, match="truncated"): + list(iter_fields(b"\x0a\x02\x01")) + with pytest.raises(ProtobufWireError, match="unsupported"): + list(iter_fields(b"\x0b")) + + +def test_decode_lio_pcl_raw_lz4() -> None: + frame = decode_lio_pcl(_pcl_payload()) + assert frame.header.seq == 7 + assert frame.header.stamp == 123456 + assert frame.header.scaler == 1000 + assert len(frame.points) == 2 + assert frame.points[0].scaled_xyz(frame.header.scaler) == (1.0, -2.0, 0.5) + assert frame.points[0].rgbi == 0x11223344 + assert frame.points[0].intensity == 0x44 + assert frame.points[1].scaled_xyz(frame.header.scaler) == (-0.25, 0.0, 4.0) + assert frame.points[1].intensity == 9 + + +def test_decode_lio_pcl_rejects_unverified_or_unsafe_frames() -> None: + with pytest.raises(UnsupportedCompressionError, match="enum 1"): + decode_lio_pcl(_pcl_payload(compression=1)) + with pytest.raises(StreamDecodeError, match="scaler is zero"): + decode_lio_pcl(_pcl_payload(scaler=0)) + with pytest.raises(StreamDecodeError, match="exceeds 1 points"): + decode_lio_pcl(_pcl_payload(), DecodeLimits(max_points_per_frame=1)) + with pytest.raises(StreamDecodeError, match="MQTT payload exceeds"): + decode_lio_pcl(_pcl_payload(), DecodeLimits(max_mqtt_payload_bytes=4)) + + +def test_decode_lio_pose() -> None: + position = _fixed64(1, 1.25) + _fixed64(2, -2.5) + _fixed64(3, 3.75) + orientation = ( + _fixed64(1, 0.1) + + _fixed64(2, 0.2) + + _fixed64(3, 0.3) + + _fixed64(4, 0.9) + ) + pose = _bytes(1, position) + _bytes(2, orientation) + stamped = _sint(1, 987654321) + _bytes(2, pose) + payload = ( + _bytes(1, _header()) + + _bytes(2, stamped) + + _fixed32(3, 12.5) + + _fixed32(4, 0.001) + ) + + frame = decode_lio_pose(payload) + assert frame.pose_stamp == 987654321 + assert frame.position_xyz == (1.25, -2.5, 3.75) + assert frame.orientation_xyzw == pytest.approx((0.1, 0.2, 0.3, 0.9)) + assert frame.distance == 12.5 + assert frame.pose_accuracy == pytest.approx(0.001) + + +def test_decode_lio_pose_rejects_nonfinite_float() -> None: + payload = _bytes(1, _header()) + _fixed32(3, math.nan) + with pytest.raises(StreamDecodeError, match="not finite"): + decode_lio_pose(payload) + + +def test_decode_legacy_pointcloud() -> None: + envelope = struct.pack(" None: + payload = struct.pack(" bytes: + encoded = bytearray() + while value > 0x7F: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def _key(number: int, wire_type: int) -> bytes: + return _varint((number << 3) | wire_type) + + +def _uint(number: int, value: int) -> bytes: + return _key(number, 0) + _varint(value) + + +def _sint(number: int, value: int) -> bytes: + zigzag = (value << 1) ^ (value >> 63) + return _uint(number, zigzag & 0xFFFFFFFFFFFFFFFF) + + +def _bytes(number: int, value: bytes) -> bytes: + return _key(number, 2) + _varint(len(value)) + value + + +def _fixed64(number: int, value: float) -> bytes: + return _key(number, 1) + struct.pack(" bytes: + return b"".join( + ( + _uint(1, 7), + _sint(2, 123456), + _sint(3, scaler), + _bytes(4, b"device-super-secret"), + _bytes(5, b"session-super-secret"), + _bytes(6, b"openapi-super-secret"), + ) + ) + + +def _pcl_payload(*, scaler: int, point_count: int) -> bytes: + points = [] + for index in range(point_count): + point = ( + _sint(1, 1000 + index) + + _sint(2, -2000 - index) + + _sint(3, 500 + index) + + _uint(4, index) + ) + points.append(_bytes(2, point)) + report = _bytes(1, _header(scaler=scaler)) + b"".join(points) + compressed = lz4.block.compress(report, store_size=False) + return _uint(3, len(report)) + _bytes(4, compressed) + + +def _pose_payload(position_xyz: tuple[float, float, float]) -> bytes: + position = b"".join( + _fixed64(field_number, value) + for field_number, value in enumerate(position_xyz, start=1) + ) + orientation = ( + _fixed64(1, 0.0) + + _fixed64(2, 0.0) + + _fixed64(3, 0.0) + + _fixed64(4, 1.0) + ) + pose = _bytes(1, position) + _bytes(2, orientation) + stamped = _sint(1, 987654321) + _bytes(2, pose) + return _bytes(1, _header(scaler=1000)) + _bytes(2, stamped) + + +def _write_capture(path: Path, frames: list[tuple[str, bytes]]) -> bytes: + raw = bytearray(RAW_MAGIC) + for topic, payload in frames: + topic_bytes = topic.encode() + raw.extend(FRAME_HEADER.pack(len(topic_bytes), len(payload))) + raw.extend(topic_bytes) + raw.extend(payload) + value = bytes(raw) + path.write_bytes(value) + return value + + +def test_summary_is_bounded_aggregate_only_and_hashes_capture(tmp_path: Path) -> None: + frames = [ + ("lixel/application/report/lio_pcl", _pcl_payload(scaler=1000, point_count=2)), + ("lixel/application/report/lio_pcl", _pcl_payload(scaler=2000, point_count=1)), + ("lixel/application/report/lio_pcl", b"bad-protobuf"), + ("lixel/application/report/lio_pose", _pose_payload((1.0, 2.0, 3.0))), + ("lixel/application/report/lio_pose", _pose_payload((4.0, 6.0, 3.0))), + ("lixel/application/report/lio_pose", b"bad-protobuf"), + ("device-super-secret/session-super-secret", b"openapi-super-secret"), + ] + capture = tmp_path / "mqtt.raw.k1mqtt" + raw = _write_capture(capture, frames) + + summary = summarize_mqtt_streams(capture) + + assert summary["source"] == { + "bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + assert summary["frames"] == { + "count": 7, + "payload_bytes": sum(len(payload) for _, payload in frames), + "encoded_frame_bytes": len(raw) - len(RAW_MAGIC), + "other_count": 1, + "other_payload_bytes": len(b"openapi-super-secret"), + } + assert summary["decoding"] == {"attempted": 6, "successes": 4, "errors": 2} + assert summary["point_cloud"]["frame_count"] == 3 + assert summary["point_cloud"]["decode_successes"] == 2 + assert summary["point_cloud"]["decode_errors"] == 1 + assert summary["point_cloud"]["points"] == { + "total": 3, + "per_frame": {"min": 1, "max": 2}, + } + assert summary["point_cloud"]["scalers"] == { + "min": 1000, + "max": 2000, + "constant": False, + } + assert summary["pose"]["frame_count"] == 3 + assert summary["pose"]["decode_successes"] == 2 + assert summary["pose"]["decode_errors"] == 1 + assert summary["pose"]["first_to_last_displacement_meters"] == 5.0 + + serialized = json.dumps(summary) + for secret in ( + "device-super-secret", + "session-super-secret", + "openapi-super-secret", + "bad-protobuf", + ): + assert secret not in serialized + assert "position_xyz" not in serialized + assert "points" in summary["point_cloud"] + + +def test_summary_rejects_a_frame_above_the_operator_limit(tmp_path: Path) -> None: + capture = tmp_path / "mqtt.raw.k1mqtt" + _write_capture(capture, [("other", b"12345")]) + + with pytest.raises(CaptureFormatError, match="exceeds 4"): + summarize_mqtt_streams(capture, max_payload_bytes=4) + + +def test_mqtt_streams_cli_writes_atomic_private_summary(tmp_path: Path) -> None: + capture = tmp_path / "mqtt.raw.k1mqtt" + raw = _write_capture(capture, []) + out = tmp_path / "captures" / "summary.json" + + result = runner.invoke( + app, + [ + "analyze", + "mqtt-streams", + "--capture", + str(capture), + "--out", + str(out), + "--max-payload-bytes", + "1024", + ], + ) + + assert result.exit_code == 0 + saved = json.loads(out.read_text()) + assert saved["source"]["sha256"] == hashlib.sha256(raw).hexdigest() + assert saved["limits"]["max_payload_bytes"] == 1024 + assert saved["frames"]["count"] == 0 + assert "aggregate-only" in result.stdout + assert stat.S_IMODE(out.stat().st_mode) == 0o600 diff --git a/tests/test_usb_snapshot.py b/tests/test_usb_snapshot.py new file mode 100644 index 0000000..cffe26d --- /dev/null +++ b/tests/test_usb_snapshot.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import json +import plistlib +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from k1link.cli import app +from k1link.usb.snapshot import ( + DISKUTIL_COMMAND, + SERIAL_IOREG_COMMAND, + USB_IOREG_COMMAND, + CommandOutput, + snapshot, +) + +runner = CliRunner() + + +def _plist_output(argv: tuple[str, ...], payload: object) -> CommandOutput: + return CommandOutput( + argv=argv, + returncode=0, + stdout=plistlib.dumps(payload), + stderr=b"", + error=None, + ) + + +def _interface( + name: str, + number: int, + interface_class: int, + subclass: int, + protocol: int, +) -> dict[str, object]: + return { + "IOObjectClass": "IOUSBHostInterface", + "IORegistryEntryName": name, + "kUSBString": name, + "bInterfaceNumber": number, + "bInterfaceClass": interface_class, + "bInterfaceSubClass": subclass, + "bInterfaceProtocol": protocol, + "bAlternateSetting": 0, + "bConfigurationValue": 1, + "bNumEndpoints": 2, + } + + +def test_snapshot_parses_xgrids_interfaces_storage_and_usbmodem() -> None: + usb_plist = [ + { + "IOObjectClass": "IOUSBHostDevice", + "IORegistryEntryName": "XGRIDS Device", + "USB Product Name": "XGRIDS Device", + "USB Vendor Name": "rockchip", + "USB Serial Number": "synthetic-serial", + "idVendor": 0x2207, + "idProduct": 0x0019, + "bDeviceClass": 0xEF, + "bDeviceSubClass": 0x02, + "bDeviceProtocol": 0x01, + "bcdUSB": 0x0210, + "bcdDevice": 0x0310, + "USBSpeed": 3, + "UsbLinkSpeed": 480_000_000, + "USB Address": 1, + "locationID": 0x01100000, + "IORegistryEntryID": 12345, + "IORegistryEntryChildren": [ + _interface("RNDIS Communications Control", 0, 0xE0, 0x01, 0x03), + _interface("RNDIS Ethernet Data", 1, 0x0A, 0x00, 0x00), + _interface("Mass Storage", 2, 0x08, 0x06, 0x50), + _interface("CDC NCM Control", 3, 0x02, 0x0D, 0x00), + _interface("CDC ACM Serial", 4, 0x02, 0x02, 0x01), + { + "IOObjectClass": "IOMedia", + "IORegistryEntryName": "Synthetic Media", + "BSD Name": "disk4", + }, + ], + }, + { + "IOObjectClass": "IOUSBHostDevice", + "IORegistryEntryName": "Unrelated Camera", + "USB Product Name": "Unrelated Camera", + "idVendor": 999, + }, + ] + serial_plist = [ + { + "IOCalloutDevice": "/dev/cu.usbmodemK1TEST", + "IODialinDevice": "/dev/tty.usbmodemK1TEST", + "IOTTYBaseName": "usbmodemK1TEST", + "IOSerialBSDClientType": "IOSerialStream", + }, + { + "IOCalloutDevice": "/dev/cu.debug-console", + "IODialinDevice": "/dev/tty.debug-console", + }, + ] + storage_plist = { + "AllDisks": ["disk4", "disk4s1"], + "WholeDisks": ["disk4"], + "VolumesFromDisks": ["K1_DATA"], + "AllDisksAndPartitions": [ + { + "DeviceIdentifier": "disk4", + "Content": "GUID_partition_scheme", + "Size": 64_000_000, + "OSInternal": False, + "Partitions": [ + { + "DeviceIdentifier": "disk4s1", + "Content": "Microsoft Basic Data", + "Size": 63_000_000, + "VolumeName": "K1_DATA", + "MountPoint": "/Volumes/K1_DATA", + "OSInternal": False, + } + ], + } + ], + } + outputs = { + USB_IOREG_COMMAND: _plist_output(USB_IOREG_COMMAND, usb_plist), + SERIAL_IOREG_COMMAND: _plist_output(SERIAL_IOREG_COMMAND, serial_plist), + DISKUTIL_COMMAND: _plist_output(DISKUTIL_COMMAND, storage_plist), + } + + result = snapshot(lambda argv: outputs[tuple(argv)]) + + assert result["xgrids_device_count"] == 1 + device = result["xgrids_devices"][0] + assert device["vendor_id_hex"] == "0x2207" + assert device["product_id_hex"] == "0x0019" + assert device["bsd_names"] == ["disk4"] + assert device["interface_capabilities"] == [ + "cdc_data", + "mass_storage", + "ncm", + "rndis", + "serial", + ] + assert [interface["interface_number"] for interface in device["interfaces"]] == [ + 0, + 1, + 2, + 3, + 4, + ] + assert all(entry["xgrids_related"] for entry in result["external_storage"]["entries"]) + assert result["usbmodem_device_names"] == [ + "/dev/cu.usbmodemK1TEST", + "/dev/tty.usbmodemK1TEST", + ] + assert len(result["usbmodem_devices"]) == 1 + assert all(source["ok"] for source in result["sources"]) + assert all("sudo" not in source["argv"] for source in result["sources"]) + assert result["safety"] == { + "metadata_only": True, + "sudo_used": False, + "device_file_contents_read": False, + "device_writes_performed": False, + } + + +def test_snapshot_reports_command_and_plist_errors_without_raising() -> None: + outputs = { + USB_IOREG_COMMAND: CommandOutput( + argv=USB_IOREG_COMMAND, + returncode=0, + stdout=b"not a plist", + stderr=b"", + error=None, + ), + SERIAL_IOREG_COMMAND: CommandOutput( + argv=SERIAL_IOREG_COMMAND, + returncode=1, + stdout=b"", + stderr=b"serial registry unavailable", + error=None, + ), + DISKUTIL_COMMAND: _plist_output( + DISKUTIL_COMMAND, + { + "AllDisks": [], + "WholeDisks": [], + "VolumesFromDisks": [], + "AllDisksAndPartitions": [], + }, + ), + } + + result = snapshot(lambda argv: outputs[tuple(argv)]) + + assert result["xgrids_devices"] == [] + statuses = {source["name"]: source for source in result["sources"]} + assert statuses["usb_ioreg"]["ok"] is False + assert statuses["usb_ioreg"]["error"].startswith("invalid plist:") + assert statuses["serial_ioreg"]["ok"] is False + assert statuses["serial_ioreg"]["error"] == "serial registry unavailable" + assert statuses["external_disks"]["ok"] is True + + +def test_snapshot_treats_empty_ioreg_output_as_no_devices() -> None: + outputs = { + USB_IOREG_COMMAND: CommandOutput(USB_IOREG_COMMAND, 0, b"", b"", None), + SERIAL_IOREG_COMMAND: CommandOutput(SERIAL_IOREG_COMMAND, 0, b"", b"", None), + DISKUTIL_COMMAND: _plist_output( + DISKUTIL_COMMAND, + { + "AllDisks": [], + "WholeDisks": [], + "VolumesFromDisks": [], + "AllDisksAndPartitions": [], + }, + ), + } + + result = snapshot(lambda argv: outputs[tuple(argv)]) + + assert result["xgrids_device_count"] == 0 + assert all(source["ok"] for source in result["sources"]) + + +def test_usb_snapshot_cli_writes_json( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + output = tmp_path / "nested" / "usb-snapshot.json" + payload = { + "schema_version": 1, + "xgrids_device_count": 0, + "safety": { + "metadata_only": True, + "sudo_used": False, + "device_file_contents_read": False, + "device_writes_performed": False, + }, + } + monkeypatch.setattr("k1link.cli.usb_snapshot", lambda: payload) + + result = runner.invoke(app, ["usb", "snapshot", "--out", str(output)]) + + assert result.exit_code == 0 + assert json.loads(output.read_text(encoding="utf-8")) == payload + assert "no sudo" in result.stdout diff --git a/tests/test_wifi_provisioning.py b/tests/test_wifi_provisioning.py new file mode 100644 index 0000000..dcf8ae2 --- /dev/null +++ b/tests/test_wifi_provisioning.py @@ -0,0 +1,69 @@ +import pytest + +from k1link.ble.wifi_provisioning import ( + FRAME_LENGTH, + build_wifi_provisioning_frame, + parse_wifi_status, +) + + +def test_build_wifi_provisioning_frame_layout() -> None: + frame = build_wifi_provisioning_frame("LabNet", "correct horse") + + assert len(frame) == FRAME_LENGTH + assert frame[0] == 6 + assert frame[1:7] == b"LabNet" + assert frame[7:33] == bytes(26) + assert frame[33] == 13 + assert frame[34:47] == b"correct horse" + assert frame[47:98] == bytes(51) + assert frame[98] == 0 + + +def test_build_wifi_provisioning_frame_uses_utf8_byte_lengths() -> None: + frame = build_wifi_provisioning_frame("Сеть", "пароль") + + assert frame[0] == len("Сеть".encode()) + assert frame[33] == len("пароль".encode()) + + +@pytest.mark.parametrize( + ("ssid", "password", "message"), + [ + ("", "password", "SSID must not be empty"), + ("network", "", "password must not be empty"), + ("x" * 33, "password", "at most 32 UTF-8 bytes"), + ("network", "x" * 65, "at most 64 UTF-8 bytes"), + ], +) +def test_build_wifi_provisioning_frame_rejects_invalid_lengths( + ssid: str, + password: str, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + build_wifi_provisioning_frame(ssid, password) + + +def test_parse_wifi_status_ap_baseline() -> None: + value = bytearray(54) + value[0] = 7 + value[1:8] = b"WIFI_AP" + value[33] = 4 + value[34:38] = bytes((192, 168, 56, 1)) + value[50] = 1 + value[52:54] = b"XX" + + assert parse_wifi_status(bytes(value)) == { + "value_length": 54, + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + "trailer_hex": "5858", + } + + +def test_parse_wifi_status_rejects_short_frame() -> None: + with pytest.raises(ValueError, match="at least 51 bytes"): + parse_wifi_status(bytes(50)) diff --git a/uv.lock b/uv.lock index a99304b..cd9ce1d 100644 --- a/uv.lock +++ b/uv.lock @@ -88,6 +88,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/65/aead61bbf3b5358593f9d4779d2a0e88eaf6ec191a6342dde36dd1df6371/librt-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1", size = 112236, upload-time = "2026-07-08T12:25:18.425Z" }, ] +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/ac/016e4f6de37d806f7cc8f13add0a46c9a7cfc41a5ddc2bc831d7954cf1ce/lz4-4.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:df5aa4cead2044bab83e0ebae56e0944cc7fcc1505c7787e9e1057d6d549897e", size = 207163, upload-time = "2025-11-03T13:01:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/8d/df/0fadac6e5bd31b6f34a1a8dbd4db6a7606e70715387c27368586455b7fc9/lz4-4.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d0bf51e7745484d2092b3a51ae6eb58c3bd3ce0300cf2b2c14f76c536d5697a", size = 207150, upload-time = "2025-11-03T13:01:47.205Z" }, + { url = "https://files.pythonhosted.org/packages/b7/17/34e36cc49bb16ca73fb57fbd4c5eaa61760c6b64bce91fcb4e0f4a97f852/lz4-4.4.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b62f94b523c251cf32aa4ab555f14d39bd1a9df385b72443fd76d7c7fb051f5", size = 1292045, upload-time = "2025-11-03T13:01:48.667Z" }, + { url = "https://files.pythonhosted.org/packages/90/1c/b1d8e3741e9fc89ed3b5f7ef5f22586c07ed6bb04e8343c2e98f0fa7ff04/lz4-4.4.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c3ea562c3af274264444819ae9b14dbbf1ab070aff214a05e97db6896c7597e", size = 1279546, upload-time = "2025-11-03T13:01:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/e3867222474f6c1b76e89f3bd914595af69f55bf2c1866e984c548afdc15/lz4-4.4.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24092635f47538b392c4eaeff14c7270d2c8e806bf4be2a6446a378591c5e69e", size = 1368249, upload-time = "2025-11-03T13:01:51.273Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e7/d667d337367686311c38b580d1ca3d5a23a6617e129f26becd4f5dc458df/lz4-4.4.5-cp312-cp312-win32.whl", hash = "sha256:214e37cfe270948ea7eb777229e211c601a3e0875541c1035ab408fbceaddf50", size = 88189, upload-time = "2025-11-03T13:01:52.605Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0b/a54cd7406995ab097fceb907c7eb13a6ddd49e0b231e448f1a81a50af65c/lz4-4.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:713a777de88a73425cf08eb11f742cd2c98628e79a8673d6a52e3c5f0c116f33", size = 99497, upload-time = "2025-11-03T13:01:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7e/dc28a952e4bfa32ca16fa2eb026e7a6ce5d1411fcd5986cd08c74ec187b9/lz4-4.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:a88cbb729cc333334ccfb52f070463c21560fca63afcf636a9f160a55fac3301", size = 91279, upload-time = "2025-11-03T13:01:54.419Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -146,6 +162,8 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "bleak" }, + { name = "lz4" }, + { name = "paho-mqtt" }, { name = "rich" }, { name = "typer" }, ] @@ -160,6 +178,8 @@ dev = [ [package.metadata] requires-dist = [ { name = "bleak", specifier = "==3.0.2" }, + { name = "lz4", specifier = ">=4.4,<5" }, + { name = "paho-mqtt", specifier = ">=2.1,<3" }, { name = "rich", specifier = ">=13.9,<15" }, { name = "typer", specifier = ">=0.15,<1" }, ] @@ -180,6 +200,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "paho-mqtt" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/15/0a6214e76d4d32e7f663b109cf71fb22561c2be0f701d67f93950cd40542/paho_mqtt-2.1.0.tar.gz", hash = "sha256:12d6e7511d4137555a3f6ea167ae846af2c7357b10bc6fa4f7c3968fc1723834", size = 148848, upload-time = "2024-04-29T19:52:55.591Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/cb/00451c3cf31790287768bb12c6bec834f5d292eaf3022afc88e14b8afc94/paho_mqtt-2.1.0-py3-none-any.whl", hash = "sha256:6db9ba9b34ed5bc6b6e3812718c7e06e2fd7444540df2455d2c51bd58808feee", size = 67219, upload-time = "2024-04-29T19:52:48.345Z" }, +] + [[package]] name = "pathspec" version = "1.1.1"