feat(planning): consolidate recorded-route localization and spatial scene

Preserve the completed teach-and-repeat laboratory stage: reference preparation, cascaded acquisition, local tracking and recovery, recording lifecycle, replay qualification, and persistent Rerun scene controls. Document the open grid-picking regression and Rerun upgrade contract. No autonomous driving or loop-closure optimization is claimed.
This commit is contained in:
DCCONSTRUCTIONS
2026-09-21 08:47:19 +03:00
parent be58d589e2
commit e515ab1b8c
189 changed files with 19074 additions and 758 deletions
+97
View File
@@ -0,0 +1,97 @@
/** Build a removable, offline-only browser probe around the production player.
* Input manifest/media must already be staged in outputRoot by a trusted local
* caller. No device API, live singleton or actual WebSocket is used by this probe.
*/
import {createRequire} from 'node:module';
import {readFile,writeFile} from 'node:fs/promises';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
const repository=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..');
const app=path.join(repository,'apps/control-station');
const outputRoot=path.resolve(process.argv[2]);
if(!outputRoot.startsWith(path.join(app,'dist')+path.sep))throw new Error('Probe must remain below the canonical static root.');
const require=createRequire(path.join(app,'package.json'));
const {build}=require('esbuild');
const player=path.join(app,'src/components/MseFmp4WebSocketPlayer.tsx');
const source=`
import React from 'react';
import {createRoot} from 'react-dom/client';
import {MseFmp4WebSocketPlayer} from ${JSON.stringify(player)};
const base=new URL('.',location.href), nativeFetch=window.fetch.bind(window);
const manifest=await (await nativeFetch(new URL('manifest.json',base))).json();
const report={schema:'missioncore.offline-browser-camera-probe/v1',startedAt:new Date().toISOString(),cases:[],unexpectedRequests:[],hardware:false,realWebSocket:false,realMediaSource:true};
let current=null,finished=false;
const output=document.querySelector('pre');
function renderReport(){output.textContent=JSON.stringify(report,null,2);}
window.fetch=async(input,options={})=>{
const url=new URL(typeof input==='string'?input:input.url??input,location.href);
if(url.origin===base.origin&&url.pathname.startsWith(base.pathname)&&(options.method??'GET')==='GET')return nativeFetch(input,options);
if(url.pathname==='/api/v1/viewer/live-diagnostics'&&options.method==='POST'){
if(current)current.events.push({atMs:performance.now()-current.started,event:JSON.parse(options.body)});
return new Response('{}',{status:200});
}
report.unexpectedRequests.push({path:url.pathname,method:options.method??'GET'});renderReport();
throw new Error('Offline probe forbids all device and unrelated HTTP requests.');
};
class ArchiveSocket extends EventTarget{
static CONNECTING=0;static OPEN=1;static CLOSING=2;static CLOSED=3;
readyState=0;binaryType='arraybuffer';timers=[];
constructor(url){
super();
if(new URL(url).pathname!==base.pathname+'offline-socket')throw new Error('Unexpected socket destination');
this.owner=current;this.owner.sockets++;this.owner.activeSockets++;
this.timers.push(setTimeout(()=>this.open(),0));
}
open(){
if(this.readyState===3)return;
this.readyState=1;this.dispatchEvent(new Event('open'));
const elapsed=this.owner.streamStarted===undefined?0:performance.now()-this.owner.streamStarted;
this.owner.streamStarted??=performance.now();
// Every replacement gets init plus only subsequent media, like a disposable
// reader joining the existing stream. It cannot rewind the archived source.
this.sendBytes(this.owner.bytes[0]);
this.owner.entries.slice(1).forEach((row,index)=>{
if(row.atMs<elapsed)return;
this.timers.push(setTimeout(()=>this.sendBytes(this.owner.bytes[index+1]),Math.max(0,row.atMs-elapsed)));
});
}
sendBytes(bytes){if(this.readyState===1){this.owner.delivered++;this.dispatchEvent(new MessageEvent('message',{data:bytes.slice(0)}));}}
close(){if(this.readyState===3)return;this.readyState=3;this.timers.forEach(clearTimeout);this.owner.activeSockets--;this.dispatchEvent(new CloseEvent('close',{code:1000}));}
send(){throw new Error('Offline camera is read-only.');}
}
window.WebSocket=ArchiveSocket;
const root=createRoot(document.querySelector('#player'));
const sampleTimer=setInterval(()=>{
if(!current||finished)return;
const video=document.querySelector('video');
const quality=video?.getVideoPlaybackQuality?.();
current.samples.push({atMs:performance.now()-current.started,status:document.querySelector('.mse-fmp4-player')?.dataset.status,currentTime:video?.currentTime??null,readyState:video?.readyState??null,error:video?.error?.code??null,decoded:quality?.totalVideoFrames??video?.webkitDecodedFrameCount??null,dropped:quality?.droppedVideoFrames??null});
renderReport();
},500);
const wait=ms=>new Promise(resolve=>setTimeout(resolve,ms));
for(const fixture of manifest.cases){
const bytes=await Promise.all(fixture.entries.map(async row=>(await nativeFetch(new URL(row.path,base))).arrayBuffer()));
current={name:fixture.name,started:performance.now(),entries:fixture.entries,bytes,samples:[],events:[],sockets:0,activeSockets:0,delivered:0};
// The public DOM report excludes binary bytes and internal timer state.
const publicCase={name:current.name,samples:current.samples,events:current.events};report.cases.push(publicCase);
document.querySelector('h1').textContent='Проверка браузерной камеры: '+fixture.name+' · только архив';
root.render(React.createElement(MseFmp4WebSocketPlayer,{key:fixture.name,label:'Сохранённая камера · тест декодера',delivery:{kind:'mse-fmp4-websocket',id:'offline-'+fixture.name,url:base.pathname+'offline-socket',mediaType:'video/mp4; codecs="avc1.641028"'},recoveryAuthorityIdentity:'offline-fixture-'+fixture.name}));
await wait(fixture.durationMs);
root.render(null);await wait(100);
Object.assign(publicCase,{sockets:current.sockets,activeSockets:current.activeSockets,delivered:current.delivered,expectedMedia:fixture.entries.length-1});
renderReport();
}
clearInterval(sampleTimer);root.unmount();finished=true;report.finishedAt=new Date().toISOString();
const gapSamples=report.cases[0].samples.filter(s=>s.atMs>10000&&s.atMs<20000);
report.checks={cleanPlayed:report.cases[0].samples.some(s=>s.status==='playing'&&s.decoded>0),cleanGapVisible:gapSamples.length>0&&gapSamples.every(s=>s.status==='buffering'),cleanNoRestarts:report.cases[0].sockets===1,cleanNoDecodeErrors:!report.cases[0].samples.some(s=>s.error)&&!report.cases[0].events.some(e=>e.event.event_code==='live_camera_transport_restart_requested'),corruptDetected:report.cases[1].samples.some(s=>s.error)||report.cases[1].events.some(e=>e.event.event_code==='live_camera_transport_restart_requested'),allReadersClosed:report.cases.every(c=>c.activeSockets===0),noUnexpectedRequests:report.unexpectedRequests.length===0};
document.querySelector('h1').textContent='Проверка браузерной камеры завершена · только архив';renderReport();
`;
await writeFile(path.join(outputRoot,'probe-source.jsx'),source);
await build({stdin:{contents:source,resolveDir:app,sourcefile:'offline-camera-probe.jsx',loader:'jsx'},absWorkingDir:app,bundle:true,format:'esm',target:'es2022',jsx:'automatic',define:{'process.env.NODE_ENV':'"production"'},outfile:path.join(outputRoot,'probe.js')});
const shell=await readFile(path.join(app,'dist/index.html'),'utf8');
const css=shell.match(/href="([^\"]+\.css)"/)?.[1];
if(!css)throw new Error('Canonical built stylesheet is missing.');
await writeFile(path.join(outputRoot,'index.html'),
`<!doctype html><meta charset="utf-8"><title>Offline camera decoder qualification</title><link rel="stylesheet" href="${css}"><div data-nodedc-ui="true" data-nodedc-theme="dark" class="nodedc-ui-root" style="background:#101010;color:#eee;padding:24px;min-height:100vh"><h1>Проверка браузерной камеры · только архив</h1><p>Штатный плеер и настоящий MediaSource. Источник — локальные файлы. Подключения к K1 нет.</p><div id="player" style="width:640px;height:400px"></div><pre style="font-size:11px;white-space:pre-wrap"></pre></div><script type="module" src="./probe.js"></script>`);
console.log(JSON.stringify({outputRoot,productionPlayer:player,hardware:false}));
@@ -0,0 +1,75 @@
"""Repeat the frozen negative controls with exactly the new entry search policy."""
import argparse
import json
from pathlib import Path
import numpy as np
from k1link.artifacts import utc_now_iso
from k1link.missions.causal_replay import digest
from k1link.missions.entry_acquisition_worker import run_entry_acquisition
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--controls", type=Path, required=True)
p.add_argument("--replay", type=Path, required=True)
p.add_argument("--output", type=Path, required=True)
args = p.parse_args()
prior = json.loads((args.controls / "report.json").read_text())
replay = json.loads((args.replay / "report.json").read_text())
path_file = args.replay / "step-003/query-path.npy"
files = {path_file: replay["artifacts"]["step-003/query-path.npy"]}
for name in ["wrong-region", "far-seed"]:
relative = name + "/registration-input.npz"
files[args.controls / relative] = prior["artifacts"][relative]
for file, expected in files.items():
if digest(file) != expected:
raise ValueError("Control input changed.")
query_path = np.load(path_file, allow_pickle=False)
direction = next(
p - query_path[0] for p in query_path[1:] if np.linalg.norm((p - query_path[0])[:2]) >= 3
)
args.output.mkdir(parents=True, exist_ok=False)
report = dict(
schema_version="missioncore.entry-controls/v1",
created_at_utc=utc_now_iso(),
source_step="step-003",
results={},
vehicle_control=False,
localization_confirmed=False,
input_digests={str(k): v for k, v in files.items()},
)
for name in ["wrong-region", "far-seed"]:
with np.load(args.controls / name / "registration-input.npz", allow_pickle=False) as data:
reference, query, initial = data["reference"], data["query"], data["initial"]
directory = args.output / name
directory.mkdir()
result = run_entry_acquisition(
directory, reference, query, initial, query_path[0], initial[:3, :3] @ direction
)
report["results"][name] = {k: v for k, v in result.items() if k != "matched_query_indices"}
print(
json.dumps(
dict(
control=name,
status=result["status"],
reasons=result["reasons"],
hypotheses=len(result["initialization"]["attempts"]),
clusters=result["initialization"]["clusters"],
elapsed_s=result["initialization"]["elapsed_s"],
)
),
flush=True,
)
report["source_integrity_verified"] = all(digest(k) == v for k, v in files.items())
report["artifacts"] = {
str(x.relative_to(args.output)): digest(x) for x in args.output.rglob("*") if x.is_file()
}
report["finished_at_utc"] = utc_now_iso()
(args.output / "report.json").write_text(json.dumps(report, allow_nan=False))
if __name__ == "__main__":
main()
+599
View File
@@ -0,0 +1,599 @@
"""Bounded archive qualification of the actual live planning service, without devices."""
import argparse
import json
import shutil
import threading
import time
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
import numpy as np
from check_stationary_bootstrap import code_hashes, write
from planning_archive_camera import ArchiveCamera
from planning_archive_source import ReceiptQueueArchiveSource
from k1link.artifacts import utc_now_iso
from k1link.device_plugins.xgrids_k1.localization_source import extract_scene_submap, extract_submap
from k1link.device_plugins.xgrids_k1.planning_replay import iter_planning_events
from k1link.missions.causal_replay import digest
from k1link.missions.live_limits import live_route_limits
from k1link.missions.live_scene_delta import decode_cursor
from k1link.missions.live_tests import PlanningLiveTests
from k1link.missions.reference_map import build_reference_map
from k1link.missions.stationary_bootstrap import BOOTSTRAP_POLICY
from k1link.missions.stationary_entry import STATIONARY_POLICY
class ArchiveSource:
"""One pending receipt, original intervals, no synthetic device authority."""
def __init__(self, raw, session, maximum_seconds=65, *, after_monotonic_ns=None):
if not 0 < maximum_seconds <= 600:
raise ValueError("This offline probe supports up to ten minutes of recorded receipts.")
self.maximum_seconds = maximum_seconds
self.iterator = iter(iter_planning_events(raw, session))
self.pending = next(self.iterator)
while after_monotonic_ns is not None and self.pending.monotonic_ns < after_monotonic_ns:
self.pending = next(self.iterator)
self.origin = self.pending.monotonic_ns
self.started = None
self.session = session
self.owner = None
self.deliveries = []
self.active = False
def snapshot(self):
return dict(
active=self.active,
session_id=self.session if self.started else None,
session_generation=1 if self.started else 0,
)
def open(self, owner):
if self.owner:
raise RuntimeError("An archive consumer already exists.")
self.owner = owner
def close(self, owner):
assert self.owner == owner
self.owner = None
self.iterator.close()
def activate(self):
self.started = time.monotonic_ns()
self.active = True
def take(self, owner):
assert self.owner == owner
if self.started is None:
time.sleep(0.02)
return None
if self.pending is None:
self.active = False
return None
delay = self.pending.monotonic_ns - self.origin
if delay > self.maximum_seconds * 1e9:
self.active = False
return None
stamp = self.started + delay
now = time.monotonic_ns()
if now < stamp:
time.sleep(min(0.02, (stamp - now) / 1e9))
return None
original = self.pending
self.pending = next(self.iterator, None)
self.deliveries.append(
dict(
sequence=original.sequence,
kind=original.kind,
original_monotonic_ns=original.monotonic_ns,
mapped_monotonic_ns=stamp,
delivered_monotonic_ns=now,
lag_s=(now - stamp) / 1e9,
)
)
return replace(original, monotonic_ns=stamp)
def main():
parser = argparse.ArgumentParser(description=__doc__)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--predecessor", type=Path)
group.add_argument("--physical-diagnosis", type=Path)
group.add_argument("--live-run", type=Path)
parser.add_argument("--reference-capture", type=Path)
parser.add_argument("--query-capture", type=Path)
parser.add_argument("--planning-source", type=Path)
parser.add_argument(
"--route-length-m",
type=float,
help="Select a longer reference for this archive probe only.",
)
parser.add_argument(
"--reference-kind", choices=["correct-entry", "wrong-region"], default="correct-entry"
)
parser.add_argument("--queue-ingress", action="store_true")
parser.add_argument(
"--drop-interval-s",
nargs=2,
type=float,
metavar=("START", "END"),
help="Omit derived receipts only; no source edits or retiming.",
)
parser.add_argument(
"--spatial-stop-monotonic-ns",
type=int,
help="Model admitted STOP at a retained receipt, with capture active until planner ends.",
)
parser.add_argument(
"--profile-planning",
action="store_true",
help="Profile the isolated planning consumer, not the canonical runtime.",
)
parser.add_argument(
"--after-monotonic-ns",
type=int,
help="Start at a retained operator retry boundary; never selects a reference position.",
)
parser.add_argument(
"--maximum-seconds",
type=float,
default=None,
help="Explicit offline replay wall budget; never a product-session limit.",
)
parser.add_argument(
"--fast-scene",
action="store_true",
help="Qualify production delta route in-process at 10 Hz, without a server.",
)
parser.add_argument("--camera-epoch", type=Path)
parser.add_argument("--ffmpeg", type=Path)
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
if args.spatial_stop_monotonic_ns is not None and not args.queue_ingress:
parser.error("--spatial-stop-monotonic-ns requires --queue-ingress")
if args.drop_interval_s is not None and not args.queue_ingress:
parser.error("--drop-interval-s requires --queue-ingress")
if args.route_length_m is not None:
live_route_limits(args.route_length_m)
if not args.live_run:
parser.error("--route-length-m requires --live-run and its frozen planning source.")
maximum_seconds = args.maximum_seconds or (120 if args.live_run else 65)
scene_cadence = 0.5 if args.live_run else 2
if args.fast_scene:
scene_cadence = 0.1
reference_provenance = None
scene_reference = scene_provenance = None
if args.camera_epoch and (not args.ffmpeg or not args.ffmpeg.is_file()):
parser.error("A retained camera requires an explicit local FFmpeg binary.")
if args.live_run:
if not all(
p is not None and p.is_file()
for p in [args.reference_capture, args.query_capture, args.planning_source]
):
parser.error(
"A live-run replay requires both raw captures and the frozen planning source."
)
original = json.loads((args.live_run / "report.json").read_text())
assert all(digest(args.live_run / p) == sha for p, sha in original["artifacts"].items())
assert (
digest(args.reference_capture)
== original["reference"]["source_digests"]["raw-transport-primary"]
)
planning = json.loads(args.planning_source.read_text())
assert planning["generation"] == original["draft"]["zone"]["generation"]
start, end = (
original["draft"]["route"]["start_index"],
original["draft"]["route"]["end_index"],
)
if args.reference_kind == "wrong-region":
start = next(i for i, p in enumerate(planning["poses"]) if p["distance_m"] >= 130)
end = next(i for i, p in enumerate(planning["poses"]) if p["distance_m"] >= 160)
if args.route_length_m is not None:
target = planning["poses"][start]["distance_m"] + args.route_length_m
if target > planning["poses"][-1]["distance_m"]:
parser.error("The reference recording does not cover the requested route.")
end = max(i for i, p in enumerate(planning["poses"]) if p["distance_m"] <= target)
path = np.array([p["position"] for p in planning["poses"][start : end + 1]])
def submap(session, generation, first, last, *, presentation=False):
extractor = extract_scene_submap if presentation else extract_submap
points, provenance = extractor(args.reference_capture, planning, first, last)
return points, {
**provenance,
**{k: planning[k] for k in ["session_id", "generation", "source_digests"]},
}
prepared = SimpleNamespace(bound=lambda *a: planning, submap=submap)
reference, reference_provenance = build_reference_map(
prepared, planning["session_id"], planning["generation"], start, end
)
scene_reference, scene_provenance = build_reference_map(
prepared, planning["session_id"], planning["generation"], start, end, presentation=True
)
files = [
args.live_run / "report.json",
args.reference_capture,
args.query_capture,
args.planning_source,
args.query_capture.with_name("mqtt.metadata.jsonl"),
args.reference_capture.with_name("mqtt.metadata.jsonl"),
]
inputs = {str(p): digest(p) for p in files}
previous = dict(
query_raw=str(args.query_capture),
query_session=original["query_session_id"],
reference_session=planning["session_id"],
)
elif args.physical_diagnosis:
if args.reference_kind != "correct-entry":
parser.error("A physical diagnosis uses its exact frozen reference.")
previous = json.loads((args.physical_diagnosis / "run/report.json").read_text())
sealed = json.loads((args.physical_diagnosis / "manifest.redacted.json").read_text())
inputs = {
str(args.physical_diagnosis / item["path"]): item["sha256"] for item in sealed["files"]
}
reference = np.load(args.physical_diagnosis / "run/reference.npy", allow_pickle=False)
path = np.array([p["position"] for p in previous["draft"]["route"]["points"]])
previous = dict(
query_raw=str(args.physical_diagnosis / "capture/captures/mqtt_live/mqtt.raw.k1mqtt"),
query_session=previous["query_session_id"],
reference_session=previous["reference"]["session_id"],
)
else:
previous = json.loads((args.predecessor / "manifest.json").read_text())
inputs = {
**previous["input_digests"],
str(args.predecessor / "manifest.json"): digest(args.predecessor / "manifest.json"),
}
with np.load(previous["references"][args.reference_kind], allow_pickle=False) as data:
reference, path = data["reference"], data["reference_path"]
assert all(digest(Path(p)) == sha for p, sha in inputs.items())
root = args.output
root.mkdir(parents=True, exist_ok=False)
camera = (
ArchiveCamera(args.camera_epoch, root / "camera", args.ffmpeg)
if args.camera_epoch
else None
)
if camera:
inputs.update(camera.inputs)
implementation = {
**code_hashes(),
"scripts/check_planning_live_bootstrap.py": digest(Path(__file__)),
"scripts/planning_archive_source.py": digest(Path("scripts/planning_archive_source.py")),
"scripts/planning_archive_camera.py": digest(Path("scripts/planning_archive_camera.py")),
"src/k1link/device_plugins/xgrids_k1/camera.py": digest(
Path("src/k1link/device_plugins/xgrids_k1/camera.py")
),
"src/k1link/web/camera_archive.py": digest(Path("src/k1link/web/camera_archive.py")),
"tests/test_planning_live.py": digest(Path("tests/test_planning_live.py")),
}
for name in [
"src/k1link/missions/live_display_buffer.py",
"src/k1link/compute/live_perception.py",
"src/k1link/device_plugins/xgrids_k1/facade.py",
"tests/test_planning_stop_lifecycle.py",
"src/k1link/missions/live_limits.py",
"src/k1link/missions/reference_map.py",
"src/k1link/missions/reference_window.py",
"src/k1link/missions/live_buffer.py",
"src/k1link/missions/live_scene_delta.py",
"src/k1link/web/planning_live_api.py",
"tests/test_planning_fast_display.py",
"apps/control-station/src/core/missions/planningSceneStream.ts",
"apps/control-station/src/components/missions/PlanningLiveScene.tsx",
]:
implementation[name] = digest(Path(name))
for name in implementation:
target = root / "executed-source" / name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(name, target)
write(
root / "manifest.json",
dict(
schema_version="missioncore.live-bootstrap-qualification/v1",
created_at_utc=utc_now_iso(),
created_monotonic_ns=time.monotonic_ns(),
input_digests=inputs,
implementation_sha256=implementation,
protocol=BOOTSTRAP_POLICY,
entry_policy=STATIONARY_POLICY,
queue_ingress=args.queue_ingress,
drop_interval_s=args.drop_interval_s,
profile_planning=args.profile_planning,
spatial_stop_monotonic_ns=args.spatial_stop_monotonic_ns,
after_monotonic_ns=args.after_monotonic_ns,
camera_replay=bool(camera),
reference_kind=args.reference_kind,
scene_cadence_s=scene_cadence,
fast_scene=args.fast_scene,
archive_maximum_seconds=maximum_seconds,
**live_route_limits(float(np.linalg.norm(np.diff(path, axis=0), axis=1).sum())),
pace=1,
expectation=(
"wrong-region: reject without tracking; correct-entry: complete prior -> "
"three disjoint fresh fits -> tracking -> end"
),
clock=(
"original receipt deltas rebased to this process monotonic start; "
"original epoch retained"
),
authority=(
"isolated archive adapter; optional in-process ASGI GET only; "
"no listening server, capture, device commands or vehicle control"
),
limitations=(
"recorded startup/motion only; stationary physical wait and scanner UI "
"require field acceptance; optional camera uses real parser/archive/queue "
"and local FFmpeg decoder, not RTSP, browser MSE or acquisition authority"
),
),
)
np.save(root / "qualified-reference.npy", reference, allow_pickle=False)
if reference_provenance is not None:
write(root / "reference-provenance.json", reference_provenance)
draft = dict(
id="archive-probe",
name="Stationary live integration · archive qualification",
revision=1,
zone=dict(session_id=previous["reference_session"], generation="frozen-archive"),
route=dict(
length_m=float(np.linalg.norm(np.diff(path, axis=0), axis=1).sum()),
start_index=0,
end_index=len(path) - 1,
points=[dict(position=p.tolist()) for p in path],
),
)
(root / "runtime").mkdir()
sources = SimpleNamespace(
store=SimpleNamespace(
get_session=lambda _: SimpleNamespace(plugin_id="archive-qualification")
),
reference_map=lambda *args, **kwargs: (
reference,
reference_provenance
or dict(session_id=previous["reference_session"], source_digests=inputs),
),
)
if scene_reference is not None:
sources.scene_reference_map = lambda *args, **kwargs: (scene_reference, scene_provenance)
drafts = SimpleNamespace(
database=root / "runtime" / "drafts.json", sources=sources, get=lambda _: draft
)
source_type = ReceiptQueueArchiveSource if args.queue_ingress else ArchiveSource
source = source_type(
Path(previous["query_raw"]),
previous["query_session"],
maximum_seconds,
after_monotonic_ns=args.after_monotonic_ns,
**(
{"spatial_stop_monotonic_ns": args.spatial_stop_monotonic_ns}
| {"drop_interval_s": args.drop_interval_s}
if args.queue_ingress
else {}
),
)
lock = threading.Lock()
service = PlanningLiveTests(drafts, {"archive-qualification": source}, lock)
if args.profile_planning:
import cProfile
original_work = service.work
def profiled_work(*work_args):
profile = cProfile.Profile()
try:
return profile.runcall(original_work, *work_args)
finally:
profile.dump_stats(str(root / "planning-consumer.prof"))
service.work = profiled_work
observations = []
started_utc = utc_now_iso()
run = service.start(draft["id"], 1)
scene_saved = False
first_tracking_state = None
last_scene = 0
scene_count = 0
scene_measurements = []
cursor = ""
client = None
if args.fast_scene:
from fastapi import FastAPI
from fastapi.testclient import TestClient
from k1link.web.planning_live_api import build_planning_live_router
app = FastAPI()
app.include_router(build_planning_live_router(service))
client = TestClient(app)
client.__enter__()
try:
deadline = time.monotonic() + 10
while service.get()["state"] == "preparing" and time.monotonic() < deadline:
time.sleep(0.02)
assert service.get()["state"] == "waiting"
source.activate()
if camera:
camera.start()
while service.thread.is_alive():
now = time.monotonic_ns()
assert now - source.started < (maximum_seconds + 40) * 1e9, (
"Functional probe wall bound exceeded."
)
state = service.get()
key = (state["state"], state.get("planning_phase"), state.get("result_source_sequence"))
if not observations or key != observations[-1]["key"]:
observations.append(
dict(
key=key,
at_s=(now - source.started) / 1e9,
frame_age_s=state["frame_age_s"],
result_age_s=state["result_age_s"],
tracking_state=state["tracking_state"],
message=state["message"],
source_active=source.snapshot().get("active", False),
recovery_attempt=state.get("recovery_attempt", 0),
accepted_sample=service.accepted_sample is not None,
)
)
print(json.dumps(observations[-1], ensure_ascii=False), flush=True)
if state["tracking_state"] == "tracking" and not scene_saved:
first_tracking_state = state
(root / "tracking.rrd").write_bytes(service.scene(run["id"], True))
scene_saved = True
if now - last_scene >= scene_cadence * 1e9:
before = time.monotonic()
extra = {}
if client:
response = client.get(
f"/api/v1/mission-planner/live-tests/{run['id']}/scene-delta.rrd",
params=dict(cursor=cursor, base=scene_count == 0),
)
assert response.status_code in (200, 204)
cursor = response.headers["X-Planning-Scene-Cursor"]
token = decode_cursor(cursor)
extra = dict(
cloud_revision=token["cloud"],
pose_sequence=token["pose"],
delta_live=token["live"],
display_cloud_age_s=float(response.headers["X-Planning-Cloud-Age"]),
at_s=(now - source.started) / 1e9,
)
payload = response.content
else:
payload = service.scene(run["id"], scene_count == 0)
scene_measurements.append(
dict(
**extra,
seconds=time.monotonic() - before,
bytes=len(payload),
presentation_state=state.get("presentation_state"),
frame_age_s=state["frame_age_s"],
result_age_s=state["result_age_s"],
pose_age_s=state.get("pose_age_s"),
)
)
last_scene, scene_count = now, scene_count + 1
time.sleep(0.01 if client else 0.05)
finally:
if client:
client.__exit__(None, None, None)
service.close()
if camera:
camera.close()
write(root / "deliveries.json", source.deliveries)
write(root / "observations.json", observations)
if args.drop_interval_s is not None:
write(root / "dropped-receipts.json", source.dropped_receipts)
result = service.get()
(root / "terminal.rrd").write_bytes(service.scene(run["id"], True))
write(root / "scene-measurements.json", scene_measurements)
directory = service.directory(run["id"])
steps = []
seen = set()
for step in sorted(directory.glob("step-*")):
sample = json.loads((step / "source.json").read_text())
decision = json.loads((step / "decision.json").read_text())
if sample["role"] == "fresh-validation":
ids = {e["sequence"] for e in sample["events"]}
assert ids and not seen.intersection(ids)
assert all(
sample["fresh_floor_ns"] < e["monotonic_ns"] <= sample["requested_monotonic_ns"]
for e in sample["events"]
)
seen.update(ids)
steps.append(dict(role=sample["role"], **decision))
checks = dict(
completed=result["state"] == "completed",
expected_tracking=scene_saved == (args.reference_kind == "correct-entry"),
prior_not_accepted=result.get("initialization_temporal", {}).get("accepted") is False,
complete_search=(
result.get("initialization_result", {}).get("initialization", {}).get("complete")
is True
and len(result["initialization_result"]["initialization"]["attempts"])
== result["initialization_result"]["initialization"]["expected_attempts"]
),
producer_ok=getattr(source, "error", None) is None,
no_live_authority_after_end=service.accepted_sample is None
and result["tracking_state"] == "lost",
lease_released=source.owner is None and not lock.locked(),
inputs_unchanged=all(digest(Path(p)) == sha for p, sha in inputs.items()),
code_unchanged=all(digest(Path(p)) == sha for p, sha in implementation.items()),
no_green_after_end=result.get("presentation_state") != "live",
terminal_transform_retained=(not scene_saved or service.presentation.result is not None),
)
if camera:
camera_report = camera.report()
write(root / "camera-report.json", camera_report)
checks["camera_passed"] = all(camera_report["checks"].values())
if args.spatial_stop_monotonic_ns is not None:
checks["commanded_stop_not_loss"] = result[
"termination_reason"
] == "spatial-stop-requested" and (
args.drop_interval_s is not None
or (
result.get("recovery_attempt", 0) == 0
and not any(t["phase"] == "lost" for t in result.get("phase_transitions", []))
)
)
checks["capture_retained_at_stop"] = bool(
source.stopping_snapshot and source.stopping_snapshot["active"]
)
if args.drop_interval_s is not None:
start, end = args.drop_interval_s
# The fault probe requires successful acquisition BEFORE the declared
# fault. Recovery may remain unconfirmed if the recorded operator kept
# walking; a synthetic stationary interval must not be manufactured.
prior = (
(first_tracking_state or {}).get("initialization_result", {}).get("initialization", {})
)
checks["complete_search"] = (
prior.get("complete") is True and len(prior["attempts"]) == prior["expected_attempts"]
)
checks["prior_not_accepted"] = (first_tracking_state or {}).get(
"initialization_temporal", {}
).get("accepted") is False
checks["tracking_before_fault"] = any(
o["tracking_state"] == "tracking" and o["at_s"] < start for o in observations
)
checks["fault_exercised"] = bool(source.dropped_receipts)
recovery = [o for o in observations if o["key"][1] == "recovering"]
checks["recovery_keeps_capture"] = bool(recovery) and all(
o["key"][0] == "running" and o["source_active"] for o in recovery
)
checks["recovery_revokes_old_authority"] = bool(recovery) and all(
o["tracking_state"] != "tracking" and not o["accepted_sample"] for o in recovery
)
write(
root / "summary.json",
dict(
started_at_utc=started_utc,
finished_at_utc=utc_now_iso(),
started_monotonic_ns=source.started,
run_id=run["id"],
checks=checks,
steps=steps,
event_count=len(source.deliveries),
scene_count=scene_count,
ingress=source.snapshot().get("queues", {}),
display=result.get("display"),
producer_error=getattr(source, "error", None),
maximum_delivery_lag_s=max(d["lag_s"] for d in source.deliveries),
vehicle_control=False,
localization_confirmed=False,
),
)
write(
root / "seal.json",
{str(p.relative_to(root)): digest(p) for p in root.rglob("*") if p.is_file()},
)
print(json.dumps(checks), flush=True)
assert all(checks.values()), "See retained integration evidence."
if __name__ == "__main__":
main()
+228
View File
@@ -0,0 +1,228 @@
"""Fixed recovery/stationary functional probes; no device or application access."""
import argparse
import json
import platform
from pathlib import Path
import numpy as np
from k1link.artifacts import utc_now_iso
from k1link.device_plugins.xgrids_k1.localization_source import extract_submap
from k1link.device_plugins.xgrids_k1.planning_replay import iter_planning_events
from k1link.missions.causal_replay import digest, replay
from k1link.missions.entry_acquisition_worker import run_entry_acquisition
from k1link.missions.replay_faults import drop_receipts
from k1link.missions.stationary_entry import STATIONARY_POLICY, stationary_prefix
def write(path, data):
path.write_text(json.dumps(data, allow_nan=False, indent=2))
def code_hashes():
root = Path(__file__).resolve().parents[1]
paths = [
Path(__file__).resolve(),
*sorted((root / "src/k1link/missions").glob("*.py")),
root / "src/k1link/device_plugins/xgrids_k1/planning_replay.py",
root / "src/k1link/device_plugins/xgrids_k1/planning_live.py",
root / "src/k1link/device_plugins/xgrids_k1/localization_source.py",
]
return {str(p.relative_to(root)): digest(p) for p in paths}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--stage", choices=["prepare", "baseline", "drop", "stationary"], required=True
)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--reference-raw", type=Path)
parser.add_argument("--reference-planning", type=Path)
parser.add_argument("--query-raw", type=Path)
parser.add_argument("--query-planning", type=Path)
parser.add_argument("--negative-controls", type=Path)
args = parser.parse_args()
root = args.output
if args.stage == "prepare":
if any(
x is None
for x in (
args.reference_raw,
args.reference_planning,
args.query_raw,
args.query_planning,
args.negative_controls,
)
):
parser.error("Preparation requires all five source arguments.")
a = json.loads(args.reference_planning.read_text())
b = json.loads(args.query_planning.read_text())
if a["session_id"] == b["session_id"]:
raise ValueError("Independent query required.")
files = {
args.reference_planning: digest(args.reference_planning),
args.query_planning: digest(args.query_planning),
}
for raw, p in ((args.reference_raw, a), (args.query_raw, b)):
files[raw] = p["source_digests"]["raw-transport-primary"]
files[raw.with_name("mqtt.metadata.jsonl")] = p["source_digests"]["raw-transport-index"]
negative = json.loads((args.negative_controls / "report.json").read_text())
negative_file = args.negative_controls / "wrong-region/registration-input.npz"
files[negative_file] = negative["artifacts"]["wrong-region/registration-input.npz"]
if any(digest(p) != h for p, h in files.items()):
raise ValueError("Input digest mismatch.")
root.mkdir(parents=True, exist_ok=False)
end = max(i for i, p in enumerate(a["poses"]) if p["distance_m"] <= 40)
reference, extraction = extract_submap(args.reference_raw, a, 0, end)
path = np.array([p["position"] for p in a["poses"][: end + 1]])
wrong_start = next(i for i, p in enumerate(a["poses"]) if p["distance_m"] >= 130)
wrong_entry = np.array(a["poses"][wrong_start]["position"])
wrong_forward = next(
np.array(p["position"]) - wrong_entry
for p in a["poses"][wrong_start + 1 :]
if np.linalg.norm((np.array(p["position"]) - wrong_entry)[:2]) >= 3
)
np.savez_compressed(root / "reference.npz", reference=reference, reference_path=path)
write(
root / "manifest.json",
dict(
schema_version="missioncore.recovery-probe/v1",
created_at_utc=utc_now_iso(),
reference_session=a["session_id"],
query_session=b["session_id"],
reference_length_m=a["poses"][end]["distance_m"],
extraction=extraction,
query_raw=str(args.query_raw),
negative_file=str(negative_file),
negative_entry=wrong_entry.tolist(),
negative_forward=wrong_forward.tolist(),
input_digests={str(p): h for p, h in files.items()},
reference_sha256=digest(root / "reference.npz"),
implementation_sha256=code_hashes(),
vehicle_control=False,
localization_confirmed=False,
),
)
print(
json.dumps(
dict(
stage="prepared",
reference_length_m=a["poses"][end]["distance_m"],
points=len(reference),
)
),
flush=True,
)
return
manifest = json.loads((root / "manifest.json").read_text())
inputs = {
**manifest["input_digests"],
str(root / "reference.npz"): manifest["reference_sha256"],
}
if any(digest(Path(p)) != h for p, h in inputs.items()):
raise ValueError("Input changed before probe.")
code = code_hashes()
with np.load(root / "reference.npz", allow_pickle=False) as data:
reference, path = data["reference"], data["reference_path"]
events = iter_planning_events(Path(manifest["query_raw"]), manifest["query_session"])
destination = root / args.stage
if args.stage in {"baseline", "drop"}:
fault = {}
if args.stage == "drop":
baseline = json.loads((root / "baseline/report.json").read_text())
if baseline["first_tracking_s"] is None or baseline["first_tracking_s"] >= 44:
raise ValueError(
"Baseline did not establish tracking before frozen fault interval."
)
events = drop_receipts(events, 44.0, 47.0, fault)
report = replay(events, reference, path, destination, mode="acquisition")
report["fault_injection"] = fault or None
else:
destination.mkdir(exist_ok=False)
report = dict(
schema_version="missioncore.stationary-probe/v1",
created_at_utc=utc_now_iso(),
policy=STATIONARY_POLICY,
results={},
)
sample, initial, forward, prefix = stationary_prefix(events, path)
events.close()
report["prefix"] = prefix
np.savez_compressed(
destination / "prefix.npz",
points=sample["points"],
path=sample["path"],
initial=initial,
forward=forward,
)
with np.load(manifest["negative_file"], allow_pickle=False) as data:
wrong_reference = data["reference"]
# Only fixed A geometry is reused. No previous B seed or fit is read.
wrong_entry = np.array(manifest["negative_entry"])
wrong_forward = np.array(manifest["negative_forward"])
wrong_initial = np.eye(4)
wrong_initial[:3, 3] = wrong_entry - sample["path"][0]
for name, ref, hint, basis in [
("correct-entry", reference, initial, forward),
("wrong-region", wrong_reference, wrong_initial, wrong_forward),
]:
job = destination / name
job.mkdir()
result = run_entry_acquisition(
job, ref, sample["points"], hint, sample["path"][0], basis, mode="stationary"
)
report["results"][name] = {
k: v for k, v in result.items() if k != "matched_query_indices"
}
print(
json.dumps(
dict(
control=name,
status=result["status"],
reasons=result["reasons"],
attempts=len(result["initialization"]["attempts"]),
clusters=result["initialization"]["clusters"],
seconds=result["initialization"]["elapsed_s"],
)
),
flush=True,
)
report["artifacts"] = {
str(p.relative_to(destination)): digest(p)
for p in destination.rglob("*")
if p.is_file()
}
report.update(
input_digests=inputs,
source_integrity_verified=all(digest(Path(p)) == h for p, h in inputs.items()),
implementation_sha256=code,
runtime=dict(
system=platform.system(), machine=platform.machine(), python=platform.python_version()
),
vehicle_control=False,
localization_confirmed=False,
finished_at_utc=utc_now_iso(),
)
write(destination / "report.json", report)
print(
json.dumps(
{
k: report.get(k)
for k in (
"mode",
"state",
"first_candidate_s",
"first_tracking_s",
"source_integrity_verified",
"transitions",
)
}
),
flush=True,
)
if __name__ == "__main__":
main()
+97
View File
@@ -0,0 +1,97 @@
"""Two bounded negative checks using an already captured causal snapshot."""
import argparse
import json
from pathlib import Path
import numpy as np
from k1link.device_plugins.xgrids_k1.localization_source import extract_submap
from k1link.missions.causal_replay import digest
from k1link.missions.registration import path_hint
from k1link.missions.registration_worker import run_registration
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--replay", type=Path, required=True)
p.add_argument("--reference-raw", type=Path, required=True)
p.add_argument("--reference-planning", type=Path, required=True)
p.add_argument("--output", type=Path, required=True)
args = p.parse_args()
base = json.loads((args.replay / "report.json").read_text())
planning = json.loads(args.reference_planning.read_text())
# Fixed third causal snapshot, never the final offline B fit.
snapshot = args.replay / "step-003/registration-input.npz"
query_path_file = snapshot.with_name("query-path.npy")
files = {
snapshot: base["artifacts"][str(snapshot.relative_to(args.replay))],
query_path_file: base["artifacts"][str(query_path_file.relative_to(args.replay))],
args.reference_raw: planning["source_digests"]["raw-transport-primary"],
args.reference_raw.with_name("mqtt.metadata.jsonl"): planning["source_digests"][
"raw-transport-index"
],
}
for file, expected in files.items():
if digest(file) != expected:
raise ValueError("Control input digest mismatch.")
args.output.mkdir(parents=True, exist_ok=False)
with np.load(snapshot, allow_pickle=False) as data:
reference, query, hint = data["reference"], data["query"], data["initial"]
path = np.load(query_path_file, allow_pickle=False)
distant = hint.copy()
distant[:3, 3] += 1000
far_dir = args.output / "far-seed"
far_dir.mkdir()
far = run_registration(far_dir, reference, query, distant)
# This disjoint A interval was specified before executing the controls.
poses = planning["poses"]
start = next(i for i, x in enumerate(poses) if x["distance_m"] >= 130)
end = next(i for i, x in enumerate(poses) if x["distance_m"] >= 155)
wrong, meta = extract_submap(args.reference_raw, planning, start, end)
wrong_path = np.array([x["position"] for x in poses[start : end + 1]])
wrong_dir = args.output / "wrong-region"
wrong_dir.mkdir()
other = run_registration(wrong_dir, wrong, query, path_hint(wrong_path, path))
def clean(value):
return {k: v for k, v in value.items() if k != "matched_query_indices"}
report = dict(
schema_version="missioncore.causal-replay-controls/v1",
source_step="step-003",
reference_interval_m=[130, 155],
reference_interval_indices=[start, end],
reference_extraction=meta,
results={"far-seed": clean(far), "wrong-region": clean(other)},
input_digests={str(k): v for k, v in files.items()},
source_integrity_verified=all(digest(k) == v for k, v in files.items()),
vehicle_control=False,
localization_confirmed=False,
)
report["artifacts"] = {
str(x.relative_to(args.output)): digest(x) for x in args.output.rglob("*") if x.is_file()
}
(args.output / "report.json").write_text(json.dumps(report, allow_nan=False))
print(
json.dumps(
{
k: {
j: v.get(j)
for j in [
"status",
"reasons",
"overlap",
"inlier_rmse_m",
"registration_seconds",
]
}
for k, v in report["results"].items()
}
),
flush=True,
)
if __name__ == "__main__":
main()
+175
View File
@@ -0,0 +1,175 @@
"""Qualify preparation on an existing complete reference, without a live service."""
import argparse
import json
import time
from pathlib import Path
from types import SimpleNamespace
import numpy as np
from k1link.artifacts import utc_now_iso
from k1link.device_plugins.xgrids_k1.localization_source import extract_submap
from k1link.device_plugins.xgrids_k1.planning_source import export_planning_source
from k1link.missions.causal_replay import digest
from k1link.missions.reference_window import ReferenceWindowIndex, reference_window
from k1link.missions.sources import PlanningSources
from k1link.sessions.models import ReplayArtifact, ReplayCommand
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--previous-manifest", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
previous = json.loads(args.previous_manifest.read_text())
inputs = {Path(p): sha for p, sha in previous["input_digests"].items()}
assert all(digest(p) == sha for p, sha in inputs.items())
run_report = next(p for p in inputs if p.name == "report.json")
physical = json.loads(run_report.read_text())
session = physical["reference"]["session_id"]
raw = next(p for p in inputs if session in str(p) and p.name == "mqtt.raw.k1mqtt")
index = raw.with_name("mqtt.metadata.jsonl")
root = args.output
root.mkdir(parents=True, exist_ok=False)
code_paths = [
Path(__file__),
*Path("src/k1link/missions").glob("*.py"),
Path("src/k1link/device_plugins/xgrids_k1/localization_source.py"),
Path("src/k1link/device_plugins/xgrids_k1/planning_source.py"),
]
code = {str(p): digest(p) for p in code_paths}
artifacts = tuple(
ReplayArtifact(
artifact_id=name,
path=p.resolve(),
media_type=media,
file_byte_length=p.stat().st_size,
replay_byte_length=p.stat().st_size,
expected_sha256=inputs[p],
)
for name, p, media in [
("raw-transport-primary", raw, "application/x-k1mqtt"),
("raw-transport-index", index, "application/x-ndjson"),
]
)
command = ReplayCommand(
session_id=session,
plugin_id="archive-qualification",
allowed_root=raw.parent.resolve(),
session_root=raw.parent.resolve(),
primary_artifact_id="raw-transport-primary",
artifacts=artifacts,
timeline_origin_epoch_ns=0,
timeline_origin_monotonic_ns=0,
speed=1.0,
loop=False,
)
detail = SimpleNamespace(
plugin_id=command.plugin_id,
summary=SimpleNamespace(replayable=True, lab=None),
as_dict=lambda: {"display_name": "Private archive preparation"},
)
store = SimpleNamespace(
data_dir=root, prepare_replay=lambda _: command, get_session=lambda _: detail
)
sources = PlanningSources(
store, {command.plugin_id: export_planning_source}, {command.plugin_id: extract_submap}
)
t0 = time.monotonic()
doc = sources.get(session)
export_s = time.monotonic() - t0
draft = physical["draft"]["route"]
measurements = []
for label, first, last in [
("physical-selected", draft["start_index"], draft["end_index"]),
("complete-recorded-reference", 0, len(doc["poses"]) - 1),
]:
t0 = time.monotonic()
points, provenance = sources.reference_map(session, doc["generation"], first, last)
seconds = time.monotonic() - t0
if label == "physical-selected":
original = run_report.parent / "reference.npy"
assert digest(original) == physical["artifacts"]["reference.npy"]
assert np.array_equal(points, np.load(original, allow_pickle=False))
t1 = time.monotonic()
spatial_index = ReferenceWindowIndex(points)
indexed_s = time.monotonic() - t1
windows = []
for step in sorted(run_report.parent.glob("step-*")):
source_file = step / "source.json"
input_file = step / "registration-input.npz"
if not input_file.is_file():
continue
assert (
digest(source_file)
== physical["artifacts"][str(source_file.relative_to(run_report.parent))]
)
assert (
digest(input_file)
== physical["artifacts"][str(input_file.relative_to(run_report.parent))]
)
saved = json.loads(source_file.read_text())
with np.load(input_file, allow_pickle=False) as sample_input:
sample = dict(points=sample_input["query"], path=np.asarray(saved["query_path"]))
t2 = time.monotonic()
full, old = reference_window(points, sample, sample_input["initial"])
t3 = time.monotonic()
local, new = reference_window(
points, sample, sample_input["initial"], index=spatial_index
)
t4 = time.monotonic()
assert np.array_equal(full, local)
windows.append(
dict(
step=step.name,
full_s=t3 - t2,
indexed_s=t4 - t3,
examined=new["examined_points"],
target=len(local),
)
)
measurement = dict(
label=label,
seconds=seconds,
points=len(points),
bytes=points.nbytes,
route_m=doc["poses"][last]["distance_m"] - doc["poses"][first]["distance_m"],
tiles=len(provenance["tiles"]),
index_s=indexed_s,
index_array_bytes=spatial_index.order.nbytes,
spatial_cells=len(spatial_index.slices),
provenance=provenance,
exact_windows=len(windows),
median_window_full_s=float(np.median([w["full_s"] for w in windows])),
median_window_indexed_s=float(np.median([w["indexed_s"] for w in windows])),
median_window_examined=float(np.median([w["examined"] for w in windows])),
window_measurements=windows,
)
measurements.append(measurement)
print(json.dumps({k: v for k, v in measurement.items() if k != "provenance"}), flush=True)
del points, spatial_index
checks = dict(
inputs_unchanged=all(digest(p) == sha for p, sha in inputs.items()),
code_unchanged=all(digest(Path(p)) == sha for p, sha in code.items()),
staging_removed=not list(sources.root.glob(".source.*")),
selected_reference_exact=True,
)
result = dict(
created_at_utc=utc_now_iso(),
inputs={str(p): sha for p, sha in inputs.items()},
implementation_sha256=code,
export_s=export_s,
pose_count=len(doc["poses"]),
trajectory_cache_bytes=(sources.root / (doc["generation"] + ".json")).stat().st_size,
checks=checks,
measurements=measurements,
limitation="Preparation of existing reference only, not independent long traversal.",
)
(root / "report.json").write_text(json.dumps(result, indent=2, allow_nan=False))
print(json.dumps(checks), flush=True)
assert all(checks.values())
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
"""Compare indexed and full-scan windows on frozen physical-pass fit inputs."""
import argparse
import json
import time
from pathlib import Path
import numpy as np
from k1link.artifacts import utc_now_iso
from k1link.missions.causal_replay import digest
from k1link.missions.reference_window import ReferenceWindowIndex, reference_window
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
report = json.loads((args.run / "report.json").read_text())
artifacts = {args.run / p: sha for p, sha in report["artifacts"].items()}
assert all(digest(p) == sha for p, sha in artifacts.items())
code = {p: digest(p) for p in [Path(__file__), Path("src/k1link/missions/reference_window.py")]}
reference = np.load(args.run / "reference.npy", allow_pickle=False)
started = time.perf_counter()
index = ReferenceWindowIndex(reference)
preparation = time.perf_counter() - started
measurements = []
for step in sorted(args.run.glob("step-*")):
source = json.loads((step / "source.json").read_text())
if source["role"] != "fresh-validation":
continue
input_path = step / "registration-input.npz"
if not input_path.exists():
continue
with np.load(input_path, allow_pickle=False) as frozen:
sample = dict(
points=frozen["query"], path=np.asarray(source["query_path"])
)
t0 = time.perf_counter()
expected, old = reference_window(reference, sample, frozen["initial"])
t1 = time.perf_counter()
actual, new = reference_window(reference, sample, frozen["initial"], index=index)
t2 = time.perf_counter()
assert np.array_equal(expected, actual)
assert np.array_equal(actual, frozen["reference"])
assert new["target_sha256"] == old["target_sha256"]
measurements.append(
dict(
step=step.name,
full_scan_s=t1 - t0,
indexed_s=t2 - t1,
map_points=len(reference),
target_points=len(actual),
examined_points=new["examined_points"],
target_sha256=new["target_sha256"],
)
)
assert measurements
assert all(digest(p) == sha for p, sha in artifacts.items())
assert all(digest(p) == sha for p, sha in code.items())
result = dict(
created_at_utc=utc_now_iso(),
run=str(args.run),
original_report_sha256=digest(args.run / "report.json"),
implementation_sha256={str(p): sha for p, sha in code.items()},
index_preparation_s=preparation,
index_bytes=index.order.nbytes,
spatial_cells=len(index.slices),
measurements=measurements,
all_exact=True,
sources_unchanged=True,
limitation="One bounded pass on real saved inputs, not kilometre qualification.",
)
with args.output.open("x") as stream:
json.dump(result, stream, indent=2, allow_nan=False)
print(
json.dumps(
dict(
exact_windows=len(measurements),
map_points=len(reference),
index_preparation_s=preparation,
median_full_s=float(np.median([m["full_scan_s"] for m in measurements])),
median_indexed_s=float(np.median([m["indexed_s"] for m in measurements])),
median_examined=float(np.median([m["examined_points"] for m in measurements])),
)
)
)
if __name__ == "__main__":
main()
+154
View File
@@ -0,0 +1,154 @@
"""Frozen stationary-to-fresh functional experiment on independent archived walks."""
import argparse
import json
import platform
import shutil
import time
from pathlib import Path
import numpy as np
from k1link.artifacts import utc_now_iso
from k1link.device_plugins.xgrids_k1.planning_replay import iter_planning_events
from k1link.missions.causal_replay import digest
from k1link.missions.stationary_bootstrap import BOOTSTRAP_POLICY
from k1link.missions.stationary_replay import replay_stationary
def write(path, value):
path.write_text(json.dumps(value, indent=2, allow_nan=False))
def code_hashes():
root = Path(__file__).resolve().parents[1]
paths = [
Path(__file__).resolve(),
*sorted((root / "src/k1link/missions").glob("*.py")),
root / "src/k1link/device_plugins/xgrids_k1/planning_replay.py",
root / "src/k1link/device_plugins/xgrids_k1/planning_live.py",
root / "src/k1link/device_plugins/xgrids_k1/localization_source.py",
root / "tests/test_stationary_bootstrap.py",
]
return {str(p.relative_to(root)): digest(p) for p in paths}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--stage", choices=["prepare", "correct-entry", "wrong-region"], required=True
)
parser.add_argument("--predecessor", type=Path)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
root = args.output
if args.stage == "prepare":
previous = args.predecessor
old = json.loads((previous / "manifest.json").read_text())
inputs = {**old["input_digests"], str(previous / "reference.npz"): old["reference_sha256"]}
inputs[str(previous / "manifest.json")] = digest(previous / "manifest.json")
if any(digest(Path(p)) != sha for p, sha in inputs.items()):
raise ValueError("Predecessor input digest mismatch.")
root.mkdir(parents=True, exist_ok=False)
code = code_hashes()
for name in code:
target = root / "executed-source" / name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(name, target)
a_path = next(
p
for p in old["input_digests"]
if p.endswith(".json")
and json.loads(Path(p).read_text()).get("session_id") == old["reference_session"]
)
poses = json.loads(Path(a_path).read_text())["poses"]
wrong_path = np.array([p["position"] for p in poses if 130 <= p["distance_m"] <= 155])
with np.load(old["negative_file"], allow_pickle=False) as data:
np.savez_compressed(
root / "wrong-reference.npz", reference=data["reference"], reference_path=wrong_path
)
# The old archive supplies fixed A geometry only; no previous B fitted
# transform, mask or future B pose is used as an initialization seed.
inputs[str(root / "wrong-reference.npz")] = digest(root / "wrong-reference.npz")
write(
root / "manifest.json",
dict(
schema_version="missioncore.stationary-bootstrap-probe/v1",
created_at_utc=utc_now_iso(),
created_monotonic_ns=time.monotonic_ns(),
query_raw=old["query_raw"],
query_session=old["query_session"],
reference_session=old["reference_session"],
references={
"correct-entry": str(previous / "reference.npz"),
"wrong-region": str(root / "wrong-reference.npz"),
},
input_digests=inputs,
implementation_sha256=code,
protocol=BOOTSTRAP_POLICY,
maximum_seconds=65.0,
maximum_distance_m=40.0,
expectations=dict(
correct_entry="complete search -> provisional -> three fresh consistent fits",
wrong_region="no current candidate or tracking",
freshness="all validation observations after ready; windows have disjoint IDs",
pace="original receipt clocks, 1x; no offline B heading or transform",
),
notes="Engineering qualification only. Frame continuity is unverified; "
"one pre-ready receipt gap permits a hypothesis, not a tracking result. "
"No new capture, device commands, UI changes or live activation.",
vehicle_control=False,
localization_confirmed=False,
),
)
print(json.dumps(dict(stage="prepared", output=str(root))), flush=True)
return
manifest = json.loads((root / "manifest.json").read_text())
inputs = manifest["input_digests"]
code = code_hashes()
if code != manifest["implementation_sha256"]:
raise ValueError("Implementation changed since protocol freeze.")
if any(digest(Path(p)) != sha for p, sha in inputs.items()):
raise ValueError("Source changed before replay.")
with np.load(manifest["references"][args.stage], allow_pickle=False) as data:
reference, path = data["reference"], data["reference_path"]
events = iter_planning_events(Path(manifest["query_raw"]), manifest["query_session"])
report = replay_stationary(
events,
reference,
path,
root / args.stage,
max_seconds=manifest["maximum_seconds"],
max_distance=manifest["maximum_distance_m"],
)
report.update(
input_digests=inputs,
source_integrity_verified=all(digest(Path(p)) == h for p, h in inputs.items()),
implementation_sha256=code,
code_integrity_verified=code_hashes() == code,
runtime=dict(
system=platform.system(), machine=platform.machine(), python=platform.python_version()
),
)
write(root / args.stage / "report.json", report)
print(
json.dumps(
{
k: report.get(k)
for k in (
"state",
"first_prior_s",
"first_candidate_s",
"first_tracking_s",
"transitions",
"source_integrity_verified",
"code_integrity_verified",
)
}
),
flush=True,
)
if __name__ == "__main__":
main()
+179
View File
@@ -0,0 +1,179 @@
"""Retained fMP4 through the real durable gateway and a bounded decoder reader.
The producer reads local files at original receipt cadence. It cannot connect to
RTSP or send device commands. This qualifies downstream camera contention, not
network transport, device authority, or browser MSE.
"""
import hashlib
import json
import subprocess
import sys
import threading
import time
from pathlib import Path
from k1link.device_plugins.xgrids_k1.camera import (
XgridsK1CameraGateway,
_CameraProducer,
_read_fmp4_stdout,
)
from k1link.missions.causal_replay import digest
from k1link.web.camera_archive import CameraArchiveWriter
def camera_inputs(epoch):
rows = [json.loads(line) for line in (epoch / "index.jsonl").read_text().splitlines()]
assert rows
inputs = {str(epoch / "index.jsonl"): digest(epoch / "index.jsonl")}
if rows[0]["kind"] != "init":
# The canonical August baseline indexes media only. Its init timestamp
# is unavailable; deliver init at the first media receipt without
# inventing a measured initialization latency. All media deltas survive.
summary = json.loads((epoch / "summary.json").read_text())
inputs[str(epoch / "summary.json")] = digest(epoch / "summary.json")
rows.insert(0, dict(kind="init", path="init.mp4", sha256=summary["init_sha256"],
host_monotonic_ns=rows[0]["host_monotonic_ns"]))
previous = rows[0]["host_monotonic_ns"]
for row in rows:
path = (epoch / row["path"]).resolve()
assert path.is_relative_to(epoch.resolve()), "Camera path escaped retained epoch."
assert row["host_monotonic_ns"] >= previous
previous = row["host_monotonic_ns"]
assert digest(path) == row["sha256"]
inputs[str(path)] = row["sha256"]
assert (previous - rows[0]["host_monotonic_ns"]) / 1e9 <= 65
return rows, inputs
class ArchiveCamera:
def __init__(self, epoch, root, ffmpeg):
self.epoch, self.root, self.ffmpeg = epoch, root, ffmpeg
self.rows, self.inputs = camera_inputs(epoch)
self.gateway = None
self.producer = None
self.decoder = None
self.threads = []
self.files = []
self.errors = []
self.commits = []
self.received = []
def start(self):
self.root.mkdir(parents=True, exist_ok=False)
self.started = time.monotonic_ns()
# Explicit local emitter only; never use the gateway's RTSP spawn path.
process = subprocess.Popen(
[sys.executable, str(Path(__file__).resolve()), str(self.epoch)],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
)
self.gateway = XgridsK1CameraGateway(
self.root, "archive-qualification", committed_segment_observer=self.observe,
)
self.producer = _CameraProducer(
1, "sensor.camera.right", process,
CameraArchiveWriter(self.root, "sensor.camera.right", 1),
)
# Private instance uses the same parser/archive/queue implementation.
# No singleton, active acquisition, authority ledger, or network endpoint.
self.gateway._producer = self.producer
self.gateway._generation = 1
self.gateway._source_id = self.producer.source_id
self.gateway._recording_root = self.root
self.gateway._phase = "connecting"
self.lease = self.gateway.open_delivery(1, require_recording=True)
self.gateway.expect_source_end_for_device_stop() # Expected fixture EOF only.
frames = (self.root / "decoded.framemd5").open("wb")
errors = (self.root / "decode.log").open("wb")
self.files.extend([frames, errors])
self.decoder = subprocess.Popen(
[str(self.ffmpeg), "-hide_banner", "-loglevel", "warning", "-threads", "1",
"-i", "pipe:0", "-map", "0:v:0", "-an", "-f", "framemd5", "pipe:1"],
stdin=subprocess.PIPE, stdout=frames, stderr=errors,
)
self.threads = [
threading.Thread(target=self.decode, name="archive-camera-preview"),
threading.Thread(
target=_read_fmp4_stdout, args=(self.gateway, self.producer),
name="archive-camera-parser",
),
]
for thread in self.threads:
thread.start()
def observe(self, segment):
self.commits.append(dict(
kind=segment.kind, sequence=segment.sequence,
at_s=(time.monotonic_ns() - self.started) / 1e9,
sha256=hashlib.sha256(segment.payload).hexdigest(),
))
def decode(self):
try:
while (segment := self.lease.segments.get()) is not None:
kind, payload = segment
self.decoder.stdin.write(payload)
self.decoder.stdin.flush()
self.received.append(dict(kind=kind, sha256=hashlib.sha256(payload).hexdigest()))
except Exception as exc:
self.errors.append(f"{type(exc).__name__}: {exc}")
finally:
self.decoder.stdin.close()
def close(self):
if self.gateway is None:
return
for thread in self.threads:
thread.join(3)
self.gateway.close()
for thread in self.threads:
thread.join(3)
for process in (self.producer.process, self.decoder):
try:
process.wait(timeout=3)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=3)
self.errors.append("Child exceeded cleanup deadline.")
for stream in self.files:
stream.close()
if self.producer.process.stdout:
self.producer.process.stdout.close()
def report(self):
archive = self.root / "media/sensor.camera.right/epoch-1"
summary = json.loads((archive / "summary.json").read_text())
expected = [row["sha256"] for row in self.rows]
frames = [line for line in (self.root / "decoded.framemd5").read_text().splitlines()
if line and not line.startswith("#")]
decoder_log = (self.root / "decode.log").read_text()
checks = dict(
archive_complete=summary["status"] == "complete",
committed_all_exact_bytes=[item["sha256"] for item in self.commits] == expected,
delivered_all_exact_bytes=[item["sha256"] for item in self.received] == expected,
preview_not_retired=self.lease.failure_code is None,
clean_decode=self.decoder.returncode == 0 and not decoder_log.strip() and bool(frames),
workers_stopped=all(not t.is_alive() for t in self.threads),
no_errors=not self.errors,
input_unchanged=all(digest(Path(p)) == sha for p, sha in self.inputs.items()),
)
return dict(checks=checks, errors=self.errors, commits=self.commits,
media_segments=len(self.rows) - 1, decoded_frames=len(frames),
decoder_returncode=self.decoder.returncode, browser_mse_tested=False,
rtsp_tested=False, acquisition_authority_tested=False)
def emit(epoch):
rows, _ = camera_inputs(epoch)
started, origin = time.monotonic_ns(), rows[0]["host_monotonic_ns"]
for row in rows:
delay = started + row["host_monotonic_ns"] - origin - time.monotonic_ns()
if delay > 0:
time.sleep(delay / 1e9)
sys.stdout.buffer.write((epoch / row["path"]).read_bytes())
sys.stdout.buffer.flush()
time.sleep(0.5) # Allow the disposable reader to drain before expected EOF.
if __name__ == "__main__":
emit(Path(sys.argv[1]))
+141
View File
@@ -0,0 +1,141 @@
"""Bounded 1x raw archive through the production derived queue and K1 decoder."""
import threading
import time
from k1link.compute.live_perception import LivePerceptionIngress
from k1link.device_plugins.xgrids_k1.planning_live import K1PlanningLiveSource
from k1link.device_plugins.xgrids_k1.viewer.replay import iter_replay_messages
class ReceiptQueueArchiveSource:
"""Private instance only: no live singleton, MQTT, capture or hardware access."""
def __init__(
self,
raw,
session,
maximum_seconds=65,
*,
after_monotonic_ns=None,
spatial_stop_monotonic_ns=None,
drop_interval_s=None,
):
if not 0 < maximum_seconds <= 600:
raise ValueError("This offline probe supports up to ten minutes of recorded receipts.")
self.maximum_seconds = maximum_seconds
if drop_interval_s is not None and not (
len(drop_interval_s) == 2
and 0 < drop_interval_s[0] < drop_interval_s[1] < maximum_seconds
):
raise ValueError("Invalid explicit archive fault interval.")
self.drop_interval_s = drop_interval_s
self.dropped_receipts = []
self.raw, self.session = raw, session
self.after_monotonic_ns = after_monotonic_ns
self.spatial_stop_monotonic_ns = spatial_stop_monotonic_ns
self.stopping_snapshot = None
self.ingress = LivePerceptionIngress()
self.adapter = K1PlanningLiveSource(self.ingress)
self.started = None
self.owner = None
self.deliveries = []
self.stop = threading.Event()
self.thread = None
self.error = None
def snapshot(self):
return self.ingress.snapshot()
def open(self, owner):
self.adapter.open(owner)
self.owner = owner
def close(self, owner):
assert self.owner == owner
self.stop.set()
if self.thread:
self.thread.join(3)
assert not self.thread.is_alive(), "Archive publisher did not stop."
self.adapter.close(owner)
self.owner = None
def activate(self):
self.ingress.begin_session(self.session)
self.started = time.monotonic_ns()
self.thread = threading.Thread(target=self.publish, name="bounded-archive-publisher")
self.thread.start()
def take(self, owner):
return self.adapter.take(owner)
def publish(self):
previous, origin = -1, None
try:
for message in iter_replay_messages(self.raw):
stamp = message.received_monotonic_ns
if stamp is None or stamp < previous:
raise ValueError("Archive has missing or regressing receipt clocks.")
previous = stamp
if self.after_monotonic_ns is not None and stamp < self.after_monotonic_ns:
continue
modality = (
"pose"
if message.topic.endswith("/lio_pose")
else "lidar"
if message.topic.endswith("/lio_pcl")
else None
)
if modality is None:
continue
if origin is None:
origin = stamp
delay = stamp - origin
if delay > self.maximum_seconds * 1e9:
break
if (
self.drop_interval_s is not None
and self.drop_interval_s[0] <= delay / 1e9 < self.drop_interval_s[1]
):
self.dropped_receipts.append(
dict(sequence=message.sequence, kind=modality, original_monotonic_ns=stamp)
)
continue # Surviving receipts retain original cadence and identity.
mapped = self.started + delay
if self.stop.wait(max(0, (mapped - time.monotonic_ns()) / 1e9)):
break
delivered = time.monotonic_ns()
admitted = self.ingress.publish(
modality=modality,
source_id=message.topic,
source_sequence=message.sequence,
captured_at_epoch_ns=message.received_at_epoch_ns,
received_monotonic_ns=mapped,
payload=message.payload,
)
self.deliveries.append(
dict(
sequence=message.sequence,
kind=modality,
original_monotonic_ns=stamp,
mapped_monotonic_ns=mapped,
delivered_monotonic_ns=delivered,
lag_s=(delivered - mapped) / 1e9,
admitted=admitted,
)
)
if (
self.spatial_stop_monotonic_ns is not None
and stamp >= self.spatial_stop_monotonic_ns
):
self.ingress.request_spatial_stop(self.session, 1)
self.stopping_snapshot = self.ingress.snapshot()
# Model retained recorder ownership. The planner must end
# first and close this private publisher, not wait for EOF.
if not self.stop.wait(10):
raise RuntimeError("Planning did not finish after the STOP boundary.")
break
except Exception as exc:
self.error = f"{type(exc).__name__}: {exc}"
finally:
self.ingress.end_session(self.session)
+133
View File
@@ -0,0 +1,133 @@
"""Run one bounded causal experiment against an immutable saved reference.
Use the repository virtual environment. This CLI never connects to hardware or
the application ingress. Inputs and reports belong in private runtime storage.
"""
import argparse
import json
import platform
import time
from pathlib import Path
import numpy as np
from k1link.device_plugins.xgrids_k1.planning_replay import iter_planning_events
from k1link.missions.causal_replay import digest, replay
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--reference-run", required=True, type=Path)
parser.add_argument("--query-raw", required=True, type=Path)
parser.add_argument("--query-planning", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument(
"--mode", choices=["baseline", "tracking", "acquisition"], default="baseline"
)
args = parser.parse_args()
started = time.monotonic()
repository = Path(__file__).resolve().parents[1]
code_paths = [Path(__file__).resolve()] + [
repository / name
for name in (
"src/k1link/missions/entry_acquisition.py",
"src/k1link/missions/entry_acquisition_worker.py",
"src/k1link/missions/causal_replay.py",
"src/k1link/missions/causal_tracking.py",
"src/k1link/missions/live_buffer.py",
"src/k1link/missions/registration.py",
"src/k1link/missions/registration_worker.py",
"src/k1link/device_plugins/xgrids_k1/planning_replay.py",
"src/k1link/device_plugins/xgrids_k1/planning_live.py",
)
]
code_hashes = {str(p.relative_to(repository)): digest(p) for p in code_paths}
original = json.loads((args.reference_run / "report.json").read_text())
query = json.loads(args.query_planning.read_text())
if query["session_id"] == original["reference"]["session_id"]:
raise ValueError("Independent replay requires a different query recording.")
files = {
args.reference_run / "report.json": digest(args.reference_run / "report.json"),
args.reference_run / "clouds.npz": original["artifacts"]["clouds.npz"],
args.query_raw: query["source_digests"]["raw-transport-primary"],
args.query_raw.with_name("mqtt.metadata.jsonl"): query["source_digests"][
"raw-transport-index"
],
args.query_planning: digest(args.query_planning),
}
for path, expected in files.items():
if digest(path) != expected:
raise ValueError(f"Source digest mismatch: {path.name}")
with np.load(args.reference_run / "clouds.npz", allow_pickle=False) as archive:
# Deliberately do not load the fitted query, its path, or the final transform.
reference = archive["reference"]
reference_path = archive["reference_path"]
prep = time.monotonic() - started
report = replay(
iter_planning_events(args.query_raw, query["session_id"]),
reference,
reference_path,
args.output,
mode=args.mode,
)
for path, expected in files.items():
if digest(path) != expected:
report.update(state="invalid", source_integrity_verified=False)
(args.output / "report.json").write_text(json.dumps(report, allow_nan=False))
raise ValueError("Source changed during replay.")
report.update(
reference_run_id=original["id"],
reference=original["reference"],
query={k: query[k] for k in ["session_id", "generation", "source_digests", "label"]},
input_digests={str(path): value for path, value in files.items()},
source_integrity_verified=True,
implementation_sha256=code_hashes,
reference_preparation_s=prep,
runtime=dict(
system=platform.system(), machine=platform.machine(), python=platform.python_version()
),
)
(args.output / "report.json").write_text(json.dumps(report, allow_nan=False))
print(
json.dumps(
{
k: report[k]
for k in [
"mode",
"state",
"elapsed_s",
"distance_m",
"first_heading_s",
"first_candidate_s",
"first_tracking_s",
"source_integrity_verified",
]
}
),
flush=True,
)
print(
json.dumps(
[
dict(
step=s["step"],
at=s["requested_s"],
distance=s["distance_m"],
seed=s["seed"],
status=s["result"]["status"],
temporal=s["temporal"],
overlap=s["result"].get("overlap"),
rmse=s["result"].get("inlier_rmse_m"),
fit_s=s["result"].get("registration_seconds"),
state=s["tracking_state"],
)
for s in report["steps"]
]
),
flush=True,
)
if __name__ == "__main__":
main()