feat(k1): add live cameras and reliable spatial following
This commit is contained in:
DCCONSTRUCTIONS
@@ -6,6 +6,10 @@ import { createServer } from "vite";
|
||||
let server;
|
||||
let xgridsK1Manifest;
|
||||
let xgridsK1ObservationSources;
|
||||
let openObservationSource;
|
||||
let shouldRestartObservationSource;
|
||||
let consumeCameraLeaseRetry;
|
||||
let resetCameraLeaseRetryBudget;
|
||||
|
||||
before(async () => {
|
||||
server = await createServer({
|
||||
@@ -19,6 +23,12 @@ before(async () => {
|
||||
({ xgridsK1ObservationSources } = await server.ssrLoadModule(
|
||||
"/src/device-plugins/xgrids-k1/observationSources.ts",
|
||||
));
|
||||
({ openObservationSource, shouldRestartObservationSource } = await server.ssrLoadModule(
|
||||
"/src/core/observation/layoutPolicy.ts",
|
||||
));
|
||||
({ consumeCameraLeaseRetry, resetCameraLeaseRetryBudget } = await server.ssrLoadModule(
|
||||
"/src/components/MseFmp4WebSocketPlayer.tsx",
|
||||
));
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
@@ -31,7 +41,30 @@ function model() {
|
||||
return activeModel;
|
||||
}
|
||||
|
||||
function declaredState() {
|
||||
function cameraRow(sourceId, label) {
|
||||
return {
|
||||
stream_id: `camera.preview.${sourceId.split(".").at(-1)}`,
|
||||
source_id: sourceId,
|
||||
semantic_channel_id: "camera.preview.live",
|
||||
label,
|
||||
sensor_kind: "camera",
|
||||
modality: "encoded-video",
|
||||
availability: "available",
|
||||
endpoint_label: "MSE · fMP4",
|
||||
activation: {
|
||||
group_id: "camera.preview.decoder",
|
||||
max_active: 1,
|
||||
selected: false,
|
||||
controllable: true,
|
||||
},
|
||||
delivery: null,
|
||||
};
|
||||
}
|
||||
|
||||
function declaredState(cameraRows = [
|
||||
cameraRow("sensor.camera.left", "K1 · камера слева"),
|
||||
cameraRow("sensor.camera.right", "K1 · камера справа"),
|
||||
]) {
|
||||
const activeModel = model();
|
||||
return {
|
||||
phase: "connected",
|
||||
@@ -60,13 +93,20 @@ function declaredState() {
|
||||
revision: "test-profile",
|
||||
streams: [
|
||||
{ stream_id: "spatial.point-cloud.live", modality: "point-cloud", availability: "observed" },
|
||||
{ stream_id: "camera.preview.live", modality: "encoded-video", availability: "observed" },
|
||||
...cameraRows,
|
||||
],
|
||||
},
|
||||
camera_preview: {
|
||||
phase: "idle",
|
||||
revision: 1,
|
||||
generation: 0,
|
||||
active_source_id: null,
|
||||
delivery: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function streamingState() {
|
||||
function pointCloudStreamingState() {
|
||||
return {
|
||||
...declaredState(),
|
||||
phase: "streaming",
|
||||
@@ -88,6 +128,36 @@ function streamingState() {
|
||||
};
|
||||
}
|
||||
|
||||
function cameraStreamingState(sourceId) {
|
||||
const state = pointCloudStreamingState();
|
||||
const delivery = {
|
||||
id: `preview-generation-7:${sourceId}`,
|
||||
kind: "mse-fmp4-websocket",
|
||||
url: "/api/v1/device-plugins/xgrids-k1/camera-preview/ws?generation=7",
|
||||
media_type: 'video/mp4; codecs="avc1.640028"',
|
||||
};
|
||||
state.sensor_catalog = {
|
||||
...state.sensor_catalog,
|
||||
streams: state.sensor_catalog.streams.map((stream) =>
|
||||
stream.source_id === sourceId
|
||||
? {
|
||||
...stream,
|
||||
availability: "streaming",
|
||||
activation: { ...stream.activation, selected: true },
|
||||
delivery,
|
||||
}
|
||||
: stream),
|
||||
};
|
||||
state.camera_preview = {
|
||||
phase: "streaming",
|
||||
revision: 4,
|
||||
generation: 7,
|
||||
active_source_id: sourceId,
|
||||
delivery,
|
||||
};
|
||||
return state;
|
||||
}
|
||||
|
||||
function collectUrlLikeStrings(value, found = []) {
|
||||
if (typeof value === "string") {
|
||||
if (value.includes("://")) found.push(value);
|
||||
@@ -103,95 +173,199 @@ function collectUrlLikeStrings(value, found = []) {
|
||||
return found;
|
||||
}
|
||||
|
||||
test("K1 maps to one primary point cloud and two auxiliary camera channels", () => {
|
||||
const sources = xgridsK1ObservationSources(declaredState(), model());
|
||||
|
||||
assert.equal(sources.length, 3);
|
||||
assert.deepEqual(
|
||||
sources.map(({ sourceId, semanticChannelId, modality, role }) => ({
|
||||
sourceId,
|
||||
semanticChannelId,
|
||||
modality,
|
||||
role,
|
||||
})),
|
||||
[
|
||||
{
|
||||
sourceId: "sensor.lidar.primary",
|
||||
semanticChannelId: "spatial.point-cloud.live",
|
||||
modality: "point-cloud",
|
||||
role: "primary",
|
||||
},
|
||||
{
|
||||
sourceId: "sensor.camera.left",
|
||||
semanticChannelId: "camera.preview.live",
|
||||
modality: "video",
|
||||
role: "auxiliary",
|
||||
},
|
||||
{
|
||||
sourceId: "sensor.camera.right",
|
||||
semanticChannelId: "camera.preview.live",
|
||||
modality: "video",
|
||||
role: "auxiliary",
|
||||
},
|
||||
],
|
||||
test("K1 maps zero, one or N catalog cameras without model-specific source ids", () => {
|
||||
const zero = xgridsK1ObservationSources(declaredState([]), model());
|
||||
const one = xgridsK1ObservationSources(
|
||||
declaredState([cameraRow("rig.front", "Передняя камера")]),
|
||||
model(),
|
||||
);
|
||||
const many = xgridsK1ObservationSources(declaredState(), model());
|
||||
|
||||
assert.equal(new Set(sources.map(({ id }) => id)).size, sources.length);
|
||||
assert.ok(sources.every(({ provider }) => provider.pluginId && provider.modelId));
|
||||
assert.ok(sources.every(({ provider }) => provider.pluginVersion));
|
||||
assert.ok(sources.every(({ binding }) => binding.deviceId === "device-k1-001"));
|
||||
assert.ok(sources.every(({ capabilities }) => capabilities.timelineMode === "live-only"));
|
||||
assert.deepEqual(zero.map(({ modality }) => modality), ["point-cloud"]);
|
||||
assert.deepEqual(one.map(({ sourceId }) => sourceId), ["sensor.lidar.primary", "rig.front"]);
|
||||
assert.deepEqual(many.map(({ sourceId }) => sourceId), [
|
||||
"sensor.lidar.primary",
|
||||
"sensor.camera.left",
|
||||
"sensor.camera.right",
|
||||
]);
|
||||
assert.equal(new Set(many.map(({ id }) => id)).size, many.length);
|
||||
assert.ok(many.every(({ provider }) => provider.pluginId && provider.modelId));
|
||||
assert.ok(many.every(({ binding }) => binding.deviceId === "device-k1-001"));
|
||||
});
|
||||
|
||||
test("observation source ids remain stable while runtime availability changes", () => {
|
||||
test("descriptor ids remain stable while point cloud and selected camera start streaming", () => {
|
||||
const declared = xgridsK1ObservationSources(declaredState(), model());
|
||||
const streaming = xgridsK1ObservationSources(streamingState(), model());
|
||||
|
||||
assert.deepEqual(
|
||||
streaming.map(({ id }) => id),
|
||||
declared.map(({ id }) => id),
|
||||
const streaming = xgridsK1ObservationSources(
|
||||
cameraStreamingState("sensor.camera.left"),
|
||||
model(),
|
||||
);
|
||||
|
||||
assert.deepEqual(streaming.map(({ id }) => id), declared.map(({ id }) => id));
|
||||
assert.equal(declared[0].availability, "available");
|
||||
assert.equal(streaming[0].availability, "streaming");
|
||||
assert.equal(streaming[0].previewUrl, "rerun+http://127.0.0.1:9877/proxy");
|
||||
});
|
||||
|
||||
test("camera descriptors stay declared and URL-free until a browser delivery adapter exists", () => {
|
||||
const sources = xgridsK1ObservationSources(streamingState(), model());
|
||||
test("only the authoritative selected camera receives browser delivery", () => {
|
||||
const sources = xgridsK1ObservationSources(
|
||||
cameraStreamingState("sensor.camera.left"),
|
||||
model(),
|
||||
);
|
||||
const cameras = sources.filter(({ modality }) => modality === "video");
|
||||
const [left, right] = cameras;
|
||||
|
||||
assert.equal(cameras.length, 2);
|
||||
for (const camera of cameras) {
|
||||
assert.equal(camera.availability, "declared");
|
||||
assert.equal(camera.transport, "rtsp");
|
||||
assert.equal(camera.previewUrl, null);
|
||||
assert.equal(camera.capabilities.spatialRegistration, "unresolved");
|
||||
assert.equal(camera.capabilities.resizable, true);
|
||||
assert.doesNotMatch(camera.endpointLabel ?? "", /:\/\//);
|
||||
}
|
||||
assert.equal(left.activation.selected, true);
|
||||
assert.equal(left.activation.maxActive, 1);
|
||||
assert.equal(left.availability, "streaming");
|
||||
assert.equal(left.delivery.kind, "mse-fmp4-websocket");
|
||||
assert.equal(left.transport, "websocket");
|
||||
assert.equal(right.activation.selected, false);
|
||||
assert.equal(right.availability, "available");
|
||||
assert.equal(right.delivery, null);
|
||||
assert.equal(left.activation.groupId, right.activation.groupId);
|
||||
assert.match(left.activation.groupId, /device-session-001/);
|
||||
});
|
||||
|
||||
test("switching left to right keeps ids stable and never exposes both deliveries", () => {
|
||||
const left = xgridsK1ObservationSources(
|
||||
cameraStreamingState("sensor.camera.left"),
|
||||
model(),
|
||||
).filter(({ modality }) => modality === "video");
|
||||
const right = xgridsK1ObservationSources(
|
||||
cameraStreamingState("sensor.camera.right"),
|
||||
model(),
|
||||
).filter(({ modality }) => modality === "video");
|
||||
|
||||
assert.deepEqual(right.map(({ id }) => id), left.map(({ id }) => id));
|
||||
assert.deepEqual(left.filter(({ delivery }) => delivery).map(({ sourceId }) => sourceId), [
|
||||
"sensor.camera.left",
|
||||
]);
|
||||
assert.deepEqual(right.filter(({ delivery }) => delivery).map(({ sourceId }) => sourceId), [
|
||||
"sensor.camera.right",
|
||||
]);
|
||||
});
|
||||
|
||||
test("camera descriptors never leak vendor RTSP endpoints or device IP addresses", () => {
|
||||
const sources = xgridsK1ObservationSources(
|
||||
cameraStreamingState("sensor.camera.left"),
|
||||
model(),
|
||||
);
|
||||
const cameras = sources.filter(({ modality }) => modality === "video");
|
||||
|
||||
assert.ok(cameras.every(({ previewUrl }) => previewUrl === null));
|
||||
assert.deepEqual(collectUrlLikeStrings(sources), [
|
||||
"rerun+http://127.0.0.1:9877/proxy",
|
||||
]);
|
||||
const serialized = JSON.stringify(sources);
|
||||
const serialized = JSON.stringify(cameras);
|
||||
assert.doesNotMatch(serialized, /rtsp:\/\//i);
|
||||
assert.doesNotMatch(serialized, /192\.168\.7\.10/);
|
||||
assert.doesNotMatch(serialized, /vendor-(?:preview|viewer)/);
|
||||
});
|
||||
|
||||
test("unattested camera catalog entries remain unverified", () => {
|
||||
const state = declaredState();
|
||||
test("unattested, duplicate and unsafe camera entries fail closed", () => {
|
||||
const duplicate = cameraRow("sensor.camera.left", "Duplicate");
|
||||
const state = cameraStreamingState("sensor.camera.left");
|
||||
state.compatibility.profile_id = null;
|
||||
state.device_session.compatibility_profile_id = null;
|
||||
state.sensor_catalog.streams = state.sensor_catalog.streams.map((stream) =>
|
||||
stream.stream_id === "camera.preview.live"
|
||||
? { ...stream, availability: "unverified", decode_status: "profile-not-attested" }
|
||||
: stream);
|
||||
state.sensor_catalog.streams.push(duplicate);
|
||||
state.camera_preview.delivery = {
|
||||
...state.camera_preview.delivery,
|
||||
url: "ws://192.168.7.10:9000/leak",
|
||||
};
|
||||
|
||||
const cameras = xgridsK1ObservationSources(state, model()).filter(
|
||||
({ modality }) => modality === "video",
|
||||
);
|
||||
assert.equal(cameras.length, 2);
|
||||
assert.ok(cameras.every(({ availability }) => availability === "unverified"));
|
||||
assert.ok(cameras.every(({ previewUrl }) => previewUrl === null));
|
||||
assert.deepEqual(cameras.map(({ sourceId }) => sourceId), ["sensor.camera.right"]);
|
||||
assert.equal(cameras[0].availability, "unverified");
|
||||
assert.equal(cameras[0].activation.controllable, false);
|
||||
assert.equal(cameras[0].delivery, null);
|
||||
});
|
||||
|
||||
test("camera delivery rejects literal and encoded endpoint or credential leaks", () => {
|
||||
const unsafeUrls = [
|
||||
"/api/preview?upstream=rtsp://camera.local/live",
|
||||
"/api/preview?upstream=rtsp%3A%2F%2Fcamera.local%2Flive",
|
||||
"/api/preview?upstream=rtsp%253A%252F%252Fcamera.local%252Flive",
|
||||
"/api/preview?endpoint=192.168.68.52:8554",
|
||||
"/api/preview?endpoint=192%2E168%2E68%2E52",
|
||||
"/api/preview?password=not-for-the-browser",
|
||||
"/api/preview?%70%61%73%73%77%6f%72%64=not-for-the-browser",
|
||||
"/api/preview/camera:secret@device",
|
||||
"/%2f%2fevil.example/preview",
|
||||
];
|
||||
|
||||
for (const url of unsafeUrls) {
|
||||
const state = cameraStreamingState("sensor.camera.left");
|
||||
state.camera_preview.delivery = { ...state.camera_preview.delivery, url };
|
||||
state.sensor_catalog.streams = state.sensor_catalog.streams.map((stream) =>
|
||||
stream.source_id === "sensor.camera.left"
|
||||
? { ...stream, delivery: { ...stream.delivery, url } }
|
||||
: stream,
|
||||
);
|
||||
const left = xgridsK1ObservationSources(state, model()).find(
|
||||
({ sourceId }) => sourceId === "sensor.camera.left",
|
||||
);
|
||||
assert.ok(left, `left camera descriptor missing for ${url}`);
|
||||
assert.equal(left.delivery, null, `unsafe delivery escaped for ${url}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("manual camera reconnect restores an exhausted lease retry budget", () => {
|
||||
let budget = resetCameraLeaseRetryBudget("delivery-7");
|
||||
for (const expectedDelay of [400, 1_000, 2_000]) {
|
||||
const retry = consumeCameraLeaseRetry(budget, "delivery-7");
|
||||
assert.equal(retry.delay, expectedDelay);
|
||||
budget = retry.budget;
|
||||
}
|
||||
assert.equal(consumeCameraLeaseRetry(budget, "delivery-7").delay, null);
|
||||
|
||||
budget = resetCameraLeaseRetryBudget("delivery-7");
|
||||
const retryAfterManualReset = consumeCameraLeaseRetry(budget, "delivery-7");
|
||||
assert.equal(retryAfterManualReset.delay, 400);
|
||||
assert.equal(retryAfterManualReset.budget.count, 1);
|
||||
});
|
||||
|
||||
test("layout policy evicts only exclusive camera peers", () => {
|
||||
const sources = xgridsK1ObservationSources(
|
||||
cameraStreamingState("sensor.camera.left"),
|
||||
model(),
|
||||
);
|
||||
const left = sources.find(({ sourceId }) => sourceId === "sensor.camera.left");
|
||||
const right = sources.find(({ sourceId }) => sourceId === "sensor.camera.right");
|
||||
const pointCloud = sources.find(({ modality }) => modality === "point-cloud");
|
||||
assert.ok(left && right && pointCloud);
|
||||
|
||||
const change = openObservationSource(
|
||||
[pointCloud.id, left.id],
|
||||
right.id,
|
||||
sources,
|
||||
);
|
||||
assert.deepEqual(change.visibleIds, [pointCloud.id, right.id]);
|
||||
assert.deepEqual(change.removedIds, [left.id]);
|
||||
});
|
||||
|
||||
test("selected camera without delivery is explicitly restartable", () => {
|
||||
const state = cameraStreamingState("sensor.camera.left");
|
||||
state.camera_preview = {
|
||||
...state.camera_preview,
|
||||
phase: "error",
|
||||
delivery: null,
|
||||
};
|
||||
state.sensor_catalog.streams = state.sensor_catalog.streams.map((stream) =>
|
||||
stream.source_id === "sensor.camera.left"
|
||||
? { ...stream, availability: "error", delivery: null }
|
||||
: stream,
|
||||
);
|
||||
const sources = xgridsK1ObservationSources(state, model());
|
||||
const left = sources.find(({ sourceId }) => sourceId === "sensor.camera.left");
|
||||
const right = sources.find(({ sourceId }) => sourceId === "sensor.camera.right");
|
||||
assert.ok(left && right);
|
||||
|
||||
assert.equal(left.activation.selected, true);
|
||||
assert.equal(left.delivery, null);
|
||||
assert.equal(left.availability, "error");
|
||||
assert.equal(shouldRestartObservationSource(left), true);
|
||||
assert.equal(shouldRestartObservationSource(right), false);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user