fix(lab): keep full route result indexable

This commit is contained in:
DCCONSTRUCTIONS
2026-08-29 13:54:10 +03:00
parent 45019f8952
commit cd16ea28a6
5 changed files with 189 additions and 23 deletions
@@ -129,6 +129,10 @@ export interface VegetationFullRouteReview {
height: 600; height: 600;
timelineStartSeconds: number; timelineStartSeconds: number;
timelineEndSeconds: number; timelineEndSeconds: number;
timelineArtifact: {
sha256: string;
byteLength: number;
};
frameSourceTimesNs: readonly number[]; frameSourceTimesNs: readonly number[];
decodeRepair: { decodeRepair: {
repairedFrameCount: 1; repairedFrameCount: 1;
@@ -742,22 +746,30 @@ function fullRouteReviewValue(value: unknown): VegetationFullRouteReview | null
if (timelineEndSeconds <= timelineStartSeconds) { if (timelineEndSeconds <= timelineStartSeconds) {
throw new VegetationShadowContractError("vegetation.route_full_review: timeline invalid."); throw new VegetationShadowContractError("vegetation.route_full_review: timeline invalid.");
} }
const frameSourceTimesNs = arrayValue( const timeline = objectValue(row.timeline, "vegetation.route_full_review.timeline");
row.frame_source_times_ns, exact(
"vegetation.route_full_review.frame_source_times_ns", timeline.path,
).map((raw, index) => { "video/frame-source-times-ns.bin",
const time = integerValue(raw, `vegetation.route_full_review.frame_source_times_ns[${index}]`); "vegetation.route_full_review.timeline.path",
if (!Number.isSafeInteger(time)) { );
throw new VegetationShadowContractError("vegetation.route_full_review: unsafe frame time."); exact(
} timeline.encoding,
return time; "uint64-le-nanoseconds",
}); "vegetation.route_full_review.timeline.encoding",
if ( );
frameSourceTimesNs.length !== 6830 exact(timeline.frame_count, 6830, "vegetation.route_full_review.timeline.frame_count");
|| frameSourceTimesNs.some((time, index) => index > 0 && time <= frameSourceTimesNs[index - 1]!) const timelineSha256 = textValue(
) { timeline.sha256,
throw new VegetationShadowContractError("vegetation.route_full_review: frame timeline changed."); "vegetation.route_full_review.timeline.sha256",
);
if (!SHA256.test(timelineSha256)) {
throw new VegetationShadowContractError("vegetation.route_full_review: timeline digest invalid.");
} }
const timelineByteLength = integerValue(
timeline.byte_length,
"vegetation.route_full_review.timeline.byte_length",
);
exact(timelineByteLength, 6830 * 8, "vegetation.route_full_review.timeline.byte_length");
const decodeRepair = objectValue( const decodeRepair = objectValue(
row.decode_repair, row.decode_repair,
"vegetation.route_full_review.decode_repair", "vegetation.route_full_review.decode_repair",
@@ -808,7 +820,8 @@ function fullRouteReviewValue(value: unknown): VegetationFullRouteReview | null
height: 600, height: 600,
timelineStartSeconds, timelineStartSeconds,
timelineEndSeconds, timelineEndSeconds,
frameSourceTimesNs, timelineArtifact: { sha256: timelineSha256, byteLength: timelineByteLength },
frameSourceTimesNs: [],
decodeRepair: { decodeRepair: {
repairedFrameCount: 1, repairedFrameCount: 1,
sequence: 6092, sequence: 6092,
@@ -935,11 +948,49 @@ export async function fetchVegetationShadowResult(
if (!response.ok) { if (!response.ok) {
throw new VegetationShadowContractError(`Vegetation LAB недоступна: HTTP ${response.status}.`); throw new VegetationShadowContractError(`Vegetation LAB недоступна: HTTP ${response.status}.`);
} }
return parseResult( const result = parseResult(
await response.json(), await response.json(),
resultId, resultId,
"/api/v1/laboratory/vegetation-shadow", "/api/v1/laboratory/vegetation-shadow",
); );
if (!result.routeFullReview) return result;
const timelineResponse = await fetcher(
`/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}/route-timeline`,
{ method: "GET", headers: { Accept: "application/octet-stream" }, signal },
);
if (!timelineResponse.ok) {
throw new VegetationShadowContractError(
`Vegetation LAB timeline недоступна: HTTP ${timelineResponse.status}.`,
);
}
if (
timelineResponse.headers.get("etag")
!== `"${result.routeFullReview.timelineArtifact.sha256}"`
) {
throw new VegetationShadowContractError("Vegetation LAB timeline digest изменён.");
}
const timelinePayload = await timelineResponse.arrayBuffer();
if (timelinePayload.byteLength !== result.routeFullReview.timelineArtifact.byteLength) {
throw new VegetationShadowContractError("Vegetation LAB timeline size изменён.");
}
const timelineView = new DataView(timelinePayload);
const frameSourceTimesNs = Array.from({ length: result.routeFullReview.frameCount }, (_, index) => {
const value = Number(timelineView.getBigUint64(index * 8, true));
if (!Number.isSafeInteger(value)) {
throw new VegetationShadowContractError("Vegetation LAB timeline содержит unsafe time.");
}
return value;
});
if (
frameSourceTimesNs[0] !== Math.round(result.routeFullReview.timelineStartSeconds * 1_000_000_000)
|| frameSourceTimesNs.some((time, index) => index > 0 && time <= frameSourceTimesNs[index - 1]!)
) {
throw new VegetationShadowContractError("Vegetation LAB timeline нарушена.");
}
return {
...result,
routeFullReview: { ...result.routeFullReview, frameSourceTimesNs },
};
} }
export async function fetchVegetationBenchmarkResult( export async function fetchVegetationBenchmarkResult(
@@ -195,7 +195,13 @@ function fullRouteReview() {
height: 600, height: 600,
timeline_start_seconds: 39.215263458, timeline_start_seconds: 39.215263458,
timeline_end_seconds: 757.260263458, timeline_end_seconds: 757.260263458,
frame_source_times_ns: Array.from({ length: 6830 }, (_, index) => 39_215_263_458 + index * 100_000_000), timeline: {
path: "video/frame-source-times-ns.bin",
sha256: "5".repeat(64),
byte_length: 6830 * 8,
encoding: "uint64-le-nanoseconds",
frame_count: 6830,
},
ground_truth: false, ground_truth: false,
decode_repair: { decode_repair: {
repaired_frame_count: 1, repaired_frame_count: 1,
@@ -301,8 +307,25 @@ test("vegetation LAB parses the full 004 pass inside the existing result contrac
catalogs: { goose: [], ravnoves: [] }, catalogs: { goose: [], ravnoves: [] },
route_full_review: fullRouteReview(), route_full_review: fullRouteReview(),
}; };
const timeline = new ArrayBuffer(6830 * 8);
const timelineView = new DataView(timeline);
for (let index = 0; index < 6830; index += 1) {
timelineView.setBigUint64(
index * 8,
BigInt(39_215_263_458 + index * 100_000_000),
true,
);
}
const result = await fetchVegetationShadowResult(resultId, { const result = await fetchVegetationShadowResult(resultId, {
fetcher: async () => new Response(JSON.stringify(payload), { fetcher: async (url) => String(url).endsWith("/route-timeline")
? new Response(timeline, {
status: 200,
headers: {
"Content-Type": "application/octet-stream",
ETag: `"${"5".repeat(64)}"`,
},
})
: new Response(JSON.stringify(payload), {
status: 200, status: 200,
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
}), }),
@@ -312,6 +335,7 @@ test("vegetation LAB parses the full 004 pass inside the existing result contrac
assert.equal(result.routeFullReview.city.taxonomy.length, 16); assert.equal(result.routeFullReview.city.taxonomy.length, 16);
assert.equal(result.routeFullReview.vegetation.taxonomy.length, 64); assert.equal(result.routeFullReview.vegetation.taxonomy.length, 64);
assert.equal(result.routeFullReview.decodeRepair.sequence, 6092); assert.equal(result.routeFullReview.decodeRepair.sequence, 6092);
assert.equal(result.routeFullReview.frameSourceTimesNs.length, 6830);
assert.equal( assert.equal(
vegetationFullRouteMaskUrl(resultId, "vegetation", 6829), vegetationFullRouteMaskUrl(resultId, "vegetation", 6829),
`/api/v1/laboratory/vegetation-shadow/${resultId}/route-masks/vegetation/6829`, `/api/v1/laboratory/vegetation-shadow/${resultId}/route-masks/vegetation/6829`,
@@ -7,6 +7,7 @@ import hashlib
import json import json
import os import os
import shutil import shutil
import struct
import tarfile import tarfile
import tempfile import tempfile
import zipfile import zipfile
@@ -705,6 +706,18 @@ def seal_mixed_route_full_video_review(
temporary / "video" / "ddrnet-semantic-masks.zip", temporary / "video" / "ddrnet-semantic-masks.zip",
FULL_ROUTE_FRAME_COUNT, FULL_ROUTE_FRAME_COUNT,
) )
timeline_destination = temporary / "video" / "frame-source-times-ns.bin"
timeline_destination.write_bytes(
struct.pack(f"<{FULL_ROUTE_FRAME_COUNT}Q", *frame_times_ns)
)
timeline_descriptor = {
"role": "full-route-frame-timeline",
"path": "video/frame-source-times-ns.bin",
"byte_length": timeline_destination.stat().st_size,
"sha256": sha256_path(timeline_destination),
"media_type": "application/octet-stream",
}
artifacts.append(timeline_descriptor)
proof_descriptors: dict[str, dict[str, object]] = {} proof_descriptors: dict[str, dict[str, object]] = {}
for key, path in ( for key, path in (
("base", base_root / "result.json"), ("base", base_root / "result.json"),
@@ -739,7 +752,13 @@ def seal_mixed_route_full_video_review(
"height": 600, "height": 600,
"timeline_start_seconds": job.timeline_start_seconds, "timeline_start_seconds": job.timeline_start_seconds,
"timeline_end_seconds": job.timeline_end_seconds, "timeline_end_seconds": job.timeline_end_seconds,
"frame_source_times_ns": frame_times_ns, "timeline": {
"path": timeline_descriptor["path"],
"sha256": timeline_descriptor["sha256"],
"byte_length": timeline_descriptor["byte_length"],
"encoding": "uint64-le-nanoseconds",
"frame_count": FULL_ROUTE_FRAME_COUNT,
},
"ground_truth": False, "ground_truth": False,
"decode_repair": { "decode_repair": {
"repaired_frame_count": 1, "repaired_frame_count": 1,
@@ -214,6 +214,54 @@ def _build_vegetation_lab_router(
raise HTTPException(status_code=404, detail="Full-route semantic mask not found") raise HTTPException(status_code=404, detail="Full-route semantic mask not found")
return _zip_mask_response(candidate.joinpath(*relative.parts), sequence) return _zip_mask_response(candidate.joinpath(*relative.parts), sequence)
@router.get("/{result_id}/route-timeline")
def get_full_route_timeline(result_id: str) -> FileResponse:
candidate = _resolve_candidate(root_provider, definition, result_id)
manifest = _read_verified(candidate, definition)
route = manifest.get("route_full_review")
timeline = route.get("timeline") if isinstance(route, dict) else None
relative_text = timeline.get("path") if isinstance(timeline, dict) else None
frame_count = timeline.get("frame_count") if isinstance(timeline, dict) else None
byte_length = timeline.get("byte_length") if isinstance(timeline, dict) else None
sha256 = timeline.get("sha256") if isinstance(timeline, dict) else None
if (
not isinstance(relative_text, str)
or frame_count != route.get("frame_count")
or byte_length != frame_count * 8
or not isinstance(sha256, str)
or len(sha256) != 64
):
raise HTTPException(status_code=404, detail="Full-route timeline not found")
relative = PurePosixPath(relative_text)
artifacts = manifest.get("artifacts")
if (
relative.is_absolute()
or str(relative) != relative_text
or any(part in {"", ".", ".."} for part in relative.parts)
or not isinstance(artifacts, list)
or not any(
isinstance(item, dict)
and item.get("path") == relative_text
and item.get("byte_length") == byte_length
and item.get("sha256") == sha256
and item.get("media_type") == "application/octet-stream"
for item in artifacts
)
):
raise HTTPException(status_code=404, detail="Full-route timeline not found")
path = candidate.joinpath(*relative.parts)
if not path.is_file() or path.is_symlink() or path.stat().st_size != byte_length:
raise HTTPException(status_code=404, detail="Full-route timeline not found")
return FileResponse(
path,
media_type="application/octet-stream",
headers={
"Cache-Control": "private, max-age=31536000, immutable",
"ETag": f'"{sha256}"',
"X-Content-Type-Options": "nosniff",
},
)
return router return router
+24
View File
@@ -4,6 +4,7 @@ import hashlib
import io import io
import json import json
import shutil import shutil
import struct
import zipfile import zipfile
from pathlib import Path from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
@@ -271,9 +272,17 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
assert mask.headers["cache-control"].endswith("immutable") assert mask.headers["cache-control"].endswith("immutable")
full_archive_payloads = (b"\x89PNG\r\n\x1a\ncity", b"\x89PNG\r\n\x1a\nvegetation") full_archive_payloads = (b"\x89PNG\r\n\x1a\ncity", b"\x89PNG\r\n\x1a\nvegetation")
full_timeline_payload = struct.pack("<2Q", 1_000_000_000, 1_100_000_000)
full_identity = dict(manifest["identity"]) full_identity = dict(manifest["identity"])
full_route = { full_route = {
"frame_count": 2, "frame_count": 2,
"timeline": {
"path": "video/frame-source-times-ns.bin",
"sha256": hashlib.sha256(full_timeline_payload).hexdigest(),
"byte_length": len(full_timeline_payload),
"encoding": "uint64-le-nanoseconds",
"frame_count": 2,
},
"layers": { "layers": {
layer: {"mask_archive": {"path": "video/full-route-masks.zip"}} layer: {"mask_archive": {"path": "video/full-route-masks.zip"}}
for layer in ("city", "vegetation") for layer in ("city", "vegetation")
@@ -296,6 +305,8 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
with zipfile.ZipFile(full_archive, "x", compression=zipfile.ZIP_STORED) as frozen: with zipfile.ZipFile(full_archive, "x", compression=zipfile.ZIP_STORED) as frozen:
for sequence, payload in enumerate(full_archive_payloads, start=1): for sequence, payload in enumerate(full_archive_payloads, start=1):
frozen.writestr(f"masks/frame-{sequence:06d}.png", payload) frozen.writestr(f"masks/frame-{sequence:06d}.png", payload)
full_timeline = full_root / "video" / "frame-source-times-ns.bin"
full_timeline.write_bytes(full_timeline_payload)
full_manifest = dict(manifest) full_manifest = dict(manifest)
full_manifest["result_id"] = full_result_id full_manifest["result_id"] = full_result_id
full_manifest["identity"] = full_identity full_manifest["identity"] = full_identity
@@ -310,6 +321,13 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
"sha256": _sha256(full_archive), "sha256": _sha256(full_archive),
"media_type": "application/zip", "media_type": "application/zip",
}, },
{
"role": "full-route-frame-timeline",
"path": "video/frame-source-times-ns.bin",
"byte_length": full_timeline.stat().st_size,
"sha256": _sha256(full_timeline),
"media_type": "application/octet-stream",
},
] ]
(full_root / "result.json").write_text( (full_root / "result.json").write_text(
json.dumps(full_manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")) json.dumps(full_manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
@@ -330,6 +348,12 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
assert client.get( assert client.get(
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/city/2" f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/city/2"
).status_code == 404 ).status_code == 404
timeline = client.get(
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-timeline"
)
assert timeline.status_code == 200
assert timeline.content == full_timeline_payload
assert timeline.headers["cache-control"].endswith("immutable")
(result_root / asset_path).write_bytes(b"tampered") (result_root / asset_path).write_bytes(b"tampered")
assert ( assert (