fix(lab): keep full route result indexable
This commit is contained in:
@@ -129,6 +129,10 @@ export interface VegetationFullRouteReview {
|
|||||||
height: 600;
|
height: 600;
|
||||||
timelineStartSeconds: number;
|
timelineStartSeconds: number;
|
||||||
timelineEndSeconds: number;
|
timelineEndSeconds: number;
|
||||||
|
timelineArtifact: {
|
||||||
|
sha256: string;
|
||||||
|
byteLength: number;
|
||||||
|
};
|
||||||
frameSourceTimesNs: readonly number[];
|
frameSourceTimesNs: readonly number[];
|
||||||
decodeRepair: {
|
decodeRepair: {
|
||||||
repairedFrameCount: 1;
|
repairedFrameCount: 1;
|
||||||
@@ -742,22 +746,30 @@ function fullRouteReviewValue(value: unknown): VegetationFullRouteReview | null
|
|||||||
if (timelineEndSeconds <= timelineStartSeconds) {
|
if (timelineEndSeconds <= timelineStartSeconds) {
|
||||||
throw new VegetationShadowContractError("vegetation.route_full_review: timeline invalid.");
|
throw new VegetationShadowContractError("vegetation.route_full_review: timeline invalid.");
|
||||||
}
|
}
|
||||||
const frameSourceTimesNs = arrayValue(
|
const timeline = objectValue(row.timeline, "vegetation.route_full_review.timeline");
|
||||||
row.frame_source_times_ns,
|
exact(
|
||||||
"vegetation.route_full_review.frame_source_times_ns",
|
timeline.path,
|
||||||
).map((raw, index) => {
|
"video/frame-source-times-ns.bin",
|
||||||
const time = integerValue(raw, `vegetation.route_full_review.frame_source_times_ns[${index}]`);
|
"vegetation.route_full_review.timeline.path",
|
||||||
if (!Number.isSafeInteger(time)) {
|
);
|
||||||
throw new VegetationShadowContractError("vegetation.route_full_review: unsafe frame time.");
|
exact(
|
||||||
}
|
timeline.encoding,
|
||||||
return time;
|
"uint64-le-nanoseconds",
|
||||||
});
|
"vegetation.route_full_review.timeline.encoding",
|
||||||
if (
|
);
|
||||||
frameSourceTimesNs.length !== 6830
|
exact(timeline.frame_count, 6830, "vegetation.route_full_review.timeline.frame_count");
|
||||||
|| frameSourceTimesNs.some((time, index) => index > 0 && time <= frameSourceTimesNs[index - 1]!)
|
const timelineSha256 = textValue(
|
||||||
) {
|
timeline.sha256,
|
||||||
throw new VegetationShadowContractError("vegetation.route_full_review: frame timeline changed.");
|
"vegetation.route_full_review.timeline.sha256",
|
||||||
|
);
|
||||||
|
if (!SHA256.test(timelineSha256)) {
|
||||||
|
throw new VegetationShadowContractError("vegetation.route_full_review: timeline digest invalid.");
|
||||||
}
|
}
|
||||||
|
const timelineByteLength = integerValue(
|
||||||
|
timeline.byte_length,
|
||||||
|
"vegetation.route_full_review.timeline.byte_length",
|
||||||
|
);
|
||||||
|
exact(timelineByteLength, 6830 * 8, "vegetation.route_full_review.timeline.byte_length");
|
||||||
const decodeRepair = objectValue(
|
const decodeRepair = objectValue(
|
||||||
row.decode_repair,
|
row.decode_repair,
|
||||||
"vegetation.route_full_review.decode_repair",
|
"vegetation.route_full_review.decode_repair",
|
||||||
@@ -808,7 +820,8 @@ function fullRouteReviewValue(value: unknown): VegetationFullRouteReview | null
|
|||||||
height: 600,
|
height: 600,
|
||||||
timelineStartSeconds,
|
timelineStartSeconds,
|
||||||
timelineEndSeconds,
|
timelineEndSeconds,
|
||||||
frameSourceTimesNs,
|
timelineArtifact: { sha256: timelineSha256, byteLength: timelineByteLength },
|
||||||
|
frameSourceTimesNs: [],
|
||||||
decodeRepair: {
|
decodeRepair: {
|
||||||
repairedFrameCount: 1,
|
repairedFrameCount: 1,
|
||||||
sequence: 6092,
|
sequence: 6092,
|
||||||
@@ -935,11 +948,49 @@ export async function fetchVegetationShadowResult(
|
|||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new VegetationShadowContractError(`Vegetation LAB недоступна: HTTP ${response.status}.`);
|
throw new VegetationShadowContractError(`Vegetation LAB недоступна: HTTP ${response.status}.`);
|
||||||
}
|
}
|
||||||
return parseResult(
|
const result = parseResult(
|
||||||
await response.json(),
|
await response.json(),
|
||||||
resultId,
|
resultId,
|
||||||
"/api/v1/laboratory/vegetation-shadow",
|
"/api/v1/laboratory/vegetation-shadow",
|
||||||
);
|
);
|
||||||
|
if (!result.routeFullReview) return result;
|
||||||
|
const timelineResponse = await fetcher(
|
||||||
|
`/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}/route-timeline`,
|
||||||
|
{ method: "GET", headers: { Accept: "application/octet-stream" }, signal },
|
||||||
|
);
|
||||||
|
if (!timelineResponse.ok) {
|
||||||
|
throw new VegetationShadowContractError(
|
||||||
|
`Vegetation LAB timeline недоступна: HTTP ${timelineResponse.status}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
timelineResponse.headers.get("etag")
|
||||||
|
!== `"${result.routeFullReview.timelineArtifact.sha256}"`
|
||||||
|
) {
|
||||||
|
throw new VegetationShadowContractError("Vegetation LAB timeline digest изменён.");
|
||||||
|
}
|
||||||
|
const timelinePayload = await timelineResponse.arrayBuffer();
|
||||||
|
if (timelinePayload.byteLength !== result.routeFullReview.timelineArtifact.byteLength) {
|
||||||
|
throw new VegetationShadowContractError("Vegetation LAB timeline size изменён.");
|
||||||
|
}
|
||||||
|
const timelineView = new DataView(timelinePayload);
|
||||||
|
const frameSourceTimesNs = Array.from({ length: result.routeFullReview.frameCount }, (_, index) => {
|
||||||
|
const value = Number(timelineView.getBigUint64(index * 8, true));
|
||||||
|
if (!Number.isSafeInteger(value)) {
|
||||||
|
throw new VegetationShadowContractError("Vegetation LAB timeline содержит unsafe time.");
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
});
|
||||||
|
if (
|
||||||
|
frameSourceTimesNs[0] !== Math.round(result.routeFullReview.timelineStartSeconds * 1_000_000_000)
|
||||||
|
|| frameSourceTimesNs.some((time, index) => index > 0 && time <= frameSourceTimesNs[index - 1]!)
|
||||||
|
) {
|
||||||
|
throw new VegetationShadowContractError("Vegetation LAB timeline нарушена.");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...result,
|
||||||
|
routeFullReview: { ...result.routeFullReview, frameSourceTimesNs },
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchVegetationBenchmarkResult(
|
export async function fetchVegetationBenchmarkResult(
|
||||||
|
|||||||
@@ -195,7 +195,13 @@ function fullRouteReview() {
|
|||||||
height: 600,
|
height: 600,
|
||||||
timeline_start_seconds: 39.215263458,
|
timeline_start_seconds: 39.215263458,
|
||||||
timeline_end_seconds: 757.260263458,
|
timeline_end_seconds: 757.260263458,
|
||||||
frame_source_times_ns: Array.from({ length: 6830 }, (_, index) => 39_215_263_458 + index * 100_000_000),
|
timeline: {
|
||||||
|
path: "video/frame-source-times-ns.bin",
|
||||||
|
sha256: "5".repeat(64),
|
||||||
|
byte_length: 6830 * 8,
|
||||||
|
encoding: "uint64-le-nanoseconds",
|
||||||
|
frame_count: 6830,
|
||||||
|
},
|
||||||
ground_truth: false,
|
ground_truth: false,
|
||||||
decode_repair: {
|
decode_repair: {
|
||||||
repaired_frame_count: 1,
|
repaired_frame_count: 1,
|
||||||
@@ -301,17 +307,35 @@ test("vegetation LAB parses the full 004 pass inside the existing result contrac
|
|||||||
catalogs: { goose: [], ravnoves: [] },
|
catalogs: { goose: [], ravnoves: [] },
|
||||||
route_full_review: fullRouteReview(),
|
route_full_review: fullRouteReview(),
|
||||||
};
|
};
|
||||||
|
const timeline = new ArrayBuffer(6830 * 8);
|
||||||
|
const timelineView = new DataView(timeline);
|
||||||
|
for (let index = 0; index < 6830; index += 1) {
|
||||||
|
timelineView.setBigUint64(
|
||||||
|
index * 8,
|
||||||
|
BigInt(39_215_263_458 + index * 100_000_000),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
const result = await fetchVegetationShadowResult(resultId, {
|
const result = await fetchVegetationShadowResult(resultId, {
|
||||||
fetcher: async () => new Response(JSON.stringify(payload), {
|
fetcher: async (url) => String(url).endsWith("/route-timeline")
|
||||||
status: 200,
|
? new Response(timeline, {
|
||||||
headers: { "Content-Type": "application/json" },
|
status: 200,
|
||||||
}),
|
headers: {
|
||||||
|
"Content-Type": "application/octet-stream",
|
||||||
|
ETag: `"${"5".repeat(64)}"`,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
: new Response(JSON.stringify(payload), {
|
||||||
|
status: 200,
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
assert.equal(result.routeVideo, null);
|
assert.equal(result.routeVideo, null);
|
||||||
assert.equal(result.routeFullReview.frameCount, 6830);
|
assert.equal(result.routeFullReview.frameCount, 6830);
|
||||||
assert.equal(result.routeFullReview.city.taxonomy.length, 16);
|
assert.equal(result.routeFullReview.city.taxonomy.length, 16);
|
||||||
assert.equal(result.routeFullReview.vegetation.taxonomy.length, 64);
|
assert.equal(result.routeFullReview.vegetation.taxonomy.length, 64);
|
||||||
assert.equal(result.routeFullReview.decodeRepair.sequence, 6092);
|
assert.equal(result.routeFullReview.decodeRepair.sequence, 6092);
|
||||||
|
assert.equal(result.routeFullReview.frameSourceTimesNs.length, 6830);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
vegetationFullRouteMaskUrl(resultId, "vegetation", 6829),
|
vegetationFullRouteMaskUrl(resultId, "vegetation", 6829),
|
||||||
`/api/v1/laboratory/vegetation-shadow/${resultId}/route-masks/vegetation/6829`,
|
`/api/v1/laboratory/vegetation-shadow/${resultId}/route-masks/vegetation/6829`,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import struct
|
||||||
import tarfile
|
import tarfile
|
||||||
import tempfile
|
import tempfile
|
||||||
import zipfile
|
import zipfile
|
||||||
@@ -705,6 +706,18 @@ def seal_mixed_route_full_video_review(
|
|||||||
temporary / "video" / "ddrnet-semantic-masks.zip",
|
temporary / "video" / "ddrnet-semantic-masks.zip",
|
||||||
FULL_ROUTE_FRAME_COUNT,
|
FULL_ROUTE_FRAME_COUNT,
|
||||||
)
|
)
|
||||||
|
timeline_destination = temporary / "video" / "frame-source-times-ns.bin"
|
||||||
|
timeline_destination.write_bytes(
|
||||||
|
struct.pack(f"<{FULL_ROUTE_FRAME_COUNT}Q", *frame_times_ns)
|
||||||
|
)
|
||||||
|
timeline_descriptor = {
|
||||||
|
"role": "full-route-frame-timeline",
|
||||||
|
"path": "video/frame-source-times-ns.bin",
|
||||||
|
"byte_length": timeline_destination.stat().st_size,
|
||||||
|
"sha256": sha256_path(timeline_destination),
|
||||||
|
"media_type": "application/octet-stream",
|
||||||
|
}
|
||||||
|
artifacts.append(timeline_descriptor)
|
||||||
proof_descriptors: dict[str, dict[str, object]] = {}
|
proof_descriptors: dict[str, dict[str, object]] = {}
|
||||||
for key, path in (
|
for key, path in (
|
||||||
("base", base_root / "result.json"),
|
("base", base_root / "result.json"),
|
||||||
@@ -739,7 +752,13 @@ def seal_mixed_route_full_video_review(
|
|||||||
"height": 600,
|
"height": 600,
|
||||||
"timeline_start_seconds": job.timeline_start_seconds,
|
"timeline_start_seconds": job.timeline_start_seconds,
|
||||||
"timeline_end_seconds": job.timeline_end_seconds,
|
"timeline_end_seconds": job.timeline_end_seconds,
|
||||||
"frame_source_times_ns": frame_times_ns,
|
"timeline": {
|
||||||
|
"path": timeline_descriptor["path"],
|
||||||
|
"sha256": timeline_descriptor["sha256"],
|
||||||
|
"byte_length": timeline_descriptor["byte_length"],
|
||||||
|
"encoding": "uint64-le-nanoseconds",
|
||||||
|
"frame_count": FULL_ROUTE_FRAME_COUNT,
|
||||||
|
},
|
||||||
"ground_truth": False,
|
"ground_truth": False,
|
||||||
"decode_repair": {
|
"decode_repair": {
|
||||||
"repaired_frame_count": 1,
|
"repaired_frame_count": 1,
|
||||||
|
|||||||
@@ -214,6 +214,54 @@ def _build_vegetation_lab_router(
|
|||||||
raise HTTPException(status_code=404, detail="Full-route semantic mask not found")
|
raise HTTPException(status_code=404, detail="Full-route semantic mask not found")
|
||||||
return _zip_mask_response(candidate.joinpath(*relative.parts), sequence)
|
return _zip_mask_response(candidate.joinpath(*relative.parts), sequence)
|
||||||
|
|
||||||
|
@router.get("/{result_id}/route-timeline")
|
||||||
|
def get_full_route_timeline(result_id: str) -> FileResponse:
|
||||||
|
candidate = _resolve_candidate(root_provider, definition, result_id)
|
||||||
|
manifest = _read_verified(candidate, definition)
|
||||||
|
route = manifest.get("route_full_review")
|
||||||
|
timeline = route.get("timeline") if isinstance(route, dict) else None
|
||||||
|
relative_text = timeline.get("path") if isinstance(timeline, dict) else None
|
||||||
|
frame_count = timeline.get("frame_count") if isinstance(timeline, dict) else None
|
||||||
|
byte_length = timeline.get("byte_length") if isinstance(timeline, dict) else None
|
||||||
|
sha256 = timeline.get("sha256") if isinstance(timeline, dict) else None
|
||||||
|
if (
|
||||||
|
not isinstance(relative_text, str)
|
||||||
|
or frame_count != route.get("frame_count")
|
||||||
|
or byte_length != frame_count * 8
|
||||||
|
or not isinstance(sha256, str)
|
||||||
|
or len(sha256) != 64
|
||||||
|
):
|
||||||
|
raise HTTPException(status_code=404, detail="Full-route timeline not found")
|
||||||
|
relative = PurePosixPath(relative_text)
|
||||||
|
artifacts = manifest.get("artifacts")
|
||||||
|
if (
|
||||||
|
relative.is_absolute()
|
||||||
|
or str(relative) != relative_text
|
||||||
|
or any(part in {"", ".", ".."} for part in relative.parts)
|
||||||
|
or not isinstance(artifacts, list)
|
||||||
|
or not any(
|
||||||
|
isinstance(item, dict)
|
||||||
|
and item.get("path") == relative_text
|
||||||
|
and item.get("byte_length") == byte_length
|
||||||
|
and item.get("sha256") == sha256
|
||||||
|
and item.get("media_type") == "application/octet-stream"
|
||||||
|
for item in artifacts
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise HTTPException(status_code=404, detail="Full-route timeline not found")
|
||||||
|
path = candidate.joinpath(*relative.parts)
|
||||||
|
if not path.is_file() or path.is_symlink() or path.stat().st_size != byte_length:
|
||||||
|
raise HTTPException(status_code=404, detail="Full-route timeline not found")
|
||||||
|
return FileResponse(
|
||||||
|
path,
|
||||||
|
media_type="application/octet-stream",
|
||||||
|
headers={
|
||||||
|
"Cache-Control": "private, max-age=31536000, immutable",
|
||||||
|
"ETag": f'"{sha256}"',
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
return router
|
return router
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import hashlib
|
|||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import shutil
|
import shutil
|
||||||
|
import struct
|
||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
@@ -271,9 +272,17 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
|
|||||||
assert mask.headers["cache-control"].endswith("immutable")
|
assert mask.headers["cache-control"].endswith("immutable")
|
||||||
|
|
||||||
full_archive_payloads = (b"\x89PNG\r\n\x1a\ncity", b"\x89PNG\r\n\x1a\nvegetation")
|
full_archive_payloads = (b"\x89PNG\r\n\x1a\ncity", b"\x89PNG\r\n\x1a\nvegetation")
|
||||||
|
full_timeline_payload = struct.pack("<2Q", 1_000_000_000, 1_100_000_000)
|
||||||
full_identity = dict(manifest["identity"])
|
full_identity = dict(manifest["identity"])
|
||||||
full_route = {
|
full_route = {
|
||||||
"frame_count": 2,
|
"frame_count": 2,
|
||||||
|
"timeline": {
|
||||||
|
"path": "video/frame-source-times-ns.bin",
|
||||||
|
"sha256": hashlib.sha256(full_timeline_payload).hexdigest(),
|
||||||
|
"byte_length": len(full_timeline_payload),
|
||||||
|
"encoding": "uint64-le-nanoseconds",
|
||||||
|
"frame_count": 2,
|
||||||
|
},
|
||||||
"layers": {
|
"layers": {
|
||||||
layer: {"mask_archive": {"path": "video/full-route-masks.zip"}}
|
layer: {"mask_archive": {"path": "video/full-route-masks.zip"}}
|
||||||
for layer in ("city", "vegetation")
|
for layer in ("city", "vegetation")
|
||||||
@@ -296,6 +305,8 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
|
|||||||
with zipfile.ZipFile(full_archive, "x", compression=zipfile.ZIP_STORED) as frozen:
|
with zipfile.ZipFile(full_archive, "x", compression=zipfile.ZIP_STORED) as frozen:
|
||||||
for sequence, payload in enumerate(full_archive_payloads, start=1):
|
for sequence, payload in enumerate(full_archive_payloads, start=1):
|
||||||
frozen.writestr(f"masks/frame-{sequence:06d}.png", payload)
|
frozen.writestr(f"masks/frame-{sequence:06d}.png", payload)
|
||||||
|
full_timeline = full_root / "video" / "frame-source-times-ns.bin"
|
||||||
|
full_timeline.write_bytes(full_timeline_payload)
|
||||||
full_manifest = dict(manifest)
|
full_manifest = dict(manifest)
|
||||||
full_manifest["result_id"] = full_result_id
|
full_manifest["result_id"] = full_result_id
|
||||||
full_manifest["identity"] = full_identity
|
full_manifest["identity"] = full_identity
|
||||||
@@ -310,6 +321,13 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
|
|||||||
"sha256": _sha256(full_archive),
|
"sha256": _sha256(full_archive),
|
||||||
"media_type": "application/zip",
|
"media_type": "application/zip",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"role": "full-route-frame-timeline",
|
||||||
|
"path": "video/frame-source-times-ns.bin",
|
||||||
|
"byte_length": full_timeline.stat().st_size,
|
||||||
|
"sha256": _sha256(full_timeline),
|
||||||
|
"media_type": "application/octet-stream",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
(full_root / "result.json").write_text(
|
(full_root / "result.json").write_text(
|
||||||
json.dumps(full_manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
json.dumps(full_manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
@@ -330,6 +348,12 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
|
|||||||
assert client.get(
|
assert client.get(
|
||||||
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/city/2"
|
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/city/2"
|
||||||
).status_code == 404
|
).status_code == 404
|
||||||
|
timeline = client.get(
|
||||||
|
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-timeline"
|
||||||
|
)
|
||||||
|
assert timeline.status_code == 200
|
||||||
|
assert timeline.content == full_timeline_payload
|
||||||
|
assert timeline.headers["cache-control"].endswith("immutable")
|
||||||
|
|
||||||
(result_root / asset_path).write_bytes(b"tampered")
|
(result_root / asset_path).write_bytes(b"tampered")
|
||||||
assert (
|
assert (
|
||||||
|
|||||||
Reference in New Issue
Block a user