feat(perception): add bounded triton runtime diagnostics

This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 02:52:11 +03:00
parent d9ea7c1129
commit 34f13ba59e
13 changed files with 1023 additions and 76 deletions
+116
View File
@@ -98,6 +98,122 @@ def test_nearest_rank_tail_metrics_keep_outlier(pilot):
assert pilot("run_joint_pilot").distribution([1, 200])["p99"] == 200
def test_layer_refresh_cadence_is_source_clocked_and_fail_closed(pilot):
refresh = pilot("run_joint_pilot").layer_refresh_due
assert refresh(None, 1_000_000_000, 50)
assert not refresh(1_000_000_000, 1_049_999_999, 50)
assert refresh(1_000_000_000, 1_050_000_000, 50)
with pytest.raises(ValueError, match="backwards"):
refresh(1_000_000_000, 999_999_999, 50)
def test_native_ddrnet_preprocess_matches_pillow_nearest_exactly(pilot):
from PIL import Image
bgr = np.random.default_rng(41).integers(0, 256, (600, 800, 3), dtype=np.uint8)
rgb = Image.fromarray(bgr[:, :, ::-1]).crop((100, 0, 700, 600))
resized = rgb.resize((512, 512), Image.Resampling.NEAREST)
expected = (np.asarray(resized, dtype=np.float32) / 255.0).transpose(2, 0, 1)[None]
actual = pilot("pilot_ddrnet_numpy").preprocess_bgr(bgr)
np.testing.assert_array_equal(actual, expected)
assert actual.flags.c_contiguous and actual.dtype == np.dtype("<f4")
with pytest.raises(ValueError, match="contract"):
pilot("pilot_ddrnet_numpy").preprocess_bgr(bgr[:512])
def test_triton_statistics_exclude_warmup_and_reject_counter_reset(pilot):
names = ("success", "fail", "queue", "compute_input", "compute_infer", "compute_output")
before = {"model": {name: {"count": "8", "ns": "1000000"} for name in names}}
after = {"model": {name: {"count": "10", "ns": "5000000"} for name in names}}
delta = pilot("pilot_triton_stats").stats_delta(before, after)
assert delta["model"]["compute_infer"] == {"count": 2, "total_ms": 4, "mean_ms": 2}
with pytest.raises(RuntimeError, match="backwards"):
pilot("pilot_triton_stats").stats_delta(after, before)
def test_ddrnet_triton_transport_is_bounded_and_validates_output_identity(pilot):
module = pilot("pilot_ddrnet_triton")
mask = np.zeros((512, 512), np.uint8)
descriptor = {"outputs": [{
"name": "mask", "datatype": "UINT8", "shape": [1, 512, 512],
"parameters": {"binary_data_size": mask.nbytes},
}]}
header = json.dumps(descriptor).encode()
class Response:
status = 200
payload = header + mask.tobytes()
def read(self, limit):
assert limit == module.MAX_RESPONSE_BYTES + 1
return self.payload[:limit]
def getheader(self, _name):
return str(len(header))
response = Response()
backend = module.TritonDdrnetMaskHttpInferenceBackend()
backend.connection = SimpleNamespace(
request=lambda *args, **kwargs: None,
getresponse=lambda: response,
close=lambda: None,
)
tensor = np.zeros((1, 3, 512, 512), np.float32)
np.testing.assert_array_equal(backend.infer(tensor), mask)
response.payload = b"x" * (module.MAX_RESPONSE_BYTES + 1)
with pytest.raises(RuntimeError, match="budget"):
backend.infer(tensor)
response.payload = header.replace(b'"UINT8"', b'"FP32" ') + mask.tobytes()
with pytest.raises(RuntimeError, match="contract"):
backend.infer(tensor)
def test_nvml_sampler_reuses_one_session_without_spawning_processes(pilot):
class Function:
def __init__(self, callback):
self.callback = callback
self.calls = 0
def __call__(self, *args):
self.calls += 1
return self.callback(*args)
def set_handle(_index, pointer):
pointer._obj.value = 7
return 0
def set_memory(_handle, pointer):
pointer._obj.total = 24 * 1024 * 1024
pointer._obj.free = 20 * 1024 * 1024
pointer._obj.used = 3 * 1024 * 1024 + 1
return 0
def set_utilization(_handle, pointer):
pointer._obj.gpu = 42
pointer._obj.memory = 3
return 0
library = SimpleNamespace(
nvmlInit_v2=Function(lambda: 0),
nvmlShutdown=Function(lambda: 0),
nvmlDeviceGetHandleByIndex_v2=Function(set_handle),
nvmlDeviceGetMemoryInfo=Function(set_memory),
nvmlDeviceGetUtilizationRates=Function(set_utilization),
)
sampler = pilot("pilot_telemetry").NvmlSampler(library)
assert sampler.sample() == {"gpu_used_mib": 4, "gpu_utilization": 42}
assert sampler.sample() == {"gpu_used_mib": 4, "gpu_utilization": 42}
sampler.close()
sampler.close()
assert library.nvmlInit_v2.calls == 1
assert library.nvmlShutdown.calls == 1
assert library.nvmlDeviceGetMemoryInfo.calls == 2
assert library.nvmlDeviceGetUtilizationRates.calls == 2
def test_body_history_is_bounded_and_never_reads_future(pilot):
store = pilot("pilot_graph").CurrentStore(None)
for index in range(100):