feat(perception): add bounded triton runtime diagnostics
This commit is contained in:
@@ -98,6 +98,122 @@ def test_nearest_rank_tail_metrics_keep_outlier(pilot):
|
||||
assert pilot("run_joint_pilot").distribution([1, 200])["p99"] == 200
|
||||
|
||||
|
||||
def test_layer_refresh_cadence_is_source_clocked_and_fail_closed(pilot):
|
||||
refresh = pilot("run_joint_pilot").layer_refresh_due
|
||||
|
||||
assert refresh(None, 1_000_000_000, 50)
|
||||
assert not refresh(1_000_000_000, 1_049_999_999, 50)
|
||||
assert refresh(1_000_000_000, 1_050_000_000, 50)
|
||||
with pytest.raises(ValueError, match="backwards"):
|
||||
refresh(1_000_000_000, 999_999_999, 50)
|
||||
|
||||
|
||||
def test_native_ddrnet_preprocess_matches_pillow_nearest_exactly(pilot):
|
||||
from PIL import Image
|
||||
|
||||
bgr = np.random.default_rng(41).integers(0, 256, (600, 800, 3), dtype=np.uint8)
|
||||
rgb = Image.fromarray(bgr[:, :, ::-1]).crop((100, 0, 700, 600))
|
||||
resized = rgb.resize((512, 512), Image.Resampling.NEAREST)
|
||||
expected = (np.asarray(resized, dtype=np.float32) / 255.0).transpose(2, 0, 1)[None]
|
||||
actual = pilot("pilot_ddrnet_numpy").preprocess_bgr(bgr)
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
assert actual.flags.c_contiguous and actual.dtype == np.dtype("<f4")
|
||||
with pytest.raises(ValueError, match="contract"):
|
||||
pilot("pilot_ddrnet_numpy").preprocess_bgr(bgr[:512])
|
||||
|
||||
|
||||
def test_triton_statistics_exclude_warmup_and_reject_counter_reset(pilot):
|
||||
names = ("success", "fail", "queue", "compute_input", "compute_infer", "compute_output")
|
||||
before = {"model": {name: {"count": "8", "ns": "1000000"} for name in names}}
|
||||
after = {"model": {name: {"count": "10", "ns": "5000000"} for name in names}}
|
||||
delta = pilot("pilot_triton_stats").stats_delta(before, after)
|
||||
assert delta["model"]["compute_infer"] == {"count": 2, "total_ms": 4, "mean_ms": 2}
|
||||
with pytest.raises(RuntimeError, match="backwards"):
|
||||
pilot("pilot_triton_stats").stats_delta(after, before)
|
||||
|
||||
|
||||
def test_ddrnet_triton_transport_is_bounded_and_validates_output_identity(pilot):
|
||||
module = pilot("pilot_ddrnet_triton")
|
||||
mask = np.zeros((512, 512), np.uint8)
|
||||
descriptor = {"outputs": [{
|
||||
"name": "mask", "datatype": "UINT8", "shape": [1, 512, 512],
|
||||
"parameters": {"binary_data_size": mask.nbytes},
|
||||
}]}
|
||||
header = json.dumps(descriptor).encode()
|
||||
|
||||
class Response:
|
||||
status = 200
|
||||
payload = header + mask.tobytes()
|
||||
|
||||
def read(self, limit):
|
||||
assert limit == module.MAX_RESPONSE_BYTES + 1
|
||||
return self.payload[:limit]
|
||||
|
||||
def getheader(self, _name):
|
||||
return str(len(header))
|
||||
|
||||
response = Response()
|
||||
backend = module.TritonDdrnetMaskHttpInferenceBackend()
|
||||
backend.connection = SimpleNamespace(
|
||||
request=lambda *args, **kwargs: None,
|
||||
getresponse=lambda: response,
|
||||
close=lambda: None,
|
||||
)
|
||||
tensor = np.zeros((1, 3, 512, 512), np.float32)
|
||||
np.testing.assert_array_equal(backend.infer(tensor), mask)
|
||||
response.payload = b"x" * (module.MAX_RESPONSE_BYTES + 1)
|
||||
with pytest.raises(RuntimeError, match="budget"):
|
||||
backend.infer(tensor)
|
||||
response.payload = header.replace(b'"UINT8"', b'"FP32" ') + mask.tobytes()
|
||||
with pytest.raises(RuntimeError, match="contract"):
|
||||
backend.infer(tensor)
|
||||
|
||||
|
||||
def test_nvml_sampler_reuses_one_session_without_spawning_processes(pilot):
|
||||
class Function:
|
||||
def __init__(self, callback):
|
||||
self.callback = callback
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, *args):
|
||||
self.calls += 1
|
||||
return self.callback(*args)
|
||||
|
||||
def set_handle(_index, pointer):
|
||||
pointer._obj.value = 7
|
||||
return 0
|
||||
|
||||
def set_memory(_handle, pointer):
|
||||
pointer._obj.total = 24 * 1024 * 1024
|
||||
pointer._obj.free = 20 * 1024 * 1024
|
||||
pointer._obj.used = 3 * 1024 * 1024 + 1
|
||||
return 0
|
||||
|
||||
def set_utilization(_handle, pointer):
|
||||
pointer._obj.gpu = 42
|
||||
pointer._obj.memory = 3
|
||||
return 0
|
||||
|
||||
library = SimpleNamespace(
|
||||
nvmlInit_v2=Function(lambda: 0),
|
||||
nvmlShutdown=Function(lambda: 0),
|
||||
nvmlDeviceGetHandleByIndex_v2=Function(set_handle),
|
||||
nvmlDeviceGetMemoryInfo=Function(set_memory),
|
||||
nvmlDeviceGetUtilizationRates=Function(set_utilization),
|
||||
)
|
||||
sampler = pilot("pilot_telemetry").NvmlSampler(library)
|
||||
|
||||
assert sampler.sample() == {"gpu_used_mib": 4, "gpu_utilization": 42}
|
||||
assert sampler.sample() == {"gpu_used_mib": 4, "gpu_utilization": 42}
|
||||
sampler.close()
|
||||
sampler.close()
|
||||
|
||||
assert library.nvmlInit_v2.calls == 1
|
||||
assert library.nvmlShutdown.calls == 1
|
||||
assert library.nvmlDeviceGetMemoryInfo.calls == 2
|
||||
assert library.nvmlDeviceGetUtilizationRates.calls == 2
|
||||
|
||||
|
||||
def test_body_history_is_bounded_and_never_reads_future(pilot):
|
||||
store = pilot("pilot_graph").CurrentStore(None)
|
||||
for index in range(100):
|
||||
|
||||
Reference in New Issue
Block a user