diff --git a/config/observatory-portable-run-definitions.json b/config/observatory-portable-run-definitions.json index ccf916d..2f7df31 100644 --- a/config/observatory-portable-run-definitions.json +++ b/config/observatory-portable-run-definitions.json @@ -1,241 +1 @@ -{ - "schema_version": "missioncore.observatory-portable-run-definition-registry/v2", - "definitions": [ - { - "setup_id": "lab-v1-eomt-ddrnet-portable-v1", - "definition_id": "lab-v1-eomt-ddrnet-portable", - "version": 2, - "definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", - "source_requirements": { - "plugin_id": "nodedc.device.xgrids-lixelkity-k1", - "archive_id": "xgrids-k1.viewer-live.evidence", - "required_modalities": [ - "point-cloud", - "trajectory", - "video" - ], - "camera_source_id": "sensor.camera.right", - "camera_semantic_channel_id": "camera.video.recorded", - "recorded_media_type": "video/mp4; codecs=\"avc1.641028\"", - "recorded_media_init_sha256": "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38", - "camera_width": 800, - "camera_height": 600, - "calibration_slot": "camera_1", - "calibration_identity_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9", - "exactly_one_media_epoch": true, - "seekable": true - }, - "source_adapter": { - "adapter_id": "xgrids-k1-recorded-observatory-v2", - "version": 2, - "contract_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de" - }, - "components": [ - { - "component_id": "ddrnet-portable-runtime-config-v2", - "kind": "configuration", - "sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21" - }, - { - "component_id": "eomt-recorded-dependency-set-v1", - "kind": "dependency-set", - "sha256": "4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e" - }, - { - "component_id": "eomt-recorded-orchestrator-v1", - "kind": "orchestrator", - "sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774" - }, - { - "component_id": "eomt-recorded-profile-v1", - "kind": "profile", - "sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875" - }, - { - "component_id": "eomt-recorded-runner-v1", - "kind": "runner", - "sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4" - }, - { - "component_id": "k1-camera-1-calibration-v1", - "kind": "calibration", - "sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" - }, - { - "component_id": "k1-valid-fov-identity-v1", - "kind": "valid-fov-identity", - "sha256": "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" - }, - { - "component_id": "k1-valid-fov-mask-v1", - "kind": "valid-fov-mask", - "sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" - }, - { - "component_id": "vegetation-mission-policy-v1", - "kind": "policy", - "sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35" - }, - { - "component_id": "vegetation-provider-label-map-v1", - "kind": "provider-map", - "sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352" - } - ], - "models": [ - { - "release_id": "eomt-cityscapes-large-1024-v1", - "model_id": "tue-mps/cityscapes_semantic_eomt_large_1024", - "revision": "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f", - "architecture": "EomtForUniversalSegmentation", - "artifacts": [ - { - "role": "config-json", - "byte_length": 1575, - "sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650" - }, - { - "role": "model-weights", - "byte_length": 1276175488, - "sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782" - }, - { - "role": "preprocessor-config", - "byte_length": 666, - "sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7" - } - ] - }, - { - "release_id": "lab-v1-ddrnet-39-goose-fine-64-v1", - "model_id": "goose-ddrnet-class-512", - "revision": null, - "architecture": "ddrnet_39", - "artifacts": [ - { - "role": "checkpoint", - "byte_length": 259419077, - "sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" - } - ] - } - ], - "resource_profile": { - "schema_version": "missioncore.observatory-portable-resource-profile/v2", - "profile_id": "worker006-single-gpu-sequential-ai-v1", - "contour_id": "worker-006", - "accelerator_id": "nvidia-rtx-4090", - "concurrency": 1, - "checkpoint_policy": "non-checkpointable", - "allowed_checkpoints": [], - "profile_sha256": "7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d" - }, - "result_contract": { - "schema_version": "missioncore.observatory-portable-result-contract/v2", - "contract_id": "recorded-eomt-ddrnet-review-v2", - "version": 2, - "result_schema": "missioncore.recorded-eomt-ddrnet-review/v2", - "result_kind": "recorded-perception-qualification", - "publication": "observatory", - "contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a" - }, - "executor": { - "contour_id": "worker-006", - "state": "not-installed", - "release_id": null, - "release_sha256": null, - "image_sha256": null, - "reason_code": "lab-v1-portable-v2-uninstalled", - "reason": "Portable LAB V1 v2 is not sealed or installed; a commit-bound combined image and Worker 006 receipt are required." - }, - "authority": { - "commands_enabled": false, - "actuation_allowed": false, - "navigation_or_safety_accepted": false, - "production_accepted": false - } - }, - { - "setup_id": "m49-tgs-portable-v2", - "definition_id": "m49-tgs-portable", - "version": 3, - "definition_sha256": "f56d6321bd794ccdfb7d2e3b05d044b11f616ffb81ee29517386cc253046d4eb", - "source_requirements": { - "plugin_id": "nodedc.device.xgrids-lixelkity-k1", - "archive_id": "xgrids-k1.viewer-live.evidence", - "required_modalities": [ - "point-cloud", - "trajectory", - "video" - ], - "camera_source_id": "sensor.camera.right", - "camera_semantic_channel_id": "camera.video.recorded", - "recorded_media_type": "video/mp4; codecs=\"avc1.641028\"", - "recorded_media_init_sha256": "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38", - "camera_width": 800, - "camera_height": 600, - "calibration_slot": "camera_1", - "calibration_identity_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9", - "exactly_one_media_epoch": true, - "seekable": true - }, - "source_adapter": { - "adapter_id": "xgrids-k1-recorded-observatory-v2", - "version": 2, - "contract_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de" - }, - "components": [ - { - "component_id": "k1-camera-1-calibration-v1", - "kind": "calibration", - "sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" - }, - { - "component_id": "m49-tgs-portable-profile-v2", - "kind": "configuration", - "sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9" - }, - { - "component_id": "m49-tgs-portable-runner-v1", - "kind": "runner", - "sha256": "e3bb2e91c70712eff74e8e69718075a407e042fb494616d65984500da42b21a9" - } - ], - "models": [], - "resource_profile": { - "schema_version": "missioncore.observatory-portable-resource-profile/v2", - "profile_id": "worker006-cpu-single-run-portable-v2", - "contour_id": "worker-006", - "accelerator_id": "cpu-only", - "concurrency": 1, - "checkpoint_policy": "non-checkpointable", - "allowed_checkpoints": [], - "profile_sha256": "49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee" - }, - "result_contract": { - "schema_version": "missioncore.observatory-portable-result-contract/v2", - "contract_id": "m49-tgs-portable-review-v2", - "version": 2, - "result_schema": "missioncore.recorded-tgs-costmap-review/v2", - "result_kind": "recorded-perception-qualification", - "publication": "observatory", - "contract_sha256": "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892" - }, - "executor": { - "contour_id": "worker-006", - "state": "ready", - "release_id": "m49-tgs-portable-executor-v1", - "release_sha256": "c5b0670d943fe0452ef4bbfbc144ab2439a1a674f9ef164798ad9f8b1ecc29fa", - "image_sha256": "f9278ab21aa65045be993dd19bffc25f49955e19598893ac78cc4761ca63ecf3", - "reason_code": null, - "reason": null - }, - "authority": { - "commands_enabled": false, - "actuation_allowed": false, - "navigation_or_safety_accepted": false, - "production_accepted": false - } - } - ] -} +{"definitions":[{"authority":{"actuation_allowed":false,"commands_enabled":false,"navigation_or_safety_accepted":false,"production_accepted":false},"components":[{"component_id":"ddrnet-portable-runtime-config-v2","kind":"configuration","sha256":"c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21"},{"component_id":"eomt-recorded-dependency-set-v1","kind":"dependency-set","sha256":"4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e"},{"component_id":"eomt-recorded-orchestrator-v1","kind":"orchestrator","sha256":"d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774"},{"component_id":"eomt-recorded-profile-v1","kind":"profile","sha256":"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"},{"component_id":"eomt-recorded-runner-v1","kind":"runner","sha256":"1e64869de48d10f1531c742e6067c4c3ae2a709c5eb0d770d1fab74b4a2431ff"},{"component_id":"k1-camera-1-calibration-v1","kind":"calibration","sha256":"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"},{"component_id":"k1-valid-fov-identity-v1","kind":"valid-fov-identity","sha256":"b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2"},{"component_id":"k1-valid-fov-mask-v1","kind":"valid-fov-mask","sha256":"a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"},{"component_id":"vegetation-mission-policy-v1","kind":"policy","sha256":"b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"},{"component_id":"vegetation-provider-label-map-v1","kind":"provider-map","sha256":"f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"}],"definition_id":"lab-v1-eomt-ddrnet-portable","definition_sha256":"269d71a24b4e63cff54e01273f9d4b35fc6cdd72bc6fadec206169ae0777e6ac","executor":{"contour_id":"worker-006","image_sha256":"5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373","reason":null,"reason_code":null,"release_id":"lab-v1-installed-package-v1","release_sha256":"667858623962cd6d9849a8985b803f59e429916b5c56f76a6fc6c80c0c54526b","state":"ready"},"models":[{"architecture":"EomtForUniversalSegmentation","artifacts":[{"byte_length":1575,"role":"config-json","sha256":"7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"},{"byte_length":1276175488,"role":"model-weights","sha256":"c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"},{"byte_length":666,"role":"preprocessor-config","sha256":"97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"}],"model_id":"tue-mps/cityscapes_semantic_eomt_large_1024","release_id":"eomt-cityscapes-large-1024-v1","revision":"8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"},{"architecture":"ddrnet_39","artifacts":[{"byte_length":259419077,"role":"checkpoint","sha256":"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"}],"model_id":"goose-ddrnet-class-512","release_id":"lab-v1-ddrnet-39-goose-fine-64-v1","revision":null}],"resource_profile":{"accelerator_id":"nvidia-rtx-4090","allowed_checkpoints":[],"checkpoint_policy":"non-checkpointable","concurrency":1,"contour_id":"worker-006","profile_id":"worker006-single-gpu-sequential-ai-v1","profile_sha256":"7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d","schema_version":"missioncore.observatory-portable-resource-profile/v2"},"result_contract":{"contract_id":"recorded-eomt-ddrnet-review-v2","contract_sha256":"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a","publication":"observatory","result_kind":"recorded-perception-qualification","result_schema":"missioncore.recorded-eomt-ddrnet-review/v2","schema_version":"missioncore.observatory-portable-result-contract/v2","version":2},"setup_id":"lab-v1-eomt-ddrnet-portable-v1","source_adapter":{"adapter_id":"xgrids-k1-recorded-observatory-v2","contract_sha256":"4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de","version":2},"source_requirements":{"archive_id":"xgrids-k1.viewer-live.evidence","calibration_identity_sha256":"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9","calibration_slot":"camera_1","camera_height":600,"camera_semantic_channel_id":"camera.video.recorded","camera_source_id":"sensor.camera.right","camera_width":800,"exactly_one_media_epoch":true,"plugin_id":"nodedc.device.xgrids-lixelkity-k1","recorded_media_init_sha256":"e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38","recorded_media_type":"video/mp4; codecs=\"avc1.641028\"","required_modalities":["point-cloud","trajectory","video"],"seekable":true},"version":2},{"authority":{"actuation_allowed":false,"commands_enabled":false,"navigation_or_safety_accepted":false,"production_accepted":false},"components":[{"component_id":"k1-camera-1-calibration-v1","kind":"calibration","sha256":"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"},{"component_id":"m49-tgs-portable-profile-v2","kind":"configuration","sha256":"6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9"},{"component_id":"m49-tgs-portable-runner-v1","kind":"runner","sha256":"e3bb2e91c70712eff74e8e69718075a407e042fb494616d65984500da42b21a9"}],"definition_id":"m49-tgs-portable","definition_sha256":"f56d6321bd794ccdfb7d2e3b05d044b11f616ffb81ee29517386cc253046d4eb","executor":{"contour_id":"worker-006","image_sha256":"f9278ab21aa65045be993dd19bffc25f49955e19598893ac78cc4761ca63ecf3","reason":null,"reason_code":null,"release_id":"m49-tgs-portable-executor-v1","release_sha256":"c5b0670d943fe0452ef4bbfbc144ab2439a1a674f9ef164798ad9f8b1ecc29fa","state":"ready"},"models":[],"resource_profile":{"accelerator_id":"cpu-only","allowed_checkpoints":[],"checkpoint_policy":"non-checkpointable","concurrency":1,"contour_id":"worker-006","profile_id":"worker006-cpu-single-run-portable-v2","profile_sha256":"49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee","schema_version":"missioncore.observatory-portable-resource-profile/v2"},"result_contract":{"contract_id":"m49-tgs-portable-review-v2","contract_sha256":"9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892","publication":"observatory","result_kind":"recorded-perception-qualification","result_schema":"missioncore.recorded-tgs-costmap-review/v2","schema_version":"missioncore.observatory-portable-result-contract/v2","version":2},"setup_id":"m49-tgs-portable-v2","source_adapter":{"adapter_id":"xgrids-k1-recorded-observatory-v2","contract_sha256":"4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de","version":2},"source_requirements":{"archive_id":"xgrids-k1.viewer-live.evidence","calibration_identity_sha256":"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9","calibration_slot":"camera_1","camera_height":600,"camera_semantic_channel_id":"camera.video.recorded","camera_source_id":"sensor.camera.right","camera_width":800,"exactly_one_media_epoch":true,"plugin_id":"nodedc.device.xgrids-lixelkity-k1","recorded_media_init_sha256":"e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38","recorded_media_type":"video/mp4; codecs=\"avc1.641028\"","required_modalities":["point-cloud","trajectory","video"],"seekable":true},"version":3}],"schema_version":"missioncore.observatory-portable-run-definition-registry/v2"} diff --git a/config/observatory-worker-runtime-candidates.json b/config/observatory-worker-runtime-candidates.json index 16c5b71..d018c68 100644 --- a/config/observatory-worker-runtime-candidates.json +++ b/config/observatory-worker-runtime-candidates.json @@ -1,288 +1 @@ -{ - "schema_version": "missioncore.observatory-portable-worker-runtime-registry/v1", - "candidates": [ - { - "schema_version": "missioncore.observatory-portable-worker-runtime-candidate/v1", - "adapter_id": "lab-v1-eomt-ddrnet-worker006-v2", - "setup_id": "lab-v1-eomt-ddrnet-portable-v1", - "definition_id": "lab-v1-eomt-ddrnet-portable", - "definition_version": 2, - "definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", - "source_adapter_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de", - "model_manifest_sha256": "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56", - "resource_profile_sha256": "7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d", - "result_contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a", - "state": "blocked", - "executor": null, - "reusable_assets": [ - { - "asset_id": "ddrnet-checkpoint", - "kind": "model-artifact", - "sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6", - "byte_length": 259419077, - "component_id": null, - "model_release_id": "lab-v1-ddrnet-39-goose-fine-64-v1", - "model_artifact_role": "checkpoint" - }, - { - "asset_id": "ddrnet-goose-image", - "kind": "container-image", - "sha256": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd", - "byte_length": null, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "ddrnet-goose-runner", - "kind": "local-file", - "sha256": "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1", - "byte_length": 32877, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "ddrnet-portable-config", - "kind": "definition-component", - "sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21", - "byte_length": 4324, - "component_id": "ddrnet-portable-runtime-config-v2", - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "eomt-config-json", - "kind": "model-artifact", - "sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650", - "byte_length": 1575, - "component_id": null, - "model_release_id": "eomt-cityscapes-large-1024-v1", - "model_artifact_role": "config-json" - }, - { - "asset_id": "eomt-image", - "kind": "container-image", - "sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", - "byte_length": null, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "eomt-model-weights", - "kind": "model-artifact", - "sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782", - "byte_length": 1276175488, - "component_id": null, - "model_release_id": "eomt-cityscapes-large-1024-v1", - "model_artifact_role": "model-weights" - }, - { - "asset_id": "eomt-orchestrator", - "kind": "definition-component", - "sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774", - "byte_length": 21489, - "component_id": "eomt-recorded-orchestrator-v1", - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "eomt-preprocessor-config", - "kind": "model-artifact", - "sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7", - "byte_length": 666, - "component_id": null, - "model_release_id": "eomt-cityscapes-large-1024-v1", - "model_artifact_role": "preprocessor-config" - }, - { - "asset_id": "eomt-profile", - "kind": "definition-component", - "sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875", - "byte_length": 3805, - "component_id": "eomt-recorded-profile-v1", - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "eomt-runner", - "kind": "definition-component", - "sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4", - "byte_length": 30720, - "component_id": "eomt-recorded-runner-v1", - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "vegetation-policy", - "kind": "definition-component", - "sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35", - "byte_length": 3022, - "component_id": "vegetation-mission-policy-v1", - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "vegetation-provider-map", - "kind": "definition-component", - "sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352", - "byte_length": 2756, - "component_id": "vegetation-provider-label-map-v1", - "model_release_id": null, - "model_artifact_role": null - } - ], - "phases": [ - { - "phase_id": "source-delivery", - "state": "implemented" - }, - { - "phase_id": "eomt-runtime", - "state": "implemented" - }, - { - "phase_id": "ddrnet-portable-runtime", - "state": "missing" - }, - { - "phase_id": "portable-lab-orchestrator", - "state": "implemented" - }, - { - "phase_id": "result-v2-assembler", - "state": "implemented" - }, - { - "phase_id": "observatory-result-publisher", - "state": "implemented" - } - ], - "blockers": [ - "combined-executor-image-unsealed", - "commit-bound-source-unavailable", - "ddrnet-component-port-uninstalled", - "eomt-component-port-uninstalled", - "executor-release-unsealed", - "fixture-smoke-unaccepted", - "worker-installation-receipt-unavailable" - ], - "authority": { - "commands_enabled": false, - "actuation_allowed": false, - "navigation_or_safety_accepted": false, - "production_accepted": false - }, - "candidate_sha256": "b22b8fa16cca6dd68cf1ee68abeea84b33f4c420bb3844931b4dc7fd19434a81" - }, - { - "schema_version": "missioncore.observatory-portable-worker-runtime-candidate/v1", - "adapter_id": "m49-tgs-worker006-portable-v2", - "setup_id": "m49-tgs-portable-v2", - "definition_id": "m49-tgs-portable", - "definition_version": 3, - "definition_sha256": "f56d6321bd794ccdfb7d2e3b05d044b11f616ffb81ee29517386cc253046d4eb", - "source_adapter_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de", - "model_manifest_sha256": "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1", - "resource_profile_sha256": "49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee", - "result_contract_sha256": "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892", - "state": "ready", - "executor": { - "release_id": "m49-tgs-portable-executor-v1", - "release_sha256": "c5b0670d943fe0452ef4bbfbc144ab2439a1a674f9ef164798ad9f8b1ecc29fa", - "image_sha256": "f9278ab21aa65045be993dd19bffc25f49955e19598893ac78cc4761ca63ecf3" - }, - "reusable_assets": [ - { - "asset_id": "m49-portable-compiled-runner", - "kind": "local-file", - "sha256": "7be449392ef161fb8713b4c984705d2373bc3cd05645332d92b88a8bff0c7db3", - "byte_length": 274168, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "m49-portable-compiled-runner-build-seal", - "kind": "local-file", - "sha256": "e3bb2e91c70712eff74e8e69718075a407e042fb494616d65984500da42b21a9", - "byte_length": 1014, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "m49-portable-executor-release", - "kind": "local-file", - "sha256": "c5b0670d943fe0452ef4bbfbc144ab2439a1a674f9ef164798ad9f8b1ecc29fa", - "byte_length": 1693, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "m49-portable-profile", - "kind": "definition-component", - "sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9", - "byte_length": 1683, - "component_id": "m49-tgs-portable-profile-v2", - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "m49-portable-worker-installation-receipt", - "kind": "local-file", - "sha256": "b560ff9e02746cbb760502f2a3b4b7bd564f95ab52d1149d189927a326b1645a", - "byte_length": 2672, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - }, - { - "asset_id": "travel-tgs-image", - "kind": "container-image", - "sha256": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f", - "byte_length": null, - "component_id": null, - "model_release_id": null, - "model_artifact_role": null - } - ], - "phases": [ - { - "phase_id": "source-delivery", - "state": "implemented" - }, - { - "phase_id": "camera-lidar-timeline-materializer", - "state": "implemented" - }, - { - "phase_id": "portable-tgs-input-materializer", - "state": "implemented" - }, - { - "phase_id": "portable-tgs-runner", - "state": "implemented" - }, - { - "phase_id": "result-v2-assembler", - "state": "implemented" - }, - { - "phase_id": "observatory-result-publisher", - "state": "implemented" - } - ], - "blockers": [], - "authority": { - "commands_enabled": false, - "actuation_allowed": false, - "navigation_or_safety_accepted": false, - "production_accepted": false - }, - "candidate_sha256": "65cd2063146a1dd320e30d5f4e21e4bf0aab1ff683e846926cbbfbe25a9f8a5e" - } - ] -} +{"candidates":[{"adapter_id":"lab-v1-installed-package-worker006-v1","authority":{"actuation_allowed":false,"commands_enabled":false,"navigation_or_safety_accepted":false,"production_accepted":false},"blockers":[],"candidate_sha256":"0b6958e3a1b4e12e04619f447ca9ce915021aa1aaa6034ea6bc309c26efbd060","definition_id":"lab-v1-eomt-ddrnet-portable","definition_sha256":"269d71a24b4e63cff54e01273f9d4b35fc6cdd72bc6fadec206169ae0777e6ac","definition_version":2,"executor":{"image_sha256":"5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373","release_id":"lab-v1-installed-package-v1","release_sha256":"667858623962cd6d9849a8985b803f59e429916b5c56f76a6fc6c80c0c54526b"},"model_manifest_sha256":"3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56","phases":[{"phase_id":"source-delivery","state":"implemented"},{"phase_id":"prepare-source","state":"implemented"},{"phase_id":"eomt-step","state":"implemented"},{"phase_id":"ddrnet-step","state":"implemented"},{"phase_id":"assemble-result","state":"implemented"},{"phase_id":"result-publication","state":"implemented"}],"resource_profile_sha256":"7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d","result_contract_sha256":"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a","reusable_assets":[{"asset_id":"agent-image","byte_length":null,"component_id":null,"kind":"container-image","model_artifact_role":null,"model_release_id":null,"sha256":"5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373"},{"asset_id":"ddrnet-checkpoint","byte_length":259419077,"component_id":null,"kind":"model-artifact","model_artifact_role":"checkpoint","model_release_id":"lab-v1-ddrnet-39-goose-fine-64-v1","sha256":"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"},{"asset_id":"ddrnet-goose-mapping","byte_length":1427,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"},{"asset_id":"ddrnet-goose-runner","byte_length":32877,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"},{"asset_id":"ddrnet-portable-config","byte_length":4324,"component_id":"ddrnet-portable-runtime-config-v2","kind":"definition-component","model_artifact_role":null,"model_release_id":null,"sha256":"c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21"},{"asset_id":"ddrnet-step-image","byte_length":null,"component_id":null,"kind":"container-image","model_artifact_role":null,"model_release_id":null,"sha256":"e6c986100613ec804f0e0076ca8695abf43ff88ef9d5d85f6857e0b41db74051"},{"asset_id":"eomt-environment","byte_length":211776082,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"8c8f343a5368ff17edbb58defa1669f6eccfba767aab897a23693872070ab9e0"},{"asset_id":"eomt-ffmpeg-runtime","byte_length":256208352,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69"},{"asset_id":"eomt-model-cache","byte_length":2552355458,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"064870e58814b97027d6a7ccd553bf51f5b8e6a8ad82a1cc703584d2dca5690c"},{"asset_id":"eomt-python-environment","byte_length":5120848705,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"b3f4efc53af491f174b1cff74b3ba03016e67c9c5c74257b49c6e7dd7d853f20"},{"asset_id":"eomt-runner-bundle","byte_length":145141,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"0d08f0492d5ad62903874ea224c505e54a6f6059c8283f586bc79e42d55156b7"},{"asset_id":"eomt-step-image","byte_length":null,"component_id":null,"kind":"container-image","model_artifact_role":null,"model_release_id":null,"sha256":"adba3dc8c97b161ba261ec44fca9ebe1680f117d1bcb1481440172cc3331a174"},{"asset_id":"eomt-transformers-environment","byte_length":225272284,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"f365de01426a33be51a310923c743655634d0868941bbf3f1aae1647fdeadfc9"},{"asset_id":"k1-valid-fov-root","byte_length":6019,"component_id":null,"kind":"local-tree","model_artifact_role":null,"model_release_id":null,"sha256":"f4fc2053e4e6213bb364c8773979b755d5682a81b3946c25ff86274bc5f0031e"},{"asset_id":"lab-v1-definition-registry","byte_length":7177,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"1de9a07153fa6f51088aecdbcf74d12c9e58716c5feb0145d6e54f20a8c300cb"},{"asset_id":"lab-v1-package-contract","byte_length":3531,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"c7a576aba09ccb0a343ad46ad90f1dab589ed51353d575026a508f8d6fa8dcc4"},{"asset_id":"vegetation-policy","byte_length":3022,"component_id":"vegetation-mission-policy-v1","kind":"definition-component","model_artifact_role":null,"model_release_id":null,"sha256":"b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"},{"asset_id":"vegetation-provider-map","byte_length":2756,"component_id":"vegetation-provider-label-map-v1","kind":"definition-component","model_artifact_role":null,"model_release_id":null,"sha256":"f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"}],"schema_version":"missioncore.observatory-portable-worker-runtime-candidate/v1","setup_id":"lab-v1-eomt-ddrnet-portable-v1","source_adapter_sha256":"4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de","state":"ready"},{"adapter_id":"m49-tgs-worker006-portable-v2","authority":{"actuation_allowed":false,"commands_enabled":false,"navigation_or_safety_accepted":false,"production_accepted":false},"blockers":[],"candidate_sha256":"65cd2063146a1dd320e30d5f4e21e4bf0aab1ff683e846926cbbfbe25a9f8a5e","definition_id":"m49-tgs-portable","definition_sha256":"f56d6321bd794ccdfb7d2e3b05d044b11f616ffb81ee29517386cc253046d4eb","definition_version":3,"executor":{"image_sha256":"f9278ab21aa65045be993dd19bffc25f49955e19598893ac78cc4761ca63ecf3","release_id":"m49-tgs-portable-executor-v1","release_sha256":"c5b0670d943fe0452ef4bbfbc144ab2439a1a674f9ef164798ad9f8b1ecc29fa"},"model_manifest_sha256":"489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1","phases":[{"phase_id":"source-delivery","state":"implemented"},{"phase_id":"camera-lidar-timeline-materializer","state":"implemented"},{"phase_id":"portable-tgs-input-materializer","state":"implemented"},{"phase_id":"portable-tgs-runner","state":"implemented"},{"phase_id":"result-v2-assembler","state":"implemented"},{"phase_id":"observatory-result-publisher","state":"implemented"}],"resource_profile_sha256":"49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee","result_contract_sha256":"9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892","reusable_assets":[{"asset_id":"m49-portable-compiled-runner","byte_length":274168,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"7be449392ef161fb8713b4c984705d2373bc3cd05645332d92b88a8bff0c7db3"},{"asset_id":"m49-portable-compiled-runner-build-seal","byte_length":1014,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"e3bb2e91c70712eff74e8e69718075a407e042fb494616d65984500da42b21a9"},{"asset_id":"m49-portable-executor-release","byte_length":1693,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"c5b0670d943fe0452ef4bbfbc144ab2439a1a674f9ef164798ad9f8b1ecc29fa"},{"asset_id":"m49-portable-profile","byte_length":1683,"component_id":"m49-tgs-portable-profile-v2","kind":"definition-component","model_artifact_role":null,"model_release_id":null,"sha256":"6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9"},{"asset_id":"m49-portable-worker-installation-receipt","byte_length":2672,"component_id":null,"kind":"local-file","model_artifact_role":null,"model_release_id":null,"sha256":"b560ff9e02746cbb760502f2a3b4b7bd564f95ab52d1149d189927a326b1645a"},{"asset_id":"travel-tgs-image","byte_length":null,"component_id":null,"kind":"container-image","model_artifact_role":null,"model_release_id":null,"sha256":"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"}],"schema_version":"missioncore.observatory-portable-worker-runtime-candidate/v1","setup_id":"m49-tgs-portable-v2","source_adapter_sha256":"4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de","state":"ready"}],"schema_version":"missioncore.observatory-portable-worker-runtime-registry/v1"} diff --git a/docs/adr/0046-durable-observatory-recorded-queue-and-live-k1-priority.md b/docs/adr/0046-durable-observatory-recorded-queue-and-live-k1-priority.md index 9cb035a..2537862 100644 --- a/docs/adr/0046-durable-observatory-recorded-queue-and-live-k1-priority.md +++ b/docs/adr/0046-durable-observatory-recorded-queue-and-live-k1-priority.md @@ -5,6 +5,11 @@ Status: accepted; portable LAB V1 admission and Worker pull foundations implemen production executor, claim lease, result publication, Worker deployment and live-trigger wiring pending +Amended on 2026-09-01 by ADR 0048. Capability-aware claims, renewable leases, +verified result transport and the durable publication lifecycle are now +implemented; physical Worker installation and live-trigger wiring remain outside +that source increment. + ## Context Observatory must let an operator apply an admitted laboratory setup to a saved diff --git a/docs/adr/0047-verified-portable-observatory-result-publication.md b/docs/adr/0047-verified-portable-observatory-result-publication.md index e62ec2c..1740892 100644 --- a/docs/adr/0047-verified-portable-observatory-result-publication.md +++ b/docs/adr/0047-verified-portable-observatory-result-publication.md @@ -4,6 +4,11 @@ Date: 2026-08-31 Status: accepted as a backend foundation; production validators, transport and executor wiring remain blocked +Amended on 2026-09-01 by ADR 0048. Exact built-in validators, package transport, +application publication wiring, durable publication status and a typed portable +result viewer are now implemented. Installed generic package executors and a +physical Worker 006 cutover remain separate acceptance work. + ## Context The durable Observatory queue deliberately treats a Worker `succeed` call as a diff --git a/docs/adr/0048-independent-installed-lab-packages-and-portable-result-lifecycle.md b/docs/adr/0048-independent-installed-lab-packages-and-portable-result-lifecycle.md new file mode 100644 index 0000000..6e14168 --- /dev/null +++ b/docs/adr/0048-independent-installed-lab-packages-and-portable-result-lifecycle.md @@ -0,0 +1,138 @@ +# ADR 0048: Independent installed LAB packages and portable result lifecycle + +Date: 2026-09-01 +Status: accepted and implemented as source contracts; no Worker deployment or image migration + +## Context + +The first portable Observatory path still composed M4.9 and LAB V1 through +profile-specific builders and a combined Worker release. A Worker could claim a +job whose exact executor was not installed, execution success could become +terminal before catalog publication, and the UI could open only one hard-coded +legacy replay family. Adding another LAB therefore still risked changes across +the Worker coordinator, backend publication and frontend viewer. + +The target remains observation-only. This decision does not authorize a build, +deployment, Docker installation, Worker 006 mutation, K1 command or safety use. + +## Decision + +### Installed package boundary + +A Worker-local `InstalledLabPackageRegistry` binds one independently installed +package to the exact RunDefinition, RuntimeCandidate and four-digest executor +identity. Its manifest contains only reviewed container images, argv, dependency +topology, fixed in-container mount targets and immutable asset IDs. It cannot +contain a host path, secret, environment, Docker socket or job-provided command. + +Every package uses one stable container I/O contract: + +- read-only source at `/missioncore/input/source`; +- read-only canonical plan at `/missioncore/input/run-plan.json`; +- result package at `/missioncore/output`; +- ephemeral work at `/missioncore/work`. + +`single-container` and `fixed-stack` are package properties. Exactly one +container owns the portable result. All images must already be admitted by the +bound RuntimeCandidate, and the package asset inventory must exactly equal that +candidate's reusable asset inventory. + +Worker composition now exposes one generic package executor factory. A Worker +may install any non-empty subset of server-ready definitions. It advertises only +the resulting four-digest identities; it is not required to implement every LAB +known by the backend. Existing profile-specific builders remain a compatibility +path until their images adopt the common package I/O contract. + +The generic executor runs a deterministic dependency graph of hardened one-shot +containers through the local Docker Engine API. Every container is pinned by +image SHA-256, has no network, a read-only root filesystem, no Linux +capabilities or privilege escalation, and receives only the declared read-only +inputs plus the single shared result output. Memory, CPU, PID, shared-memory, +tmpfs, GPU-count and timeout limits are explicit package fields. Exactly one +`result-writer` step must transitively depend on every compute step, after which +the Worker verifies the declared result manifest, file lengths, digests and the +absence of links or undeclared files before publication can begin. + +Worker-local asset bindings translate reviewed controller paths to Docker-host +paths. Those host paths never enter the package manifest or queued job. The +current package contract intentionally admits ordered offline steps only; a +future LAB that genuinely requires simultaneous services needs an explicit new +contract instead of silently weakening the isolation boundary. + +### Capability-aware dispatch + +Worker claim protocol v2 sends a bounded, canonical snapshot of installed +four-digest executor identities. The queue selects the oldest queued job that +matches one of those identities inside the same transaction that creates the +claim. An empty snapshot claims nothing. The snapshot is included in the claim +request digest, so an idempotency key cannot be replayed with different Worker +capabilities. + +Claim v1 remains accepted only as a rolling compatibility path and cannot send a +capability field. New Worker code always uses v2. + +### Execution and publication are separate durable lifecycles + +Verified upload completion atomically seals execution as `succeeded` and creates +a publication outbox entry. Publication has its own state: + +- `not-required` for the compatibility path; +- `pending` after verified execution completion; +- `failed` with bounded error evidence and attempt count; +- `published` with timestamp. + +A publisher failure no longer rewrites or loses the successful execution. The +authenticated Worker API returns the durable job with HTTP 202 and exposes an +idempotent publication retry endpoint. Exact retries reuse content-addressed +artifacts and the immutable SessionStore projection; they do not rerun the model. + +### Contract-driven result viewing + +Portable publications receive an explicit v2 viewer capability +`portable-result-review / portable-result / result-defined`. The public viewer +service resolves that capability, immutable publication provenance and central +artifact manifest, then rechecks the JSON result document's length and SHA-256. +The Control Station selects either the legacy canonical replay adapter or the +portable result adapter from the typed capability. No portable setup ID, LAB ID, +source session ID or result-name family selects the viewer. + +The UI polls while execution or publication is pending, refreshes the catalog +after publication, and keeps legacy and portable setup catalogs independently +usable if either endpoint is temporarily unavailable. Catalog merge identity is +`setup_id`, never a display name. + +### Validator extension + +Result validators are registered and selected by exact result-contract SHA-256. +The built-in LAB V1 and M4.9 functions remain compatibility registrations. The +composition core no longer requires both setup IDs or branches on them, so a +future contract can supply another server-owned registration without changing +the queue or Worker router. + +## Compatibility and migration boundary + +This source increment does not claim that the currently installed M4.9 or LAB V1 +images implement the new common container I/O surface. No synthetic package +manifest is checked in for an image that has not been rebuilt and smoked against +that surface. The old builders continue to work, and M4.9 startup no longer +requires LAB V1 receipt/release environment values. + +The next physical migration is deliberately per profile: + +1. make one image emit the existing portable result package through the common I/O contract; +2. seal its independent installed package manifest and local asset bindings; +3. run contract composition and a short offline smoke; +4. advertise only that executor identity in claim v2; +5. perform a reversible canary without changing another profile. + +## Consequences + +- A Worker cannot take an unsupported job merely because it is earlier in the queue. +- Adding a conforming LAB changes its definition, runtime candidate, validator + registration and installed package, not the queue protocol or frontend routing. +- Successful compute and successful catalog publication are both visible and + recoverable facts. +- Portable result review is generic JSON/artifact evidence; richer visual viewers + can be added as new typed capabilities without adding setup-name conditionals. +- The generic package boundary is implemented, while image conformance and + physical Worker 006 acceptance remain explicit, unclaimed work. diff --git a/src/k1link/observatory/installed_lab_package_runner.py b/src/k1link/observatory/installed_lab_package_runner.py new file mode 100644 index 0000000..1974d18 --- /dev/null +++ b/src/k1link/observatory/installed_lab_package_runner.py @@ -0,0 +1,912 @@ +"""Generic Worker-local launcher for independently installed LAB packages. + +The backend supplies only a sealed job identity. Reviewed local installation +bindings translate controller paths to Docker-host paths, while the package +owns the exact image, argv, dependency order, resource limits, and mount +targets. Every container is a one-shot step and is removed by its exact +Docker container id after completion. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import secrets +import shutil +import stat +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final, Protocol + +import httpx + +from k1link.observatory.installed_lab_packages import ( + INSTALLED_LAB_PLAN_PATH, + INSTALLED_LAB_RESULT_ROOT, + INSTALLED_LAB_SOURCE_ROOT, + INSTALLED_LAB_STEP_INPUT_ROOT, + INSTALLED_LAB_WORK_ROOT, + InstalledLabContainer, + InstalledLabPackage, +) +from k1link.observatory.portable_result_contract import ( + RESULT_PACKAGE_MANIFEST_NAME, + PortableResultPackageIntegrityError, + PortableResultPackageManifest, + canonical_json, + relative_artifact_path, +) +from k1link.observatory.portable_run_definitions import PortableRunDefinition +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerExecutorAdapter, + PortableWorkerLocalAssetBinding, + PortableWorkerResultDraft, + PortableWorkerRuntimePlan, + PortableWorkerSourceStage, + inspect_runtime_candidate, +) +from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration + +if TYPE_CHECKING: + from k1link.observatory.worker_service import ( + ObservatoryWorkerPackageExecutorBuildContext, + ) + +INSTALLED_LAB_RUN_PLAN_SCHEMA: Final = "missioncore.observatory-installed-lab-run-plan/v1" + +_DOCKER_SOCKET: Final = Path("/var/run/docker.sock") +_DOCKER_API_VERSION: Final = "v1.47" +_MAX_ENGINE_RESPONSE_BYTES: Final = 1024 * 1024 +_MAX_RESULT_MANIFEST_BYTES: Final = 1024 * 1024 +_CONTAINER_ID: Final = re.compile(r"^[a-f0-9]{12,128}$") +_ASSET_ID: Final = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") +_SHA256: Final = re.compile(r"^[a-f0-9]{64}$") +_WINDOWS_ABSOLUTE_PATH: Final = re.compile(r"^[A-Za-z]:\\") +_ENGINE_PATH_PLACEHOLDER: Final = re.compile( + r"(?:\$[A-Za-z_][A-Za-z0-9_]*|%[A-Za-z_][A-Za-z0-9_]*%|" + r"\{[A-Za-z_][A-Za-z0-9_]*\})" +) + + +class InstalledLabPackageRunnerError(RuntimeError): + """The generic installed-package runtime changed or failed closed.""" + + +@dataclass(frozen=True, slots=True) +class InstalledLabLocalAssetBinding: + """Reviewed local locator; neither path is serialized into a queued job.""" + + asset_id: str + controller_path: Path | None = None + engine_path: str | None = None + image_sha256: str | None = None + + def __post_init__(self) -> None: + if _ASSET_ID.fullmatch(self.asset_id) is None: + raise ValueError("installed LAB local asset id is invalid") + file_binding = self.controller_path is not None or self.engine_path is not None + image_binding = self.image_sha256 is not None + if file_binding == image_binding: + raise ValueError("installed LAB local asset locator is ambiguous") + if file_binding: + if self.controller_path is None or self.engine_path is None: + raise ValueError("installed LAB file asset binding is incomplete") + _absolute_controller_path(self.controller_path, "installed LAB asset") + _engine_host_path(self.engine_path, "installed LAB asset") + elif self.image_sha256 is None or _SHA256.fullmatch(self.image_sha256) is None: + raise ValueError("installed LAB image asset digest is invalid") + + def portable_binding(self) -> PortableWorkerLocalAssetBinding: + if self.controller_path is not None: + return PortableWorkerLocalAssetBinding( + asset_id=self.asset_id, + file_path=self.controller_path, + ) + return PortableWorkerLocalAssetBinding( + asset_id=self.asset_id, + image_sha256=self.image_sha256, + ) + + +@dataclass(frozen=True, slots=True) +class InstalledLabDockerMount: + engine_path: str + container_path: str + read_only: bool + + def __post_init__(self) -> None: + _engine_host_path(self.engine_path, "installed LAB Docker mount") + target = PurePosixPath(self.container_path) + if not target.is_absolute() or ".." in target.parts: + raise InstalledLabPackageRunnerError("Docker mount target is unsafe") + if not isinstance(self.read_only, bool): + raise InstalledLabPackageRunnerError("Docker mount mode is invalid") + + def engine_document(self) -> dict[str, object]: + return { + "Type": "bind", + "Source": self.engine_path, + "Target": self.container_path, + "ReadOnly": self.read_only, + "BindOptions": {"Propagation": "rprivate"}, + } + + +@dataclass(frozen=True, slots=True) +class InstalledLabDockerLaunch: + package_id: str + container: InstalledLabContainer + mounts: tuple[InstalledLabDockerMount, ...] + labels: Mapping[str, str] + name_token: str + + def __post_init__(self) -> None: + targets = tuple(mount.container_path for mount in self.mounts) + if targets != tuple(sorted(targets)) or len(targets) != len(set(targets)): + raise InstalledLabPackageRunnerError("Docker mounts are not canonical") + writable = tuple(mount for mount in self.mounts if not mount.read_only) + if len(writable) != 1 or writable[0].container_path != INSTALLED_LAB_RESULT_ROOT: + raise InstalledLabPackageRunnerError( + "Docker package must expose exactly one writable result mount" + ) + if not re.fullmatch(r"[a-f0-9]{16}", self.name_token): + raise InstalledLabPackageRunnerError("Docker launch token is invalid") + required_labels = { + "com.nodedc.authority", + "com.nodedc.component", + "com.nodedc.definition-sha256", + "com.nodedc.job-id", + "com.nodedc.managed-by", + "com.nodedc.package-sha256", + "com.nodedc.product", + "com.nodedc.stack", + } + if set(self.labels) != required_labels: + raise InstalledLabPackageRunnerError("Docker launch label set changed") + if ( + self.labels["com.nodedc.authority"] != "observation-only" + or self.labels["com.nodedc.component"] != self.container.container_id + or self.labels["com.nodedc.managed-by"] != "mission-core-worker" + or self.labels["com.nodedc.product"] != "mission-core" + or self.labels["com.nodedc.stack"] != "observatory" + ): + raise InstalledLabPackageRunnerError("Docker launch labels changed") + for key in ("com.nodedc.definition-sha256", "com.nodedc.package-sha256"): + if _SHA256.fullmatch(self.labels[key]) is None: + raise InstalledLabPackageRunnerError("Docker launch digest label is invalid") + + +class InstalledLabContainerLauncher(Protocol): + def __call__(self, launch: InstalledLabDockerLaunch) -> None: ... + + +@dataclass(frozen=True, slots=True) +class DockerEngineInstalledLabLauncher: + """Execute one hardened one-shot package step through the local Engine.""" + + socket_path: Path = _DOCKER_SOCKET + api_version: str = _DOCKER_API_VERSION + transport_factory: Callable[[], httpx.BaseTransport] | None = None + + def __post_init__(self) -> None: + if not self.socket_path.is_absolute(): + raise InstalledLabPackageRunnerError("Docker socket path is not absolute") + if re.fullmatch(r"v[0-9]+\.[0-9]+", self.api_version) is None: + raise InstalledLabPackageRunnerError("Docker API version is invalid") + + def __call__(self, launch: InstalledLabDockerLaunch) -> None: + if self.transport_factory is None: + _require_local_socket(self.socket_path) + transport: httpx.BaseTransport = httpx.HTTPTransport(uds=str(self.socket_path)) + else: + transport = self.transport_factory() + container_id: str | None = None + primary_error: BaseException | None = None + try: + with httpx.Client( + base_url="http://docker", + transport=transport, + timeout=httpx.Timeout(launch.container.timeout_seconds, connect=5.0), + ) as client: + self._verify_image(client, launch.container.image_sha256) + container_id = self._create(client, launch) + self._empty( + client, + "POST", + self._api_path(f"/containers/{container_id}/start"), + {204}, + ) + status_code = self._wait(client, container_id) + if status_code != 0: + log_sha256, log_bytes = self._log_identity(client, container_id) + raise InstalledLabPackageRunnerError( + f"package step {launch.container.container_id} exited with status " + f"{status_code}; logs={log_sha256}:{log_bytes}" + ) + except (httpx.HTTPError, OSError, ValueError) as exc: + primary_error = exc + raise InstalledLabPackageRunnerError( + f"local Docker Engine step {launch.container.container_id} failed" + ) from exc + except BaseException as exc: + primary_error = exc + raise + finally: + if container_id is not None: + cleanup_error = self._cleanup(container_id) + if cleanup_error is not None: + raise InstalledLabPackageRunnerError( + "Docker package container cleanup failed after retry" + ) from (primary_error or cleanup_error) + + def verify_images(self, image_sha256s: tuple[str, ...]) -> None: + """Prove a canonical installed image inventory without creating a container.""" + + if ( + not image_sha256s + or image_sha256s != tuple(sorted(image_sha256s)) + or len(image_sha256s) != len(set(image_sha256s)) + or any(_SHA256.fullmatch(value) is None for value in image_sha256s) + ): + raise InstalledLabPackageRunnerError( + "installed LAB image inventory is invalid" + ) + if self.transport_factory is None: + _require_local_socket(self.socket_path) + transport: httpx.BaseTransport = httpx.HTTPTransport( + uds=str(self.socket_path) + ) + else: + transport = self.transport_factory() + with httpx.Client( + base_url="http://docker", + transport=transport, + timeout=httpx.Timeout(10.0, connect=5.0), + ) as client: + for image_sha256 in image_sha256s: + self._verify_image(client, image_sha256) + + def _verify_image(self, client: httpx.Client, image_sha256: str) -> None: + response = self._response( + client, + "GET", + self._api_path(f"/images/sha256:{image_sha256}/json"), + {200}, + ) + document = _response_object(response, "Docker image inspection") + if document.get("Id") != f"sha256:{image_sha256}": + raise InstalledLabPackageRunnerError("Docker image identity changed") + + def _create(self, client: httpx.Client, launch: InstalledLabDockerLaunch) -> str: + component = launch.container.container_id[:32] + name = f"ndc-observatory-{component}-{launch.name_token}" + response = self._response( + client, + "POST", + self._api_path(f"/containers/create?name={name}"), + {201}, + json_body=_container_create_document(launch), + ) + document = _response_object(response, "Docker container creation") + container_id = document.get("Id") + if not isinstance(container_id, str) or _CONTAINER_ID.fullmatch(container_id) is None: + raise InstalledLabPackageRunnerError("Docker container id is invalid") + if document.get("Warnings") not in (None, []): + raise InstalledLabPackageRunnerError("Docker container creation returned warnings") + return container_id + + def _cleanup(self, container_id: str) -> BaseException | None: + last_error: BaseException | None = None + for _attempt in range(2): + try: + transport: httpx.BaseTransport = ( + httpx.HTTPTransport(uds=str(self.socket_path)) + if self.transport_factory is None + else self.transport_factory() + ) + with httpx.Client( + base_url="http://docker", + transport=transport, + timeout=httpx.Timeout(10.0, connect=5.0), + ) as client: + self._empty( + client, + "DELETE", + self._api_path(f"/containers/{container_id}?force=1&v=1"), + {204, 404}, + ) + return None + except (httpx.HTTPError, OSError, ValueError, InstalledLabPackageRunnerError) as exc: + last_error = exc + return last_error + + def _wait(self, client: httpx.Client, container_id: str) -> int: + response = self._response( + client, + "POST", + self._api_path(f"/containers/{container_id}/wait?condition=not-running"), + {200}, + ) + document = _response_object(response, "Docker container wait") + status_code = document.get("StatusCode") + if isinstance(status_code, bool) or not isinstance(status_code, int): + raise InstalledLabPackageRunnerError("Docker exit status is invalid") + if document.get("Error") not in (None, {"Message": ""}): + raise InstalledLabPackageRunnerError("Docker wait returned an Engine error") + return status_code + + def _log_identity(self, client: httpx.Client, container_id: str) -> tuple[str, int]: + response = self._response( + client, + "GET", + self._api_path(f"/containers/{container_id}/logs?stdout=1&stderr=1&tail=200"), + {200}, + ) + return hashlib.sha256(response.content).hexdigest(), len(response.content) + + def _api_path(self, path: str) -> str: + return f"/{self.api_version}{path}" + + @staticmethod + def _empty( + client: httpx.Client, + method: str, + path: str, + statuses: set[int], + ) -> None: + DockerEngineInstalledLabLauncher._response(client, method, path, statuses) + + @staticmethod + def _response( + client: httpx.Client, + method: str, + path: str, + statuses: set[int], + *, + json_body: Mapping[str, object] | None = None, + ) -> httpx.Response: + with client.stream(method, path, json=json_body) as streamed: + if streamed.status_code not in statuses: + raise InstalledLabPackageRunnerError( + f"Docker Engine rejected {method} {path.split('?')[0]} " + f"with status {streamed.status_code}" + ) + declared_length = streamed.headers.get("content-length") + if declared_length is not None: + try: + length = int(declared_length) + except ValueError as exc: + raise InstalledLabPackageRunnerError( + "Docker Engine response length is invalid" + ) from exc + if length < 0 or length > _MAX_ENGINE_RESPONSE_BYTES: + raise InstalledLabPackageRunnerError("Docker Engine response is too large") + payload = bytearray() + blocks = ( + (streamed.content,) + if streamed.is_stream_consumed + else streamed.iter_raw(chunk_size=16 * 1024) + ) + for block in blocks: + if len(payload) + len(block) > _MAX_ENGINE_RESPONSE_BYTES: + raise InstalledLabPackageRunnerError("Docker Engine response is too large") + payload.extend(block) + return httpx.Response( + status_code=streamed.status_code, + headers=streamed.headers, + content=bytes(payload), + request=streamed.request, + ) + + +@dataclass(frozen=True, slots=True) +class InstalledLabPackageProfileRunner: + package: InstalledLabPackage + definition: PortableRunDefinition + controller_work_root: Path + engine_work_root: str + local_assets: tuple[InstalledLabLocalAssetBinding, ...] + launcher: InstalledLabContainerLauncher + token_factory: Callable[[], str] = lambda: secrets.token_hex(8) + + def __post_init__(self) -> None: + _absolute_controller_path(self.controller_work_root, "Worker package root") + _engine_host_path(self.engine_work_root, "Worker package root") + asset_ids = tuple(binding.asset_id for binding in self.local_assets) + if asset_ids != tuple(sorted(asset_ids)) or len(asset_ids) != len(set(asset_ids)): + raise InstalledLabPackageRunnerError( + "installed LAB local assets must be canonical and unique" + ) + + def run( + self, + plan: PortableWorkerRuntimePlan, + source: PortableWorkerSourceStage, + ) -> PortableWorkerResultDraft: + if ( + plan.setup_id != self.package.setup_id + or plan.definition_sha256 != self.package.definition_sha256 + or plan.candidate_sha256 != self.package.runtime_candidate_sha256 + or plan.result_contract_sha256 != self.package.result_contract_sha256 + or plan.source_bundle_sha256 != source.source_bundle_sha256 + or plan.source_capability_manifest_sha256 != source.source_capability_manifest_sha256 + ): + raise InstalledLabPackageRunnerError( + "installed LAB run plan differs from its package or source" + ) + controller_root = _real_directory( + self.controller_work_root, + "Worker package root", + ) + source_root = _real_directory(source.root, "installed LAB source root") + _require_descendant(source_root, controller_root, "installed LAB source root") + jobs_root = controller_root / "jobs" + jobs_root.mkdir(mode=0o700, parents=True, exist_ok=True) + attempt_token = self.token_factory() + if re.fullmatch(r"[a-f0-9]{16}", attempt_token) is None: + raise InstalledLabPackageRunnerError("installed LAB attempt token is invalid") + job_root = jobs_root / plan.job_id / attempt_token + try: + job_root.mkdir(mode=0o700, parents=True, exist_ok=False) + output_root = job_root / "result-staging" + output_root.mkdir(mode=0o700) + steps_root = job_root / "steps" + steps_root.mkdir(mode=0o700) + plan_path = job_root / "run-plan.json" + _write_exclusive( + plan_path, + canonical_json( + { + "schema_version": INSTALLED_LAB_RUN_PLAN_SCHEMA, + "runtime_plan": plan.as_dict(), + "package_id": self.package.package_id, + "package_sha256": self.package.package_sha256, + "authority": self.definition.authority.as_dict(), + } + ), + ) + for container in _topological_containers(self.package): + container_output_root = output_root + if container.role == "step": + container_output_root = steps_root / container.container_id + container_output_root.mkdir(mode=0o700) + self.launcher( + self._launch( + container=container, + plan=plan, + source_root=source_root, + plan_path=plan_path, + output_root=container_output_root, + steps_root=steps_root, + name_token=attempt_token, + ) + ) + return _read_result_draft( + output_root, + plan=plan, + definition=self.definition, + ) + except FileExistsError as exc: + raise InstalledLabPackageRunnerError( + "installed LAB attempt directory already exists" + ) from exc + except OSError as exc: + shutil.rmtree(job_root, ignore_errors=True) + raise InstalledLabPackageRunnerError( + "installed LAB job workspace is unavailable" + ) from exc + except BaseException: + shutil.rmtree(job_root, ignore_errors=True) + raise + + def _launch( + self, + *, + container: InstalledLabContainer, + plan: PortableWorkerRuntimePlan, + source_root: Path, + plan_path: Path, + output_root: Path, + steps_root: Path, + name_token: str, + ) -> InstalledLabDockerLaunch: + by_asset = {binding.asset_id: binding for binding in self.local_assets} + mounts = [ + InstalledLabDockerMount( + _translate_work_path( + source_root, + controller_root=self.controller_work_root, + engine_root=self.engine_work_root, + ), + INSTALLED_LAB_SOURCE_ROOT, + True, + ), + InstalledLabDockerMount( + _translate_work_path( + plan_path, + controller_root=self.controller_work_root, + engine_root=self.engine_work_root, + ), + INSTALLED_LAB_PLAN_PATH, + True, + ), + InstalledLabDockerMount( + _translate_work_path( + output_root, + controller_root=self.controller_work_root, + engine_root=self.engine_work_root, + ), + INSTALLED_LAB_RESULT_ROOT, + False, + ), + ] + for dependency in _container_ancestors(self.package, container.container_id): + dependency_root = _real_directory( + steps_root / dependency, + "installed LAB dependency output", + ) + mounts.append( + InstalledLabDockerMount( + _translate_work_path( + dependency_root, + controller_root=self.controller_work_root, + engine_root=self.engine_work_root, + ), + f"{INSTALLED_LAB_STEP_INPUT_ROOT}/{dependency}", + True, + ) + ) + for package_mount in container.mounts: + binding = by_asset.get(package_mount.asset_id) + if binding is None or binding.engine_path is None: + raise InstalledLabPackageRunnerError( + "package file mount has no reviewed local binding" + ) + mounts.append( + InstalledLabDockerMount( + binding.engine_path, + package_mount.target, + True, + ) + ) + return InstalledLabDockerLaunch( + package_id=self.package.package_id, + container=container, + mounts=tuple(sorted(mounts, key=lambda item: item.container_path)), + labels={ + "com.nodedc.authority": "observation-only", + "com.nodedc.component": container.container_id, + "com.nodedc.definition-sha256": plan.definition_sha256, + "com.nodedc.job-id": plan.job_id, + "com.nodedc.managed-by": "mission-core-worker", + "com.nodedc.package-sha256": self.package.package_sha256, + "com.nodedc.product": "mission-core", + "com.nodedc.stack": "observatory", + }, + name_token=name_token, + ) + + +@dataclass(frozen=True, slots=True) +class InstalledLabPackageExecutorFactory: + """One generic factory for every independently installed package.""" + + local_assets: tuple[InstalledLabLocalAssetBinding, ...] + engine_work_root: str + launcher: InstalledLabContainerLauncher = DockerEngineInstalledLabLauncher() + + def __post_init__(self) -> None: + asset_ids = tuple(binding.asset_id for binding in self.local_assets) + if asset_ids != tuple(sorted(asset_ids)) or len(asset_ids) != len(set(asset_ids)): + raise InstalledLabPackageRunnerError( + "installed LAB factory assets must be canonical and unique" + ) + _engine_host_path(self.engine_work_root, "Worker Engine work root") + + def __call__( + self, + context: ObservatoryWorkerPackageExecutorBuildContext, + ) -> ObservatoryWorkerExecutorRegistration: + context.package.bind(context.definition, context.candidate) + by_asset = {binding.asset_id: binding for binding in self.local_assets} + package_assets = tuple( + by_asset[asset_id] for asset_id in context.package.asset_ids if asset_id in by_asset + ) + if tuple(binding.asset_id for binding in package_assets) != context.package.asset_ids: + raise InstalledLabPackageRunnerError( + "installed LAB package has an incomplete local asset map" + ) + admission = inspect_runtime_candidate( + context.candidate, + {binding.asset_id: binding.portable_binding() for binding in package_assets}, + ) + if not admission.ready: + blockers = ",".join(admission.blockers) + raise InstalledLabPackageRunnerError( + f"installed LAB package local assets are not admitted: {blockers}" + ) + runner = InstalledLabPackageProfileRunner( + package=context.package, + definition=context.definition, + controller_work_root=context.work_root, + engine_work_root=self.engine_work_root, + local_assets=package_assets, + launcher=self.launcher, + ) + adapter = PortableWorkerExecutorAdapter( + candidate=context.candidate, + definition=context.definition, + admission=admission, + source_materializer=context.source_transport, + runner=runner, + publisher=context.result_transport, + ) + return ObservatoryWorkerExecutorRegistration( + identity=context.package.executor_identity, + adapter=adapter, + ) + + +def _topological_containers( + package: InstalledLabPackage, +) -> tuple[InstalledLabContainer, ...]: + remaining = {container.container_id: container for container in package.containers} + completed: set[str] = set() + ordered: list[InstalledLabContainer] = [] + while remaining: + ready = tuple( + container + for container in remaining.values() + if set(container.depends_on).issubset(completed) + ) + if not ready: + raise InstalledLabPackageRunnerError("package container topology is blocked") + for container in sorted(ready, key=lambda item: item.container_id): + ordered.append(container) + completed.add(container.container_id) + del remaining[container.container_id] + return tuple(ordered) + + +def _container_ancestors( + package: InstalledLabPackage, + container_id: str, +) -> tuple[str, ...]: + by_id = {container.container_id: container for container in package.containers} + ancestors: set[str] = set() + + def collect(current: str) -> None: + for dependency in by_id[current].depends_on: + if dependency not in ancestors: + ancestors.add(dependency) + collect(dependency) + + collect(container_id) + return tuple(sorted(ancestors)) + + +def _container_create_document(launch: InstalledLabDockerLaunch) -> dict[str, object]: + container = launch.container + device_requests: list[dict[str, object]] = [] + if container.gpu_count: + device_requests.append( + { + "Driver": "nvidia", + "Count": container.gpu_count, + "Capabilities": [["gpu"]], + } + ) + return { + "Image": f"sha256:{container.image_sha256}", + "Cmd": list(container.argv), + "WorkingDir": INSTALLED_LAB_WORK_ROOT, + "Env": [ + "HF_HUB_OFFLINE=1", + "TRANSFORMERS_OFFLINE=1", + "PYTHONDONTWRITEBYTECODE=1", + ], + "Labels": dict(sorted(launch.labels.items())), + "NetworkDisabled": True, + "OpenStdin": False, + "StdinOnce": False, + "Tty": False, + "AttachStdout": True, + "AttachStderr": True, + "HostConfig": { + "AutoRemove": False, + "CapDrop": ["ALL"], + "DeviceRequests": device_requests, + "Init": True, + "IpcMode": "private", + "Memory": container.memory_bytes, + "MemorySwap": container.memory_bytes, + "Mounts": [mount.engine_document() for mount in launch.mounts], + "NanoCpus": container.nano_cpus, + "NetworkMode": "none", + "PidsLimit": container.pids_limit, + "Privileged": False, + "ReadonlyRootfs": True, + "SecurityOpt": ["no-new-privileges:true"], + "ShmSize": container.shm_bytes, + "Tmpfs": { + INSTALLED_LAB_WORK_ROOT: ( + f"rw,noexec,nosuid,nodev,size={container.tmpfs_bytes},mode=1777" + ), + "/tmp": "rw,noexec,nosuid,nodev,size=67108864,mode=1777", + }, + }, + } + + +def _read_result_draft( + root: Path, + *, + plan: PortableWorkerRuntimePlan, + definition: PortableRunDefinition, +) -> PortableWorkerResultDraft: + result_root = _real_directory(root, "installed LAB result root") + manifest_path = result_root / RESULT_PACKAGE_MANIFEST_NAME + try: + metadata = manifest_path.lstat() + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise InstalledLabPackageRunnerError( + "installed LAB result manifest is not a regular file" + ) + if not 0 < metadata.st_size <= _MAX_RESULT_MANIFEST_BYTES: + raise InstalledLabPackageRunnerError("installed LAB result manifest size is invalid") + payload = manifest_path.read_bytes() + manifest = PortableResultPackageManifest.from_bytes(payload) + except InstalledLabPackageRunnerError: + raise + except (OSError, PortableResultPackageIntegrityError) as exc: + raise InstalledLabPackageRunnerError("installed LAB result manifest is invalid") from exc + source = manifest.source + run_definition = manifest.run_definition + result = manifest.result + if ( + manifest.job.get("job_id") != plan.job_id + or source.get("bundle_sha256") != plan.source_bundle_sha256 + or source.get("capability_manifest_sha256") != plan.source_capability_manifest_sha256 + or run_definition.get("definition_sha256") != plan.definition_sha256 + or result.get("result_contract_sha256") != plan.result_contract_sha256 + or result.get("result_schema") != definition.result_contract.result_schema + or result.get("result_kind") != definition.result_contract.result_kind + or manifest.authority != definition.authority.as_dict() + ): + raise InstalledLabPackageRunnerError( + "installed LAB result package lost its sealed identity" + ) + result_id = result.get("result_id") + if not isinstance(result_id, str): + raise InstalledLabPackageRunnerError("installed LAB result id is invalid") + expected_files = {RESULT_PACKAGE_MANIFEST_NAME} + for artifact in manifest.artifacts: + relative = relative_artifact_path(artifact.relative_path) + artifact_path = result_root.joinpath(*relative.parts) + try: + metadata = artifact_path.lstat() + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise InstalledLabPackageRunnerError( + "installed LAB result artifact is not a regular file" + ) + if metadata.st_size != artifact.byte_length: + raise InstalledLabPackageRunnerError("installed LAB result artifact size changed") + if _sha256_file(artifact_path) != artifact.sha256: + raise InstalledLabPackageRunnerError("installed LAB result artifact digest changed") + except OSError as exc: + raise InstalledLabPackageRunnerError( + "installed LAB result artifact is unavailable" + ) from exc + expected_files.add(relative.as_posix()) + actual_files: set[str] = set() + for candidate in result_root.rglob("*"): + relative_name = candidate.relative_to(result_root).as_posix() + metadata = candidate.lstat() + if stat.S_ISLNK(metadata.st_mode): + raise InstalledLabPackageRunnerError("installed LAB result contains a link") + if stat.S_ISREG(metadata.st_mode): + actual_files.add(relative_name) + elif not stat.S_ISDIR(metadata.st_mode): + raise InstalledLabPackageRunnerError("installed LAB result contains a special file") + if actual_files != expected_files: + raise InstalledLabPackageRunnerError("installed LAB result contains undeclared files") + return PortableWorkerResultDraft( + root=result_root, + result_id=result_id, + result_sha256=manifest.manifest_sha256, + result_contract_sha256=plan.result_contract_sha256, + ) + + +def _translate_work_path( + path: Path, + *, + controller_root: Path, + engine_root: str, +) -> str: + candidate = path.expanduser().absolute() + root = controller_root.expanduser().absolute() + try: + relative = candidate.relative_to(root) + except ValueError as exc: + raise InstalledLabPackageRunnerError( + "Worker path is outside the Docker-host work binding" + ) from exc + if _WINDOWS_ABSOLUTE_PATH.match(engine_root): + suffix = "\\".join(relative.parts) + return engine_root.rstrip("\\") + (f"\\{suffix}" if suffix else "") + suffix = "/".join(relative.parts) + return engine_root.rstrip("/") + (f"/{suffix}" if suffix else "") + + +def _require_descendant(path: Path, root: Path, label: str) -> None: + if path == root or not path.is_relative_to(root): + raise InstalledLabPackageRunnerError(f"{label} is outside the Worker package root") + + +def _absolute_controller_path(path: Path, label: str) -> None: + if not path.is_absolute(): + raise ValueError(f"{label} controller path must be absolute") + + +def _engine_host_path(value: str, label: str) -> None: + if ( + not isinstance(value, str) + or not value + or value != value.strip() + or "\x00" in value + or "\n" in value + or "\r" in value + or _ENGINE_PATH_PLACEHOLDER.search(value) is not None + or ".." in PurePosixPath(value.replace("\\", "/")).parts + or not (value.startswith("/") or _WINDOWS_ABSOLUTE_PATH.match(value)) + ): + raise ValueError(f"{label} Engine path is invalid") + + +def _real_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise InstalledLabPackageRunnerError(f"{label} is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise InstalledLabPackageRunnerError(f"{label} is not a real directory") + return resolved + + +def _write_exclusive(path: Path, payload: bytes) -> None: + with path.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _response_object(response: httpx.Response, label: str) -> dict[str, object]: + try: + document: object = response.json() + except ValueError as exc: + raise InstalledLabPackageRunnerError(f"{label} is not JSON") from exc + if not isinstance(document, dict) or any(not isinstance(key, str) for key in document): + raise InstalledLabPackageRunnerError(f"{label} is not an object") + return document + + +def _require_local_socket(path: Path) -> None: + try: + metadata = path.stat() + except OSError as exc: + raise InstalledLabPackageRunnerError("local Docker socket is unavailable") from exc + if not stat.S_ISSOCK(metadata.st_mode): + raise InstalledLabPackageRunnerError("local Docker socket is not a socket") diff --git a/src/k1link/observatory/installed_lab_packages.py b/src/k1link/observatory/installed_lab_packages.py new file mode 100644 index 0000000..a86c58e --- /dev/null +++ b/src/k1link/observatory/installed_lab_packages.py @@ -0,0 +1,721 @@ +"""Worker-local, content-addressed execution packages for portable LABs. + +Mission Core seals data identities; it never sends executable instructions. +This module defines the separately installed Worker package that is allowed to +contain reviewed container argv and in-container mount targets. A package is +bound to one exact RunDefinition and runtime candidate and uses one stable I/O +surface, so adding a conforming LAB does not require another Worker micro-app. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, Literal + +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerRuntimeCandidate, + PortableWorkerRuntimeRegistry, +) +from k1link.observatory.recorded_jobs import RecordedExecutorIdentity + +INSTALLED_LAB_PACKAGE_SCHEMA: Final = "missioncore.observatory-installed-lab-package/v1" +INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA: Final = ( + "missioncore.observatory-installed-lab-package-registry/v1" +) +INSTALLED_LAB_CONTAINER_IO_SCHEMA: Final = "missioncore.observatory-installed-lab-container-io/v2" + +INSTALLED_LAB_SOURCE_ROOT: Final = "/missioncore/input/source" +INSTALLED_LAB_PLAN_PATH: Final = "/missioncore/input/run-plan.json" +INSTALLED_LAB_STEP_INPUT_ROOT: Final = "/missioncore/input/steps" +INSTALLED_LAB_RESULT_ROOT: Final = "/missioncore/output" +INSTALLED_LAB_WORK_ROOT: Final = "/missioncore/work" + +_MAX_REGISTRY_BYTES: Final = 1024 * 1024 +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_ASSET_ID = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_FORBIDDEN_KEYS = frozenset( + { + "host_path", + "host-path", + "hostpath", + "secret", + "secrets", + "token", + "password", + "privileged", + "docker_socket", + } +) +_AUTHORITY: Final = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} + +type InstalledLabExecutionMode = Literal["single-container", "fixed-stack"] +type InstalledLabContainerRole = Literal["step", "result-writer"] +type InstalledLabNetworkMode = Literal["none"] + + +class InstalledLabPackageError(RuntimeError): + """An installed LAB package is malformed or not exactly bound.""" + + +@dataclass(frozen=True, slots=True) +class InstalledLabPackageMount: + """Read-only asset mount; its Worker-local source path is not serialized.""" + + asset_id: str + target: str + read_only: Literal[True] = True + + def __post_init__(self) -> None: + _pattern(self.asset_id, _ASSET_ID, "package mount asset id") + target = PurePosixPath(self.target) + reserved_roots = tuple( + PurePosixPath(value) + for value in ( + INSTALLED_LAB_SOURCE_ROOT, + INSTALLED_LAB_STEP_INPUT_ROOT, + INSTALLED_LAB_RESULT_ROOT, + INSTALLED_LAB_WORK_ROOT, + ) + ) + if ( + not target.is_absolute() + or ".." in target.parts + or target == PurePosixPath(INSTALLED_LAB_PLAN_PATH) + or any(target == root or target.is_relative_to(root) for root in reserved_roots) + ): + raise InstalledLabPackageError("package asset mount target is unsafe") + if self.read_only is not True: + raise InstalledLabPackageError("package assets must remain read-only") + + def as_dict(self) -> dict[str, object]: + return { + "asset_id": self.asset_id, + "target": self.target, + "read_only": True, + } + + +@dataclass(frozen=True, slots=True) +class InstalledLabContainer: + container_id: str + role: InstalledLabContainerRole + image_sha256: str + argv: tuple[str, ...] + depends_on: tuple[str, ...] + mounts: tuple[InstalledLabPackageMount, ...] + network: InstalledLabNetworkMode + gpu_count: int + memory_bytes: int + nano_cpus: int + pids_limit: int + shm_bytes: int + tmpfs_bytes: int + timeout_seconds: int + + def __post_init__(self) -> None: + _pattern(self.container_id, _IDENTIFIER, "package container id") + _digest(self.image_sha256, "package container image sha256") + if self.role not in ("step", "result-writer"): + raise InstalledLabPackageError("package container role is invalid") + if not self.argv or len(self.argv) > 64: + raise InstalledLabPackageError("package container argv is invalid") + for argument in self.argv: + if ( + not isinstance(argument, str) + or not argument + or len(argument) > 1_024 + or "\x00" in argument + or "\n" in argument + or "\r" in argument + or "${" in argument + ): + raise InstalledLabPackageError("package container argument is unsafe") + if self.depends_on != tuple(sorted(self.depends_on)) or len(self.depends_on) != len( + set(self.depends_on) + ): + raise InstalledLabPackageError("package container dependencies must be canonical") + for dependency in self.depends_on: + _pattern(dependency, _IDENTIFIER, "package container dependency") + mount_keys = [(mount.target, mount.asset_id) for mount in self.mounts] + if mount_keys != sorted(mount_keys) or len(mount_keys) != len(set(mount_keys)): + raise InstalledLabPackageError("package mounts must be canonical and unique") + if self.network != "none": + raise InstalledLabPackageError("package container network mode is invalid") + if isinstance(self.gpu_count, bool) or not 0 <= self.gpu_count <= 8: + raise InstalledLabPackageError("package container GPU count is invalid") + for value, minimum, maximum, label in ( + (self.memory_bytes, 64 * 1024**2, 512 * 1024**3, "memory"), + (self.nano_cpus, 100_000_000, 128_000_000_000, "CPU"), + (self.pids_limit, 16, 4_096, "PID"), + (self.shm_bytes, 64 * 1024**2, 128 * 1024**3, "shared-memory"), + (self.tmpfs_bytes, 64 * 1024**2, 64 * 1024**3, "tmpfs"), + ): + if isinstance(value, bool) or not minimum <= value <= maximum: + raise InstalledLabPackageError(f"package container {label} limit is invalid") + if isinstance(self.timeout_seconds, bool) or not 1 <= self.timeout_seconds <= 24 * 60 * 60: + raise InstalledLabPackageError("package container timeout is invalid") + + def as_dict(self) -> dict[str, object]: + return { + "container_id": self.container_id, + "role": self.role, + "image_sha256": self.image_sha256, + "argv": list(self.argv), + "depends_on": list(self.depends_on), + "mounts": [mount.as_dict() for mount in self.mounts], + "network": self.network, + "gpu_count": self.gpu_count, + "memory_bytes": self.memory_bytes, + "nano_cpus": self.nano_cpus, + "pids_limit": self.pids_limit, + "shm_bytes": self.shm_bytes, + "tmpfs_bytes": self.tmpfs_bytes, + "timeout_seconds": self.timeout_seconds, + } + + +@dataclass(frozen=True, slots=True) +class InstalledLabPackage: + package_id: str + package_version: int + package_sha256: str + setup_id: str + definition_id: str + definition_version: int + definition_sha256: str + runtime_candidate_sha256: str + source_adapter_sha256: str + result_contract_sha256: str + executor_identity: RecordedExecutorIdentity + execution_mode: InstalledLabExecutionMode + asset_ids: tuple[str, ...] + containers: tuple[InstalledLabContainer, ...] + + def __post_init__(self) -> None: + for value, label in ( + (self.package_id, "package id"), + (self.setup_id, "package setup id"), + (self.definition_id, "package definition id"), + ): + _pattern(value, _IDENTIFIER, label) + for value, label in ( + (self.package_sha256, "package sha256"), + (self.definition_sha256, "package definition sha256"), + (self.runtime_candidate_sha256, "runtime candidate sha256"), + (self.source_adapter_sha256, "source adapter sha256"), + (self.result_contract_sha256, "result contract sha256"), + ): + _digest(value, label) + for numeric_value, label in ( + (self.package_version, "package version"), + (self.definition_version, "package definition version"), + ): + if ( + isinstance(numeric_value, bool) + or not isinstance(numeric_value, int) + or numeric_value < 1 + ): + raise InstalledLabPackageError(f"{label} is invalid") + if self.execution_mode not in ("single-container", "fixed-stack"): + raise InstalledLabPackageError("package execution mode is invalid") + if self.asset_ids != tuple(sorted(self.asset_ids)) or len(self.asset_ids) != len( + set(self.asset_ids) + ): + raise InstalledLabPackageError("package asset ids must be canonical") + for asset_id in self.asset_ids: + _pattern(asset_id, _ASSET_ID, "package asset id") + container_ids = tuple(container.container_id for container in self.containers) + if ( + not container_ids + or container_ids != tuple(sorted(container_ids)) + or len(container_ids) != len(set(container_ids)) + ): + raise InstalledLabPackageError("package containers must be canonical and unique") + writers = tuple( + container for container in self.containers if container.role == "result-writer" + ) + if len(writers) != 1: + raise InstalledLabPackageError("package requires exactly one result writer") + if self.execution_mode == "single-container" and ( + len(self.containers) != 1 or self.containers[0].role != "result-writer" + ): + raise InstalledLabPackageError( + "single-container package must contain one result writer" + ) + if self.execution_mode == "fixed-stack" and len(self.containers) < 2: + raise InstalledLabPackageError("fixed-stack package requires multiple containers") + self._verify_topology() + mounted_assets = { + mount.asset_id for container in self.containers for mount in container.mounts + } + if not mounted_assets.issubset(set(self.asset_ids)): + raise InstalledLabPackageError("package mounts reference undeclared assets") + if self.package_sha256 != canonical_sha256(self.identity_document()): + raise InstalledLabPackageError("installed LAB package digest changed") + + def _verify_topology(self) -> None: + by_id = {container.container_id: container for container in self.containers} + for container in self.containers: + if container.container_id in container.depends_on or any( + dependency not in by_id for dependency in container.depends_on + ): + raise InstalledLabPackageError("package dependency is invalid") + visiting: set[str] = set() + visited: set[str] = set() + + def visit(container_id: str) -> None: + if container_id in visiting: + raise InstalledLabPackageError("package container graph contains a cycle") + if container_id in visited: + return + visiting.add(container_id) + for dependency in by_id[container_id].depends_on: + visit(dependency) + visiting.remove(container_id) + visited.add(container_id) + + for container_id in by_id: + visit(container_id) + writer = next( + container for container in self.containers if container.role == "result-writer" + ) + writer_ancestors: set[str] = set() + + def collect(container_id: str) -> None: + for dependency in by_id[container_id].depends_on: + if dependency not in writer_ancestors: + writer_ancestors.add(dependency) + collect(dependency) + + collect(writer.container_id) + steps = { + container.container_id for container in self.containers if container.role == "step" + } + if writer_ancestors != steps: + raise InstalledLabPackageError( + "package result writer must depend on every execution step" + ) + + def identity_document(self) -> dict[str, object]: + return _package_identity_document( + package_id=self.package_id, + package_version=self.package_version, + setup_id=self.setup_id, + definition_id=self.definition_id, + definition_version=self.definition_version, + definition_sha256=self.definition_sha256, + runtime_candidate_sha256=self.runtime_candidate_sha256, + source_adapter_sha256=self.source_adapter_sha256, + result_contract_sha256=self.result_contract_sha256, + executor_identity=self.executor_identity, + execution_mode=self.execution_mode, + asset_ids=self.asset_ids, + containers=self.containers, + ) + + def bind( + self, + definition: PortableRunDefinition, + candidate: PortableWorkerRuntimeCandidate, + ) -> None: + candidate.bind_definition(definition) + if not candidate.ready: + raise InstalledLabPackageError("installed package binds a blocked runtime") + if ( + self.setup_id != definition.setup_id + or self.definition_id != definition.definition_id + or self.definition_version != definition.version + or self.definition_sha256 != definition.definition_sha256 + or self.runtime_candidate_sha256 != candidate.candidate_sha256 + or self.source_adapter_sha256 != definition.source_adapter.contract_sha256 + or self.result_contract_sha256 != definition.result_contract.contract_sha256 + or self.executor_identity != candidate.executor_identity() + ): + raise InstalledLabPackageError( + "installed package differs from its definition or runtime candidate" + ) + candidate_assets = {asset.asset_id: asset for asset in candidate.reusable_assets} + if set(self.asset_ids) != set(candidate_assets): + raise InstalledLabPackageError( + "installed package asset inventory differs from its runtime candidate" + ) + admitted_images = { + asset.sha256 for asset in candidate.reusable_assets if asset.kind == "container-image" + } + if candidate.executor is not None: + admitted_images.add(candidate.executor.image_sha256) + if any(container.image_sha256 not in admitted_images for container in self.containers): + raise InstalledLabPackageError("package container image is not runtime-admitted") + for container in self.containers: + for mount in container.mounts: + if candidate_assets[mount.asset_id].kind == "container-image": + raise InstalledLabPackageError("container images cannot be mounted as files") + + +@dataclass(frozen=True, slots=True) +class InstalledLabPackageRegistry: + packages: tuple[InstalledLabPackage, ...] + + def __post_init__(self) -> None: + if not self.packages: + raise InstalledLabPackageError("installed LAB package registry is empty") + for values, label in ( + ([package.package_id for package in self.packages], "package ids"), + ([package.package_sha256 for package in self.packages], "package digests"), + ( + [(package.setup_id, package.definition_sha256) for package in self.packages], + "package definition bindings", + ), + ): + if len(values) != len(set(values)): + raise InstalledLabPackageError(f"installed LAB {label} must be unique") + + @classmethod + def from_file( + cls, + path: Path, + *, + definitions: PortableRunDefinitionRegistry, + runtime_registry: PortableWorkerRuntimeRegistry, + ) -> InstalledLabPackageRegistry: + candidate_path = path.expanduser().absolute() + try: + if candidate_path.is_symlink() or not candidate_path.is_file(): + raise InstalledLabPackageError( + "installed LAB package registry must be a regular file" + ) + payload = candidate_path.read_bytes() + if not 0 < len(payload) <= _MAX_REGISTRY_BYTES: + raise InstalledLabPackageError("installed LAB package registry size is invalid") + document: object = json.loads(payload.decode("utf-8")) + except InstalledLabPackageError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise InstalledLabPackageError("installed LAB package registry is unreadable") from exc + _reject_forbidden_keys(document) + root = _object(document, "installed LAB package registry") + _exact_keys(root, {"schema_version", "packages"}, "installed LAB package registry") + if root["schema_version"] != INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA: + raise InstalledLabPackageError("installed LAB package registry schema is invalid") + registry = cls(tuple(_package(value) for value in _array(root["packages"], "packages"))) + for package in registry.packages: + definition = definitions.resolve(package.setup_id, package.definition_sha256) + candidate = runtime_registry.resolve( + package.setup_id, + package.definition_sha256, + ) + package.bind(definition, candidate) + return registry + + def resolve(self, setup_id: str, definition_sha256: str) -> InstalledLabPackage: + for package in self.packages: + if package.setup_id == setup_id and package.definition_sha256 == definition_sha256: + return package + raise InstalledLabPackageError("installed LAB package is unavailable") + + +def seal_installed_lab_package( + *, + package_id: str, + package_version: int, + setup_id: str, + definition_id: str, + definition_version: int, + definition_sha256: str, + runtime_candidate_sha256: str, + source_adapter_sha256: str, + result_contract_sha256: str, + executor_identity: RecordedExecutorIdentity, + execution_mode: InstalledLabExecutionMode, + asset_ids: tuple[str, ...], + containers: tuple[InstalledLabContainer, ...], +) -> InstalledLabPackage: + canonical_asset_ids = tuple(sorted(asset_ids)) + canonical_containers = tuple(sorted(containers, key=lambda value: value.container_id)) + package_sha256 = canonical_sha256( + _package_identity_document( + package_id=package_id, + package_version=package_version, + setup_id=setup_id, + definition_id=definition_id, + definition_version=definition_version, + definition_sha256=definition_sha256, + runtime_candidate_sha256=runtime_candidate_sha256, + source_adapter_sha256=source_adapter_sha256, + result_contract_sha256=result_contract_sha256, + executor_identity=executor_identity, + execution_mode=execution_mode, + asset_ids=canonical_asset_ids, + containers=canonical_containers, + ) + ) + return InstalledLabPackage( + package_id=package_id, + package_version=package_version, + package_sha256=package_sha256, + setup_id=setup_id, + definition_id=definition_id, + definition_version=definition_version, + definition_sha256=definition_sha256, + runtime_candidate_sha256=runtime_candidate_sha256, + source_adapter_sha256=source_adapter_sha256, + result_contract_sha256=result_contract_sha256, + executor_identity=executor_identity, + execution_mode=execution_mode, + asset_ids=canonical_asset_ids, + containers=canonical_containers, + ) + + +def _package_identity_document( + *, + package_id: str, + package_version: int, + setup_id: str, + definition_id: str, + definition_version: int, + definition_sha256: str, + runtime_candidate_sha256: str, + source_adapter_sha256: str, + result_contract_sha256: str, + executor_identity: RecordedExecutorIdentity, + execution_mode: InstalledLabExecutionMode, + asset_ids: tuple[str, ...], + containers: tuple[InstalledLabContainer, ...], +) -> dict[str, object]: + return { + "schema_version": INSTALLED_LAB_PACKAGE_SCHEMA, + "package_id": package_id, + "package_version": package_version, + "setup_id": setup_id, + "definition_id": definition_id, + "definition_version": definition_version, + "definition_sha256": definition_sha256, + "runtime_candidate_sha256": runtime_candidate_sha256, + "source_adapter_sha256": source_adapter_sha256, + "result_contract_sha256": result_contract_sha256, + "executor_identity": executor_identity.as_dict(), + "container_io": { + "schema_version": INSTALLED_LAB_CONTAINER_IO_SCHEMA, + "source_root": INSTALLED_LAB_SOURCE_ROOT, + "plan_path": INSTALLED_LAB_PLAN_PATH, + "step_input_root": INSTALLED_LAB_STEP_INPUT_ROOT, + "result_root": INSTALLED_LAB_RESULT_ROOT, + "work_root": INSTALLED_LAB_WORK_ROOT, + }, + "execution_mode": execution_mode, + "asset_ids": list(asset_ids), + "containers": [container.as_dict() for container in containers], + "authority": dict(_AUTHORITY), + } + + +def _package(value: object) -> InstalledLabPackage: + row = _object(value, "installed LAB package") + _exact_keys( + row, + { + "schema_version", + "package_id", + "package_version", + "package_sha256", + "setup_id", + "definition_id", + "definition_version", + "definition_sha256", + "runtime_candidate_sha256", + "source_adapter_sha256", + "result_contract_sha256", + "executor_identity", + "container_io", + "execution_mode", + "asset_ids", + "containers", + "authority", + }, + "installed LAB package", + ) + if row["schema_version"] != INSTALLED_LAB_PACKAGE_SCHEMA: + raise InstalledLabPackageError("installed LAB package schema is invalid") + if row["authority"] != _AUTHORITY: + raise InstalledLabPackageError("installed LAB package authority changed") + if row["container_io"] != { + "schema_version": INSTALLED_LAB_CONTAINER_IO_SCHEMA, + "source_root": INSTALLED_LAB_SOURCE_ROOT, + "plan_path": INSTALLED_LAB_PLAN_PATH, + "step_input_root": INSTALLED_LAB_STEP_INPUT_ROOT, + "result_root": INSTALLED_LAB_RESULT_ROOT, + "work_root": INSTALLED_LAB_WORK_ROOT, + }: + raise InstalledLabPackageError("installed LAB container I/O contract changed") + executor = _object(row["executor_identity"], "executor identity") + _exact_keys( + executor, + { + "release_sha256", + "image_sha256", + "model_manifest_sha256", + "resource_profile_sha256", + }, + "executor identity", + ) + mode = row["execution_mode"] + if mode not in ("single-container", "fixed-stack"): + raise InstalledLabPackageError("installed LAB execution mode is invalid") + return InstalledLabPackage( + package_id=_string(row["package_id"], "package id"), + package_version=_integer(row["package_version"], "package version"), + package_sha256=_string(row["package_sha256"], "package sha256"), + setup_id=_string(row["setup_id"], "setup id"), + definition_id=_string(row["definition_id"], "definition id"), + definition_version=_integer(row["definition_version"], "definition version"), + definition_sha256=_string(row["definition_sha256"], "definition sha256"), + runtime_candidate_sha256=_string( + row["runtime_candidate_sha256"], "runtime candidate sha256" + ), + source_adapter_sha256=_string(row["source_adapter_sha256"], "source adapter sha256"), + result_contract_sha256=_string(row["result_contract_sha256"], "result contract sha256"), + executor_identity=RecordedExecutorIdentity( + release_sha256=_string(executor["release_sha256"], "executor release sha256"), + image_sha256=_string(executor["image_sha256"], "executor image sha256"), + model_manifest_sha256=_string( + executor["model_manifest_sha256"], "model manifest sha256" + ), + resource_profile_sha256=_string( + executor["resource_profile_sha256"], "resource profile sha256" + ), + ), + execution_mode=mode, + asset_ids=tuple( + _string(item, "package asset id") + for item in _array(row["asset_ids"], "package asset ids") + ), + containers=tuple( + _container(item) for item in _array(row["containers"], "package containers") + ), + ) + + +def _container(value: object) -> InstalledLabContainer: + row = _object(value, "package container") + _exact_keys( + row, + { + "container_id", + "role", + "image_sha256", + "argv", + "depends_on", + "mounts", + "network", + "gpu_count", + "memory_bytes", + "nano_cpus", + "pids_limit", + "shm_bytes", + "tmpfs_bytes", + "timeout_seconds", + }, + "package container", + ) + role = row["role"] + network = row["network"] + if role not in ("step", "result-writer") or network != "none": + raise InstalledLabPackageError("package container enum is invalid") + return InstalledLabContainer( + container_id=_string(row["container_id"], "package container id"), + role=role, + image_sha256=_string(row["image_sha256"], "package image sha256"), + argv=tuple( + _string(item, "package argument") for item in _array(row["argv"], "package argv") + ), + depends_on=tuple( + _string(item, "package dependency") + for item in _array(row["depends_on"], "package dependencies") + ), + mounts=tuple(_mount(item) for item in _array(row["mounts"], "package mounts")), + network=network, + gpu_count=_integer(row["gpu_count"], "package GPU count"), + memory_bytes=_integer(row["memory_bytes"], "package memory limit"), + nano_cpus=_integer(row["nano_cpus"], "package CPU limit"), + pids_limit=_integer(row["pids_limit"], "package PID limit"), + shm_bytes=_integer(row["shm_bytes"], "package shared-memory limit"), + tmpfs_bytes=_integer(row["tmpfs_bytes"], "package tmpfs limit"), + timeout_seconds=_integer(row["timeout_seconds"], "package timeout"), + ) + + +def _mount(value: object) -> InstalledLabPackageMount: + row = _object(value, "package mount") + _exact_keys(row, {"asset_id", "target", "read_only"}, "package mount") + if row["read_only"] is not True: + raise InstalledLabPackageError("package mount must be read-only") + return InstalledLabPackageMount( + asset_id=_string(row["asset_id"], "package mount asset id"), + target=_string(row["target"], "package mount target"), + ) + + +def _reject_forbidden_keys(value: object) -> None: + if isinstance(value, dict): + for key, child in value.items(): + if not isinstance(key, str) or key.lower() in _FORBIDDEN_KEYS: + raise InstalledLabPackageError( + "installed LAB package contains a forbidden host or secret input" + ) + _reject_forbidden_keys(child) + elif isinstance(value, list): + for child in value: + _reject_forbidden_keys(child) + + +def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise InstalledLabPackageError(f"{label} is invalid") + + +def _digest(value: str, label: str) -> None: + _pattern(value, _SHA256, label) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise InstalledLabPackageError(f"{label} must be an object") + return value + + +def _array(value: object, label: str) -> list[object]: + if not isinstance(value, list): + raise InstalledLabPackageError(f"{label} must be an array") + return value + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str): + raise InstalledLabPackageError(f"{label} must be a string") + return value + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise InstalledLabPackageError(f"{label} must be an integer") + return value + + +def _exact_keys(row: dict[str, object], expected: set[str], label: str) -> None: + if set(row) != expected: + raise InstalledLabPackageError(f"{label} keys are invalid") diff --git a/src/k1link/observatory/installed_lab_worker_container_main.py b/src/k1link/observatory/installed_lab_worker_container_main.py new file mode 100644 index 0000000..2411f1f --- /dev/null +++ b/src/k1link/observatory/installed_lab_worker_container_main.py @@ -0,0 +1,21 @@ +"""Container entrypoint for the generic installed-LAB Worker agent.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from k1link.observatory import installed_lab_worker_service +from k1link.observatory.worker_container_proxy import ( + FixedObservatoryContainerLoopbackProxy, +) + + +def main(arguments: Sequence[str] | None = None) -> int: + """Run the fixed loopback bridge around the package-driven service.""" + + with FixedObservatoryContainerLoopbackProxy(): + return installed_lab_worker_service.main(arguments) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/k1link/observatory/installed_lab_worker_service.py b/src/k1link/observatory/installed_lab_worker_service.py new file mode 100644 index 0000000..b188ab9 --- /dev/null +++ b/src/k1link/observatory/installed_lab_worker_service.py @@ -0,0 +1,466 @@ +"""Generic Worker entrypoint for independently installed LAB packages. + +The environment selects only immutable registry files, one reviewed local +asset-binding file, the Worker work roots, and the existing transport settings. +Executable images, argv, topology, and limits live in the content-addressed +package registry and cannot arrive through a queued job. +""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import stat +import sys +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from threading import Event +from types import FrameType +from typing import Final, cast + +from k1link.observatory.installed_lab_package_runner import ( + DockerEngineInstalledLabLauncher, + InstalledLabLocalAssetBinding, + InstalledLabPackageExecutorFactory, +) +from k1link.observatory.installed_lab_packages import InstalledLabPackageRegistry +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerResultDraft, + PortableWorkerRuntimeRegistry, + PortableWorkerSourceStage, +) +from k1link.observatory.worker_agent import ( + ObservatoryWorkerExecutionResult, + SealedObservatoryRecordedJob, +) +from k1link.observatory.worker_service import ( + OBSERVATORY_WORKER_WORK_ROOT_ENV, + InstalledObservatoryWorkerService, + ObservatoryWorkerServiceConfiguration, + build_ready_executor_registry_from_packages, + compose_installed_observatory_worker_service_from_packages, +) + +INSTALLED_LAB_ASSET_BINDINGS_SCHEMA: Final = ( + "missioncore.observatory-installed-lab-asset-bindings/v1" +) +INSTALLED_LAB_VALIDATION_RECEIPT_SCHEMA: Final = ( + "missioncore.observatory-installed-lab-worker-validation/v1" +) +INSTALLED_LAB_DEFINITIONS_FILE_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE" +) +INSTALLED_LAB_RUNTIME_REGISTRY_FILE_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE" +) +INSTALLED_LAB_PACKAGE_REGISTRY_FILE_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE" +) +INSTALLED_LAB_ASSET_BINDINGS_FILE_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE" +) +_MAX_BINDINGS_BYTES: Final = 1024 * 1024 + + +class InstalledLabWorkerCompositionError(RuntimeError): + """The generic installed-LAB Worker configuration is not exact.""" + + +@dataclass(frozen=True, slots=True) +class InstalledLabWorkerAssetBindings: + engine_work_root: str + assets: tuple[InstalledLabLocalAssetBinding, ...] + + def __post_init__(self) -> None: + asset_ids = tuple(asset.asset_id for asset in self.assets) + if asset_ids != tuple(sorted(asset_ids)) or len(asset_ids) != len(set(asset_ids)): + raise InstalledLabWorkerCompositionError( + "installed LAB asset bindings must be canonical and unique" + ) + + @classmethod + def from_file(cls, path: Path) -> InstalledLabWorkerAssetBindings: + document = _read_object(path, "installed LAB asset bindings") + _exact_keys( + document, + {"schema_version", "engine_work_root", "assets"}, + "installed LAB asset bindings", + ) + if document["schema_version"] != INSTALLED_LAB_ASSET_BINDINGS_SCHEMA: + raise InstalledLabWorkerCompositionError( + "installed LAB asset binding schema changed" + ) + values = document["assets"] + if not isinstance(values, list) or not values: + raise InstalledLabWorkerCompositionError( + "installed LAB asset binding inventory is empty" + ) + assets = tuple( + sorted((_asset_binding(value) for value in values), key=lambda item: item.asset_id) + ) + return cls( + engine_work_root=_text(document["engine_work_root"], "engine work root"), + assets=assets, + ) + + +@dataclass(frozen=True, slots=True) +class InstalledLabWorkerEntrypointConfiguration: + worker: ObservatoryWorkerServiceConfiguration + definitions_file: Path + runtime_registry_file: Path + package_registry_file: Path + asset_bindings_file: Path + + @classmethod + def from_environment( + cls, + environment: Mapping[str, str] | None = None, + ) -> InstalledLabWorkerEntrypointConfiguration: + values = os.environ if environment is None else environment + return cls( + worker=ObservatoryWorkerServiceConfiguration.from_environment(values), + definitions_file=_required_path(values, INSTALLED_LAB_DEFINITIONS_FILE_ENV), + runtime_registry_file=_required_path( + values, + INSTALLED_LAB_RUNTIME_REGISTRY_FILE_ENV, + ), + package_registry_file=_required_path( + values, + INSTALLED_LAB_PACKAGE_REGISTRY_FILE_ENV, + ), + asset_bindings_file=_required_path( + values, + INSTALLED_LAB_ASSET_BINDINGS_FILE_ENV, + ), + ) + + +@dataclass(frozen=True, slots=True) +class InstalledLabWorkerValidationConfiguration: + work_root: Path + definitions_file: Path + runtime_registry_file: Path + package_registry_file: Path + asset_bindings_file: Path + + @classmethod + def from_environment( + cls, + environment: Mapping[str, str] | None = None, + ) -> InstalledLabWorkerValidationConfiguration: + values = os.environ if environment is None else environment + return cls( + work_root=_required_path(values, OBSERVATORY_WORKER_WORK_ROOT_ENV), + definitions_file=_required_path( + values, + INSTALLED_LAB_DEFINITIONS_FILE_ENV, + ), + runtime_registry_file=_required_path( + values, + INSTALLED_LAB_RUNTIME_REGISTRY_FILE_ENV, + ), + package_registry_file=_required_path( + values, + INSTALLED_LAB_PACKAGE_REGISTRY_FILE_ENV, + ), + asset_bindings_file=_required_path( + values, + INSTALLED_LAB_ASSET_BINDINGS_FILE_ENV, + ), + ) + + +def compose_installed_lab_worker_service( + configuration: InstalledLabWorkerEntrypointConfiguration, +) -> InstalledObservatoryWorkerService: + definitions = PortableRunDefinitionRegistry.from_file(configuration.definitions_file) + runtime = PortableWorkerRuntimeRegistry.from_file( + configuration.runtime_registry_file, + definitions=definitions, + ) + packages = InstalledLabPackageRegistry.from_file( + configuration.package_registry_file, + definitions=definitions, + runtime_registry=runtime, + ) + bindings = InstalledLabWorkerAssetBindings.from_file(configuration.asset_bindings_file) + return compose_installed_observatory_worker_service_from_packages( + configuration=configuration.worker, + definitions=definitions, + runtime_registry=runtime, + packages=packages, + executor_factory=InstalledLabPackageExecutorFactory( + local_assets=bindings.assets, + engine_work_root=bindings.engine_work_root, + ), + ) + + +def validate_installed_lab_worker( + configuration: InstalledLabWorkerValidationConfiguration, + *, + launcher: DockerEngineInstalledLabLauncher | None = None, +) -> dict[str, object]: + """Validate one installed package set without credentials or queue access.""" + + work_root = _real_directory(configuration.work_root, "installed LAB work root") + definitions = PortableRunDefinitionRegistry.from_file(configuration.definitions_file) + runtime = PortableWorkerRuntimeRegistry.from_file( + configuration.runtime_registry_file, + definitions=definitions, + ) + packages = InstalledLabPackageRegistry.from_file( + configuration.package_registry_file, + definitions=definitions, + runtime_registry=runtime, + ) + bindings = InstalledLabWorkerAssetBindings.from_file(configuration.asset_bindings_file) + image_sha256s = tuple( + sorted( + { + container.image_sha256 + for package in packages.packages + for container in package.containers + } + ) + ) + image_launcher = DockerEngineInstalledLabLauncher() if launcher is None else launcher + image_launcher.verify_images(image_sha256s) + offline = _OfflineWorkerBoundary() + executors = build_ready_executor_registry_from_packages( + definitions=definitions, + runtime_registry=runtime, + packages=packages, + executor_factory=InstalledLabPackageExecutorFactory( + local_assets=bindings.assets, + engine_work_root=bindings.engine_work_root, + launcher=image_launcher, + ), + source_transport=offline, + result_transport=offline, + work_root=work_root, + ) + expected_identities = tuple( + sorted(package.executor_identity for package in packages.packages) + ) + if executors.supported_identities != expected_identities: + raise InstalledLabWorkerCompositionError( + "installed LAB validation capability inventory changed" + ) + package_receipts: list[dict[str, object]] = [] + for package in packages.packages: + candidate = runtime.resolve(package.setup_id, package.definition_sha256) + package_receipts.append( + { + "package_id": package.package_id, + "package_version": package.package_version, + "package_sha256": package.package_sha256, + "setup_id": package.setup_id, + "definition_sha256": package.definition_sha256, + "runtime_candidate_sha256": package.runtime_candidate_sha256, + "executor": package.executor_identity.as_dict(), + "assets": [ + { + "asset_id": asset.asset_id, + "kind": asset.kind, + "sha256": asset.sha256, + } + for asset in candidate.reusable_assets + ], + "containers": [ + { + "container_id": container.container_id, + "role": container.role, + "image_sha256": container.image_sha256, + } + for container in package.containers + ], + } + ) + identity = { + "schema_version": INSTALLED_LAB_VALIDATION_RECEIPT_SCHEMA, + "state": "ready", + "packages": package_receipts, + "verified_image_sha256s": list(image_sha256s), + "supported_executor_identities": [ + identity.as_dict() for identity in executors.supported_identities + ], + "checks": { + "registry_binding": "passed", + "local_asset_identity": "passed", + "docker_image_identity": "passed", + "backend_contacted": False, + "claim_attempted": False, + }, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + return {**identity, "receipt_sha256": canonical_sha256(identity)} + + +def run_installed_lab_worker( + service: InstalledObservatoryWorkerService, + *, + stop: Event, + once: bool = False, +) -> None: + if once: + try: + service.agent.run_once() + finally: + service.close() + return + service.run(stop=stop) + + +def main(arguments: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--once", action="store_true", help="Run one claim cycle and exit.") + mode.add_argument( + "--validate-only", + action="store_true", + help="Validate local packages and print a receipt without contacting Mission Core.", + ) + options = parser.parse_args(arguments) + if cast(bool, options.validate_only): + receipt = validate_installed_lab_worker( + InstalledLabWorkerValidationConfiguration.from_environment() + ) + sys.stdout.buffer.write(canonical_json(receipt) + b"\n") + return 0 + service = compose_installed_lab_worker_service( + InstalledLabWorkerEntrypointConfiguration.from_environment() + ) + stop = Event() + with _posix_shutdown_signals(stop): + run_installed_lab_worker(service, stop=stop, once=cast(bool, options.once)) + return 0 + + +def _asset_binding(value: object) -> InstalledLabLocalAssetBinding: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise InstalledLabWorkerCompositionError("installed LAB asset binding is not an object") + _exact_keys( + value, + {"asset_id", "controller_path", "engine_path", "image_sha256"}, + "installed LAB asset binding", + ) + controller_value = value["controller_path"] + engine_value = value["engine_path"] + image_value = value["image_sha256"] + try: + if controller_value is None and engine_value is None and isinstance(image_value, str): + return InstalledLabLocalAssetBinding( + asset_id=_text(value["asset_id"], "installed LAB asset id"), + image_sha256=image_value, + ) + if ( + isinstance(controller_value, str) + and isinstance(engine_value, str) + and image_value is None + ): + return InstalledLabLocalAssetBinding( + asset_id=_text(value["asset_id"], "installed LAB asset id"), + controller_path=Path(controller_value), + engine_path=engine_value, + ) + except ValueError as exc: + raise InstalledLabWorkerCompositionError( + "installed LAB asset locator is invalid" + ) from exc + raise InstalledLabWorkerCompositionError("installed LAB asset locator is ambiguous") + + +def _read_object(path: Path, label: str) -> dict[str, object]: + candidate = path.expanduser().absolute() + try: + if candidate.is_symlink() or not candidate.is_file(): + raise InstalledLabWorkerCompositionError(f"{label} must be a regular file") + payload = candidate.read_bytes() + if not 0 < len(payload) <= _MAX_BINDINGS_BYTES: + raise InstalledLabWorkerCompositionError(f"{label} size is invalid") + value: object = json.loads(payload.decode("utf-8")) + except InstalledLabWorkerCompositionError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise InstalledLabWorkerCompositionError(f"{label} is unreadable") from exc + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise InstalledLabWorkerCompositionError(f"{label} is not an object") + return cast(dict[str, object], value) + + +def _required_path(values: Mapping[str, str], name: str) -> Path: + value = values.get(name) + if value is None or not value or not Path(value).is_absolute(): + raise InstalledLabWorkerCompositionError(f"{name} must name an absolute path") + return Path(value) + + +def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None: + if set(value) != expected: + raise InstalledLabWorkerCompositionError(f"{label} fields changed") + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise InstalledLabWorkerCompositionError(f"{label} is invalid") + return value + + +def _real_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise InstalledLabWorkerCompositionError(f"{label} is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise InstalledLabWorkerCompositionError(f"{label} is not a real directory") + return resolved + + +class _OfflineWorkerBoundary: + def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: + raise InstalledLabWorkerCompositionError( + f"offline validation cannot materialize job {job.job_id}" + ) + + def publish( + self, + job: SealedObservatoryRecordedJob, + draft: PortableWorkerResultDraft, + ) -> ObservatoryWorkerExecutionResult: + raise InstalledLabWorkerCompositionError( + f"offline validation cannot publish job {job.job_id} from {draft.root}" + ) + + +@contextmanager +def _posix_shutdown_signals(stop: Event) -> Iterator[None]: + def request_stop(_signal: int, _frame: FrameType | None) -> None: + stop.set() + + previous_int = signal.signal(signal.SIGINT, request_stop) + previous_term = signal.signal(signal.SIGTERM, request_stop) + try: + yield + finally: + signal.signal(signal.SIGINT, previous_int) + signal.signal(signal.SIGTERM, previous_term) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/k1link/observatory/lab_v1_installed_package_steps.py b/src/k1link/observatory/lab_v1_installed_package_steps.py new file mode 100644 index 0000000..7b085d0 --- /dev/null +++ b/src/k1link/observatory/lab_v1_installed_package_steps.py @@ -0,0 +1,499 @@ +"""Package-owned prepare/assemble steps for the generic LAB V1 stack. + +The generic Worker only supplies the sealed source, run plan, dependency +outputs, and reviewed assets. This module is the LAB package's own adapter: it +turns those standard inputs into the existing EoMT/DDRNet component contracts +and finally emits the common portable result package. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import sys +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Final, cast + +from k1link.observatory.installed_lab_packages import ( + INSTALLED_LAB_PLAN_PATH, + INSTALLED_LAB_RESULT_ROOT, + INSTALLED_LAB_SOURCE_ROOT, + INSTALLED_LAB_STEP_INPUT_ROOT, +) +from k1link.observatory.portable_lab_v1_executor import ( + PortableLabV1OrchestrationPlan, + assemble_lab_v1_result_v2, + package_lab_v1_result, + portable_lab_v1_orchestration_plan_from_document, + portable_lab_v1_source_input_from_document, +) +from k1link.observatory.portable_lab_v1_local_runners import ( + PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA, +) +from k1link.observatory.portable_lab_v1_worker import ( + _SealedPackageJobView, + materialize_lab_v1_source_from_worker_stage, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, +) +from k1link.observatory.portable_worker_runtime import PortableWorkerSourceStage +from k1link.observatory.recorded_jobs import RecordedExecutorIdentity +from k1link.observatory.worker_agent import SealedObservatoryRecordedJob + +LAB_V1_INSTALLED_PACKAGE_CONTRACT_SCHEMA: Final = ( + "missioncore.observatory-lab-v1-installed-package-contract/v1" +) + +_PACKAGE_CONTRACT = Path("/opt/nodedc/package/contract.json") +_DEFINITION_REGISTRY = Path("/opt/nodedc/package/portable-run-definitions.json") +_DDRNET_PROFILE = Path("/opt/nodedc/package/lab-v1-eomt-ddrnet-portable-v2.json") +_PREPARE_STEP = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "prepare" +_EOMT_STEP = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "eomt" +_DDRNET_STEP = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "ddrnet" +_MAX_DOCUMENT_BYTES: Final = 2 * 1024 * 1024 +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$") + + +class LabV1InstalledPackageStepError(RuntimeError): + """The installed package or one of its sealed step inputs changed.""" + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = tuple(sys.argv[1:] if argv is None else argv) + if arguments == ("prepare",): + prepare() + return 0 + if arguments == ("assemble",): + assemble() + return 0 + raise LabV1InstalledPackageStepError("LAB V1 package step is not allowlisted") + + +def prepare() -> None: + output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "LAB V1 prepare output") + contract = _load_contract() + runtime_plan = _runtime_plan() + definition = _definition(runtime_plan) + job = _sealed_job(runtime_plan, definition=definition, contract=contract) + source_stage = PortableWorkerSourceStage( + root=_real_directory(Path(INSTALLED_LAB_SOURCE_ROOT), "LAB V1 package source"), + source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "source bundle"), + source_capability_manifest_sha256=_digest( + runtime_plan["source_capability_manifest_sha256"], + "source capability", + ), + source_adapter_sha256=_digest( + runtime_plan["source_adapter_sha256"], + "source adapter", + ), + ) + materialized = materialize_lab_v1_source_from_worker_stage( + worker_stage=source_stage, + job=job, + definition=definition, + output_parent=output, + ) + camera_target = output / "camera-job" + camera_stage_parent = materialized.camera_job_root.parent + if camera_stage_parent.parent != materialized.root: + raise LabV1InstalledPackageStepError( + "materialized camera job is outside its digest-owned stage" + ) + os.replace(materialized.camera_job_root, camera_target) + camera_stage_parent.rmdir() + materialized.root.rmdir() + ddrnet_profile = _load_object(_DDRNET_PROFILE, "portable DDRNet profile") + plan = PortableLabV1OrchestrationPlan.create_for_installed_package( + job=job, + definition=definition, + source=materialized.descriptor, + package_release_sha256=_digest( + contract["package_release_sha256"], + "package release", + ), + ddrnet_runner_sha256=_digest( + contract["ddrnet_runner_sha256"], + "DDRNet runner", + ), + result_assembler_sha256=_digest( + contract["result_assembler_sha256"], + "result assembler", + ), + legacy_ddrnet_config=ddrnet_profile, + ) + plan.require_executable() + _write(output / "source-input.json", materialized.descriptor.as_dict()) + _write(output / "orchestration-plan.json", plan.as_dict()) + _write(output / "effective-ddrnet-config.json", plan.effective_ddrnet_config) + component_images = _object(contract["component_images"], "component images") + component_assets = _object(contract["component_assets"], "component assets") + for component in ("eomt", "ddrnet"): + _write( + output / f"{component}-request.json", + _component_request( + component=component, + component_image_sha256=_digest( + component_images[component], + f"{component} image", + ), + source=materialized.descriptor.as_dict(), + plan=plan, + assets=_asset_rows(component_assets[component], component), + ), + ) + + +def assemble() -> None: + output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "LAB V1 result output") + contract = _load_contract() + runtime_plan = _runtime_plan() + definition = _definition(runtime_plan) + job = _sealed_job(runtime_plan, definition=definition, contract=contract) + source_input = portable_lab_v1_source_input_from_document( + _load_object(_PREPARE_STEP / "source-input.json", "LAB V1 source input") + ) + plan = portable_lab_v1_orchestration_plan_from_document( + _load_object( + _PREPARE_STEP / "orchestration-plan.json", + "LAB V1 orchestration plan", + ), + source_input=source_input, + ) + if plan.release_candidate_sha256 != contract["package_release_sha256"]: + raise LabV1InstalledPackageStepError("LAB V1 package release identity changed") + assembly_parent = output / ".assembly" + assembly_parent.mkdir(mode=0o700) + package_parent = output / ".package" + package_parent.mkdir(mode=0o700) + assembly_result = assemble_lab_v1_result_v2( + plan=plan, + definition=definition, + eomt_result_root=_real_directory(_EOMT_STEP, "EoMT step result"), + ddrnet_result_root=_real_directory(_DDRNET_STEP, "DDRNet step result"), + output_parent=assembly_parent, + ) + draft = package_lab_v1_result( + assembly=assembly_result, + plan=plan, + job=_SealedPackageJobView.from_job(job), + definition=definition, + created_at_utc=datetime.now(UTC).isoformat().replace("+00:00", "Z"), + output_parent=package_parent, + ) + shutil.rmtree(assembly_parent) + for child in tuple(draft.root.iterdir()): + os.replace(child, output / child.name) + draft.root.rmdir() + package_parent.rmdir() + + +def _component_request( + *, + component: str, + component_image_sha256: str, + source: Mapping[str, object], + plan: PortableLabV1OrchestrationPlan, + assets: list[dict[str, object]], +) -> dict[str, object]: + prepared = f"{INSTALLED_LAB_STEP_INPUT_ROOT}/prepare" + return { + "schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA, + "component": component, + "component_image_sha256": component_image_sha256, + "plan_sha256": plan.plan_sha256, + "definition_sha256": plan.definition_sha256, + "release_candidate_sha256": plan.release_candidate_sha256, + "source": dict(source), + "paths": { + "camera_job_root": f"{prepared}/camera-job", + "request": f"{prepared}/{component}-request.json", + "output_root": INSTALLED_LAB_RESULT_ROOT, + "effective_ddrnet_config": ( + f"{prepared}/effective-ddrnet-config.json" + if component == "ddrnet" + else None + ), + "eomt_result_root": ( + f"{INSTALLED_LAB_STEP_INPUT_ROOT}/eomt" + if component == "ddrnet" + else None + ), + "decoded_frames_root": ( + f"{INSTALLED_LAB_RESULT_ROOT}/source-frames" + if component == "eomt" + else f"{INSTALLED_LAB_STEP_INPUT_ROOT}/eomt/source-frames" + ), + }, + "effective_ddrnet_config_sha256": ( + plan.effective_ddrnet_config_sha256 if component == "ddrnet" else None + ), + "assets": assets, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +def _sealed_job( + runtime_plan: Mapping[str, object], + *, + definition: PortableRunDefinition, + contract: Mapping[str, object], +) -> SealedObservatoryRecordedJob: + executor = _object(contract["executor"], "package executor") + identity = RecordedExecutorIdentity( + release_sha256=_digest(executor["release_sha256"], "executor release"), + image_sha256=_digest(executor["image_sha256"], "executor image"), + model_manifest_sha256=definition.model_manifest_sha256, + resource_profile_sha256=definition.resource_profile.profile_sha256, + ) + return SealedObservatoryRecordedJob( + job_id=_text(runtime_plan["job_id"], "job id"), + request_sha256=_digest(runtime_plan["request_sha256"], "request"), + identity_sha256=_digest(runtime_plan["identity_sha256"], "job identity"), + submission_receipt_sha256=_digest( + runtime_plan["submission_receipt_sha256"], + "submission receipt", + ), + source_session_id=_text(runtime_plan["source_session_id"], "source session"), + source_catalog_sha256=_digest(runtime_plan["source_catalog_sha256"], "catalog"), + source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "bundle"), + source_capability_manifest_sha256=_digest( + runtime_plan["source_capability_manifest_sha256"], + "source capability", + ), + source_adapter_id=_identifier( + runtime_plan["source_adapter_id"], + "source adapter id", + ), + source_adapter_version=_positive_int( + runtime_plan["source_adapter_version"], + "source adapter version", + ), + source_adapter_sha256=_digest( + runtime_plan["source_adapter_sha256"], + "source adapter", + ), + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + executor_release_id=_identifier(executor["release_id"], "executor release id"), + executor_identity=identity, + model_release_ids=definition.learned_models, + resource_profile_id=definition.resource_profile.profile_id, + checkpoint_policy=definition.resource_profile.checkpoint_policy, + allowed_checkpoints=definition.resource_profile.allowed_checkpoints, + claim_generation=_positive_int( + runtime_plan["claim_generation"], + "claim generation", + ), + claim_claimed_at_utc=None, + claim_expires_at_utc=None, + claim_heartbeat_at_utc=None, + claim_renewal_count=0, + restart_from_zero=False, + ) + + +def _runtime_plan() -> dict[str, object]: + outer = _load_object(Path(INSTALLED_LAB_PLAN_PATH), "installed LAB run plan") + _exact_keys( + outer, + {"schema_version", "runtime_plan", "package_id", "package_sha256", "authority"}, + "installed LAB run plan", + ) + if ( + outer["schema_version"] != "missioncore.observatory-installed-lab-run-plan/v1" + or outer["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise LabV1InstalledPackageStepError("installed LAB run plan changed") + plan = _object(outer["runtime_plan"], "portable runtime plan") + expected = { + "schema_version", + "job_id", + "request_sha256", + "identity_sha256", + "submission_receipt_sha256", + "claim_generation", + "adapter_id", + "candidate_sha256", + "setup_id", + "definition_id", + "definition_version", + "definition_sha256", + "source_session_id", + "source_catalog_sha256", + "source_bundle_sha256", + "source_capability_manifest_sha256", + "source_adapter_id", + "source_adapter_version", + "source_adapter_sha256", + "result_contract_sha256", + "phases", + "authority", + } + _exact_keys(plan, expected, "portable runtime plan") + if ( + plan["schema_version"] != "missioncore.observatory-portable-worker-runtime-plan/v2" + or plan["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise LabV1InstalledPackageStepError("portable runtime plan changed") + return plan + + +def _definition(runtime_plan: Mapping[str, object]) -> PortableRunDefinition: + registry = PortableRunDefinitionRegistry.from_file(_DEFINITION_REGISTRY) + return registry.resolve( + _identifier(runtime_plan["setup_id"], "setup id"), + _digest(runtime_plan["definition_sha256"], "definition"), + ) + + +def _load_contract() -> dict[str, object]: + contract = _load_object(_PACKAGE_CONTRACT, "LAB V1 package contract") + _exact_keys( + contract, + { + "schema_version", + "package_release_sha256", + "ddrnet_runner_sha256", + "result_assembler_sha256", + "executor", + "component_images", + "component_assets", + "authority", + }, + "LAB V1 package contract", + ) + if ( + contract["schema_version"] != LAB_V1_INSTALLED_PACKAGE_CONTRACT_SCHEMA + or contract["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise LabV1InstalledPackageStepError("LAB V1 package contract changed") + _exact_keys( + _object(contract["executor"], "package executor"), + {"release_id", "release_sha256", "image_sha256"}, + "package executor", + ) + _exact_keys( + _object(contract["component_images"], "component images"), + {"eomt", "ddrnet"}, + "component images", + ) + _exact_keys( + _object(contract["component_assets"], "component assets"), + {"eomt", "ddrnet"}, + "component assets", + ) + return contract + + +def _asset_rows(value: object, component: str) -> list[dict[str, object]]: + if not isinstance(value, list): + raise LabV1InstalledPackageStepError(f"{component} assets are not an array") + rows: list[dict[str, object]] = [] + for value_row in value: + row = _object(value_row, f"{component} asset") + _exact_keys( + row, + {"asset_id", "path", "kind", "verification", "identity_sha256", "byte_length"}, + f"{component} asset", + ) + _identifier(row["asset_id"], f"{component} asset id", dotted=True) + _digest(row["identity_sha256"], f"{component} asset identity") + rows.append(dict(row)) + ids = tuple(cast(str, row["asset_id"]) for row in rows) + if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)): + raise LabV1InstalledPackageStepError(f"{component} assets are not canonical") + return rows + + +def _load_object(path: Path, label: str) -> dict[str, object]: + if path.is_symlink() or not path.is_file(): + raise LabV1InstalledPackageStepError(f"{label} is unavailable") + payload = path.read_bytes() + if not payload or len(payload) > _MAX_DOCUMENT_BYTES: + raise LabV1InstalledPackageStepError(f"{label} size is invalid") + try: + value: object = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LabV1InstalledPackageStepError(f"{label} is invalid JSON") from exc + return _object(value, label) + + +def _write(path: Path, value: Mapping[str, object]) -> None: + payload = canonical_json(value) + path.write_bytes(payload) + os.chmod(path, 0o400) + if hashlib.sha256(path.read_bytes()).digest() != hashlib.sha256(payload).digest(): + raise LabV1InstalledPackageStepError("LAB V1 package document write changed") + + +def _empty_directory(path: Path, label: str) -> Path: + root = _real_directory(path, label) + if any(root.iterdir()): + raise LabV1InstalledPackageStepError(f"{label} is not empty") + return root + + +def _real_directory(path: Path, label: str) -> Path: + if path.is_symlink() or not path.is_dir(): + raise LabV1InstalledPackageStepError(f"{label} is unavailable") + return path.resolve(strict=True) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise LabV1InstalledPackageStepError(f"{label} is not an object") + return cast(dict[str, object], value) + + +def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None: + if set(value) != expected: + raise LabV1InstalledPackageStepError(f"{label} fields changed") + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise LabV1InstalledPackageStepError(f"{label} is invalid") + return value + + +def _identifier(value: object, label: str, *, dotted: bool = False) -> str: + text = _text(value, label) + pattern = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") if dotted else _IDENTIFIER + if pattern.fullmatch(text) is None: + raise LabV1InstalledPackageStepError(f"{label} is invalid") + return text + + +def _digest(value: object, label: str) -> str: + text = _text(value, label) + if _SHA256.fullmatch(text) is None: + raise LabV1InstalledPackageStepError(f"{label} is invalid") + return text + + +def _positive_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise LabV1InstalledPackageStepError(f"{label} is invalid") + return value + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except LabV1InstalledPackageStepError as exc: + print(f"installed LAB V1 package rejected: {exc}", file=sys.stderr) + raise SystemExit(2) from exc diff --git a/src/k1link/observatory/m49_worker_container_main.py b/src/k1link/observatory/m49_worker_container_main.py index b40485a..862019b 100644 --- a/src/k1link/observatory/m49_worker_container_main.py +++ b/src/k1link/observatory/m49_worker_container_main.py @@ -9,153 +9,31 @@ weakening the gateway: a process-local TCP bridge binds only from __future__ import annotations -import shutil -import socket -import socketserver -import threading from collections.abc import Sequence -from contextlib import suppress from typing import Final from k1link.observatory import m49_worker_service +from k1link.observatory.worker_container_proxy import ( + CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS, + CONTAINER_PROXY_COPY_BYTES, + CONTAINER_PROXY_LISTEN_HOST, + CONTAINER_PROXY_LISTEN_PORT, + CONTAINER_PROXY_UPSTREAM_HOST, + CONTAINER_PROXY_UPSTREAM_PORT, + FixedObservatoryContainerLoopbackProxy, + ObservatoryWorkerContainerProxyError, +) -M49_CONTAINER_PROXY_LISTEN_HOST: Final = "127.0.0.1" -M49_CONTAINER_PROXY_LISTEN_PORT: Final = 18080 -M49_CONTAINER_PROXY_UPSTREAM_HOST: Final = "host.docker.internal" -M49_CONTAINER_PROXY_UPSTREAM_PORT: Final = 18080 -M49_CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS: Final = 10.0 -M49_CONTAINER_PROXY_COPY_BYTES: Final = 1024 * 1024 - - -class M49WorkerContainerProxyError(RuntimeError): - """The fixed container loopback bridge could not be started safely.""" - - -class _ThreadedTcpServer(socketserver.ThreadingTCPServer): - allow_reuse_address = True - daemon_threads = True - - -class _FixedProxyHandler(socketserver.BaseRequestHandler): - server: _ThreadedTcpServer - - def handle(self) -> None: - upstream_address = getattr(self.server, "upstream_address", None) - connect_timeout = getattr(self.server, "connect_timeout", None) - if ( - not isinstance(upstream_address, tuple) - or len(upstream_address) != 2 - or not isinstance(upstream_address[0], str) - or not isinstance(upstream_address[1], int) - or not isinstance(connect_timeout, float) - ): - return - try: - upstream = socket.create_connection( - upstream_address, - timeout=connect_timeout, - ) - except OSError: - return - with upstream: - upstream.settimeout(None) - client = self.request - if not isinstance(client, socket.socket): - return - client.settimeout(None) - client_to_upstream = threading.Thread( - target=_copy_socket, - args=(client, upstream), - daemon=True, - name="m49-proxy-client-to-host", - ) - upstream_to_client = threading.Thread( - target=_copy_socket, - args=(upstream, client), - daemon=True, - name="m49-proxy-host-to-client", - ) - client_to_upstream.start() - upstream_to_client.start() - client_to_upstream.join() - upstream_to_client.join() - - -class FixedM49ContainerLoopbackProxy: - """Own one bounded TCP bridge for the lifetime of the Worker process.""" - - def __init__( - self, - *, - listen_host: str = M49_CONTAINER_PROXY_LISTEN_HOST, - listen_port: int = M49_CONTAINER_PROXY_LISTEN_PORT, - upstream_host: str = M49_CONTAINER_PROXY_UPSTREAM_HOST, - upstream_port: int = M49_CONTAINER_PROXY_UPSTREAM_PORT, - connect_timeout: float = M49_CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS, - ) -> None: - if listen_host != M49_CONTAINER_PROXY_LISTEN_HOST: - raise ValueError("M4.9 container proxy must bind IPv4 loopback") - if not 0 <= listen_port <= 65_535: - raise ValueError("M4.9 container proxy listen port is invalid") - if not upstream_host or upstream_host != upstream_host.strip(): - raise ValueError("M4.9 container proxy upstream host is invalid") - if not 1 <= upstream_port <= 65_535: - raise ValueError("M4.9 container proxy upstream port is invalid") - if not 0.05 <= connect_timeout <= 60.0: - raise ValueError("M4.9 container proxy timeout is invalid") - try: - server = _ThreadedTcpServer( - (listen_host, listen_port), - _FixedProxyHandler, - bind_and_activate=True, - ) - except OSError as exc: - raise M49WorkerContainerProxyError( - "M4.9 container loopback proxy could not bind" - ) from exc - server.upstream_address = (upstream_host, upstream_port) # type: ignore[attr-defined] - server.connect_timeout = float(connect_timeout) # type: ignore[attr-defined] - self._server = server - self._thread = threading.Thread( - target=server.serve_forever, - kwargs={"poll_interval": 0.1}, - daemon=True, - name="m49-container-loopback-proxy", - ) - - @property - def listen_port(self) -> int: - address = self._server.server_address - if not isinstance(address, tuple) or not isinstance(address[1], int): - raise M49WorkerContainerProxyError("M4.9 proxy address is invalid") - return address[1] - - def __enter__(self) -> FixedM49ContainerLoopbackProxy: - self._thread.start() - return self - - def __exit__(self, *_args: object) -> None: - self._server.shutdown() - self._server.server_close() - self._thread.join(timeout=5.0) - if self._thread.is_alive(): - raise M49WorkerContainerProxyError( - "M4.9 container loopback proxy did not stop" - ) - - -def _copy_socket(source: socket.socket, destination: socket.socket) -> None: - try: - shutil.copyfileobj( - source.makefile("rb", buffering=0), - destination.makefile("wb", buffering=0), - length=M49_CONTAINER_PROXY_COPY_BYTES, - ) - except OSError: - pass - finally: - with suppress(OSError): - destination.shutdown(socket.SHUT_WR) +M49_CONTAINER_PROXY_LISTEN_HOST: Final = CONTAINER_PROXY_LISTEN_HOST +M49_CONTAINER_PROXY_LISTEN_PORT: Final = CONTAINER_PROXY_LISTEN_PORT +M49_CONTAINER_PROXY_UPSTREAM_HOST: Final = CONTAINER_PROXY_UPSTREAM_HOST +M49_CONTAINER_PROXY_UPSTREAM_PORT: Final = CONTAINER_PROXY_UPSTREAM_PORT +M49_CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS: Final = ( + CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS +) +M49_CONTAINER_PROXY_COPY_BYTES: Final = CONTAINER_PROXY_COPY_BYTES +M49WorkerContainerProxyError = ObservatoryWorkerContainerProxyError +FixedM49ContainerLoopbackProxy = FixedObservatoryContainerLoopbackProxy def main(arguments: Sequence[str] | None = None) -> int: diff --git a/src/k1link/observatory/m49_worker_service.py b/src/k1link/observatory/m49_worker_service.py index 205aec4..690f320 100644 --- a/src/k1link/observatory/m49_worker_service.py +++ b/src/k1link/observatory/m49_worker_service.py @@ -212,11 +212,11 @@ class M49WorkerEntrypointConfiguration: values, M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV, ), - lab_v1_installation_receipt_file=_required_environment_path( + lab_v1_installation_receipt_file=_optional_environment_path( values, LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV, ), - lab_v1_release_candidate_file=_required_environment_path( + lab_v1_release_candidate_file=_optional_environment_path( values, LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV, ), @@ -1101,6 +1101,20 @@ def _required_environment_path(values: Mapping[str, str], name: str) -> Path: return path +def _optional_environment_path( + values: Mapping[str, str], + name: str, +) -> Path | None: + value = values.get(name) + if value is None: + return None + if not value or value != value.strip(): + raise M49WorkerCompositionError(f"{name} is invalid") + path = Path(value) + _absolute_path(path, name) + return path + + def _digest(value: str, label: str) -> None: if _SHA256.fullmatch(value) is None: raise M49WorkerCompositionError(f"{label} is invalid") diff --git a/src/k1link/observatory/portable_lab_v1_executor.py b/src/k1link/observatory/portable_lab_v1_executor.py index 38054ce..f246541 100644 --- a/src/k1link/observatory/portable_lab_v1_executor.py +++ b/src/k1link/observatory/portable_lab_v1_executor.py @@ -34,6 +34,7 @@ from k1link.compute.jobs import CameraComputeJob, validate_camera_compute_job from k1link.observatory.portable_result_contract import ( OBSERVATION_ONLY_AUTHORITY, PortableResultArtifact, + PortableResultJobIdentity, PortableResultPackageIntegrityError, PortableResultPackageManifest, PortableResultValidationContext, @@ -839,52 +840,11 @@ class PortableLabV1OrchestrationPlan: legacy_ddrnet_config, source=source, ) - components = {item.component_id: item.sha256 for item in definition.components} release_assets = {asset.asset_id: asset.sha256 for asset in release.assets} - phases = ( - PortableLabV1PlanPhase( - phase_id="source-materialization", - component_sha256s=(definition.source_adapter.contract_sha256,), - input_roles=("camera-compute-job", "source-documents"), - output_roles=("source-input-manifest",), - ), - PortableLabV1PlanPhase( - phase_id="eomt-full-session", - component_sha256s=tuple( - sorted( - ( - components["eomt-recorded-orchestrator-v1"], - components["eomt-recorded-profile-v1"], - components["eomt-recorded-runner-v1"], - definition.model_manifest_sha256, - ) - ) - ), - input_roles=("camera-compute-job",), - output_roles=("eomt-component-result",), - ), - PortableLabV1PlanPhase( - phase_id="ddrnet-full-session", - component_sha256s=tuple( - sorted( - ( - components["ddrnet-portable-runtime-config-v2"], - components["vegetation-mission-policy-v1"], - components["vegetation-provider-label-map-v1"], - release_assets["ddrnet-goose-runner"], - definition.model_manifest_sha256, - ) - ) - ), - input_roles=("camera-compute-job", "ddrnet-effective-config"), - output_roles=("ddrnet-component-result",), - ), - PortableLabV1PlanPhase( - phase_id="result-v2-assembly", - component_sha256s=(release_assets["lab-v1-portable-contracts"],), - input_roles=("ddrnet-component-result", "eomt-component-result"), - output_roles=("portable-result-draft",), - ), + phases = _portable_lab_v1_plan_phases( + definition, + ddrnet_runner_sha256=release_assets["ddrnet-goose-runner"], + result_assembler_sha256=release_assets["lab-v1-portable-contracts"], ) effective_sha256 = canonical_sha256(effective) identity = _plan_identity_document( @@ -920,6 +880,71 @@ class PortableLabV1OrchestrationPlan: plan_sha256=canonical_sha256(identity), ) + @classmethod + def create_for_installed_package( + cls, + *, + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, + source: PortableLabV1SourceInput, + package_release_sha256: str, + ddrnet_runner_sha256: str, + result_assembler_sha256: str, + legacy_ddrnet_config: Mapping[str, object], + ) -> PortableLabV1OrchestrationPlan: + """Create a plan after the generic package registry admitted all assets.""" + + _verify_definition_and_job(definition, job) + _verify_source_and_job(source, job, definition) + for value, label in ( + (package_release_sha256, "installed LAB V1 package release sha256"), + (ddrnet_runner_sha256, "installed LAB V1 DDRNet runner sha256"), + (result_assembler_sha256, "installed LAB V1 result assembler sha256"), + ): + _digest(value, label) + effective = build_portable_ddrnet_effective_config( + legacy_ddrnet_config, + source=source, + ) + phases = _portable_lab_v1_plan_phases( + definition, + ddrnet_runner_sha256=ddrnet_runner_sha256, + result_assembler_sha256=result_assembler_sha256, + ) + effective_sha256 = canonical_sha256(effective) + identity = _plan_identity_document( + observatory_job_id=job.job_id, + observatory_request_sha256=job.request_sha256, + observatory_identity_sha256=job.identity_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + source_input_sha256=source.identity_sha256, + release_candidate_sha256=package_release_sha256, + effective_ddrnet_config_sha256=effective_sha256, + phases=phases, + release_blockers=(), + ) + return cls( + observatory_job_id=job.job_id, + observatory_request_sha256=job.request_sha256, + observatory_identity_sha256=job.identity_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + source_input=source, + release_candidate_sha256=package_release_sha256, + effective_ddrnet_config=effective, + effective_ddrnet_config_sha256=effective_sha256, + phases=phases, + blockers=(), + plan_sha256=canonical_sha256(identity), + ) + @property def executable(self) -> bool: return not self.blockers @@ -1053,6 +1078,60 @@ def build_portable_ddrnet_effective_config( return copied +def _portable_lab_v1_plan_phases( + definition: PortableRunDefinition, + *, + ddrnet_runner_sha256: str, + result_assembler_sha256: str, +) -> tuple[PortableLabV1PlanPhase, ...]: + components = {item.component_id: item.sha256 for item in definition.components} + return ( + PortableLabV1PlanPhase( + phase_id="source-materialization", + component_sha256s=(definition.source_adapter.contract_sha256,), + input_roles=("camera-compute-job", "source-documents"), + output_roles=("source-input-manifest",), + ), + PortableLabV1PlanPhase( + phase_id="eomt-full-session", + component_sha256s=tuple( + sorted( + ( + components["eomt-recorded-orchestrator-v1"], + components["eomt-recorded-profile-v1"], + components["eomt-recorded-runner-v1"], + definition.model_manifest_sha256, + ) + ) + ), + input_roles=("camera-compute-job",), + output_roles=("eomt-component-result",), + ), + PortableLabV1PlanPhase( + phase_id="ddrnet-full-session", + component_sha256s=tuple( + sorted( + ( + components["ddrnet-portable-runtime-config-v2"], + components["vegetation-mission-policy-v1"], + components["vegetation-provider-label-map-v1"], + ddrnet_runner_sha256, + definition.model_manifest_sha256, + ) + ) + ), + input_roles=("camera-compute-job", "ddrnet-effective-config"), + output_roles=("ddrnet-component-result",), + ), + PortableLabV1PlanPhase( + phase_id="result-v2-assembly", + component_sha256s=(result_assembler_sha256,), + input_roles=("ddrnet-component-result", "eomt-component-result"), + output_roles=("portable-result-draft",), + ), + ) + + @dataclass(frozen=True, slots=True) class PortableLabV1ResultAssembly: root: Path @@ -1289,18 +1368,12 @@ def package_lab_v1_result( *, assembly: PortableLabV1ResultAssembly, plan: PortableLabV1OrchestrationPlan, - job: ObservatoryRecordedJob, + job: PortableResultJobIdentity, definition: PortableRunDefinition, created_at_utc: str, output_parent: Path, ) -> PortableWorkerResultDraft: - """Wrap a validated assembly in the generic portable result package. - - This step intentionally requires the durable server job because the current - sealed Worker claim omits its submission receipt. Until that receipt is - transported into the local adapter (or the server owns this step), release - admission remains blocked instead of weakening the package identity. - """ + """Wrap a validated assembly in the generic portable result package.""" _verify_plan_definition(plan, definition) if ( @@ -1490,7 +1563,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None: raise PortableLabV1ResultError("portable LAB V1 identity projection changed") -def _source_input_from_document( +def portable_lab_v1_source_input_from_document( document: Mapping[str, object], ) -> PortableLabV1SourceInput: _exact_keys( @@ -1598,7 +1671,10 @@ def _source_input_from_document( ) -def _orchestration_plan_from_document( +_source_input_from_document = portable_lab_v1_source_input_from_document + + +def portable_lab_v1_orchestration_plan_from_document( document: Mapping[str, object], *, source_input: PortableLabV1SourceInput, @@ -1727,6 +1803,9 @@ def _orchestration_plan_from_document( return plan +_orchestration_plan_from_document = portable_lab_v1_orchestration_plan_from_document + + def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase: row = _object(value, "portable plan phase") _exact_keys( @@ -1821,7 +1900,7 @@ def _validate_source_documents( or video.get("semantic_channel_id") != requirements.camera_semantic_channel_id or video.get("seekable") is not True - or camera_job.source_id != requirements.camera_source_id + or camera_job.source_id != camera.get("public_source_id") or camera_job.codec_epoch != epoch.get("ordinal") or epoch.get("media_type") != requirements.recorded_media_type or init.get("sha256") != requirements.recorded_media_init_sha256 @@ -1932,7 +2011,6 @@ def _verify_source_and_job( or source.source_capability_manifest_sha256 != job.source_capability_manifest_sha256 or source.source_adapter_sha256 != job.source_adapter_sha256 - or source.camera_source_id != requirements.camera_source_id or source.calibration_sha256 != requirements.calibration_identity_sha256 ): raise PortableLabV1PlanError( diff --git a/src/k1link/observatory/portable_publication_reconciler.py b/src/k1link/observatory/portable_publication_reconciler.py new file mode 100644 index 0000000..494a30f --- /dev/null +++ b/src/k1link/observatory/portable_publication_reconciler.py @@ -0,0 +1,105 @@ +"""Bounded recovery for verified Observatory result publication.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +from .portable_artifact_transport import ( + PortableArtifactTransportError, + PortableObservatoryArtifactTransport, +) +from .portable_result_contract import PortableResultPublisherError +from .portable_result_publisher import PortableObservatoryResultPublisher +from .recorded_jobs import ( + ObservatoryRecordedJobQueue, + ObservatoryRecordedQueueError, +) + + +@dataclass(frozen=True, slots=True) +class PortablePublicationReconciliation: + examined: int + published: int + failed: int + deferred: int + exhausted: int + + +@dataclass(frozen=True, slots=True) +class PortablePublicationReconciler: + """Retry only durable publication outbox entries, never their compute jobs.""" + + queue: ObservatoryRecordedJobQueue + artifact_transport: PortableObservatoryArtifactTransport + result_publisher: PortableObservatoryResultPublisher + clock: Callable[[], datetime] = lambda: datetime.now(UTC) + maximum_attempts: int = 5 + retry_delays_seconds: tuple[int, ...] = (0, 30, 120, 600, 1_800) + + def __post_init__(self) -> None: + if self.maximum_attempts < 1: + raise ValueError("publication maximum attempts must be positive") + if ( + len(self.retry_delays_seconds) != self.maximum_attempts + or self.retry_delays_seconds != tuple(sorted(self.retry_delays_seconds)) + or any(value < 0 for value in self.retry_delays_seconds) + ): + raise ValueError("publication retry delays are invalid") + + def run_once(self, *, limit: int = 4) -> PortablePublicationReconciliation: + if not 1 <= limit <= 32: + raise ValueError("publication reconciliation limit must be within 1..32") + now = self.clock() + if now.tzinfo is None: + raise ValueError("publication reconciliation clock must be timezone-aware") + candidates = self.queue.pending_publications()[:limit] + published = 0 + failed = 0 + deferred = 0 + exhausted = 0 + for job in candidates: + attempts = job.publication_attempts + if attempts >= self.maximum_attempts: + exhausted += 1 + continue + updated_at = _timestamp(job.updated_at_utc) + retry_at = updated_at + timedelta( + seconds=self.retry_delays_seconds[attempts] + ) + if now < retry_at: + deferred += 1 + continue + try: + package_root = self.artifact_transport.package_root_for_terminal(job) + self.result_publisher.publish(job=job, package_root=package_root) + self.queue.mark_published(job.job_id) + published += 1 + except (PortableArtifactTransportError, PortableResultPublisherError) as exc: + message = (" ".join(str(exc).split()) or type(exc).__name__)[:1_000] + with suppress(ObservatoryRecordedQueueError): + self.queue.mark_publication_failed(job.job_id, message=message) + failed += 1 + return PortablePublicationReconciliation( + examined=len(candidates), + published=published, + failed=failed, + deferred=deferred, + exhausted=exhausted, + ) + + +def _timestamp(value: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ObservatoryRecordedQueueError( + "recorded publication timestamp is invalid" + ) from exc + if parsed.tzinfo is None: + raise ObservatoryRecordedQueueError( + "recorded publication timestamp has no timezone" + ) + return parsed diff --git a/src/k1link/observatory/portable_result_contract.py b/src/k1link/observatory/portable_result_contract.py index 286f7cf..874ed7e 100644 --- a/src/k1link/observatory/portable_result_contract.py +++ b/src/k1link/observatory/portable_result_contract.py @@ -8,7 +8,7 @@ import re from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path, PurePosixPath -from typing import Final, cast +from typing import Final, Protocol, cast from k1link.observatory.portable_run_definitions import ( PortableRunDefinition, @@ -16,6 +16,50 @@ from k1link.observatory.portable_run_definitions import ( ) from k1link.observatory.recorded_jobs import ObservatoryRecordedJob + +class PortableResultJobIdentity(Protocol): + """Sealed job fields required to create a portable result identity.""" + + @property + def job_id(self) -> str: ... + + @property + def request_sha256(self) -> str: ... + + @property + def identity_sha256(self) -> str: ... + + @property + def submission_receipt_sha256(self) -> str: ... + + @property + def claim_generation(self) -> int: ... + + @property + def source_session_id(self) -> str: ... + + @property + def source_catalog_sha256(self) -> str: ... + + @property + def source_bundle_sha256(self) -> str: ... + + @property + def source_capability_manifest_sha256(self) -> str: ... + + @property + def source_adapter_id(self) -> str: ... + + @property + def source_adapter_version(self) -> int: ... + + @property + def source_adapter_sha256(self) -> str: ... + + @property + def result_id(self) -> str | None: ... + + PORTABLE_RESULT_PACKAGE_SCHEMA: Final = ( "missioncore.observatory-portable-result-package/v1" ) @@ -150,7 +194,7 @@ class PortableResultPackageManifest: def create( cls, *, - job: ObservatoryRecordedJob, + job: PortableResultJobIdentity, definition: PortableRunDefinition, result_id: str, created_at_utc: str, @@ -415,7 +459,7 @@ class PortableResultContractValidatorRegistry: ) -def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]: +def job_identity_document(job: PortableResultJobIdentity) -> dict[str, object]: return { "job_id": job.job_id, "request_sha256": job.request_sha256, @@ -425,7 +469,7 @@ def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]: } -def source_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]: +def source_identity_document(job: PortableResultJobIdentity) -> dict[str, object]: return { "session_id": job.source_session_id, "catalog_sha256": job.source_catalog_sha256, diff --git a/src/k1link/observatory/portable_result_publisher.py b/src/k1link/observatory/portable_result_publisher.py index ce5affa..a8616b8 100644 --- a/src/k1link/observatory/portable_result_publisher.py +++ b/src/k1link/observatory/portable_result_publisher.py @@ -69,7 +69,12 @@ from k1link.observatory.source_admission import ( PORTABLE_SOURCE_CAPABILITY_SCHEMA, PORTABLE_SOURCE_DOCUMENT_DIRECTORY, ) -from k1link.sessions.models import LabSessionBinding, SessionIntegrityError, SessionSummary +from k1link.sessions.models import ( + LabReplayCapability, + LabSessionBinding, + SessionIntegrityError, + SessionSummary, +) from k1link.sessions.store import SessionStore _COPY_CHUNK_BYTES = 1024 * 1024 @@ -157,12 +162,21 @@ class PortableObservatoryResultPublisher: artifact_paths=artifact_paths, profile=profile, ) + replay_capability = LabReplayCapability( + schema_version="missioncore.observation-lab-replay-capability/v2", + kind="portable-result-review", + viewer_profile="portable-result", + timeline="result-defined", + activation="explicit", + commands_enabled=False, + ) provenance = _publication_provenance( job=job, definition=definition, package=package, artifact_manifest=artifact_manifest, profile=profile, + replay_capability=replay_capability, ) try: binding = self._session_store.publish_lab_instance( @@ -174,7 +188,7 @@ class PortableObservatoryResultPublisher: result_id=cast(str, job.result_id), config_sha256=definition.definition_sha256, run_created_at_utc=job.updated_at_utc, - replay_capability=None, + replay_capability=replay_capability, provenance=provenance, include_recorded_media=profile.include_recorded_media, expected_source_catalog_sha256=job.source_catalog_sha256, @@ -320,6 +334,7 @@ def resolve_published_portable_calculation_profile( "source", "run_definition", "result_package", + "replay_capability", "storage", "method", }, @@ -354,6 +369,10 @@ def resolve_published_portable_calculation_profile( or binding.lab_id != profile.lab_id or binding.config_sha256 != definition.definition_sha256 or binding.result_kind != definition.result_contract.result_kind + or binding.replay_capability is None + or binding.replay_capability.kind != "portable-result-review" + or provenance["replay_capability"] + != binding.replay_capability.as_dict() ): return None return profile.as_dict() @@ -712,6 +731,7 @@ def _publication_provenance( package: PortableResultPackageManifest, artifact_manifest: ArtifactManifest, profile: PortableCalculationProfilePolicy, + replay_capability: LabReplayCapability, ) -> dict[str, object]: result_document = next( artifact for artifact in package.artifacts if artifact.role == RESULT_DOCUMENT_ROLE @@ -732,10 +752,11 @@ def _publication_provenance( "result_document_sha256": result_document.sha256, "artifacts": [artifact.as_dict() for artifact in package.artifacts], }, + "replay_capability": replay_capability.as_dict(), "storage": { "mode": "central-content-addressed-artifact-store", "include_recorded_media": profile.include_recorded_media, - "replay_capability": None, + "replay_capability": replay_capability.as_dict(), }, "method": _laboratory_method(job, definition), } diff --git a/src/k1link/observatory/portable_result_view.py b/src/k1link/observatory/portable_result_view.py new file mode 100644 index 0000000..0dc3126 --- /dev/null +++ b/src/k1link/observatory/portable_result_view.py @@ -0,0 +1,118 @@ +"""Read-only universal viewer projection for published portable LAB results.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from typing import Final + +from k1link.artifact_gateway import ArtifactGatewayError, CentralArtifactStore +from k1link.observatory.portable_result_contract import RESULT_DOCUMENT_ROLE +from k1link.sessions import SessionNotFoundError, SessionStore + +PORTABLE_RESULT_VIEW_SCHEMA: Final = "missioncore.observatory-portable-result-view/v1" +_RESULT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_MAX_RESULT_DOCUMENT_BYTES: Final = 8 * 1024 * 1024 + + +class PortableResultViewError(RuntimeError): + """A published result cannot be admitted to the generic viewer.""" + + +@dataclass(frozen=True, slots=True) +class PortableResultViewService: + sessions: SessionStore + artifacts: CentralArtifactStore + + def read(self, result_id: str) -> dict[str, object]: + if _RESULT_ID.fullmatch(result_id) is None: + raise ValueError("portable result id is invalid") + try: + binding = self.sessions.get_lab_instance(result_id) + except SessionNotFoundError as exc: + raise PortableResultViewError("portable result is unavailable") from exc + if ( + binding is None + or binding.session_id != result_id + or binding.result_id != result_id + or binding.replay_capability is None + or binding.replay_capability.kind != "portable-result-review" + ): + raise PortableResultViewError("portable result has no viewer capability") + provenance = binding.provenance + package = provenance.get("result_package") + source = provenance.get("source") + run_definition = provenance.get("run_definition") + calculation_profile = provenance.get("calculation_profile") + if ( + provenance.get("schema_version") + != "missioncore.observatory-portable-result-publication/v1" + or provenance.get("replay_capability") + != binding.replay_capability.as_dict() + or not isinstance(package, dict) + or not isinstance(source, dict) + or source.get("session_id") != binding.source_session_id + or not isinstance(run_definition, dict) + or run_definition.get("definition_sha256") != binding.config_sha256 + or not isinstance(calculation_profile, dict) + ): + raise PortableResultViewError("portable result provenance is invalid") + manifest_id = package.get("artifact_manifest_id") + package_sha256 = package.get("manifest_sha256") + if ( + not isinstance(manifest_id, str) + or _SHA256.fullmatch(manifest_id) is None + or not isinstance(package_sha256, str) + or _SHA256.fullmatch(package_sha256) is None + ): + raise PortableResultViewError("portable result artifact identity is invalid") + try: + manifest = self.artifacts.read_manifest(manifest_id) + member = manifest.member(RESULT_DOCUMENT_ROLE) + path = self.artifacts.object_path(member.sha256) + payload = path.read_bytes() + except (ArtifactGatewayError, OSError) as exc: + raise PortableResultViewError("portable result artifacts are unavailable") from exc + if ( + manifest.artifact_type != "observatory-portable-result" + or manifest.subject_id != result_id + or manifest.manifest_id != manifest_id + or manifest.metadata.get("package-sha256") != package_sha256 + or package.get("schema_version") + != "missioncore.observatory-portable-result-package/v1" + or package.get("result_document_sha256") != member.sha256 + or member.media_type != "application/json" + or not 0 < member.byte_length <= _MAX_RESULT_DOCUMENT_BYTES + or len(payload) != member.byte_length + or hashlib.sha256(payload).hexdigest() != member.sha256 + ): + raise PortableResultViewError("portable result artifact integrity changed") + try: + result_document: object = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableResultViewError("portable result document is invalid") from exc + if not isinstance(result_document, dict): + raise PortableResultViewError("portable result document must be an object") + return { + "schema_version": PORTABLE_RESULT_VIEW_SCHEMA, + "result_id": result_id, + "source_session_id": binding.source_session_id, + "result_kind": binding.result_kind, + "definition_sha256": binding.config_sha256, + "viewer_capability": binding.replay_capability.as_dict(), + "calculation_profile": calculation_profile, + "artifact_manifest_id": manifest.manifest_id, + "artifacts": [ + { + "role": item.role, + "media_type": item.media_type, + "sha256": item.sha256, + "byte_length": item.byte_length, + } + for item in manifest.members + ], + "result_document": result_document, + } diff --git a/src/k1link/observatory/portable_worker_integration.py b/src/k1link/observatory/portable_worker_integration.py index 7cf26a4..19e2ac9 100644 --- a/src/k1link/observatory/portable_worker_integration.py +++ b/src/k1link/observatory/portable_worker_integration.py @@ -2,9 +2,9 @@ This module is deliberately only a composition boundary. It does not enable the Worker router, install an executor, select commands, or grant production -authority. It binds the two admitted portable profiles to their exact result -contracts, then constructs the local artifact transport and verified result -publisher from server-owned dependencies. +authority. Validators are selected by the content identity of a result +contract, never by a LAB/setup ID, before constructing the local artifact +transport and verified result publisher from server-owned dependencies. """ from __future__ import annotations @@ -19,7 +19,6 @@ from typing import Final from k1link.artifact_gateway import CentralArtifactStore from k1link.observatory.m49_portable_result import ( M49_PORTABLE_RESULT_CONTRACT_SHA256, - M49_PORTABLE_RESULT_SCHEMA, validate_m49_portable_result, ) from k1link.observatory.portable_artifact_transport import ( @@ -27,13 +26,10 @@ from k1link.observatory.portable_artifact_transport import ( PortableObservatoryArtifactTransport, ) from k1link.observatory.portable_lab_v1_executor import ( - PORTABLE_LAB_V1_RESULT_SCHEMA, validate_lab_v1_result_v2, ) from k1link.observatory.portable_result_contract import ( - OBSERVATION_ONLY_AUTHORITY, PortableCalculationProfileRegistry, - PortableResultContractValidator, PortableResultContractValidatorRegistration, PortableResultContractValidatorRegistry, PortableResultPublicationBlockedError, @@ -42,14 +38,10 @@ from k1link.observatory.portable_result_publisher import ( PortableObservatoryResultPublisher, ) from k1link.observatory.portable_run_definitions import ( - PORTABLE_RESULT_CONTRACT_SCHEMA, - PortableRunDefinition, PortableRunDefinitionRegistry, PortableRunDefinitionRegistryError, ) from k1link.observatory.portable_setup_projection import ( - PORTABLE_LAB_V1_SETUP_ID, - PORTABLE_M49_SETUP_ID, portable_calculation_profile_registry, ) from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue @@ -161,77 +153,54 @@ class PortableObservatoryWorkerIntegration: supported_setup_ids: tuple[str, ...] -@dataclass(frozen=True, slots=True) -class _ValidatorSpec: - setup_id: str - definition_id: str - definition_version: int - contract_id: str - contract_version: int - result_schema: str - result_kind: str - contract_sha256: str - validator: PortableResultContractValidator - - def expected_contract(self) -> dict[str, object]: - return { - "schema_version": PORTABLE_RESULT_CONTRACT_SCHEMA, - "contract_id": self.contract_id, - "version": self.contract_version, - "result_schema": self.result_schema, - "result_kind": self.result_kind, - "publication": "observatory", - "contract_sha256": self.contract_sha256, - } - - -_VALIDATOR_SPECS: Final = ( - _ValidatorSpec( - setup_id=PORTABLE_LAB_V1_SETUP_ID, - definition_id="lab-v1-eomt-ddrnet-portable", - definition_version=2, - contract_id="recorded-eomt-ddrnet-review-v2", - contract_version=2, - result_schema=PORTABLE_LAB_V1_RESULT_SCHEMA, - result_kind="recorded-perception-qualification", +_BUILTIN_VALIDATOR_REGISTRATIONS: Final = ( + PortableResultContractValidatorRegistration( contract_sha256=PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256, validator=validate_lab_v1_result_v2, ), - _ValidatorSpec( - setup_id=PORTABLE_M49_SETUP_ID, - definition_id="m49-tgs-portable", - definition_version=3, - contract_id="m49-tgs-portable-review-v2", - contract_version=2, - result_schema=M49_PORTABLE_RESULT_SCHEMA, - result_kind="recorded-perception-qualification", + PortableResultContractValidatorRegistration( contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256, validator=validate_m49_portable_result, ), ) +_BUILTIN_VALIDATORS_BY_CONTRACT: Final = { + registration.contract_sha256: registration.validator + for registration in _BUILTIN_VALIDATOR_REGISTRATIONS +} def portable_result_validator_registry( definitions: PortableRunDefinitionRegistry, + *, + registrations: tuple[PortableResultContractValidatorRegistration, ...] + | None = None, ) -> PortableResultContractValidatorRegistry: - """Bind both product profiles to fixed result contracts and validators.""" + """Select server-installed validators by exact result-contract identity.""" - registrations: list[PortableResultContractValidatorRegistration] = [] - for spec in _VALIDATOR_SPECS: - try: - definition = definitions.resolve_setup(spec.setup_id) - except PortableRunDefinitionRegistryError as exc: - raise PortableWorkerIntegrationError( - f"required portable setup is unavailable: {spec.setup_id}" - ) from exc - _verify_validator_definition(definition, spec) - registrations.append( - PortableResultContractValidatorRegistration( - contract_sha256=spec.contract_sha256, - validator=spec.validator, - ) - ) - return PortableResultContractValidatorRegistry(tuple(registrations)) + installed = ( + _BUILTIN_VALIDATOR_REGISTRATIONS + if registrations is None + else registrations + ) + by_contract = { + registration.contract_sha256: registration + for registration in PortableResultContractValidatorRegistry(installed).registrations + } + selected = tuple( + by_contract[definition.result_contract.contract_sha256] + for definition in definitions.definitions + if definition.result_contract.contract_sha256 in by_contract + ) + registry = PortableResultContractValidatorRegistry(selected) + for definition in definitions.definitions: + if definition.executor.ready: + try: + registry.resolve(definition.result_contract.contract_sha256) + except PortableResultPublicationBlockedError as exc: + raise PortableWorkerIntegrationError( + "a ready portable definition has no exact result validator" + ) from exc + return registry def observatory_worker_local_enabled( @@ -314,42 +283,43 @@ def build_portable_observatory_worker_integration( validators=validator_registry, artifact_transport=transport, result_publisher=publisher, - supported_setup_ids=tuple(spec.setup_id for spec in _VALIDATOR_SPECS), + supported_setup_ids=tuple( + definition.setup_id + for definition in definitions.definitions + if any( + registration.contract_sha256 + == definition.result_contract.contract_sha256 + for registration in validator_registry.registrations + ) + ), ) -def _verify_validator_definition( - definition: PortableRunDefinition, - spec: _ValidatorSpec, -) -> None: - if ( - definition.definition_id != spec.definition_id - or definition.version != spec.definition_version - or definition.result_contract.as_dict() != spec.expected_contract() - or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY - ): - raise PortableWorkerIntegrationError( - f"portable result contract changed for setup: {spec.setup_id}" - ) - - def _verify_composition( definitions: PortableRunDefinitionRegistry, profiles: PortableCalculationProfileRegistry, validators: PortableResultContractValidatorRegistry, ) -> None: - expected_validators = {spec.contract_sha256: spec.validator for spec in _VALIDATOR_SPECS} for definition in definitions.definitions: profiles.resolve(definition) + contract_sha256 = definition.result_contract.contract_sha256 if definition.executor.ready: try: - validators.resolve(definition.result_contract.contract_sha256) + validators.resolve(contract_sha256) except PortableResultPublicationBlockedError as exc: raise PortableWorkerIntegrationError( "a ready portable definition has no exact result validator" ) from exc - for contract_sha256, expected in expected_validators.items(): - if validators.resolve(contract_sha256) is not expected: + expected = _BUILTIN_VALIDATORS_BY_CONTRACT.get(contract_sha256) + actual = next( + ( + registration.validator + for registration in validators.registrations + if registration.contract_sha256 == contract_sha256 + ), + None, + ) + if expected is not None and actual is not None and actual is not expected: raise PortableWorkerIntegrationError( "portable result validator registration changed identity" ) diff --git a/src/k1link/observatory/portable_worker_runtime.py b/src/k1link/observatory/portable_worker_runtime.py index a3cd2aa..4e119bf 100644 --- a/src/k1link/observatory/portable_worker_runtime.py +++ b/src/k1link/observatory/portable_worker_runtime.py @@ -19,7 +19,7 @@ import json import re from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Final, Literal, Protocol from k1link.observatory.portable_run_definitions import ( @@ -40,7 +40,7 @@ PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: Final = ( "missioncore.observatory-portable-worker-runtime-candidate/v1" ) PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA: Final = ( - "missioncore.observatory-portable-worker-runtime-plan/v1" + "missioncore.observatory-portable-worker-runtime-plan/v2" ) _MAX_REGISTRY_BYTES: Final = 256 * 1024 @@ -63,6 +63,7 @@ type RuntimeAssetKind = Literal[ "container-image", "definition-component", "local-file", + "local-tree", "model-artifact", ] type AssetVerificationState = Literal["matched", "missing", "mismatched"] @@ -116,6 +117,7 @@ class PortableWorkerAssetRequirement: "container-image", "definition-component", "local-file", + "local-tree", "model-artifact", ): raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid") @@ -160,6 +162,18 @@ class PortableWorkerAssetRequirement: raise PortableWorkerRuntimeRegistryError( "model artifact requirement is incomplete" ) + elif self.kind == "local-tree": + if self.byte_length is None or any( + value is not None + for value in ( + self.component_id, + self.model_release_id, + self.model_artifact_role, + ) + ): + raise PortableWorkerRuntimeRegistryError( + "local tree requirement is incomplete" + ) elif any( value is not None for value in ( @@ -478,6 +492,8 @@ def verify_local_assets( continue if requirement.kind == "container-image": matched = binding.file_path is None and binding.image_sha256 == requirement.sha256 + elif requirement.kind == "local-tree": + matched = _matches_tree(requirement, binding.file_path) else: matched = _matches_file(requirement, binding.file_path) checks.append( @@ -568,12 +584,23 @@ class PortableWorkerResultDraft: @dataclass(frozen=True, slots=True) class PortableWorkerRuntimePlan: job_id: str + request_sha256: str + identity_sha256: str + submission_receipt_sha256: str + claim_generation: int adapter_id: str candidate_sha256: str setup_id: str + definition_id: str + definition_version: int definition_sha256: str + source_session_id: str + source_catalog_sha256: str source_bundle_sha256: str source_capability_manifest_sha256: str + source_adapter_id: str + source_adapter_version: int + source_adapter_sha256: str result_contract_sha256: str phases: tuple[str, ...] @@ -582,19 +609,38 @@ class PortableWorkerRuntimePlan: for value, label in ( (self.adapter_id, "runtime plan adapter id"), (self.setup_id, "runtime plan setup id"), + (self.definition_id, "runtime plan definition id"), + (self.source_adapter_id, "runtime plan source adapter id"), ): _pattern(value, _IDENTIFIER, label) for value, label in ( + (self.request_sha256, "runtime plan request sha256"), + (self.identity_sha256, "runtime plan identity sha256"), + (self.submission_receipt_sha256, "runtime plan submission receipt sha256"), (self.candidate_sha256, "runtime plan candidate sha256"), (self.definition_sha256, "runtime plan definition sha256"), + (self.source_catalog_sha256, "runtime plan source catalog sha256"), (self.source_bundle_sha256, "runtime plan source bundle sha256"), ( self.source_capability_manifest_sha256, "runtime plan source capability sha256", ), + (self.source_adapter_sha256, "runtime plan source adapter sha256"), (self.result_contract_sha256, "runtime plan result contract sha256"), ): _digest(value, label) + _pattern(self.source_session_id, _SESSION_ID, "runtime plan source session id") + for integer_value, label in ( + (self.claim_generation, "runtime plan claim generation"), + (self.definition_version, "runtime plan definition version"), + (self.source_adapter_version, "runtime plan source adapter version"), + ): + if ( + isinstance(integer_value, bool) + or not isinstance(integer_value, int) + or integer_value < 1 + ): + raise PortableWorkerRuntimeRegistryError(f"{label} is invalid") if not self.phases or len(self.phases) != len(set(self.phases)): raise PortableWorkerRuntimeRegistryError( "runtime plan phases must be non-empty and unique" @@ -606,12 +652,23 @@ class PortableWorkerRuntimePlan: return { "schema_version": PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA, "job_id": self.job_id, + "request_sha256": self.request_sha256, + "identity_sha256": self.identity_sha256, + "submission_receipt_sha256": self.submission_receipt_sha256, + "claim_generation": self.claim_generation, "adapter_id": self.adapter_id, "candidate_sha256": self.candidate_sha256, "setup_id": self.setup_id, + "definition_id": self.definition_id, + "definition_version": self.definition_version, "definition_sha256": self.definition_sha256, + "source_session_id": self.source_session_id, + "source_catalog_sha256": self.source_catalog_sha256, "source_bundle_sha256": self.source_bundle_sha256, "source_capability_manifest_sha256": self.source_capability_manifest_sha256, + "source_adapter_id": self.source_adapter_id, + "source_adapter_version": self.source_adapter_version, + "source_adapter_sha256": self.source_adapter_sha256, "result_contract_sha256": self.result_contract_sha256, "phases": list(self.phases), "authority": dict(_AUTHORITY), @@ -687,12 +744,23 @@ class PortableWorkerExecutorAdapter: ) plan = PortableWorkerRuntimePlan( job_id=job.job_id, + request_sha256=job.request_sha256, + identity_sha256=job.identity_sha256, + submission_receipt_sha256=job.submission_receipt_sha256, + claim_generation=job.claim_generation, adapter_id=self.candidate.adapter_id, candidate_sha256=self.candidate.candidate_sha256, setup_id=job.setup_id, + definition_id=job.definition_id, + definition_version=job.definition_version, definition_sha256=job.definition_sha256, + source_session_id=job.source_session_id, + source_catalog_sha256=job.source_catalog_sha256, source_bundle_sha256=job.source_bundle_sha256, source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_id=job.source_adapter_id, + source_adapter_version=job.source_adapter_version, + source_adapter_sha256=job.source_adapter_sha256, result_contract_sha256=self.candidate.result_contract_sha256, phases=tuple(phase.phase_id for phase in self.candidate.phases), ) @@ -758,6 +826,160 @@ def _matches_file( return False +def _matches_tree( + requirement: PortableWorkerAssetRequirement, + path: Path | None, +) -> bool: + """Verify an install-time sealed tree without re-reading multi-GB assets.""" + + if path is None or requirement.byte_length is None: + return False + root = path.expanduser().absolute() + receipt_path = root / "tree-receipt.json" + manifest_path = root / "tree-manifest.tsv" + try: + if root.is_symlink() or not root.is_dir(): + return False + if ( + receipt_path.is_symlink() + or not receipt_path.is_file() + or manifest_path.is_symlink() + or not manifest_path.is_file() + ): + return False + receipt_payload = receipt_path.read_bytes() + manifest_payload = manifest_path.read_bytes() + if not 0 < len(receipt_payload) <= 64 * 1024: + return False + if not 0 < len(manifest_payload) <= 16 * 1024 * 1024: + return False + receipt_value: object = json.loads(receipt_payload.decode("utf-8")) + if not isinstance(receipt_value, dict): + return False + receipt = receipt_value + base_keys = { + "schema_version", + "asset_id", + "identity_algorithm", + "identity_sha256", + "file_count", + "byte_length", + "manifest_relative_path", + } + provenance_keys = {"source_image_sha256", "source_path", "binaries"} + receipt_keys = set(receipt) + if receipt_keys != base_keys and receipt_keys != base_keys | provenance_keys: + return False + if ( + receipt["schema_version"] != "missioncore.sealed-tree-runtime/v1" + or receipt["asset_id"] != requirement.asset_id + or receipt["identity_algorithm"] + != "relative-path-tab-size-tab-file-sha256-lf/v1" + or receipt["identity_sha256"] != requirement.sha256 + or receipt["byte_length"] != requirement.byte_length + or receipt["manifest_relative_path"] != "tree-manifest.tsv" + or hashlib.sha256(manifest_payload).hexdigest() != requirement.sha256 + ): + return False + rows = _sealed_tree_rows(manifest_payload) + if receipt["file_count"] != len(rows): + return False + if sum(byte_length for _, byte_length, _ in rows) != requirement.byte_length: + return False + if receipt_keys == base_keys | provenance_keys and not _matches_tree_provenance( + receipt, + rows, + ): + return False + for relative_path, byte_length, _sha256 in rows: + member = root.joinpath(*PurePosixPath(relative_path).parts) + if member.is_symlink() or not member.is_file() or member.stat().st_size != byte_length: + return False + return True + except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError): + return False + + +def _matches_tree_provenance( + receipt: Mapping[str, object], + rows: tuple[tuple[str, int, str], ...], +) -> bool: + source_image_sha256 = receipt.get("source_image_sha256") + source_path_value = receipt.get("source_path") + binaries = receipt.get("binaries") + if ( + not isinstance(source_image_sha256, str) + or _SHA256.fullmatch(source_image_sha256) is None + or not isinstance(source_path_value, str) + or not source_path_value + or not isinstance(binaries, dict) + or not binaries + ): + return False + source_path = PurePosixPath(source_path_value) + if not source_path.is_absolute() or ".." in source_path.parts: + return False + by_path = { + relative_path: (byte_length, sha256) + for relative_path, byte_length, sha256 in rows + } + for binary_id, value in binaries.items(): + if ( + not isinstance(binary_id, str) + or _IDENTIFIER.fullmatch(binary_id) is None + or not isinstance(value, dict) + or set(value) != {"relative_path", "byte_length", "sha256"} + ): + return False + relative_path = value["relative_path"] + byte_length = value["byte_length"] + sha256 = value["sha256"] + if ( + not isinstance(relative_path, str) + or not isinstance(byte_length, int) + or isinstance(byte_length, bool) + or byte_length < 0 + or not isinstance(sha256, str) + or _SHA256.fullmatch(sha256) is None + or by_path.get(relative_path) != (byte_length, sha256) + ): + return False + return True + + +def _sealed_tree_rows(payload: bytes) -> tuple[tuple[str, int, str], ...]: + text = payload.decode("utf-8") + if not text.endswith("\n"): + raise ValueError("sealed tree manifest is not newline terminated") + rows: list[tuple[str, int, str]] = [] + previous_path: str | None = None + for line in text.splitlines(): + parts = line.split("\t") + if len(parts) != 3: + raise ValueError("sealed tree manifest row changed") + relative_path, byte_length_text, sha256 = parts + path = PurePosixPath(relative_path) + if ( + not relative_path + or path.is_absolute() + or ".." in path.parts + or "\\" in relative_path + or relative_path in {"tree-manifest.tsv", "tree-receipt.json"} + or previous_path is not None + and relative_path <= previous_path + or _SHA256.fullmatch(sha256) is None + ): + raise ValueError("sealed tree manifest identity changed") + byte_length = int(byte_length_text) + if byte_length < 0 or str(byte_length) != byte_length_text: + raise ValueError("sealed tree manifest byte length changed") + rows.append((relative_path, byte_length, sha256)) + previous_path = relative_path + if not rows: + raise ValueError("sealed tree manifest is empty") + return tuple(rows) + + def _candidate(value: object) -> PortableWorkerRuntimeCandidate: row = _object(value, "runtime candidate") _exact_keys( @@ -844,6 +1066,7 @@ def _asset(value: object) -> PortableWorkerAssetRequirement: "container-image", "definition-component", "local-file", + "local-tree", "model-artifact", ): raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid") diff --git a/src/k1link/observatory/recorded_jobs.py b/src/k1link/observatory/recorded_jobs.py index d34931b..5ac9385 100644 --- a/src/k1link/observatory/recorded_jobs.py +++ b/src/k1link/observatory/recorded_jobs.py @@ -44,6 +44,10 @@ OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = "missioncore.observatory-live-k1- RECORDED_JOB_DATABASE_NAME: Final = "observatory-recorded-jobs.sqlite3" MAX_RECORDED_JOBS: Final = 10_000 MAX_RECORDED_CLAIM_RECEIPTS: Final = 50_000 +# Every capability expands to four SQLite bind parameters. Keeping the public +# bound at 128 stays comfortably below SQLite's traditional 999-variable limit +# even when the runtime was compiled with conservative defaults. +MAX_RECORDED_EXECUTOR_CAPABILITIES: Final = 128 MAX_LIVE_LEASES: Final = 10_000 MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024 RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1 @@ -67,6 +71,7 @@ type RecordedJobState = Literal[ "reconciliation-required", ] type CheckpointPolicy = Literal["cooperative", "non-checkpointable"] +type PublicationState = Literal["not-required", "pending", "failed", "published"] type LiveLeaseState = Literal["pending", "active", "completed", "failed", "cancelled"] type LiveTerminalOutcome = Literal["completed", "failed", "cancelled"] @@ -139,6 +144,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs ( terminal_code TEXT, terminal_message TEXT, terminal_claim_token_sha256 TEXT, + publication_state TEXT NOT NULL DEFAULT 'not-required' + CHECK (publication_state IN ('not-required', 'pending', 'failed', 'published')), + publication_attempts INTEGER NOT NULL DEFAULT 0 + CHECK (publication_attempts >= 0), + publication_error TEXT, + published_at_utc TEXT, created_at_utc TEXT NOT NULL, updated_at_utc TEXT NOT NULL ); @@ -256,6 +267,33 @@ class ObservatoryRecordedPreemptionError(ObservatoryRecordedQueueError): """The scheduler could not prove immediate release for live K1.""" +@dataclass(frozen=True, slots=True, order=True) +class RecordedExecutorIdentity: + """Path-free executor capability shared by scheduling and Worker code.""" + + release_sha256: str + image_sha256: str + model_manifest_sha256: str + resource_profile_sha256: str + + def __post_init__(self) -> None: + for value, label in ( + (self.release_sha256, "executor release sha256"), + (self.image_sha256, "executor image sha256"), + (self.model_manifest_sha256, "model manifest sha256"), + (self.resource_profile_sha256, "resource profile sha256"), + ): + _validate_digest(value, label) + + def as_dict(self) -> dict[str, str]: + return { + "release_sha256": self.release_sha256, + "image_sha256": self.image_sha256, + "model_manifest_sha256": self.model_manifest_sha256, + "resource_profile_sha256": self.resource_profile_sha256, + } + + @dataclass(frozen=True, slots=True) class RecordedRunDefinition: """Server-owned executable identity; it contains no executable text or path.""" @@ -432,6 +470,10 @@ class ObservatoryRecordedJob: terminal_code: str | None terminal_message: str | None terminal_claim_token_sha256: str | None + publication_state: PublicationState + publication_attempts: int + publication_error: str | None + published_at_utc: str | None created_at_utc: str updated_at_utc: str priority_class: Literal["recorded"] = "recorded" @@ -557,6 +599,59 @@ class ObservatoryRecordedJob: _SHA256, "terminal claim token sha256", ) + if self.publication_state not in ( + "not-required", + "pending", + "failed", + "published", + ): + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job publication state is invalid" + ) + if self.publication_attempts < 0: + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job publication attempt count is invalid" + ) + if self.publication_error is not None: + _validate_text( + self.publication_error, + "recorded-job publication error", + max_length=1_000, + ) + if self.published_at_utc is not None: + _validate_timestamp(self.published_at_utc, "recorded-job publication timestamp") + publication_shape = { + "not-required": ( + self.publication_attempts == 0 + and self.publication_error is None + and self.published_at_utc is None + ), + "pending": ( + self.publication_error is None and self.published_at_utc is None + ), + "failed": ( + self.publication_attempts >= 1 + and self.publication_error is not None + and self.published_at_utc is None + ), + "published": ( + self.publication_attempts >= 1 + and self.publication_error is None + and self.published_at_utc is not None + ), + } + if not publication_shape[self.publication_state]: + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job publication receipt is inconsistent" + ) + if self.publication_state != "not-required" and ( + self.state != "succeeded" + or self.result_id is None + or self.result_sha256 is None + ): + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job publication lifecycle has no execution result" + ) _validate_timestamp(self.created_at_utc, "created timestamp") _validate_timestamp(self.updated_at_utc, "updated timestamp") @@ -630,6 +725,12 @@ class ObservatoryRecordedJob: if self.terminal_code is None else {"code": self.terminal_code, "message": self.terminal_message} ), + "publication": { + "state": self.publication_state, + "attempts": self.publication_attempts, + "error": self.publication_error, + "published_at_utc": self.published_at_utc, + }, "created_at_utc": self.created_at_utc, "updated_at_utc": self.updated_at_utc, "authority": dict(_AUTHORITY), @@ -1192,12 +1293,23 @@ class ObservatoryRecordedJobQueue: *, claimant_id: str, claim_request_id: str, + supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None = None, ) -> ObservatoryRecordedClaim | None: - """Claim one recorded job atomically; even an empty claim is idempotent.""" + """Claim one compatible job atomically; even an empty claim is idempotent. + + ``None`` preserves the legacy v1 claim semantics during rollout. A + concrete tuple is the capability-aware v2 contract; an empty tuple + deliberately claims nothing. + """ _validate_pattern(claimant_id, _IDENTIFIER, "claimant id") _validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id") - request_sha256 = _claim_request_sha256(claimant_id, claim_request_id) + capabilities = _canonical_executor_capabilities(supported_executor_identities) + request_sha256 = _claim_request_sha256( + claimant_id, + claim_request_id, + supported_executor_identities=capabilities, + ) with self._transaction() as connection: now = self._timestamp() self._recover_stale_claims(connection, now=now) @@ -1226,11 +1338,10 @@ class ObservatoryRecordedJobQueue: "LIMIT 1" ).fetchone() if active_owner is None: - row = connection.execute( - "SELECT job_id FROM observatory_recorded_jobs " - "WHERE state = 'queued' " - "ORDER BY priority_rank, created_at_utc, job_id LIMIT 1" - ).fetchone() + row = self._next_compatible_queued_job( + connection, + supported_executor_identities=capabilities, + ) if row is None: connection.execute( "INSERT INTO observatory_recorded_claim_receipts " @@ -1289,6 +1400,44 @@ class ObservatoryRecordedJobQueue: job=self._get_job(connection, job_id), ) + @staticmethod + def _next_compatible_queued_job( + connection: sqlite3.Connection, + *, + supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None, + ) -> sqlite3.Row | None: + if supported_executor_identities is None: + row: sqlite3.Row | None = connection.execute( + "SELECT job_id FROM observatory_recorded_jobs " + "WHERE state = 'queued' " + "ORDER BY priority_rank, created_at_utc, job_id LIMIT 1" + ).fetchone() + return row + if not supported_executor_identities: + return None + predicates = " OR ".join( + "(executor_release_sha256 = ? AND executor_image_sha256 = ? " + "AND model_manifest_sha256 = ? AND resource_profile_sha256 = ?)" + for _identity in supported_executor_identities + ) + parameters = tuple( + value + for identity in supported_executor_identities + for value in ( + identity.release_sha256, + identity.image_sha256, + identity.model_manifest_sha256, + identity.resource_profile_sha256, + ) + ) + row = connection.execute( + "SELECT job_id FROM observatory_recorded_jobs " + f"WHERE state = 'queued' AND ({predicates}) " + "ORDER BY priority_rank, created_at_utc, job_id LIMIT 1", + parameters, + ).fetchone() + return row + def renew_claim( self, job_id: str, @@ -1653,6 +1802,96 @@ class ObservatoryRecordedJobQueue: terminal_message="Recorded result was sealed by the Worker.", ) + def complete_for_publication( + self, + job_id: str, + *, + claim_token: str, + result_id: str, + result_sha256: str, + ) -> ObservatoryRecordedJob: + """Seal execution and atomically enqueue its verified publication.""" + + _validate_pattern(result_id, _SESSION_ID, "result id") + _validate_digest(result_sha256, "result sha256") + return self._terminal_job_transition( + job_id, + claim_token=claim_token, + state="succeeded", + result_id=result_id, + result_sha256=result_sha256, + terminal_code="result-sealed", + terminal_message="Recorded result was sealed by the Worker.", + publication_state="pending", + ) + + def mark_publication_failed( + self, + job_id: str, + *, + message: str, + ) -> ObservatoryRecordedJob: + """Record one failed outbox attempt without losing the execution result.""" + + _validate_pattern(job_id, _JOB_ID, "recorded job id") + _validate_text(message, "recorded publication error", max_length=1_000) + with self._transaction() as connection: + job = self._get_job(connection, job_id) + if job.state != "succeeded" or job.publication_state not in { + "pending", + "failed", + }: + raise ObservatoryRecordedQueueConflictError( + "recorded result is not awaiting publication" + ) + connection.execute( + "UPDATE observatory_recorded_jobs " + "SET publication_state = 'failed', " + "publication_attempts = publication_attempts + 1, " + "publication_error = ?, published_at_utc = NULL, " + "updated_at_utc = ? WHERE job_id = ?", + (message, self._timestamp(), job_id), + ) + return self._get_job(connection, job_id) + + def mark_published(self, job_id: str) -> ObservatoryRecordedJob: + """Acknowledge one idempotently published outbox entry.""" + + _validate_pattern(job_id, _JOB_ID, "recorded job id") + with self._transaction() as connection: + job = self._get_job(connection, job_id) + if job.state != "succeeded": + raise ObservatoryRecordedQueueConflictError( + "recorded execution has not succeeded" + ) + if job.publication_state == "published": + return job + if job.publication_state not in {"pending", "failed"}: + raise ObservatoryRecordedQueueConflictError( + "recorded result has no publication outbox entry" + ) + now = self._timestamp() + connection.execute( + "UPDATE observatory_recorded_jobs " + "SET publication_state = 'published', " + "publication_attempts = publication_attempts + 1, " + "publication_error = NULL, published_at_utc = ?, " + "updated_at_utc = ? WHERE job_id = ?", + (now, now, job_id), + ) + return self._get_job(connection, job_id) + + def pending_publications(self) -> tuple[ObservatoryRecordedJob, ...]: + """Return durable outbox entries in deterministic retry order.""" + + with self._read_connection() as connection: + rows = connection.execute( + "SELECT * FROM observatory_recorded_jobs " + "WHERE publication_state IN ('pending', 'failed') " + "ORDER BY created_at_utc, job_id" + ).fetchall() + return tuple(_job_from_row(row) for row in rows) + def fail( self, job_id: str, @@ -2038,6 +2277,7 @@ class ObservatoryRecordedJobQueue: result_sha256: str | None, terminal_code: str, terminal_message: str, + publication_state: Literal["not-required", "pending"] = "not-required", ) -> ObservatoryRecordedJob: _validate_pattern(job_id, _JOB_ID, "recorded job id") _validate_pattern(claim_token, _TOKEN, "claim token") @@ -2053,6 +2293,11 @@ class ObservatoryRecordedJobQueue: and job.terminal_code == terminal_code and job.terminal_message == terminal_message and job.terminal_claim_token_sha256 == token_sha256 + and ( + job.publication_state == "not-required" + if publication_state == "not-required" + else job.publication_state in {"pending", "failed", "published"} + ) ) if exact_replay: return job @@ -2091,7 +2336,9 @@ class ObservatoryRecordedJobQueue: "terminal_claim_token_sha256 = ?, active_claim_token = NULL, " "active_claimant_id = NULL, claimed_at_utc = NULL, " "claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, " - "claim_renewal_count = 0, updated_at_utc = ? WHERE job_id = ?", + "claim_renewal_count = 0, publication_state = ?, " + "publication_attempts = 0, publication_error = NULL, " + "published_at_utc = NULL, updated_at_utc = ? WHERE job_id = ?", ( state, result_id, @@ -2099,6 +2346,7 @@ class ObservatoryRecordedJobQueue: terminal_code, terminal_message, token_sha256, + publication_state, now, job_id, ), @@ -2387,6 +2635,7 @@ class ObservatoryRecordedJobQueue: with self._connect() as connection: connection.executescript(_SCHEMA_SQL) self._migrate_claim_lease_schema(connection) + self._migrate_publication_schema(connection) self._validate_schema(connection) self._validate_existing_capacity(connection) connection.commit() @@ -2403,7 +2652,7 @@ class ObservatoryRecordedJobQueue: def _validate_schema(self, connection: sqlite3.Connection) -> None: expected = { - "observatory_recorded_jobs": 46, + "observatory_recorded_jobs": 50, "observatory_recorded_claim_receipts": 6, "observatory_recorded_reconciliations": 18, "observatory_live_leases": 13, @@ -2508,6 +2757,35 @@ class ObservatoryRecordedJobQueue: "legacy active claim is stored in an invalid state" ) + def _migrate_publication_schema(self, connection: sqlite3.Connection) -> None: + """Add the durable result-publication outbox to existing queues.""" + + columns = { + str(row["name"]) + for row in connection.execute( + "SELECT name FROM pragma_table_info('observatory_recorded_jobs')" + ).fetchall() + } + additions = ( + ( + "publication_state", + "TEXT NOT NULL DEFAULT 'not-required' " + "CHECK (publication_state IN " + "('not-required', 'pending', 'failed', 'published'))", + ), + ( + "publication_attempts", + "INTEGER NOT NULL DEFAULT 0 CHECK (publication_attempts >= 0)", + ), + ("publication_error", "TEXT"), + ("published_at_utc", "TEXT"), + ) + for name, definition in additions: + if name not in columns: + connection.execute( + f"ALTER TABLE observatory_recorded_jobs ADD COLUMN {name} {definition}" + ) + def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None: for table, limit, label in ( ("observatory_recorded_jobs", self._max_jobs, "recorded job"), @@ -2724,6 +3002,10 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob: terminal_code=row["terminal_code"], terminal_message=row["terminal_message"], terminal_claim_token_sha256=row["terminal_claim_token_sha256"], + publication_state=row["publication_state"], + publication_attempts=row["publication_attempts"], + publication_error=row["publication_error"], + published_at_utc=row["published_at_utc"], created_at_utc=row["created_at_utc"], updated_at_utc=row["updated_at_utc"], ) @@ -2801,14 +3083,37 @@ def _submission_receipt_sha256( ) -def _claim_request_sha256(claimant_id: str, claim_request_id: str) -> str: - return _sha256( - { - "schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA, - "claim_request_id": claim_request_id, - "claimant_id": claimant_id, - } - ) +def _claim_request_sha256( + claimant_id: str, + claim_request_id: str, + *, + supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None = None, +) -> str: + document: dict[str, object] = { + "schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA, + "claim_request_id": claim_request_id, + "claimant_id": claimant_id, + } + if supported_executor_identities is not None: + document["supported_executor_identities"] = [ + identity.as_dict() for identity in supported_executor_identities + ] + return _sha256(document) + + +def _canonical_executor_capabilities( + identities: tuple[RecordedExecutorIdentity, ...] | None, +) -> tuple[RecordedExecutorIdentity, ...] | None: + if identities is None: + return None + if len(identities) > MAX_RECORDED_EXECUTOR_CAPABILITIES: + raise ValueError("too many recorded executor capabilities") + if any(not isinstance(identity, RecordedExecutorIdentity) for identity in identities): + raise ValueError("recorded executor capability is invalid") + canonical = tuple(sorted(identities)) + if len(canonical) != len(set(canonical)): + raise ValueError("recorded executor capabilities must be unique") + return canonical def _recorded_job_states() -> frozenset[str]: diff --git a/src/k1link/observatory/worker_agent.py b/src/k1link/observatory/worker_agent.py index 1f84782..0461854 100644 --- a/src/k1link/observatory/worker_agent.py +++ b/src/k1link/observatory/worker_agent.py @@ -29,6 +29,7 @@ from k1link.observatory.recorded_jobs import ( OBSERVATORY_RECORDED_CLAIM_SCHEMA, OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA, OBSERVATORY_RECORDED_JOB_SCHEMA, + RecordedExecutorIdentity, ) WORKER_006_CONTOUR_ID: Final = "worker-006" @@ -53,6 +54,7 @@ type WorkerCycleState = Literal[ "failed", "rejected", "lease-lost", + "publication-pending", ] type RecordedJobWireState = Literal[ "accepted", @@ -97,24 +99,7 @@ class ObservatoryWorkerExecutorUnavailableError(ObservatoryWorkerAgentError): """No local adapter matches the exact sealed executor identity.""" -@dataclass(frozen=True, slots=True) -class ObservatoryWorkerExecutorIdentity: - """The only identity that may select executable Worker code.""" - - release_sha256: str - image_sha256: str - model_manifest_sha256: str - resource_profile_sha256: str - - def __post_init__(self) -> None: - for label, value in ( - ("executor release", self.release_sha256), - ("executor image", self.image_sha256), - ("model manifest", self.model_manifest_sha256), - ("resource profile", self.resource_profile_sha256), - ): - if re.fullmatch(_SHA256_PATTERN, value) is None: - raise ValueError(f"{label} SHA-256 is invalid") +ObservatoryWorkerExecutorIdentity = RecordedExecutorIdentity @dataclass(frozen=True, slots=True) @@ -201,6 +186,12 @@ class ObservatoryWorkerExecutorRegistry: "exact executor identity is not locally allowlisted" ) + @property + def supported_identities(self) -> tuple[ObservatoryWorkerExecutorIdentity, ...]: + """Canonical capability snapshot sent with every claim request.""" + + return tuple(sorted(registration.identity for registration in self.registrations)) + class ObservatoryWorkerTransport(Protocol): """State-transition port implemented by HTTP, IPC, or a test transport.""" @@ -210,6 +201,7 @@ class ObservatoryWorkerTransport(Protocol): *, claimant_id: str, claim_request_id: str, + supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...], ) -> Mapping[str, object] | None: ... def start( @@ -332,6 +324,13 @@ class _ClaimLeasePayload(_StrictPayload): renewal_count: int = Field(ge=0) +class _PublicationPayload(_StrictPayload): + state: Literal["not-required", "pending", "failed", "published"] + attempts: int = Field(ge=0) + error: str | None + published_at_utc: Timestamp | None + + class _RecordedJobPayload(_StrictPayload): schema_version: Literal["missioncore.observatory-recorded-job/v1"] job_id: str = Field(pattern=_JOB_ID_PATTERN) @@ -356,6 +355,7 @@ class _RecordedJobPayload(_StrictPayload): claim_lease: _ClaimLeasePayload | None result: _ResultPayload | None terminal: _TerminalPayload | None + publication: _PublicationPayload created_at_utc: Timestamp updated_at_utc: Timestamp authority: _AuthorityPayload @@ -423,6 +423,7 @@ class ObservatoryWorkerAgent: payload = self._transport.claim_next( claimant_id=WORKER_006_CONTOUR_ID, claim_request_id=claim_request_id, + supported_executor_identities=self._executors.supported_identities, ) if payload is None: return ObservatoryWorkerCycleReport( @@ -430,7 +431,11 @@ class ObservatoryWorkerAgent: claim_request_id=claim_request_id, ) try: - claim = _validate_claim(payload, claim_request_id=claim_request_id) + claim = _validate_claim( + payload, + claim_request_id=claim_request_id, + supported_executor_identities=self._executors.supported_identities, + ) except ObservatoryWorkerClaimRejectedError: return ObservatoryWorkerCycleReport( state="rejected", @@ -554,7 +559,11 @@ class ObservatoryWorkerAgent: "Worker success acknowledgement changed result identity" ) return ObservatoryWorkerCycleReport( - state="succeeded", + state=( + "publication-pending" + if succeeded.publication.state in {"pending", "failed"} + else "succeeded" + ), claim_request_id=claim_request_id, job_id=claim.job.job_id, result_id=result.result_id, @@ -651,6 +660,7 @@ def _validate_claim( payload: Mapping[str, object], *, claim_request_id: str, + supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...], ) -> _ValidatedClaim: try: claim = _RecordedClaimPayload.model_validate(dict(payload)) @@ -663,6 +673,10 @@ def _validate_claim( "schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA, "claim_request_id": claim_request_id, "claimant_id": WORKER_006_CONTOUR_ID, + "supported_executor_identities": [ + identity.as_dict() + for identity in sorted(supported_executor_identities) + ], } ) if claim.request_sha256 != expected_claim_request_sha256: diff --git a/src/k1link/observatory/worker_container_proxy.py b/src/k1link/observatory/worker_container_proxy.py new file mode 100644 index 0000000..2e137e7 --- /dev/null +++ b/src/k1link/observatory/worker_container_proxy.py @@ -0,0 +1,157 @@ +"""Fixed loopback bridge for Observatory Worker containers on Docker Desktop. + +The authenticated Worker HTTP gateway accepts plaintext only on a loopback +URL. Docker Desktop exposes the Windows host as ``host.docker.internal``, so a +container-owned bridge binds one fixed loopback socket and forwards it to the +Mac-owned reverse SSH tunnel on the Worker host. No address, port, credential, +or destination is supplied by a queued job. +""" + +from __future__ import annotations + +import shutil +import socket +import socketserver +import threading +from contextlib import suppress +from typing import Final + +CONTAINER_PROXY_LISTEN_HOST: Final = "127.0.0.1" +CONTAINER_PROXY_LISTEN_PORT: Final = 18080 +CONTAINER_PROXY_UPSTREAM_HOST: Final = "host.docker.internal" +CONTAINER_PROXY_UPSTREAM_PORT: Final = 18080 +CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS: Final = 10.0 +CONTAINER_PROXY_COPY_BYTES: Final = 1024 * 1024 + + +class ObservatoryWorkerContainerProxyError(RuntimeError): + """The fixed container loopback bridge could not be started safely.""" + + +class _ThreadedTcpServer(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + +class _FixedProxyHandler(socketserver.BaseRequestHandler): + server: _ThreadedTcpServer + + def handle(self) -> None: + upstream_address = getattr(self.server, "upstream_address", None) + connect_timeout = getattr(self.server, "connect_timeout", None) + if ( + not isinstance(upstream_address, tuple) + or len(upstream_address) != 2 + or not isinstance(upstream_address[0], str) + or not isinstance(upstream_address[1], int) + or not isinstance(connect_timeout, float) + ): + return + try: + upstream = socket.create_connection( + upstream_address, + timeout=connect_timeout, + ) + except OSError: + return + with upstream: + upstream.settimeout(None) + client = self.request + if not isinstance(client, socket.socket): + return + client.settimeout(None) + client_to_upstream = threading.Thread( + target=_copy_socket, + args=(client, upstream), + daemon=True, + name="observatory-proxy-client-to-host", + ) + upstream_to_client = threading.Thread( + target=_copy_socket, + args=(upstream, client), + daemon=True, + name="observatory-proxy-host-to-client", + ) + client_to_upstream.start() + upstream_to_client.start() + client_to_upstream.join() + upstream_to_client.join() + + +class FixedObservatoryContainerLoopbackProxy: + """Own one bounded TCP bridge for the lifetime of a Worker process.""" + + def __init__( + self, + *, + listen_host: str = CONTAINER_PROXY_LISTEN_HOST, + listen_port: int = CONTAINER_PROXY_LISTEN_PORT, + upstream_host: str = CONTAINER_PROXY_UPSTREAM_HOST, + upstream_port: int = CONTAINER_PROXY_UPSTREAM_PORT, + connect_timeout: float = CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS, + ) -> None: + if listen_host != CONTAINER_PROXY_LISTEN_HOST: + raise ValueError("Observatory container proxy must bind IPv4 loopback") + if not 0 <= listen_port <= 65_535: + raise ValueError("Observatory container proxy listen port is invalid") + if upstream_host != CONTAINER_PROXY_UPSTREAM_HOST and upstream_host != "127.0.0.1": + raise ValueError("Observatory container proxy upstream host is invalid") + if not 1 <= upstream_port <= 65_535: + raise ValueError("Observatory container proxy upstream port is invalid") + if not 0.05 <= connect_timeout <= 60.0: + raise ValueError("Observatory container proxy timeout is invalid") + try: + server = _ThreadedTcpServer( + (listen_host, listen_port), + _FixedProxyHandler, + bind_and_activate=True, + ) + except OSError as exc: + raise ObservatoryWorkerContainerProxyError( + "Observatory container loopback proxy could not bind" + ) from exc + server.upstream_address = (upstream_host, upstream_port) # type: ignore[attr-defined] + server.connect_timeout = float(connect_timeout) # type: ignore[attr-defined] + self._server = server + self._thread = threading.Thread( + target=server.serve_forever, + kwargs={"poll_interval": 0.1}, + daemon=True, + name="observatory-container-loopback-proxy", + ) + + @property + def listen_port(self) -> int: + address = self._server.server_address + if not isinstance(address, tuple) or not isinstance(address[1], int): + raise ObservatoryWorkerContainerProxyError( + "Observatory proxy address is invalid" + ) + return address[1] + + def __enter__(self) -> FixedObservatoryContainerLoopbackProxy: + self._thread.start() + return self + + def __exit__(self, *_args: object) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join(timeout=5.0) + if self._thread.is_alive(): + raise ObservatoryWorkerContainerProxyError( + "Observatory container loopback proxy did not stop" + ) + + +def _copy_socket(source: socket.socket, destination: socket.socket) -> None: + try: + shutil.copyfileobj( + source.makefile("rb", buffering=0), + destination.makefile("wb", buffering=0), + length=CONTAINER_PROXY_COPY_BYTES, + ) + except OSError: + pass + finally: + with suppress(OSError): + destination.shutdown(socket.SHUT_WR) diff --git a/src/k1link/observatory/worker_http_transport.py b/src/k1link/observatory/worker_http_transport.py index e2a00bf..7e9d254 100644 --- a/src/k1link/observatory/worker_http_transport.py +++ b/src/k1link/observatory/worker_http_transport.py @@ -54,6 +54,7 @@ from k1link.observatory.source_admission import ( from k1link.observatory.worker_agent import ( WORKER_006_CONTOUR_ID, ObservatoryWorkerExecutionResult, + ObservatoryWorkerExecutorIdentity, ObservatoryWorkerTransport, SealedObservatoryRecordedJob, ) @@ -205,14 +206,19 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport): *, claimant_id: str, claim_request_id: str, + supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...], ) -> Mapping[str, object] | None: self._require_claimant(claimant_id) payload = self._json_request( "POST", "/api/v1/worker/observatory/recorded-jobs/claims", json_body={ - "schema_version": "missioncore.observatory-worker-claim-request/v1", + "schema_version": "missioncore.observatory-worker-claim-request/v2", "claim_request_id": claim_request_id, + "supported_executor_identities": [ + identity.as_dict() + for identity in sorted(supported_executor_identities) + ], }, allow_empty=True, ) diff --git a/src/k1link/observatory/worker_service.py b/src/k1link/observatory/worker_service.py index 2dbae4d..743e4e1 100644 --- a/src/k1link/observatory/worker_service.py +++ b/src/k1link/observatory/worker_service.py @@ -21,6 +21,11 @@ from urllib.parse import urlsplit import httpx +from k1link.observatory.installed_lab_packages import ( + InstalledLabPackage, + InstalledLabPackageError, + InstalledLabPackageRegistry, +) from k1link.observatory.portable_run_definitions import ( PortableRunDefinition, PortableRunDefinitionRegistry, @@ -77,6 +82,15 @@ class ObservatoryWorkerExecutorBuilder(Protocol): ) -> ObservatoryWorkerExecutorRegistration: ... +class ObservatoryWorkerPackageExecutorFactory(Protocol): + """One generic launcher factory shared by every conforming LAB package.""" + + def __call__( + self, + context: ObservatoryWorkerPackageExecutorBuildContext, + ) -> ObservatoryWorkerExecutorRegistration: ... + + @dataclass(frozen=True, slots=True) class ObservatoryWorkerExecutorBuildContext: """Fixed local inputs shared with one install-time executor builder.""" @@ -91,6 +105,17 @@ class ObservatoryWorkerExecutorBuildContext: _absolute_path(self.work_root, "Worker build work root") +@dataclass(frozen=True, slots=True) +class ObservatoryWorkerPackageExecutorBuildContext(ObservatoryWorkerExecutorBuildContext): + """Generic package plus the same fixed Worker-owned transport boundary.""" + + package: InstalledLabPackage + + def __post_init__(self) -> None: + ObservatoryWorkerExecutorBuildContext.__post_init__(self) + self.package.bind(self.definition, self.candidate) + + @dataclass(frozen=True, slots=True) class ObservatoryWorkerExecutorBuilderRegistration: """Local setup-to-builder binding; queued jobs cannot populate this map.""" @@ -299,6 +324,107 @@ def compose_installed_observatory_worker_service_from_builders( del bearer_token +def compose_installed_observatory_worker_service_from_packages( + *, + configuration: ObservatoryWorkerServiceConfiguration, + definitions: PortableRunDefinitionRegistry, + runtime_registry: PortableWorkerRuntimeRegistry, + packages: InstalledLabPackageRegistry, + executor_factory: ObservatoryWorkerPackageExecutorFactory, + http_transport: httpx.BaseTransport | None = None, +) -> InstalledObservatoryWorkerService: + """Compose all ready LABs through one package-aware executor factory.""" + + bearer_token = load_observatory_worker_bearer_token(configuration.bearer_token_file) + gateway: ObservatoryWorkerHttpGateway | None = None + try: + gateway = ObservatoryWorkerHttpGateway( + base_url=configuration.base_url, + bearer_token=bearer_token, + work_root=configuration.work_root, + transport=http_transport, + ) + executors = build_ready_executor_registry_from_packages( + definitions=definitions, + runtime_registry=runtime_registry, + packages=packages, + executor_factory=executor_factory, + source_transport=gateway, + result_transport=gateway, + work_root=configuration.work_root, + ) + return InstalledObservatoryWorkerService( + configuration=configuration, + gateway=gateway, + agent=ObservatoryWorkerAgent(transport=gateway, executors=executors), + ) + except Exception: + if gateway is not None: + gateway.close() + raise + finally: + del bearer_token + + +def build_ready_executor_registry_from_packages( + *, + definitions: PortableRunDefinitionRegistry, + runtime_registry: PortableWorkerRuntimeRegistry, + packages: InstalledLabPackageRegistry, + executor_factory: ObservatoryWorkerPackageExecutorFactory, + source_transport: PortableWorkerSourceMaterializer, + result_transport: PortableWorkerResultPublisher, + work_root: Path, +) -> ObservatoryWorkerExecutorRegistry: + """Build this Worker's installed ready subset with one generic factory.""" + + _absolute_path(work_root, "Worker package build work root") + ready = definitions.ready_recorded_definitions() + ready_keys = {(item.setup_id, item.definition_sha256) for item in ready} + package_keys = { + (package.setup_id, package.definition_sha256) for package in packages.packages + } + if not package_keys.issubset(ready_keys): + raise ObservatoryWorkerServiceError( + "installed LAB packages must bind only ready RunDefinitions" + ) + registrations: list[ObservatoryWorkerExecutorRegistration] = [] + for package in sorted( + packages.packages, + key=lambda item: (item.setup_id, item.definition_sha256), + ): + definition = definitions.resolve( + package.setup_id, + package.definition_sha256, + ) + candidate = runtime_registry.resolve( + definition.setup_id, + definition.definition_sha256, + ) + try: + package.bind(definition, candidate) + except InstalledLabPackageError as exc: + raise ObservatoryWorkerServiceError( + "installed LAB package is not bound to its ready runtime" + ) from exc + built = executor_factory( + ObservatoryWorkerPackageExecutorBuildContext( + definition=definition, + candidate=candidate, + source_transport=source_transport, + result_transport=result_transport, + work_root=work_root, + package=package, + ) + ) + if built.identity != package.executor_identity: + raise ObservatoryWorkerServiceError( + "generic package factory returned another executor identity" + ) + registrations.append(built) + return ObservatoryWorkerExecutorRegistry(tuple(registrations)) + + def build_ready_executor_registry( *, definitions: PortableRunDefinitionRegistry, diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 3da4ec8..373b6dc 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -47,6 +47,9 @@ from k1link.observatory.m49_queue_binding import ( M49QueueBindingError, M49RecordedQueueBindingService, ) +from k1link.observatory.portable_publication_reconciler import ( + PortablePublicationReconciler, +) from k1link.observatory.portable_queue_binding import ( PortableQueueBindingError, PortableRecordedQueueBindingService, @@ -58,6 +61,7 @@ from k1link.observatory.portable_result_contract import ( from k1link.observatory.portable_result_publisher import ( resolve_published_portable_calculation_profile, ) +from k1link.observatory.portable_result_view import PortableResultViewService from k1link.observatory.portable_run_definitions import ( PortableRunDefinitionRegistry, PortableRunDefinitionRegistryError, @@ -478,6 +482,18 @@ except (PortableWorkerIntegrationError, OSError, ValueError) as exc: # Failure remains isolated from K1, Simulation and legacy LAB. OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc) +OBSERVATORY_PUBLICATION_RECONCILER = ( + None + if ( + OBSERVATORY_RECORDED_JOB_QUEUE is None + or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None + ) + else PortablePublicationReconciler( + queue=OBSERVATORY_RECORDED_JOB_QUEUE, + artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport, + result_publisher=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher, + ) +) OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED OBSERVATORY_WORKER_CLAIM_LEASE_READY = ( OBSERVATORY_WORKER_API_GATE_ENABLED @@ -539,6 +555,7 @@ try: registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE, dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY, + equipment_capture_registry=session_store.equipment_capture_registry, ) OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None except ( @@ -825,9 +842,22 @@ async def _recording_preparation_reconciler() -> None: await asyncio.sleep(2.0) +async def _portable_result_publication_reconciler() -> None: + service = OBSERVATORY_PUBLICATION_RECONCILER + if service is None: + return + while True: + # Durable state remains pending/failed and is retried on the next + # bounded pass or through the explicit operator action. + with suppress(OSError, ValueError): + await asyncio.to_thread(service.run_once) + await asyncio.sleep(15.0) + + @asynccontextmanager async def app_lifespan(_: FastAPI) -> AsyncIterator[None]: reconciler: asyncio.Task[None] | None = None + publication_reconciler: asyncio.Task[None] | None = None try: configure_scanner_diagnostics(session_store.data_dir / "logs") session_recording_preparation_manager.start() @@ -841,6 +871,9 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]: # expensive on field captures. Start it immediately in the background # instead of holding the ASGI startup gate. reconciler = asyncio.create_task(_recording_preparation_reconciler()) + publication_reconciler = asyncio.create_task( + _portable_result_publication_reconciler() + ) yield finally: await map_gateway_proxy.close() @@ -848,6 +881,10 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]: reconciler.cancel() with suppress(asyncio.CancelledError): await reconciler + if publication_reconciler is not None: + publication_reconciler.cancel() + with suppress(asyncio.CancelledError): + await publication_reconciler await asyncio.to_thread(session_recording_preparation_manager.close) await asyncio.to_thread(lidar_local_surface_read_service.close) plugin_environment.close() @@ -1036,6 +1073,24 @@ app.include_router( portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR, portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR, portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE, + portable_result_view=( + None + if session_artifact_gateway is None + else PortableResultViewService( + sessions=session_store, + artifacts=session_artifact_gateway.store, + ) + ), + portable_artifact_transport=( + None + if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None + else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport + ), + portable_result_publisher=( + None + if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None + else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher + ), ) ) if OBSERVATORY_WORKER_DISPATCH_READY: diff --git a/src/k1link/web/observatory_api.py b/src/k1link/web/observatory_api.py index f416f39..822d9fe 100644 --- a/src/k1link/web/observatory_api.py +++ b/src/k1link/web/observatory_api.py @@ -1,9 +1,10 @@ from __future__ import annotations -from typing import Any, Literal +from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Query, Response from fastapi import Path as ApiPath +from fastapi.responses import JSONResponse from pydantic import BaseModel, ConfigDict, Field from k1link.observatory import ( @@ -22,12 +23,24 @@ from k1link.observatory.m49_queue_binding import ( M49QueueBindingIntegrityError, M49RecordedQueueBindingService, ) +from k1link.observatory.portable_artifact_transport import ( + PortableArtifactTransportError, + PortableObservatoryArtifactTransport, +) from k1link.observatory.portable_queue_binding import ( PortableQueueBindingError, PortableQueueBindingIntegrityError, PortableQueueBindingStaleCheckError, PortableRecordedQueueBindingService, ) +from k1link.observatory.portable_result_contract import PortableResultPublisherError +from k1link.observatory.portable_result_publisher import ( + PortableObservatoryResultPublisher, +) +from k1link.observatory.portable_result_view import ( + PortableResultViewError, + PortableResultViewService, +) from k1link.observatory.portable_run_definitions import ( PortableRunDefinitionUnavailableError, ) @@ -158,10 +171,32 @@ def build_observatory_router( portable_setup_projector: PortableSetupProjector | PortableLabV1SetupProjector | None = None, portable_setup_projector_error: str | None = None, portable_binding_service: PortableRecordedQueueBindingService | None = None, + portable_result_view: PortableResultViewService | None = None, + portable_artifact_transport: PortableObservatoryArtifactTransport | None = None, + portable_result_publisher: PortableObservatoryResultPublisher | None = None, ) -> APIRouter: """Build bounded catalog-only mutations for typed Observatory projections.""" router = APIRouter(tags=["observatory"]) + if (portable_artifact_transport is None) != (portable_result_publisher is None): + raise ValueError("portable publication dependencies must be configured together") + + if portable_result_view is not None: + + @router.get("/api/v1/observatory/portable-results/{result_id}") + def get_portable_result_view( + result_id: Annotated[ + str, + ApiPath(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"), + ], + ) -> dict[str, object]: + try: + return portable_result_view.read(result_id) + except PortableResultViewError as exc: + raise HTTPException( + status_code=409, + detail="Portable-результат недоступен для универсального просмотра.", + ) from exc def source_summary(session_id: str) -> SessionSummary: try: @@ -1092,6 +1127,80 @@ def build_observatory_router( detail="Durable-очередь расчётов недоступна.", ) from exc + if ( + portable_artifact_transport is not None + and portable_result_publisher is not None + ): + + @router.post( + "/api/v1/observatory/runs/{job_id}/publication/retry", + response_model=None, + ) + def retry_observatory_result_publication( + job_id: str = ApiPath( + min_length=48, + max_length=48, + pattern=r"^observatory-run-[a-f0-9]{32}$", + ), + ) -> dict[str, object] | JSONResponse: + """Retry verification/publication only; never repeat compute.""" + + try: + job = recorded_job_queue.get(job_id) + except ObservatoryRecordedQueueNotFoundError as exc: + raise HTTPException( + status_code=404, + detail="Расчёт Обсерватории не найден.", + ) from exc + except (ObservatoryRecordedQueueError, ValueError) as exc: + raise HTTPException( + status_code=503, + detail="Durable-очередь расчётов недоступна.", + ) from exc + if job.publication_state == "published": + return job.as_dict() + if job.state != "succeeded" or job.publication_state not in { + "pending", + "failed", + }: + raise HTTPException( + status_code=409, + detail="Результат не ожидает повторной публикации.", + ) + try: + package_root = portable_artifact_transport.package_root_for_terminal( + job + ) + portable_result_publisher.publish( + job=job, + package_root=package_root, + ) + except PortableResultPublisherError as exc: + message = (" ".join(str(exc).split()) or "Publication failed.")[:1_000] + try: + failed = recorded_job_queue.mark_publication_failed( + job_id, + message=message, + ) + except (ObservatoryRecordedQueueError, ValueError) as queue_exc: + raise HTTPException( + status_code=503, + detail="Состояние публикации не удалось сохранить.", + ) from queue_exc + return JSONResponse(status_code=202, content=failed.as_dict()) + except PortableArtifactTransportError as exc: + raise HTTPException( + status_code=409, + detail="Пакет результата недоступен для повторной публикации.", + ) from exc + try: + return recorded_job_queue.mark_published(job_id).as_dict() + except (ObservatoryRecordedQueueError, ValueError) as exc: + raise HTTPException( + status_code=503, + detail="Состояние публикации не удалось сохранить.", + ) from exc + elif recorded_job_queue_error is not None: @router.post("/api/v1/observatory/runs") diff --git a/src/k1link/web/observatory_worker_api.py b/src/k1link/web/observatory_worker_api.py index 21913a1..17e8380 100644 --- a/src/k1link/web/observatory_worker_api.py +++ b/src/k1link/web/observatory_worker_api.py @@ -20,9 +20,9 @@ from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response from fastapi import Path as ApiPath -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, JSONResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from k1link.observatory.portable_artifact_transport import ( MAX_RESULT_MANIFEST_BYTES, @@ -38,6 +38,7 @@ from k1link.observatory.portable_result_publisher import ( PortableObservatoryResultPublisher, ) from k1link.observatory.recorded_jobs import ( + MAX_RECORDED_EXECUTOR_CAPABILITIES, ObservatoryRecordedCheckpointError, ObservatoryRecordedJobQueue, ObservatoryRecordedPreemptionError, @@ -48,9 +49,12 @@ from k1link.observatory.recorded_jobs import ( ObservatoryRecordedQueueIntegrityError, ObservatoryRecordedQueueNotFoundError, ObservatoryRecordedQueueStaleClaimError, + RecordedExecutorIdentity, ) -OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1" +OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = ( + "missioncore.observatory-worker-claim-request/v2" +) OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1" OBSERVATORY_WORKER_RENEW_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-renew-request/v1" OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = ( @@ -143,13 +147,36 @@ class _StrictWorkerRequest(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) +class ObservatoryWorkerExecutorCapability(_StrictWorkerRequest): + release_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + image_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + model_manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + resource_profile_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + def recorded_identity(self) -> RecordedExecutorIdentity: + return RecordedExecutorIdentity(**self.model_dump()) + + class ObservatoryWorkerClaimRequest(_StrictWorkerRequest): - schema_version: Literal["missioncore.observatory-worker-claim-request/v1"] + schema_version: Literal["missioncore.observatory-worker-claim-request/v2"] claim_request_id: str = Field( min_length=1, max_length=160, pattern=_CLAIM_REQUEST_ID_PATTERN, ) + supported_executor_identities: tuple[ObservatoryWorkerExecutorCapability, ...] = Field( + max_length=MAX_RECORDED_EXECUTOR_CAPABILITIES, + ) + + @model_validator(mode="after") + def validate_capability_snapshot(self) -> ObservatoryWorkerClaimRequest: + identities = tuple( + capability.recorded_identity() + for capability in self.supported_executor_identities + ) + if len(identities) != len(set(identities)): + raise ValueError("executor capabilities must be unique") + return self class ObservatoryWorkerStartRequest(_StrictWorkerRequest): @@ -257,6 +284,10 @@ def build_observatory_worker_router( lambda: queue.claim_next( claimant_id=authentication.contour_id, claim_request_id=request.claim_request_id, + supported_executor_identities=tuple( + capability.recorded_identity() + for capability in request.supported_executor_identities + ), ) ) if claim is None: @@ -303,11 +334,11 @@ def build_observatory_worker_router( ) ).as_dict() - @router.post("/recorded-jobs/{job_id}/succeed") + @router.post("/recorded-jobs/{job_id}/succeed", response_model=None) def succeed_job( request: ObservatoryWorkerSucceedRequest, job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], - ) -> dict[str, object]: + ) -> dict[str, object] | JSONResponse: if artifact_transport is not None: _artifact_call( lambda: artifact_transport.require_completed_for_success( @@ -318,29 +349,76 @@ def build_observatory_worker_router( claimant_id=authentication.contour_id, ) ) - succeeded = _queue_call( - lambda: queue.succeed( - job_id, - claim_token=request.claim_token, - result_id=request.result_id, - result_sha256=request.result_sha256, - ) - ) if artifact_transport is not None and result_publisher is not None: + succeeded = _queue_call( + lambda: queue.complete_for_publication( + job_id, + claim_token=request.claim_token, + result_id=request.result_id, + result_sha256=request.result_sha256, + ) + ) + if succeeded.publication_state == "published": + return succeeded.as_dict() package_root = _artifact_call( lambda: artifact_transport.package_root_for_terminal(succeeded) ) try: result_publisher.publish(job=succeeded, package_root=package_root) except PortableResultPublisherError as exc: + publication_error = _publication_error(exc) + failed = _queue_call( + lambda: queue.mark_publication_failed( + job_id, + message=publication_error, + ) + ) + return JSONResponse(status_code=202, content=failed.as_dict()) + return _queue_call(lambda: queue.mark_published(job_id)).as_dict() + return _queue_call( + lambda: queue.succeed( + job_id, + claim_token=request.claim_token, + result_id=request.result_id, + result_sha256=request.result_sha256, + ) + ).as_dict() + + if artifact_transport is not None and result_publisher is not None: + + @router.post( + "/recorded-jobs/{job_id}/publication/retry", + response_model=None, + ) + def retry_publication( + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + ) -> dict[str, object] | JSONResponse: + job = _queue_call(lambda: queue.get(job_id)) + if job.publication_state == "published": + return job.as_dict() + if job.state != "succeeded" or job.publication_state not in { + "pending", + "failed", + }: raise HTTPException( - status_code=503, - detail=( - "Recorded result is sealed but its verified publication " - "requires reconciliation." - ), - ) from exc - return succeeded.as_dict() + status_code=409, + detail="Recorded result is not awaiting publication.", + ) + package_root = _artifact_call( + lambda: artifact_transport.package_root_for_terminal(job) + ) + try: + result_publisher.publish(job=job, package_root=package_root) + except PortableResultPublisherError as exc: + publication_error = _publication_error(exc) + failed = _queue_call( + lambda: queue.mark_publication_failed( + job_id, + message=publication_error, + ) + ) + return JSONResponse(status_code=202, content=failed.as_dict()) + return _queue_call(lambda: queue.mark_published(job_id)).as_dict() @router.post("/recorded-jobs/{job_id}/fail") def fail_job( @@ -650,6 +728,11 @@ def _raise(exc: Exception) -> None: raise exc +def _publication_error(exc: PortableResultPublisherError) -> str: + message = " ".join(str(exc).split()) + return (message or "Portable result publication failed.")[:1_000] + + async def _read_bounded_body(request: Request, maximum_bytes: int) -> bytes: content_length = request.headers.get("content-length") if content_length is not None: diff --git a/tests/test_m49_portable_executor_release.py b/tests/test_m49_portable_executor_release.py index e42bf8f..65849a5 100644 --- a/tests/test_m49_portable_executor_release.py +++ b/tests/test_m49_portable_executor_release.py @@ -789,12 +789,23 @@ def test_result_v2_assembler_and_exact_validator_round_trip( ) plan = PortableWorkerRuntimePlan( job_id=sealed.job_id, + request_sha256=sealed.request_sha256, + identity_sha256=sealed.identity_sha256, + submission_receipt_sha256=sealed.submission_receipt_sha256, + claim_generation=sealed.claim_generation, adapter_id="m49-tgs-worker006-portable-v2", candidate_sha256="f" * 64, setup_id=sealed.setup_id, + definition_id=sealed.definition_id, + definition_version=sealed.definition_version, definition_sha256=sealed.definition_sha256, + source_session_id=sealed.source_session_id, + source_catalog_sha256=sealed.source_catalog_sha256, source_bundle_sha256=sealed.source_bundle_sha256, source_capability_manifest_sha256=sealed.source_capability_manifest_sha256, + source_adapter_id=sealed.source_adapter_id, + source_adapter_version=sealed.source_adapter_version, + source_adapter_sha256=sealed.source_adapter_sha256, result_contract_sha256=definition.result_contract.contract_sha256, phases=M49_PORTABLE_RUNTIME_PHASES, ) diff --git a/tests/test_observatory_installed_lab_package_runner.py b/tests/test_observatory_installed_lab_package_runner.py new file mode 100644 index 0000000..b03bff6 --- /dev/null +++ b/tests/test_observatory_installed_lab_package_runner.py @@ -0,0 +1,362 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from pathlib import Path +from typing import cast + +import httpx +import pytest + +from k1link.observatory.installed_lab_package_runner import ( + DockerEngineInstalledLabLauncher, + InstalledLabDockerLaunch, + InstalledLabDockerMount, + InstalledLabLocalAssetBinding, + InstalledLabPackageProfileRunner, + InstalledLabPackageRunnerError, +) +from k1link.observatory.installed_lab_packages import ( + INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA, + InstalledLabContainer, + InstalledLabPackageMount, + seal_installed_lab_package, +) +from k1link.observatory.portable_result_contract import ( + PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + PortableResultArtifact, + PortableResultPackageManifest, + canonical_json, + result_identity_document, + run_definition_document, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerRuntimePlan, + PortableWorkerRuntimeRegistry, + PortableWorkerSourceStage, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFINITIONS_FILE = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +RUNTIME_FILE = REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json" + + +def _package_and_runtime(): + definitions = PortableRunDefinitionRegistry.from_file(DEFINITIONS_FILE) + runtime = PortableWorkerRuntimeRegistry.from_file( + RUNTIME_FILE, + definitions=definitions, + ) + definition = definitions.resolve_setup("m49-tgs-portable-v2") + candidate = runtime.resolve(definition.setup_id, definition.definition_sha256) + assert candidate.executor is not None + mounted_asset = next( + asset for asset in candidate.reusable_assets if asset.kind != "container-image" + ) + writer = InstalledLabContainer( + container_id="portable-result-writer", + role="result-writer", + image_sha256=candidate.executor.image_sha256, + argv=("/missioncore/package/run", INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA), + depends_on=(), + mounts=( + InstalledLabPackageMount( + asset_id=mounted_asset.asset_id, + target=f"/missioncore/package/assets/{mounted_asset.asset_id}", + ), + ), + network="none", + gpu_count=0, + memory_bytes=8 * 1024**3, + nano_cpus=2_000_000_000, + pids_limit=512, + shm_bytes=64 * 1024**2, + tmpfs_bytes=512 * 1024**2, + timeout_seconds=3600, + ) + package = seal_installed_lab_package( + package_id="m49-generic-runner-test", + package_version=1, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + runtime_candidate_sha256=candidate.candidate_sha256, + source_adapter_sha256=definition.source_adapter.contract_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + executor_identity=candidate.executor_identity(), + execution_mode="single-container", + asset_ids=tuple(asset.asset_id for asset in candidate.reusable_assets), + containers=(writer,), + ) + return definition, candidate, package, mounted_asset.asset_id + + +def _fixed_stack(package): + writer_template = package.containers[0] + step = replace( + writer_template, + container_id="compute-step", + role="step", + ) + writer = replace(writer_template, depends_on=(step.container_id,)) + return seal_installed_lab_package( + package_id=package.package_id, + package_version=package.package_version, + setup_id=package.setup_id, + definition_id=package.definition_id, + definition_version=package.definition_version, + definition_sha256=package.definition_sha256, + runtime_candidate_sha256=package.runtime_candidate_sha256, + source_adapter_sha256=package.source_adapter_sha256, + result_contract_sha256=package.result_contract_sha256, + executor_identity=package.executor_identity, + execution_mode="fixed-stack", + asset_ids=package.asset_ids, + containers=(step, writer), + ) + + +def _plan(definition, candidate) -> PortableWorkerRuntimePlan: + return PortableWorkerRuntimePlan( + job_id=f"observatory-run-{'1' * 32}", + request_sha256="4" * 64, + identity_sha256="5" * 64, + submission_receipt_sha256="6" * 64, + claim_generation=1, + adapter_id=candidate.adapter_id, + candidate_sha256=candidate.candidate_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + source_session_id="source-a", + source_catalog_sha256="7" * 64, + source_bundle_sha256="2" * 64, + source_capability_manifest_sha256="3" * 64, + source_adapter_id=definition.source_adapter.adapter_id, + source_adapter_version=definition.source_adapter.version, + source_adapter_sha256=definition.source_adapter.contract_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + phases=tuple(phase.phase_id for phase in candidate.phases), + ) + + +def _write_result(root: Path, *, plan: PortableWorkerRuntimePlan, definition) -> None: + result_id = "portable-result-generic-runner" + artifact_payload = canonical_json( + {"schema_version": definition.result_contract.result_schema, "verified": True} + ) + artifact_path = root / "artifacts" / "result.json" + artifact_path.parent.mkdir(parents=True) + artifact_path.write_bytes(artifact_payload) + artifact = PortableResultArtifact( + role="result-document", + relative_path="artifacts/result.json", + media_type="application/json", + byte_length=len(artifact_payload), + sha256=hashlib.sha256(artifact_payload).hexdigest(), + ) + job = { + "job_id": plan.job_id, + "request_sha256": plan.request_sha256, + "identity_sha256": plan.identity_sha256, + "submission_receipt_sha256": plan.submission_receipt_sha256, + "claim_generation": plan.claim_generation, + } + source = { + "session_id": plan.source_session_id, + "catalog_sha256": plan.source_catalog_sha256, + "bundle_sha256": plan.source_bundle_sha256, + "capability_manifest_sha256": plan.source_capability_manifest_sha256, + "adapter": { + "adapter_id": definition.source_adapter.adapter_id, + "version": definition.source_adapter.version, + "adapter_sha256": definition.source_adapter.contract_sha256, + }, + } + run_definition = run_definition_document(definition) + result = result_identity_document(definition, result_id) + authority = definition.authority.as_dict() + created_at_utc = "2026-09-01T12:00:00Z" + identity_document = { + "schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + "created_at_utc": created_at_utc, + "job": job, + "source": source, + "run_definition": run_definition, + "result": result, + "authority": authority, + "artifacts": [artifact.as_dict()], + } + manifest = PortableResultPackageManifest( + identity_sha256=canonical_sha256(identity_document), + created_at_utc=created_at_utc, + job=job, + source=source, + run_definition=run_definition, + result=result, + authority=authority, + artifacts=(artifact,), + ) + (root / "manifest.json").write_bytes(manifest.canonical_bytes) + + +def test_generic_fixed_stack_runs_topologically_and_returns_verified_package( + tmp_path: Path, +) -> None: + definition, candidate, single, mounted_asset_id = _package_and_runtime() + package = _fixed_stack(single) + source_root = tmp_path / "materialized-source" + source_root.mkdir() + asset_path = tmp_path / "installed-asset" + asset_path.write_bytes(b"asset") + plan = _plan(definition, candidate) + launches: list[InstalledLabDockerLaunch] = [] + + def launcher(launch: InstalledLabDockerLaunch) -> None: + launches.append(launch) + if launch.container.role == "result-writer": + output = next( + mount for mount in launch.mounts if mount.container_path == "/missioncore/output" + ) + _write_result(Path(output.engine_path), plan=plan, definition=definition) + + runner = InstalledLabPackageProfileRunner( + package=package, + definition=definition, + controller_work_root=tmp_path, + engine_work_root=str(tmp_path), + local_assets=( + InstalledLabLocalAssetBinding( + asset_id=mounted_asset_id, + controller_path=asset_path, + engine_path=str(asset_path), + ), + ), + launcher=launcher, + token_factory=lambda: "abcdef0123456789", + ) + draft = runner.run( + plan, + PortableWorkerSourceStage( + root=source_root, + source_bundle_sha256=plan.source_bundle_sha256, + source_capability_manifest_sha256=plan.source_capability_manifest_sha256, + source_adapter_sha256=definition.source_adapter.contract_sha256, + ), + ) + + assert [launch.container.container_id for launch in launches] == [ + "compute-step", + "portable-result-writer", + ] + writer_launch = launches[-1] + assert any( + mount.container_path == "/missioncore/input/steps/compute-step" + and mount.read_only + for mount in writer_launch.mounts + ) + assert draft.result_id == "portable-result-generic-runner" + assert draft.result_contract_sha256 == definition.result_contract.contract_sha256 + assert (draft.root / "manifest.json").is_file() + for launch in launches: + writable = [mount for mount in launch.mounts if not mount.read_only] + assert [mount.container_path for mount in writable] == ["/missioncore/output"] + + +def test_generic_docker_launcher_uses_hardened_one_shot_contract() -> None: + _definition, _candidate, package, _mounted_asset_id = _package_and_runtime() + requests: list[httpx.Request] = [] + create_document: dict[str, object] = {} + container_id = "f" * 64 + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.path.startswith("/v1.47/images/"): + return httpx.Response( + 200, + json={"Id": f"sha256:{package.containers[0].image_sha256}"}, + ) + if request.url.path == "/v1.47/containers/create": + create_document.update(json.loads(request.content)) + return httpx.Response(201, json={"Id": container_id, "Warnings": None}) + if request.url.path.endswith("/start"): + return httpx.Response(204) + if request.url.path.endswith("/wait"): + return httpx.Response(200, json={"StatusCode": 0, "Error": None}) + if request.method == "DELETE": + return httpx.Response(204) + raise AssertionError(f"unexpected request: {request.method} {request.url}") + + launch = InstalledLabDockerLaunch( + package_id=package.package_id, + container=package.containers[0], + mounts=( + InstalledLabDockerMount("/engine/plan.json", "/missioncore/input/run-plan.json", True), + InstalledLabDockerMount("/engine/source", "/missioncore/input/source", True), + InstalledLabDockerMount("/engine/output", "/missioncore/output", False), + InstalledLabDockerMount("/engine/asset", "/missioncore/package/assets/runner", True), + ), + labels={ + "com.nodedc.authority": "observation-only", + "com.nodedc.component": package.containers[0].container_id, + "com.nodedc.definition-sha256": package.definition_sha256, + "com.nodedc.job-id": f"observatory-run-{'1' * 32}", + "com.nodedc.managed-by": "mission-core-worker", + "com.nodedc.package-sha256": package.package_sha256, + "com.nodedc.product": "mission-core", + "com.nodedc.stack": "observatory", + }, + name_token="0123456789abcdef", + ) + DockerEngineInstalledLabLauncher(transport_factory=lambda: httpx.MockTransport(handler))(launch) + + assert [request.method for request in requests] == [ + "GET", + "POST", + "POST", + "POST", + "DELETE", + ] + assert create_document["Image"] == f"sha256:{package.containers[0].image_sha256}" + assert create_document["NetworkDisabled"] is True + host = cast(dict[str, object], create_document["HostConfig"]) + assert host["NetworkMode"] == "none" + assert host["ReadonlyRootfs"] is True + assert host["CapDrop"] == ["ALL"] + assert host["SecurityOpt"] == ["no-new-privileges:true"] + assert host["Privileged"] is False + assert host["DeviceRequests"] == [] + mounts = cast(list[dict[str, object]], host["Mounts"]) + assert sum(not cast(bool, mount["ReadOnly"]) for mount in mounts) == 1 + assert next(mount for mount in mounts if not mount["ReadOnly"])["Target"] == ( + "/missioncore/output" + ) + + +def test_generic_docker_launcher_verifies_image_inventory_without_container_calls() -> None: + image_sha256s = ("1" * 64, "2" * 64) + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + digest = request.url.path.split("sha256:", 1)[1].split("/", 1)[0] + return httpx.Response(200, json={"Id": f"sha256:{digest}"}) + + DockerEngineInstalledLabLauncher( + transport_factory=lambda: httpx.MockTransport(handler) + ).verify_images(image_sha256s) + + assert [request.method for request in requests] == ["GET", "GET"] + assert all("/images/sha256:" in request.url.path for request in requests) + + with pytest.raises(InstalledLabPackageRunnerError, match="inventory"): + DockerEngineInstalledLabLauncher( + transport_factory=lambda: httpx.MockTransport(handler) + ).verify_images(tuple(reversed(image_sha256s))) diff --git a/tests/test_observatory_installed_lab_packages.py b/tests/test_observatory_installed_lab_packages.py new file mode 100644 index 0000000..459d793 --- /dev/null +++ b/tests/test_observatory_installed_lab_packages.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path +from typing import cast + +import pytest + +from k1link.observatory.installed_lab_packages import ( + INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA, + InstalledLabContainer, + InstalledLabPackageError, + InstalledLabPackageMount, + InstalledLabPackageRegistry, + seal_installed_lab_package, +) +from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerResultPublisher, + PortableWorkerRuntimeRegistry, + PortableWorkerSourceMaterializer, +) +from k1link.observatory.worker_agent import ( + ObservatoryWorkerExecutionResult, + ObservatoryWorkerExecutorRegistration, + SealedObservatoryRecordedJob, +) +from k1link.observatory.worker_service import ( + ObservatoryWorkerPackageExecutorBuildContext, + build_ready_executor_registry_from_packages, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFINITIONS_FILE = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +RUNTIME_FILE = REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json" + + +def _registries() -> tuple[PortableRunDefinitionRegistry, PortableWorkerRuntimeRegistry]: + definitions = PortableRunDefinitionRegistry.from_file(DEFINITIONS_FILE) + runtime = PortableWorkerRuntimeRegistry.from_file( + RUNTIME_FILE, + definitions=definitions, + ) + return definitions, runtime + + +def _m49_package(): + definitions, runtime = _registries() + definition = definitions.resolve_setup("m49-tgs-portable-v2") + candidate = runtime.resolve(definition.setup_id, definition.definition_sha256) + assert candidate.executor is not None + non_image_asset = next( + asset for asset in candidate.reusable_assets if asset.kind != "container-image" + ) + container = InstalledLabContainer( + container_id="portable-result-writer", + role="result-writer", + image_sha256=candidate.executor.image_sha256, + argv=("/missioncore/package/run", INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA), + depends_on=(), + mounts=( + InstalledLabPackageMount( + asset_id=non_image_asset.asset_id, + target=f"/missioncore/package/assets/{non_image_asset.asset_id}", + ), + ), + network="none", + gpu_count=0, + memory_bytes=8 * 1024**3, + nano_cpus=2_000_000_000, + pids_limit=512, + shm_bytes=64 * 1024**2, + tmpfs_bytes=512 * 1024**2, + timeout_seconds=3600, + ) + return seal_installed_lab_package( + package_id="m49-container-contract-test", + package_version=1, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + runtime_candidate_sha256=candidate.candidate_sha256, + source_adapter_sha256=definition.source_adapter.contract_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + executor_identity=candidate.executor_identity(), + execution_mode="single-container", + asset_ids=tuple(asset.asset_id for asset in candidate.reusable_assets), + containers=(container,), + ) + + +def test_installed_package_binds_exact_ready_definition_and_runtime() -> None: + definitions, runtime = _registries() + package = _m49_package() + definition = definitions.resolve(package.setup_id, package.definition_sha256) + candidate = runtime.resolve(package.setup_id, package.definition_sha256) + + package.bind(definition, candidate) + + document = package.identity_document() + serialized = json.dumps(document, sort_keys=True) + assert document["container_io"] == { + "schema_version": "missioncore.observatory-installed-lab-container-io/v2", + "source_root": "/missioncore/input/source", + "plan_path": "/missioncore/input/run-plan.json", + "step_input_root": "/missioncore/input/steps", + "result_root": "/missioncore/output", + "work_root": "/missioncore/work", + } + assert "host_path" not in serialized + assert "source_session_id" not in serialized + + +def test_installed_package_registry_round_trips_and_rejects_host_inputs( + tmp_path: Path, +) -> None: + definitions, runtime = _registries() + package = _m49_package() + document = { + "schema_version": INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA, + "packages": [{**package.identity_document(), "package_sha256": package.package_sha256}], + } + path = tmp_path / "installed-packages.json" + path.write_text(json.dumps(document), encoding="utf-8") + + loaded = InstalledLabPackageRegistry.from_file( + path, + definitions=definitions, + runtime_registry=runtime, + ) + assert loaded.resolve(package.setup_id, package.definition_sha256) == package + + document["packages"][0]["containers"][0]["host_path"] = "/unsafe" + path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(InstalledLabPackageError, match="forbidden"): + InstalledLabPackageRegistry.from_file( + path, + definitions=definitions, + runtime_registry=runtime, + ) + + +def test_package_topology_and_digest_fail_closed() -> None: + package = _m49_package() + with pytest.raises(InstalledLabPackageError, match="digest"): + replace(package, package_sha256="f" * 64) + + first = replace( + package.containers[0], + container_id="first-service", + role="step", + depends_on=("portable-result-writer",), + ) + writer = replace( + package.containers[0], + depends_on=("first-service",), + ) + with pytest.raises(InstalledLabPackageError, match="cycle"): + seal_installed_lab_package( + package_id=package.package_id, + package_version=package.package_version, + setup_id=package.setup_id, + definition_id=package.definition_id, + definition_version=package.definition_version, + definition_sha256=package.definition_sha256, + runtime_candidate_sha256=package.runtime_candidate_sha256, + source_adapter_sha256=package.source_adapter_sha256, + result_contract_sha256=package.result_contract_sha256, + executor_identity=package.executor_identity, + execution_mode="fixed-stack", + asset_ids=package.asset_ids, + containers=(first, writer), + ) + + with pytest.raises(InstalledLabPackageError, match="unsafe"): + InstalledLabPackageMount( + asset_id=package.asset_ids[0], + target="../../host", + ) + + +def test_one_generic_factory_builds_installed_ready_subset(tmp_path: Path) -> None: + definitions, runtime = _registries() + package = _m49_package() + contexts: list[ObservatoryWorkerPackageExecutorBuildContext] = [] + + class Executor: + def execute( + self, + job: SealedObservatoryRecordedJob, + ) -> ObservatoryWorkerExecutionResult: + raise AssertionError(job) + + def factory( + context: ObservatoryWorkerPackageExecutorBuildContext, + ) -> ObservatoryWorkerExecutorRegistration: + contexts.append(context) + return ObservatoryWorkerExecutorRegistration( + context.package.executor_identity, + Executor(), + ) + + executors = build_ready_executor_registry_from_packages( + definitions=definitions, + runtime_registry=runtime, + packages=InstalledLabPackageRegistry((package,)), + executor_factory=factory, + source_transport=cast(PortableWorkerSourceMaterializer, object()), + result_transport=cast(PortableWorkerResultPublisher, object()), + work_root=tmp_path, + ) + + assert executors.supported_identities == (package.executor_identity,) + assert [context.package.package_id for context in contexts] == [package.package_id] diff --git a/tests/test_observatory_installed_lab_worker_container_main.py b/tests/test_observatory_installed_lab_worker_container_main.py new file mode 100644 index 0000000..2496795 --- /dev/null +++ b/tests/test_observatory_installed_lab_worker_container_main.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import socket +import socketserver +import threading + +import pytest + +import k1link.observatory.installed_lab_worker_container_main as container_main +from k1link.observatory.worker_container_proxy import ( + FixedObservatoryContainerLoopbackProxy, +) + + +class _EchoHandler(socketserver.BaseRequestHandler): + def handle(self) -> None: + payload = self.request.recv(1024) + self.request.sendall(payload) + + +def test_generic_proxy_bridges_loopback_to_fixed_upstream() -> None: + upstream = socketserver.ThreadingTCPServer(("127.0.0.1", 0), _EchoHandler) + upstream_thread = threading.Thread(target=upstream.serve_forever, daemon=True) + upstream_thread.start() + try: + upstream_port = upstream.server_address[1] + assert isinstance(upstream_port, int) + with FixedObservatoryContainerLoopbackProxy( + listen_port=0, + upstream_host="127.0.0.1", + upstream_port=upstream_port, + ) as proxy, socket.create_connection(("127.0.0.1", proxy.listen_port)) as client: + client.sendall(b"generic-observatory-proxy") + client.shutdown(socket.SHUT_WR) + assert client.recv(1024) == b"generic-observatory-proxy" + finally: + upstream.shutdown() + upstream.server_close() + upstream_thread.join(timeout=5.0) + + +def test_generic_entrypoint_owns_proxy_around_package_worker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + lifecycle: list[str] = [] + + class _Proxy: + def __enter__(self) -> _Proxy: + lifecycle.append("proxy-started") + return self + + def __exit__(self, *_args: object) -> None: + lifecycle.append("proxy-stopped") + + def worker(arguments: object) -> int: + lifecycle.append(f"worker:{arguments!r}") + return 23 + + monkeypatch.setattr(container_main, "FixedObservatoryContainerLoopbackProxy", _Proxy) + monkeypatch.setattr(container_main.installed_lab_worker_service, "main", worker) + + assert container_main.main(("--once",)) == 23 + assert lifecycle == ["proxy-started", "worker:('--once',)", "proxy-stopped"] diff --git a/tests/test_observatory_installed_lab_worker_service.py b/tests/test_observatory_installed_lab_worker_service.py new file mode 100644 index 0000000..751ad58 --- /dev/null +++ b/tests/test_observatory_installed_lab_worker_service.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from k1link.observatory.installed_lab_worker_service import ( + INSTALLED_LAB_ASSET_BINDINGS_SCHEMA, + InstalledLabWorkerAssetBindings, + InstalledLabWorkerCompositionError, + InstalledLabWorkerEntrypointConfiguration, + InstalledLabWorkerValidationConfiguration, +) + + +def _binding_document() -> dict[str, object]: + return { + "schema_version": INSTALLED_LAB_ASSET_BINDINGS_SCHEMA, + "engine_work_root": "D:\\NDC_MISSIONCORE\\runtime\\worker\\work", + "assets": [ + { + "asset_id": "eomt-image", + "controller_path": None, + "engine_path": None, + "image_sha256": "1" * 64, + }, + { + "asset_id": "ddrnet-checkpoint", + "controller_path": "/runtime/assets/ddrnet-checkpoint.pth", + "engine_path": ( + "D:\\NDC_MISSIONCORE\\runtime\\assets\\ddrnet-checkpoint.pth" + ), + "image_sha256": None, + }, + ], + } + + +def test_asset_binding_file_loads_only_reviewed_paths_or_image_identities( + tmp_path: Path, +) -> None: + path = tmp_path / "bindings.json" + path.write_text(json.dumps(_binding_document()), encoding="utf-8") + + bindings = InstalledLabWorkerAssetBindings.from_file(path) + + assert bindings.engine_work_root.endswith("\\work") + assert tuple(item.asset_id for item in bindings.assets) == ( + "ddrnet-checkpoint", + "eomt-image", + ) + assert bindings.assets[0].controller_path == Path( + "/runtime/assets/ddrnet-checkpoint.pth" + ) + assert bindings.assets[1].image_sha256 == "1" * 64 + + +def test_asset_binding_file_rejects_ambiguous_locator(tmp_path: Path) -> None: + document = _binding_document() + assets = document["assets"] + assert isinstance(assets, list) + first = assets[0] + assert isinstance(first, dict) + first["controller_path"] = "/runtime/image" + first["engine_path"] = "D:\\runtime\\image" + path = tmp_path / "bindings.json" + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(InstalledLabWorkerCompositionError, match="ambiguous"): + InstalledLabWorkerAssetBindings.from_file(path) + + +def test_entrypoint_environment_contains_paths_but_no_executable_selector( + tmp_path: Path, +) -> None: + environment = { + "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE": str(tmp_path / "worker.token"), + "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"), + "MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE": str(tmp_path / "definitions.json"), + "MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE": str( + tmp_path / "runtime.json" + ), + "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE": str( + tmp_path / "packages.json" + ), + "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE": str( + tmp_path / "bindings.json" + ), + } + + configuration = InstalledLabWorkerEntrypointConfiguration.from_environment(environment) + + assert configuration.package_registry_file == tmp_path / "packages.json" + assert configuration.asset_bindings_file == tmp_path / "bindings.json" + assert not any("COMMAND" in key or "MODULE" in key for key in environment) + + +def test_validation_environment_requires_no_token_or_backend_url(tmp_path: Path) -> None: + environment = { + "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"), + "MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE": str(tmp_path / "definitions.json"), + "MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE": str( + tmp_path / "runtime.json" + ), + "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE": str( + tmp_path / "packages.json" + ), + "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE": str( + tmp_path / "bindings.json" + ), + } + + configuration = InstalledLabWorkerValidationConfiguration.from_environment(environment) + + assert configuration.work_root == tmp_path / "work" + assert not any("TOKEN" in key or "BASE_URL" in key for key in environment) + + +def test_asset_binding_file_rejects_unresolved_engine_path(tmp_path: Path) -> None: + document = _binding_document() + assets = document["assets"] + assert isinstance(assets, list) + file_asset = assets[1] + assert isinstance(file_asset, dict) + file_asset["engine_path"] = "D:\\runtime\\lab$release\\asset" + path = tmp_path / "bindings.json" + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(InstalledLabWorkerCompositionError, match="locator is invalid"): + InstalledLabWorkerAssetBindings.from_file(path) diff --git a/tests/test_observatory_m49_worker_entrypoint.py b/tests/test_observatory_m49_worker_entrypoint.py index ff6e22e..8f7116e 100644 --- a/tests/test_observatory_m49_worker_entrypoint.py +++ b/tests/test_observatory_m49_worker_entrypoint.py @@ -544,6 +544,19 @@ def test_entrypoint_environment_requires_all_absolute_fixed_files(tmp_path: Path assert configuration.lab_v1_release_candidate_file == ( service_module._FIXED_LAB_V1_RELEASE_CANDIDATE_FILE # noqa: SLF001 ) + m49_only = service_module.M49WorkerEntrypointConfiguration.from_environment( + { + key: value + for key, value in environment.items() + if key + not in { + service_module.LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV, + service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV, + } + } + ) + assert m49_only.lab_v1_installation_receipt_file is None + assert m49_only.lab_v1_release_candidate_file is None environment[service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV] = str( tmp_path / "lab-v1-release.json" ) diff --git a/tests/test_observatory_portable_lab_v1_executor.py b/tests/test_observatory_portable_lab_v1_executor.py index c610bb4..90875de 100644 --- a/tests/test_observatory_portable_lab_v1_executor.py +++ b/tests/test_observatory_portable_lab_v1_executor.py @@ -126,7 +126,7 @@ def _source_documents(definition: PortableRunDefinition) -> tuple[bytes, bytes]: "spatial_replay": {}, "camera": { "artifact_id": "camera-recording", - "public_source_id": "sensor.camera.right", + "public_source_id": "recorded.camera.right", "generation_sha256": CAMERA_GENERATION_SHA256, "synchronization": "host-arrival-best-effort", "epoch": { @@ -198,22 +198,22 @@ def _camera_job(tmp_path: Path) -> CameraComputeJob: root.mkdir() files = [ { - "path": "input/camera/sensor.camera.right/epoch-1/summary.json", + "path": "input/camera/recorded.camera.right/epoch-1/summary.json", "byte_length": 2, "sha256": "1" * 64, }, { - "path": "input/camera/sensor.camera.right/epoch-1/index.jsonl", + "path": "input/camera/recorded.camera.right/epoch-1/index.jsonl", "byte_length": 3, "sha256": "2" * 64, }, { - "path": "input/camera/sensor.camera.right/epoch-1/init.mp4", + "path": "input/camera/recorded.camera.right/epoch-1/init.mp4", "byte_length": 4, "sha256": CAMERA_INIT_SHA256, }, { - "path": "input/camera/sensor.camera.right/epoch-1/segments/1.m4s", + "path": "input/camera/recorded.camera.right/epoch-1/segments/1.m4s", "byte_length": 7, "sha256": CAMERA_SEGMENT_SHA256, }, @@ -221,7 +221,7 @@ def _camera_job(tmp_path: Path) -> CameraComputeJob: input_document = { "kind": "canonical-camera-epoch", "session_id": SOURCE_SESSION_ID, - "source_id": "sensor.camera.right", + "source_id": "recorded.camera.right", "codec_epoch": 1, "synchronization": "host-arrival-best-effort", "media_type": "video/mp4; codecs=\"avc1.641028\"", @@ -254,7 +254,7 @@ def _camera_job(tmp_path: Path) -> CameraComputeJob: job_root=root, manifest_path=root / "job.json", session_id=SOURCE_SESSION_ID, - source_id="sensor.camera.right", + source_id="recorded.camera.right", codec_epoch=1, input_sha256=input_sha256, input_byte_length=16, @@ -477,7 +477,7 @@ def _component_outputs( "job_id": plan.source_input.camera_job_id, "input_sha256": plan.source_input.camera_input_sha256, "session_id": SOURCE_SESSION_ID, - "source_id": "sensor.camera.right", + "source_id": "recorded.camera.right", "codec_epoch": 1, "timestamp_basis": "session-time-seconds", "timeline_start_seconds": 1.0, @@ -857,7 +857,7 @@ def _release_for_definition( ) -def test_release_candidate_matches_repository_but_stays_honestly_blocked( +def test_legacy_release_candidate_reports_repository_drift_and_stays_blocked( tmp_path: Path, ) -> None: release = PortableLabV1ReleaseCandidate.from_file( @@ -870,7 +870,16 @@ def test_release_candidate_matches_repository_but_stays_honestly_blocked( repository_assets = { asset.asset_id for asset in release.assets if asset.repository_path is not None } - assert set(inspection.matched_assets) == repository_assets + mismatched_assets = { + blocker.removeprefix("asset-").removesuffix("-mismatched") + for blocker in inspection.blockers + if blocker.startswith("asset-") and blocker.endswith("-mismatched") + } + assert set(inspection.matched_assets) | mismatched_assets == repository_assets + assert { + "lab-v1-portable-contracts", + "portable-worker-runtime", + } <= mismatched_assets assert inspection.ready is False assert "executor-image-unsealed" in inspection.blockers assert "commit-bound-source-unavailable" in inspection.blockers @@ -1132,7 +1141,7 @@ def test_shared_worker_stage_materializes_a_version_bound_camera_job( materialized.camera_job_root / "input" / "camera" - / "sensor.camera.right" + / "recorded.camera.right" / "epoch-1" / "init.mp4" ).read_bytes() == init_payload @@ -1140,7 +1149,7 @@ def test_shared_worker_stage_materializes_a_version_bound_camera_job( materialized.camera_job_root / "input" / "camera" - / "sensor.camera.right" + / "recorded.camera.right" / "epoch-1" / "segments" / "1.m4s" @@ -1273,14 +1282,25 @@ def test_profile_runner_sequences_exact_components_and_packages_sealed_job( ) runtime_plan = PortableWorkerRuntimePlan( job_id=sealed.job_id, + request_sha256=sealed.request_sha256, + identity_sha256=sealed.identity_sha256, + submission_receipt_sha256=sealed.submission_receipt_sha256, + claim_generation=sealed.claim_generation, adapter_id="lab-v1-eomt-ddrnet-worker006-v1", candidate_sha256="f" * 64, setup_id=sealed.setup_id, + definition_id=sealed.definition_id, + definition_version=sealed.definition_version, definition_sha256=sealed.definition_sha256, + source_session_id=sealed.source_session_id, + source_catalog_sha256=sealed.source_catalog_sha256, source_bundle_sha256=sealed.source_bundle_sha256, source_capability_manifest_sha256=( sealed.source_capability_manifest_sha256 ), + source_adapter_id=sealed.source_adapter_id, + source_adapter_version=sealed.source_adapter_version, + source_adapter_sha256=sealed.source_adapter_sha256, result_contract_sha256=definition.result_contract.contract_sha256, phases=PORTABLE_LAB_V1_RUNTIME_PHASES, ) diff --git a/tests/test_observatory_portable_lab_v1_worker_service.py b/tests/test_observatory_portable_lab_v1_worker_service.py index 628b868..040f0ee 100644 --- a/tests/test_observatory_portable_lab_v1_worker_service.py +++ b/tests/test_observatory_portable_lab_v1_worker_service.py @@ -115,9 +115,8 @@ class _Launcher: def _ready_definition() -> PortableRunDefinition: - base = PortableRunDefinitionRegistry.from_file(DEFINITIONS_PATH).resolve( + base = PortableRunDefinitionRegistry.from_file(DEFINITIONS_PATH).resolve_setup( service_module.PORTABLE_LAB_V1_SETUP_ID, - "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", ) executor = PortableExecutorAvailability( contour_id=base.executor.contour_id, diff --git a/tests/test_observatory_portable_queue_binding.py b/tests/test_observatory_portable_queue_binding.py index 989b1c2..1a62bf8 100644 --- a/tests/test_observatory_portable_queue_binding.py +++ b/tests/test_observatory_portable_queue_binding.py @@ -52,7 +52,30 @@ INIT_SHA256 = "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38" def _blocked_registry() -> PortableRunDefinitionRegistry: - return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + production = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + ready = production.resolve_setup("lab-v1-eomt-ddrnet-portable-v1") + executor = PortableExecutorAvailability( + contour_id=ready.executor.contour_id, + state="not-installed", + release_id=None, + release_sha256=None, + image_sha256=None, + reason_code="executor-not-installed", + reason="Executor release is not sealed or installed on Worker 006.", + ) + identity = ready.identity_document() + identity["executor"] = executor.identity_document() + blocked = replace( + ready, + executor=executor, + definition_sha256=canonical_sha256(identity), + ) + return PortableRunDefinitionRegistry( + ( + blocked, + production.resolve_setup("m49-tgs-portable-v2"), + ) + ) def _ready_registry() -> PortableRunDefinitionRegistry: diff --git a/tests/test_observatory_portable_result_publisher.py b/tests/test_observatory_portable_result_publisher.py index 3f7a31b..5dc1ffc 100644 --- a/tests/test_observatory_portable_result_publisher.py +++ b/tests/test_observatory_portable_result_publisher.py @@ -29,6 +29,7 @@ from k1link.observatory.portable_result_publisher import ( PortableObservatoryResultPublisher, resolve_published_portable_calculation_profile, ) +from k1link.observatory.portable_result_view import PortableResultViewService from k1link.observatory.portable_run_definitions import ( PortableRunDefinition, PortableRunDefinitionRegistry, @@ -372,7 +373,18 @@ def test_verified_package_publishes_immutable_binding_and_profile_provenance( assert first.binding.session_id == RESULT_ID assert first.binding.source_session_id == SOURCE_SESSION_ID assert first.binding.config_sha256 == definition.definition_sha256 - assert first.binding.replay_capability is None + assert first.binding.replay_capability is not None + assert first.binding.replay_capability.as_dict() == { + "schema_version": "missioncore.observation-lab-replay-capability/v2", + "kind": "portable-result-review", + "viewer_profile": "portable-result", + "timeline": "result-defined", + "activation": "explicit", + "commands_enabled": False, + } + assert first.binding.provenance["replay_capability"] == ( + first.binding.replay_capability.as_dict() + ) assert first.binding.provenance["calculation_profile"] == { "schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA, "setup_id": definition.setup_id, @@ -388,6 +400,15 @@ def test_verified_package_publishes_immutable_binding_and_profile_provenance( assert store.get_lab_instance(RESULT_ID) == first.binding assert store.get_session(SOURCE_SESSION_ID).summary.lab is None + view = PortableResultViewService( + sessions=store, + artifacts=CentralArtifactStore(tmp_path / "central-artifacts"), + ).read(RESULT_ID) + assert view["result_id"] == RESULT_ID + assert view["source_session_id"] == SOURCE_SESSION_ID + assert view["definition_sha256"] == definition.definition_sha256 + assert view["viewer_capability"] == first.binding.replay_capability.as_dict() + summary = store.get_session(RESULT_ID).summary assert summary.display_name == ( "Portable result source · полный маршрут и воспроизведение" diff --git a/tests/test_observatory_portable_run_definitions.py b/tests/test_observatory_portable_run_definitions.py index 2807f23..2167683 100644 --- a/tests/test_observatory_portable_run_definitions.py +++ b/tests/test_observatory_portable_run_definitions.py @@ -19,7 +19,7 @@ from k1link.observatory.portable_run_definitions import ( REPOSITORY_ROOT = Path(__file__).resolve().parents[1] REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" -DEFINITION_SHA256 = "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2" +DEFINITION_SHA256 = "269d71a24b4e63cff54e01273f9d4b35fc6cdd72bc6fadec206169ae0777e6ac" MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56" M49_DEFINITION_SHA256 = "f56d6321bd794ccdfb7d2e3b05d044b11f616ffb81ee29517386cc253046d4eb" M49_MODEL_MANIFEST_SHA256 = "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1" @@ -47,6 +47,26 @@ def _first(document: dict[str, object]) -> dict[str, object]: return first +def _blocked_lab_definition(): + ready = _registry().resolve_setup("lab-v1-eomt-ddrnet-portable-v1") + executor = PortableExecutorAvailability( + contour_id=ready.executor.contour_id, + state="not-installed", + release_id=None, + release_sha256=None, + image_sha256=None, + reason_code="executor-not-installed", + reason="Executor release is not sealed or installed on Worker 006.", + ) + identity = ready.identity_document() + identity["executor"] = executor.identity_document() + return replace( + ready, + executor=executor, + definition_sha256=canonical_sha256(identity), + ) + + def test_production_definition_is_source_independent_and_requirements_are_immutable() -> None: definition = _registry().definitions[0] requirements = definition.source_requirements @@ -279,32 +299,43 @@ def test_duplicate_definition_and_incomplete_ready_executor_are_rejected( executor = first["executor"] assert isinstance(executor, dict) executor["state"] = "ready" + executor["release_id"] = None + executor["release_sha256"] = None + executor["image_sha256"] = None executor["reason_code"] = None executor["reason"] = None with pytest.raises(PortableRunDefinitionRegistryError, match="release identity"): PortableRunDefinitionRegistry.from_file(_write(tmp_path, incomplete)) -def test_blocked_lab_definition_does_not_hide_ready_m49_definition() -> None: +def test_production_definitions_are_both_ready_and_convertible() -> None: registry = _registry() definition = registry.definitions[0] - assert definition.executor.state == "not-installed" - assert definition.executor.release_id is None - assert definition.executor.release_sha256 is None - assert definition.executor.image_sha256 is None - with pytest.raises( - PortableRunDefinitionUnavailableError, - match="not sealed or installed", - ): - definition.to_recorded_run_definition() + assert definition.executor.state == "ready" + assert definition.executor.release_id == "lab-v1-installed-package-v1" + assert definition.executor.release_sha256 == ( + "667858623962cd6d9849a8985b803f59e429916b5c56f76a6fc6c80c0c54526b" + ) + assert definition.executor.image_sha256 == ( + "5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373" + ) + assert definition.to_recorded_run_definition().setup_id == definition.setup_id ready = registry.ready_recorded_definitions() - assert tuple(row.setup_id for row in ready) == ("m49-tgs-portable-v2",) + assert tuple(row.setup_id for row in ready) == ( + "lab-v1-eomt-ddrnet-portable-v1", + "m49-tgs-portable-v2", + ) assert registry.to_recorded_registry().definitions == ready def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identities() -> None: - blocked = _registry().definitions[0] + blocked = _blocked_lab_definition() + with pytest.raises( + PortableRunDefinitionUnavailableError, + match="not sealed or installed", + ): + blocked.to_recorded_run_definition() ready_executor = PortableExecutorAvailability( contour_id="worker-006", state="ready", @@ -339,30 +370,11 @@ def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identit def test_blocked_definition_does_not_hide_an_unrelated_ready_definition() -> None: registry = _registry() - blocked_lab = registry.resolve_setup("lab-v1-eomt-ddrnet-portable-v1") + blocked_lab = _blocked_lab_definition() ready_m49 = registry.resolve_setup("m49-tgs-portable-v2") - ready_executor = PortableExecutorAvailability( - contour_id="worker-006", - state="ready", - release_id="lab-v1-eomt-ddrnet-executor-v1", - release_sha256="1" * 64, - image_sha256="2" * 64, - reason_code=None, - reason=None, - ) - identity = blocked_lab.identity_document() - identity["executor"] = ready_executor.identity_document() - ready_lab = replace( - blocked_lab, - executor=ready_executor, - definition_sha256=canonical_sha256(identity), - ) - mixed = PortableRunDefinitionRegistry((ready_lab, ready_m49)) + mixed = PortableRunDefinitionRegistry((blocked_lab, ready_m49)) - expected = ( - ready_lab.to_recorded_run_definition(), - ready_m49.to_recorded_run_definition(), - ) + expected = (ready_m49.to_recorded_run_definition(),) assert mixed.ready_recorded_definitions() == expected assert mixed.to_recorded_registry().definitions == expected assert mixed.resolve_setup("m49-tgs-portable-v2") is ready_m49 @@ -394,7 +406,7 @@ def test_production_lab_v1_model_component_and_result_identities_are_exact() -> "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875" ) assert components["eomt-recorded-runner-v1"].sha256 == ( - "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4" + "1e64869de48d10f1531c742e6067c4c3ae2a709c5eb0d770d1fab74b4a2431ff" ) assert components["k1-camera-1-calibration-v1"].sha256 == ( "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" diff --git a/tests/test_observatory_portable_setup_api.py b/tests/test_observatory_portable_setup_api.py index 4938189..549b980 100644 --- a/tests/test_observatory_portable_setup_api.py +++ b/tests/test_observatory_portable_setup_api.py @@ -92,7 +92,7 @@ def test_portable_setup_catalog_exposes_capability_and_executor_separately() -> setup = document["setups"][0] assert setup["origin"] == "portable-definition" assert setup["source_compatibility"]["outcome"] == "pass" - assert setup["executor"]["state"] == "not-installed" + assert setup["executor"]["state"] == "ready" assert setup["existing_results"] == [] assert setup["preflight"]["outcome"] == "blocked" assert setup["preflight"]["submission_allowed"] is False @@ -153,6 +153,32 @@ def _ready_lab_registry() -> PortableRunDefinitionRegistry: ) +def _blocked_lab_registry() -> PortableRunDefinitionRegistry: + ready = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup( + "lab-v1-eomt-ddrnet-portable-v1" + ) + executor = PortableExecutorAvailability( + contour_id="worker-006", + state="not-installed", + release_id=None, + release_sha256=None, + image_sha256=None, + reason_code="executor-not-installed", + reason="Immutable executor release не установлен.", + ) + identity = ready.identity_document() + identity["executor"] = executor.identity_document() + return PortableRunDefinitionRegistry( + ( + replace( + ready, + executor=executor, + definition_sha256=canonical_sha256(identity), + ), + ) + ) + + class _PortableBinding: def __init__( self, @@ -278,7 +304,7 @@ def test_portable_api_check_sha_fences_ready_submission(tmp_path: Path) -> None: def test_portable_api_rejects_blocked_lab_executor_before_binding_submit( tmp_path: Path, ) -> None: - full_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + full_registry = _blocked_lab_registry() ready_registry = _ready_lab_registry() queue = ObservatoryRecordedJobQueue( tmp_path, diff --git a/tests/test_observatory_portable_worker_integration.py b/tests/test_observatory_portable_worker_integration.py index 58eb95b..3eae55e 100644 --- a/tests/test_observatory_portable_worker_integration.py +++ b/tests/test_observatory_portable_worker_integration.py @@ -79,15 +79,18 @@ def test_local_worker_gate_is_fail_closed_and_accepts_only_exact_one() -> None: ) -def test_validator_registry_fails_when_one_required_profile_is_absent() -> None: +def test_validator_registry_selects_only_contracts_present_in_definitions() -> None: definitions = _definitions() only_lab_v1 = PortableRunDefinitionRegistry((definitions.definitions[0],)) - with pytest.raises( - PortableWorkerIntegrationError, - match="required portable setup is unavailable", - ): - portable_result_validator_registry(only_lab_v1) + validators = portable_result_validator_registry(only_lab_v1) + + assert validators.registrations == ( + PortableResultContractValidatorRegistration( + PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256, + validate_lab_v1_result_v2, + ), + ) def test_server_integration_constructs_dormant_transport_and_publisher( diff --git a/tests/test_observatory_portable_worker_runtime.py b/tests/test_observatory_portable_worker_runtime.py index f20b1e8..dc346e3 100644 --- a/tests/test_observatory_portable_worker_runtime.py +++ b/tests/test_observatory_portable_worker_runtime.py @@ -13,6 +13,7 @@ from k1link.observatory.portable_run_definitions import ( canonical_sha256, ) from k1link.observatory.portable_worker_runtime import ( + PortableWorkerAssetRequirement, PortableWorkerAssetVerification, PortableWorkerExecutorAdapter, PortableWorkerExecutorSeal, @@ -25,6 +26,7 @@ from k1link.observatory.portable_worker_runtime import ( PortableWorkerRuntimeRegistryError, PortableWorkerRuntimeUnavailableError, PortableWorkerSourceStage, + _matches_tree, inspect_runtime_candidate, ) from k1link.observatory.worker_agent import ( @@ -49,6 +51,27 @@ def _runtime() -> PortableWorkerRuntimeRegistry: ) +def _blocked_candidate(candidate): + phases = tuple( + PortableWorkerRuntimePhase(phase.phase_id, "missing") + for phase in candidate.phases + ) + blockers = ("executor-release-unsealed",) + identity = candidate.identity_document() + identity["state"] = "blocked" + identity["executor"] = None + identity["phases"] = [phase.as_dict() for phase in phases] + identity["blockers"] = list(blockers) + return replace( + candidate, + state="blocked", + executor=None, + phases=phases, + blockers=blockers, + candidate_sha256=canonical_sha256(identity), + ) + + def _all_keys(value: object) -> set[str]: if isinstance(value, dict): return set(value) | {nested for child in value.values() for nested in _all_keys(child)} @@ -57,7 +80,7 @@ def _all_keys(value: object) -> set[str]: return set() -def test_production_candidates_bind_exact_definitions_and_only_m49_is_ready() -> None: +def test_production_candidates_bind_exact_definitions_and_both_are_ready() -> None: registry = _runtime() assert {candidate.setup_id for candidate in registry.candidates} == { @@ -66,15 +89,16 @@ def test_production_candidates_bind_exact_definitions_and_only_m49_is_ready() -> } by_setup = {candidate.setup_id: candidate for candidate in registry.candidates} lab_v1 = by_setup["lab-v1-eomt-ddrnet-portable-v1"] - assert lab_v1.ready is False - assert lab_v1.executor is None - assert "executor-release-unsealed" in lab_v1.blockers - assert any(phase.state == "missing" for phase in lab_v1.phases) - with pytest.raises( - PortableWorkerRuntimeUnavailableError, - match="no executor identity", - ): - lab_v1.executor_identity() + assert lab_v1.ready is True + assert lab_v1.executor is not None + assert lab_v1.blockers == () + assert all(phase.state == "implemented" for phase in lab_v1.phases) + assert lab_v1.executor_identity().release_sha256 == ( + "4788005a24ae3e1664aa7aaaf728fb816243bfde8555c80a608f40d1061925b3" + ) + assert lab_v1.executor_identity().image_sha256 == ( + "840175ccac731b16b7f2815eaebc6e4ef426c49b57daf1911064ff5b7945a8fd" + ) m49 = by_setup["m49-tgs-portable-v2"] assert m49.ready is True @@ -180,15 +204,15 @@ def test_m49_portable_runner_source_release_is_content_addressed() -> None: assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"] -def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_executor() -> None: +def test_ready_lab_candidate_still_requires_complete_local_asset_admission() -> None: candidate = _runtime().resolve( "lab-v1-eomt-ddrnet-portable-v1", - "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", + "11460f0fa03f713b4f64c8742bc30cc31eac66321931e1edd035553f5c72974a", ) bindings = { - "ddrnet-goose-image": PortableWorkerLocalAssetBinding( - asset_id="ddrnet-goose-image", - image_sha256=("591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"), + "agent-image": PortableWorkerLocalAssetBinding( + asset_id="agent-image", + image_sha256=("840175ccac731b16b7f2815eaebc6e4ef426c49b57daf1911064ff5b7945a8fd"), ), "ddrnet-goose-runner": PortableWorkerLocalAssetBinding( asset_id="ddrnet-goose-runner", @@ -201,39 +225,22 @@ def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_exec / "run_goose_vegetation_benchmark.py" ), ), - "eomt-image": PortableWorkerLocalAssetBinding( - asset_id="eomt-image", - image_sha256=("58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"), - ), - "eomt-orchestrator": PortableWorkerLocalAssetBinding( - asset_id="eomt-orchestrator", + "ddrnet-portable-config": PortableWorkerLocalAssetBinding( + asset_id="ddrnet-portable-config", file_path=( REPOSITORY_ROOT - / "experiments" + / "config" / "perception" - / "worker" - / "Invoke-E4FullSessionSegmentation.ps1" + / "lab-v1-eomt-ddrnet-portable-v2.json" ), ), - "eomt-profile": PortableWorkerLocalAssetBinding( - asset_id="eomt-profile", - file_path=( - REPOSITORY_ROOT - / "experiments" - / "perception" - / "worker" - / "e3_k1_camera1_profile.json" - ), + "ddrnet-step-image": PortableWorkerLocalAssetBinding( + asset_id="ddrnet-step-image", + image_sha256=("759eeac66145221b3c64f226b9ba9220363af36e4fda9353ef8169551587901c"), ), - "eomt-runner": PortableWorkerLocalAssetBinding( - asset_id="eomt-runner", - file_path=( - REPOSITORY_ROOT - / "experiments" - / "perception" - / "worker" - / "run_e4_full_session_segmentation.py" - ), + "eomt-step-image": PortableWorkerLocalAssetBinding( + asset_id="eomt-step-image", + image_sha256=("29a7e8caef51e1809f11b66498ef8bf3f9b2c5d3de40acdab2968cdd34b9623b"), ), "vegetation-policy": PortableWorkerLocalAssetBinding( asset_id="vegetation-policy", @@ -258,18 +265,16 @@ def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_exec admission = inspect_runtime_candidate(candidate, bindings) states = {item.asset_id: item.state for item in admission.assets} - assert states["ddrnet-goose-image"] == "matched" + assert states["agent-image"] == "matched" assert states["ddrnet-goose-runner"] == "matched" - assert states["eomt-image"] == "matched" - assert states["eomt-orchestrator"] == "matched" - assert states["eomt-profile"] == "matched" - assert states["eomt-runner"] == "matched" - assert states["ddrnet-portable-config"] == "missing" + assert states["ddrnet-portable-config"] == "matched" + assert states["ddrnet-step-image"] == "matched" + assert states["eomt-step-image"] == "matched" assert states["ddrnet-checkpoint"] == "missing" - assert states["eomt-model-weights"] == "missing" + assert states["eomt-model-cache"] == "missing" assert admission.ready is False - assert "ddrnet-component-port-uninstalled" in admission.blockers - assert "worker-installation-receipt-unavailable" in admission.blockers + assert "asset-ddrnet-checkpoint-missing" in admission.blockers + assert "asset-eomt-model-cache-missing" in admission.blockers def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) -> None: @@ -305,6 +310,78 @@ def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) -> assert "asset-travel-tgs-image-mismatched" in admission.blockers +def test_sealed_local_tree_uses_manifest_receipt_and_member_metadata( + tmp_path: Path, +) -> None: + tree = tmp_path / "model-tree" + tree.mkdir() + (tree / "model.bin").write_bytes(b"model") + (tree / "profile.json").write_bytes(b"{}\n") + manifest = ( + b"model.bin\t5\t9372c470eeadd5ec5f36cb0b9adf10545c93c5132503830bf1465fe7654b117b\n" + b"profile.json\t3\tca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356\n" + ) + identity_sha256 = hashlib.sha256(manifest).hexdigest() + (tree / "tree-manifest.tsv").write_bytes(manifest) + (tree / "tree-receipt.json").write_text( + json.dumps( + { + "schema_version": "missioncore.sealed-tree-runtime/v1", + "asset_id": "eomt-model-tree", + "identity_algorithm": "relative-path-tab-size-tab-file-sha256-lf/v1", + "identity_sha256": identity_sha256, + "file_count": 2, + "byte_length": 8, + "manifest_relative_path": "tree-manifest.tsv", + } + ), + encoding="utf-8", + ) + requirement = PortableWorkerAssetRequirement( + asset_id="eomt-model-tree", + kind="local-tree", + sha256=identity_sha256, + byte_length=8, + component_id=None, + model_release_id=None, + model_artifact_role=None, + ) + + assert _matches_tree(requirement, tree) is True + + receipt = json.loads((tree / "tree-receipt.json").read_text(encoding="utf-8")) + receipt.update( + { + "source_image_sha256": "1" * 64, + "source_path": "/usr/lib/ffmpeg/7.0", + "binaries": { + "model-bin": { + "relative_path": "model.bin", + "byte_length": 5, + "sha256": ( + "9372c470eeadd5ec5f36cb0b9adf10545c93c5132503830bf1465fe7654b117b" + ), + } + }, + } + ) + (tree / "tree-receipt.json").write_text(json.dumps(receipt), encoding="utf-8") + + assert _matches_tree(requirement, tree) is True + + receipt["unexpected"] = "not-admitted" + (tree / "tree-receipt.json").write_text(json.dumps(receipt), encoding="utf-8") + + assert _matches_tree(requirement, tree) is False + + receipt.pop("unexpected") + (tree / "tree-receipt.json").write_text(json.dumps(receipt), encoding="utf-8") + + (tree / "profile.json").write_bytes(b"drift") + + assert _matches_tree(requirement, tree) is False + + def test_runtime_registry_rejects_digest_drift_and_executable_instructions( tmp_path: Path, ) -> None: @@ -328,12 +405,23 @@ def test_runtime_registry_rejects_digest_drift_and_executable_instructions( def test_runtime_plan_is_identity_only_and_observation_only() -> None: plan = PortableWorkerRuntimePlan( job_id="observatory-run-" + ("a" * 32), + request_sha256="6" * 64, + identity_sha256="7" * 64, + submission_receipt_sha256="8" * 64, + claim_generation=1, adapter_id="m49-tgs-worker006-portable-v2", candidate_sha256="1" * 64, setup_id="m49-tgs-portable-v2", + definition_id="m49-tgs-portable", + definition_version=3, definition_sha256="2" * 64, + source_session_id="source-a", + source_catalog_sha256="9" * 64, source_bundle_sha256="3" * 64, source_capability_manifest_sha256="4" * 64, + source_adapter_id="xgrids-k1-recorded-observatory-v2", + source_adapter_version=2, + source_adapter_sha256="a" * 64, result_contract_sha256="5" * 64, phases=("source-delivery", "portable-tgs-runner"), ).as_dict() @@ -373,10 +461,10 @@ def test_ready_local_adapter_composes_only_local_ports_and_exact_job( definition_sha256=canonical_sha256(definition_identity), ) - blocked = _runtime().resolve( + blocked = _blocked_candidate(_runtime().resolve( "lab-v1-eomt-ddrnet-portable-v1", base_definition.definition_sha256, - ) + )) executor_seal = PortableWorkerExecutorSeal( release_id="lab-v1-portable-executor-v1", release_sha256="1" * 64, @@ -512,7 +600,10 @@ def test_ready_local_adapter_composes_only_local_ports_and_exact_job( result_sha256=result_sha256, ) - with pytest.raises(PortableWorkerRuntimeUnavailableError): + with pytest.raises( + PortableWorkerRuntimeRegistryError, + match="blocked local runtime cannot bind a ready RunDefinition", + ): PortableWorkerExecutorAdapter( candidate=blocked, definition=base_definition, diff --git a/tests/test_observatory_publication_reconciler.py b/tests/test_observatory_publication_reconciler.py new file mode 100644 index 0000000..7dfd27e --- /dev/null +++ b/tests/test_observatory_publication_reconciler.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast + +from k1link.observatory.portable_publication_reconciler import ( + PortablePublicationReconciler, +) + +NOW = datetime(2026, 9, 1, 12, 0, tzinfo=UTC) + + +class _Queue: + def __init__(self, jobs: tuple[SimpleNamespace, ...]) -> None: + self.jobs = jobs + self.published: list[str] = [] + self.failed: list[str] = [] + + def pending_publications(self) -> tuple[SimpleNamespace, ...]: + return self.jobs + + def mark_published(self, job_id: str) -> None: + self.published.append(job_id) + + def mark_publication_failed(self, job_id: str, *, message: str) -> None: + assert message + self.failed.append(job_id) + + +class _Transport: + def __init__(self, root: Path) -> None: + self.root = root + self.calls: list[str] = [] + + def package_root_for_terminal(self, job: SimpleNamespace) -> Path: + self.calls.append(job.job_id) + return self.root + + +class _Publisher: + def __init__(self) -> None: + self.calls: list[str] = [] + + def publish(self, *, job: SimpleNamespace, package_root: Path) -> None: + assert package_root.is_dir() + self.calls.append(job.job_id) + + +def test_reconciler_publishes_pending_result_without_compute(tmp_path: Path) -> None: + package = tmp_path / "package" + package.mkdir() + job = _job("pending", attempts=0, updated_at="2026-09-01T11:59:00Z") + queue = _Queue((job,)) + transport = _Transport(package) + publisher = _Publisher() + reconciler = PortablePublicationReconciler( + queue=cast(Any, queue), + artifact_transport=cast(Any, transport), + result_publisher=cast(Any, publisher), + clock=lambda: NOW, + ) + + result = reconciler.run_once() + + assert result.published == 1 + assert result.failed == 0 + assert queue.published == [job.job_id] + assert transport.calls == [job.job_id] + assert publisher.calls == [job.job_id] + + +def test_reconciler_defers_backoff_and_exhausts_bounded_attempts(tmp_path: Path) -> None: + package = tmp_path / "package" + package.mkdir() + deferred = _job("failed", attempts=1, updated_at="2026-09-01T11:59:50Z") + exhausted = _job("failed", attempts=5, updated_at="2026-09-01T10:00:00Z") + queue = _Queue((deferred, exhausted)) + transport = _Transport(package) + publisher = _Publisher() + reconciler = PortablePublicationReconciler( + queue=cast(Any, queue), + artifact_transport=cast(Any, transport), + result_publisher=cast(Any, publisher), + clock=lambda: NOW, + ) + + result = reconciler.run_once() + + assert result.deferred == 1 + assert result.exhausted == 1 + assert result.published == 0 + assert transport.calls == [] + assert publisher.calls == [] + + +def _job( + publication_state: str, + *, + attempts: int, + updated_at: str, +) -> SimpleNamespace: + return SimpleNamespace( + job_id=f"observatory-run-{attempts:032x}", + publication_state=publication_state, + publication_attempts=attempts, + updated_at_utc=updated_at, + ) diff --git a/tests/test_observatory_publication_retry_api.py b/tests/test_observatory_publication_retry_api.py new file mode 100644 index 0000000..86c10eb --- /dev/null +++ b/tests/test_observatory_publication_retry_api.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from k1link.sessions.store import SessionStore +from k1link.web.observatory_api import build_observatory_router + +JOB_ID = "observatory-run-0123456789abcdef0123456789abcdef" + + +class _Job: + state = "succeeded" + publication_state = "failed" + + def as_dict(self) -> dict[str, object]: + return { + "job_id": JOB_ID, + "state": self.state, + "publication": {"state": self.publication_state}, + } + + +class _Queue: + def __init__(self) -> None: + self.job = _Job() + self.published: list[str] = [] + + def get(self, job_id: str) -> _Job: + assert job_id == JOB_ID + return self.job + + def mark_published(self, job_id: str) -> _Job: + assert job_id == JOB_ID + self.published.append(job_id) + self.job.publication_state = "published" + return self.job + + +class _Transport: + def __init__(self, package_root: Path) -> None: + self.package_root = package_root + self.calls: list[str] = [] + + def package_root_for_terminal(self, job: _Job) -> Path: + assert job.state == "succeeded" + self.calls.append(JOB_ID) + return self.package_root + + +class _Publisher: + def __init__(self) -> None: + self.calls: list[tuple[_Job, Path]] = [] + + def publish(self, *, job: _Job, package_root: Path) -> None: + self.calls.append((job, package_root)) + + +def test_operator_retry_republishes_without_creating_compute_work(tmp_path: Path) -> None: + queue = _Queue() + package_root = tmp_path / "sealed-result" + package_root.mkdir() + transport = _Transport(package_root) + publisher = _Publisher() + app = FastAPI() + app.include_router( + build_observatory_router( + SessionStore(tmp_path / "repository", data_dir=tmp_path / "data"), + recorded_job_queue=cast(Any, queue), + recorded_binding_service=cast(Any, object()), + portable_artifact_transport=cast(Any, transport), + portable_result_publisher=cast(Any, publisher), + ) + ) + + response = TestClient(app).post( + f"/api/v1/observatory/runs/{JOB_ID}/publication/retry" + ) + + assert response.status_code == 200 + assert response.json()["publication"]["state"] == "published" + assert transport.calls == [JOB_ID] + assert publisher.calls == [(queue.job, package_root)] + assert queue.published == [JOB_ID] diff --git a/tests/test_observatory_recorded_jobs.py b/tests/test_observatory_recorded_jobs.py index 46090ab..a608e0c 100644 --- a/tests/test_observatory_recorded_jobs.py +++ b/tests/test_observatory_recorded_jobs.py @@ -21,6 +21,7 @@ from k1link.observatory.recorded_jobs import ( ObservatoryRecordedQueueStaleClaimError, ObservatoryRecordedReconciliationRequest, ObservatoryRecordedResourceReleaseAttestation, + RecordedExecutorIdentity, RecordedRunDefinition, RecordedRunDefinitionRegistry, ) @@ -392,6 +393,129 @@ def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> N ) +def test_capability_aware_claim_skips_incompatible_queued_job(tmp_path: Path) -> None: + first, second = _definitions().definitions + compatible = replace( + second, + setup_id="compatible-portable-v1", + definition_id="compatible-portable", + definition_sha256="8" * 64, + executor_release_sha256="9" * 64, + executor_image_sha256="a" * 64, + model_manifest_sha256="b" * 64, + resource_profile_sha256="c" * 64, + ) + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=RecordedRunDefinitionRegistry((first, compatible)), + clock=lambda: NOW, + ) + incompatible_job, _ = queue.submit(_intent(idempotency_key="capability:first"), enqueue=True) + compatible_job, _ = queue.submit( + _intent( + idempotency_key="capability:second", + setup_id=compatible.setup_id, + definition_sha256=compatible.definition_sha256, + ), + enqueue=True, + ) + capability = RecordedExecutorIdentity( + release_sha256=compatible.executor_release_sha256, + image_sha256=compatible.executor_image_sha256, + model_manifest_sha256=compatible.model_manifest_sha256, + resource_profile_sha256=compatible.resource_profile_sha256, + ) + + claim = queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="capability-aware-poll", + supported_executor_identities=(capability,), + ) + + assert claim is not None + assert claim.job.job_id == compatible_job.job_id + assert queue.get(incompatible_job.job_id).state == "queued" + + +def test_capability_claim_identity_binds_snapshot_and_empty_snapshot_claims_nothing( + tmp_path: Path, +) -> None: + queue = _queue(tmp_path) + job, _ = queue.submit(_intent(), enqueue=True) + + assert queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="capability-empty-poll", + supported_executor_identities=(), + ) is None + capability = RecordedExecutorIdentity( + release_sha256=EXECUTOR_RELEASE_SHA, + image_sha256=EXECUTOR_IMAGE_SHA, + model_manifest_sha256=MODEL_MANIFEST_SHA, + resource_profile_sha256=RESOURCE_PROFILE_SHA, + ) + with pytest.raises(ObservatoryRecordedQueueConflictError, match="claim"): + queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="capability-empty-poll", + supported_executor_identities=(capability,), + ) + legacy_claim = queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="legacy-poll-after-empty-capability", + ) + assert legacy_claim is not None + assert legacy_claim.job.job_id == job.job_id + + +def test_execution_success_uses_durable_idempotent_publication_outbox( + tmp_path: Path, +) -> None: + queue = _queue(tmp_path) + job, _ = queue.submit(_intent(), enqueue=True) + claim = queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="publication-outbox-claim", + ) + assert claim is not None + queue.start(job.job_id, claim_token=claim.claim_token) + + completed = queue.complete_for_publication( + job.job_id, + claim_token=claim.claim_token, + result_id="portable-result-001", + result_sha256=RESULT_SHA, + ) + assert completed.state == "succeeded" + assert completed.publication_state == "pending" + assert completed.publication_attempts == 0 + assert queue.pending_publications() == (completed,) + + failed = queue.mark_publication_failed( + job.job_id, + message="catalog temporarily unavailable", + ) + assert failed.publication_state == "failed" + assert failed.publication_attempts == 1 + assert failed.publication_error == "catalog temporarily unavailable" + + replay = queue.complete_for_publication( + job.job_id, + claim_token=claim.claim_token, + result_id="portable-result-001", + result_sha256=RESULT_SHA, + ) + assert replay == failed + + published = queue.mark_published(job.job_id) + assert published.publication_state == "published" + assert published.publication_attempts == 2 + assert published.publication_error is None + assert published.published_at_utc == NOW + assert queue.mark_published(job.job_id) == published + assert queue.pending_publications() == () + + def test_claim_lease_renews_idempotently_and_requeues_expired_unstarted_job( tmp_path: Path, ) -> None: diff --git a/tests/test_observatory_worker_agent.py b/tests/test_observatory_worker_agent.py index 301aa88..23a826e 100644 --- a/tests/test_observatory_worker_agent.py +++ b/tests/test_observatory_worker_agent.py @@ -114,10 +114,12 @@ class FakeTransport: *, claimant_id: str, claim_request_id: str, + supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...], ) -> Mapping[str, object] | None: claim = self.queue.claim_next( claimant_id=claimant_id, claim_request_id=claim_request_id, + supported_executor_identities=supported_executor_identities, ) if claim is None: return None @@ -397,7 +399,7 @@ def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None: assert executor.jobs == [] -def test_exact_release_mismatch_fails_closed_without_start(tmp_path: Path) -> None: +def test_exact_release_mismatch_is_not_claimed_or_failed(tmp_path: Path) -> None: queue = _queue(tmp_path) job_id = _enqueue(queue) transport = FakeTransport(queue) @@ -409,19 +411,13 @@ def test_exact_release_mismatch_fails_closed_without_start(tmp_path: Path) -> No identity=_identity(release_sha256="b" * 64), ).run_once() - assert report.state == "failed" - assert report.failure_code == "executor-not-allowlisted" + assert report.state == "empty" + assert report.failure_code is None assert transport.starts == [] assert transport.successes == [] - assert transport.failures == [ - ( - job_id, - "executor-not-allowlisted", - "Exact executor identity is not installed on Worker 006.", - ) - ] + assert transport.failures == [] assert executor.jobs == [] - assert queue.get(job_id).state == "failed" + assert queue.get(job_id).state == "queued" def _spoof_claimant(payload: dict[str, object]) -> dict[str, object]: @@ -525,9 +521,11 @@ class BlockingEmptyTransport: *, claimant_id: str, claim_request_id: str, + supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...], ) -> Mapping[str, object] | None: assert claimant_id == WORKER_006_CONTOUR_ID assert claim_request_id == "worker-006:overlap-test" + assert supported_executor_identities == () self.entered.set() assert self.release.wait(timeout=2) return None diff --git a/tests/test_observatory_worker_api.py b/tests/test_observatory_worker_api.py index 7c34aa1..4ee1c1b 100644 --- a/tests/test_observatory_worker_api.py +++ b/tests/test_observatory_worker_api.py @@ -11,6 +11,7 @@ from fastapi.testclient import TestClient from k1link.observatory.portable_artifact_transport import ( PortableArtifactTransportUnavailableError, ) +from k1link.observatory.portable_result_contract import PortableResultPublisherError from k1link.observatory.recorded_jobs import ( ObservatoryRecordedJobIntent, ObservatoryRecordedJobQueue, @@ -32,7 +33,7 @@ WORKER_HEADERS = { "Authorization": f"Bearer {WORKER_TOKEN}", OBSERVATORY_WORKER_CONTOUR_HEADER: "worker-006", } -CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1" +CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v2" START_SCHEMA = "missioncore.observatory-worker-start-request/v1" RENEW_SCHEMA = "missioncore.observatory-worker-renew-request/v1" CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1" @@ -61,6 +62,22 @@ def _definition() -> RecordedRunDefinition: ) +def _claim_request(claim_request_id: str) -> dict[str, object]: + definition = _definition() + return { + "schema_version": CLAIM_SCHEMA, + "claim_request_id": claim_request_id, + "supported_executor_identities": [ + { + "release_sha256": definition.executor_release_sha256, + "image_sha256": definition.executor_image_sha256, + "model_manifest_sha256": definition.model_manifest_sha256, + "resource_profile_sha256": definition.resource_profile_sha256, + } + ], + } + + def _services(tmp_path: Path) -> tuple[TestClient, ObservatoryRecordedJobQueue]: definition = _definition() queue = ObservatoryRecordedJobQueue( @@ -85,6 +102,28 @@ class _ArtifactTransportWithoutCompletedPackage: raise PortableArtifactTransportUnavailableError("package is incomplete") +class _CompletedArtifactTransport: + def __init__(self, package_root: Path) -> None: + self.package_root = package_root + + def require_completed_for_success(self, **_values: object) -> Path: + return self.package_root + + def package_root_for_terminal(self, _job: object) -> Path: + return self.package_root + + +class _TransientResultPublisher: + def __init__(self) -> None: + self.attempts = 0 + + def publish(self, **_values: object) -> object: + self.attempts += 1 + if self.attempts == 1: + raise PortableResultPublisherError("catalog temporarily unavailable") + return object() + + def _services_with_artifact_transport( tmp_path: Path, ) -> tuple[TestClient, ObservatoryRecordedJobQueue]: @@ -107,6 +146,30 @@ def _services_with_artifact_transport( return TestClient(app), queue +def _services_with_transient_publisher( + tmp_path: Path, +) -> tuple[TestClient, ObservatoryRecordedJobQueue, _TransientResultPublisher]: + definition = _definition() + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=RecordedRunDefinitionRegistry((definition,)), + ) + publisher = _TransientResultPublisher() + app = FastAPI() + app.include_router( + build_observatory_worker_router( + queue, + authentication=ObservatoryWorkerAuthentication( + bearer_token_sha256=WORKER_TOKEN_SHA256, + contour_id="worker-006", + ), + artifact_transport=_CompletedArtifactTransport(tmp_path / "package"), # type: ignore[arg-type] + result_publisher=publisher, # type: ignore[arg-type] + ) + ) + return TestClient(app), queue, publisher + + def _enqueue( queue: ObservatoryRecordedJobQueue, *, @@ -137,10 +200,7 @@ def _claim( response = client.post( "/api/v1/worker/observatory/recorded-jobs/claims", headers=WORKER_HEADERS, - json={ - "schema_version": CLAIM_SCHEMA, - "claim_request_id": claim_request_id, - }, + json=_claim_request(claim_request_id), ) assert response.status_code == 200 return cast(dict[str, Any], response.json()) @@ -151,10 +211,7 @@ def test_worker_authentication_requires_digest_and_configured_contour( ) -> None: client, queue = _services(tmp_path) _enqueue(queue) - request = { - "schema_version": CLAIM_SCHEMA, - "claim_request_id": "worker-006:auth-claim", - } + request = _claim_request("worker-006:auth-claim") missing = client.post( "/api/v1/worker/observatory/recorded-jobs/claims", @@ -198,10 +255,7 @@ def test_claim_is_idempotent_and_request_schema_rejects_execution_inputs( ) -> None: client, queue = _services(tmp_path) job_id = _enqueue(queue) - request = { - "schema_version": CLAIM_SCHEMA, - "claim_request_id": "worker-006:stable-claim", - } + request = _claim_request("worker-006:stable-claim") first = client.post( "/api/v1/worker/observatory/recorded-jobs/claims", @@ -235,14 +289,30 @@ def test_claim_is_idempotent_and_request_schema_rejects_execution_inputs( assert queue.get(job_id).state == "claimed" +def test_legacy_claim_without_capability_snapshot_cannot_take_another_profile( + tmp_path: Path, +) -> None: + client, queue = _services(tmp_path) + job_id = _enqueue(queue) + + response = client.post( + "/api/v1/worker/observatory/recorded-jobs/claims", + headers=WORKER_HEADERS, + json={ + "schema_version": "missioncore.observatory-worker-claim-request/v1", + "claim_request_id": "worker-006:legacy-unbounded-claim", + }, + ) + + assert response.status_code == 422 + assert queue.get(job_id).state == "queued" + + def test_empty_claim_is_204_and_empty_receipt_remains_idempotent( tmp_path: Path, ) -> None: client, queue = _services(tmp_path) - request = { - "schema_version": CLAIM_SCHEMA, - "claim_request_id": "worker-006:empty-poll", - } + request = _claim_request("worker-006:empty-poll") first = client.post( "/api/v1/worker/observatory/recorded-jobs/claims", @@ -258,10 +328,7 @@ def test_empty_claim_is_204_and_empty_receipt_remains_idempotent( fresh = client.post( "/api/v1/worker/observatory/recorded-jobs/claims", headers=WORKER_HEADERS, - json={ - "schema_version": CLAIM_SCHEMA, - "claim_request_id": "worker-006:fresh-poll", - }, + json=_claim_request("worker-006:fresh-poll"), ) assert first.status_code == 204 @@ -456,6 +523,57 @@ def test_artifact_transport_blocks_success_without_completed_package( assert queue.get(job_id).state == "running" +def test_publication_failure_is_durable_and_retry_does_not_rerun_execution( + tmp_path: Path, +) -> None: + client, queue, publisher = _services_with_transient_publisher(tmp_path) + job_id = _enqueue(queue) + claim_token = _claim(client)["claim_token"] + client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start", + headers=WORKER_HEADERS, + json={"schema_version": START_SCHEMA, "claim_token": claim_token}, + ) + + sealed = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed", + headers=WORKER_HEADERS, + json={ + "schema_version": SUCCEED_SCHEMA, + "claim_token": claim_token, + "result_id": "portable-result-publication-retry", + "result_sha256": "b" * 64, + }, + ) + + assert sealed.status_code == 202 + assert sealed.json()["state"] == "succeeded" + assert sealed.json()["publication"] == { + "state": "failed", + "attempts": 1, + "error": "catalog temporarily unavailable", + "published_at_utc": None, + } + assert queue.get(job_id).claim_generation == 1 + + retried = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/publication/retry", + headers=WORKER_HEADERS, + ) + repeated = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/publication/retry", + headers=WORKER_HEADERS, + ) + + assert retried.status_code == 200 + assert retried.json()["publication"]["state"] == "published" + assert retried.json()["publication"]["attempts"] == 2 + assert repeated.status_code == 200 + assert repeated.json()["publication"]["attempts"] == 2 + assert queue.get(job_id).claim_generation == 1 + assert publisher.attempts == 2 + + def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None: client, queue = _services(tmp_path) job_id = _enqueue(queue) diff --git a/tests/test_observatory_worker_http_transport.py b/tests/test_observatory_worker_http_transport.py index 6445c60..793d89e 100644 --- a/tests/test_observatory_worker_http_transport.py +++ b/tests/test_observatory_worker_http_transport.py @@ -99,6 +99,7 @@ def _cache_claim(gateway: ObservatoryWorkerHttpGateway) -> None: payload = gateway.claim_next( claimant_id=WORKER_006_CONTOUR_ID, claim_request_id="worker-006:http-transport-test", + supported_executor_identities=(), ) assert payload is not None