feat(observatory): install local M49 worker path
This commit is contained in:
DCCONSTRUCTIONS
@@ -12,6 +12,16 @@ from typing import Final, cast
|
||||
|
||||
MISSION_CORE_LAUNCH_AGENT_LABEL: Final = "com.nodedc.mission-core.local"
|
||||
MISSION_CORE_LAUNCH_AGENT_SCHEMA: Final = "missioncore.local-launch-agent-plan/v1"
|
||||
OBSERVATORY_LOCAL_WORKER_ENABLED_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_LOCAL_ENABLED"
|
||||
)
|
||||
OBSERVATORY_SOURCE_CAS_ROOT_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT"
|
||||
)
|
||||
OBSERVATORY_RESULT_STAGING_ROOT_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
|
||||
)
|
||||
ARTIFACT_STORE_ROOT_ENV: Final = "MISSIONCORE_ARTIFACT_STORE_ROOT"
|
||||
|
||||
|
||||
class MissionCoreLaunchAgentError(RuntimeError):
|
||||
@@ -28,6 +38,7 @@ class MissionCoreLaunchAgentPlan:
|
||||
preserved_data_directory: Path | None
|
||||
current_program_arguments: tuple[str, ...]
|
||||
desired_program_arguments: tuple[str, ...]
|
||||
local_observatory_worker_enabled: bool
|
||||
desired_payload: bytes
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
@@ -61,6 +72,9 @@ class MissionCoreLaunchAgentPlan:
|
||||
"bounded_launchd_exit_timeout_seconds": 20,
|
||||
"keep_alive": True,
|
||||
"process_group_owned": True,
|
||||
"local_observatory_worker_enabled": (
|
||||
self.local_observatory_worker_enabled
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -70,6 +84,7 @@ def plan_mission_core_launch_agent(
|
||||
repository_root: Path,
|
||||
agent_path: Path,
|
||||
expected_current_repository_root: Path | None = None,
|
||||
enable_local_observatory_worker: bool = False,
|
||||
) -> MissionCoreLaunchAgentPlan:
|
||||
repository = repository_root.expanduser().resolve(strict=True)
|
||||
expected_current_repository = (
|
||||
@@ -131,6 +146,28 @@ def plan_mission_core_launch_agent(
|
||||
desired_environment["MISSIONCORE_SERVICE_WATCHDOG"] = "1"
|
||||
if preserved_data_directory is not None:
|
||||
desired_environment["MISSIONCORE_DATA_DIR"] = str(preserved_data_directory)
|
||||
if enable_local_observatory_worker:
|
||||
data_directory = _local_observatory_data_directory(
|
||||
repository=repository,
|
||||
environment=desired_environment,
|
||||
)
|
||||
artifact_store = _private_local_worker_directory(
|
||||
data_directory / "observatory-artifact-store",
|
||||
"local Observatory artifact store",
|
||||
)
|
||||
source_cas = _private_local_worker_directory(
|
||||
data_directory / "observatory-worker-source-cas",
|
||||
"local Observatory source CAS",
|
||||
)
|
||||
result_staging = _private_local_worker_directory(
|
||||
data_directory / "observatory-worker-result-staging",
|
||||
"local Observatory result staging",
|
||||
)
|
||||
desired_environment["MISSIONCORE_DATA_DIR"] = str(data_directory)
|
||||
desired_environment[ARTIFACT_STORE_ROOT_ENV] = str(artifact_store)
|
||||
desired_environment[OBSERVATORY_SOURCE_CAS_ROOT_ENV] = str(source_cas)
|
||||
desired_environment[OBSERVATORY_RESULT_STAGING_ROOT_ENV] = str(result_staging)
|
||||
desired_environment[OBSERVATORY_LOCAL_WORKER_ENABLED_ENV] = "1"
|
||||
log_path = repository / ".runtime/mission-core/k1link-serve-launchd.log"
|
||||
desired_program_arguments = (
|
||||
str(uv_entrypoint),
|
||||
@@ -163,10 +200,44 @@ def plan_mission_core_launch_agent(
|
||||
preserved_data_directory=preserved_data_directory,
|
||||
current_program_arguments=current_arguments,
|
||||
desired_program_arguments=desired_program_arguments,
|
||||
local_observatory_worker_enabled=enable_local_observatory_worker,
|
||||
desired_payload=desired_payload,
|
||||
)
|
||||
|
||||
|
||||
def _local_observatory_data_directory(
|
||||
*,
|
||||
repository: Path,
|
||||
environment: dict[str, str],
|
||||
) -> Path:
|
||||
configured = environment.get("MISSIONCORE_DATA_DIR", "").strip()
|
||||
candidate = Path(configured) if configured else repository / ".runtime" / "mission-core"
|
||||
return _private_local_worker_directory(
|
||||
candidate,
|
||||
"Mission Core data directory",
|
||||
)
|
||||
|
||||
|
||||
def _private_local_worker_directory(path: Path, label: str) -> Path:
|
||||
if not path.is_absolute():
|
||||
raise MissionCoreLaunchAgentError(f"{label} is not an absolute directory")
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise MissionCoreLaunchAgentError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
resolved != path
|
||||
or not stat.S_ISDIR(metadata.st_mode)
|
||||
or stat.S_IMODE(metadata.st_mode) != 0o700
|
||||
or metadata.st_uid != os.getuid()
|
||||
):
|
||||
raise MissionCoreLaunchAgentError(
|
||||
f"{label} is not a private canonical directory"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _program_arguments(document: dict[str, object]) -> tuple[str, ...]:
|
||||
value = document.get("ProgramArguments")
|
||||
if not isinstance(value, list) or not value or any(not isinstance(item, str) for item in value):
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Runnable fixed entrypoint for the portable M4.9 Observatory Worker."""
|
||||
|
||||
from k1link.observatory.m49_worker_service import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,716 @@
|
||||
"""Fixed POSIX service composition for the sealed portable M4.9 executor.
|
||||
|
||||
The entrypoint reads only immutable registries, one installation receipt and
|
||||
path-only Worker service settings. Jobs cannot select commands, providers,
|
||||
modules, images or filesystem locations. Every runtime asset is resolved from
|
||||
the fixed release layout and re-verified against the ready runtime candidate
|
||||
before the first queue claim.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import stat
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from threading import Event
|
||||
from types import FrameType
|
||||
from typing import Final, cast
|
||||
|
||||
import httpx
|
||||
|
||||
from k1link.observatory.m49_portable_executor import (
|
||||
M49_PORTABLE_COMPILED_RUNNER_ASSET_ID,
|
||||
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID,
|
||||
M49_PORTABLE_PROFILE_ASSET_ID,
|
||||
M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID,
|
||||
M49PortableRunnerInstallation,
|
||||
compose_m49_portable_executor_adapter,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import OBSERVATION_ONLY_AUTHORITY
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerLocalAssetBinding,
|
||||
PortableWorkerRuntimeCandidate,
|
||||
PortableWorkerRuntimeRegistry,
|
||||
inspect_runtime_candidate,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
WORKER_006_CONTOUR_ID,
|
||||
ObservatoryWorkerAgent,
|
||||
ObservatoryWorkerExecutorRegistration,
|
||||
ObservatoryWorkerExecutorRegistry,
|
||||
)
|
||||
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
|
||||
from k1link.observatory.worker_service import (
|
||||
InstalledObservatoryWorkerService,
|
||||
ObservatoryWorkerServiceConfiguration,
|
||||
load_observatory_worker_bearer_token,
|
||||
require_ready_executor_coverage,
|
||||
)
|
||||
|
||||
M49_WORKER_DEFINITIONS_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE"
|
||||
M49_WORKER_RUNTIME_REGISTRY_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE"
|
||||
M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_M49_INSTALLATION_RECEIPT_FILE"
|
||||
)
|
||||
|
||||
M49_WORKER_INSTALLATION_RECEIPT_SCHEMA: Final = (
|
||||
"missioncore.m49-tgs-portable-worker-installation-ready-receipt/v1"
|
||||
)
|
||||
M49_WORKER_SETUP_ID: Final = "m49-tgs-portable-v2"
|
||||
M49_WORKER_ADAPTER_ID: Final = "m49-tgs-worker006-portable-v2"
|
||||
M49_WORKER_RELEASE_ID: Final = "m49-tgs-portable-executor-v1"
|
||||
|
||||
M49_EXECUTOR_RELEASE_ASSET_ID: Final = "m49-portable-executor-release"
|
||||
M49_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = "m49-portable-worker-installation-receipt"
|
||||
|
||||
_MAX_RECEIPT_BYTES: Final = 256 * 1024
|
||||
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
_SOURCE_REVISION: Final = re.compile(r"^[a-f0-9]{40}$")
|
||||
_WORKER_COMPUTER_NAME: Final = "DESKTOP-OPJ8J04"
|
||||
_FIXED_RUNTIME_RELEASE_ROOT: Final = Path("/release")
|
||||
_PROTECTED_RUNTIME_NAMES: Final = (
|
||||
"ndc-mission-core-triton",
|
||||
"ndc-mission-core-perception-worker",
|
||||
"ndc-gaussian-pipeline-gaussian-gateway-1",
|
||||
"ndc-gaussian-pipeline-gaussian-pipeline-1",
|
||||
"ndc-gaussian-pipeline-gaussian-terrain-executor-1",
|
||||
)
|
||||
_AUDIT_RELEASE_ROOT: Final = re.compile(
|
||||
r"^D:\\NDC_MISSIONCORE\\runtime\\releases\\observatory-portable\\"
|
||||
r"m49-tgs-portable-candidate-([a-f0-9]{64})\\ready$"
|
||||
)
|
||||
|
||||
_FIXED_FILE_ASSET_PATHS: Final = {
|
||||
M49_PORTABLE_COMPILED_RUNNER_ASSET_ID: PurePosixPath("run_m49_tgs_portable"),
|
||||
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID: PurePosixPath("compiled-runner-build.json"),
|
||||
M49_PORTABLE_PROFILE_ASSET_ID: PurePosixPath("m49-tgs-portable-v2.json"),
|
||||
M49_EXECUTOR_RELEASE_ASSET_ID: PurePosixPath("executor-release.json"),
|
||||
}
|
||||
_REQUIRED_RUNTIME_ASSET_IDS: Final = frozenset(
|
||||
{
|
||||
*_FIXED_FILE_ASSET_PATHS,
|
||||
M49_WORKER_INSTALLATION_RECEIPT_ASSET_ID,
|
||||
M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class M49WorkerCompositionError(RuntimeError):
|
||||
"""The fixed M4.9 Worker installation cannot be composed safely."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49WorkerEntrypointConfiguration:
|
||||
"""Path-only service inputs selected before the Worker process starts."""
|
||||
|
||||
worker: ObservatoryWorkerServiceConfiguration
|
||||
definitions_file: Path
|
||||
runtime_registry_file: Path
|
||||
installation_receipt_file: Path
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for path, label in (
|
||||
(self.definitions_file, "portable RunDefinition registry"),
|
||||
(self.runtime_registry_file, "portable runtime registry"),
|
||||
(self.installation_receipt_file, "M4.9 installation receipt"),
|
||||
):
|
||||
_absolute_path(path, label)
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> M49WorkerEntrypointConfiguration:
|
||||
values = os.environ if environment is None else environment
|
||||
return cls(
|
||||
worker=ObservatoryWorkerServiceConfiguration.from_environment(values),
|
||||
definitions_file=_required_environment_path(
|
||||
values,
|
||||
M49_WORKER_DEFINITIONS_FILE_ENV,
|
||||
),
|
||||
runtime_registry_file=_required_environment_path(
|
||||
values,
|
||||
M49_WORKER_RUNTIME_REGISTRY_FILE_ENV,
|
||||
),
|
||||
installation_receipt_file=_required_environment_path(
|
||||
values,
|
||||
M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49WorkerReceiptFile:
|
||||
asset_id: str
|
||||
relative_path: PurePosixPath
|
||||
byte_length: int
|
||||
sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
expected = _FIXED_FILE_ASSET_PATHS.get(self.asset_id)
|
||||
if expected is None or self.relative_path != expected:
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 installation receipt contains an unknown release file"
|
||||
)
|
||||
if isinstance(self.byte_length, bool) or not 1 <= self.byte_length <= 2**40:
|
||||
raise M49WorkerCompositionError("M4.9 installation receipt file length is invalid")
|
||||
_digest(self.sha256, "M4.9 installation receipt file SHA-256")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49WorkerInstallationReceipt:
|
||||
"""Strict installed-ready receipt; paths remain confined to one release root."""
|
||||
|
||||
path: Path
|
||||
release_root: Path
|
||||
source_revision: str
|
||||
candidate_sha256: str
|
||||
candidate_release_sha256: str
|
||||
candidate_worker_installation_receipt_sha256: str
|
||||
release_id: str
|
||||
release_sha256: str
|
||||
base_image_sha256: str
|
||||
executor_image_sha256: str
|
||||
files: tuple[M49WorkerReceiptFile, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.release_id != M49_WORKER_RELEASE_ID:
|
||||
raise M49WorkerCompositionError("M4.9 installation receipt release changed")
|
||||
if _SOURCE_REVISION.fullmatch(self.source_revision) is None:
|
||||
raise M49WorkerCompositionError("M4.9 installation receipt source revision is invalid")
|
||||
for value, label in (
|
||||
(self.candidate_sha256, "M4.9 source candidate SHA-256"),
|
||||
(self.candidate_release_sha256, "M4.9 candidate release SHA-256"),
|
||||
(
|
||||
self.candidate_worker_installation_receipt_sha256,
|
||||
"M4.9 candidate installation receipt SHA-256",
|
||||
),
|
||||
(self.release_sha256, "M4.9 executor release SHA-256"),
|
||||
(self.base_image_sha256, "M4.9 base image SHA-256"),
|
||||
(self.executor_image_sha256, "M4.9 executor image SHA-256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
asset_ids = tuple(row.asset_id for row in self.files)
|
||||
if asset_ids != tuple(sorted(_FIXED_FILE_ASSET_PATHS)):
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 installation receipt file inventory is incomplete"
|
||||
)
|
||||
release_file = next(
|
||||
row for row in self.files if row.asset_id == M49_EXECUTOR_RELEASE_ASSET_ID
|
||||
)
|
||||
if release_file.sha256 != self.release_sha256:
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 executor release digest differs from its installation receipt"
|
||||
)
|
||||
if self.path.parent != self.release_root:
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 installation receipt is outside its fixed release root"
|
||||
)
|
||||
|
||||
def asset_bindings(
|
||||
self,
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
) -> dict[str, PortableWorkerLocalAssetBinding]:
|
||||
"""Resolve every candidate asset or reject the unknown inventory."""
|
||||
|
||||
candidate_asset_ids = {asset.asset_id for asset in candidate.reusable_assets}
|
||||
if candidate_asset_ids != _REQUIRED_RUNTIME_ASSET_IDS:
|
||||
raise M49WorkerCompositionError(
|
||||
"ready M4.9 runtime asset inventory differs from the fixed release"
|
||||
)
|
||||
rows = {row.asset_id: row for row in self.files}
|
||||
bindings: dict[str, PortableWorkerLocalAssetBinding] = {}
|
||||
for requirement in candidate.reusable_assets:
|
||||
if requirement.asset_id == M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID:
|
||||
bindings[requirement.asset_id] = PortableWorkerLocalAssetBinding(
|
||||
asset_id=requirement.asset_id,
|
||||
image_sha256=self.base_image_sha256,
|
||||
)
|
||||
continue
|
||||
if requirement.asset_id == M49_WORKER_INSTALLATION_RECEIPT_ASSET_ID:
|
||||
bindings[requirement.asset_id] = PortableWorkerLocalAssetBinding(
|
||||
asset_id=requirement.asset_id,
|
||||
file_path=self.path,
|
||||
)
|
||||
continue
|
||||
row = rows.get(requirement.asset_id)
|
||||
if row is None:
|
||||
raise M49WorkerCompositionError(
|
||||
"ready M4.9 runtime requires an unmapped local asset"
|
||||
)
|
||||
path = _confined_release_file(self.release_root, row.relative_path)
|
||||
metadata = path.stat()
|
||||
if metadata.st_size != row.byte_length or _sha256_file(path) != row.sha256:
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 release file differs from its installation receipt"
|
||||
)
|
||||
bindings[requirement.asset_id] = PortableWorkerLocalAssetBinding(
|
||||
asset_id=requirement.asset_id,
|
||||
file_path=path,
|
||||
)
|
||||
if set(bindings) != {asset.asset_id for asset in candidate.reusable_assets}:
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 runtime asset bindings do not cover the exact candidate"
|
||||
)
|
||||
return bindings
|
||||
|
||||
|
||||
def load_m49_worker_installation_receipt(
|
||||
path: Path,
|
||||
) -> M49WorkerInstallationReceipt:
|
||||
"""Load one exact regular installed-ready receipt without following links."""
|
||||
|
||||
receipt_path = _regular_file(path, "M4.9 installation receipt")
|
||||
try:
|
||||
payload = receipt_path.read_bytes()
|
||||
document: object = json.loads(payload.decode("utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise M49WorkerCompositionError("M4.9 installation receipt is unreadable") from exc
|
||||
if not 0 < len(payload) <= _MAX_RECEIPT_BYTES:
|
||||
raise M49WorkerCompositionError("M4.9 installation receipt size is invalid")
|
||||
row = _object(document, "M4.9 installation receipt")
|
||||
_exact_keys(
|
||||
row,
|
||||
{
|
||||
"schema_version",
|
||||
"receipt_state",
|
||||
"worker_id",
|
||||
"computer_name",
|
||||
"source_revision",
|
||||
"candidate_sha256",
|
||||
"candidate_release_sha256",
|
||||
"candidate_worker_installation_receipt_sha256",
|
||||
"release_id",
|
||||
"release_sha256",
|
||||
"release_root",
|
||||
"runtime_release_root",
|
||||
"base_image_sha256",
|
||||
"executor_image_sha256",
|
||||
"files",
|
||||
"fixture_smoke",
|
||||
"protected_runtime",
|
||||
"legacy_m49_task_state",
|
||||
"blockers",
|
||||
"authority",
|
||||
},
|
||||
"M4.9 installation receipt",
|
||||
)
|
||||
blockers = _array(row["blockers"], "M4.9 installation blockers")
|
||||
if (
|
||||
row["schema_version"] != M49_WORKER_INSTALLATION_RECEIPT_SCHEMA
|
||||
or row["receipt_state"] != "installed-ready"
|
||||
or row["worker_id"] != WORKER_006_CONTOUR_ID
|
||||
or row["computer_name"] != _WORKER_COMPUTER_NAME
|
||||
or row["fixture_smoke"] != "passed"
|
||||
or blockers
|
||||
or row["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 installation receipt is not an accepted ready installation"
|
||||
)
|
||||
candidate_sha256 = _string(
|
||||
row["candidate_sha256"],
|
||||
"M4.9 source candidate SHA-256",
|
||||
)
|
||||
audit_release_root = _string(row["release_root"], "M4.9 audit release root")
|
||||
audit_match = _AUDIT_RELEASE_ROOT.fullmatch(audit_release_root)
|
||||
if audit_match is None or audit_match.group(1) != candidate_sha256:
|
||||
raise M49WorkerCompositionError("M4.9 audit release root identity changed")
|
||||
release_root = _real_directory(
|
||||
Path(_string(row["runtime_release_root"], "M4.9 runtime release root")),
|
||||
"M4.9 runtime release root",
|
||||
)
|
||||
if release_root != _FIXED_RUNTIME_RELEASE_ROOT:
|
||||
raise M49WorkerCompositionError("M4.9 runtime release root is not fixed")
|
||||
file_rows = tuple(
|
||||
sorted(
|
||||
(_receipt_file(value) for value in _array(row["files"], "M4.9 release files")),
|
||||
key=lambda value: value.asset_id,
|
||||
)
|
||||
)
|
||||
receipt = M49WorkerInstallationReceipt(
|
||||
path=receipt_path,
|
||||
release_root=release_root,
|
||||
source_revision=_string(row["source_revision"], "M4.9 source revision"),
|
||||
candidate_sha256=candidate_sha256,
|
||||
candidate_release_sha256=_string(
|
||||
row["candidate_release_sha256"],
|
||||
"M4.9 candidate release SHA-256",
|
||||
),
|
||||
candidate_worker_installation_receipt_sha256=_string(
|
||||
row["candidate_worker_installation_receipt_sha256"],
|
||||
"M4.9 candidate installation receipt SHA-256",
|
||||
),
|
||||
release_id=_string(row["release_id"], "M4.9 release id"),
|
||||
release_sha256=_string(row["release_sha256"], "M4.9 release SHA-256"),
|
||||
base_image_sha256=_string(
|
||||
row["base_image_sha256"],
|
||||
"M4.9 base image SHA-256",
|
||||
),
|
||||
executor_image_sha256=_string(
|
||||
row["executor_image_sha256"],
|
||||
"M4.9 executor image SHA-256",
|
||||
),
|
||||
files=file_rows,
|
||||
)
|
||||
protected = _array(row["protected_runtime"], "M4.9 protected runtime")
|
||||
protected_names: list[str] = []
|
||||
for value in protected:
|
||||
protected_row = _object(value, "M4.9 protected runtime row")
|
||||
_exact_keys(
|
||||
protected_row,
|
||||
{"name", "container_id"},
|
||||
"M4.9 protected runtime row",
|
||||
)
|
||||
protected_names.append(
|
||||
_nonempty_string(
|
||||
protected_row["name"],
|
||||
"M4.9 protected runtime name",
|
||||
)
|
||||
)
|
||||
container_id = _nonempty_string(
|
||||
protected_row["container_id"],
|
||||
"M4.9 protected runtime container id",
|
||||
)
|
||||
_digest(container_id, "M4.9 protected runtime container id")
|
||||
if tuple(protected_names) != _PROTECTED_RUNTIME_NAMES:
|
||||
raise M49WorkerCompositionError("M4.9 protected runtime inventory changed")
|
||||
legacy_state = row["legacy_m49_task_state"]
|
||||
if legacy_state is not None:
|
||||
_nonempty_string(legacy_state, "M4.9 legacy task state")
|
||||
return receipt
|
||||
|
||||
|
||||
def compose_installed_m49_worker_service(
|
||||
configuration: M49WorkerEntrypointConfiguration,
|
||||
*,
|
||||
http_transport: httpx.BaseTransport | None = None,
|
||||
) -> InstalledObservatoryWorkerService:
|
||||
"""Compose one fixed M4.9 executor around one shared HTTP gateway."""
|
||||
|
||||
if os.name != "posix":
|
||||
raise M49WorkerCompositionError("portable M4.9 Worker requires a POSIX runtime")
|
||||
definitions = PortableRunDefinitionRegistry.from_file(configuration.definitions_file)
|
||||
definition = definitions.resolve_setup(M49_WORKER_SETUP_ID)
|
||||
runtime_registry = PortableWorkerRuntimeRegistry.from_file(
|
||||
configuration.runtime_registry_file,
|
||||
definitions=definitions,
|
||||
)
|
||||
candidate = runtime_registry.resolve(
|
||||
definition.setup_id,
|
||||
definition.definition_sha256,
|
||||
)
|
||||
_verify_ready_identity(definitions, definition, candidate)
|
||||
receipt = load_m49_worker_installation_receipt(configuration.installation_receipt_file)
|
||||
_verify_receipt_identity(receipt, definition, candidate)
|
||||
bindings = receipt.asset_bindings(candidate)
|
||||
admission = inspect_runtime_candidate(candidate, bindings)
|
||||
if not admission.ready:
|
||||
raise M49WorkerCompositionError("portable M4.9 local asset admission is not ready")
|
||||
profile = _bound_file(bindings, M49_PORTABLE_PROFILE_ASSET_ID)
|
||||
runner = _bound_file(bindings, M49_PORTABLE_COMPILED_RUNNER_ASSET_ID)
|
||||
build_seal = _bound_file(
|
||||
bindings,
|
||||
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID,
|
||||
)
|
||||
installation = M49PortableRunnerInstallation(
|
||||
profile_path=profile,
|
||||
runner_binary_path=runner,
|
||||
runner_build_seal_path=build_seal,
|
||||
runner_build_seal_sha256=_asset_sha256(
|
||||
candidate,
|
||||
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID,
|
||||
),
|
||||
output_parent=configuration.worker.work_root / "m49-portable" / "runner-output",
|
||||
)
|
||||
token = load_observatory_worker_bearer_token(configuration.worker.bearer_token_file)
|
||||
gateway: ObservatoryWorkerHttpGateway | None = None
|
||||
try:
|
||||
gateway = ObservatoryWorkerHttpGateway(
|
||||
base_url=configuration.worker.base_url,
|
||||
bearer_token=token,
|
||||
work_root=configuration.worker.work_root,
|
||||
transport=http_transport,
|
||||
)
|
||||
adapter = compose_m49_portable_executor_adapter(
|
||||
candidate=candidate,
|
||||
definition=definition,
|
||||
admission=admission,
|
||||
source_transport=gateway,
|
||||
result_transport=gateway,
|
||||
installation=installation,
|
||||
source_output_parent=(
|
||||
configuration.worker.work_root / "m49-portable" / "source-output"
|
||||
),
|
||||
created_at_utc=_utc_now,
|
||||
)
|
||||
executors = ObservatoryWorkerExecutorRegistry(
|
||||
(
|
||||
ObservatoryWorkerExecutorRegistration(
|
||||
identity=candidate.executor_identity(),
|
||||
adapter=adapter,
|
||||
),
|
||||
)
|
||||
)
|
||||
require_ready_executor_coverage(
|
||||
definitions=definitions,
|
||||
executors=executors,
|
||||
)
|
||||
return InstalledObservatoryWorkerService(
|
||||
configuration=configuration.worker,
|
||||
gateway=gateway,
|
||||
agent=ObservatoryWorkerAgent(transport=gateway, executors=executors),
|
||||
)
|
||||
except Exception:
|
||||
if gateway is not None:
|
||||
gateway.close()
|
||||
raise
|
||||
finally:
|
||||
del token
|
||||
|
||||
|
||||
def run_installed_m49_worker(
|
||||
service: InstalledObservatoryWorkerService,
|
||||
*,
|
||||
stop: Event,
|
||||
once: bool = False,
|
||||
) -> None:
|
||||
"""Run one claim or the bounded-failure polling loop until POSIX shutdown."""
|
||||
|
||||
if once:
|
||||
try:
|
||||
service.agent.run_once()
|
||||
finally:
|
||||
service.close()
|
||||
return
|
||||
service.run(stop=stop)
|
||||
|
||||
|
||||
def main(arguments: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Run at most one claim cycle and exit.",
|
||||
)
|
||||
options = parser.parse_args(arguments)
|
||||
configuration = M49WorkerEntrypointConfiguration.from_environment()
|
||||
service = compose_installed_m49_worker_service(configuration)
|
||||
stop = Event()
|
||||
with _posix_shutdown_signals(stop):
|
||||
run_installed_m49_worker(service, stop=stop, once=cast(bool, options.once))
|
||||
return 0
|
||||
|
||||
|
||||
def _verify_ready_identity(
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
definition: PortableRunDefinition,
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
) -> None:
|
||||
ready = definitions.ready_recorded_definitions()
|
||||
if (
|
||||
len(ready) != 1
|
||||
or ready[0].setup_id != M49_WORKER_SETUP_ID
|
||||
or definition.setup_id != M49_WORKER_SETUP_ID
|
||||
or definition.executor.contour_id != WORKER_006_CONTOUR_ID
|
||||
or candidate.adapter_id != M49_WORKER_ADAPTER_ID
|
||||
or not definition.executor.ready
|
||||
or not candidate.ready
|
||||
):
|
||||
raise M49WorkerCompositionError(
|
||||
"fixed M4.9 Worker requires exactly one ready M4.9 definition"
|
||||
)
|
||||
|
||||
|
||||
def _verify_receipt_identity(
|
||||
receipt: M49WorkerInstallationReceipt,
|
||||
definition: PortableRunDefinition,
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
) -> None:
|
||||
executor = candidate.executor
|
||||
if (
|
||||
executor is None
|
||||
or definition.executor.release_id != receipt.release_id
|
||||
or definition.executor.release_sha256 != receipt.release_sha256
|
||||
or definition.executor.image_sha256 != receipt.executor_image_sha256
|
||||
or executor.release_id != receipt.release_id
|
||||
or executor.release_sha256 != receipt.release_sha256
|
||||
or executor.image_sha256 != receipt.executor_image_sha256
|
||||
):
|
||||
raise M49WorkerCompositionError(
|
||||
"M4.9 installation receipt differs from the sealed executor"
|
||||
)
|
||||
|
||||
|
||||
def _receipt_file(value: object) -> M49WorkerReceiptFile:
|
||||
row = _object(value, "M4.9 release file")
|
||||
_exact_keys(
|
||||
row,
|
||||
{"asset_id", "relative_path", "byte_length", "sha256"},
|
||||
"M4.9 release file",
|
||||
)
|
||||
relative = _safe_relative_path(_string(row["relative_path"], "M4.9 release relative path"))
|
||||
byte_length = row["byte_length"]
|
||||
if not isinstance(byte_length, int):
|
||||
raise M49WorkerCompositionError("M4.9 release file length is invalid")
|
||||
return M49WorkerReceiptFile(
|
||||
asset_id=_string(row["asset_id"], "M4.9 release asset id"),
|
||||
relative_path=relative,
|
||||
byte_length=byte_length,
|
||||
sha256=_string(row["sha256"], "M4.9 release file SHA-256"),
|
||||
)
|
||||
|
||||
|
||||
def _bound_file(
|
||||
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
|
||||
asset_id: str,
|
||||
) -> Path:
|
||||
binding = bindings.get(asset_id)
|
||||
if binding is None or binding.file_path is None:
|
||||
raise M49WorkerCompositionError("M4.9 fixed file asset is unavailable")
|
||||
return binding.file_path
|
||||
|
||||
|
||||
def _asset_sha256(candidate: PortableWorkerRuntimeCandidate, asset_id: str) -> str:
|
||||
for asset in candidate.reusable_assets:
|
||||
if asset.asset_id == asset_id:
|
||||
return asset.sha256
|
||||
raise M49WorkerCompositionError("M4.9 fixed asset is absent from the runtime")
|
||||
|
||||
|
||||
def _confined_release_file(root: Path, relative: PurePosixPath) -> Path:
|
||||
candidate = _regular_file(root.joinpath(*relative.parts), "M4.9 release file")
|
||||
if not candidate.is_relative_to(root):
|
||||
raise M49WorkerCompositionError("M4.9 release file escapes its release root")
|
||||
return candidate
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _regular_file(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M49WorkerCompositionError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or not os.path.samefile(candidate, resolved)
|
||||
):
|
||||
raise M49WorkerCompositionError(f"{label} is unsafe")
|
||||
return resolved
|
||||
|
||||
|
||||
def _real_directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M49WorkerCompositionError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISDIR(metadata.st_mode)
|
||||
or not os.path.samefile(candidate, resolved)
|
||||
):
|
||||
raise M49WorkerCompositionError(f"{label} is unsafe")
|
||||
return resolved
|
||||
|
||||
|
||||
def _safe_relative_path(value: str) -> PurePosixPath:
|
||||
path = PurePosixPath(value)
|
||||
if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts):
|
||||
raise M49WorkerCompositionError("M4.9 release relative path is unsafe")
|
||||
return path
|
||||
|
||||
|
||||
def _absolute_path(path: Path, label: str) -> None:
|
||||
if not path.is_absolute() or str(path) != str(path).strip():
|
||||
raise M49WorkerCompositionError(f"{label} must be an absolute path")
|
||||
|
||||
|
||||
def _required_environment_path(values: Mapping[str, str], name: str) -> Path:
|
||||
value = values.get(name, "")
|
||||
if not value or value != value.strip():
|
||||
raise M49WorkerCompositionError(f"{name} is required")
|
||||
path = Path(value)
|
||||
_absolute_path(path, name)
|
||||
return path
|
||||
|
||||
|
||||
def _digest(value: str, label: str) -> None:
|
||||
if _SHA256.fullmatch(value) is None:
|
||||
raise M49WorkerCompositionError(f"{label} is invalid")
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise M49WorkerCompositionError(f"{label} must be an object")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _array(value: object, label: str) -> list[object]:
|
||||
if not isinstance(value, list):
|
||||
raise M49WorkerCompositionError(f"{label} must be an array")
|
||||
return value
|
||||
|
||||
|
||||
def _exact_keys(row: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||
if set(row) != expected:
|
||||
raise M49WorkerCompositionError(f"{label} fields are invalid")
|
||||
|
||||
|
||||
def _string(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise M49WorkerCompositionError(f"{label} must be a string")
|
||||
return value
|
||||
|
||||
|
||||
def _nonempty_string(value: object, label: str) -> str:
|
||||
result = _string(value, label)
|
||||
if not result or result != result.strip() or len(result) > 512:
|
||||
raise M49WorkerCompositionError(f"{label} is invalid")
|
||||
return result
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _posix_shutdown_signals(stop: Event) -> Iterator[None]:
|
||||
def request_stop(_signal: int, _frame: FrameType | None) -> None:
|
||||
stop.set()
|
||||
|
||||
previous_int = signal.signal(signal.SIGINT, request_stop)
|
||||
previous_term = signal.signal(signal.SIGTERM, request_stop)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
signal.signal(signal.SIGINT, previous_int)
|
||||
signal.signal(signal.SIGTERM, previous_term)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -63,6 +63,9 @@ OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
|
||||
)
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_LOCAL_ENABLED"
|
||||
)
|
||||
|
||||
|
||||
class PortableWorkerIntegrationError(RuntimeError):
|
||||
@@ -197,7 +200,7 @@ _VALIDATOR_SPECS: Final = (
|
||||
_ValidatorSpec(
|
||||
setup_id=PORTABLE_M49_SETUP_ID,
|
||||
definition_id="m49-tgs-portable",
|
||||
definition_version=2,
|
||||
definition_version=3,
|
||||
contract_id="m49-tgs-portable-review-v2",
|
||||
contract_version=2,
|
||||
result_schema=M49_PORTABLE_RESULT_SCHEMA,
|
||||
@@ -231,6 +234,27 @@ def portable_result_validator_registry(
|
||||
return PortableResultContractValidatorRegistry(tuple(registrations))
|
||||
|
||||
|
||||
def observatory_worker_local_enabled(
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""Return the explicit local-only Worker API gate.
|
||||
|
||||
Absence is the fail-closed default. The single accepted enabled value is
|
||||
deliberately exact so misspelled or whitespace-padded service settings
|
||||
cannot expose the pull API.
|
||||
"""
|
||||
|
||||
values = os.environ if environment is None else environment
|
||||
raw = values.get(OBSERVATORY_WORKER_LOCAL_ENABLED_ENV)
|
||||
if raw is None or raw == "":
|
||||
return False
|
||||
if raw != "1":
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"{OBSERVATORY_WORKER_LOCAL_ENABLED_ENV} must be exactly 1 when enabled"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def build_portable_observatory_worker_integration(
|
||||
*,
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
|
||||
+44
-21
@@ -68,10 +68,12 @@ from k1link.observatory.portable_setup_projection import (
|
||||
portable_calculation_profile_registry,
|
||||
)
|
||||
from k1link.observatory.portable_worker_integration import (
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV,
|
||||
PortableObservatoryWorkerIntegration,
|
||||
PortableWorkerIntegrationError,
|
||||
PortableWorkerStorageRoots,
|
||||
build_portable_observatory_worker_integration,
|
||||
observatory_worker_local_enabled,
|
||||
portable_result_validator_registry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
@@ -385,9 +387,15 @@ except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token"
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED: bool
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR: str | None
|
||||
try:
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED = observatory_worker_local_enabled()
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR = None
|
||||
except PortableWorkerIntegrationError as exc:
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED = False
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
||||
OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None
|
||||
OBSERVATORY_WORKER_API_ERROR: str | None
|
||||
@@ -470,32 +478,47 @@ except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
||||
# Failure remains isolated from K1, Simulation and legacy LAB.
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_API_ERROR = (
|
||||
"Worker pull API is hard-disabled pending sealed installed executors, "
|
||||
"a configured Worker credential, explicit integration acceptance and the "
|
||||
"production gate"
|
||||
+ (
|
||||
""
|
||||
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is None
|
||||
else (
|
||||
"; authentication unavailable: "
|
||||
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
||||
)
|
||||
)
|
||||
+ (
|
||||
""
|
||||
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is None
|
||||
else f"; integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
||||
)
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = (
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
)
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = (
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED
|
||||
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||
)
|
||||
OBSERVATORY_WORKER_DISPATCH_READY = (
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||
)
|
||||
if OBSERVATORY_WORKER_DISPATCH_READY:
|
||||
OBSERVATORY_WORKER_API_ERROR = None
|
||||
else:
|
||||
worker_api_errors: list[str] = []
|
||||
if not OBSERVATORY_WORKER_API_GATE_ENABLED:
|
||||
worker_api_errors.append(
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR
|
||||
or (
|
||||
"local-only Worker API gate is disabled; set "
|
||||
f"{OBSERVATORY_WORKER_LOCAL_ENABLED_ENV}=1 to enable it"
|
||||
)
|
||||
)
|
||||
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None:
|
||||
worker_api_errors.append(
|
||||
"authentication unavailable: "
|
||||
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
||||
)
|
||||
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None:
|
||||
worker_api_errors.append(
|
||||
"integration unavailable: "
|
||||
f"{OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
||||
)
|
||||
OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(
|
||||
worker_api_errors
|
||||
)
|
||||
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||
|
||||
Reference in New Issue
Block a user