feat(deploy): register Device Control Core releases
This commit is contained in:
@@ -259,6 +259,9 @@ DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V2_REL = (
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_REL = (
|
||||
"deployment/device-edge-core-channel-upgrade-v4.json"
|
||||
)
|
||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_REL = (
|
||||
"deployment/device-control-core-release-v1.json"
|
||||
)
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_COMPOSE_REL = (
|
||||
"docker-compose.device-edge-core-channel.yml"
|
||||
)
|
||||
@@ -311,6 +314,21 @@ DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_ENTRIES = (
|
||||
"services/device-control-core",
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_REL,
|
||||
)
|
||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_ENTRIES = (
|
||||
".dockerignore",
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"packages/device-protocol-contract",
|
||||
"packages/device-edge-channel-contract",
|
||||
"services/device-control-core",
|
||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_REL,
|
||||
)
|
||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_PATCH_ID = (
|
||||
"device-edge-core-channel-upgrade-v4-20260812-023"
|
||||
)
|
||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_ARTIFACT_SHA256 = (
|
||||
"c10d5b6b7d55ab239f85b6c8130e34ce9f84985e3b46e6e5534733156c7982fc"
|
||||
)
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_BOOTSTRAP_PREDECESSOR_PATCH_ID = (
|
||||
"device-edge-core-channel-bootstrap-20260812-018"
|
||||
)
|
||||
@@ -3875,6 +3893,7 @@ def allowed_payload_path(component, rel):
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_REL,
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V2_REL,
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_REL,
|
||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_REL,
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
|
||||
"packages/device-protocol-contract",
|
||||
@@ -8927,6 +8946,14 @@ def load_artifact(artifact, work_dir):
|
||||
payload_dir,
|
||||
expected_release_id=manifest["id"],
|
||||
)
|
||||
if is_device_plane_control_core_release_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
):
|
||||
validate_device_plane_control_core_release_payload(
|
||||
payload_dir,
|
||||
expected_release_id=manifest["id"],
|
||||
)
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
@@ -9485,6 +9512,14 @@ def is_device_plane_edge_core_channel_upgrade_v4_slice(component, entries):
|
||||
)
|
||||
|
||||
|
||||
def is_device_plane_control_core_release_slice(component, entries):
|
||||
return (
|
||||
component == "device-plane"
|
||||
and entries is not None
|
||||
and tuple(entries) == DEVICE_PLANE_CONTROL_CORE_RELEASE_ENTRIES
|
||||
)
|
||||
|
||||
|
||||
def is_device_plane_manager_v2_control_plane_slice(component, entries):
|
||||
return (
|
||||
component == "device-plane"
|
||||
@@ -9785,6 +9820,49 @@ def expected_device_plane_edge_core_channel_upgrade_v4_descriptor(
|
||||
}
|
||||
|
||||
|
||||
def expected_device_plane_control_core_release_descriptor(
|
||||
release_id,
|
||||
predecessor,
|
||||
):
|
||||
return {
|
||||
"schemaVersion": (
|
||||
"nodedc.device-plane.device-control-core-release.v1"
|
||||
),
|
||||
"releaseId": release_id,
|
||||
"action": "upgrade",
|
||||
"predecessor": predecessor,
|
||||
"service": "device-control-core",
|
||||
"composeActivation": "preserve-active-v4-topology",
|
||||
"identity": (
|
||||
"reuse-existing-runner-managed-host-local-private-key-"
|
||||
"public-certificate-export"
|
||||
),
|
||||
"identityRecovery": "forbidden-valid-existing-identity-required",
|
||||
"tlsPurpose": "clientAuth",
|
||||
"direction": "core-initiated",
|
||||
"endpointPolicy": "public-ipv4-standard-https-tcp-443-only",
|
||||
"coreNetworks": [
|
||||
"device-plane-private",
|
||||
"device-plane-egress",
|
||||
],
|
||||
"publicIngress": "none-on-synology",
|
||||
"edgeRegistrations": "preserved",
|
||||
"commandTransport": "disabled",
|
||||
"gelios": "untouched",
|
||||
"preservedServices": [
|
||||
"device-manager",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
"device-backhaul-target",
|
||||
],
|
||||
"healthGate": (
|
||||
"bounded-container-grace+core-edge-contract+"
|
||||
"exact-private-egress-network-boundary"
|
||||
),
|
||||
"rollback": "restore-preapply-source-and-core-runtime",
|
||||
}
|
||||
|
||||
|
||||
def expected_device_plane_manager_failed_control_plane_descriptor():
|
||||
return {
|
||||
"schemaVersion": (
|
||||
@@ -10202,6 +10280,70 @@ def validate_device_plane_edge_core_channel_upgrade_v4_payload(
|
||||
return descriptor
|
||||
|
||||
|
||||
def validate_device_plane_control_core_release_payload(
|
||||
payload_dir,
|
||||
*,
|
||||
expected_release_id=None,
|
||||
):
|
||||
descriptor = read_strict_json(
|
||||
payload_dir / DEVICE_PLANE_CONTROL_CORE_RELEASE_REL,
|
||||
"Device Control Core release descriptor",
|
||||
max_bytes=16 * 1024,
|
||||
)
|
||||
release_id = descriptor.get("releaseId")
|
||||
predecessor = descriptor.get("predecessor")
|
||||
valid_predecessor = (
|
||||
isinstance(predecessor, dict)
|
||||
and set(predecessor) == {"kind", "patchId", "artifactSha256"}
|
||||
and predecessor.get("kind") in (
|
||||
"edge-core-channel-upgrade-v4",
|
||||
"release",
|
||||
)
|
||||
and isinstance(predecessor.get("patchId"), str)
|
||||
and re.fullmatch(
|
||||
r"[A-Za-z0-9._-]{1,96}",
|
||||
predecessor["patchId"],
|
||||
)
|
||||
and isinstance(predecessor.get("artifactSha256"), str)
|
||||
and re.fullmatch(r"[0-9a-f]{64}", predecessor["artifactSha256"])
|
||||
)
|
||||
if valid_predecessor and predecessor["kind"] == (
|
||||
"edge-core-channel-upgrade-v4"
|
||||
):
|
||||
valid_predecessor = (
|
||||
predecessor["patchId"]
|
||||
== DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_PATCH_ID
|
||||
and predecessor["artifactSha256"]
|
||||
== DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_ARTIFACT_SHA256
|
||||
)
|
||||
elif valid_predecessor:
|
||||
valid_predecessor = (
|
||||
predecessor["patchId"].startswith(
|
||||
"device-control-core-release-"
|
||||
)
|
||||
and predecessor["patchId"] != release_id
|
||||
)
|
||||
if (
|
||||
not isinstance(release_id, str)
|
||||
or not re.fullmatch(
|
||||
r"device-control-core-release-[A-Za-z0-9._-]{1,67}",
|
||||
release_id,
|
||||
)
|
||||
or (
|
||||
expected_release_id is not None
|
||||
and release_id != expected_release_id
|
||||
)
|
||||
or not valid_predecessor
|
||||
or descriptor
|
||||
!= expected_device_plane_control_core_release_descriptor(
|
||||
release_id,
|
||||
predecessor,
|
||||
)
|
||||
):
|
||||
die("Device Control Core release descriptor mismatch")
|
||||
return descriptor
|
||||
|
||||
|
||||
def validate_device_plane_edge_core_channel_compose(payload_dir):
|
||||
compose = payload_dir / DEVICE_PLANE_EDGE_CORE_CHANNEL_COMPOSE_REL
|
||||
if sha256_file(compose) != DEVICE_PLANE_EDGE_CORE_CHANNEL_COMPOSE_SHA256:
|
||||
@@ -12389,6 +12531,114 @@ def validate_device_plane_edge_core_channel_upgrade_v4_predecessor(
|
||||
}
|
||||
|
||||
|
||||
def validate_device_plane_control_core_release_predecessor(payload_dir):
|
||||
descriptor = validate_device_plane_control_core_release_payload(
|
||||
payload_dir
|
||||
)
|
||||
predecessor = descriptor["predecessor"]
|
||||
artifact_name = f"nodedc-device-plane-{predecessor['patchId']}.tgz"
|
||||
artifact = APPLIED_DIR / artifact_name
|
||||
if (
|
||||
not artifact.is_file()
|
||||
or artifact.is_symlink()
|
||||
or sha256_file(artifact) != predecessor["artifactSha256"]
|
||||
):
|
||||
die("Device Control Core release predecessor mismatch")
|
||||
records = [
|
||||
row for row in load_state(STATE_FILE)
|
||||
if row.get("id") == predecessor["patchId"]
|
||||
and row.get("sha256") == predecessor["artifactSha256"]
|
||||
]
|
||||
if (
|
||||
len(records) != 1
|
||||
or records[0].get("status") != "ok"
|
||||
or records[0].get("component") != "device-plane"
|
||||
or records[0].get("artifact") != artifact_name
|
||||
):
|
||||
die("Device Control Core release predecessor journal mismatch")
|
||||
|
||||
installed_release = DEVICE_PLANE_ROOT / DEVICE_PLANE_CONTROL_CORE_RELEASE_REL
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="device-control-core-release-predecessor-",
|
||||
dir=TMP_DIR,
|
||||
) as directory:
|
||||
manifest, entries, predecessor_payload = load_artifact(
|
||||
artifact,
|
||||
Path(directory),
|
||||
)
|
||||
if (
|
||||
manifest.get("id") != predecessor["patchId"]
|
||||
or manifest.get("component") != "device-plane"
|
||||
or manifest.get("type") != "app-overlay"
|
||||
):
|
||||
die("Device Control Core release predecessor type mismatch")
|
||||
if predecessor["kind"] == "edge-core-channel-upgrade-v4":
|
||||
if tuple(entries) != DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_ENTRIES:
|
||||
die("Device Control Core release v4 predecessor type mismatch")
|
||||
validate_device_plane_edge_core_channel_upgrade_v4_payload(
|
||||
predecessor_payload,
|
||||
expected_transition_id=predecessor["patchId"],
|
||||
)
|
||||
source_entries = tuple(
|
||||
rel for rel in DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_ENTRIES
|
||||
if rel not in (
|
||||
"docker-compose.device-plane.yml",
|
||||
DEVICE_PLANE_EDGE_CORE_CHANNEL_UPGRADE_V4_REL,
|
||||
)
|
||||
)
|
||||
if installed_release.exists() or installed_release.is_symlink():
|
||||
die("Device Control Core first release is already installed")
|
||||
else:
|
||||
if tuple(entries) != DEVICE_PLANE_CONTROL_CORE_RELEASE_ENTRIES:
|
||||
die("Device Control Core release predecessor type mismatch")
|
||||
expected_installed = validate_device_plane_control_core_release_payload(
|
||||
predecessor_payload,
|
||||
expected_release_id=predecessor["patchId"],
|
||||
)
|
||||
installed_descriptor = read_strict_json(
|
||||
installed_release,
|
||||
"installed Device Control Core release predecessor",
|
||||
max_bytes=16 * 1024,
|
||||
)
|
||||
if installed_descriptor != expected_installed:
|
||||
die("Device Control Core release predecessor is not current")
|
||||
source_entries = DEVICE_PLANE_CONTROL_CORE_RELEASE_ENTRIES
|
||||
expected_source = collect_exact_files(
|
||||
predecessor_payload,
|
||||
source_entries,
|
||||
"Device Control Core release predecessor source",
|
||||
)
|
||||
actual_source = collect_exact_files(
|
||||
DEVICE_PLANE_ROOT,
|
||||
source_entries,
|
||||
"installed Device Control Core release predecessor source",
|
||||
)
|
||||
if actual_source != expected_source:
|
||||
die("installed Device Control Core release source drift detected")
|
||||
|
||||
component_compose_files("device-plane")
|
||||
identity_state = inspect_device_edge_channel_core_identity_state()
|
||||
if identity_state != "valid-reuse-at-apply":
|
||||
die("Device Control Core release requires the valid active identity")
|
||||
for service in (
|
||||
"device-control-core",
|
||||
"device-manager",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
):
|
||||
healthcheck_compose_service("device-plane", service)
|
||||
validate_device_manager_control_plane_runtime(
|
||||
require_edge_channel=True,
|
||||
core_network_mode="private-egress",
|
||||
)
|
||||
return {
|
||||
"mode": "active-device-control-core-forward-release",
|
||||
"descriptor": descriptor,
|
||||
"identityState": identity_state,
|
||||
"predecessorArtifact": artifact,
|
||||
}
|
||||
|
||||
|
||||
def validate_device_plane_manager_v2_reconciliation_backup():
|
||||
backup_dir = (
|
||||
BACKUPS_DIR / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID
|
||||
@@ -15833,6 +16083,9 @@ def is_platform_provider_catalog_only(entries):
|
||||
|
||||
|
||||
def component_services(component, entries=None):
|
||||
if is_device_plane_control_core_release_slice(component, entries):
|
||||
return ("device-control-core",)
|
||||
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(component, entries):
|
||||
return ("device-control-core",)
|
||||
|
||||
@@ -16409,7 +16662,13 @@ def component_builds(component, entries=None):
|
||||
if is_platform_device_manager_public_route_slice(component, entries):
|
||||
return ()
|
||||
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(component, entries):
|
||||
if (
|
||||
is_device_plane_control_core_release_slice(component, entries)
|
||||
or is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
):
|
||||
return ((
|
||||
DEVICE_PLANE_ROOT,
|
||||
(
|
||||
@@ -18481,6 +18740,7 @@ def plan_artifact(artifact):
|
||||
device_plane_b2_recovery_preflight = None
|
||||
device_plane_manager_activation_preflight = None
|
||||
device_plane_edge_core_channel_preflight = None
|
||||
device_plane_control_core_release_preflight = None
|
||||
device_plane_manager_reconciliation_preflight = None
|
||||
device_plane_manager_v2_reconciliation_preflight = None
|
||||
device_plane_backhaul_preflight = None
|
||||
@@ -18711,6 +18971,15 @@ def plan_artifact(artifact):
|
||||
payload_dir
|
||||
)
|
||||
)
|
||||
if is_device_plane_control_core_release_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
):
|
||||
device_plane_control_core_release_preflight = (
|
||||
validate_device_plane_control_core_release_predecessor(
|
||||
payload_dir
|
||||
)
|
||||
)
|
||||
if is_device_plane_manager_reconciliation_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
@@ -20129,9 +20398,12 @@ def plan_artifact(artifact):
|
||||
"runtime_secret=runner-managed:"
|
||||
f"{PLATFORM_DEVICE_CORE_INTERNAL_TOKEN_FILE}"
|
||||
)
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
if (
|
||||
is_device_plane_control_core_release_slice(component, entries)
|
||||
or is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
):
|
||||
print(
|
||||
"runtime_secret=runner-managed:"
|
||||
@@ -20331,6 +20603,46 @@ def plan_artifact(artifact):
|
||||
"device_plane_rollback="
|
||||
"source+reconciled-baseline-runtime"
|
||||
)
|
||||
if device_plane_control_core_release_preflight is not None:
|
||||
core_release = device_plane_control_core_release_preflight
|
||||
descriptor = core_release["descriptor"]
|
||||
predecessor = descriptor["predecessor"]
|
||||
print(
|
||||
"device_plane_transition="
|
||||
f"{core_release['mode']}"
|
||||
)
|
||||
print(f"device_control_core_release={descriptor['releaseId']}")
|
||||
print(
|
||||
"device_plane_predecessor_kind="
|
||||
f"{predecessor['kind']}"
|
||||
)
|
||||
print(
|
||||
"device_plane_predecessor_patch="
|
||||
f"{predecessor['patchId']}"
|
||||
)
|
||||
print(
|
||||
"device_plane_predecessor_artifact_sha256="
|
||||
f"{predecessor['artifactSha256']}"
|
||||
)
|
||||
print(
|
||||
"device_edge_channel_identity_preflight="
|
||||
f"{core_release['identityState']}"
|
||||
)
|
||||
print("device_plane_runtime_mutation=build+recreate:device-control-core")
|
||||
print(
|
||||
"device_plane_runtime_services="
|
||||
"preserved:device-manager,device-gateway,device-postgres,"
|
||||
"device-backhaul-target"
|
||||
)
|
||||
print("device_edge_channel=preserved:core-initiated:pinned-mtls:registered-edges-only")
|
||||
print("device_edge_channel_networks=preserved:device-plane-private,device-plane-egress")
|
||||
print("device_edge_channel_registrations=preserved")
|
||||
print("device_edge_channel_commands=disabled")
|
||||
print("device_manager=preserved:active")
|
||||
print("device_manager_public_route=unchanged:active")
|
||||
print("device_gateway_tcp_9921=preserved:loopback-only")
|
||||
print("gelios=untouched")
|
||||
print("device_plane_rollback=source+preapply-core-runtime")
|
||||
if device_plane_edge_core_channel_preflight is not None:
|
||||
print(
|
||||
"device_plane_transition="
|
||||
@@ -21096,7 +21408,25 @@ def rollback_device_plane_apply(
|
||||
baseline_entries,
|
||||
baseline_services,
|
||||
)
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
if is_device_plane_control_core_release_slice(
|
||||
"device-plane",
|
||||
entries,
|
||||
):
|
||||
for service in (
|
||||
"device-control-core",
|
||||
"device-manager",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
):
|
||||
healthcheck_compose_service_with_grace(
|
||||
"device-plane",
|
||||
service,
|
||||
)
|
||||
validate_device_manager_control_plane_runtime(
|
||||
require_edge_channel=True,
|
||||
core_network_mode="private-egress",
|
||||
)
|
||||
elif is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
"device-plane",
|
||||
entries,
|
||||
):
|
||||
@@ -21921,9 +22251,12 @@ def prepare_component_runtime(component, entries=None):
|
||||
MAP_GATEWAY_SECRET_RE,
|
||||
"Device Core Hub handoff",
|
||||
)
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
if (
|
||||
is_device_plane_control_core_release_slice(component, entries)
|
||||
or is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
):
|
||||
ensure_platform_runtime_secret(
|
||||
DEVICE_PLANE_MANAGEMENT_CORE_TOKEN_FILE,
|
||||
@@ -21932,7 +22265,11 @@ def prepare_component_runtime(component, entries=None):
|
||||
)
|
||||
ensure_device_edge_channel_core_identity(
|
||||
allow_invalid_unexported_recovery=(
|
||||
not is_device_plane_edge_core_channel_upgrade_slice(
|
||||
not is_device_plane_control_core_release_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
and not is_device_plane_edge_core_channel_upgrade_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
@@ -22245,7 +22582,13 @@ def component_healthchecks(component, entries=None, services=None):
|
||||
"commandTransport": "disabled",
|
||||
},
|
||||
},)
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(component, entries):
|
||||
if (
|
||||
is_device_plane_control_core_release_slice(component, entries)
|
||||
or is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
):
|
||||
return ({
|
||||
"url": "http://127.0.0.1:18120/healthz",
|
||||
"expected_json": {
|
||||
@@ -22811,6 +23154,27 @@ process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plan
|
||||
|
||||
|
||||
def run_healthchecks(component, entries=None, services=None):
|
||||
if is_device_plane_control_core_release_slice(component, entries):
|
||||
if tuple(services or ()) != ("device-control-core",):
|
||||
die("Device Control Core release service set mismatch")
|
||||
for service in (
|
||||
"device-control-core",
|
||||
"device-manager",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
):
|
||||
healthcheck_compose_service_with_grace(
|
||||
"device-plane",
|
||||
service,
|
||||
)
|
||||
for check in component_healthchecks(component, entries, services):
|
||||
healthcheck_url(check)
|
||||
validate_device_manager_control_plane_runtime(
|
||||
require_edge_channel=True,
|
||||
core_network_mode="private-egress",
|
||||
)
|
||||
return
|
||||
|
||||
if is_device_plane_manager_reconciliation_slice(component, entries):
|
||||
if tuple(services or ()) != ():
|
||||
die("Device Manager reconciliation service set mismatch")
|
||||
@@ -23854,6 +24218,13 @@ def apply_artifact(artifact):
|
||||
validate_device_plane_edge_core_channel_bootstrap_predecessor(
|
||||
payload_dir
|
||||
)
|
||||
if is_device_plane_control_core_release_slice(
|
||||
component,
|
||||
entries,
|
||||
):
|
||||
validate_device_plane_control_core_release_predecessor(
|
||||
payload_dir
|
||||
)
|
||||
if is_device_plane_manager_reconciliation_slice(
|
||||
component,
|
||||
entries,
|
||||
@@ -24174,9 +24545,15 @@ def apply_artifact(artifact):
|
||||
*inventory_services,
|
||||
"device-manager",
|
||||
)
|
||||
if is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
if (
|
||||
is_device_plane_control_core_release_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
or is_device_plane_edge_core_channel_bootstrap_slice(
|
||||
component,
|
||||
entries,
|
||||
)
|
||||
):
|
||||
inventory_services = (
|
||||
*inventory_services,
|
||||
|
||||
Reference in New Issue
Block a user