SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): stabilize pre-capture methodology

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-28 17:47:06 +03:00
parent 1f20e0d7d9
commit d729abab31
65 changed files with 9698 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/telemetry-plane/.env.example
+2
View File
@@ -4,7 +4,9 @@ MISSIONCORE_TELEMETRY_QUERY_PORT=18030
MISSIONCORE_DB_NAME=missioncore_telemetry
MISSIONCORE_DB_USER=missioncore_ingest
MISSIONCORE_DB_PASSWORD=replace-with-a-random-local-secret
MISSIONCORE_DB_INGEST_PASSWORD=replace-with-a-separate-ingest-secret
MISSIONCORE_MQTT_INGEST_USER=missioncore-ingest
MISSIONCORE_MQTT_INGEST_PASSWORD=replace-with-a-random-local-secret
MISSIONCORE_MQTT_WORKER_006_USER=worker-006
MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006
MISSIONCORE_MQTT_WORKER_006_PASSWORD=replace-with-a-different-random-local-secret
deploy/telemetry-plane/README.md
+22 -2
View File
@@ -2,12 +2,13 @@
Portable local-only telemetry infrastructure for compute contours.
One Docker Compose project, `ndc-mission-core-telemetry`, owns three isolated
containers:
One Docker Compose project, `ndc-mission-core-telemetry`, owns three long-running
containers and one bounded bootstrap job:
- `ndc-mission-core-mqtt-broker` — Eclipse Mosquitto `2.1.2-alpine`;
- `ndc-mission-core-telemetry-timescaledb` — TimescaleDB HA OSS
`pg16.14-ts2.28.2-all-oss`;
- `ndc-mission-core-telemetry-bootstrap` — one-shot database role/bootstrap job;
- `ndc-mission-core-telemetry-normalizer` — Mission Core telemetry normalizer.
The Telegraf configuration templates cover Windows and Linux, but the agent runs as a
@@ -17,6 +18,11 @@ Windows performance counters, Docker Desktop and NVIDIA telemetry.
The normalizer exposes a read-only normalized telemetry adapter on
`127.0.0.1:18030`. Mission Core reads this adapter; the browser never receives MQTT or
Timescale credentials, and Timescale is not published on a host port.
The normalizer uses a separate `missioncore_normalizer` database role with only
`SELECT`, `INSERT`, `UPDATE`, and bounded retention `DELETE` on the telemetry
hypertable. Raw telemetry older than 30 days is removed at most once per day by the
normalizer. This stays compatible with the Apache-licensed Timescale image without
depending on the Timescale License retention scheduler.
The stack is one deployment contour, not one multi-process container. Keeping broker,
normalizer and database in separate containers preserves independent health checks,
@@ -62,6 +68,20 @@ directory are ignored by Git.
The product architecture and topic contract are defined in
`docs/adr/0031-local-compute-contour-telemetry-plane.md`.
## Security boundary
The current MQTT listener is authenticated and contour-scoped, but intentionally
uses plaintext MQTT inside one owner-controlled laboratory LAN. Do not expose port
1883 through a router, cellular WAN, public Wi-Fi, or an Internet-facing host. A
remote or shared-network deployment requires a reviewed TLS listener, a private CA
distributed to every agent, and credential rotation. Wi-Fi link encryption is not a
replacement for MQTT TLS.
Each agent credential is bound to one exact
`contours/<contour-id>/agents/<agent-id>/+` prefix. Adding another contour requires
issuing another password entry and explicit ACL row; the wildcard contour writer is
not permitted.
## Worker agent
Worker 006 uses the official Windows Telegraf distribution as the host service
deploy/telemetry-plane/compose.yaml
+57 -1
View File
@@ -10,6 +10,17 @@ services:
image: eclipse-mosquitto:2.1.2-alpine
container_name: ndc-mission-core-mqtt-broker
restart: unless-stopped
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
mem_limit: 128m
pids_limit: 128
labels:
<<: *ndc-labels
com.nodedc.role: mqtt-broker
@@ -23,6 +34,8 @@ services:
- ./runtime/mosquitto/passwords:/mosquitto/config/passwords:ro
- ./runtime/mosquitto/acl:/mosquitto/config/acl:ro
- broker-data:/mosquitto/data
tmpfs:
- /tmp:size=16m,noexec,nosuid
healthcheck:
test:
- CMD-SHELL
@@ -38,6 +51,10 @@ services:
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
container_name: ndc-mission-core-telemetry-timescaledb
restart: unless-stopped
security_opt:
- no-new-privileges:true
mem_limit: 1024m
pids_limit: 256
labels:
<<: *ndc-labels
com.nodedc.role: telemetry-database
@@ -54,6 +71,34 @@ services:
timeout: 5s
retries: 10
database-bootstrap:
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
container_name: ndc-mission-core-telemetry-bootstrap
restart: "no"
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
mem_limit: 128m
pids_limit: 64
labels:
<<: *ndc-labels
com.nodedc.role: telemetry-database-bootstrap
depends_on:
timescale:
condition: service_healthy
environment:
MISSIONCORE_DB_NAME: "${MISSIONCORE_DB_NAME}"
MISSIONCORE_DB_ADMIN_USER: "${MISSIONCORE_DB_USER}"
MISSIONCORE_DB_ADMIN_PASSWORD: "${MISSIONCORE_DB_PASSWORD}"
MISSIONCORE_DB_INGEST_PASSWORD: "${MISSIONCORE_DB_INGEST_PASSWORD}"
volumes:
- ./timescale/002_roles_and_retention.sh:/bootstrap.sh:ro
tmpfs:
- /tmp:size=16m,noexec,nosuid
entrypoint: ["/bin/sh", "/bootstrap.sh"]
normalizer:
image: nodedc/mission-core-telemetry-normalizer:local
container_name: ndc-mission-core-telemetry-normalizer
@@ -61,6 +106,13 @@ services:
context: ../..
dockerfile: deploy/telemetry-plane/normalizer/Dockerfile
restart: unless-stopped
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
mem_limit: 256m
pids_limit: 128
labels:
<<: *ndc-labels
com.nodedc.role: telemetry-normalizer
@@ -69,6 +121,8 @@ services:
condition: service_healthy
timescale:
condition: service_healthy
database-bootstrap:
condition: service_completed_successfully
ports:
- "127.0.0.1:${MISSIONCORE_TELEMETRY_QUERY_PORT:-18030}:18030"
environment:
@@ -77,9 +131,11 @@ services:
MISSIONCORE_MQTT_USERNAME: "${MISSIONCORE_MQTT_INGEST_USER}"
MISSIONCORE_MQTT_PASSWORD: "${MISSIONCORE_MQTT_INGEST_PASSWORD}"
MISSIONCORE_DATABASE_DSN: >-
postgresql://${MISSIONCORE_DB_USER}:${MISSIONCORE_DB_PASSWORD}@timescale:5432/${MISSIONCORE_DB_NAME}
postgresql://missioncore_normalizer:${MISSIONCORE_DB_INGEST_PASSWORD}@timescale:5432/${MISSIONCORE_DB_NAME}
MISSIONCORE_QUERY_HOST: "0.0.0.0"
MISSIONCORE_QUERY_PORT: "18030"
tmpfs:
- /tmp:size=32m,noexec,nosuid
healthcheck:
test:
- CMD
deploy/telemetry-plane/normalizer/normalizer.py
+191 -19
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import contextlib
import json
import os
import re
@@ -25,6 +26,40 @@ SOURCE_SCHEMAS: Final = {
TELEGRAF_SCHEMA: Final = "telegraf.metric-json/v1"
QUERY_SCHEMA: Final = "missioncore.telemetry-query/v1"
SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
MAX_TAGS: Final = 256
MAX_SERIES_KEY_BYTES: Final = 4096
RETENTION_INTERVAL_SECONDS: Final = 86_400
ALLOWED_TELEMETRY_TAGS: Final = frozenset(
{
"agent_id",
"container",
"container_id",
"container_image",
"container_name",
"container_status",
"container_version",
"contour_id",
"cpu",
"device",
"engine_host",
"gpu_name",
"host",
"interface",
"lab_id",
"method_id",
"name",
"node_id",
"path",
"request_id",
"run_id",
"server_version",
"source_id",
"source_package_id",
"stage_id",
"stage_state",
}
)
def _required(name: str) -> str:
@@ -68,26 +103,48 @@ def _series_key(document: dict[str, Any]) -> str:
if not isinstance(tags, dict):
value = document.get("series_key")
return value if isinstance(value, str) else ""
if len(tags) > MAX_TAGS:
raise ValueError("telemetry payload contains too many tags")
stable_tags = {
str(name): str(value)
for name, value in tags.items()
if name not in {"agent_id", "contour_id", "host", "node_id"}
}
return json.dumps(
encoded = json.dumps(
stable_tags,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
if len(encoded.encode("utf-8")) > MAX_SERIES_KEY_BYTES:
raise ValueError("telemetry series identity is too large")
return encoded
def _sanitize_tags(document: dict[str, Any]) -> dict[str, Any]:
tags = document.get("tags")
if not isinstance(tags, dict):
return document
if len(tags) > MAX_TAGS:
raise ValueError("telemetry payload contains too many tags")
sanitized = {
str(name): str(value)
for name, value in tags.items()
if name in ALLOWED_TELEMETRY_TAGS
}
return {**document, "tags": sanitized}
def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
if len(payload) > MAX_PAYLOAD_BYTES:
raise ValueError("telemetry payload exceeds the 1 MiB contract")
match = TOPIC_RE.fullmatch(topic)
if match is None:
raise ValueError("topic is outside the telemetry contract")
document = json.loads(payload.decode("utf-8"))
if not isinstance(document, dict):
raise ValueError("payload must be an object")
document = _sanitize_tags(document)
kind = match.group("kind")
source_schema = document.get("schema_version")
if source_schema == SOURCE_SCHEMAS[kind]:
@@ -101,14 +158,18 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
source_schema = TELEGRAF_SCHEMA
observed_value = document.get("timestamp")
measurement = document["name"].strip()
if not measurement:
if not measurement or len(measurement) > 128:
raise ValueError("Telegraf measurement name is required")
else:
raise ValueError("payload schema does not match topic kind")
node_id = document.get("node_id") or _tag_text(document, "node_id")
if not node_id:
node_id = _tag_text(document, "host")
if not isinstance(node_id, str) or not node_id.strip():
if (
not isinstance(node_id, str)
or not node_id.strip()
or len(node_id.strip()) > 128
):
raise ValueError("node_id is required")
observed_at = _timestamp(observed_value)
return (
@@ -132,9 +193,53 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
class TelemetryQueryServer(ThreadingHTTPServer):
daemon_threads = True
def __init__(self, server_address: tuple[str, int], dsn: str) -> None:
def __init__(
self,
server_address: tuple[str, int],
dsn: str,
runtime_health: TelemetryRuntimeHealth,
) -> None:
super().__init__(server_address, TelemetryQueryHandler)
self.dsn = dsn
self.runtime_health = runtime_health
class TelemetryRuntimeHealth:
"""Thread-safe health of the actual MQTT-to-database ingestion path."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._mqtt_connected = False
self._database_connected = False
self._last_ingested_at_utc: str | None = None
self._last_error: str | None = None
def set_mqtt(self, connected: bool, error: str | None = None) -> None:
with self._lock:
self._mqtt_connected = connected
self._last_error = error
def set_database(self, connected: bool, error: str | None = None) -> None:
with self._lock:
self._database_connected = connected
self._last_error = error
def mark_ingested(self) -> None:
with self._lock:
self._database_connected = True
self._last_ingested_at_utc = (
datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z")
)
self._last_error = None
def snapshot(self) -> dict[str, object]:
with self._lock:
return {
"mqtt_connected": self._mqtt_connected,
"database_connected": self._database_connected,
"last_ingested_at_utc": self._last_ingested_at_utc,
"last_error": self._last_error,
}
class TelemetryQueryHandler(BaseHTTPRequestHandler):
@@ -174,6 +279,7 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler):
def _health(self) -> None:
import psycopg # type: ignore[import-not-found]
database_reachable = False
try:
with (
psycopg.connect(self.server.dsn) as connection,
@@ -181,13 +287,24 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler):
):
cursor.execute("SELECT 1")
cursor.fetchone()
database_reachable = True
except psycopg.Error:
self._json(
HTTPStatus.SERVICE_UNAVAILABLE,
{"ok": False, "schema_version": QUERY_SCHEMA},
)
return
self._json(HTTPStatus.OK, {"ok": True, "schema_version": QUERY_SCHEMA})
database_reachable = False
runtime = self.server.runtime_health.snapshot()
ok = (
database_reachable
and runtime["mqtt_connected"] is True
and runtime["database_connected"] is True
)
self._json(
HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE,
{
"ok": ok,
"schema_version": QUERY_SCHEMA,
"database_reachable": database_reachable,
**runtime,
},
)
def _latest(
self,
@@ -270,10 +387,13 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler):
self.wfile.write(body)
def _start_query_server(dsn: str) -> ThreadingHTTPServer:
def _start_query_server(
dsn: str,
runtime_health: TelemetryRuntimeHealth,
) -> ThreadingHTTPServer:
host = os.environ.get("MISSIONCORE_QUERY_HOST", "0.0.0.0")
port = int(os.environ.get("MISSIONCORE_QUERY_PORT", "18030"))
server = TelemetryQueryServer((host, port), dsn)
server = TelemetryQueryServer((host, port), dsn, runtime_health)
thread = threading.Thread(
target=server.serve_forever,
name="telemetry-query",
@@ -294,8 +414,11 @@ def main() -> None:
port = int(os.environ.get("MISSIONCORE_MQTT_PORT", "1883"))
username = _required("MISSIONCORE_MQTT_USERNAME")
password = _required("MISSIONCORE_MQTT_PASSWORD")
_start_query_server(dsn)
runtime_health = TelemetryRuntimeHealth()
connection = psycopg.connect(dsn, autocommit=True)
last_retention_monotonic = 0.0
runtime_health.set_database(True)
_start_query_server(dsn, runtime_health)
client = mqtt.Client(
callback_api_version=CallbackAPIVersion.VERSION2,
client_id="ndc-mission-core-telemetry-normalizer",
@@ -311,14 +434,44 @@ def main() -> None:
_properties: Any,
) -> None:
if reason_code.is_failure:
runtime_health.set_mqtt(False, f"MQTT connection rejected: {reason_code}")
raise RuntimeError(f"MQTT connection rejected: {reason_code}")
connected_client.subscribe(TOPIC, qos=1)
runtime_health.set_mqtt(True)
def on_disconnect(
_connected_client: Any,
_userdata: object,
_disconnect_flags: Any,
reason_code: Any,
_properties: Any,
) -> None:
runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}")
def on_message(_client: Any, _userdata: object, message: Any) -> None:
nonlocal connection, last_retention_monotonic
try:
row = _normalize(message.topic, message.payload)
with connection.cursor() as cursor:
cursor.execute(
except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
print(f"telemetry sample rejected: {exc}", flush=True)
return
for attempt in range(2):
try:
with connection.cursor() as cursor:
now_monotonic = time.monotonic()
if (
now_monotonic - last_retention_monotonic
>= RETENTION_INTERVAL_SECONDS
):
cursor.execute(
"""
DELETE FROM contour_telemetry_samples
WHERE observed_at
< CURRENT_TIMESTAMP - INTERVAL '30 days'
"""
)
last_retention_monotonic = now_monotonic
cursor.execute(
"""
INSERT INTO contour_telemetry_samples (
observed_at, contour_id, agent_id, node_id, kind, measurement,
@@ -358,18 +511,37 @@ def main() -> None:
ELSE EXCLUDED.payload
END
""",
(*row, TELEGRAF_SCHEMA, TELEGRAF_SCHEMA),
)
except (ValueError, UnicodeDecodeError, json.JSONDecodeError, psycopg.Error) as exc:
print(f"telemetry sample rejected: {exc}", flush=True)
(*row, TELEGRAF_SCHEMA, TELEGRAF_SCHEMA),
)
runtime_health.mark_ingested()
return
except psycopg.Error as exc:
runtime_health.set_database(False, f"database ingestion failed: {exc}")
with contextlib.suppress(psycopg.Error):
connection.close()
if attempt == 0:
try:
connection = psycopg.connect(dsn, autocommit=True)
runtime_health.set_database(True)
except psycopg.Error as reconnect_error:
runtime_health.set_database(
False,
f"database reconnect failed: {reconnect_error}",
)
break
else:
break
print("telemetry sample rejected: database unavailable", flush=True)
client.on_connect = on_connect
client.on_disconnect = on_disconnect
client.on_message = on_message
while True:
try:
client.connect(host, port, keepalive=30)
client.loop_forever(retry_first_connection=True)
except (OSError, MQTTException, psycopg.Error) as exc:
runtime_health.set_mqtt(False, f"MQTT reconnecting: {exc}")
print(f"telemetry normalizer reconnecting: {exc}", flush=True)
time.sleep(3)
deploy/telemetry-plane/prepare.py
+71 -5
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import ipaddress
import os
import re
import secrets
import subprocess
from pathlib import Path
@@ -13,6 +14,9 @@ ENV_PATH: Final = ROOT / ".env"
RUNTIME: Final = ROOT / "runtime" / "mosquitto"
IMAGE: Final = "eclipse-mosquitto:2.1.2-alpine"
PLACEHOLDER: Final = "replace-with-"
SAFE_IDENTIFIER: Final = re.compile(
r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$"
)
def _initialize_environment(bind_address: str, *, overwrite: bool = False) -> None:
@@ -26,9 +30,11 @@ def _initialize_environment(bind_address: str, *, overwrite: bool = False) -> No
"MISSIONCORE_DB_NAME": "missioncore_telemetry",
"MISSIONCORE_DB_USER": "missioncore_ingest",
"MISSIONCORE_DB_PASSWORD": secrets.token_urlsafe(36),
"MISSIONCORE_DB_INGEST_PASSWORD": secrets.token_urlsafe(36),
"MISSIONCORE_MQTT_INGEST_USER": "missioncore-ingest",
"MISSIONCORE_MQTT_INGEST_PASSWORD": secrets.token_urlsafe(36),
"MISSIONCORE_MQTT_WORKER_006_USER": "worker-006",
"MISSIONCORE_MQTT_WORKER_006_CONTOUR": "worker-006",
"MISSIONCORE_MQTT_WORKER_006_PASSWORD": secrets.token_urlsafe(36),
}
ENV_PATH.write_text(
@@ -51,6 +57,23 @@ def _environment() -> dict[str, str]:
return values
def _migrate_environment() -> None:
values = _environment()
additions: dict[str, str] = {}
if "MISSIONCORE_DB_INGEST_PASSWORD" not in values:
additions["MISSIONCORE_DB_INGEST_PASSWORD"] = secrets.token_urlsafe(36)
if "MISSIONCORE_MQTT_WORKER_006_CONTOUR" not in values:
additions["MISSIONCORE_MQTT_WORKER_006_CONTOUR"] = "worker-006"
if not additions:
return
with ENV_PATH.open("a", encoding="utf-8", newline="\n") as stream:
for name, value in additions.items():
stream.write(f"{name}={value}\n")
stream.flush()
os.fsync(stream.fileno())
os.chmod(ENV_PATH, 0o600)
def _required(values: dict[str, str], name: str) -> str:
value = values.get(name, "")
if not value or value.startswith(PLACEHOLDER):
@@ -58,6 +81,13 @@ def _required(values: dict[str, str], name: str) -> str:
return value
def _identifier(values: dict[str, str], name: str) -> str:
value = _required(values, name)
if SAFE_IDENTIFIER.fullmatch(value) is None:
raise RuntimeError(f"{name} must contain a DNS-safe lowercase identifier")
return value
def _password_entry(path: Path, username: str, password: str, *, create: bool) -> None:
command = [
"docker",
@@ -80,6 +110,22 @@ def _password_entry(path: Path, username: str, password: str, *, create: bool) -
)
def _prepare_password_entries(
path: Path,
ingest_user: str,
ingest_password: str,
worker_user: str,
worker_password: str,
) -> None:
_password_entry(
path,
ingest_user,
ingest_password,
create=not path.exists(),
)
_password_entry(path, worker_user, worker_password, create=False)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -87,6 +133,11 @@ def main() -> None:
action="store_true",
help="create a private .env with generated local credentials",
)
parser.add_argument(
"--migrate",
action="store_true",
help="add newly required generated secrets without replacing existing values",
)
parser.add_argument(
"--mqtt-bind-address",
default="127.0.0.1",
@@ -95,17 +146,28 @@ def main() -> None:
arguments = parser.parse_args()
if arguments.initialize:
_initialize_environment(arguments.mqtt_bind_address)
if arguments.migrate:
_migrate_environment()
values = _environment()
ingest_user = _required(values, "MISSIONCORE_MQTT_INGEST_USER")
ingest_password = _required(values, "MISSIONCORE_MQTT_INGEST_PASSWORD")
worker_user = _required(values, "MISSIONCORE_MQTT_WORKER_006_USER")
worker_user = _identifier(values, "MISSIONCORE_MQTT_WORKER_006_USER")
worker_contour = _identifier(
values,
"MISSIONCORE_MQTT_WORKER_006_CONTOUR",
)
worker_password = _required(values, "MISSIONCORE_MQTT_WORKER_006_PASSWORD")
_required(values, "MISSIONCORE_DB_PASSWORD")
RUNTIME.mkdir(parents=True, exist_ok=True)
password_path = RUNTIME / "passwords"
_password_entry(password_path, ingest_user, ingest_password, create=True)
_password_entry(password_path, worker_user, worker_password, create=False)
_prepare_password_entries(
password_path,
ingest_user,
ingest_password,
worker_user,
worker_password,
)
acl_path = RUNTIME / "acl"
acl_path.write_text(
"\n".join(
@@ -114,8 +176,12 @@ def main() -> None:
"topic read mission-core/v1/contours/+/agents/+/+",
"topic read $SYS/broker/uptime",
"",
"# Agent username must equal its stable agent id.",
"pattern write mission-core/v1/contours/+/agents/%u/+",
"# Every credential is scoped to one contour and one stable agent.",
f"user {worker_user}",
(
"topic write mission-core/v1/contours/"
f"{worker_contour}/agents/{worker_user}/+"
),
"",
]
),
deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1
+4 -2
View File
@@ -20,7 +20,8 @@ foreach ($name in @(
"MISSIONCORE_MQTT_HOST",
"MISSIONCORE_MQTT_PORT",
"MISSIONCORE_MQTT_USERNAME",
"MISSIONCORE_MQTT_PASSWORD"
"MISSIONCORE_MQTT_PASSWORD",
"MISSIONCORE_TELEMETRY_INTERVAL"
)) {
$value = $payload.$name
if (-not $value) {
@@ -87,7 +88,8 @@ try {
"MISSIONCORE_MQTT_HOST=$($payload.MISSIONCORE_MQTT_HOST)",
"MISSIONCORE_MQTT_PORT=$($payload.MISSIONCORE_MQTT_PORT)",
"MISSIONCORE_MQTT_USERNAME=$($payload.MISSIONCORE_MQTT_USERNAME)",
"MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)"
"MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)",
"MISSIONCORE_TELEMETRY_INTERVAL=$($payload.MISSIONCORE_TELEMETRY_INTERVAL)"
)
$serviceRegistryPath = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName"
Set-ItemProperty -Path $serviceRegistryPath -Name Environment `
deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1
+3
View File
@@ -26,6 +26,9 @@ foreach ($entry in @((Get-ItemProperty -Path $serviceRegistryPath).Environment))
Set-Item -Path "Env:$name" -Value $value
}
}
if (-not $env:MISSIONCORE_TELEMETRY_INTERVAL) {
$env:MISSIONCORE_TELEMETRY_INTERVAL = "2s"
}
$temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-update-$([Guid]::NewGuid().ToString('N'))"
$validationOutput = Join-Path $temporaryRoot "validation.out.log"
deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl
+1 -1
View File
@@ -1,5 +1,5 @@
[agent]
interval = "2s"
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
round_interval = true
omit_hostname = false
deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl
+1 -1
View File
@@ -1,5 +1,5 @@
[agent]
interval = "2s"
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
round_interval = true
omit_hostname = false
deploy/telemetry-plane/timescale/002_roles_and_retention.sh
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
set -eu
export PGPASSWORD="${MISSIONCORE_DB_ADMIN_PASSWORD}"
psql \
--host timescale \
--username "${MISSIONCORE_DB_ADMIN_USER}" \
--dbname "${MISSIONCORE_DB_NAME}" \
--set ON_ERROR_STOP=1 \
--set database_name="${MISSIONCORE_DB_NAME}" \
--set ingest_password="${MISSIONCORE_DB_INGEST_PASSWORD}" <<'SQL'
DO $roles$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_roles WHERE rolname = 'missioncore_normalizer'
) THEN
CREATE ROLE missioncore_normalizer LOGIN;
END IF;
END
$roles$;
ALTER ROLE missioncore_normalizer PASSWORD :'ingest_password';
GRANT CONNECT ON DATABASE :"database_name" TO missioncore_normalizer;
GRANT USAGE ON SCHEMA public TO missioncore_normalizer;
GRANT SELECT, INSERT, UPDATE, DELETE ON contour_telemetry_samples
TO missioncore_normalizer;
DELETE FROM contour_telemetry_samples
WHERE observed_at < CURRENT_TIMESTAMP - INTERVAL '30 days';
SQL