Fix onboard WebKit preview and archive board telemetry locally

apps/node-agent/packaging/build_deb.py

@@ -11,7 +11,7 @@ import sys
 
 
 ROOT = Path(__file__).resolve().parents[1]
-VERSION = "0.8.14"
+VERSION = "0.8.15"
 sys.path.insert(0, str(ROOT.parents[1] / "scripts/packaging"))
 from debian import package
 
@@ -43,7 +43,7 @@ Architecture: amd64
 Maintainer: NODE.DC local build <noreply@example.invalid>
 Section: admin
 Priority: optional
-Depends: adduser, systemd, python3, python3-gi, gir1.2-gtk-3.0, gir1.2-webkit2-4.1, pkexec, polkitd, ca-certificates, hicolor-icon-theme, network-manager, iproute2
+Depends: adduser, systemd, python3, python3-gi, gir1.2-gtk-3.0, gir1.2-webkit2-4.1, pkexec, polkitd, ca-certificates, hicolor-icon-theme, network-manager, iproute2, python3-psycopg2, postgresql-16 (>= 16.15), timescaledb-2-oss-postgresql-16 (= 2.29.2~ubuntu24.04-1615)
 Description: Mission Core onboard computer configuration
  Local graphical setup, host inventory, SSH access and persistent node identity.
 """.encode()
@@ -67,8 +67,13 @@ Description: Mission Core onboard computer configuration
         ("environment_helper.py", "usr/lib/mission-core-node/environment_helper.py", 0o644),
         ("mission-core-node-environment.service", "usr/lib/systemd/system/mission-core-node-environment.service", 0o644),
         ("60-environment.conf", "usr/share/mission-core-node/60-environment.conf", 0o644),
+        ("setup-monitor", "usr/lib/mission-core-node/setup-monitor", 0o755),
+        ("mission-core-node-monitor.service", "usr/lib/systemd/system/mission-core-node-monitor.service", 0o644),
     ]:
         files.append((path, (p / source).read_bytes(), mode))
+    for path in sorted((ROOT / 'monitor').iterdir()):
+        if path.suffix in {'.py', '.sql'}:
+            files.append(('usr/lib/mission-core-node/monitor/' + path.name, path.read_bytes(), 0o644))
     files.append(("usr/share/mission-core-node/environment-profile.json", (ROOT / "internal/node/environment-profile.json").read_bytes(), 0o644))
     for name in ("realsense_prepare.py", "realsense_iio_access.py"):
         files.append(("usr/lib/mission-core-node/" + name, (p / name).read_bytes(), 0o644))

apps/node-agent/packaging/fetch_monitor_dependencies.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Stage verified Timescale OSS Ubuntu packages, without APT/system mutation."""
+
+import argparse
+import gzip
+import hashlib
+import json
+import subprocess
+import urllib.request
+from pathlib import Path
+
+BASE = "https://packagecloud.io/timescale/timescaledb/ubuntu/"
+VERSION = "2.29.2"
+
+
+def fetch(url, limit):
+    with urllib.request.urlopen(url, timeout=30) as response:
+        data = response.read(limit + 1)
+    if len(data) > limit:
+        raise ValueError("Download budget exceeded")
+    return data
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("destination", type=Path)
+    args = parser.parse_args()
+    root = args.destination
+    root.mkdir(parents=True, exist_ok=True)
+    key = root / "timescale.asc"
+    key.write_bytes(fetch("https://packagecloud.io/timescale/timescaledb/gpgkey", 65536))
+    keyring = root / "timescale.gpg"
+    subprocess.run(
+        ["gpg", "--batch", "--yes", "--dearmor", "--output", str(keyring), str(key)], check=True
+    )
+    release = root / "InRelease"
+    release.write_bytes(fetch(BASE + "dists/noble/InRelease", 1048576))
+    subprocess.run(["gpgv", "--keyring", str(keyring.resolve()), str(release)], check=True)
+    lines = release.read_text().splitlines()
+    hashes = {}
+    inside = False
+    for line in lines:
+        if line == "SHA256:":
+            inside = True
+            continue
+        if inside and not line.startswith(" "):
+            break
+        if inside:
+            sha, size, path = line.split()
+            hashes[path] = (sha, int(size))
+    sha, size = hashes["main/binary-amd64/Packages.gz"]
+    compressed = fetch(BASE + "dists/noble/main/binary-amd64/Packages.gz", 16000000)
+    if len(compressed) != size or hashlib.sha256(compressed).hexdigest() != sha:
+        raise ValueError("Signed package index mismatch")
+    (root / "Packages.gz").write_bytes(compressed)
+    wanted = {
+        name: VERSION + "~ubuntu24.04-1615"
+        for name in ("timescaledb-2-oss-postgresql-16", "timescaledb-2-loader-postgresql-16")
+    }
+    artifacts = []
+    for block in gzip.decompress(compressed).decode().split("\n\n"):
+        fields = dict(
+            line.split(": ", 1)
+            for line in block.splitlines()
+            if ": " in line and not line.startswith(" ")
+        )
+        if (
+            fields.get("Package") not in wanted
+            or fields.get("Version") != wanted[fields["Package"]]
+        ):
+            continue
+        path = fields["Filename"]
+        if not path.startswith("pool/") or ".." in path.split("/"):
+            raise ValueError("Invalid package path")
+        data = fetch(BASE + path, 64000000)
+        if hashlib.sha256(data).hexdigest() != fields["SHA256"]:
+            raise ValueError("Package hash mismatch")
+        target = root / Path(path).name
+        target.write_bytes(data)
+        artifacts.append(
+            dict(
+                package=fields["Package"],
+                version=fields["Version"],
+                file=target.name,
+                sha256=fields["SHA256"],
+                bytes=len(data),
+                depends=fields["Depends"],
+            )
+        )
+    if len(artifacts) != len(wanted):
+        raise ValueError("Pinned package not found")
+    manifest = dict(
+        source=BASE,
+        version=VERSION,
+        key_sha256=hashlib.sha256(key.read_bytes()).hexdigest(),
+        inrelease_sha256=hashlib.sha256(release.read_bytes()).hexdigest(),
+        artifacts=artifacts,
+    )
+    (root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+    print(json.dumps(manifest, indent=2))
+
+
+if __name__ == "__main__":
+    main()

apps/node-agent/packaging/install-monitor-release

@@ -0,0 +1,56 @@
+#!/bin/sh
+# Run by the owner-facing release installer after hash verification. Scope is
+# this product's packages and its dedicated cluster; never remove another DB.
+set -eu
+test "$(id -u)" = 0
+mc_release_dir=$1
+cd "$mc_release_dir"
+sha256sum --check SHA256SUMS
+mc_pg_guard=/etc/postgresql-common/createcluster.d/60-mission-core-install.conf
+mc_created_guard=0
+cleanup() {
+  if [ "$mc_created_guard" = 1 ]; then
+    if [ "$(cat "$mc_pg_guard")" = 'create_main_cluster = false' ]; then
+      rm "$mc_pg_guard"
+    fi
+  fi
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' HUP TERM
+if ! dpkg-query -W -f='${Status}' postgresql-16 2>/dev/null | grep -qx 'install ok installed'; then
+  # postgresql-common explicitly supports this drop-in. Suppress only automatic
+  # creation of the unrelated default main cluster during the first install.
+  test ! -e "$mc_pg_guard"
+  install -d -m 0755 /etc/postgresql-common/createcluster.d
+  printf '%s\n' 'create_main_cluster = false' > "$mc_pg_guard"
+  chmod 0644 "$mc_pg_guard"
+  mc_created_guard=1
+fi
+DEBIAN_FRONTEND=noninteractive apt-get install -y --no-remove \
+  "$mc_release_dir/timescaledb-2-loader-postgresql-16_2.29.2~ubuntu24.04-1615_amd64.deb" \
+  "$mc_release_dir/timescaledb-2-oss-postgresql-16_2.29.2~ubuntu24.04-1615_amd64.deb" \
+  "$mc_release_dir/mission-core-node_0.8.15_amd64.deb" \
+  "$mc_release_dir/mission-core-xgrids-k1_0.1.14_amd64.deb"
+systemctl is-active --quiet mission-core-node.service
+systemctl is-active --quiet mission-core-k1.service
+systemctl is-active --quiet postgresql@16-ndc-monitor.service
+systemctl is-active --quiet mission-core-node-monitor.service
+runuser -u mission-core-monitor -- /usr/bin/python3 - <<'PY'
+import http.client,json,socket,time
+deadline=time.monotonic()+20
+while True:
+    c=http.client.HTTPConnection('monitor',timeout=3)
+    c.sock=socket.socket(socket.AF_UNIX);c.sock.settimeout(3)
+    try:
+        c.sock.connect('/run/mission-core-monitor/monitor.sock')
+        c.request('GET','/status');r=c.getresponse();v=json.load(r)
+        if r.status==200 and v['storage']=='ready' and v['latest']['seq']>0:break
+    except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException):
+        pass
+    finally:c.close()
+    if time.monotonic()>deadline:raise SystemExit('Local telemetry did not become ready')
+    time.sleep(1)
+print('Local telemetry: ready')
+PY
+printf '%s\n' 'Mission Core Node R18: package and local archive checks passed.'

apps/node-agent/packaging/launcher.py

@@ -9,6 +9,7 @@ import re
 import socket
 import subprocess
 import threading
+import syslog
 from urllib.parse import urlsplit
 
 import gi
@@ -261,11 +262,15 @@ class NodeApplication(Gtk.Application):
         return False
 
     def load_failed(self, _view, _event, uri, _error):
+        syslog.openlog('mission-core-node-ui')
+        syslog.syslog(syslog.LOG_ERR, 'node-ui-load-failed')
         if local_url(uri):
             self.problem("Служба ноды недоступна. Повторно откройте приложение после восстановления службы.")
         return True
 
-    def process_failed(self, *_):
+    def process_failed(self, _view, reason):
+        syslog.openlog('mission-core-node-ui')
+        syslog.syslog(syslog.LOG_ERR, 'node-ui-process-terminated reason=' + str(int(reason)))
         self.problem("Окно приложения остановилось. Закройте и повторно откройте Mission Core Node. Служба борта продолжает работать отдельно.")
 
     def deny_permission(self, _view, permission):

apps/node-agent/packaging/mission-core-node-monitor.service

@@ -0,0 +1,38 @@
+[Unit]
+Description=Mission Core onboard system telemetry collector
+After=postgresql@16-ndc-monitor.service
+Wants=postgresql@16-ndc-monitor.service
+
+[Service]
+Type=simple
+User=mission-core-monitor
+Group=mission-core-node
+SupplementaryGroups=systemd-journal
+ExecStart=/usr/bin/python3 /usr/lib/mission-core-node/monitor/collector.py
+RuntimeDirectory=mission-core-monitor
+RuntimeDirectoryMode=0750
+StateDirectory=mission-core-monitor
+StateDirectoryMode=0750
+UMask=0007
+Restart=on-failure
+RestartSec=5
+MemoryHigh=96M
+MemoryMax=128M
+CPUQuota=15%
+TasksMax=24
+Nice=10
+IOSchedulingClass=idle
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictAddressFamilies=AF_UNIX
+CapabilityBoundingSet=
+RestrictSUIDSGID=yes
+
+[Install]
+WantedBy=multi-user.target

apps/node-agent/packaging/postinst

@@ -23,6 +23,7 @@ case "$1" in
         fi
         rm -f /run/mission-core-node-k1-upgrade-active
       fi
+      /usr/lib/mission-core-node/setup-monitor
     fi
     ;;
 esac

apps/node-agent/packaging/preinst

@@ -34,6 +34,7 @@ if [ "$1" = install ] || [ "$1" = upgrade ]; then
       (umask 077; : > /run/mission-core-node-k1-upgrade-active)
       systemctl stop mission-core-k1.service
     fi
+    systemctl stop mission-core-node-monitor.service 2>/dev/null || true
   fi
   . /etc/os-release
   if [ "${ID:-}" != ubuntu ] || [ "${VERSION_ID:-}" != 24.04 ]; then

apps/node-agent/packaging/prerm

@@ -52,6 +52,10 @@ case "$1" in
       systemctl disable mission-core-realsense.service || true
       systemctl stop mission-core-node.service
       systemctl disable mission-core-node.service
+      systemctl stop mission-core-node-monitor.service
+      systemctl disable mission-core-node-monitor.service
+      systemctl stop postgresql@16-ndc-monitor.service
+      systemctl disable postgresql@16-ndc-monitor.service
     fi
     ;;
 esac

apps/node-agent/packaging/setup-monitor

@@ -0,0 +1,54 @@
+#!/bin/sh
+# Fixed local database bootstrap; no supplied SQL, credentials or network address.
+set -eu
+test "$(id -u)" = 0
+if ! getent passwd mission-core-monitor >/dev/null; then
+ adduser --system --group --home /var/lib/mission-core-monitor --no-create-home --disabled-login mission-core-monitor
+fi
+install -d -m 0750 -o mission-core-monitor -g mission-core-node /var/lib/mission-core-monitor
+install -d -m 0755 -o postgres -g postgres /run/mission-core-monitor-db
+install -d -m 0755 /etc/tmpfiles.d
+printf '%s\n' 'd /run/mission-core-monitor-db 0755 postgres postgres -' > /etc/tmpfiles.d/mission-core-monitor.conf
+if [ ! -d /etc/postgresql/16/ndc-monitor ]; then
+ pg_createcluster 16 ndc-monitor --port=5433 --socketdir=/run/mission-core-monitor-db --datadir=/var/lib/mission-core-monitor-db --start-conf=auto -- --auth-local=peer --auth-host=reject
+fi
+install -d -m 0755 /etc/postgresql/16/ndc-monitor/conf.d
+cat > /etc/postgresql/16/ndc-monitor/conf.d/60-mission-core-monitor.conf <<'CONF'
+listen_addresses = ''
+unix_socket_directories = '/run/mission-core-monitor-db'
+shared_preload_libraries = 'timescaledb'
+shared_buffers = '32MB'
+work_mem = '2MB'
+maintenance_work_mem = '16MB'
+max_connections = 12
+max_worker_processes = 4
+max_parallel_workers = 0
+timescaledb.max_background_workers = 2
+timescaledb.telemetry_level = 'off'
+max_wal_size = '128MB'
+min_wal_size = '32MB'
+temp_file_limit = '32MB'
+statement_timeout = '5s'
+log_statement = 'none'
+log_min_error_statement = 'panic'
+CONF
+install -d -m 0755 /etc/systemd/system/postgresql@16-ndc-monitor.service.d
+cat > /etc/systemd/system/postgresql@16-ndc-monitor.service.d/60-mission-core-monitor.conf <<'CONF'
+[Service]
+ExecStartPre=+/usr/bin/install -d -m 0755 -o postgres -g postgres /run/mission-core-monitor-db
+MemoryHigh=256M
+MemoryMax=384M
+CPUQuota=25%
+TasksMax=32
+Nice=10
+CONF
+systemctl daemon-reload
+systemctl restart postgresql@16-ndc-monitor.service
+runuser -u postgres -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d postgres <<'SQL'
+SELECT 'CREATE ROLE "mission-core-monitor" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE' WHERE NOT EXISTS(SELECT FROM pg_roles WHERE rolname='mission-core-monitor') \gexec
+SELECT 'CREATE DATABASE mission_core_monitor OWNER "mission-core-monitor"' WHERE NOT EXISTS(SELECT FROM pg_database WHERE datname='mission_core_monitor') \gexec
+SQL
+runuser -u postgres -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d mission_core_monitor -c 'CREATE EXTENSION IF NOT EXISTS timescaledb'
+runuser -u mission-core-monitor -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d mission_core_monitor -f /usr/lib/mission-core-node/monitor/schema.sql
+systemctl enable mission-core-node-monitor.service
+systemctl restart mission-core-node-monitor.service