Fix onboard WebKit preview and archive board telemetry locally
This commit is contained in:
@@ -69,12 +69,14 @@ func run() error {
|
||||
return err
|
||||
}
|
||||
app := &node.Server{Store: store, Assets: assets, Origin: "http://" + *listen, Version: version, Inventory: func() node.Inventory { return node.Host("/") }}
|
||||
app.Monitor = node.NewMonitor()
|
||||
app.Presentation = node.NewPresentationStore(*dir)
|
||||
pairing, err := node.OpenPairing(store, *dir, version, app.Inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
app.Pairing = pairing
|
||||
pairing.Monitor = app.Monitor
|
||||
nodeID, _ := store.Public()
|
||||
app.Sensors, err = node.OpenSensors(*dir, nodeID)
|
||||
if err != nil {
|
||||
@@ -122,6 +124,7 @@ func run() error {
|
||||
go func() { errs <- private.Serve(unix) }()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
go app.Monitor.Run(ctx)
|
||||
go app.Sensors.WatchUSB(ctx)
|
||||
go pairing.Run(ctx)
|
||||
log.Print("Mission Core Node " + version + " listening on loopback")
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Local WebKit requires completed bounded HTTP responses. The same plugin
|
||||
// delivery owns RRD/fMP4 framing, backpressure, resume and acquisition isolation.
|
||||
func (s *Server) localPreviewRoutes(mux *http.ServeMux) {
|
||||
capacity := make(chan struct{}, 2)
|
||||
for _, route := range []string{"/api/devices/preview", "/api/devices/preview/read"} {
|
||||
mux.HandleFunc("POST "+route, func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(w, r) {
|
||||
return
|
||||
}
|
||||
var value json.RawMessage
|
||||
if !decode(w, r, &value) {
|
||||
return
|
||||
}
|
||||
if s.DeviceEnrollment == nil {
|
||||
http.Error(w, "Preview unavailable", 503)
|
||||
return
|
||||
}
|
||||
path := "/local-preview/read"
|
||||
if r.URL.Path == "/api/devices/preview" {
|
||||
var command SensorCommand
|
||||
if json.Unmarshal(value, &command) != nil || command.Action != "offer" || !sensorID.MatchString(command.Session.DeviceID) || !strings.HasPrefix(command.Session.DeviceID, "k1_") {
|
||||
http.Error(w, "Invalid preview", 400)
|
||||
return
|
||||
}
|
||||
path = "/local-preview"
|
||||
}
|
||||
select {
|
||||
case capacity <- struct{}{}:
|
||||
defer func() { <-capacity }()
|
||||
default:
|
||||
http.Error(w, "Close another viewer", 429)
|
||||
return
|
||||
}
|
||||
request, err := http.NewRequestWithContext(r.Context(), "POST", "http://k1"+path, bytes.NewReader(value))
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid preview", 400)
|
||||
return
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("X-Node-Id", s.DeviceEnrollment.nodeID)
|
||||
client := &http.Client{Transport: s.DeviceEnrollment.client.Transport, Timeout: 5 * time.Second}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
http.Error(w, "Preview unavailable", 503)
|
||||
return
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != 200 {
|
||||
http.Error(w, "Preview unavailable", response.StatusCode)
|
||||
return
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(response.Body, 196609))
|
||||
if err != nil || len(data) > 196608 {
|
||||
http.Error(w, "Preview unavailable", 503)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.missioncore.preview")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_, _ = w.Write(data)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The collector owns sampling/storage. This bounded cache keeps monitoring I/O
|
||||
// out of the paired control heartbeat and out of acquisition lifecycles.
|
||||
type Monitor struct {
|
||||
mu sync.Mutex
|
||||
client *http.Client
|
||||
source string
|
||||
after int64
|
||||
sent int64
|
||||
value map[string]any
|
||||
}
|
||||
|
||||
func NewMonitor() *Monitor {
|
||||
return &Monitor{client: &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/run/mission-core-monitor/monitor.sock")
|
||||
}}}}
|
||||
}
|
||||
func (m *Monitor) read(ctx context.Context, path string, body []byte) (map[string]any, error) {
|
||||
method := "GET"
|
||||
if body != nil {
|
||||
method = "POST"
|
||||
}
|
||||
r, e := http.NewRequestWithContext(ctx, method, "http://monitor"+path, bytes.NewReader(body))
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
response, e := m.client.Do(r)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != 200 {
|
||||
return nil, errors.New("monitor unavailable")
|
||||
}
|
||||
data, e := io.ReadAll(io.LimitReader(response.Body, 262145))
|
||||
if e != nil || len(data) > 262144 {
|
||||
return nil, errors.New("monitor response budget")
|
||||
}
|
||||
var value map[string]any
|
||||
e = json.Unmarshal(data, &value)
|
||||
return value, e
|
||||
}
|
||||
func (m *Monitor) Run(ctx context.Context) {
|
||||
defer m.client.CloseIdleConnections()
|
||||
timer := time.NewTicker(2 * time.Second)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
status, e := m.read(ctx, "/status", nil)
|
||||
if e == nil {
|
||||
source, _ := status["source_id"].(string)
|
||||
m.mu.Lock()
|
||||
if source != m.source {
|
||||
m.source = source
|
||||
m.after = 0
|
||||
m.sent = 0
|
||||
}
|
||||
after := m.after
|
||||
m.mu.Unlock()
|
||||
batch, be := m.read(ctx, "/batch?after="+strconv.FormatInt(after, 10), nil)
|
||||
if be == nil && batch["source_id"] == source {
|
||||
status["batch"] = batch
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.value = status
|
||||
m.mu.Unlock()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
func (m *Monitor) Snapshot() map[string]any {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.value == nil {
|
||||
return map[string]any{"storage": "unavailable"}
|
||||
}
|
||||
if batch, ok := m.value["batch"].(map[string]any); ok {
|
||||
if rows, ok := batch["samples"].([]any); ok {
|
||||
for _, raw := range rows {
|
||||
if row, ok := raw.(map[string]any); ok {
|
||||
if seq, ok := row["seq"].(float64); ok && int64(seq) > m.sent {
|
||||
m.sent = int64(seq)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return m.value
|
||||
}
|
||||
func (m *Monitor) Acknowledge(raw json.RawMessage) {
|
||||
var ack struct {
|
||||
Source string `json:"source_id"`
|
||||
After int64 `json:"after"`
|
||||
}
|
||||
if json.Unmarshal(raw, &ack) != nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
// A receiver may already have a later cursor from before the Node restarted.
|
||||
// Its authenticated ACK is bounded by the collector's durable latest record.
|
||||
latest := m.sent
|
||||
if row, ok := m.value["latest"].(map[string]any); ok {
|
||||
if seq, ok := row["seq"].(float64); ok {
|
||||
latest = int64(seq)
|
||||
}
|
||||
}
|
||||
if ack.Source == m.source && ack.After >= m.after && ack.After <= latest {
|
||||
m.after = ack.After
|
||||
}
|
||||
}
|
||||
func (s *Server) monitorRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/monitor", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(w, r) {
|
||||
return
|
||||
}
|
||||
if s.Monitor == nil {
|
||||
reply(w, 503, map[string]string{"error": "Мониторинг недоступен"})
|
||||
return
|
||||
}
|
||||
reply(w, 200, s.Monitor.Snapshot())
|
||||
})
|
||||
mux.HandleFunc("POST /api/monitor/events", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(w, r) {
|
||||
return
|
||||
}
|
||||
var value json.RawMessage
|
||||
if !decode(w, r, &value) {
|
||||
return
|
||||
}
|
||||
if len(value) > 2048 || s.Monitor == nil {
|
||||
http.Error(w, "Monitoring unavailable", 503)
|
||||
return
|
||||
}
|
||||
if _, e := s.Monitor.read(r.Context(), "/events", value); e != nil {
|
||||
http.Error(w, "Monitoring unavailable", 503)
|
||||
return
|
||||
}
|
||||
reply(w, 200, map[string]bool{"ok": true})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMonitorAcknowledgementsRequireSameArchiveAndDurableBounds(t *testing.T) {
|
||||
m := NewMonitor()
|
||||
m.source = "archive-one"
|
||||
m.value = map[string]any{"latest": map[string]any{"seq": float64(100)}}
|
||||
for _, raw := range []string{`{"source_id":"another","after":50}`, `{"source_id":"archive-one","after":101}`, `{"source_id":"archive-one","after":-1}`} {
|
||||
m.Acknowledge(json.RawMessage(raw))
|
||||
if m.after != 0 {
|
||||
t.Fatal("accepted invalid cursor")
|
||||
}
|
||||
}
|
||||
m.Acknowledge(json.RawMessage(`{"source_id":"archive-one","after":80}`))
|
||||
if m.after != 80 {
|
||||
t.Fatal("did not restore committed receiver cursor after restart")
|
||||
}
|
||||
m.Acknowledge(json.RawMessage(`{"source_id":"archive-one","after":10}`))
|
||||
if m.after != 80 {
|
||||
t.Fatal("regressed cursor")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorAndPreviewCannotBypassNodeSession(t *testing.T) {
|
||||
s := newTestServer(t)
|
||||
for _, path := range []string{"/api/devices/preview", "/api/devices/preview/read", "/api/monitor/events"} {
|
||||
if got := call(s, "POST", path, `{}`, nil).Code; got != 401 {
|
||||
t.Fatalf("%s: %d", path, got)
|
||||
}
|
||||
}
|
||||
if got := call(s, "GET", "/api/monitor", "", nil).Code; got != 401 {
|
||||
t.Fatal(got)
|
||||
}
|
||||
cookie := login(t, s)
|
||||
if got := call(s, "POST", "/api/devices/preview", `{}`, cookie).Code; got != 503 {
|
||||
t.Fatal(got)
|
||||
}
|
||||
s.Monitor = NewMonitor()
|
||||
if got := call(s, "GET", "/api/monitor", "", cookie).Code; got != 200 {
|
||||
t.Fatal(got)
|
||||
}
|
||||
// An unavailable collector remains independent from the authenticated UI.
|
||||
if got := call(s, "GET", "/api/status", "", cookie).Code; got != 200 {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ type PairState struct {
|
||||
Revocations []CoreBinding `json:"revocations,omitempty"`
|
||||
}
|
||||
type Pairing struct {
|
||||
Monitor *Monitor
|
||||
Sensors *Sensors
|
||||
DeviceEnrollment *DeviceEnrollment
|
||||
mu sync.Mutex
|
||||
|
||||
@@ -286,10 +286,16 @@ func (p *Pairing) channel(ctx context.Context) {
|
||||
payload["device_enrollment"] = p.DeviceEnrollment.Status()
|
||||
payload["enrollment_results"] = p.DeviceEnrollment.RemoteResults()
|
||||
}
|
||||
if p.Monitor != nil {
|
||||
payload["monitor"] = p.Monitor.Snapshot()
|
||||
}
|
||||
result, status, e := p.send(ctx, *binding, "/v1/node/heartbeat", payload)
|
||||
p.mu.Lock()
|
||||
if p.state.Phase == "paired" && p.state.Binding != nil && p.state.Binding.BindingID == binding.BindingID {
|
||||
if e == nil && status == 200 {
|
||||
if p.Monitor != nil {
|
||||
p.Monitor.Acknowledge(result["monitor_ack"])
|
||||
}
|
||||
p.connection = "online"
|
||||
p.lastSeen = p.now().Unix()
|
||||
if p.DeviceEnrollment != nil {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
Monitor *Monitor
|
||||
Store *Store
|
||||
Pairing *Pairing
|
||||
Sensors *Sensors
|
||||
@@ -81,6 +82,8 @@ func reply(w http.ResponseWriter, status int, v any) {
|
||||
|
||||
func (s *Server) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
s.localPreviewRoutes(mux)
|
||||
s.monitorRoutes(mux)
|
||||
if s.Presentation != nil {
|
||||
s.presentationRoutes(mux)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/python3
|
||||
"""Independent one-second collector, local archive and private Node read adapter."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from linux_metrics import LinuxMetrics
|
||||
from storage import SCHEMA, Archive
|
||||
|
||||
SOCKET = Path("/run/mission-core-monitor/monitor.sock")
|
||||
|
||||
|
||||
class Collector:
|
||||
def __init__(self):
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.latest = None
|
||||
self.definitions = {}
|
||||
self.events = deque(maxlen=64)
|
||||
self.storage = "starting"
|
||||
self.database_bytes = None
|
||||
self.archive = None
|
||||
self.boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
||||
|
||||
def event(self, value):
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) - {"code", "kind", "locations"}
|
||||
or value.get("code") not in {"ui-error", "ui-rejection", "ui-render-error"}
|
||||
or not isinstance(value.get("kind"), str)
|
||||
or not re.fullmatch("[A-Za-z]{1,48}", value["kind"])
|
||||
or not isinstance(value.get("locations", []), list)
|
||||
or len(value.get("locations", [])) > 8
|
||||
or any(
|
||||
not isinstance(v, str)
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]{1,96}\.js:[0-9]{1,9}:[0-9]{1,9}", v)
|
||||
for v in value.get("locations", [])
|
||||
)
|
||||
):
|
||||
raise ValueError("invalid-event")
|
||||
with self.lock:
|
||||
if len(self.events) < 64:
|
||||
self.events.append({**value, "at": time.time()})
|
||||
|
||||
def run(self):
|
||||
metrics = LinuxMetrics()
|
||||
maintenance = 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
values = metrics.sample()
|
||||
with self.lock:
|
||||
events = list(self.events)
|
||||
sample = dict(
|
||||
at=time.time(), boot_id=self.boot, uptime=started, values=values, events=events
|
||||
)
|
||||
if self.archive is None:
|
||||
self.archive = Archive()
|
||||
# Maintenance also runs while writes are paused by a disk budget.
|
||||
if started - maintenance > 60:
|
||||
self.database_bytes = self.archive.maintain()
|
||||
maintenance = started
|
||||
record = self.archive.append(sample, metrics.definitions)
|
||||
with self.lock:
|
||||
for _ in events:
|
||||
self.events.popleft()
|
||||
self.latest = record
|
||||
self.definitions = metrics.definitions
|
||||
self.storage = "ready"
|
||||
except Exception as error:
|
||||
logging.warning("monitor sample failed class=%s", type(error).__name__)
|
||||
with self.lock:
|
||||
self.storage = (
|
||||
str(error)
|
||||
if str(error) in {"disk-reserve", "archive-budget"}
|
||||
else "unavailable"
|
||||
)
|
||||
self.stop.wait(max(0.05, 1 - (time.monotonic() - started)))
|
||||
|
||||
def status(self):
|
||||
with self.lock:
|
||||
return dict(
|
||||
schema=SCHEMA,
|
||||
source_id=self.archive.source if self.archive else None,
|
||||
latest=self.latest,
|
||||
definitions=self.definitions,
|
||||
storage=self.storage,
|
||||
database_bytes=self.database_bytes,
|
||||
sample_interval_seconds=1,
|
||||
retention_days=7,
|
||||
database_budget_bytes=2 * 1024**3,
|
||||
free_reserve_bytes=2 * 1024**3,
|
||||
)
|
||||
|
||||
|
||||
class Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer):
|
||||
daemon_threads = True
|
||||
block_on_close = False
|
||||
|
||||
def __init__(self, collector):
|
||||
self.collector = collector
|
||||
self.capacity = threading.BoundedSemaphore(6)
|
||||
super().__init__(str(SOCKET), Handler)
|
||||
|
||||
def process_request(self, request, address):
|
||||
if not self.capacity.acquire(False):
|
||||
request.close()
|
||||
return
|
||||
request.settimeout(5)
|
||||
super().process_request(request, address)
|
||||
|
||||
def process_request_thread(self, request, address):
|
||||
try:
|
||||
super().process_request_thread(request, address)
|
||||
finally:
|
||||
self.capacity.release()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def reply(self, value, status=200):
|
||||
payload = json.dumps(value, allow_nan=False, separators=(",", ":")).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self):
|
||||
path = urlsplit(self.path)
|
||||
try:
|
||||
if path.path == "/status":
|
||||
self.reply(self.server.collector.status())
|
||||
elif path.path == "/batch":
|
||||
after = int(parse_qs(path.query).get("after", ["0"])[0])
|
||||
if not 0 <= after <= 2**53 - 1:
|
||||
raise ValueError()
|
||||
archive = self.server.collector.archive
|
||||
if archive is None:
|
||||
self.reply({"error": "Archive unavailable"}, 503)
|
||||
else:
|
||||
self.reply(archive.batch(after))
|
||||
else:
|
||||
self.reply({"error": "Unknown request"}, 404)
|
||||
except Exception:
|
||||
self.reply({"error": "Archive unavailable"}, 503)
|
||||
|
||||
def do_POST(self):
|
||||
try:
|
||||
size = int(self.headers.get("Content-Length", "0"))
|
||||
if self.path != "/events" or not 0 < size <= 2048:
|
||||
raise ValueError()
|
||||
self.server.collector.event(json.loads(self.rfile.read(size)))
|
||||
self.reply({"ok": True})
|
||||
except (ValueError, TypeError):
|
||||
self.reply({"error": "Invalid event"}, 400)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
os.umask(0o007)
|
||||
collector = Collector()
|
||||
from journal_events import watch
|
||||
|
||||
threading.Thread(
|
||||
target=watch, args=(collector,), name="node-system-events", daemon=True
|
||||
).start()
|
||||
SOCKET.unlink(missing_ok=True)
|
||||
server = Server(collector)
|
||||
worker = threading.Thread(target=collector.run, name="node-system-sampler", daemon=True)
|
||||
worker.start()
|
||||
|
||||
def stop(*_):
|
||||
collector.stop.set()
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.5)
|
||||
finally:
|
||||
collector.stop.set()
|
||||
server.server_close()
|
||||
worker.join(5)
|
||||
SOCKET.unlink(missing_ok=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Classify bounded journal records; never persist free-form messages or payloads."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
RULES = (
|
||||
("system-oom", r"out of memory|oom-kill|killed process"),
|
||||
("gpu-reset", r"GPU HANG|GPU reset|Resetting.*(gpu|chip)|i915.*(hang|reset|error)"),
|
||||
("disk-error", r"I/O error|EXT4-fs error|nvme.*(timeout|reset)|ata.*failed command"),
|
||||
("usb-disconnected", r"USB disconnect"),
|
||||
("usb-error", r"usb.*(error -|unable to enumerate|reset.*device)"),
|
||||
("service-failed", r"Failed with result|Main process exited|Failed to start"),
|
||||
("ui-process-exit", r"node-ui-process-terminated"),
|
||||
("ui-load-failed", r"node-ui-load-failed"),
|
||||
)
|
||||
|
||||
|
||||
def classify(record):
|
||||
message = record.get("MESSAGE")
|
||||
if not isinstance(message, str):
|
||||
return None
|
||||
code = next((code for code, pattern in RULES if re.search(pattern, message, re.I)), None)
|
||||
if code is None:
|
||||
return None
|
||||
try:
|
||||
at = int(record["__REALTIME_TIMESTAMP"]) / 1_000_000
|
||||
except (KeyError, ValueError, TypeError):
|
||||
return None
|
||||
return dict(
|
||||
code=code,
|
||||
kind="Kernel" if record.get("_TRANSPORT") == "kernel" else "System",
|
||||
at=at,
|
||||
locations=[],
|
||||
)
|
||||
|
||||
|
||||
def watch(collector):
|
||||
args = [
|
||||
"/usr/bin/journalctl",
|
||||
"--follow",
|
||||
"--lines=0",
|
||||
"--output=json",
|
||||
"--no-pager",
|
||||
"_TRANSPORT=kernel",
|
||||
"+",
|
||||
"SYSLOG_IDENTIFIER=mission-core-node-ui",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=mission-core-node.service",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=mission-core-k1.service",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=mission-core-node-monitor.service",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=postgresql@16-ndc-monitor.service",
|
||||
]
|
||||
for unit in (
|
||||
"mission-core-node.service",
|
||||
"mission-core-k1.service",
|
||||
"mission-core-node-monitor.service",
|
||||
"postgresql@16-ndc-monitor.service",
|
||||
):
|
||||
args.extend(["+", "UNIT=" + unit])
|
||||
while not collector.stop.is_set():
|
||||
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
def retire(process=process):
|
||||
while process.poll() is None:
|
||||
if collector.stop.wait(1):
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
return
|
||||
|
||||
guard = threading.Thread(target=retire, daemon=True)
|
||||
guard.start()
|
||||
try:
|
||||
while not collector.stop.is_set():
|
||||
line = process.stdout.readline(65537)
|
||||
if not line:
|
||||
break
|
||||
if len(line) > 65536:
|
||||
while line and not line.endswith(b"\n"):
|
||||
line = process.stdout.readline(65537)
|
||||
continue
|
||||
try:
|
||||
event = classify(json.loads(line))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if event:
|
||||
with collector.lock:
|
||||
if len(collector.events) < 64:
|
||||
collector.events.append(event)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
process.stdout.close()
|
||||
collector.stop.wait(10)
|
||||
@@ -0,0 +1,353 @@
|
||||
"""Read-only Linux counters. Missing or reset counters never manufacture zero."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class LinuxMetrics:
|
||||
def __init__(self, root=Path("/"), clock=time.monotonic):
|
||||
self.root, self.clock = Path(root), clock
|
||||
self.previous = {}
|
||||
self.previous_time = None
|
||||
self.definitions = {}
|
||||
|
||||
def text(self, path):
|
||||
try:
|
||||
return (self.root / str(path).lstrip("/")).read_text()[:262144].strip()
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
|
||||
def number(self, path):
|
||||
try:
|
||||
return float(self.text(path))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def paths(self, pattern):
|
||||
return sorted(self.root.glob(pattern))[:128]
|
||||
|
||||
def sample(self):
|
||||
now = self.clock()
|
||||
elapsed = None if self.previous_time is None else now - self.previous_time
|
||||
self.previous_time = now
|
||||
values, definitions, counters = {}, {}, {}
|
||||
|
||||
def metric(
|
||||
key, label, group, resource, unit, value, reason="Счётчик недоступен в этой системе"
|
||||
):
|
||||
values[key] = None if value is None else round(value, 3)
|
||||
definitions[key] = dict(
|
||||
label=label,
|
||||
group=group,
|
||||
resource=resource,
|
||||
unit=unit,
|
||||
reason=reason if value is None else None,
|
||||
)
|
||||
|
||||
def delta(key, value):
|
||||
if value is None:
|
||||
return None
|
||||
counters[key] = value
|
||||
previous = self.previous.get(key)
|
||||
return (
|
||||
None
|
||||
if previous is None or value < previous or not elapsed or elapsed > 10
|
||||
else (value - previous) / elapsed
|
||||
)
|
||||
|
||||
stat = self.text("proc/stat") or ""
|
||||
cpus = 0
|
||||
for line in stat.splitlines():
|
||||
parts = line.split()
|
||||
if not re.fullmatch(r"cpu\d*", parts[0]):
|
||||
continue
|
||||
if parts[0] != "cpu":
|
||||
cpus += 1
|
||||
data = [int(v) for v in parts[1:9]]
|
||||
total = delta(parts[0] + ".total", sum(data))
|
||||
idle = delta(parts[0] + ".idle", data[3] + data[4])
|
||||
wait = delta(parts[0] + ".iowait", data[4])
|
||||
for name, label, value in [
|
||||
(
|
||||
"usage",
|
||||
"Загрузка",
|
||||
None if not total or idle is None else 100 * (1 - idle / total),
|
||||
),
|
||||
(
|
||||
"iowait",
|
||||
"Ожидание диска",
|
||||
None if not total or wait is None else 100 * wait / total,
|
||||
),
|
||||
]:
|
||||
metric(
|
||||
parts[0] + "." + name,
|
||||
label,
|
||||
"CPU",
|
||||
parts[0],
|
||||
"%",
|
||||
value,
|
||||
"Ожидаем два последовательных измерения",
|
||||
)
|
||||
if not cpus:
|
||||
metric("cpu.usage", "Загрузка", "CPU", "CPU", "%", None)
|
||||
memory = {}
|
||||
for line in (self.text("proc/meminfo") or "").splitlines():
|
||||
parts = line.replace(":", "").split()
|
||||
if len(parts) > 1:
|
||||
memory[parts[0]] = int(parts[1]) * 1024
|
||||
for key, label in [
|
||||
("MemTotal", "Всего"),
|
||||
("MemAvailable", "Доступно"),
|
||||
("SwapTotal", "Swap всего"),
|
||||
("SwapFree", "Swap свободно"),
|
||||
("Dirty", "Ожидает записи"),
|
||||
("Writeback", "Записывается"),
|
||||
]:
|
||||
metric("memory." + key, label, "RAM", "Память", "bytes", memory.get(key))
|
||||
total, available = memory.get("MemTotal"), memory.get("MemAvailable")
|
||||
metric(
|
||||
"memory.usage",
|
||||
"Использовано",
|
||||
"RAM",
|
||||
"Память",
|
||||
"%",
|
||||
None if not total or available is None else 100 * (1 - available / total),
|
||||
)
|
||||
for index, load in enumerate((self.text("proc/loadavg") or "").split()[:3]):
|
||||
metric(
|
||||
"load." + str(index),
|
||||
["За 1 минуту", "За 5 минут", "За 15 минут"][index],
|
||||
"CPU",
|
||||
"Load average",
|
||||
"",
|
||||
float(load),
|
||||
)
|
||||
for resource in ["cpu", "memory", "io"]:
|
||||
for line in (self.text("proc/pressure/" + resource) or "").splitlines():
|
||||
parts = line.split()
|
||||
fields = dict(v.split("=") for v in parts[1:])
|
||||
metric(
|
||||
"pressure." + resource + "." + parts[0],
|
||||
"Задержка за 10 секунд",
|
||||
"Ожидание ресурсов",
|
||||
resource + " · " + parts[0],
|
||||
"%",
|
||||
float(fields["avg10"]),
|
||||
)
|
||||
for interface in self.paths("sys/class/net/*"):
|
||||
name = interface.name
|
||||
for field, label in [
|
||||
("rx_bytes", "Приём"),
|
||||
("tx_bytes", "Передача"),
|
||||
("rx_errors", "Ошибки приёма"),
|
||||
("tx_errors", "Ошибки передачи"),
|
||||
("rx_dropped", "Потери приёма"),
|
||||
("tx_dropped", "Потери передачи"),
|
||||
]:
|
||||
raw = self.number(interface.relative_to(self.root) / "statistics" / field)
|
||||
metric(
|
||||
"net." + name + "." + field,
|
||||
label,
|
||||
"Сеть",
|
||||
name,
|
||||
"bytes/s" if field.endswith("bytes") else "1/s",
|
||||
delta("net." + name + "." + field, raw),
|
||||
)
|
||||
speed = self.number(interface.relative_to(self.root) / "speed")
|
||||
metric(
|
||||
"net." + name + ".speed",
|
||||
"Скорость соединения",
|
||||
"Сеть",
|
||||
name,
|
||||
"Mbit/s",
|
||||
speed if speed is not None and speed > 0 else None,
|
||||
)
|
||||
diskstats = (self.text("proc/diskstats") or "").splitlines()
|
||||
for line in diskstats:
|
||||
p = line.split()
|
||||
if (
|
||||
len(p) < 14
|
||||
or p[2].startswith(("loop", "ram"))
|
||||
or not (self.root / "sys/block" / p[2]).exists()
|
||||
):
|
||||
continue
|
||||
name = p[2]
|
||||
for index, label, unit, multiplier in [
|
||||
(5, "Чтение", "bytes/s", 512),
|
||||
(9, "Запись", "bytes/s", 512),
|
||||
(3, "Операции чтения", "1/s", 1),
|
||||
(7, "Операции записи", "1/s", 1),
|
||||
(12, "Занятость", "%", 0.1),
|
||||
]:
|
||||
key = "disk." + name + "." + str(index)
|
||||
rate = delta(key, int(p[index]))
|
||||
metric(key, label, "Диски", name, unit, None if rate is None else rate * multiplier)
|
||||
if self.root == Path("/"):
|
||||
mounts = set()
|
||||
for line in (self.text("proc/mounts") or "").splitlines():
|
||||
p = line.split()
|
||||
if len(p) < 3 or not p[0].startswith("/dev/") or p[1] in mounts:
|
||||
continue
|
||||
mount = p[1].replace("\\040", " ")
|
||||
mounts.add(mount)
|
||||
try:
|
||||
st = os.statvfs(mount)
|
||||
key = "fs." + str(len(mounts))
|
||||
metric(
|
||||
key + ".free",
|
||||
"Свободно",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"bytes",
|
||||
st.f_bavail * st.f_frsize,
|
||||
)
|
||||
metric(
|
||||
key + ".usage",
|
||||
"Использовано",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"%",
|
||||
100 * (1 - st.f_bfree / st.f_blocks) if st.f_blocks else None,
|
||||
)
|
||||
metric(
|
||||
key + ".inodes",
|
||||
"Использовано inode",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"%",
|
||||
100 * (1 - st.f_ffree / st.f_files) if st.f_files else None,
|
||||
)
|
||||
except OSError:
|
||||
metric(
|
||||
"fs." + str(len(mounts)) + ".free",
|
||||
"Свободно",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"bytes",
|
||||
None,
|
||||
)
|
||||
for path in self.paths("sys/class/hwmon/hwmon*/temp*_input"):
|
||||
relative = path.relative_to(self.root)
|
||||
name = self.text(path.parent.relative_to(self.root) / "name") or path.parent.name
|
||||
raw = self.number(relative)
|
||||
metric(
|
||||
"temperature." + path.parent.name + "." + path.stem,
|
||||
"Температура",
|
||||
"Температура",
|
||||
name + " · " + path.stem,
|
||||
"°C",
|
||||
None if raw is None else raw / 1000,
|
||||
)
|
||||
for card in self.paths("sys/class/drm/card[0-9]"):
|
||||
base = card.relative_to(self.root)
|
||||
metric(
|
||||
"gpu." + card.name + ".usage",
|
||||
"Загрузка",
|
||||
"GPU",
|
||||
card.name,
|
||||
"%",
|
||||
self.number(base / "device/gpu_busy_percent"),
|
||||
"Драйвер не публикует общий счётчик загрузки GPU",
|
||||
)
|
||||
metric(
|
||||
"gpu." + card.name + ".frequency",
|
||||
"Частота",
|
||||
"GPU",
|
||||
card.name,
|
||||
"MHz",
|
||||
self.number(base / "gt_cur_freq_mhz"),
|
||||
)
|
||||
for field, label in [
|
||||
("mem_info_vram_used", "Видеопамять занята"),
|
||||
("mem_info_vram_total", "Видеопамять всего"),
|
||||
]:
|
||||
metric(
|
||||
"gpu." + card.name + "." + field,
|
||||
label,
|
||||
"GPU",
|
||||
card.name,
|
||||
"bytes",
|
||||
self.number(base / "device" / field),
|
||||
"Выделенная видеопамять или её счётчик недоступны",
|
||||
)
|
||||
for usb in self.paths("sys/bus/usb/devices/*"):
|
||||
if not (usb / "idVendor").exists():
|
||||
continue
|
||||
base = usb.relative_to(self.root)
|
||||
name = usb.name
|
||||
metric(
|
||||
"usb." + name + ".speed",
|
||||
"Скорость соединения",
|
||||
"USB",
|
||||
name,
|
||||
"Mbit/s",
|
||||
self.number(base / "speed"),
|
||||
)
|
||||
metric(
|
||||
"usb." + name + ".traffic",
|
||||
"Трафик",
|
||||
"USB",
|
||||
name,
|
||||
"bytes/s",
|
||||
None,
|
||||
"USB-драйвер не публикует счётчик трафика порта",
|
||||
)
|
||||
groups = self.paths("sys/fs/cgroup/system.slice/mission-core*.service")
|
||||
groups += self.paths(
|
||||
"sys/fs/cgroup/system.slice/system-postgresql.slice/postgresql@16-ndc-monitor.service"
|
||||
)
|
||||
groups += self.paths(
|
||||
"sys/fs/cgroup/user.slice/user-*.slice/user@*.service/app.slice/app-gnome-org.nodedc.MissionCoreNode-*.scope"
|
||||
)
|
||||
for path in groups:
|
||||
base = path.relative_to(self.root)
|
||||
name = path.name
|
||||
key = "service." + re.sub("[^A-Za-z0-9_.-]", "_", name)
|
||||
metric(
|
||||
key + ".memory",
|
||||
"Память",
|
||||
"Приложения",
|
||||
name,
|
||||
"bytes",
|
||||
self.number(base / "memory.current"),
|
||||
)
|
||||
metric(
|
||||
key + ".peak",
|
||||
"Пик памяти",
|
||||
"Приложения",
|
||||
name,
|
||||
"bytes",
|
||||
self.number(base / "memory.peak"),
|
||||
)
|
||||
fields = dict(
|
||||
line.split() for line in (self.text(base / "cpu.stat") or "").splitlines()
|
||||
)
|
||||
rate = delta(
|
||||
key + ".cpu", float(fields["usage_usec"]) if "usage_usec" in fields else None
|
||||
)
|
||||
metric(
|
||||
key + ".cpu",
|
||||
"Загрузка CPU",
|
||||
"Приложения",
|
||||
name,
|
||||
"%",
|
||||
None if rate is None else rate / 10000 / max(cpus, 1),
|
||||
)
|
||||
fields = dict(
|
||||
line.split() for line in (self.text(base / "memory.events") or "").splitlines()
|
||||
)
|
||||
metric(
|
||||
key + ".oom",
|
||||
"Убийств из-за нехватки памяти",
|
||||
"Приложения",
|
||||
name,
|
||||
"",
|
||||
float(fields["oom_kill"]) if "oom_kill" in fields else None,
|
||||
)
|
||||
self.previous = counters
|
||||
# A bounded hardware inventory protects the control-plane payload budget.
|
||||
keys = list(values)[:384]
|
||||
self.definitions = {key: definitions[key] for key in keys}
|
||||
return {key: values[key] for key in keys}
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
||||
CREATE TABLE IF NOT EXISTS monitor_identity (singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton), source_id text NOT NULL);
|
||||
INSERT INTO monitor_identity(singleton,source_id) VALUES(true,gen_random_uuid()::text) ON CONFLICT DO NOTHING;
|
||||
CREATE TABLE IF NOT EXISTS monitor_samples (
|
||||
seq bigint GENERATED ALWAYS AS IDENTITY,
|
||||
observed_at timestamptz NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
PRIMARY KEY(observed_at,seq)
|
||||
);
|
||||
SELECT create_hypertable('monitor_samples',by_range('observed_at',INTERVAL '1 hour'),if_not_exists=>true);
|
||||
CREATE INDEX IF NOT EXISTS monitor_samples_cursor ON monitor_samples(seq);
|
||||
CREATE TABLE IF NOT EXISTS monitor_definitions(singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton),payload jsonb NOT NULL);
|
||||
REVOKE ALL ON DATABASE mission_core_monitor FROM PUBLIC;
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Bounded local Timescale archive; transport acknowledgements never delete data."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
|
||||
SCHEMA = "missioncore.node-system-monitor/v1"
|
||||
MAX_BATCH_BYTES = 196608
|
||||
DATABASE_BUDGET = 2 * 1024**3
|
||||
|
||||
|
||||
class Archive:
|
||||
def __init__(self):
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
|
||||
self.pool = ThreadedConnectionPool(
|
||||
1,
|
||||
4,
|
||||
host="/run/mission-core-monitor-db",
|
||||
port=5433,
|
||||
dbname="mission_core_monitor",
|
||||
user="mission-core-monitor",
|
||||
connect_timeout=3,
|
||||
options="-c statement_timeout=3000",
|
||||
)
|
||||
try:
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
cursor.execute("SELECT source_id FROM monitor_identity")
|
||||
self.source = cursor.fetchone()[0]
|
||||
except Exception:
|
||||
self.pool.closeall()
|
||||
raise
|
||||
self.definitions = None
|
||||
self.database_bytes = None
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
db = self.pool.getconn()
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except BaseException:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
self.pool.putconn(db)
|
||||
|
||||
def append(self, sample, definitions):
|
||||
# This reserve belongs to telemetry, never to acquisition storage.
|
||||
if shutil.disk_usage("/var/lib/mission-core-monitor").free < 2 * 1024**3:
|
||||
raise RuntimeError("disk-reserve")
|
||||
if self.database_bytes is None or self.database_bytes >= DATABASE_BUDGET:
|
||||
raise RuntimeError("archive-budget")
|
||||
payload = json.dumps(sample, separators=(",", ":"), allow_nan=False)
|
||||
if len(payload.encode()) > 32768:
|
||||
raise ValueError("sample-budget")
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
if definitions != self.definitions:
|
||||
cursor.execute(
|
||||
"INSERT INTO monitor_definitions VALUES(true,%s) ON CONFLICT(singleton) "
|
||||
"DO UPDATE SET payload=excluded.payload",
|
||||
(json.dumps(definitions),),
|
||||
)
|
||||
cursor.execute(
|
||||
"INSERT INTO monitor_samples(observed_at,payload) VALUES(%s,%s) RETURNING seq",
|
||||
(datetime.fromtimestamp(sample["at"], UTC), payload),
|
||||
)
|
||||
seq = cursor.fetchone()[0]
|
||||
self.definitions = definitions
|
||||
return {**sample, "seq": seq}
|
||||
|
||||
def batch(self, after):
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT seq,payload FROM monitor_samples WHERE seq>%s ORDER BY seq LIMIT 32",
|
||||
(after,),
|
||||
)
|
||||
rows = []
|
||||
size = 0
|
||||
for seq, payload in cursor.fetchall():
|
||||
record = {**payload, "seq": seq}
|
||||
size += len(json.dumps(record).encode())
|
||||
if size > MAX_BATCH_BYTES:
|
||||
break
|
||||
rows.append(record)
|
||||
cursor.execute("SELECT min(seq),max(seq),min(observed_at) FROM monitor_samples")
|
||||
first, last, oldest = cursor.fetchone()
|
||||
return dict(
|
||||
schema=SCHEMA,
|
||||
source_id=self.source,
|
||||
samples=rows,
|
||||
first_seq=first,
|
||||
last_seq=last,
|
||||
retained_since=oldest.timestamp() if oldest else None,
|
||||
)
|
||||
|
||||
def maintain(self):
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
cursor.execute("SELECT drop_chunks('monitor_samples',INTERVAL '7 days')")
|
||||
cursor.execute("SELECT pg_database_size(current_database())")
|
||||
size = cursor.fetchone()[0]
|
||||
if size >= DATABASE_BUDGET:
|
||||
# Drop one oldest completed chunk per pass, retaining the live hour.
|
||||
cursor.execute(
|
||||
"SELECT min(range_end) FROM timescaledb_information.chunks "
|
||||
"WHERE hypertable_name='monitor_samples' AND range_end<NOW()"
|
||||
)
|
||||
boundary = cursor.fetchone()[0]
|
||||
if boundary:
|
||||
cursor.execute(
|
||||
"SELECT drop_chunks('monitor_samples',older_than=>%s::timestamptz)",
|
||||
(boundary,),
|
||||
)
|
||||
cursor.execute("SELECT pg_database_size(current_database())")
|
||||
size = cursor.fetchone()[0]
|
||||
self.database_bytes = size
|
||||
return size
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
VERSION = "0.8.14"
|
||||
VERSION = "0.8.15"
|
||||
sys.path.insert(0, str(ROOT.parents[1] / "scripts/packaging"))
|
||||
from debian import package
|
||||
|
||||
@@ -43,7 +43,7 @@ Architecture: amd64
|
||||
Maintainer: NODE.DC local build <noreply@example.invalid>
|
||||
Section: admin
|
||||
Priority: optional
|
||||
Depends: adduser, systemd, python3, python3-gi, gir1.2-gtk-3.0, gir1.2-webkit2-4.1, pkexec, polkitd, ca-certificates, hicolor-icon-theme, network-manager, iproute2
|
||||
Depends: adduser, systemd, python3, python3-gi, gir1.2-gtk-3.0, gir1.2-webkit2-4.1, pkexec, polkitd, ca-certificates, hicolor-icon-theme, network-manager, iproute2, python3-psycopg2, postgresql-16 (>= 16.15), timescaledb-2-oss-postgresql-16 (= 2.29.2~ubuntu24.04-1615)
|
||||
Description: Mission Core onboard computer configuration
|
||||
Local graphical setup, host inventory, SSH access and persistent node identity.
|
||||
""".encode()
|
||||
@@ -67,8 +67,13 @@ Description: Mission Core onboard computer configuration
|
||||
("environment_helper.py", "usr/lib/mission-core-node/environment_helper.py", 0o644),
|
||||
("mission-core-node-environment.service", "usr/lib/systemd/system/mission-core-node-environment.service", 0o644),
|
||||
("60-environment.conf", "usr/share/mission-core-node/60-environment.conf", 0o644),
|
||||
("setup-monitor", "usr/lib/mission-core-node/setup-monitor", 0o755),
|
||||
("mission-core-node-monitor.service", "usr/lib/systemd/system/mission-core-node-monitor.service", 0o644),
|
||||
]:
|
||||
files.append((path, (p / source).read_bytes(), mode))
|
||||
for path in sorted((ROOT / 'monitor').iterdir()):
|
||||
if path.suffix in {'.py', '.sql'}:
|
||||
files.append(('usr/lib/mission-core-node/monitor/' + path.name, path.read_bytes(), 0o644))
|
||||
files.append(("usr/share/mission-core-node/environment-profile.json", (ROOT / "internal/node/environment-profile.json").read_bytes(), 0o644))
|
||||
for name in ("realsense_prepare.py", "realsense_iio_access.py"):
|
||||
files.append(("usr/lib/mission-core-node/" + name, (p / name).read_bytes(), 0o644))
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stage verified Timescale OSS Ubuntu packages, without APT/system mutation."""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
BASE = "https://packagecloud.io/timescale/timescaledb/ubuntu/"
|
||||
VERSION = "2.29.2"
|
||||
|
||||
|
||||
def fetch(url, limit):
|
||||
with urllib.request.urlopen(url, timeout=30) as response:
|
||||
data = response.read(limit + 1)
|
||||
if len(data) > limit:
|
||||
raise ValueError("Download budget exceeded")
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("destination", type=Path)
|
||||
args = parser.parse_args()
|
||||
root = args.destination
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
key = root / "timescale.asc"
|
||||
key.write_bytes(fetch("https://packagecloud.io/timescale/timescaledb/gpgkey", 65536))
|
||||
keyring = root / "timescale.gpg"
|
||||
subprocess.run(
|
||||
["gpg", "--batch", "--yes", "--dearmor", "--output", str(keyring), str(key)], check=True
|
||||
)
|
||||
release = root / "InRelease"
|
||||
release.write_bytes(fetch(BASE + "dists/noble/InRelease", 1048576))
|
||||
subprocess.run(["gpgv", "--keyring", str(keyring.resolve()), str(release)], check=True)
|
||||
lines = release.read_text().splitlines()
|
||||
hashes = {}
|
||||
inside = False
|
||||
for line in lines:
|
||||
if line == "SHA256:":
|
||||
inside = True
|
||||
continue
|
||||
if inside and not line.startswith(" "):
|
||||
break
|
||||
if inside:
|
||||
sha, size, path = line.split()
|
||||
hashes[path] = (sha, int(size))
|
||||
sha, size = hashes["main/binary-amd64/Packages.gz"]
|
||||
compressed = fetch(BASE + "dists/noble/main/binary-amd64/Packages.gz", 16000000)
|
||||
if len(compressed) != size or hashlib.sha256(compressed).hexdigest() != sha:
|
||||
raise ValueError("Signed package index mismatch")
|
||||
(root / "Packages.gz").write_bytes(compressed)
|
||||
wanted = {
|
||||
name: VERSION + "~ubuntu24.04-1615"
|
||||
for name in ("timescaledb-2-oss-postgresql-16", "timescaledb-2-loader-postgresql-16")
|
||||
}
|
||||
artifacts = []
|
||||
for block in gzip.decompress(compressed).decode().split("\n\n"):
|
||||
fields = dict(
|
||||
line.split(": ", 1)
|
||||
for line in block.splitlines()
|
||||
if ": " in line and not line.startswith(" ")
|
||||
)
|
||||
if (
|
||||
fields.get("Package") not in wanted
|
||||
or fields.get("Version") != wanted[fields["Package"]]
|
||||
):
|
||||
continue
|
||||
path = fields["Filename"]
|
||||
if not path.startswith("pool/") or ".." in path.split("/"):
|
||||
raise ValueError("Invalid package path")
|
||||
data = fetch(BASE + path, 64000000)
|
||||
if hashlib.sha256(data).hexdigest() != fields["SHA256"]:
|
||||
raise ValueError("Package hash mismatch")
|
||||
target = root / Path(path).name
|
||||
target.write_bytes(data)
|
||||
artifacts.append(
|
||||
dict(
|
||||
package=fields["Package"],
|
||||
version=fields["Version"],
|
||||
file=target.name,
|
||||
sha256=fields["SHA256"],
|
||||
bytes=len(data),
|
||||
depends=fields["Depends"],
|
||||
)
|
||||
)
|
||||
if len(artifacts) != len(wanted):
|
||||
raise ValueError("Pinned package not found")
|
||||
manifest = dict(
|
||||
source=BASE,
|
||||
version=VERSION,
|
||||
key_sha256=hashlib.sha256(key.read_bytes()).hexdigest(),
|
||||
inrelease_sha256=hashlib.sha256(release.read_bytes()).hexdigest(),
|
||||
artifacts=artifacts,
|
||||
)
|
||||
(root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
print(json.dumps(manifest, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
# Run by the owner-facing release installer after hash verification. Scope is
|
||||
# this product's packages and its dedicated cluster; never remove another DB.
|
||||
set -eu
|
||||
test "$(id -u)" = 0
|
||||
mc_release_dir=$1
|
||||
cd "$mc_release_dir"
|
||||
sha256sum --check SHA256SUMS
|
||||
mc_pg_guard=/etc/postgresql-common/createcluster.d/60-mission-core-install.conf
|
||||
mc_created_guard=0
|
||||
cleanup() {
|
||||
if [ "$mc_created_guard" = 1 ]; then
|
||||
if [ "$(cat "$mc_pg_guard")" = 'create_main_cluster = false' ]; then
|
||||
rm "$mc_pg_guard"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' HUP TERM
|
||||
if ! dpkg-query -W -f='${Status}' postgresql-16 2>/dev/null | grep -qx 'install ok installed'; then
|
||||
# postgresql-common explicitly supports this drop-in. Suppress only automatic
|
||||
# creation of the unrelated default main cluster during the first install.
|
||||
test ! -e "$mc_pg_guard"
|
||||
install -d -m 0755 /etc/postgresql-common/createcluster.d
|
||||
printf '%s\n' 'create_main_cluster = false' > "$mc_pg_guard"
|
||||
chmod 0644 "$mc_pg_guard"
|
||||
mc_created_guard=1
|
||||
fi
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-remove \
|
||||
"$mc_release_dir/timescaledb-2-loader-postgresql-16_2.29.2~ubuntu24.04-1615_amd64.deb" \
|
||||
"$mc_release_dir/timescaledb-2-oss-postgresql-16_2.29.2~ubuntu24.04-1615_amd64.deb" \
|
||||
"$mc_release_dir/mission-core-node_0.8.15_amd64.deb" \
|
||||
"$mc_release_dir/mission-core-xgrids-k1_0.1.14_amd64.deb"
|
||||
systemctl is-active --quiet mission-core-node.service
|
||||
systemctl is-active --quiet mission-core-k1.service
|
||||
systemctl is-active --quiet postgresql@16-ndc-monitor.service
|
||||
systemctl is-active --quiet mission-core-node-monitor.service
|
||||
runuser -u mission-core-monitor -- /usr/bin/python3 - <<'PY'
|
||||
import http.client,json,socket,time
|
||||
deadline=time.monotonic()+20
|
||||
while True:
|
||||
c=http.client.HTTPConnection('monitor',timeout=3)
|
||||
c.sock=socket.socket(socket.AF_UNIX);c.sock.settimeout(3)
|
||||
try:
|
||||
c.sock.connect('/run/mission-core-monitor/monitor.sock')
|
||||
c.request('GET','/status');r=c.getresponse();v=json.load(r)
|
||||
if r.status==200 and v['storage']=='ready' and v['latest']['seq']>0:break
|
||||
except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException):
|
||||
pass
|
||||
finally:c.close()
|
||||
if time.monotonic()>deadline:raise SystemExit('Local telemetry did not become ready')
|
||||
time.sleep(1)
|
||||
print('Local telemetry: ready')
|
||||
PY
|
||||
printf '%s\n' 'Mission Core Node R18: package and local archive checks passed.'
|
||||
@@ -9,6 +9,7 @@ import re
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import syslog
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import gi
|
||||
@@ -261,11 +262,15 @@ class NodeApplication(Gtk.Application):
|
||||
return False
|
||||
|
||||
def load_failed(self, _view, _event, uri, _error):
|
||||
syslog.openlog('mission-core-node-ui')
|
||||
syslog.syslog(syslog.LOG_ERR, 'node-ui-load-failed')
|
||||
if local_url(uri):
|
||||
self.problem("Служба ноды недоступна. Повторно откройте приложение после восстановления службы.")
|
||||
return True
|
||||
|
||||
def process_failed(self, *_):
|
||||
def process_failed(self, _view, reason):
|
||||
syslog.openlog('mission-core-node-ui')
|
||||
syslog.syslog(syslog.LOG_ERR, 'node-ui-process-terminated reason=' + str(int(reason)))
|
||||
self.problem("Окно приложения остановилось. Закройте и повторно откройте Mission Core Node. Служба борта продолжает работать отдельно.")
|
||||
|
||||
def deny_permission(self, _view, permission):
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
[Unit]
|
||||
Description=Mission Core onboard system telemetry collector
|
||||
After=postgresql@16-ndc-monitor.service
|
||||
Wants=postgresql@16-ndc-monitor.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=mission-core-monitor
|
||||
Group=mission-core-node
|
||||
SupplementaryGroups=systemd-journal
|
||||
ExecStart=/usr/bin/python3 /usr/lib/mission-core-node/monitor/collector.py
|
||||
RuntimeDirectory=mission-core-monitor
|
||||
RuntimeDirectoryMode=0750
|
||||
StateDirectory=mission-core-monitor
|
||||
StateDirectoryMode=0750
|
||||
UMask=0007
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
MemoryHigh=96M
|
||||
MemoryMax=128M
|
||||
CPUQuota=15%
|
||||
TasksMax=24
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictAddressFamilies=AF_UNIX
|
||||
CapabilityBoundingSet=
|
||||
RestrictSUIDSGID=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -23,6 +23,7 @@ case "$1" in
|
||||
fi
|
||||
rm -f /run/mission-core-node-k1-upgrade-active
|
||||
fi
|
||||
/usr/lib/mission-core-node/setup-monitor
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -34,6 +34,7 @@ if [ "$1" = install ] || [ "$1" = upgrade ]; then
|
||||
(umask 077; : > /run/mission-core-node-k1-upgrade-active)
|
||||
systemctl stop mission-core-k1.service
|
||||
fi
|
||||
systemctl stop mission-core-node-monitor.service 2>/dev/null || true
|
||||
fi
|
||||
. /etc/os-release
|
||||
if [ "${ID:-}" != ubuntu ] || [ "${VERSION_ID:-}" != 24.04 ]; then
|
||||
|
||||
@@ -52,6 +52,10 @@ case "$1" in
|
||||
systemctl disable mission-core-realsense.service || true
|
||||
systemctl stop mission-core-node.service
|
||||
systemctl disable mission-core-node.service
|
||||
systemctl stop mission-core-node-monitor.service
|
||||
systemctl disable mission-core-node-monitor.service
|
||||
systemctl stop postgresql@16-ndc-monitor.service
|
||||
systemctl disable postgresql@16-ndc-monitor.service
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/bin/sh
|
||||
# Fixed local database bootstrap; no supplied SQL, credentials or network address.
|
||||
set -eu
|
||||
test "$(id -u)" = 0
|
||||
if ! getent passwd mission-core-monitor >/dev/null; then
|
||||
adduser --system --group --home /var/lib/mission-core-monitor --no-create-home --disabled-login mission-core-monitor
|
||||
fi
|
||||
install -d -m 0750 -o mission-core-monitor -g mission-core-node /var/lib/mission-core-monitor
|
||||
install -d -m 0755 -o postgres -g postgres /run/mission-core-monitor-db
|
||||
install -d -m 0755 /etc/tmpfiles.d
|
||||
printf '%s\n' 'd /run/mission-core-monitor-db 0755 postgres postgres -' > /etc/tmpfiles.d/mission-core-monitor.conf
|
||||
if [ ! -d /etc/postgresql/16/ndc-monitor ]; then
|
||||
pg_createcluster 16 ndc-monitor --port=5433 --socketdir=/run/mission-core-monitor-db --datadir=/var/lib/mission-core-monitor-db --start-conf=auto -- --auth-local=peer --auth-host=reject
|
||||
fi
|
||||
install -d -m 0755 /etc/postgresql/16/ndc-monitor/conf.d
|
||||
cat > /etc/postgresql/16/ndc-monitor/conf.d/60-mission-core-monitor.conf <<'CONF'
|
||||
listen_addresses = ''
|
||||
unix_socket_directories = '/run/mission-core-monitor-db'
|
||||
shared_preload_libraries = 'timescaledb'
|
||||
shared_buffers = '32MB'
|
||||
work_mem = '2MB'
|
||||
maintenance_work_mem = '16MB'
|
||||
max_connections = 12
|
||||
max_worker_processes = 4
|
||||
max_parallel_workers = 0
|
||||
timescaledb.max_background_workers = 2
|
||||
timescaledb.telemetry_level = 'off'
|
||||
max_wal_size = '128MB'
|
||||
min_wal_size = '32MB'
|
||||
temp_file_limit = '32MB'
|
||||
statement_timeout = '5s'
|
||||
log_statement = 'none'
|
||||
log_min_error_statement = 'panic'
|
||||
CONF
|
||||
install -d -m 0755 /etc/systemd/system/postgresql@16-ndc-monitor.service.d
|
||||
cat > /etc/systemd/system/postgresql@16-ndc-monitor.service.d/60-mission-core-monitor.conf <<'CONF'
|
||||
[Service]
|
||||
ExecStartPre=+/usr/bin/install -d -m 0755 -o postgres -g postgres /run/mission-core-monitor-db
|
||||
MemoryHigh=256M
|
||||
MemoryMax=384M
|
||||
CPUQuota=25%
|
||||
TasksMax=32
|
||||
Nice=10
|
||||
CONF
|
||||
systemctl daemon-reload
|
||||
systemctl restart postgresql@16-ndc-monitor.service
|
||||
runuser -u postgres -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d postgres <<'SQL'
|
||||
SELECT 'CREATE ROLE "mission-core-monitor" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE' WHERE NOT EXISTS(SELECT FROM pg_roles WHERE rolname='mission-core-monitor') \gexec
|
||||
SELECT 'CREATE DATABASE mission_core_monitor OWNER "mission-core-monitor"' WHERE NOT EXISTS(SELECT FROM pg_database WHERE datname='mission_core_monitor') \gexec
|
||||
SQL
|
||||
runuser -u postgres -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d mission_core_monitor -c 'CREATE EXTENSION IF NOT EXISTS timescaledb'
|
||||
runuser -u mission-core-monitor -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d mission_core_monitor -f /usr/lib/mission-core-node/monitor/schema.sql
|
||||
systemctl enable mission-core-node-monitor.service
|
||||
systemctl restart mission-core-node-monitor.service
|
||||
@@ -3,5 +3,5 @@ import {createIsolatedRerunHost} from '../../../control-station/src/components/r
|
||||
import {SensorWorkspace} from '../../../../packages/sensor-ui/src/SensorWorkspace';
|
||||
import type {SensorTransport} from '../../../../packages/sensor-ui/src/contracts';
|
||||
import {request} from './api';
|
||||
const transport:SensorTransport={enrollment:{state:()=>request("/api/devices/enrollment"),submit:value=>request("/api/devices/enrollment/operations","POST",value),operation:id=>request("/api/devices/enrollment/operations/"+encodeURIComponent(id))},subscribe:(receive,unavailable)=>{const events=new EventSource('/api/devices/events');events.onmessage=e=>{try{receive(JSON.parse(e.data));}catch{unavailable();}};events.onerror=unavailable;return()=>events.close();},inventory:()=>request('/api/devices'),submit:value=>request('/api/devices/operations','POST',value),operation:id=>request('/api/devices/operations/'+encodeURIComponent(id))};
|
||||
const transport:SensorTransport={localPreview:{open:(command,signal)=>fetch("/api/devices/preview",{method:"POST",credentials:"same-origin",headers:{"Content-Type":"application/json"},body:JSON.stringify(command),signal}),read:(peer,after,signal)=>fetch("/api/devices/preview/read",{method:"POST",credentials:"same-origin",headers:{"Content-Type":"application/json"},body:JSON.stringify({peer_id:peer,after}),signal})},enrollment:{state:()=>request("/api/devices/enrollment"),submit:value=>request("/api/devices/enrollment/operations","POST",value),operation:id=>request("/api/devices/enrollment/operations/"+encodeURIComponent(id))},subscribe:(receive,unavailable)=>{const events=new EventSource('/api/devices/events');events.onmessage=e=>{try{receive(JSON.parse(e.data));}catch{unavailable();}};events.onerror=unavailable;return()=>events.close();},inventory:()=>request('/api/devices'),submit:value=>request('/api/devices/operations','POST',value),operation:id=>request('/api/devices/operations/'+encodeURIComponent(id))};
|
||||
export function NodeSensors(){return <SensorWorkspace contributions={[xgridsK1SensorUi]} createRerunHost={createIsolatedRerunHost} transport={transport}/>;}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import {Component,type ReactNode} from 'react';
|
||||
import {Button,SettingsCard} from '@nodedc/ui-react';
|
||||
|
||||
export function runtimeDiagnostic(error:unknown,code='ui-error') {
|
||||
const kind=error instanceof Error&&/^[A-Za-z]{1,48}$/.test(error.name)?error.name:'Error';
|
||||
// Never send exception messages, URLs, form values or arbitrary stack text.
|
||||
const locations=error instanceof Error?(error.stack??'').match(/[A-Za-z0-9_-]+\.js:\d+:\d+/g)?.slice(0,8)??[]:[];
|
||||
void fetch('/api/monitor/events',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({code,kind,locations}),keepalive:true}).catch(()=>{});
|
||||
}
|
||||
export function observeRuntime(){
|
||||
window.addEventListener('error',event=>runtimeDiagnostic(event.error));
|
||||
window.addEventListener('unhandledrejection',event=>runtimeDiagnostic(event.reason,'ui-rejection'));
|
||||
}
|
||||
export class RuntimeBoundary extends Component<{children:ReactNode;resetKey?:string},{failed:boolean}> {
|
||||
state={failed:false};
|
||||
static getDerivedStateFromError(){return {failed:true};}
|
||||
componentDidCatch(error:Error){runtimeDiagnostic(error,'ui-render-error');}
|
||||
componentDidUpdate(previous:Readonly<{children:ReactNode;resetKey?:string}>){if(this.state.failed&&previous.resetKey!==this.props.resetKey)this.setState({failed:false});}
|
||||
render(){return this.state.failed?<SettingsCard title="Не удалось открыть представление" description="Вернитесь к устройствам или повторите открытие."><Button onClick={()=>this.setState({failed:false})}>Открыть повторно</Button></SettingsCard>:this.props.children;}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import "./node.css";
|
||||
import { NodeSensors } from "./NodeSensors";
|
||||
import { Home, HomeSettings } from "./Home";
|
||||
import { usePresentation } from "./usePresentation";
|
||||
import {RuntimeBoundary,observeRuntime} from './RuntimeBoundary';
|
||||
|
||||
function App() {
|
||||
const node = useNode();
|
||||
@@ -53,7 +54,7 @@ function App() {
|
||||
contexts={value ? [{ id: "board", label: value.name, description: value.host.hostname, icon: <Icon name="activity" />, onSelect: () => openView("overview") }] : []}
|
||||
items={views.filter(item => item.root === root).map(item => ({ id: item.id, label: item.label, icon: <Icon name={item.icon} /> }))} activeId={workspace.activeView ?? undefined} onItemChange={id => openView(id as ViewId)} footer={<span>Mission Core Node · {value?.version}</span>} />}
|
||||
content={currentView && <ApplicationPanel title={currentView.label} eyebrow={currentRoot.label} expanded={workspace.contentExpanded} onExpandedChange={workspace.setContentExpanded} onClose={workspace.closeView}
|
||||
utilityActions={[...(workspace.activeView === "ssh" ? [{ label: "Добавить доверенное устройство", icon: "plus" as const, onClick: () => setAdding(true) }] : []), { label: "Обновить сведения", icon: "refresh", disabled: pending || environment.running, onClick: refreshAll }]}>{content}</ApplicationPanel>}
|
||||
utilityActions={[...(workspace.activeView === "ssh" ? [{ label: "Добавить доверенное устройство", icon: "plus" as const, onClick: () => setAdding(true) }] : []), { label: "Обновить сведения", icon: "refresh", disabled: pending || environment.running, onClick: refreshAll }]}><RuntimeBoundary resetKey={workspace.activeView??undefined}>{content}</RuntimeBoundary></ApplicationPanel>}
|
||||
stage={value ? <Home page={presentation.settings.pages.home} openView={openView} /> : <div className="node-stage" aria-busy={pending}>
|
||||
<SettingsCard className="node-entry" title={pending ? "Подключаемся к ноде" : locked ? "Вход в Mission Core Node" : "Нода недоступна"}>
|
||||
{pending ? <ActivityIndicator label="Получение сведений о ноде" /> : <p className="node-note">{locked ? "Подтвердите доступ в системном окне." : "Не удалось связаться со службой. Повторите подключение."}</p>}
|
||||
@@ -64,4 +65,5 @@ function App() {
|
||||
<HomeSettings open={!!value && settingsOpen} onClose={() => setSettingsOpen(false)} presentation={presentation} />
|
||||
</>;
|
||||
}
|
||||
createRoot(document.getElementById("root")!).render(<App />);
|
||||
observeRuntime();
|
||||
createRoot(document.getElementById("root")!).render(<RuntimeBoundary><App /></RuntimeBoundary>);
|
||||
|
||||
Reference in New Issue
Block a user