Fix onboard WebKit preview and archive board telemetry locally

This commit is contained in:
DCCONSTRUCTIONS
2026-09-08 01:37:52 +03:00
parent d8afb61229
commit f0802d2713
46 changed files with 2623 additions and 40 deletions
@@ -6,19 +6,17 @@ interface TelemetrySeriesProps {
ceiling?: number;
}
function linePoints(values: Array<number | null>, ceiling?: number): string {
const finite = values.filter((value): value is number =>
value !== null && Number.isFinite(value),
);
if (finite.length === 0) return "";
const maximum = Math.max(ceiling ?? 0, ...finite, 1);
const denominator = Math.max(1, values.length - 1);
return values.map((value, index) => {
const x = index / denominator * 100;
const normalized = value === null ? 0 : Math.max(0, Math.min(maximum, value));
const y = 36 - normalized / maximum * 34;
return `${x.toFixed(2)},${y.toFixed(2)}`;
}).join(" ");
export function lineSegments(values: Array<number | null>, ceiling?: number): string[] {
const finite = values.filter((value): value is number => value !== null && Number.isFinite(value));
if (!finite.length) return [];
const maximum = Math.max(ceiling ?? 0, ...finite, 1), denominator = Math.max(1, values.length - 1);
const segments:string[]=[];let segment:string[]=[];
values.forEach((value,index)=>{
if(value===null||!Number.isFinite(value)){if(segment.length)segments.push(segment.join(' '));segment=[];return;}
segment.push(`${(index/denominator*100).toFixed(2)},${(36-Math.max(0,Math.min(maximum,value))/maximum*34).toFixed(2)}`);
});
if(segment.length)segments.push(segment.join(' '));
return segments;
}
export function TelemetrySeries({
@@ -28,7 +26,7 @@ export function TelemetrySeries({
resource,
ceiling,
}: TelemetrySeriesProps) {
const points = linePoints(values, ceiling);
const segments = lineSegments(values, ceiling);
return (
<div className="system-telemetry-series">
<div>
@@ -45,7 +43,7 @@ export function TelemetrySeries({
aria-label={`${label}: ${value}`}
>
<path d="M0 37 H100" />
{points ? <polyline points={points} /> : null}
{segments.map((points,index)=><polyline key={index} points={points} />)}
</svg>
</div>
);
@@ -0,0 +1,49 @@
import {useEffect,useState} from 'react';
export interface MetricDefinition {label:string;group:string;resource:string;unit:string;reason:string|null}
export interface BoardMonitor {
available:boolean;fresh:boolean;storage:string;source_id:string;
latest:{seq:number;at:number;boot_id:string;values:Record<string,number|null>}|null;
definitions:Record<string,MetricDefinition>;received_at:number;backlog:number;
database_bytes:number|null;database_budget_bytes:number;archive_since:number|null;
start:number;end:number;bucket_seconds:number;
series:{at:number;min:number|null;max:number|null;mean:number|null;count:number}[];
events:{at:number;code:string;kind:string;locations:string[]}[];
}
export function useBoardMonitor(vehicle:string,metric:string,window:number,end:number|null){
const [value,setValue]=useState<BoardMonitor|null>(null),[error,setError]=useState('');
useEffect(()=>{
let active=true;let pending:AbortController|undefined;let timer:ReturnType<typeof setTimeout>|undefined;
setValue(null);setError('');
const read=async()=>{
const controller=new AbortController();pending=controller;
const deadline=setTimeout(()=>controller.abort(),12000);
try {
const query=new URLSearchParams({metric,window:String(window),...(end===null?{}:{end:String(end)})});
const response=await fetch(`/api/v1/fleet/${encodeURIComponent(vehicle)}/monitor?${query}`,{credentials:'same-origin',cache:'no-store',signal:controller.signal});
if(!response.ok)throw new Error('Мониторинг БК недоступен.');
const next=await response.json() as BoardMonitor;
if(active){setValue(next);setError('');}
}catch {if(active)setError('Не удалось обновить мониторинг БК.');}
finally {clearTimeout(deadline);if(active)timer=setTimeout(()=>void read(),5000);}
};
void read();return()=>{active=false;pending?.abort();clearTimeout(timer);};
},[vehicle,metric,window,end]);
return {value,error,dismissError:()=>setError('')};
}
export function monitorSeries(value:BoardMonitor,mode:'min'|'max'|'mean') {
const count=Math.ceil((value.end-value.start)/value.bucket_seconds)+1;
const values:Array<number|null>=Array(Math.min(count,302)).fill(null);
const first=Math.floor(value.start/value.bucket_seconds)*value.bucket_seconds;
for(const point of value.series){const index=Math.round((point.at-first)/value.bucket_seconds);if(index>=0&&index<values.length)values[index]=point[mode];}
return values;
}
export function monitorValue(value:number|null|undefined,unit=''){
if(value==null||!Number.isFinite(value))return 'Нет измерения';
if(unit==='bytes'||unit==='bytes/s'){
const labels=unit==='bytes'?['Б','КиБ','МиБ','ГиБ']:['Б/с','КиБ/с','МиБ/с','ГиБ/с'];let index=0;
while(value>=1024&&index<3){value/=1024;index++;}
return `${value.toLocaleString('ru-RU',{maximumFractionDigits:1})} ${labels[index]}`;
}
return `${value.toLocaleString('ru-RU',{maximumFractionDigits:1})}${unit?' '+unit:''}`;
}
@@ -0,0 +1,45 @@
import {useState} from 'react';
import {ActivityIndicator,Button,Icon,ResourceList,ResourceRow,Select,SettingsCard,StatusBadge,TextField,ToastStack} from '@nodedc/ui-react';
import {useBoardMonitor,monitorSeries,monitorValue} from '../../core/fleet/boardMonitor';
import {TelemetrySeries} from '../../components/system/TelemetrySeries';
import '../../styles/system-telemetry.css';
import './board-monitor.css';
const eventLabels:Record<string,string>={'ui-error':'Ошибка интерфейса','ui-rejection':'Ошибка асинхронной операции','ui-render-error':'Ошибка представления','system-oom':'Нехватка памяти','gpu-reset':'Сброс GPU','disk-error':'Ошибка диска','usb-disconnected':'Отключение USB','usb-error':'Ошибка USB','service-failed':'Сбой службы','ui-process-exit':'Завершение процесса окна','ui-load-failed':'Ошибка загрузки окна'};
const time=(value:number)=>new Date(value*1000).toLocaleString('ru-RU');
export function BoardMonitorView({vehicle,name,back}:{vehicle:string;name:string;back:()=>void}) {
const [metric,setMetric]=useState('cpu.usage'),[group,setGroup]=useState('CPU'),[window,setWindow]=useState(900),[end,setEnd]=useState<number|null>(null),[mode,setMode]=useState<'min'|'max'|'mean'>('max');
const {value,error,dismissError}=useBoardMonitor(vehicle,metric,window,end);
const definitions=value?.definitions??{},definition=definitions[metric];
const groups=[...new Set(Object.values(definitions).map(item=>item.group))];
const metrics=Object.entries(definitions).filter(([,item])=>item.group===group);
const latest=value?.latest;
return <div className="board-monitor">
<div><Button onClick={back}><Icon name="chevron-left"/>К аппарату</Button></div>
<SettingsCard title="Мониторинг системы БК" description={name} actions={<StatusBadge tone={value?.fresh&&!error?'success':'neutral'}>{value?.fresh&&!error?'Данные поступают':'Нет свежих данных'}</StatusBadge>}>
{latest&&<p>Измерено на БК: {time(latest.at)}. {value?.storage==='ready'?'Локальная история записывается.':'Запись локальной истории недоступна.'} {value?.backlog?'История догружается в Mission Core.':''}</p>}
{value?.database_bytes!=null&&<p>Архив на БК: {monitorValue(value.database_bytes,'bytes')} из {monitorValue(value.database_budget_bytes,'bytes')}. {value.archive_since?`Копия в Mission Core с ${time(value.archive_since)}.`:''}</p>}
</SettingsCard>
{!value?<ActivityIndicator label="Получаем мониторинг БК"/>:!value.available?<SettingsCard title="История БК пока не получена" description="Данные появятся после подключения бортового сборщика телеметрии."/>:<>
<SettingsCard title="История нагрузки">
<div className="board-monitor-controls">
<Select label="Ресурс" value={group} options={groups.map(label=>({value:label,label}))} onChange={next=>{setGroup(next);const first=Object.entries(definitions).find(([,item])=>item.group===next);if(first)setMetric(first[0]);}}/>
<Select label="Показатель" value={metric} options={metrics.map(([key,item])=>({value:key,label:`${item.resource} · ${item.label}`}))} onChange={setMetric}/>
<Select label="Период" value={String(window)} options={[{value:'900',label:'15 минут'},{value:'3600',label:'1 час'},{value:'21600',label:'6 часов'},{value:'86400',label:'24 часа'},{value:'604800',label:'7 дней'}]} onChange={next=>setWindow(Number(next))}/>
<Select label="Значение в интервале" value={mode} options={[{value:'max',label:'Максимум'},{value:'mean',label:'Среднее'},{value:'min',label:'Минимум'}]} onChange={next=>setMode(next as typeof mode)}/>
</div>
<div className="board-monitor-controls"><Button onClick={()=>setEnd(end===null?Date.now()/1000:null)}>{end===null?'Зафиксировать время':'В реальном времени'}</Button>{end!==null&&<TextField label="Конец периода" type="datetime-local" step="1" value={new Date((end-new Date(end*1000).getTimezoneOffset()*60)*1000).toISOString().slice(0,19)} onChange={event=>{const next=new Date(event.target.value).getTime()/1000;if(Number.isFinite(next))setEnd(next);}}/>}</div>
<TelemetrySeries label={definition?`${definition.resource} · ${definition.label}`:'Загрузка CPU'} resource="Последний интервал периода" value={monitorValue(value.series.at(-1)?.[mode],definition?.unit)} values={monitorSeries(value,mode)} ceiling={definition?.unit==='%'?100:undefined}/>
<p>{time(value.start)} {time(value.end)} · интервал {value.bucket_seconds} с. Разрывы означают отсутствие измерений.</p>
{!value.series.length&&<p>За выбранный период измерений нет.</p>}
</SettingsCard>
<SettingsCard title={`Последние измерения · ${group}`} description={latest?time(latest.at):'Ожидаем измерения'}>
<ResourceList aria-label="Показатели системы">{metrics.map(([key,item])=><li key={key}><ResourceRow title={`${item.resource} · ${item.label}`} description={latest?.values[key]==null?item.reason??'Счётчик недоступен':undefined} metadata={monitorValue(latest?.values[key],item.unit)}/></li>)}</ResourceList>
</SettingsCard>
<SettingsCard title="События системы и приложения" description="За выбранный период">
{value.events.length?<ResourceList aria-label="События БК">{value.events.map((event,index)=><li key={`${event.at}-${index}`}><ResourceRow title={eventLabels[event.code]??'Событие системы'} description={time(event.at)}/></li>)}</ResourceList>:<p>Событий не получено.</p>}
</SettingsCard>
</>}
<ToastStack items={error?[{id:'monitor-error',title:error,tone:'error',durationMs:null}]:[]} onDismiss={dismissError}/>
</div>;
}
@@ -1,3 +1,4 @@
import {BoardMonitorView} from './BoardMonitorView';
import { useEffect, useRef, useState } from "react";
import { ActivityIndicator, Button, ConfirmationModal, Icon, IconButton, ResourceList, ResourceRow, Select, SettingsCard, StatusBadge, TextAreaField, TextField, Window, WindowFooterActions } from "@nodedc/ui-react";
import { fleetRequest, useFleet, type FleetPreview, type Vehicle } from "../../core/fleet/useFleet";
@@ -19,6 +20,7 @@ export function VehiclesWorkspace({ createRequest = 0 }: { createRequest?: numbe
const [error, setError] = useState("");
const [selected, setSelected] = useState<string | null>(null);
const [sensorOpen, setSensorOpen] = useState(false);
const [monitorOpen,setMonitorOpen]=useState(false);
const [revoking, setRevoking] = useState<Vehicle | null>(null);
const lastCreateRequest = useRef(createRequest);
useEffect(() => { if (createRequest !== lastCreateRequest.current) { lastCreateRequest.current = createRequest; setAdding(true); setError(""); } }, [createRequest]);
@@ -40,6 +42,7 @@ export function VehiclesWorkspace({ createRequest = 0 }: { createRequest?: numbe
finally { setPending(false); }
}
const detail = fleet.items?.find(item => item.id === selected);
if(detail&&monitorOpen)return <BoardMonitorView vehicle={detail.id} name={detail.name} back={()=>setMonitorOpen(false)}/>;
return <div className="fleet-workspace">
{fleet.error && <p role="alert">{fleet.error}</p>}
{!adding && error && <p role="alert">{error}</p>}
@@ -51,6 +54,7 @@ export function VehiclesWorkspace({ createRequest = 0 }: { createRequest?: numbe
<div><dt>Последняя связь</dt><dd>{detail.last_seen ? new Date(detail.last_seen * 1000).toLocaleString("ru-RU") : "Соединение ещё не получено"}</dd></div>
{detail.host && <><div><dt>Имя БК в системе</dt><dd>{detail.host.hostname}</dd></div><div><dt>Операционная система</dt><dd>{detail.host.os}</dd></div><div><dt>Архитектура</dt><dd>{detail.host.architecture}</dd></div><div><dt>Процессоры</dt><dd>{detail.host.cpus}</dd></div><div><dt>Память</dt><dd>{detail.host.memory_kib ? `${(detail.host.memory_kib / 1024 / 1024).toFixed(1)} ГиБ` : "Нет сведений"}</dd></div></>}
</dl>
<Button onClick={()=>setMonitorOpen(true)}><Icon name="activity"/>Мониторинг системы БК</Button>
{detail.enrollment !== "revoked" && <Button onClick={() => setRevoking(detail)}>Отозвать привязку БК</Button>}
</SettingsCard>}
<SettingsCard title="Устройства аппарата"><VehicleSensors onDetailChange={setSensorOpen} vehicleID={detail.id} enabled={!fleet.error && detail.enrollment === "paired" && detail.connectivity === "online"} /></SettingsCard>
@@ -0,0 +1,3 @@
.board-monitor { display: grid; gap: var(--nodedc-space-4); min-width: 0; }
.board-monitor-controls { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: var(--nodedc-space-3); align-items: end; margin-bottom: var(--nodedc-space-3); }
.board-monitor p { font-size: var(--nodedc-font-size-sm); color: var(--nodedc-text-secondary); }
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import {before,after,test} from 'node:test';
import {createServer} from 'vite';
let server,series,segments,frames,localPreview,MEDIA_PROTOCOL;
before(async()=>{
server=await createServer({appType:'custom',logLevel:'silent',server:{middlewareMode:true}});
({monitorSeries:series}=await server.ssrLoadModule('/src/core/fleet/boardMonitor.ts'));
({lineSegments:segments}=await server.ssrLoadModule('/src/components/system/TelemetrySeries.tsx'));
({MEDIA_PROTOCOL}=await server.ssrLoadModule('@xgrids-k1/frontend/sensors/previewFrames.ts'));
({localPreviewFrames:frames,localPreview}=await server.ssrLoadModule('@xgrids-k1/frontend/sensors/localPreview.ts'));
});
after(async()=>{await server?.close();});
test('archive gaps break the plot instead of manufacturing zero or connecting missing time',()=>{
const values=series({start:0,end:20,bucket_seconds:5,series:[{at:0,max:80},{at:5,max:90},{at:20,max:10}]},'max');
assert.deepEqual(values,[80,90,null,null,10]);
const result=segments(values,100);
assert.equal(result.length,2);
assert.match(result[0],/^0.00,8.80 25.00,5.40$/);
assert.deepEqual(segments([null,NaN,null]),[]);
});
test('HTTP media framing survives every byte boundary and refuses oversized frames',()=>{
const output=[],receive=frames((kind,bytes)=>output.push([kind,[...bytes]]));
const encoded=Uint8Array.of(2,0,0,0,3,4,5,6,3,0,0,0,2,7,8);
for(const byte of encoded)receive(Uint8Array.of(byte));
assert.deepEqual(output,[[2,[4,5,6]],[3,[7,8]]]);
assert.throws(()=>frames(()=>{})(Uint8Array.of(2,0,1,0,0)));
assert.throws(()=>frames(()=>{})(Uint8Array.of(7,0,0,0,0)));
});
test('local carrier delivers registered channels and ACKs through completed responses',async()=>{
const encode=(kind,data)=>{const bytes=typeof data==='string'?new TextEncoder().encode(data):Uint8Array.from(data);const out=new Uint8Array(bytes.length+5);out[0]=kind;new DataView(out.buffer).setUint32(1,bytes.length);out.set(bytes,5);return out;};
const peer='peer_'+ 'a'.repeat(32), received=[], closed=[];let preview;
const device={id:'synthetic-k1',snapshot:{context:{session_id:'synthetic-session'}}};
const transport={
submit:async request=>{closed.push(request);return {state:'complete'};},
localPreview:{
open:async request=>{assert.equal(request.action_id,'offer');return new Response(encode(0,JSON.stringify({peer_id:peer,media_protocol:MEDIA_PROTOCOL})));},
read:async(id,after)=>{
assert.equal(id,peer);
if(after===0)return new Response(new Uint8Array([...encode(2,[8,9]),...encode(3,'camera-ready')]));
assert.equal(after,7);preview.close();return new Response();
}
}
};
preview=localPreview(device,transport,{},()=>assert.fail('carrier failed'));
preview.rrd.onmessage=event=>{received.push([...new Uint8Array(event.data)]);preview.rrd.send(JSON.stringify({ack:7}));};
preview.camera.onmessage=event=>received.push(event.data);
await preview.start();
assert.deepEqual(received,[[8,9],'camera-ready']);
assert.equal(preview.rrd.readyState,'closed');assert.equal(closed.length,1);
assert.equal(closed[0].action_id,'close-peer');assert.equal(closed[0].parameters.peer_id,peer);
});
+3
View File
@@ -69,12 +69,14 @@ func run() error {
return err
}
app := &node.Server{Store: store, Assets: assets, Origin: "http://" + *listen, Version: version, Inventory: func() node.Inventory { return node.Host("/") }}
app.Monitor = node.NewMonitor()
app.Presentation = node.NewPresentationStore(*dir)
pairing, err := node.OpenPairing(store, *dir, version, app.Inventory)
if err != nil {
return err
}
app.Pairing = pairing
pairing.Monitor = app.Monitor
nodeID, _ := store.Public()
app.Sensors, err = node.OpenSensors(*dir, nodeID)
if err != nil {
@@ -122,6 +124,7 @@ func run() error {
go func() { errs <- private.Serve(unix) }()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
go app.Monitor.Run(ctx)
go app.Sensors.WatchUSB(ctx)
go pairing.Run(ctx)
log.Print("Mission Core Node " + version + " listening on loopback")
@@ -0,0 +1,73 @@
package node
import (
"bytes"
"encoding/json"
"io"
"net/http"
"strings"
"time"
)
// Local WebKit requires completed bounded HTTP responses. The same plugin
// delivery owns RRD/fMP4 framing, backpressure, resume and acquisition isolation.
func (s *Server) localPreviewRoutes(mux *http.ServeMux) {
capacity := make(chan struct{}, 2)
for _, route := range []string{"/api/devices/preview", "/api/devices/preview/read"} {
mux.HandleFunc("POST "+route, func(w http.ResponseWriter, r *http.Request) {
if !s.authorized(w, r) {
return
}
var value json.RawMessage
if !decode(w, r, &value) {
return
}
if s.DeviceEnrollment == nil {
http.Error(w, "Preview unavailable", 503)
return
}
path := "/local-preview/read"
if r.URL.Path == "/api/devices/preview" {
var command SensorCommand
if json.Unmarshal(value, &command) != nil || command.Action != "offer" || !sensorID.MatchString(command.Session.DeviceID) || !strings.HasPrefix(command.Session.DeviceID, "k1_") {
http.Error(w, "Invalid preview", 400)
return
}
path = "/local-preview"
}
select {
case capacity <- struct{}{}:
defer func() { <-capacity }()
default:
http.Error(w, "Close another viewer", 429)
return
}
request, err := http.NewRequestWithContext(r.Context(), "POST", "http://k1"+path, bytes.NewReader(value))
if err != nil {
http.Error(w, "Invalid preview", 400)
return
}
request.Header.Set("Content-Type", "application/json")
request.Header.Set("X-Node-Id", s.DeviceEnrollment.nodeID)
client := &http.Client{Transport: s.DeviceEnrollment.client.Transport, Timeout: 5 * time.Second}
response, err := client.Do(request)
if err != nil {
http.Error(w, "Preview unavailable", 503)
return
}
defer response.Body.Close()
if response.StatusCode != 200 {
http.Error(w, "Preview unavailable", response.StatusCode)
return
}
data, err := io.ReadAll(io.LimitReader(response.Body, 196609))
if err != nil || len(data) > 196608 {
http.Error(w, "Preview unavailable", 503)
return
}
w.Header().Set("Content-Type", "application/vnd.missioncore.preview")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(data)
})
}
}
+159
View File
@@ -0,0 +1,159 @@
package node
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"strconv"
"sync"
"time"
)
// The collector owns sampling/storage. This bounded cache keeps monitoring I/O
// out of the paired control heartbeat and out of acquisition lifecycles.
type Monitor struct {
mu sync.Mutex
client *http.Client
source string
after int64
sent int64
value map[string]any
}
func NewMonitor() *Monitor {
return &Monitor{client: &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/run/mission-core-monitor/monitor.sock")
}}}}
}
func (m *Monitor) read(ctx context.Context, path string, body []byte) (map[string]any, error) {
method := "GET"
if body != nil {
method = "POST"
}
r, e := http.NewRequestWithContext(ctx, method, "http://monitor"+path, bytes.NewReader(body))
if e != nil {
return nil, e
}
r.Header.Set("Content-Type", "application/json")
response, e := m.client.Do(r)
if e != nil {
return nil, e
}
defer response.Body.Close()
if response.StatusCode != 200 {
return nil, errors.New("monitor unavailable")
}
data, e := io.ReadAll(io.LimitReader(response.Body, 262145))
if e != nil || len(data) > 262144 {
return nil, errors.New("monitor response budget")
}
var value map[string]any
e = json.Unmarshal(data, &value)
return value, e
}
func (m *Monitor) Run(ctx context.Context) {
defer m.client.CloseIdleConnections()
timer := time.NewTicker(2 * time.Second)
defer timer.Stop()
for {
status, e := m.read(ctx, "/status", nil)
if e == nil {
source, _ := status["source_id"].(string)
m.mu.Lock()
if source != m.source {
m.source = source
m.after = 0
m.sent = 0
}
after := m.after
m.mu.Unlock()
batch, be := m.read(ctx, "/batch?after="+strconv.FormatInt(after, 10), nil)
if be == nil && batch["source_id"] == source {
status["batch"] = batch
}
m.mu.Lock()
m.value = status
m.mu.Unlock()
}
select {
case <-ctx.Done():
return
case <-timer.C:
}
}
}
func (m *Monitor) Snapshot() map[string]any {
m.mu.Lock()
defer m.mu.Unlock()
if m.value == nil {
return map[string]any{"storage": "unavailable"}
}
if batch, ok := m.value["batch"].(map[string]any); ok {
if rows, ok := batch["samples"].([]any); ok {
for _, raw := range rows {
if row, ok := raw.(map[string]any); ok {
if seq, ok := row["seq"].(float64); ok && int64(seq) > m.sent {
m.sent = int64(seq)
}
}
}
}
}
return m.value
}
func (m *Monitor) Acknowledge(raw json.RawMessage) {
var ack struct {
Source string `json:"source_id"`
After int64 `json:"after"`
}
if json.Unmarshal(raw, &ack) != nil {
return
}
m.mu.Lock()
defer m.mu.Unlock()
// A receiver may already have a later cursor from before the Node restarted.
// Its authenticated ACK is bounded by the collector's durable latest record.
latest := m.sent
if row, ok := m.value["latest"].(map[string]any); ok {
if seq, ok := row["seq"].(float64); ok {
latest = int64(seq)
}
}
if ack.Source == m.source && ack.After >= m.after && ack.After <= latest {
m.after = ack.After
}
}
func (s *Server) monitorRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /api/monitor", func(w http.ResponseWriter, r *http.Request) {
if !s.authorized(w, r) {
return
}
if s.Monitor == nil {
reply(w, 503, map[string]string{"error": "Мониторинг недоступен"})
return
}
reply(w, 200, s.Monitor.Snapshot())
})
mux.HandleFunc("POST /api/monitor/events", func(w http.ResponseWriter, r *http.Request) {
if !s.authorized(w, r) {
return
}
var value json.RawMessage
if !decode(w, r, &value) {
return
}
if len(value) > 2048 || s.Monitor == nil {
http.Error(w, "Monitoring unavailable", 503)
return
}
if _, e := s.Monitor.read(r.Context(), "/events", value); e != nil {
http.Error(w, "Monitoring unavailable", 503)
return
}
reply(w, 200, map[string]bool{"ok": true})
})
}
@@ -0,0 +1,50 @@
package node
import (
"encoding/json"
"testing"
)
func TestMonitorAcknowledgementsRequireSameArchiveAndDurableBounds(t *testing.T) {
m := NewMonitor()
m.source = "archive-one"
m.value = map[string]any{"latest": map[string]any{"seq": float64(100)}}
for _, raw := range []string{`{"source_id":"another","after":50}`, `{"source_id":"archive-one","after":101}`, `{"source_id":"archive-one","after":-1}`} {
m.Acknowledge(json.RawMessage(raw))
if m.after != 0 {
t.Fatal("accepted invalid cursor")
}
}
m.Acknowledge(json.RawMessage(`{"source_id":"archive-one","after":80}`))
if m.after != 80 {
t.Fatal("did not restore committed receiver cursor after restart")
}
m.Acknowledge(json.RawMessage(`{"source_id":"archive-one","after":10}`))
if m.after != 80 {
t.Fatal("regressed cursor")
}
}
func TestMonitorAndPreviewCannotBypassNodeSession(t *testing.T) {
s := newTestServer(t)
for _, path := range []string{"/api/devices/preview", "/api/devices/preview/read", "/api/monitor/events"} {
if got := call(s, "POST", path, `{}`, nil).Code; got != 401 {
t.Fatalf("%s: %d", path, got)
}
}
if got := call(s, "GET", "/api/monitor", "", nil).Code; got != 401 {
t.Fatal(got)
}
cookie := login(t, s)
if got := call(s, "POST", "/api/devices/preview", `{}`, cookie).Code; got != 503 {
t.Fatal(got)
}
s.Monitor = NewMonitor()
if got := call(s, "GET", "/api/monitor", "", cookie).Code; got != 200 {
t.Fatal(got)
}
// An unavailable collector remains independent from the authenticated UI.
if got := call(s, "GET", "/api/status", "", cookie).Code; got != 200 {
t.Fatal(got)
}
}
+1
View File
@@ -39,6 +39,7 @@ type PairState struct {
Revocations []CoreBinding `json:"revocations,omitempty"`
}
type Pairing struct {
Monitor *Monitor
Sensors *Sensors
DeviceEnrollment *DeviceEnrollment
mu sync.Mutex
@@ -286,10 +286,16 @@ func (p *Pairing) channel(ctx context.Context) {
payload["device_enrollment"] = p.DeviceEnrollment.Status()
payload["enrollment_results"] = p.DeviceEnrollment.RemoteResults()
}
if p.Monitor != nil {
payload["monitor"] = p.Monitor.Snapshot()
}
result, status, e := p.send(ctx, *binding, "/v1/node/heartbeat", payload)
p.mu.Lock()
if p.state.Phase == "paired" && p.state.Binding != nil && p.state.Binding.BindingID == binding.BindingID {
if e == nil && status == 200 {
if p.Monitor != nil {
p.Monitor.Acknowledge(result["monitor_ack"])
}
p.connection = "online"
p.lastSeen = p.now().Unix()
if p.DeviceEnrollment != nil {
+3
View File
@@ -13,6 +13,7 @@ import (
)
type Server struct {
Monitor *Monitor
Store *Store
Pairing *Pairing
Sensors *Sensors
@@ -81,6 +82,8 @@ func reply(w http.ResponseWriter, status int, v any) {
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
s.localPreviewRoutes(mux)
s.monitorRoutes(mux)
if s.Presentation != nil {
s.presentationRoutes(mux)
}
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/python3
"""Independent one-second collector, local archive and private Node read adapter."""
import json
import logging
import os
import re
import signal
import socketserver
import threading
import time
from collections import deque
from http.server import BaseHTTPRequestHandler
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from linux_metrics import LinuxMetrics
from storage import SCHEMA, Archive
SOCKET = Path("/run/mission-core-monitor/monitor.sock")
class Collector:
def __init__(self):
self.stop = threading.Event()
self.lock = threading.Lock()
self.latest = None
self.definitions = {}
self.events = deque(maxlen=64)
self.storage = "starting"
self.database_bytes = None
self.archive = None
self.boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
def event(self, value):
if (
not isinstance(value, dict)
or set(value) - {"code", "kind", "locations"}
or value.get("code") not in {"ui-error", "ui-rejection", "ui-render-error"}
or not isinstance(value.get("kind"), str)
or not re.fullmatch("[A-Za-z]{1,48}", value["kind"])
or not isinstance(value.get("locations", []), list)
or len(value.get("locations", [])) > 8
or any(
not isinstance(v, str)
or not re.fullmatch(r"[A-Za-z0-9_-]{1,96}\.js:[0-9]{1,9}:[0-9]{1,9}", v)
for v in value.get("locations", [])
)
):
raise ValueError("invalid-event")
with self.lock:
if len(self.events) < 64:
self.events.append({**value, "at": time.time()})
def run(self):
metrics = LinuxMetrics()
maintenance = 0
while not self.stop.is_set():
started = time.monotonic()
try:
values = metrics.sample()
with self.lock:
events = list(self.events)
sample = dict(
at=time.time(), boot_id=self.boot, uptime=started, values=values, events=events
)
if self.archive is None:
self.archive = Archive()
# Maintenance also runs while writes are paused by a disk budget.
if started - maintenance > 60:
self.database_bytes = self.archive.maintain()
maintenance = started
record = self.archive.append(sample, metrics.definitions)
with self.lock:
for _ in events:
self.events.popleft()
self.latest = record
self.definitions = metrics.definitions
self.storage = "ready"
except Exception as error:
logging.warning("monitor sample failed class=%s", type(error).__name__)
with self.lock:
self.storage = (
str(error)
if str(error) in {"disk-reserve", "archive-budget"}
else "unavailable"
)
self.stop.wait(max(0.05, 1 - (time.monotonic() - started)))
def status(self):
with self.lock:
return dict(
schema=SCHEMA,
source_id=self.archive.source if self.archive else None,
latest=self.latest,
definitions=self.definitions,
storage=self.storage,
database_bytes=self.database_bytes,
sample_interval_seconds=1,
retention_days=7,
database_budget_bytes=2 * 1024**3,
free_reserve_bytes=2 * 1024**3,
)
class Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer):
daemon_threads = True
block_on_close = False
def __init__(self, collector):
self.collector = collector
self.capacity = threading.BoundedSemaphore(6)
super().__init__(str(SOCKET), Handler)
def process_request(self, request, address):
if not self.capacity.acquire(False):
request.close()
return
request.settimeout(5)
super().process_request(request, address)
def process_request_thread(self, request, address):
try:
super().process_request_thread(request, address)
finally:
self.capacity.release()
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def reply(self, value, status=200):
payload = json.dumps(value, allow_nan=False, separators=(",", ":")).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def do_GET(self):
path = urlsplit(self.path)
try:
if path.path == "/status":
self.reply(self.server.collector.status())
elif path.path == "/batch":
after = int(parse_qs(path.query).get("after", ["0"])[0])
if not 0 <= after <= 2**53 - 1:
raise ValueError()
archive = self.server.collector.archive
if archive is None:
self.reply({"error": "Archive unavailable"}, 503)
else:
self.reply(archive.batch(after))
else:
self.reply({"error": "Unknown request"}, 404)
except Exception:
self.reply({"error": "Archive unavailable"}, 503)
def do_POST(self):
try:
size = int(self.headers.get("Content-Length", "0"))
if self.path != "/events" or not 0 < size <= 2048:
raise ValueError()
self.server.collector.event(json.loads(self.rfile.read(size)))
self.reply({"ok": True})
except (ValueError, TypeError):
self.reply({"error": "Invalid event"}, 400)
def main():
logging.basicConfig(level=logging.WARNING)
os.umask(0o007)
collector = Collector()
from journal_events import watch
threading.Thread(
target=watch, args=(collector,), name="node-system-events", daemon=True
).start()
SOCKET.unlink(missing_ok=True)
server = Server(collector)
worker = threading.Thread(target=collector.run, name="node-system-sampler", daemon=True)
worker.start()
def stop(*_):
collector.stop.set()
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
server.serve_forever(poll_interval=0.5)
finally:
collector.stop.set()
server.server_close()
worker.join(5)
SOCKET.unlink(missing_ok=True)
if __name__ == "__main__":
main()
+103
View File
@@ -0,0 +1,103 @@
"""Classify bounded journal records; never persist free-form messages or payloads."""
import json
import re
import subprocess
import threading
RULES = (
("system-oom", r"out of memory|oom-kill|killed process"),
("gpu-reset", r"GPU HANG|GPU reset|Resetting.*(gpu|chip)|i915.*(hang|reset|error)"),
("disk-error", r"I/O error|EXT4-fs error|nvme.*(timeout|reset)|ata.*failed command"),
("usb-disconnected", r"USB disconnect"),
("usb-error", r"usb.*(error -|unable to enumerate|reset.*device)"),
("service-failed", r"Failed with result|Main process exited|Failed to start"),
("ui-process-exit", r"node-ui-process-terminated"),
("ui-load-failed", r"node-ui-load-failed"),
)
def classify(record):
message = record.get("MESSAGE")
if not isinstance(message, str):
return None
code = next((code for code, pattern in RULES if re.search(pattern, message, re.I)), None)
if code is None:
return None
try:
at = int(record["__REALTIME_TIMESTAMP"]) / 1_000_000
except (KeyError, ValueError, TypeError):
return None
return dict(
code=code,
kind="Kernel" if record.get("_TRANSPORT") == "kernel" else "System",
at=at,
locations=[],
)
def watch(collector):
args = [
"/usr/bin/journalctl",
"--follow",
"--lines=0",
"--output=json",
"--no-pager",
"_TRANSPORT=kernel",
"+",
"SYSLOG_IDENTIFIER=mission-core-node-ui",
"+",
"_SYSTEMD_UNIT=mission-core-node.service",
"+",
"_SYSTEMD_UNIT=mission-core-k1.service",
"+",
"_SYSTEMD_UNIT=mission-core-node-monitor.service",
"+",
"_SYSTEMD_UNIT=postgresql@16-ndc-monitor.service",
]
for unit in (
"mission-core-node.service",
"mission-core-k1.service",
"mission-core-node-monitor.service",
"postgresql@16-ndc-monitor.service",
):
args.extend(["+", "UNIT=" + unit])
while not collector.stop.is_set():
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
def retire(process=process):
while process.poll() is None:
if collector.stop.wait(1):
if process.poll() is None:
process.terminate()
return
guard = threading.Thread(target=retire, daemon=True)
guard.start()
try:
while not collector.stop.is_set():
line = process.stdout.readline(65537)
if not line:
break
if len(line) > 65536:
while line and not line.endswith(b"\n"):
line = process.stdout.readline(65537)
continue
try:
event = classify(json.loads(line))
except (ValueError, TypeError):
continue
if event:
with collector.lock:
if len(collector.events) < 64:
collector.events.append(event)
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=3)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
process.stdout.close()
collector.stop.wait(10)
+353
View File
@@ -0,0 +1,353 @@
"""Read-only Linux counters. Missing or reset counters never manufacture zero."""
import os
import re
import time
from pathlib import Path
class LinuxMetrics:
def __init__(self, root=Path("/"), clock=time.monotonic):
self.root, self.clock = Path(root), clock
self.previous = {}
self.previous_time = None
self.definitions = {}
def text(self, path):
try:
return (self.root / str(path).lstrip("/")).read_text()[:262144].strip()
except (OSError, UnicodeError):
return None
def number(self, path):
try:
return float(self.text(path))
except (ValueError, TypeError):
return None
def paths(self, pattern):
return sorted(self.root.glob(pattern))[:128]
def sample(self):
now = self.clock()
elapsed = None if self.previous_time is None else now - self.previous_time
self.previous_time = now
values, definitions, counters = {}, {}, {}
def metric(
key, label, group, resource, unit, value, reason="Счётчик недоступен в этой системе"
):
values[key] = None if value is None else round(value, 3)
definitions[key] = dict(
label=label,
group=group,
resource=resource,
unit=unit,
reason=reason if value is None else None,
)
def delta(key, value):
if value is None:
return None
counters[key] = value
previous = self.previous.get(key)
return (
None
if previous is None or value < previous or not elapsed or elapsed > 10
else (value - previous) / elapsed
)
stat = self.text("proc/stat") or ""
cpus = 0
for line in stat.splitlines():
parts = line.split()
if not re.fullmatch(r"cpu\d*", parts[0]):
continue
if parts[0] != "cpu":
cpus += 1
data = [int(v) for v in parts[1:9]]
total = delta(parts[0] + ".total", sum(data))
idle = delta(parts[0] + ".idle", data[3] + data[4])
wait = delta(parts[0] + ".iowait", data[4])
for name, label, value in [
(
"usage",
"Загрузка",
None if not total or idle is None else 100 * (1 - idle / total),
),
(
"iowait",
"Ожидание диска",
None if not total or wait is None else 100 * wait / total,
),
]:
metric(
parts[0] + "." + name,
label,
"CPU",
parts[0],
"%",
value,
"Ожидаем два последовательных измерения",
)
if not cpus:
metric("cpu.usage", "Загрузка", "CPU", "CPU", "%", None)
memory = {}
for line in (self.text("proc/meminfo") or "").splitlines():
parts = line.replace(":", "").split()
if len(parts) > 1:
memory[parts[0]] = int(parts[1]) * 1024
for key, label in [
("MemTotal", "Всего"),
("MemAvailable", "Доступно"),
("SwapTotal", "Swap всего"),
("SwapFree", "Swap свободно"),
("Dirty", "Ожидает записи"),
("Writeback", "Записывается"),
]:
metric("memory." + key, label, "RAM", "Память", "bytes", memory.get(key))
total, available = memory.get("MemTotal"), memory.get("MemAvailable")
metric(
"memory.usage",
"Использовано",
"RAM",
"Память",
"%",
None if not total or available is None else 100 * (1 - available / total),
)
for index, load in enumerate((self.text("proc/loadavg") or "").split()[:3]):
metric(
"load." + str(index),
["За 1 минуту", "За 5 минут", "За 15 минут"][index],
"CPU",
"Load average",
"",
float(load),
)
for resource in ["cpu", "memory", "io"]:
for line in (self.text("proc/pressure/" + resource) or "").splitlines():
parts = line.split()
fields = dict(v.split("=") for v in parts[1:])
metric(
"pressure." + resource + "." + parts[0],
"Задержка за 10 секунд",
"Ожидание ресурсов",
resource + " · " + parts[0],
"%",
float(fields["avg10"]),
)
for interface in self.paths("sys/class/net/*"):
name = interface.name
for field, label in [
("rx_bytes", "Приём"),
("tx_bytes", "Передача"),
("rx_errors", "Ошибки приёма"),
("tx_errors", "Ошибки передачи"),
("rx_dropped", "Потери приёма"),
("tx_dropped", "Потери передачи"),
]:
raw = self.number(interface.relative_to(self.root) / "statistics" / field)
metric(
"net." + name + "." + field,
label,
"Сеть",
name,
"bytes/s" if field.endswith("bytes") else "1/s",
delta("net." + name + "." + field, raw),
)
speed = self.number(interface.relative_to(self.root) / "speed")
metric(
"net." + name + ".speed",
"Скорость соединения",
"Сеть",
name,
"Mbit/s",
speed if speed is not None and speed > 0 else None,
)
diskstats = (self.text("proc/diskstats") or "").splitlines()
for line in diskstats:
p = line.split()
if (
len(p) < 14
or p[2].startswith(("loop", "ram"))
or not (self.root / "sys/block" / p[2]).exists()
):
continue
name = p[2]
for index, label, unit, multiplier in [
(5, "Чтение", "bytes/s", 512),
(9, "Запись", "bytes/s", 512),
(3, "Операции чтения", "1/s", 1),
(7, "Операции записи", "1/s", 1),
(12, "Занятость", "%", 0.1),
]:
key = "disk." + name + "." + str(index)
rate = delta(key, int(p[index]))
metric(key, label, "Диски", name, unit, None if rate is None else rate * multiplier)
if self.root == Path("/"):
mounts = set()
for line in (self.text("proc/mounts") or "").splitlines():
p = line.split()
if len(p) < 3 or not p[0].startswith("/dev/") or p[1] in mounts:
continue
mount = p[1].replace("\\040", " ")
mounts.add(mount)
try:
st = os.statvfs(mount)
key = "fs." + str(len(mounts))
metric(
key + ".free",
"Свободно",
"Хранилище",
mount,
"bytes",
st.f_bavail * st.f_frsize,
)
metric(
key + ".usage",
"Использовано",
"Хранилище",
mount,
"%",
100 * (1 - st.f_bfree / st.f_blocks) if st.f_blocks else None,
)
metric(
key + ".inodes",
"Использовано inode",
"Хранилище",
mount,
"%",
100 * (1 - st.f_ffree / st.f_files) if st.f_files else None,
)
except OSError:
metric(
"fs." + str(len(mounts)) + ".free",
"Свободно",
"Хранилище",
mount,
"bytes",
None,
)
for path in self.paths("sys/class/hwmon/hwmon*/temp*_input"):
relative = path.relative_to(self.root)
name = self.text(path.parent.relative_to(self.root) / "name") or path.parent.name
raw = self.number(relative)
metric(
"temperature." + path.parent.name + "." + path.stem,
"Температура",
"Температура",
name + " · " + path.stem,
"°C",
None if raw is None else raw / 1000,
)
for card in self.paths("sys/class/drm/card[0-9]"):
base = card.relative_to(self.root)
metric(
"gpu." + card.name + ".usage",
"Загрузка",
"GPU",
card.name,
"%",
self.number(base / "device/gpu_busy_percent"),
"Драйвер не публикует общий счётчик загрузки GPU",
)
metric(
"gpu." + card.name + ".frequency",
"Частота",
"GPU",
card.name,
"MHz",
self.number(base / "gt_cur_freq_mhz"),
)
for field, label in [
("mem_info_vram_used", "Видеопамять занята"),
("mem_info_vram_total", "Видеопамять всего"),
]:
metric(
"gpu." + card.name + "." + field,
label,
"GPU",
card.name,
"bytes",
self.number(base / "device" / field),
"Выделенная видеопамять или её счётчик недоступны",
)
for usb in self.paths("sys/bus/usb/devices/*"):
if not (usb / "idVendor").exists():
continue
base = usb.relative_to(self.root)
name = usb.name
metric(
"usb." + name + ".speed",
"Скорость соединения",
"USB",
name,
"Mbit/s",
self.number(base / "speed"),
)
metric(
"usb." + name + ".traffic",
"Трафик",
"USB",
name,
"bytes/s",
None,
"USB-драйвер не публикует счётчик трафика порта",
)
groups = self.paths("sys/fs/cgroup/system.slice/mission-core*.service")
groups += self.paths(
"sys/fs/cgroup/system.slice/system-postgresql.slice/postgresql@16-ndc-monitor.service"
)
groups += self.paths(
"sys/fs/cgroup/user.slice/user-*.slice/user@*.service/app.slice/app-gnome-org.nodedc.MissionCoreNode-*.scope"
)
for path in groups:
base = path.relative_to(self.root)
name = path.name
key = "service." + re.sub("[^A-Za-z0-9_.-]", "_", name)
metric(
key + ".memory",
"Память",
"Приложения",
name,
"bytes",
self.number(base / "memory.current"),
)
metric(
key + ".peak",
"Пик памяти",
"Приложения",
name,
"bytes",
self.number(base / "memory.peak"),
)
fields = dict(
line.split() for line in (self.text(base / "cpu.stat") or "").splitlines()
)
rate = delta(
key + ".cpu", float(fields["usage_usec"]) if "usage_usec" in fields else None
)
metric(
key + ".cpu",
"Загрузка CPU",
"Приложения",
name,
"%",
None if rate is None else rate / 10000 / max(cpus, 1),
)
fields = dict(
line.split() for line in (self.text(base / "memory.events") or "").splitlines()
)
metric(
key + ".oom",
"Убийств из-за нехватки памяти",
"Приложения",
name,
"",
float(fields["oom_kill"]) if "oom_kill" in fields else None,
)
self.previous = counters
# A bounded hardware inventory protects the control-plane payload budget.
keys = list(values)[:384]
self.definitions = {key: definitions[key] for key in keys}
return {key: values[key] for key in keys}
+13
View File
@@ -0,0 +1,13 @@
CREATE EXTENSION IF NOT EXISTS timescaledb;
CREATE TABLE IF NOT EXISTS monitor_identity (singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton), source_id text NOT NULL);
INSERT INTO monitor_identity(singleton,source_id) VALUES(true,gen_random_uuid()::text) ON CONFLICT DO NOTHING;
CREATE TABLE IF NOT EXISTS monitor_samples (
seq bigint GENERATED ALWAYS AS IDENTITY,
observed_at timestamptz NOT NULL,
payload jsonb NOT NULL,
PRIMARY KEY(observed_at,seq)
);
SELECT create_hypertable('monitor_samples',by_range('observed_at',INTERVAL '1 hour'),if_not_exists=>true);
CREATE INDEX IF NOT EXISTS monitor_samples_cursor ON monitor_samples(seq);
CREATE TABLE IF NOT EXISTS monitor_definitions(singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton),payload jsonb NOT NULL);
REVOKE ALL ON DATABASE mission_core_monitor FROM PUBLIC;
+118
View File
@@ -0,0 +1,118 @@
"""Bounded local Timescale archive; transport acknowledgements never delete data."""
import json
import shutil
from contextlib import contextmanager
from datetime import UTC, datetime
SCHEMA = "missioncore.node-system-monitor/v1"
MAX_BATCH_BYTES = 196608
DATABASE_BUDGET = 2 * 1024**3
class Archive:
def __init__(self):
from psycopg2.pool import ThreadedConnectionPool
self.pool = ThreadedConnectionPool(
1,
4,
host="/run/mission-core-monitor-db",
port=5433,
dbname="mission_core_monitor",
user="mission-core-monitor",
connect_timeout=3,
options="-c statement_timeout=3000",
)
try:
with self.connection() as db, db.cursor() as cursor:
cursor.execute("SELECT source_id FROM monitor_identity")
self.source = cursor.fetchone()[0]
except Exception:
self.pool.closeall()
raise
self.definitions = None
self.database_bytes = None
@contextmanager
def connection(self):
db = self.pool.getconn()
try:
yield db
db.commit()
except BaseException:
db.rollback()
raise
finally:
self.pool.putconn(db)
def append(self, sample, definitions):
# This reserve belongs to telemetry, never to acquisition storage.
if shutil.disk_usage("/var/lib/mission-core-monitor").free < 2 * 1024**3:
raise RuntimeError("disk-reserve")
if self.database_bytes is None or self.database_bytes >= DATABASE_BUDGET:
raise RuntimeError("archive-budget")
payload = json.dumps(sample, separators=(",", ":"), allow_nan=False)
if len(payload.encode()) > 32768:
raise ValueError("sample-budget")
with self.connection() as db, db.cursor() as cursor:
if definitions != self.definitions:
cursor.execute(
"INSERT INTO monitor_definitions VALUES(true,%s) ON CONFLICT(singleton) "
"DO UPDATE SET payload=excluded.payload",
(json.dumps(definitions),),
)
cursor.execute(
"INSERT INTO monitor_samples(observed_at,payload) VALUES(%s,%s) RETURNING seq",
(datetime.fromtimestamp(sample["at"], UTC), payload),
)
seq = cursor.fetchone()[0]
self.definitions = definitions
return {**sample, "seq": seq}
def batch(self, after):
with self.connection() as db, db.cursor() as cursor:
cursor.execute(
"SELECT seq,payload FROM monitor_samples WHERE seq>%s ORDER BY seq LIMIT 32",
(after,),
)
rows = []
size = 0
for seq, payload in cursor.fetchall():
record = {**payload, "seq": seq}
size += len(json.dumps(record).encode())
if size > MAX_BATCH_BYTES:
break
rows.append(record)
cursor.execute("SELECT min(seq),max(seq),min(observed_at) FROM monitor_samples")
first, last, oldest = cursor.fetchone()
return dict(
schema=SCHEMA,
source_id=self.source,
samples=rows,
first_seq=first,
last_seq=last,
retained_since=oldest.timestamp() if oldest else None,
)
def maintain(self):
with self.connection() as db, db.cursor() as cursor:
cursor.execute("SELECT drop_chunks('monitor_samples',INTERVAL '7 days')")
cursor.execute("SELECT pg_database_size(current_database())")
size = cursor.fetchone()[0]
if size >= DATABASE_BUDGET:
# Drop one oldest completed chunk per pass, retaining the live hour.
cursor.execute(
"SELECT min(range_end) FROM timescaledb_information.chunks "
"WHERE hypertable_name='monitor_samples' AND range_end<NOW()"
)
boundary = cursor.fetchone()[0]
if boundary:
cursor.execute(
"SELECT drop_chunks('monitor_samples',older_than=>%s::timestamptz)",
(boundary,),
)
cursor.execute("SELECT pg_database_size(current_database())")
size = cursor.fetchone()[0]
self.database_bytes = size
return size
+7 -2
View File
@@ -11,7 +11,7 @@ import sys
ROOT = Path(__file__).resolve().parents[1]
VERSION = "0.8.14"
VERSION = "0.8.15"
sys.path.insert(0, str(ROOT.parents[1] / "scripts/packaging"))
from debian import package
@@ -43,7 +43,7 @@ Architecture: amd64
Maintainer: NODE.DC local build <noreply@example.invalid>
Section: admin
Priority: optional
Depends: adduser, systemd, python3, python3-gi, gir1.2-gtk-3.0, gir1.2-webkit2-4.1, pkexec, polkitd, ca-certificates, hicolor-icon-theme, network-manager, iproute2
Depends: adduser, systemd, python3, python3-gi, gir1.2-gtk-3.0, gir1.2-webkit2-4.1, pkexec, polkitd, ca-certificates, hicolor-icon-theme, network-manager, iproute2, python3-psycopg2, postgresql-16 (>= 16.15), timescaledb-2-oss-postgresql-16 (= 2.29.2~ubuntu24.04-1615)
Description: Mission Core onboard computer configuration
Local graphical setup, host inventory, SSH access and persistent node identity.
""".encode()
@@ -67,8 +67,13 @@ Description: Mission Core onboard computer configuration
("environment_helper.py", "usr/lib/mission-core-node/environment_helper.py", 0o644),
("mission-core-node-environment.service", "usr/lib/systemd/system/mission-core-node-environment.service", 0o644),
("60-environment.conf", "usr/share/mission-core-node/60-environment.conf", 0o644),
("setup-monitor", "usr/lib/mission-core-node/setup-monitor", 0o755),
("mission-core-node-monitor.service", "usr/lib/systemd/system/mission-core-node-monitor.service", 0o644),
]:
files.append((path, (p / source).read_bytes(), mode))
for path in sorted((ROOT / 'monitor').iterdir()):
if path.suffix in {'.py', '.sql'}:
files.append(('usr/lib/mission-core-node/monitor/' + path.name, path.read_bytes(), 0o644))
files.append(("usr/share/mission-core-node/environment-profile.json", (ROOT / "internal/node/environment-profile.json").read_bytes(), 0o644))
for name in ("realsense_prepare.py", "realsense_iio_access.py"):
files.append(("usr/lib/mission-core-node/" + name, (p / name).read_bytes(), 0o644))
@@ -0,0 +1,104 @@
#!/usr/bin/env python3
"""Stage verified Timescale OSS Ubuntu packages, without APT/system mutation."""
import argparse
import gzip
import hashlib
import json
import subprocess
import urllib.request
from pathlib import Path
BASE = "https://packagecloud.io/timescale/timescaledb/ubuntu/"
VERSION = "2.29.2"
def fetch(url, limit):
with urllib.request.urlopen(url, timeout=30) as response:
data = response.read(limit + 1)
if len(data) > limit:
raise ValueError("Download budget exceeded")
return data
def main():
parser = argparse.ArgumentParser()
parser.add_argument("destination", type=Path)
args = parser.parse_args()
root = args.destination
root.mkdir(parents=True, exist_ok=True)
key = root / "timescale.asc"
key.write_bytes(fetch("https://packagecloud.io/timescale/timescaledb/gpgkey", 65536))
keyring = root / "timescale.gpg"
subprocess.run(
["gpg", "--batch", "--yes", "--dearmor", "--output", str(keyring), str(key)], check=True
)
release = root / "InRelease"
release.write_bytes(fetch(BASE + "dists/noble/InRelease", 1048576))
subprocess.run(["gpgv", "--keyring", str(keyring.resolve()), str(release)], check=True)
lines = release.read_text().splitlines()
hashes = {}
inside = False
for line in lines:
if line == "SHA256:":
inside = True
continue
if inside and not line.startswith(" "):
break
if inside:
sha, size, path = line.split()
hashes[path] = (sha, int(size))
sha, size = hashes["main/binary-amd64/Packages.gz"]
compressed = fetch(BASE + "dists/noble/main/binary-amd64/Packages.gz", 16000000)
if len(compressed) != size or hashlib.sha256(compressed).hexdigest() != sha:
raise ValueError("Signed package index mismatch")
(root / "Packages.gz").write_bytes(compressed)
wanted = {
name: VERSION + "~ubuntu24.04-1615"
for name in ("timescaledb-2-oss-postgresql-16", "timescaledb-2-loader-postgresql-16")
}
artifacts = []
for block in gzip.decompress(compressed).decode().split("\n\n"):
fields = dict(
line.split(": ", 1)
for line in block.splitlines()
if ": " in line and not line.startswith(" ")
)
if (
fields.get("Package") not in wanted
or fields.get("Version") != wanted[fields["Package"]]
):
continue
path = fields["Filename"]
if not path.startswith("pool/") or ".." in path.split("/"):
raise ValueError("Invalid package path")
data = fetch(BASE + path, 64000000)
if hashlib.sha256(data).hexdigest() != fields["SHA256"]:
raise ValueError("Package hash mismatch")
target = root / Path(path).name
target.write_bytes(data)
artifacts.append(
dict(
package=fields["Package"],
version=fields["Version"],
file=target.name,
sha256=fields["SHA256"],
bytes=len(data),
depends=fields["Depends"],
)
)
if len(artifacts) != len(wanted):
raise ValueError("Pinned package not found")
manifest = dict(
source=BASE,
version=VERSION,
key_sha256=hashlib.sha256(key.read_bytes()).hexdigest(),
inrelease_sha256=hashlib.sha256(release.read_bytes()).hexdigest(),
artifacts=artifacts,
)
(root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
print(json.dumps(manifest, indent=2))
if __name__ == "__main__":
main()
@@ -0,0 +1,56 @@
#!/bin/sh
# Run by the owner-facing release installer after hash verification. Scope is
# this product's packages and its dedicated cluster; never remove another DB.
set -eu
test "$(id -u)" = 0
mc_release_dir=$1
cd "$mc_release_dir"
sha256sum --check SHA256SUMS
mc_pg_guard=/etc/postgresql-common/createcluster.d/60-mission-core-install.conf
mc_created_guard=0
cleanup() {
if [ "$mc_created_guard" = 1 ]; then
if [ "$(cat "$mc_pg_guard")" = 'create_main_cluster = false' ]; then
rm "$mc_pg_guard"
fi
fi
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' HUP TERM
if ! dpkg-query -W -f='${Status}' postgresql-16 2>/dev/null | grep -qx 'install ok installed'; then
# postgresql-common explicitly supports this drop-in. Suppress only automatic
# creation of the unrelated default main cluster during the first install.
test ! -e "$mc_pg_guard"
install -d -m 0755 /etc/postgresql-common/createcluster.d
printf '%s\n' 'create_main_cluster = false' > "$mc_pg_guard"
chmod 0644 "$mc_pg_guard"
mc_created_guard=1
fi
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-remove \
"$mc_release_dir/timescaledb-2-loader-postgresql-16_2.29.2~ubuntu24.04-1615_amd64.deb" \
"$mc_release_dir/timescaledb-2-oss-postgresql-16_2.29.2~ubuntu24.04-1615_amd64.deb" \
"$mc_release_dir/mission-core-node_0.8.15_amd64.deb" \
"$mc_release_dir/mission-core-xgrids-k1_0.1.14_amd64.deb"
systemctl is-active --quiet mission-core-node.service
systemctl is-active --quiet mission-core-k1.service
systemctl is-active --quiet postgresql@16-ndc-monitor.service
systemctl is-active --quiet mission-core-node-monitor.service
runuser -u mission-core-monitor -- /usr/bin/python3 - <<'PY'
import http.client,json,socket,time
deadline=time.monotonic()+20
while True:
c=http.client.HTTPConnection('monitor',timeout=3)
c.sock=socket.socket(socket.AF_UNIX);c.sock.settimeout(3)
try:
c.sock.connect('/run/mission-core-monitor/monitor.sock')
c.request('GET','/status');r=c.getresponse();v=json.load(r)
if r.status==200 and v['storage']=='ready' and v['latest']['seq']>0:break
except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException):
pass
finally:c.close()
if time.monotonic()>deadline:raise SystemExit('Local telemetry did not become ready')
time.sleep(1)
print('Local telemetry: ready')
PY
printf '%s\n' 'Mission Core Node R18: package and local archive checks passed.'
+6 -1
View File
@@ -9,6 +9,7 @@ import re
import socket
import subprocess
import threading
import syslog
from urllib.parse import urlsplit
import gi
@@ -261,11 +262,15 @@ class NodeApplication(Gtk.Application):
return False
def load_failed(self, _view, _event, uri, _error):
syslog.openlog('mission-core-node-ui')
syslog.syslog(syslog.LOG_ERR, 'node-ui-load-failed')
if local_url(uri):
self.problem("Служба ноды недоступна. Повторно откройте приложение после восстановления службы.")
return True
def process_failed(self, *_):
def process_failed(self, _view, reason):
syslog.openlog('mission-core-node-ui')
syslog.syslog(syslog.LOG_ERR, 'node-ui-process-terminated reason=' + str(int(reason)))
self.problem("Окно приложения остановилось. Закройте и повторно откройте Mission Core Node. Служба борта продолжает работать отдельно.")
def deny_permission(self, _view, permission):
@@ -0,0 +1,38 @@
[Unit]
Description=Mission Core onboard system telemetry collector
After=postgresql@16-ndc-monitor.service
Wants=postgresql@16-ndc-monitor.service
[Service]
Type=simple
User=mission-core-monitor
Group=mission-core-node
SupplementaryGroups=systemd-journal
ExecStart=/usr/bin/python3 /usr/lib/mission-core-node/monitor/collector.py
RuntimeDirectory=mission-core-monitor
RuntimeDirectoryMode=0750
StateDirectory=mission-core-monitor
StateDirectoryMode=0750
UMask=0007
Restart=on-failure
RestartSec=5
MemoryHigh=96M
MemoryMax=128M
CPUQuota=15%
TasksMax=24
Nice=10
IOSchedulingClass=idle
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
RestrictAddressFamilies=AF_UNIX
CapabilityBoundingSet=
RestrictSUIDSGID=yes
[Install]
WantedBy=multi-user.target
+1
View File
@@ -23,6 +23,7 @@ case "$1" in
fi
rm -f /run/mission-core-node-k1-upgrade-active
fi
/usr/lib/mission-core-node/setup-monitor
fi
;;
esac
+1
View File
@@ -34,6 +34,7 @@ if [ "$1" = install ] || [ "$1" = upgrade ]; then
(umask 077; : > /run/mission-core-node-k1-upgrade-active)
systemctl stop mission-core-k1.service
fi
systemctl stop mission-core-node-monitor.service 2>/dev/null || true
fi
. /etc/os-release
if [ "${ID:-}" != ubuntu ] || [ "${VERSION_ID:-}" != 24.04 ]; then
+4
View File
@@ -52,6 +52,10 @@ case "$1" in
systemctl disable mission-core-realsense.service || true
systemctl stop mission-core-node.service
systemctl disable mission-core-node.service
systemctl stop mission-core-node-monitor.service
systemctl disable mission-core-node-monitor.service
systemctl stop postgresql@16-ndc-monitor.service
systemctl disable postgresql@16-ndc-monitor.service
fi
;;
esac
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
# Fixed local database bootstrap; no supplied SQL, credentials or network address.
set -eu
test "$(id -u)" = 0
if ! getent passwd mission-core-monitor >/dev/null; then
adduser --system --group --home /var/lib/mission-core-monitor --no-create-home --disabled-login mission-core-monitor
fi
install -d -m 0750 -o mission-core-monitor -g mission-core-node /var/lib/mission-core-monitor
install -d -m 0755 -o postgres -g postgres /run/mission-core-monitor-db
install -d -m 0755 /etc/tmpfiles.d
printf '%s\n' 'd /run/mission-core-monitor-db 0755 postgres postgres -' > /etc/tmpfiles.d/mission-core-monitor.conf
if [ ! -d /etc/postgresql/16/ndc-monitor ]; then
pg_createcluster 16 ndc-monitor --port=5433 --socketdir=/run/mission-core-monitor-db --datadir=/var/lib/mission-core-monitor-db --start-conf=auto -- --auth-local=peer --auth-host=reject
fi
install -d -m 0755 /etc/postgresql/16/ndc-monitor/conf.d
cat > /etc/postgresql/16/ndc-monitor/conf.d/60-mission-core-monitor.conf <<'CONF'
listen_addresses = ''
unix_socket_directories = '/run/mission-core-monitor-db'
shared_preload_libraries = 'timescaledb'
shared_buffers = '32MB'
work_mem = '2MB'
maintenance_work_mem = '16MB'
max_connections = 12
max_worker_processes = 4
max_parallel_workers = 0
timescaledb.max_background_workers = 2
timescaledb.telemetry_level = 'off'
max_wal_size = '128MB'
min_wal_size = '32MB'
temp_file_limit = '32MB'
statement_timeout = '5s'
log_statement = 'none'
log_min_error_statement = 'panic'
CONF
install -d -m 0755 /etc/systemd/system/postgresql@16-ndc-monitor.service.d
cat > /etc/systemd/system/postgresql@16-ndc-monitor.service.d/60-mission-core-monitor.conf <<'CONF'
[Service]
ExecStartPre=+/usr/bin/install -d -m 0755 -o postgres -g postgres /run/mission-core-monitor-db
MemoryHigh=256M
MemoryMax=384M
CPUQuota=25%
TasksMax=32
Nice=10
CONF
systemctl daemon-reload
systemctl restart postgresql@16-ndc-monitor.service
runuser -u postgres -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d postgres <<'SQL'
SELECT 'CREATE ROLE "mission-core-monitor" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE' WHERE NOT EXISTS(SELECT FROM pg_roles WHERE rolname='mission-core-monitor') \gexec
SELECT 'CREATE DATABASE mission_core_monitor OWNER "mission-core-monitor"' WHERE NOT EXISTS(SELECT FROM pg_database WHERE datname='mission_core_monitor') \gexec
SQL
runuser -u postgres -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d mission_core_monitor -c 'CREATE EXTENSION IF NOT EXISTS timescaledb'
runuser -u mission-core-monitor -- psql -X -v ON_ERROR_STOP=1 -h /run/mission-core-monitor-db -p 5433 -d mission_core_monitor -f /usr/lib/mission-core-node/monitor/schema.sql
systemctl enable mission-core-node-monitor.service
systemctl restart mission-core-node-monitor.service
+1 -1
View File
@@ -3,5 +3,5 @@ import {createIsolatedRerunHost} from '../../../control-station/src/components/r
import {SensorWorkspace} from '../../../../packages/sensor-ui/src/SensorWorkspace';
import type {SensorTransport} from '../../../../packages/sensor-ui/src/contracts';
import {request} from './api';
const transport:SensorTransport={enrollment:{state:()=>request("/api/devices/enrollment"),submit:value=>request("/api/devices/enrollment/operations","POST",value),operation:id=>request("/api/devices/enrollment/operations/"+encodeURIComponent(id))},subscribe:(receive,unavailable)=>{const events=new EventSource('/api/devices/events');events.onmessage=e=>{try{receive(JSON.parse(e.data));}catch{unavailable();}};events.onerror=unavailable;return()=>events.close();},inventory:()=>request('/api/devices'),submit:value=>request('/api/devices/operations','POST',value),operation:id=>request('/api/devices/operations/'+encodeURIComponent(id))};
const transport:SensorTransport={localPreview:{open:(command,signal)=>fetch("/api/devices/preview",{method:"POST",credentials:"same-origin",headers:{"Content-Type":"application/json"},body:JSON.stringify(command),signal}),read:(peer,after,signal)=>fetch("/api/devices/preview/read",{method:"POST",credentials:"same-origin",headers:{"Content-Type":"application/json"},body:JSON.stringify({peer_id:peer,after}),signal})},enrollment:{state:()=>request("/api/devices/enrollment"),submit:value=>request("/api/devices/enrollment/operations","POST",value),operation:id=>request("/api/devices/enrollment/operations/"+encodeURIComponent(id))},subscribe:(receive,unavailable)=>{const events=new EventSource('/api/devices/events');events.onmessage=e=>{try{receive(JSON.parse(e.data));}catch{unavailable();}};events.onerror=unavailable;return()=>events.close();},inventory:()=>request('/api/devices'),submit:value=>request('/api/devices/operations','POST',value),operation:id=>request('/api/devices/operations/'+encodeURIComponent(id))};
export function NodeSensors(){return <SensorWorkspace contributions={[xgridsK1SensorUi]} createRerunHost={createIsolatedRerunHost} transport={transport}/>;}
@@ -0,0 +1,20 @@
import {Component,type ReactNode} from 'react';
import {Button,SettingsCard} from '@nodedc/ui-react';
export function runtimeDiagnostic(error:unknown,code='ui-error') {
const kind=error instanceof Error&&/^[A-Za-z]{1,48}$/.test(error.name)?error.name:'Error';
// Never send exception messages, URLs, form values or arbitrary stack text.
const locations=error instanceof Error?(error.stack??'').match(/[A-Za-z0-9_-]+\.js:\d+:\d+/g)?.slice(0,8)??[]:[];
void fetch('/api/monitor/events',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({code,kind,locations}),keepalive:true}).catch(()=>{});
}
export function observeRuntime(){
window.addEventListener('error',event=>runtimeDiagnostic(event.error));
window.addEventListener('unhandledrejection',event=>runtimeDiagnostic(event.reason,'ui-rejection'));
}
export class RuntimeBoundary extends Component<{children:ReactNode;resetKey?:string},{failed:boolean}> {
state={failed:false};
static getDerivedStateFromError(){return {failed:true};}
componentDidCatch(error:Error){runtimeDiagnostic(error,'ui-render-error');}
componentDidUpdate(previous:Readonly<{children:ReactNode;resetKey?:string}>){if(this.state.failed&&previous.resetKey!==this.props.resetKey)this.setState({failed:false});}
render(){return this.state.failed?<SettingsCard title="Не удалось открыть представление" description="Вернитесь к устройствам или повторите открытие."><Button onClick={()=>this.setState({failed:false})}>Открыть повторно</Button></SettingsCard>:this.props.children;}
}
+4 -2
View File
@@ -18,6 +18,7 @@ import "./node.css";
import { NodeSensors } from "./NodeSensors";
import { Home, HomeSettings } from "./Home";
import { usePresentation } from "./usePresentation";
import {RuntimeBoundary,observeRuntime} from './RuntimeBoundary';
function App() {
const node = useNode();
@@ -53,7 +54,7 @@ function App() {
contexts={value ? [{ id: "board", label: value.name, description: value.host.hostname, icon: <Icon name="activity" />, onSelect: () => openView("overview") }] : []}
items={views.filter(item => item.root === root).map(item => ({ id: item.id, label: item.label, icon: <Icon name={item.icon} /> }))} activeId={workspace.activeView ?? undefined} onItemChange={id => openView(id as ViewId)} footer={<span>Mission Core Node · {value?.version}</span>} />}
content={currentView && <ApplicationPanel title={currentView.label} eyebrow={currentRoot.label} expanded={workspace.contentExpanded} onExpandedChange={workspace.setContentExpanded} onClose={workspace.closeView}
utilityActions={[...(workspace.activeView === "ssh" ? [{ label: "Добавить доверенное устройство", icon: "plus" as const, onClick: () => setAdding(true) }] : []), { label: "Обновить сведения", icon: "refresh", disabled: pending || environment.running, onClick: refreshAll }]}>{content}</ApplicationPanel>}
utilityActions={[...(workspace.activeView === "ssh" ? [{ label: "Добавить доверенное устройство", icon: "plus" as const, onClick: () => setAdding(true) }] : []), { label: "Обновить сведения", icon: "refresh", disabled: pending || environment.running, onClick: refreshAll }]}><RuntimeBoundary resetKey={workspace.activeView??undefined}>{content}</RuntimeBoundary></ApplicationPanel>}
stage={value ? <Home page={presentation.settings.pages.home} openView={openView} /> : <div className="node-stage" aria-busy={pending}>
<SettingsCard className="node-entry" title={pending ? "Подключаемся к ноде" : locked ? "Вход в Mission Core Node" : "Нода недоступна"}>
{pending ? <ActivityIndicator label="Получение сведений о ноде" /> : <p className="node-note">{locked ? "Подтвердите доступ в системном окне." : "Не удалось связаться со службой. Повторите подключение."}</p>}
@@ -64,4 +65,5 @@ function App() {
<HomeSettings open={!!value && settingsOpen} onClose={() => setSettingsOpen(false)} presentation={presentation} />
</>;
}
createRoot(document.getElementById("root")!).render(<App />);
observeRuntime();
createRoot(document.getElementById("root")!).render(<RuntimeBoundary><App /></RuntimeBoundary>);