Fix onboard WebKit preview and archive board telemetry locally
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/python3
|
||||
"""Independent one-second collector, local archive and private Node read adapter."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from linux_metrics import LinuxMetrics
|
||||
from storage import SCHEMA, Archive
|
||||
|
||||
SOCKET = Path("/run/mission-core-monitor/monitor.sock")
|
||||
|
||||
|
||||
class Collector:
|
||||
def __init__(self):
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.latest = None
|
||||
self.definitions = {}
|
||||
self.events = deque(maxlen=64)
|
||||
self.storage = "starting"
|
||||
self.database_bytes = None
|
||||
self.archive = None
|
||||
self.boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
||||
|
||||
def event(self, value):
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) - {"code", "kind", "locations"}
|
||||
or value.get("code") not in {"ui-error", "ui-rejection", "ui-render-error"}
|
||||
or not isinstance(value.get("kind"), str)
|
||||
or not re.fullmatch("[A-Za-z]{1,48}", value["kind"])
|
||||
or not isinstance(value.get("locations", []), list)
|
||||
or len(value.get("locations", [])) > 8
|
||||
or any(
|
||||
not isinstance(v, str)
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]{1,96}\.js:[0-9]{1,9}:[0-9]{1,9}", v)
|
||||
for v in value.get("locations", [])
|
||||
)
|
||||
):
|
||||
raise ValueError("invalid-event")
|
||||
with self.lock:
|
||||
if len(self.events) < 64:
|
||||
self.events.append({**value, "at": time.time()})
|
||||
|
||||
def run(self):
|
||||
metrics = LinuxMetrics()
|
||||
maintenance = 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
values = metrics.sample()
|
||||
with self.lock:
|
||||
events = list(self.events)
|
||||
sample = dict(
|
||||
at=time.time(), boot_id=self.boot, uptime=started, values=values, events=events
|
||||
)
|
||||
if self.archive is None:
|
||||
self.archive = Archive()
|
||||
# Maintenance also runs while writes are paused by a disk budget.
|
||||
if started - maintenance > 60:
|
||||
self.database_bytes = self.archive.maintain()
|
||||
maintenance = started
|
||||
record = self.archive.append(sample, metrics.definitions)
|
||||
with self.lock:
|
||||
for _ in events:
|
||||
self.events.popleft()
|
||||
self.latest = record
|
||||
self.definitions = metrics.definitions
|
||||
self.storage = "ready"
|
||||
except Exception as error:
|
||||
logging.warning("monitor sample failed class=%s", type(error).__name__)
|
||||
with self.lock:
|
||||
self.storage = (
|
||||
str(error)
|
||||
if str(error) in {"disk-reserve", "archive-budget"}
|
||||
else "unavailable"
|
||||
)
|
||||
self.stop.wait(max(0.05, 1 - (time.monotonic() - started)))
|
||||
|
||||
def status(self):
|
||||
with self.lock:
|
||||
return dict(
|
||||
schema=SCHEMA,
|
||||
source_id=self.archive.source if self.archive else None,
|
||||
latest=self.latest,
|
||||
definitions=self.definitions,
|
||||
storage=self.storage,
|
||||
database_bytes=self.database_bytes,
|
||||
sample_interval_seconds=1,
|
||||
retention_days=7,
|
||||
database_budget_bytes=2 * 1024**3,
|
||||
free_reserve_bytes=2 * 1024**3,
|
||||
)
|
||||
|
||||
|
||||
class Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer):
|
||||
daemon_threads = True
|
||||
block_on_close = False
|
||||
|
||||
def __init__(self, collector):
|
||||
self.collector = collector
|
||||
self.capacity = threading.BoundedSemaphore(6)
|
||||
super().__init__(str(SOCKET), Handler)
|
||||
|
||||
def process_request(self, request, address):
|
||||
if not self.capacity.acquire(False):
|
||||
request.close()
|
||||
return
|
||||
request.settimeout(5)
|
||||
super().process_request(request, address)
|
||||
|
||||
def process_request_thread(self, request, address):
|
||||
try:
|
||||
super().process_request_thread(request, address)
|
||||
finally:
|
||||
self.capacity.release()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def reply(self, value, status=200):
|
||||
payload = json.dumps(value, allow_nan=False, separators=(",", ":")).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self):
|
||||
path = urlsplit(self.path)
|
||||
try:
|
||||
if path.path == "/status":
|
||||
self.reply(self.server.collector.status())
|
||||
elif path.path == "/batch":
|
||||
after = int(parse_qs(path.query).get("after", ["0"])[0])
|
||||
if not 0 <= after <= 2**53 - 1:
|
||||
raise ValueError()
|
||||
archive = self.server.collector.archive
|
||||
if archive is None:
|
||||
self.reply({"error": "Archive unavailable"}, 503)
|
||||
else:
|
||||
self.reply(archive.batch(after))
|
||||
else:
|
||||
self.reply({"error": "Unknown request"}, 404)
|
||||
except Exception:
|
||||
self.reply({"error": "Archive unavailable"}, 503)
|
||||
|
||||
def do_POST(self):
|
||||
try:
|
||||
size = int(self.headers.get("Content-Length", "0"))
|
||||
if self.path != "/events" or not 0 < size <= 2048:
|
||||
raise ValueError()
|
||||
self.server.collector.event(json.loads(self.rfile.read(size)))
|
||||
self.reply({"ok": True})
|
||||
except (ValueError, TypeError):
|
||||
self.reply({"error": "Invalid event"}, 400)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
os.umask(0o007)
|
||||
collector = Collector()
|
||||
from journal_events import watch
|
||||
|
||||
threading.Thread(
|
||||
target=watch, args=(collector,), name="node-system-events", daemon=True
|
||||
).start()
|
||||
SOCKET.unlink(missing_ok=True)
|
||||
server = Server(collector)
|
||||
worker = threading.Thread(target=collector.run, name="node-system-sampler", daemon=True)
|
||||
worker.start()
|
||||
|
||||
def stop(*_):
|
||||
collector.stop.set()
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.5)
|
||||
finally:
|
||||
collector.stop.set()
|
||||
server.server_close()
|
||||
worker.join(5)
|
||||
SOCKET.unlink(missing_ok=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Classify bounded journal records; never persist free-form messages or payloads."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
RULES = (
|
||||
("system-oom", r"out of memory|oom-kill|killed process"),
|
||||
("gpu-reset", r"GPU HANG|GPU reset|Resetting.*(gpu|chip)|i915.*(hang|reset|error)"),
|
||||
("disk-error", r"I/O error|EXT4-fs error|nvme.*(timeout|reset)|ata.*failed command"),
|
||||
("usb-disconnected", r"USB disconnect"),
|
||||
("usb-error", r"usb.*(error -|unable to enumerate|reset.*device)"),
|
||||
("service-failed", r"Failed with result|Main process exited|Failed to start"),
|
||||
("ui-process-exit", r"node-ui-process-terminated"),
|
||||
("ui-load-failed", r"node-ui-load-failed"),
|
||||
)
|
||||
|
||||
|
||||
def classify(record):
|
||||
message = record.get("MESSAGE")
|
||||
if not isinstance(message, str):
|
||||
return None
|
||||
code = next((code for code, pattern in RULES if re.search(pattern, message, re.I)), None)
|
||||
if code is None:
|
||||
return None
|
||||
try:
|
||||
at = int(record["__REALTIME_TIMESTAMP"]) / 1_000_000
|
||||
except (KeyError, ValueError, TypeError):
|
||||
return None
|
||||
return dict(
|
||||
code=code,
|
||||
kind="Kernel" if record.get("_TRANSPORT") == "kernel" else "System",
|
||||
at=at,
|
||||
locations=[],
|
||||
)
|
||||
|
||||
|
||||
def watch(collector):
|
||||
args = [
|
||||
"/usr/bin/journalctl",
|
||||
"--follow",
|
||||
"--lines=0",
|
||||
"--output=json",
|
||||
"--no-pager",
|
||||
"_TRANSPORT=kernel",
|
||||
"+",
|
||||
"SYSLOG_IDENTIFIER=mission-core-node-ui",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=mission-core-node.service",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=mission-core-k1.service",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=mission-core-node-monitor.service",
|
||||
"+",
|
||||
"_SYSTEMD_UNIT=postgresql@16-ndc-monitor.service",
|
||||
]
|
||||
for unit in (
|
||||
"mission-core-node.service",
|
||||
"mission-core-k1.service",
|
||||
"mission-core-node-monitor.service",
|
||||
"postgresql@16-ndc-monitor.service",
|
||||
):
|
||||
args.extend(["+", "UNIT=" + unit])
|
||||
while not collector.stop.is_set():
|
||||
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
def retire(process=process):
|
||||
while process.poll() is None:
|
||||
if collector.stop.wait(1):
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
return
|
||||
|
||||
guard = threading.Thread(target=retire, daemon=True)
|
||||
guard.start()
|
||||
try:
|
||||
while not collector.stop.is_set():
|
||||
line = process.stdout.readline(65537)
|
||||
if not line:
|
||||
break
|
||||
if len(line) > 65536:
|
||||
while line and not line.endswith(b"\n"):
|
||||
line = process.stdout.readline(65537)
|
||||
continue
|
||||
try:
|
||||
event = classify(json.loads(line))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if event:
|
||||
with collector.lock:
|
||||
if len(collector.events) < 64:
|
||||
collector.events.append(event)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
process.stdout.close()
|
||||
collector.stop.wait(10)
|
||||
@@ -0,0 +1,353 @@
|
||||
"""Read-only Linux counters. Missing or reset counters never manufacture zero."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class LinuxMetrics:
|
||||
def __init__(self, root=Path("/"), clock=time.monotonic):
|
||||
self.root, self.clock = Path(root), clock
|
||||
self.previous = {}
|
||||
self.previous_time = None
|
||||
self.definitions = {}
|
||||
|
||||
def text(self, path):
|
||||
try:
|
||||
return (self.root / str(path).lstrip("/")).read_text()[:262144].strip()
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
|
||||
def number(self, path):
|
||||
try:
|
||||
return float(self.text(path))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def paths(self, pattern):
|
||||
return sorted(self.root.glob(pattern))[:128]
|
||||
|
||||
def sample(self):
|
||||
now = self.clock()
|
||||
elapsed = None if self.previous_time is None else now - self.previous_time
|
||||
self.previous_time = now
|
||||
values, definitions, counters = {}, {}, {}
|
||||
|
||||
def metric(
|
||||
key, label, group, resource, unit, value, reason="Счётчик недоступен в этой системе"
|
||||
):
|
||||
values[key] = None if value is None else round(value, 3)
|
||||
definitions[key] = dict(
|
||||
label=label,
|
||||
group=group,
|
||||
resource=resource,
|
||||
unit=unit,
|
||||
reason=reason if value is None else None,
|
||||
)
|
||||
|
||||
def delta(key, value):
|
||||
if value is None:
|
||||
return None
|
||||
counters[key] = value
|
||||
previous = self.previous.get(key)
|
||||
return (
|
||||
None
|
||||
if previous is None or value < previous or not elapsed or elapsed > 10
|
||||
else (value - previous) / elapsed
|
||||
)
|
||||
|
||||
stat = self.text("proc/stat") or ""
|
||||
cpus = 0
|
||||
for line in stat.splitlines():
|
||||
parts = line.split()
|
||||
if not re.fullmatch(r"cpu\d*", parts[0]):
|
||||
continue
|
||||
if parts[0] != "cpu":
|
||||
cpus += 1
|
||||
data = [int(v) for v in parts[1:9]]
|
||||
total = delta(parts[0] + ".total", sum(data))
|
||||
idle = delta(parts[0] + ".idle", data[3] + data[4])
|
||||
wait = delta(parts[0] + ".iowait", data[4])
|
||||
for name, label, value in [
|
||||
(
|
||||
"usage",
|
||||
"Загрузка",
|
||||
None if not total or idle is None else 100 * (1 - idle / total),
|
||||
),
|
||||
(
|
||||
"iowait",
|
||||
"Ожидание диска",
|
||||
None if not total or wait is None else 100 * wait / total,
|
||||
),
|
||||
]:
|
||||
metric(
|
||||
parts[0] + "." + name,
|
||||
label,
|
||||
"CPU",
|
||||
parts[0],
|
||||
"%",
|
||||
value,
|
||||
"Ожидаем два последовательных измерения",
|
||||
)
|
||||
if not cpus:
|
||||
metric("cpu.usage", "Загрузка", "CPU", "CPU", "%", None)
|
||||
memory = {}
|
||||
for line in (self.text("proc/meminfo") or "").splitlines():
|
||||
parts = line.replace(":", "").split()
|
||||
if len(parts) > 1:
|
||||
memory[parts[0]] = int(parts[1]) * 1024
|
||||
for key, label in [
|
||||
("MemTotal", "Всего"),
|
||||
("MemAvailable", "Доступно"),
|
||||
("SwapTotal", "Swap всего"),
|
||||
("SwapFree", "Swap свободно"),
|
||||
("Dirty", "Ожидает записи"),
|
||||
("Writeback", "Записывается"),
|
||||
]:
|
||||
metric("memory." + key, label, "RAM", "Память", "bytes", memory.get(key))
|
||||
total, available = memory.get("MemTotal"), memory.get("MemAvailable")
|
||||
metric(
|
||||
"memory.usage",
|
||||
"Использовано",
|
||||
"RAM",
|
||||
"Память",
|
||||
"%",
|
||||
None if not total or available is None else 100 * (1 - available / total),
|
||||
)
|
||||
for index, load in enumerate((self.text("proc/loadavg") or "").split()[:3]):
|
||||
metric(
|
||||
"load." + str(index),
|
||||
["За 1 минуту", "За 5 минут", "За 15 минут"][index],
|
||||
"CPU",
|
||||
"Load average",
|
||||
"",
|
||||
float(load),
|
||||
)
|
||||
for resource in ["cpu", "memory", "io"]:
|
||||
for line in (self.text("proc/pressure/" + resource) or "").splitlines():
|
||||
parts = line.split()
|
||||
fields = dict(v.split("=") for v in parts[1:])
|
||||
metric(
|
||||
"pressure." + resource + "." + parts[0],
|
||||
"Задержка за 10 секунд",
|
||||
"Ожидание ресурсов",
|
||||
resource + " · " + parts[0],
|
||||
"%",
|
||||
float(fields["avg10"]),
|
||||
)
|
||||
for interface in self.paths("sys/class/net/*"):
|
||||
name = interface.name
|
||||
for field, label in [
|
||||
("rx_bytes", "Приём"),
|
||||
("tx_bytes", "Передача"),
|
||||
("rx_errors", "Ошибки приёма"),
|
||||
("tx_errors", "Ошибки передачи"),
|
||||
("rx_dropped", "Потери приёма"),
|
||||
("tx_dropped", "Потери передачи"),
|
||||
]:
|
||||
raw = self.number(interface.relative_to(self.root) / "statistics" / field)
|
||||
metric(
|
||||
"net." + name + "." + field,
|
||||
label,
|
||||
"Сеть",
|
||||
name,
|
||||
"bytes/s" if field.endswith("bytes") else "1/s",
|
||||
delta("net." + name + "." + field, raw),
|
||||
)
|
||||
speed = self.number(interface.relative_to(self.root) / "speed")
|
||||
metric(
|
||||
"net." + name + ".speed",
|
||||
"Скорость соединения",
|
||||
"Сеть",
|
||||
name,
|
||||
"Mbit/s",
|
||||
speed if speed is not None and speed > 0 else None,
|
||||
)
|
||||
diskstats = (self.text("proc/diskstats") or "").splitlines()
|
||||
for line in diskstats:
|
||||
p = line.split()
|
||||
if (
|
||||
len(p) < 14
|
||||
or p[2].startswith(("loop", "ram"))
|
||||
or not (self.root / "sys/block" / p[2]).exists()
|
||||
):
|
||||
continue
|
||||
name = p[2]
|
||||
for index, label, unit, multiplier in [
|
||||
(5, "Чтение", "bytes/s", 512),
|
||||
(9, "Запись", "bytes/s", 512),
|
||||
(3, "Операции чтения", "1/s", 1),
|
||||
(7, "Операции записи", "1/s", 1),
|
||||
(12, "Занятость", "%", 0.1),
|
||||
]:
|
||||
key = "disk." + name + "." + str(index)
|
||||
rate = delta(key, int(p[index]))
|
||||
metric(key, label, "Диски", name, unit, None if rate is None else rate * multiplier)
|
||||
if self.root == Path("/"):
|
||||
mounts = set()
|
||||
for line in (self.text("proc/mounts") or "").splitlines():
|
||||
p = line.split()
|
||||
if len(p) < 3 or not p[0].startswith("/dev/") or p[1] in mounts:
|
||||
continue
|
||||
mount = p[1].replace("\\040", " ")
|
||||
mounts.add(mount)
|
||||
try:
|
||||
st = os.statvfs(mount)
|
||||
key = "fs." + str(len(mounts))
|
||||
metric(
|
||||
key + ".free",
|
||||
"Свободно",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"bytes",
|
||||
st.f_bavail * st.f_frsize,
|
||||
)
|
||||
metric(
|
||||
key + ".usage",
|
||||
"Использовано",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"%",
|
||||
100 * (1 - st.f_bfree / st.f_blocks) if st.f_blocks else None,
|
||||
)
|
||||
metric(
|
||||
key + ".inodes",
|
||||
"Использовано inode",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"%",
|
||||
100 * (1 - st.f_ffree / st.f_files) if st.f_files else None,
|
||||
)
|
||||
except OSError:
|
||||
metric(
|
||||
"fs." + str(len(mounts)) + ".free",
|
||||
"Свободно",
|
||||
"Хранилище",
|
||||
mount,
|
||||
"bytes",
|
||||
None,
|
||||
)
|
||||
for path in self.paths("sys/class/hwmon/hwmon*/temp*_input"):
|
||||
relative = path.relative_to(self.root)
|
||||
name = self.text(path.parent.relative_to(self.root) / "name") or path.parent.name
|
||||
raw = self.number(relative)
|
||||
metric(
|
||||
"temperature." + path.parent.name + "." + path.stem,
|
||||
"Температура",
|
||||
"Температура",
|
||||
name + " · " + path.stem,
|
||||
"°C",
|
||||
None if raw is None else raw / 1000,
|
||||
)
|
||||
for card in self.paths("sys/class/drm/card[0-9]"):
|
||||
base = card.relative_to(self.root)
|
||||
metric(
|
||||
"gpu." + card.name + ".usage",
|
||||
"Загрузка",
|
||||
"GPU",
|
||||
card.name,
|
||||
"%",
|
||||
self.number(base / "device/gpu_busy_percent"),
|
||||
"Драйвер не публикует общий счётчик загрузки GPU",
|
||||
)
|
||||
metric(
|
||||
"gpu." + card.name + ".frequency",
|
||||
"Частота",
|
||||
"GPU",
|
||||
card.name,
|
||||
"MHz",
|
||||
self.number(base / "gt_cur_freq_mhz"),
|
||||
)
|
||||
for field, label in [
|
||||
("mem_info_vram_used", "Видеопамять занята"),
|
||||
("mem_info_vram_total", "Видеопамять всего"),
|
||||
]:
|
||||
metric(
|
||||
"gpu." + card.name + "." + field,
|
||||
label,
|
||||
"GPU",
|
||||
card.name,
|
||||
"bytes",
|
||||
self.number(base / "device" / field),
|
||||
"Выделенная видеопамять или её счётчик недоступны",
|
||||
)
|
||||
for usb in self.paths("sys/bus/usb/devices/*"):
|
||||
if not (usb / "idVendor").exists():
|
||||
continue
|
||||
base = usb.relative_to(self.root)
|
||||
name = usb.name
|
||||
metric(
|
||||
"usb." + name + ".speed",
|
||||
"Скорость соединения",
|
||||
"USB",
|
||||
name,
|
||||
"Mbit/s",
|
||||
self.number(base / "speed"),
|
||||
)
|
||||
metric(
|
||||
"usb." + name + ".traffic",
|
||||
"Трафик",
|
||||
"USB",
|
||||
name,
|
||||
"bytes/s",
|
||||
None,
|
||||
"USB-драйвер не публикует счётчик трафика порта",
|
||||
)
|
||||
groups = self.paths("sys/fs/cgroup/system.slice/mission-core*.service")
|
||||
groups += self.paths(
|
||||
"sys/fs/cgroup/system.slice/system-postgresql.slice/postgresql@16-ndc-monitor.service"
|
||||
)
|
||||
groups += self.paths(
|
||||
"sys/fs/cgroup/user.slice/user-*.slice/user@*.service/app.slice/app-gnome-org.nodedc.MissionCoreNode-*.scope"
|
||||
)
|
||||
for path in groups:
|
||||
base = path.relative_to(self.root)
|
||||
name = path.name
|
||||
key = "service." + re.sub("[^A-Za-z0-9_.-]", "_", name)
|
||||
metric(
|
||||
key + ".memory",
|
||||
"Память",
|
||||
"Приложения",
|
||||
name,
|
||||
"bytes",
|
||||
self.number(base / "memory.current"),
|
||||
)
|
||||
metric(
|
||||
key + ".peak",
|
||||
"Пик памяти",
|
||||
"Приложения",
|
||||
name,
|
||||
"bytes",
|
||||
self.number(base / "memory.peak"),
|
||||
)
|
||||
fields = dict(
|
||||
line.split() for line in (self.text(base / "cpu.stat") or "").splitlines()
|
||||
)
|
||||
rate = delta(
|
||||
key + ".cpu", float(fields["usage_usec"]) if "usage_usec" in fields else None
|
||||
)
|
||||
metric(
|
||||
key + ".cpu",
|
||||
"Загрузка CPU",
|
||||
"Приложения",
|
||||
name,
|
||||
"%",
|
||||
None if rate is None else rate / 10000 / max(cpus, 1),
|
||||
)
|
||||
fields = dict(
|
||||
line.split() for line in (self.text(base / "memory.events") or "").splitlines()
|
||||
)
|
||||
metric(
|
||||
key + ".oom",
|
||||
"Убийств из-за нехватки памяти",
|
||||
"Приложения",
|
||||
name,
|
||||
"",
|
||||
float(fields["oom_kill"]) if "oom_kill" in fields else None,
|
||||
)
|
||||
self.previous = counters
|
||||
# A bounded hardware inventory protects the control-plane payload budget.
|
||||
keys = list(values)[:384]
|
||||
self.definitions = {key: definitions[key] for key in keys}
|
||||
return {key: values[key] for key in keys}
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
||||
CREATE TABLE IF NOT EXISTS monitor_identity (singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton), source_id text NOT NULL);
|
||||
INSERT INTO monitor_identity(singleton,source_id) VALUES(true,gen_random_uuid()::text) ON CONFLICT DO NOTHING;
|
||||
CREATE TABLE IF NOT EXISTS monitor_samples (
|
||||
seq bigint GENERATED ALWAYS AS IDENTITY,
|
||||
observed_at timestamptz NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
PRIMARY KEY(observed_at,seq)
|
||||
);
|
||||
SELECT create_hypertable('monitor_samples',by_range('observed_at',INTERVAL '1 hour'),if_not_exists=>true);
|
||||
CREATE INDEX IF NOT EXISTS monitor_samples_cursor ON monitor_samples(seq);
|
||||
CREATE TABLE IF NOT EXISTS monitor_definitions(singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton),payload jsonb NOT NULL);
|
||||
REVOKE ALL ON DATABASE mission_core_monitor FROM PUBLIC;
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Bounded local Timescale archive; transport acknowledgements never delete data."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
|
||||
SCHEMA = "missioncore.node-system-monitor/v1"
|
||||
MAX_BATCH_BYTES = 196608
|
||||
DATABASE_BUDGET = 2 * 1024**3
|
||||
|
||||
|
||||
class Archive:
|
||||
def __init__(self):
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
|
||||
self.pool = ThreadedConnectionPool(
|
||||
1,
|
||||
4,
|
||||
host="/run/mission-core-monitor-db",
|
||||
port=5433,
|
||||
dbname="mission_core_monitor",
|
||||
user="mission-core-monitor",
|
||||
connect_timeout=3,
|
||||
options="-c statement_timeout=3000",
|
||||
)
|
||||
try:
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
cursor.execute("SELECT source_id FROM monitor_identity")
|
||||
self.source = cursor.fetchone()[0]
|
||||
except Exception:
|
||||
self.pool.closeall()
|
||||
raise
|
||||
self.definitions = None
|
||||
self.database_bytes = None
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
db = self.pool.getconn()
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except BaseException:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
self.pool.putconn(db)
|
||||
|
||||
def append(self, sample, definitions):
|
||||
# This reserve belongs to telemetry, never to acquisition storage.
|
||||
if shutil.disk_usage("/var/lib/mission-core-monitor").free < 2 * 1024**3:
|
||||
raise RuntimeError("disk-reserve")
|
||||
if self.database_bytes is None or self.database_bytes >= DATABASE_BUDGET:
|
||||
raise RuntimeError("archive-budget")
|
||||
payload = json.dumps(sample, separators=(",", ":"), allow_nan=False)
|
||||
if len(payload.encode()) > 32768:
|
||||
raise ValueError("sample-budget")
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
if definitions != self.definitions:
|
||||
cursor.execute(
|
||||
"INSERT INTO monitor_definitions VALUES(true,%s) ON CONFLICT(singleton) "
|
||||
"DO UPDATE SET payload=excluded.payload",
|
||||
(json.dumps(definitions),),
|
||||
)
|
||||
cursor.execute(
|
||||
"INSERT INTO monitor_samples(observed_at,payload) VALUES(%s,%s) RETURNING seq",
|
||||
(datetime.fromtimestamp(sample["at"], UTC), payload),
|
||||
)
|
||||
seq = cursor.fetchone()[0]
|
||||
self.definitions = definitions
|
||||
return {**sample, "seq": seq}
|
||||
|
||||
def batch(self, after):
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT seq,payload FROM monitor_samples WHERE seq>%s ORDER BY seq LIMIT 32",
|
||||
(after,),
|
||||
)
|
||||
rows = []
|
||||
size = 0
|
||||
for seq, payload in cursor.fetchall():
|
||||
record = {**payload, "seq": seq}
|
||||
size += len(json.dumps(record).encode())
|
||||
if size > MAX_BATCH_BYTES:
|
||||
break
|
||||
rows.append(record)
|
||||
cursor.execute("SELECT min(seq),max(seq),min(observed_at) FROM monitor_samples")
|
||||
first, last, oldest = cursor.fetchone()
|
||||
return dict(
|
||||
schema=SCHEMA,
|
||||
source_id=self.source,
|
||||
samples=rows,
|
||||
first_seq=first,
|
||||
last_seq=last,
|
||||
retained_since=oldest.timestamp() if oldest else None,
|
||||
)
|
||||
|
||||
def maintain(self):
|
||||
with self.connection() as db, db.cursor() as cursor:
|
||||
cursor.execute("SELECT drop_chunks('monitor_samples',INTERVAL '7 days')")
|
||||
cursor.execute("SELECT pg_database_size(current_database())")
|
||||
size = cursor.fetchone()[0]
|
||||
if size >= DATABASE_BUDGET:
|
||||
# Drop one oldest completed chunk per pass, retaining the live hour.
|
||||
cursor.execute(
|
||||
"SELECT min(range_end) FROM timescaledb_information.chunks "
|
||||
"WHERE hypertable_name='monitor_samples' AND range_end<NOW()"
|
||||
)
|
||||
boundary = cursor.fetchone()[0]
|
||||
if boundary:
|
||||
cursor.execute(
|
||||
"SELECT drop_chunks('monitor_samples',older_than=>%s::timestamptz)",
|
||||
(boundary,),
|
||||
)
|
||||
cursor.execute("SELECT pg_database_size(current_database())")
|
||||
size = cursor.fetchone()[0]
|
||||
self.database_bytes = size
|
||||
return size
|
||||
Reference in New Issue
Block a user