From cfaed054d1ef39d0a578b48ea65e21b650408481 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 22 Jun 2026 18:14:31 +0300 Subject: [PATCH] Add BIM Viewer to canonical deploy runner --- infra/deploy-runner/README.md | 46 ++ infra/deploy-runner/nodedc-deploy | 1247 +++++++++++++++++++++++++++++ infra/synology/README.md | 40 +- 3 files changed, 1323 insertions(+), 10 deletions(-) create mode 100644 infra/deploy-runner/README.md create mode 100755 infra/deploy-runner/nodedc-deploy diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md new file mode 100644 index 0000000..60e94b8 --- /dev/null +++ b/infra/deploy-runner/README.md @@ -0,0 +1,46 @@ +# NODE.DC deploy runner + +This directory stores the versioned source for the Synology canonical deploy runner. + +Live runner: + +```text +/usr/local/sbin/nodedc-deploy +``` + +Synology staging candidate: + +```text +/volume1/docker/nodedc-deploy/runner-install/nodedc-deploy +``` + +The runner accepts data-only app-overlay artifacts from: + +```text +/volume1/docker/nodedc-deploy/inbox +``` + +Supported components in this source: + +- `engine` +- `launcher` +- `platform` +- `tasker` +- `ops-agents` +- `bim-viewer` + +Install or update the root-owned live runner on Synology: + +```bash +sudo install -o root -g root -m 0755 \ + /volume1/docker/nodedc-deploy/runner-install/nodedc-deploy \ + /usr/local/sbin/nodedc-deploy +sudo /usr/local/sbin/nodedc-deploy verify-install +``` + +Normal service deploys must still use explicit artifacts: + +```bash +sudo /usr/local/sbin/nodedc-deploy plan /volume1/docker/nodedc-deploy/inbox/.tgz +sudo /usr/local/sbin/nodedc-deploy apply /volume1/docker/nodedc-deploy/inbox/.tgz +``` diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy new file mode 100755 index 0000000..c523592 --- /dev/null +++ b/infra/deploy-runner/nodedc-deploy @@ -0,0 +1,1247 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + + +EXPECTED_SELF = Path("/usr/local/sbin/nodedc-deploy") +DEPLOY_ROOT = Path("/volume1/docker/nodedc-deploy") +INBOX = DEPLOY_ROOT / "inbox" +APPLIED_DIR = DEPLOY_ROOT / "applied" +FAILED_DIR = DEPLOY_ROOT / "failed" +BACKUPS_DIR = DEPLOY_ROOT / "backups" +STATE_DIR = DEPLOY_ROOT / "state" +TMP_DIR = DEPLOY_ROOT / ".tmp" +STATE_FILE = STATE_DIR / "applied.jsonl" +FAILED_STATE_FILE = STATE_DIR / "failed.jsonl" +LOCK_DIR = STATE_DIR / "deploy.lock" +DOCKER = Path("/usr/local/bin/docker") + +MAX_ARTIFACT_BYTES = 512 * 1024 * 1024 +MAX_MEMBER_COUNT = 20000 +MAX_PAYLOAD_BYTES = 1024 * 1024 * 1024 +MAX_FILE_BYTES = 256 * 1024 * 1024 + +PATCH_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,96}$") +MANIFEST_KEYS = {"id", "component", "type"} + +COMPONENTS = { + "engine": { + "payload_root": Path("/volume2/nodedc-demo"), + "compose_root": Path("/volume2/nodedc-demo"), + "services": ("nodedc-backend", "app"), + "compose_no_deps": True, + "publish_dist": True, + "healthchecks": ( + "http://127.0.0.1:8080/", + "http://127.0.0.1:3001/health", + ), + }, + "launcher": { + "payload_root": Path("/volume1/docker/nodedc-platform/launcher/source"), + "build_root": Path("/volume1/docker/nodedc-platform/launcher/source"), + "build": ("build", "--no-cache", "-t", "nodedc/launcher:local", "."), + "compose_root": Path("/volume1/docker/nodedc-platform/platform"), + "compose_env_file": Path("/volume1/docker/nodedc-platform/platform/.env.synology"), + "compose_files": ( + Path("/volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml"), + ), + "compose_no_deps": True, + "services": ("launcher",), + "healthchecks": ( + { + "url": "http://127.0.0.1:18080/healthz", + "headers": {"Host": "hub.nodedc.ru"}, + }, + ), + }, + "platform": { + "payload_root": Path("/volume1/docker/nodedc-platform"), + "compose_root": Path("/volume1/docker/nodedc-platform/platform"), + "compose_env_file": Path("/volume1/docker/nodedc-platform/platform/.env.synology"), + "compose_files": ( + Path("/volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml"), + ), + "compose_no_deps": True, + "services": ("notification-postgres", "notification-core", "ai-workspace-hub", "launcher", "reverse-proxy"), + "healthchecks": ( + "http://127.0.0.1:5185/healthz", + "http://127.0.0.1:18081/healthz", + { + "url": "http://127.0.0.1:18080/healthz", + "headers": {"Host": "hub.nodedc.ru"}, + }, + { + "url": "http://127.0.0.1:18080/", + "headers": {"Host": "id.nodedc.ru"}, + }, + ), + }, + "tasker": { + "payload_root": Path("/volume1/docker/nodedc-platform/tasker"), + "compose_root": Path("/volume1/docker/nodedc-platform/tasker/plane-app"), + "compose_env_file": Path("/volume1/docker/nodedc-platform/tasker/plane-app/.env.synology"), + "compose_files": ( + Path("/volume1/docker/nodedc-platform/tasker/plane-app/docker-compose.yaml"), + Path("/volume1/docker/nodedc-platform/tasker/plane-app/docker-compose.synology.override.yml"), + ), + "services": ("api", "worker", "beat-worker", "web"), + "healthchecks": ( + { + "url": "http://127.0.0.1:18090/", + "headers": {"Host": "ops.nodedc.ru"}, + }, + ), + }, + "ops-agents": { + "payload_root": Path("/volume1/docker/nodedc-platform/ops-agents"), + "compose_root": Path("/volume1/docker/nodedc-platform/ops-agents"), + "compose_env_file": Path("/volume1/docker/nodedc-platform/ops-agents/.env"), + "compose_files": ( + Path("/volume1/docker/nodedc-platform/ops-agents/docker-compose.synology.yml"), + ), + "compose_build": True, + "compose_no_deps": True, + "services": ("agent-gateway",), + "healthchecks": ( + "http://172.22.0.222:18190/readyz", + ), + }, + "bim-viewer": { + "payload_root": Path("/volume1/docker/nodedc-platform/bim-viewer/source"), + "build_root": Path("/volume1/docker/nodedc-platform/bim-viewer/source/converter"), + "build": ("build", "--no-cache", "-t", "nodedc/bim-converter:local", "."), + "compose_root": Path("/volume1/docker/nodedc-platform/bim-viewer/source"), + "compose_env_file": Path("/volume1/docker/nodedc-platform/bim-viewer/source/.env"), + "compose_files": ( + Path("/volume1/docker/nodedc-platform/bim-viewer/source/docker-compose.beam.yml"), + ), + "compose_no_deps": True, + "services": ("ndc-beam-viewer", "nodedc-bim-converter"), + "healthchecks": ( + "http://127.0.0.1:18100/api/auth/session", + ), + } +} + + +class NoRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +NO_REDIRECT_OPENER = urllib.request.build_opener(NoRedirectHandler) + + +class DeployError(Exception): + pass + + +def die(message): + raise DeployError(message) + + +def utc_now(): + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def stamp(): + return datetime.now().strftime("%Y%m%d-%H%M%S") + + +def sha256_file(path): + import hashlib + + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def safe_name(value): + return re.sub(r"[^A-Za-z0-9._-]", "_", value) + + +def is_relative_to(child, parent): + try: + child.relative_to(parent) + return True + except ValueError: + return False + + +def ensure_layout(): + for path in (INBOX, APPLIED_DIR, FAILED_DIR, BACKUPS_DIR, STATE_DIR, TMP_DIR): + path.mkdir(parents=True, exist_ok=True) + + for path in (STATE_DIR, BACKUPS_DIR, TMP_DIR): + os.chown(path, 0, 0) + path.chmod(0o700) + + for path in (APPLIED_DIR, FAILED_DIR): + os.chown(path, 0, 0) + path.chmod(0o755) + + +def require_root(): + if os.geteuid() != 0: + die("run with sudo") + + +def verify_file_mode(path, require_root_owner): + try: + st = path.lstat() + except FileNotFoundError: + die(f"missing path: {path}") + + if stat.S_ISLNK(st.st_mode): + die(f"path must not be a symlink: {path}") + + if require_root_owner and (st.st_uid != 0 or st.st_gid != 0): + die(f"path must be root:root: {path}") + + if st.st_mode & stat.S_IWGRP: + die(f"path must not be group-writable: {path}") + + if st.st_mode & stat.S_IWOTH: + die(f"path must not be world-writable: {path}") + + +def verify_install(): + invoked = Path(sys.argv[0]).absolute() + if invoked != EXPECTED_SELF: + die(f"runner must be invoked as {EXPECTED_SELF}, current: {invoked}") + + verify_file_mode(EXPECTED_SELF, require_root_owner=True) + verify_file_mode(Path("/usr/local"), require_root_owner=True) + verify_file_mode(Path("/usr/local/sbin"), require_root_owner=True) + + print(f"path={EXPECTED_SELF}") + print(f"sha256={sha256_file(EXPECTED_SELF)}") + print("verify-install-ok") + + +def validate_artifact_location(path): + if path.suffix != ".tgz": + die("artifact must have .tgz extension") + + try: + st = path.lstat() + except FileNotFoundError: + die(f"artifact not found: {path}") + + if stat.S_ISLNK(st.st_mode): + die(f"artifact must not be a symlink: {path}") + + if not stat.S_ISREG(st.st_mode): + die(f"artifact must be a regular file: {path}") + + if st.st_size > MAX_ARTIFACT_BYTES: + die(f"artifact too large: {st.st_size} bytes") + + inbox_real = INBOX.resolve() + parent_real = path.parent.resolve() + if parent_real != inbox_real: + die(f"artifact must be inside {INBOX}") + + name = path.name + if "/" in name or ".." in name or name.startswith("."): + die(f"unsafe artifact filename: {name}") + + +def validate_posix_path(value): + if not value: + die("empty path") + if "\\" in value: + die(f"backslash is not allowed in path: {value}") + pure = PurePosixPath(value) + if pure.is_absolute(): + die(f"absolute path rejected: {value}") + if any(part in ("", ".", "..") for part in pure.parts): + die(f"path escape rejected: {value}") + if any(ord(ch) < 32 for ch in value): + die(f"control character rejected in path: {value!r}") + return pure + + +def tar_name_to_path(work_dir, name): + pure = validate_posix_path(name) + target = work_dir.joinpath(*pure.parts) + resolved = target.resolve(strict=False) + if not is_relative_to(resolved, work_dir.resolve()): + die(f"tar path escaped work dir: {name}") + return target + + +def validate_tar_member(member): + name = member.name + validate_posix_path(name) + + if name not in ("manifest.env", "files.txt", "payload") and not name.startswith("payload/"): + die(f"unexpected tar member: {name}") + + if not (member.isfile() or member.isdir()): + die(f"unsupported tar member type: {name}") + + if member.mode & stat.S_ISUID or member.mode & stat.S_ISGID: + die(f"setuid/setgid tar member rejected: {name}") + + if member.isfile() and member.size > MAX_FILE_BYTES: + die(f"file too large in artifact: {name}") + + +def scan_tar(artifact): + member_count = 0 + payload_bytes = 0 + names = set() + + with tarfile.open(artifact, "r:gz") as tar: + for member in tar: + member_count += 1 + if member_count > MAX_MEMBER_COUNT: + die("too many files in artifact") + + validate_tar_member(member) + if member.name in names: + die(f"duplicate tar member rejected: {member.name}") + names.add(member.name) + + if member.isfile() and member.name.startswith("payload/"): + payload_bytes += member.size + if payload_bytes > MAX_PAYLOAD_BYTES: + die("payload is too large") + + if "manifest.env" not in names: + die("manifest.env missing") + if "files.txt" not in names: + die("files.txt missing") + if not any(name == "payload" or name.startswith("payload/") for name in names): + die("payload missing") + + +def safe_extract(artifact, work_dir): + scan_tar(artifact) + + with tarfile.open(artifact, "r:gz") as tar: + for member in tar: + validate_tar_member(member) + target = tar_name_to_path(work_dir, member.name) + + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + target.chmod(0o755) + continue + + if target.exists() and target.is_dir(): + die(f"file target already exists as directory: {member.name}") + + target.parent.mkdir(parents=True, exist_ok=True) + source = tar.extractfile(member) + if source is None: + die(f"cannot read tar member: {member.name}") + + tmp = target.with_name(f"{target.name}.extracting") + with source, tmp.open("wb") as out: + shutil.copyfileobj(source, out, 1024 * 1024) + tmp.chmod(0o644) + os.replace(tmp, target) + + +def parse_manifest(path): + data = {} + with path.open("r", encoding="utf-8") as f: + for lineno, line in enumerate(f, 1): + line = line.rstrip("\n") + if not line: + continue + if "=" not in line: + die(f"manifest line {lineno} must be key=value") + key, value = line.split("=", 1) + if key not in MANIFEST_KEYS: + die(f"unsupported manifest key: {key}") + if key in data: + die(f"duplicate manifest key: {key}") + if any(ord(ch) < 32 for ch in value): + die(f"control character in manifest value: {key}") + data[key] = value + + missing = sorted(MANIFEST_KEYS - set(data)) + if missing: + die(f"manifest keys missing: {', '.join(missing)}") + + if not PATCH_ID_RE.match(data["id"]): + die("manifest id contains unsafe characters") + + if data["type"] != "app-overlay": + die(f"unsupported artifact type: {data['type']}") + + if data["component"] not in COMPONENTS: + die(f"unsupported component: {data['component']}") + + return data + + +def parse_files_list(path): + entries = [] + seen = set() + with path.open("r", encoding="utf-8") as f: + for lineno, raw in enumerate(f, 1): + rel = raw.rstrip("\n") + if not rel: + continue + if rel.strip() != rel: + die(f"files.txt line {lineno} has leading/trailing whitespace") + validate_posix_path(rel) + if rel in seen: + die(f"duplicate files.txt entry: {rel}") + seen.add(rel) + entries.append(rel) + + if not entries: + die("files.txt is empty") + + return entries + + +def denied_payload_path(component, rel): + lower = rel.lower() + parts = lower.split("/") + base = parts[-1] + + if component == "engine" and rel == "docker-compose.yml": + pass + elif component == "platform" and rel in ( + "platform/docker-compose.platform-http.yml", + "platform/notification-core/Dockerfile", + "platform/ai-workspace-hub/Dockerfile", + "platform/ai-workspace-assistant/Dockerfile", + ): + pass + elif component == "bim-viewer" and rel in ( + "converter/Dockerfile", + ): + pass + elif base in (".env", "dockerfile", "docker-compose.yml", "compose.yml"): + return "sensitive or infrastructure filename" + + if base.endswith(".env") or base.endswith((".pem", ".crt", ".key", ".p12", ".pfx")): + return "secret-like file extension" + + if base.endswith((".sh", ".bash", ".zsh")): + return "shell files are not allowed in app-overlay artifacts" + + if base.endswith(".zip"): + return "zip backup files are not deployable artifacts" + + if any(part in (".git", "node_modules") for part in parts): + return "repository/build dependency directory" + + if any(token in lower for token in ("secret", "token", "password")): + return "secret-like path" + + runtime_prefixes = () + if component == "engine": + runtime_prefixes = ( + "nodedc-source/server/data", + "nodedc-source/server/storage", + "nodedc-source/server/logs", + "nodedc-source/public/storage", + "nodedc-source/dist/storage", + ) + if rel == "nodedc-source/server/.env": + return "server env file" + elif component == "launcher": + runtime_prefixes = ( + "server/data", + "server/storage", + "server/logs", + "public/storage", + "dist/storage", + ) + if rel == "server/.env": + return "server env file" + elif component == "platform": + runtime_prefixes = ( + "backups", + "launcher", + "ops-agents", + "tasker", + "platform/.env.synology", + "platform/authentik", + "platform/docs", + "authentik/data", + "authentik/certs", + "authentik/media", + "authentik/postgresql", + ) + if rel.startswith("platform/.env"): + return "platform env file" + if rel.startswith(("platform/Caddyfile.http.bak", "platform/docker-compose.platform-http.yml.bak")): + return "platform backup file" + elif component == "tasker": + runtime_prefixes = ( + "plane-app/.local-web-root", + "plane-app/archive", + "plane-app/backup", + "plane-app/plane.env", + "plane-app/plane.env.example", + "plane-app/plane.env.staging.example", + "plane-app/plane.env.synology", + "plane-app/plane.env.synology.base", + "plane-src/.cache", + "plane-src/.next", + "plane-src/.pnpm-store", + "plane-src/.react-router", + "plane-src/.turbo", + "plane-src/build", + "plane-src/coverage", + "plane-src/dist", + "plane-src/playwright-report", + "plane-src/test-results", + ) + if rel.startswith(("plane-app/docker-compose.yaml.bak", "plane-app/docker-compose.synology.override.yml.bak")): + return "tasker backup file" + elif component == "ops-agents": + runtime_prefixes = ( + "dist", + ) + if rel.startswith(".env"): + return "ops-agents env file" + elif component == "bim-viewer": + runtime_prefixes = ( + "dist", + "frontend/dist", + "server/data", + "server/logs", + "server/storage", + "server/tmp", + ) + if rel == ".env" or rel.startswith(".env."): + if rel != ".env.synology.example": + return "bim-viewer env file" + if rel.startswith(("docker-compose.beam.yml.bak",)): + return "bim-viewer backup file" + + if any(rel == prefix or rel.startswith(prefix + "/") for prefix in runtime_prefixes): + return "runtime data path" + + return None + + +def allowed_payload_path(component, rel): + validate_posix_path(rel) + reason = denied_payload_path(component, rel) + if reason: + die(f"path rejected: {rel}: {reason}") + + if component == "engine": + if rel in ("nodedc-source/package.json", "nodedc-source/package-lock.json"): + return True + if rel == "docker-compose.yml": + return True + if rel.startswith(( + "nodedc-source/server/", + "nodedc-source/src/", + "nodedc-source/dist/", + "nodedc-source/public/", + "nodedc-source/workers/ai-workspace-bridge/", + )): + return True + if rel == "nodedc-source/dist": + return True + + if component == "launcher": + if rel in ( + "index.html", + "package.json", + "package-lock.json", + "tsconfig.json", + "tsconfig.app.json", + "tsconfig.node.json", + "vite.config.ts", + ): + return True + if rel.startswith(("server/", "src/", "public/", "scripts/", "dc-ui-guideline/")): + return True + + if component == "platform": + if rel in ( + "platform/Caddyfile.http", + "platform/docker-compose.platform-http.yml", + ): + return True + if rel == "platform/notification-core" or rel.startswith("platform/notification-core/"): + return True + if rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/"): + return True + if rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/"): + return True + if rel == "authentik/custom-templates" or rel.startswith("authentik/custom-templates/"): + return True + + if component == "tasker": + if rel in ( + "plane-app/docker-compose.yaml", + "plane-app/docker-compose.synology.override.yml", + ): + return True + if rel.startswith(( + "plane-src/apps/api/", + "plane-src/apps/web/", + "plane-src/packages/", + )): + return True + + if component == "ops-agents": + if rel in ( + ".dockerignore", + ".env.synology.example", + ".gitignore", + "Dockerfile", + "README.md", + "docker-compose.local.yml", + "docker-compose.synology.yml", + "package.json", + "package-lock.json", + "tsconfig.json", + ): + return True + if rel.startswith(( + "docs/", + "migrations/", + "src/", + )): + return True + + if component == "bim-viewer": + if rel in ( + "converter", + "embeddedDemos", + "frontend", + "images", + "locales", + "processor", + "report", + "src", + ): + return True + if rel in ( + ".env.synology.example", + ".esdoc.json", + ".gitattributes", + ".gitignore", + ".release-please-manifest.json", + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "LICENSE", + "README.md", + "SECURITY.md", + "_config.yml", + "changelog-template.hbs", + "docker-compose.beam.yml", + "embeddedViewer.html", + "index.js", + "package.json", + "package-lock.json", + "release-please-config.json", + "rollup.config.js", + "rollup.dev.config.js", + "webComponentExample.html", + "xeokit-bim-viewer.css", + ): + return True + if rel.startswith(( + "converter/", + "embeddedDemos/", + "frontend/", + "images/", + "locales/", + "processor/", + "report/", + "server/", + "src/", + )): + return True + + die(f"path allowlist rejected: {rel}") + + +def path_is_covered_by_files_list(rel, entries): + return any(rel == entry or rel.startswith(entry.rstrip("/") + "/") for entry in entries) + + +def validate_payload_tree(component, payload_dir, entries): + root = payload_dir.resolve() + for path in payload_dir.rglob("*"): + resolved = path.resolve(strict=False) + if not is_relative_to(resolved, root): + die(f"payload path escaped payload dir: {path}") + + rel = path.relative_to(payload_dir).as_posix() + if path.is_symlink(): + die(f"payload symlink rejected: {rel}") + if not path.is_file() and not path.is_dir(): + die(f"payload special file rejected: {rel}") + + if path.is_dir(): + continue + + allowed_payload_path(component, rel) + if not path_is_covered_by_files_list(rel, entries): + die(f"payload file is not covered by files.txt: {rel}") + + +def load_artifact(artifact, work_dir): + safe_extract(artifact, work_dir) + + manifest_path = work_dir / "manifest.env" + files_path = work_dir / "files.txt" + payload_dir = work_dir / "payload" + + if not manifest_path.is_file(): + die("manifest.env missing after extract") + if not files_path.is_file(): + die("files.txt missing after extract") + if not payload_dir.is_dir(): + die("payload directory missing after extract") + + manifest = parse_manifest(manifest_path) + entries = parse_files_list(files_path) + + for rel in entries: + allowed_payload_path(manifest["component"], rel) + if not (payload_dir / rel).exists(): + die(f"files.txt entry missing in payload: {rel}") + + validate_payload_tree(manifest["component"], payload_dir, entries) + return manifest, entries, payload_dir + + +def load_state(path): + if not path.exists(): + return [] + rows = [] + with path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + +def append_jsonl(path, row): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + + +def state_has_sha(sha): + return any(row.get("sha256") == sha for row in load_state(STATE_FILE)) + + +def state_has_patch_id(patch_id): + return any(row.get("id") == patch_id for row in load_state(STATE_FILE)) + + +class DeployLock: + def __enter__(self): + try: + LOCK_DIR.mkdir() + except FileExistsError: + die(f"deploy lock is busy: {LOCK_DIR}") + (LOCK_DIR / "pid").write_text(str(os.getpid()), encoding="utf-8") + return self + + def __exit__(self, exc_type, exc, tb): + shutil.rmtree(LOCK_DIR, ignore_errors=True) + + +def component_root(component): + return COMPONENTS[component]["payload_root"] + + +def component_compose_root(component): + return COMPONENTS[component].get("compose_root", component_root(component)) + + +def component_services(component, entries=None): + if component == "platform" and entries is not None: + selected = [] + + def add(*services): + for service in services: + if service not in selected: + selected.append(service) + + touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries) + touches_caddy = any(rel == "platform/Caddyfile.http" for rel in entries) + touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries) + touches_ai_workspace = any(rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/") for rel in entries) + touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries) + touches_authentik = any(rel == "authentik/custom-templates" or rel.startswith("authentik/custom-templates/") for rel in entries) + + if touches_notification or touches_compose: + add("notification-postgres", "notification-core", "launcher", "reverse-proxy") + if touches_ai_workspace or touches_compose: + add("ai-workspace-hub", "reverse-proxy") + if touches_ai_workspace_assistant or touches_compose: + add("ai-workspace-postgres", "ai-workspace-assistant") + if touches_authentik: + add("authentik-server", "authentik-worker", "reverse-proxy") + if touches_caddy: + add("reverse-proxy") + + if selected: + return tuple(selected) + + return COMPONENTS[component]["services"] + + +def component_compose_files(component): + return COMPONENTS[component].get("compose_files", ()) + + +def component_compose_env_file(component): + return COMPONENTS[component].get("compose_env_file") + + +def component_build_root(component): + return COMPONENTS[component].get("build_root") + + +def component_build_args(component, entries=None): + if component == "platform" and entries is not None: + touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries) + return COMPONENTS[component].get("build") if touches_notification else None + + return COMPONENTS[component].get("build") + + +def component_builds(component, entries=None): + if component == "platform" and entries is not None: + touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries) + touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries) + touches_ai_workspace = any(rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/") for rel in entries) + touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries) + builds = [] + if touches_notification or touches_compose: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform/notification-core"), + ("build", "--no-cache", "-t", "nodedc/notification-core:local", "."), + )) + if touches_ai_workspace or touches_compose: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform/ai-workspace-hub"), + ("build", "--no-cache", "-t", "nodedc/ai-workspace-hub:local", "."), + )) + if touches_ai_workspace_assistant or touches_compose: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform/ai-workspace-assistant"), + ("build", "--no-cache", "-t", "nodedc/ai-workspace-assistant:local", "."), + )) + return tuple(builds) + + if component == "tasker": + plane_src = Path("/volume1/docker/nodedc-platform/tasker/plane-src") + return ( + ( + plane_src / "apps/api", + ( + "build", + "--network=host", + "-t", + "nodedc/plane-backend:local", + "-f", + "Dockerfile.api", + ".", + ), + ), + ( + plane_src, + ( + "build", + "--network=host", + "--build-arg", + "VITE_NODEDC_LAUNCHER_URL=https://hub.nodedc.ru", + "--build-arg", + "VITE_NODEDC_OIDC_LOGIN_ENABLED=1", + "-t", + "nodedc/plane-frontend:ru", + "-f", + "apps/web/Dockerfile.web.nas-legacy", + ".", + ), + ), + ) + + build_root = component_build_root(component) + build_args = component_build_args(component, entries) + if build_root and build_args: + return ((build_root, build_args),) + return () + + +def plan_artifact(artifact): + validate_artifact_location(artifact) + ensure_layout() + + sha = sha256_file(artifact) + with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: + manifest, entries, _payload_dir = load_artifact(artifact, Path(tmp)) + + component = manifest["component"] + root = component_root(component) + compose_root = component_compose_root(component) + services = component_services(component, entries) + builds = component_builds(component, entries) + + print("== plan ==") + print(f"artifact={artifact.name}") + print(f"sha256={sha}") + print(f"id={manifest['id']}") + print(f"component={component}") + print(f"type={manifest['type']}") + print(f"payload_root={root}") + print(f"compose_root={compose_root}") + for build_root, build_args in builds: + print(f"build_root={build_root}") + print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}") + print(f"services={' '.join(services)}") + if state_has_sha(sha): + print("state=sha-already-applied") + elif state_has_patch_id(manifest["id"]): + print("state=patch-id-already-applied") + else: + print("state=new") + print("== files ==") + for rel in entries: + print(f" {rel}") + + +def add_backup_path(tar, root, rel): + src = root / rel + if src.exists(): + tar.add(src, arcname=rel, recursive=True) + return True + return False + + +def create_backup(root, backup_dir, entries, include_nginx_html): + existing = [] + missing = [] + + with tarfile.open(backup_dir / "source-before.tgz", "w:gz") as tar: + for rel in entries: + if add_backup_path(tar, root, rel): + existing.append(rel) + else: + missing.append(rel) + + if include_nginx_html and (root / "nginx-html").exists(): + tar.add(root / "nginx-html", arcname="nginx-html", recursive=True) + existing.append("nginx-html") + + (backup_dir / "existing-files.txt").write_text("\n".join(existing) + ("\n" if existing else ""), encoding="utf-8") + (backup_dir / "missing-files.txt").write_text("\n".join(missing) + ("\n" if missing else ""), encoding="utf-8") + + +def replace_file(src, dst, current_stamp): + dst.parent.mkdir(parents=True, exist_ok=True) + tmp = dst.with_name(f"{dst.name}.next-{current_stamp}") + if tmp.exists(): + die(f"staging path already exists: {tmp}") + shutil.copy2(src, tmp) + os.replace(tmp, dst) + + +def replace_directory(src, dst, current_stamp): + dst.parent.mkdir(parents=True, exist_ok=True) + next_path = dst.with_name(f"{dst.name}.next-{current_stamp}") + prev_path = dst.with_name(f"{dst.name}.prev-{current_stamp}") + if next_path.exists() or prev_path.exists(): + die(f"staging path already exists near: {dst}") + shutil.copytree(src, next_path) + if dst.exists(): + dst.rename(prev_path) + next_path.rename(dst) + + +def copy_payload_path(payload_dir, root, rel, current_stamp): + src = payload_dir / rel + dst = root / rel + root_real = root.resolve() + dst_real = dst.resolve(strict=False) + if not is_relative_to(dst_real, root_real): + die(f"target escaped component root: {rel}") + + if src.is_dir(): + replace_directory(src, dst, current_stamp) + elif src.is_file(): + replace_file(src, dst, current_stamp) + else: + die(f"payload path is neither file nor directory: {rel}") + + +def publish_engine_dist(root, current_stamp): + dist = root / "nodedc-source" / "dist" + if not dist.is_dir(): + return + + dst = root / "nginx-html" + next_path = root / f"nginx-html.next-{current_stamp}" + prev_path = root / f"nginx-html.prev-{current_stamp}" + if next_path.exists() or prev_path.exists(): + die("nginx-html staging path already exists") + + shutil.copytree(dist, next_path) + if dst.exists(): + dst.rename(prev_path) + next_path.rename(dst) + + +def run_build(component, entries=None): + for build_root, build_args in component_builds(component, entries): + dockerfile = "Dockerfile" + for idx, arg in enumerate(build_args): + if arg == "-f" and idx + 1 < len(build_args): + dockerfile = build_args[idx + 1] + break + + if not (build_root / dockerfile).is_file(): + die(f"Dockerfile not found in build root: {build_root / dockerfile}") + + cmd = [str(DOCKER), *build_args] + env = os.environ.copy() + if component == "tasker": + env["DOCKER_BUILDKIT"] = "0" + subprocess.run(cmd, cwd=str(build_root), env=env, check=True) + + +def compose_base_cmd(component): + cmd = [str(DOCKER), "compose"] + env_file = component_compose_env_file(component) + if env_file: + cmd.extend(["--env-file", str(env_file)]) + for compose_file in component_compose_files(component): + cmd.extend(["-f", str(compose_file)]) + return cmd + + +def run_compose(component, services): + compose_root = component_compose_root(component) + cmd = [*compose_base_cmd(component), "up", "-d", "--force-recreate"] + if COMPONENTS[component].get("compose_build"): + cmd.append("--build") + if COMPONENTS[component].get("compose_no_deps"): + cmd.append("--no-deps") + cmd.extend(services) + subprocess.run(cmd, cwd=str(compose_root), check=True) + subprocess.run([*compose_base_cmd(component), "ps"], cwd=str(compose_root), check=True) + + +def run_component_runtime(component, entries, services): + run_build(component, entries) + run_compose(component, services) + + +def healthcheck_url(check): + headers = {} + if isinstance(check, dict): + url = check.get("url") + headers = check.get("headers") or {} + else: + url = check + + last_error = None + for _attempt in range(1, 61): + try: + request = urllib.request.Request(url, headers=headers) + with NO_REDIRECT_OPENER.open(request, timeout=10) as response: + if 200 <= response.status < 400: + return + last_error = f"HTTP {response.status}" + except urllib.error.HTTPError as exc: + if 200 <= exc.code < 400: + return + last_error = f"HTTP {exc.code}" + except Exception as exc: + last_error = str(exc) + time.sleep(5) + die(f"healthcheck failed for {url}: {last_error}") + + +def component_healthchecks(component, entries=None): + checks = list(COMPONENTS[component].get("healthchecks", ())) + if component == "platform" and entries is not None: + touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries) + touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries) + if touches_compose or touches_ai_workspace_assistant: + checks.append("http://127.0.0.1:18082/healthz") + return tuple(checks) + + +def run_healthchecks(component, entries=None): + for url in component_healthchecks(component, entries): + healthcheck_url(url) + + +def move_artifact(src, target_dir, suffix=""): + target_dir.mkdir(parents=True, exist_ok=True) + name = src.name + suffix + dst = target_dir / name + if dst.exists(): + dst = target_dir / f"{src.name}.{stamp()}" + src.rename(dst) + return dst + + +def apply_artifact(artifact): + require_root() + validate_artifact_location(artifact) + ensure_layout() + + sha = sha256_file(artifact) + backup_id = None + manifest = None + apply_started = False + current_stamp = stamp() + + with DeployLock(): + try: + with tempfile.TemporaryDirectory(prefix=f"apply-{current_stamp}-", dir=TMP_DIR) as tmp: + work = Path(tmp) + manifest, entries, payload_dir = load_artifact(artifact, work) + patch_id = manifest["id"] + component = manifest["component"] + root = component_root(component) + compose_root = component_compose_root(component) + services = component_services(component, entries) + + if state_has_sha(sha): + die(f"artifact sha already applied: {sha}") + if state_has_patch_id(patch_id): + die(f"patch id already applied: {patch_id}") + if not root.is_dir(): + die(f"component payload root not found: {root}") + if not compose_root.is_dir(): + die(f"component compose root not found: {compose_root}") + compose_files = component_compose_files(component) + if compose_files: + for compose_file in compose_files: + if not compose_file.is_file(): + die(f"compose file not found: {compose_file}") + elif not (compose_root / "docker-compose.yml").is_file(): + die(f"docker-compose.yml not found in component compose root: {compose_root}") + compose_env_file = component_compose_env_file(component) + if compose_env_file and not compose_env_file.is_file(): + die(f"compose env file not found: {compose_env_file}") + if not DOCKER.is_file(): + die(f"docker not found: {DOCKER}") + + backup_id = safe_name(f"{component}-{patch_id}-{current_stamp}") + backup_dir = BACKUPS_DIR / backup_id + backup_dir.mkdir(parents=True, exist_ok=False) + (backup_dir / "manifest.env").write_text((work / "manifest.env").read_text(encoding="utf-8"), encoding="utf-8") + (backup_dir / "files.txt").write_text((work / "files.txt").read_text(encoding="utf-8"), encoding="utf-8") + + include_nginx_html = component == "engine" and any( + rel == "nodedc-source/dist" or rel.startswith("nodedc-source/dist/") + for rel in entries + ) + create_backup(root, backup_dir, entries, include_nginx_html) + + apply_started = True + for rel in entries: + copy_payload_path(payload_dir, root, rel, current_stamp) + + if COMPONENTS[component].get("publish_dist"): + publish_engine_dist(root, current_stamp) + + run_component_runtime(component, entries, services) + run_healthchecks(component, entries) + + applied_path = move_artifact(artifact, APPLIED_DIR) + append_jsonl(STATE_FILE, { + "applied_at": utc_now(), + "artifact": applied_path.name, + "backup_id": backup_id, + "component": component, + "id": patch_id, + "sha256": sha, + "status": "ok", + }) + print(f"deploy-ok patch={patch_id} component={component} backup={backup_id}") + except Exception as exc: + failed_path = None + if artifact.exists(): + try: + failed_path = move_artifact(artifact, FAILED_DIR, suffix=f".{current_stamp}") + except Exception: + failed_path = None + + append_jsonl(FAILED_STATE_FILE, { + "artifact": str(failed_path.name if failed_path else artifact.name), + "backup_id": backup_id, + "component": manifest.get("component") if manifest else None, + "failed_at": utc_now(), + "id": manifest.get("id") if manifest else None, + "message": str(exc), + "sha256": sha, + "started_apply": apply_started, + "status": "failed", + }) + if backup_id: + print(f"backup={backup_id}", file=sys.stderr) + raise + + +def main(argv): + parser = argparse.ArgumentParser(prog="nodedc-deploy") + sub = parser.add_subparsers(dest="cmd", required=True) + + plan = sub.add_parser("plan") + plan.add_argument("artifact") + + apply = sub.add_parser("apply") + apply.add_argument("artifact") + + sub.add_parser("verify-install") + + args = parser.parse_args(argv) + + if args.cmd == "verify-install": + verify_install() + return 0 + + artifact = Path(args.artifact).absolute() + if args.cmd == "plan": + plan_artifact(artifact) + return 0 + if args.cmd == "apply": + apply_artifact(artifact) + return 0 + + parser.print_help() + return 2 + + +if __name__ == "__main__": + try: + raise SystemExit(main(sys.argv[1:])) + except DeployError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/infra/synology/README.md b/infra/synology/README.md index e47265a..94096ce 100644 --- a/infra/synology/README.md +++ b/infra/synology/README.md @@ -251,19 +251,39 @@ echo 'launcher-pending-gate-ok' `bim.nodedc.tech` на Synology можно вести напрямую через DSM Reverse Proxy на `172.22.0.222:18100`. В этом режиме Platform Caddy не участвует в публичном BIM-трафике, но `Caddyfile.http` всё равно содержит BIM route как запасной вариант для маршрута через `172.22.0.222:18080`. -Первый старт или обновление BIM после `BIM_REPO` sync: +Canonical BIM deploy now uses `nodedc-deploy` with `component=bim-viewer`; do not deploy BIM by manually running `docker compose up` from the source directory as the normal path. -```bash -cd /volume1/docker/nodedc-platform/bim-viewer/source -sudo cp -n .env.synology.example .env -# Проверить .env: NODEDC_INTERNAL_ACCESS_TOKEN должен совпадать с platform/.env.synology. -sudo /usr/local/bin/docker compose \ - --env-file .env \ - -f docker-compose.beam.yml \ - up -d --build --force-recreate ndc-beam-viewer nodedc-bim-converter +Live BIM root: + +```text +/volume1/docker/nodedc-platform/bim-viewer/source ``` -Минимальная проверка на Synology: +Live env stays outside artifacts: + +```text +/volume1/docker/nodedc-platform/bim-viewer/source/.env +``` + +Before the first canonical apply, prepare `.env` from `.env.synology.example` and verify that `NODEDC_INTERNAL_ACCESS_TOKEN` matches `platform/.env.synology`. After that, `.env` must not be included in deploy artifacts. + +Canonical deploy commands: + +```bash +sudo /usr/local/sbin/nodedc-deploy verify-install +sudo /usr/local/sbin/nodedc-deploy plan /volume1/docker/nodedc-deploy/inbox/.tgz +sudo /usr/local/sbin/nodedc-deploy apply /volume1/docker/nodedc-deploy/inbox/.tgz +``` + +The `bim-viewer` runner component: + +- applies payload under `/volume1/docker/nodedc-platform/bim-viewer/source`; +- rejects `.env`, `server/data`, upload/model/comment/share runtime storage, `node_modules`, `.git`, shell hooks and backup files; +- builds `nodedc/bim-converter:local` from `converter/Dockerfile`; +- recreates only `ndc-beam-viewer` and `nodedc-bim-converter`; +- healthchecks `http://127.0.0.1:18100/api/auth/session`. + +Минимальная проверка после apply: ```bash curl -k -fsS http://127.0.0.1:18100/api/auth/session