From 09b1173697d5fef525da6ea482fe9587425c289c Mon Sep 17 00:00:00 2001 From: CODEX Date: Thu, 4 Jun 2026 21:21:58 +0300 Subject: [PATCH] beam: produce XKT preview artifacts --- converter/Dockerfile | 4 + converter/worker.py | 194 ++++++++++++++++++++++++++++++++++++++----- server/index.js | 8 +- 3 files changed, 181 insertions(+), 25 deletions(-) diff --git a/converter/Dockerfile b/converter/Dockerfile index dbc4c4a..18a9645 100644 --- a/converter/Dockerfile +++ b/converter/Dockerfile @@ -9,6 +9,8 @@ RUN apt-get update \ libgl1 \ libglib2.0-0 \ libglu1-mesa \ + nodejs \ + npm \ libsm6 \ libxext6 \ libxrender1 \ @@ -18,6 +20,8 @@ WORKDIR /app COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt +RUN npm install -g @xeokit/xeokit-convert@1.3.2 \ + && npm cache clean --force COPY . ./ diff --git a/converter/worker.py b/converter/worker.py index 6f348b6..48984fa 100644 --- a/converter/worker.py +++ b/converter/worker.py @@ -2,6 +2,7 @@ import hashlib import json import os import struct +import subprocess import sys import time from datetime import datetime, timezone @@ -21,13 +22,16 @@ from OCP.XCAFDoc import XCAFDoc_DocumentTool, XCAFDoc_ShapeTool CONVERTER_NAME = "NodeDcBimConverter" -CONVERTER_VERSION = "0.3.0" +CONVERTER_VERSION = "0.4.0" UPLOADS_DIR = Path(os.environ.get("NODEDC_BIM_CONVERTER_UPLOADS_DIR", "/beam/uploads")).resolve() POLL_INTERVAL_SECONDS = float(os.environ.get("NODEDC_BIM_CONVERTER_INTERVAL_SECONDS", "10")) PROCESS_ONCE = os.environ.get("NODEDC_BIM_CONVERTER_ONCE", "").lower() in {"1", "true", "yes"} REPROCESS_READY_ON_VERSION_CHANGE = os.environ.get( "NODEDC_BIM_CONVERTER_REPROCESS_READY_ON_VERSION_CHANGE", "" ).lower() in {"1", "true", "yes"} +XKT_ENABLED = os.environ.get("NODEDC_BIM_CONVERTER_XKT_ENABLED", "1").lower() not in {"0", "false", "no"} +XKT_COMMAND = os.environ.get("NODEDC_BIM_CONVERTER_XKT_COMMAND", "xeokit-convert") +NODE_OPTIONS = os.environ.get("NODE_OPTIONS", "--max-old-space-size=8192") STEP_MESH_LINEAR_DEFLECTION = float(os.environ.get("NODEDC_BIM_CONVERTER_STEP_LINEAR_DEFLECTION", "0.1")) STEP_MESH_ANGULAR_DEFLECTION = float(os.environ.get("NODEDC_BIM_CONVERTER_STEP_ANGULAR_DEFLECTION", "0.1")) STEP_EXTENSIONS = {".step", ".stp"} @@ -257,6 +261,20 @@ def normalize_glb_node_names(glb_path: Path) -> tuple[dict[str, Any], int]: node["name"] = mesh_name renamed += 1 + seen_names: dict[str, int] = {} + for index, node in enumerate(nodes): + base_name = str(node.get("name") or f"node_{index}") + occurrence = seen_names.get(base_name, 0) + 1 + seen_names[base_name] = occurrence + if occurrence > 1: + extras = node.get("extras") + if not isinstance(extras, dict): + extras = {} + extras.setdefault("originalName", base_name) + node["extras"] = extras + node["name"] = f"{base_name}__{occurrence:03d}" + renamed += 1 + if renamed: write_glb_json(glb_path, gltf_json, chunks) return gltf_json, renamed @@ -389,20 +407,98 @@ def convert_step_cadquery_to_glb(source_path: Path, glb_path: Path) -> tuple[dic return tree, stats -def convert_step_to_glb(source_path: Path, glb_path: Path, metadata_path: Path) -> dict[str, Any]: - import_strategy = "occt-xcaf-gltf" +def read_log_tail(path: Path, max_lines: int = 30) -> str: try: - tree, stats = convert_step_xcaf_to_glb(source_path, glb_path) - except Exception as exc: - print(f"[{CONVERTER_NAME}] OCCT/XCAF path failed for {source_path}: {exc}", file=sys.stderr, flush=True) - tree, stats = convert_step_cadquery_to_glb(source_path, glb_path) - import_strategy = stats.get("importStrategy") or "cadquery" + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except Exception: + return "" + return "\n".join(lines[-max_lines:]) + + +def convert_glb_to_xkt(glb_path: Path, xkt_path: Path) -> dict[str, Any]: + if not XKT_ENABLED: + return {"enabled": False} + + tmp_xkt_path = xkt_path.with_name(f"{xkt_path.name}.tmp") + log_path = xkt_path.with_name(f"{xkt_path.name}.convert.log") + if tmp_xkt_path.exists(): + tmp_xkt_path.unlink() + + env = {**os.environ, "NODE_OPTIONS": NODE_OPTIONS} + command = [ + XKT_COMMAND, + "-s", + str(glb_path), + "-f", + "glb", + "-o", + str(tmp_xkt_path), + "-t", + "-n", + "-e", + "0", + "-b", + ] + + print(f"[{CONVERTER_NAME}] XKT export {glb_path.name}", flush=True) + with log_path.open("w", encoding="utf-8", errors="replace") as log_file: + result = subprocess.run( + command, + cwd=str(glb_path.parent), + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + + if result.returncode != 0: + tail = read_log_tail(log_path) + raise RuntimeError(f"XKT export failed with code {result.returncode}: {tail}") + if not tmp_xkt_path.exists() or tmp_xkt_path.stat().st_size == 0: + raise RuntimeError("XKT export produced no output.") + + tmp_xkt_path.replace(xkt_path) + try: + log_path.unlink() + except Exception: + pass + + return { + "enabled": True, + "command": XKT_COMMAND, + "nodeOptions": NODE_OPTIONS, + "edgeBuffers": False, + "textures": False, + "normals": False, + "compressedBuffers": True, + "sizeBytes": xkt_path.stat().st_size, + } + + +def build_step_metadata( + source_path: Path, + glb_path: Path, + xkt_path: Path, + metadata_path: Path, + *, + tree: dict[str, Any], + stats: dict[str, Any], + import_strategy: str, +) -> dict[str, Any]: + xkt_stats = convert_glb_to_xkt(glb_path, xkt_path) + has_xkt = bool(xkt_stats.get("enabled") and xkt_path.exists()) + preferred_artifact_path = xkt_path if has_xkt else glb_path + preferred_artifact_type = "xkt" if has_xkt else "gltf" metadata = { "source": src_from_path(source_path), - "artifact": src_from_path(glb_path), + "artifact": src_from_path(preferred_artifact_path), + "artifactType": preferred_artifact_type, + "glbArtifact": src_from_path(glb_path), + "xktArtifact": src_from_path(xkt_path) if has_xkt else None, "format": "step", - "targetFormat": "glb", + "targetFormat": "xkt" if has_xkt else "glb", "importStrategy": import_strategy, "componentTree": tree, "componentCount": count_nodes(tree), @@ -414,14 +510,65 @@ def convert_step_to_glb(source_path: Path, glb_path: Path, metadata_path: Path) "cadqueryVersion": stats.get("cadqueryVersion"), }, "stats": stats, + "xkt": xkt_stats, } write_json_atomic(metadata_path, metadata) return metadata -def should_process(source_path: Path, manifest: dict[str, Any], glb_path: Path) -> bool: +def convert_step_assets(source_path: Path, glb_path: Path, xkt_path: Path, metadata_path: Path) -> dict[str, Any]: + previous_metadata = read_json(metadata_path) + + if XKT_ENABLED and glb_path.exists() and previous_metadata.get("componentTree"): + print(f"[{CONVERTER_NAME}] upgrading existing GLB to XKT {glb_path.name}", flush=True) + try: + gltf_json, renamed_nodes = normalize_glb_node_names(glb_path) + except Exception as exc: + print(f"[{CONVERTER_NAME}] existing GLB normalize failed for {glb_path}: {exc}", file=sys.stderr, flush=True) + gltf_json = {} + renamed_nodes = 0 + + previous_stats = previous_metadata.get("stats") if isinstance(previous_metadata.get("stats"), dict) else {} + stats = { + **previous_stats, + "gltfNodeCount": len(gltf_json.get("nodes", [])) or previous_stats.get("gltfNodeCount"), + "gltfMeshCount": len(gltf_json.get("meshes", [])) or previous_stats.get("gltfMeshCount"), + "renamedNodeCount": (previous_stats.get("renamedNodeCount") or 0) + renamed_nodes, + "reusedExistingGlb": True, + } + return build_step_metadata( + source_path, + glb_path, + xkt_path, + metadata_path, + tree=previous_metadata["componentTree"], + stats=stats, + import_strategy=previous_metadata.get("importStrategy") or "existing-glb", + ) + + import_strategy = "occt-xcaf-gltf" + try: + tree, stats = convert_step_xcaf_to_glb(source_path, glb_path) + except Exception as exc: + print(f"[{CONVERTER_NAME}] OCCT/XCAF path failed for {source_path}: {exc}", file=sys.stderr, flush=True) + tree, stats = convert_step_cadquery_to_glb(source_path, glb_path) + import_strategy = stats.get("importStrategy") or "cadquery" + + return build_step_metadata( + source_path, + glb_path, + xkt_path, + metadata_path, + tree=tree, + stats=stats, + import_strategy=import_strategy, + ) + + +def should_process(source_path: Path, manifest: dict[str, Any], glb_path: Path, xkt_path: Path) -> bool: status = manifest.get("status") - if status == "ready" and glb_path.exists(): + ready_artifact_path = xkt_path if XKT_ENABLED else glb_path + if status == "ready" and ready_artifact_path.exists(): return REPROCESS_READY_ON_VERSION_CHANGE and manifest.get("converterVersion") != CONVERTER_VERSION if status == "processing": updated_at = manifest.get("updatedAt") @@ -439,10 +586,11 @@ def process_one(source_path: Path) -> bool: manifest_file = manifest_path(source_path) stem = source_path.stem glb_path = source_path.with_name(f"{stem}.glb") + xkt_path = source_path.with_name(f"{stem}.xkt") metadata_path = source_path.with_name(f"{stem}.metadata.json") manifest = read_json(manifest_file) - if not should_process(source_path, manifest, glb_path): + if not should_process(source_path, manifest, glb_path, xkt_path): return False base_manifest = { @@ -452,14 +600,14 @@ def process_one(source_path: Path) -> bool: "converterVersion": CONVERTER_VERSION, "sourceFormat": "step", "sourceSrc": src_from_path(source_path), - "targetFormat": "glb", + "targetFormat": "xkt" if XKT_ENABLED else "glb", } write_json_atomic( manifest_file, { **base_manifest, - "message": "Preparing GLB and component tree.", + "message": "Preparing XKT preview and component tree.", "status": "processing", "updatedAt": now_iso(), }, @@ -467,21 +615,25 @@ def process_one(source_path: Path) -> bool: print(f"[{CONVERTER_NAME}] converting {source_path}", flush=True) try: - metadata = convert_step_to_glb(source_path, glb_path, metadata_path) + metadata = convert_step_assets(source_path, glb_path, xkt_path, metadata_path) + artifact_src = metadata.get("artifact") or src_from_path(xkt_path if xkt_path.exists() else glb_path) + artifact_type = metadata.get("artifactType") or ("xkt" if xkt_path.exists() else "gltf") write_json_atomic( manifest_file, { **base_manifest, - "artifactSrc": src_from_path(glb_path), - "artifactType": "gltf", + "artifactSrc": artifact_src, + "artifactType": artifact_type, + "fallbackArtifactSrc": src_from_path(glb_path), + "fallbackArtifactType": "gltf", "componentCount": metadata.get("componentCount"), "metadataSrc": src_from_path(metadata_path), - "message": "GLB and component tree are ready.", + "message": "XKT preview and component tree are ready.", "status": "ready", "updatedAt": now_iso(), }, ) - print(f"[{CONVERTER_NAME}] ready {glb_path}", flush=True) + print(f"[{CONVERTER_NAME}] ready {artifact_src}", flush=True) return True except Exception as exc: write_json_atomic( @@ -489,7 +641,7 @@ def process_one(source_path: Path) -> bool: { **base_manifest, "error": str(exc), - "message": "Failed to prepare GLB and component tree.", + "message": "Failed to prepare XKT preview and component tree.", "status": "failed", "updatedAt": now_iso(), }, diff --git a/server/index.js b/server/index.js index 77f2daa..beac572 100644 --- a/server/index.js +++ b/server/index.js @@ -293,11 +293,11 @@ const handleRawUpload = async (req, res, url) => { if (sourceFormat) { const conversion = { componentTreeRequired: true, - message: "Original STEP uploaded. GLB/component tree is waiting for NodeDcBimConverter.", + message: "Original STEP uploaded. XKT preview/component tree is waiting for NodeDcBimConverter.", sourceFormat, sourceSrc: src, status: "conversion_required", - targetFormat: "glb", + targetFormat: "xkt", updatedAt: nowIso() }; await writeJSONFile(manifestPathForSource(targetPath), conversion); @@ -334,11 +334,11 @@ const handleConversionStatus = async (res, searchParams) => { if (!manifest) { sendJSON(res, 200, { componentTreeRequired: true, - message: "GLB/component tree is waiting for NodeDcBimConverter.", + message: "XKT preview/component tree is waiting for NodeDcBimConverter.", sourceFormat, sourceSrc: srcFromUploadPath(sourcePath), status: "conversion_required", - targetFormat: "glb", + targetFormat: "xkt", updatedAt: nowIso() }); return;