Add tracked CMS publishing progress

2026-07-10 00:32:56 +03:00
parent 808d867dd7
commit 0579976ffc
4 changed files with 513 additions and 30 deletions
@@ -1,6 +1,6 @@
 import { createServer } from "node:http";
 import { createReadStream, statSync } from "node:fs";
-import { copyFile, cp, mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { appendFile, copyFile, cp, mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { extname, join, normalize, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
@@ -16,6 +16,9 @@ const host = process.env.HOST || "127.0.0.1";
 const activeProjectId = process.env.CMS_PROJECT || "nodedc";
 const projectConfigPath = join(cmsRoot, "projects", `${activeProjectId}.json`);
 const projectSecretsPath = join(cmsRoot, "projects", `${activeProjectId}.secrets.json`);
+const publishLogPath = join(cmsRoot, "projects", `${activeProjectId}.publish.log.jsonl`);
+const publishJobs = new Map();
+const PUBLISH_JOB_RETENTION_MS = 24 * 60 * 60 * 1000;

 async function loadProjectConfig() {
  const source = await readFile(projectConfigPath, "utf8");
@@ -1725,12 +1728,12 @@ async function unpackSiteArchive({ fileName, fileBuffer }) {
 async function readDeployConfig() {
  const deploy = projectConfig.deploy || {};
  const method = String(deploy.method || "sftp").trim().toLowerCase();
-  const password = await readDeployPasswordSecret();

  if (!deploy.enabled) throw new Error("Деплой выключен в настройках проекта");
  if (method !== "sftp") throw new Error("Сейчас поддержан только SFTP. Для McHost используйте метод sftp и порт 22");
  if (!deploy.host) throw new Error("Укажите FTP / SFTP host");
  if (!deploy.username) throw new Error("Укажите логин SFTP");
+  const password = await readDeployPasswordSecret();

  return {
    method,
@@ -1760,7 +1763,77 @@ async function shouldSkipRemoteFile(client, localFile, remotePath) {
  }
 }

-async function publishProject({ rendered = false } = {}) {
+function publishJobPercent(job) {
+  if (job.state === "completed") return 100;
+  if (!job.totalBytes) return 0;
+  return Math.max(0, Math.min(100, Math.round((job.processedBytes / job.totalBytes) * 1000) / 10));
+}
+
+function publicPublishJob(job) {
+  if (!job) return null;
+  return {
+    id: job.id,
+    state: job.state,
+    rendered: job.rendered,
+    host: job.host,
+    remoteRoot: job.remoteRoot,
+    totalFiles: job.totalFiles,
+    totalBytes: job.totalBytes,
+    totalLabel: formatProjectSize(job.totalBytes),
+    processedFiles: job.processedFiles,
+    processedBytes: job.processedBytes,
+    processedLabel: formatProjectSize(job.processedBytes),
+    uploadedFiles: job.uploadedFiles,
+    uploadedBytes: job.uploadedBytes,
+    uploadedLabel: formatProjectSize(job.uploadedBytes),
+    skippedFiles: job.skippedFiles,
+    skippedBytes: job.skippedBytes,
+    skippedLabel: formatProjectSize(job.skippedBytes),
+    currentFile: job.currentFile,
+    currentFileBytes: job.currentFileBytes,
+    currentFileSize: job.currentFileSize,
+    percent: publishJobPercent(job),
+    error: job.error,
+    startedAt: job.startedAt,
+    updatedAt: job.updatedAt,
+    completedAt: job.completedAt,
+  };
+}
+
+function cleanupPublishJobs() {
+  const cutoff = Date.now() - PUBLISH_JOB_RETENTION_MS;
+  for (const [id, job] of publishJobs) {
+    const timestamp = Date.parse(job.completedAt || job.updatedAt || job.startedAt || 0);
+    if (timestamp && timestamp < cutoff) publishJobs.delete(id);
+  }
+}
+
+function activePublishJob() {
+  return [...publishJobs.values()].find((job) => !["completed", "failed"].includes(job.state)) || null;
+}
+
+function latestPublishJob() {
+  return [...publishJobs.values()].sort((left, right) => Date.parse(right.startedAt) - Date.parse(left.startedAt))[0] || null;
+}
+
+async function writePublishLog(job, event, extra = {}) {
+  const entry = {
+    timestamp: new Date().toISOString(),
+    event,
+    project: activeProjectId,
+    job: publicPublishJob(job),
+    ...extra,
+  };
+  await appendFile(publishLogPath, `${JSON.stringify(entry)}\n`, "utf8").catch((error) => {
+    console.error(`Publish log write failed: ${error.message}`);
+  });
+}
+
+function touchPublishJob(job, patch = {}) {
+  Object.assign(job, patch, { updatedAt: new Date().toISOString() });
+}
+
+async function publishProject({ rendered = false, job = null } = {}) {
  const deployConfig = await readDeployConfig();
  const files = await collectPublicDeployFiles();
  if (!files.length) throw new Error("Нет публичных файлов для публикации");
@@ -1777,9 +1850,22 @@ async function publishProject({ rendered = false } = {}) {
    uploadedFiles: 0,
    skippedFiles: 0,
    uploadedBytes: 0,
+    skippedBytes: 0,
+    processedFiles: 0,
+    processedBytes: 0,
    totalBytes: files.reduce((sum, file) => sum + file.size, 0),
  };

+  if (job) {
+    touchPublishJob(job, {
+      state: "connecting",
+      host: report.host,
+      remoteRoot: report.remoteRoot,
+      totalFiles: report.totalFiles,
+      totalBytes: report.totalBytes,
+    });
+  }
+
  try {
    await client.connect({
      host: deployConfig.host,
@@ -1789,30 +1875,156 @@ async function publishProject({ rendered = false } = {}) {
      readyTimeout: 30000,
    });

+    if (job) touchPublishJob(job, { state: "publishing" });
+
    await ensureRemoteDirectory(client, deployConfig.remoteRoot, directoryCache);

    for (const file of files) {
      const remotePath = remoteJoin(deployConfig.remoteRoot, file.relativePath);
+      const completedBytes = report.processedBytes;
+      if (job) {
+        touchPublishJob(job, {
+          currentFile: file.relativePath,
+          currentFileBytes: 0,
+          currentFileSize: file.size,
+        });
+      }
      await ensureRemoteDirectory(client, remoteParent(remotePath), directoryCache);

      if (await shouldSkipRemoteFile(client, file, remotePath)) {
        report.skippedFiles += 1;
+        report.skippedBytes += file.size;
+        report.processedFiles += 1;
+        report.processedBytes += file.size;
+        if (job) {
+          touchPublishJob(job, {
+            skippedFiles: report.skippedFiles,
+            skippedBytes: report.skippedBytes,
+            processedFiles: report.processedFiles,
+            processedBytes: report.processedBytes,
+            currentFileBytes: file.size,
+          });
+        }
        continue;
      }

-      await client.fastPut(file.absolutePath, remotePath);
+      await client.fastPut(file.absolutePath, remotePath, {
+        step: (transferred) => {
+          if (!job) return;
+          const currentFileBytes = Math.max(0, Math.min(file.size, Number(transferred || 0)));
+          touchPublishJob(job, {
+            currentFileBytes,
+            processedBytes: completedBytes + currentFileBytes,
+          });
+        },
+      });
      report.uploadedFiles += 1;
      report.uploadedBytes += file.size;
+      report.processedFiles += 1;
+      report.processedBytes += file.size;
+      if (job) {
+        touchPublishJob(job, {
+          uploadedFiles: report.uploadedFiles,
+          uploadedBytes: report.uploadedBytes,
+          processedFiles: report.processedFiles,
+          processedBytes: report.processedBytes,
+          currentFileBytes: file.size,
+        });
+      }
    }

    report.totalLabel = formatProjectSize(report.totalBytes);
    report.uploadedLabel = formatProjectSize(report.uploadedBytes);
+    report.skippedLabel = formatProjectSize(report.skippedBytes);
    return report;
  } finally {
    await client.end().catch(() => {});
  }
 }

+function startPublishJob({ rendered = false } = {}) {
+  cleanupPublishJobs();
+  const activeJob = activePublishJob();
+  if (activeJob) return { job: publicPublishJob(activeJob), reused: true };
+
+  const now = new Date().toISOString();
+  const job = {
+    id: `${Date.now().toString(36)}-${randomBytes(5).toString("hex")}`,
+    state: "queued",
+    rendered,
+    host: "",
+    remoteRoot: "",
+    totalFiles: 0,
+    totalBytes: 0,
+    processedFiles: 0,
+    processedBytes: 0,
+    uploadedFiles: 0,
+    uploadedBytes: 0,
+    skippedFiles: 0,
+    skippedBytes: 0,
+    currentFile: "",
+    currentFileBytes: 0,
+    currentFileSize: 0,
+    error: "",
+    startedAt: now,
+    updatedAt: now,
+    completedAt: "",
+  };
+  publishJobs.set(job.id, job);
+  writePublishLog(job, "started");
+
+  void (async () => {
+    try {
+      touchPublishJob(job, { state: "scanning" });
+      const report = await publishProject({ rendered, job });
+      touchPublishJob(job, {
+        state: "completed",
+        processedFiles: report.totalFiles,
+        processedBytes: report.totalBytes,
+        uploadedFiles: report.uploadedFiles,
+        uploadedBytes: report.uploadedBytes,
+        skippedFiles: report.skippedFiles,
+        skippedBytes: report.skippedBytes,
+        currentFile: "",
+        currentFileBytes: 0,
+        currentFileSize: 0,
+        completedAt: new Date().toISOString(),
+      });
+      await writePublishLog(job, "completed");
+    } catch (error) {
+      touchPublishJob(job, {
+        state: "failed",
+        error: error?.message || String(error),
+        completedAt: new Date().toISOString(),
+      });
+      console.error(`Publish job ${job.id} failed`, error);
+      await writePublishLog(job, "failed", { stack: error?.stack || "" });
+    }
+  })();
+
+  return { job: publicPublishJob(job), reused: false };
+}
+
+async function recentPublishLog(limit = 20) {
+  const source = await readFile(publishLogPath, "utf8").catch((error) => {
+    if (error?.code === "ENOENT") return "";
+    throw error;
+  });
+  return source
+    .trim()
+    .split("\n")
+    .filter(Boolean)
+    .slice(-Math.max(1, Math.min(100, Number(limit || 20))))
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return { timestamp: "", event: "invalid-log-entry", message: line };
+      }
+    })
+    .reverse();
+}
+
 async function maybePublishAfterRender(renderResult) {
  if (!projectConfig.deploy?.enabled) {
    return {
@@ -2605,6 +2817,28 @@ const server = createServer(async (req, res) => {
      return;
    }

+    if (req.method === "POST" && url.pathname === "/api/project/publish/start") {
+      if (!ensureSiteReady(res)) return;
+      sendJson(res, 202, { ok: true, ...startPublishJob() });
+      return;
+    }
+
+    if (req.method === "GET" && url.pathname === "/api/project/publish/status") {
+      const id = String(url.searchParams.get("jobId") || "").trim();
+      const job = id ? publishJobs.get(id) : latestPublishJob();
+      if (!job) {
+        sendJson(res, 404, { ok: false, error: "Публикация не найдена" });
+        return;
+      }
+      sendJson(res, 200, { ok: true, job: publicPublishJob(job) });
+      return;
+    }
+
+    if (req.method === "GET" && url.pathname === "/api/project/publish/log") {
+      sendJson(res, 200, { ok: true, entries: await recentPublishLog(url.searchParams.get("limit")) });
+      return;
+    }
+
    if (req.method === "POST" && url.pathname === "/api/project/upload-site") {
      if (!isMultipartRequest(req)) {
        sendJson(res, 400, { ok: false, error: "Ожидается multipart/form-data с ZIP-архивом сайта" });
@@ -2764,6 +2998,7 @@ const server = createServer(async (req, res) => {

    await serveStatic(req, res);
  } catch (error) {
+    console.error(`${req.method || "REQUEST"} ${req.url || "/"} failed`, error);
    sendJson(res, 500, { ok: false, error: error.message });
  }
 });