fix(simulation): reserve disk space before artifact import

This commit is contained in:
DCCONSTRUCTIONS
2026-08-29 16:18:56 +03:00
parent e6df8cb264
commit 76694cc869
2 changed files with 138 additions and 0 deletions
+81
View File
@@ -55,6 +55,7 @@ PROVIDER_JOB_STATES: Final = {
}
PROVIDER_POLL_TIMEOUT_SECONDS: Final = 2 * 60 * 60 + 5 * 60
PROVIDER_UNAVAILABLE_RETRY_LIMIT: Final = 6
IMPORT_DISK_RESERVE_BYTES: Final = 512 * 1024 * 1024
class SimulationProjectError(RuntimeError):
@@ -700,6 +701,7 @@ class SimulationProjectService:
self._raise_if_cancelled(project_id)
artifacts = _artifact_descriptors(result.get("artifacts"))
artifacts_root = self.store.artifacts_root(project_id)
self._ensure_import_capacity(project_id, artifacts, artifacts_root)
for descriptor in artifacts:
self._raise_if_cancelled(project_id)
_retry_provider_unavailable(
@@ -747,6 +749,55 @@ class SimulationProjectService:
self._queued_ids.add(project_id)
self._condition.notify_all()
def _ensure_import_capacity(
self,
project_id: str,
artifacts: list[dict[str, Any]],
artifacts_root: Path,
) -> None:
required_bytes = _remaining_import_bytes(artifacts_root, artifacts)
available_bytes = shutil.disk_usage(artifacts_root).free
if available_bytes >= required_bytes:
return
project = self.store.get(project_id)
source = project.get("source")
provider = project.get("provider")
if (
isinstance(source, dict)
and source.get("kind") == "archive"
and isinstance(provider, dict)
and isinstance(provider.get("job_id"), str)
):
source_files = source.get("files")
if isinstance(source_files, list) and len(source_files) == 1:
source_file = source_files[0]
if isinstance(source_file, dict):
logical_path = source_file.get("logical_path")
byte_length = source_file.get("byte_length")
if isinstance(logical_path, str) and isinstance(byte_length, int):
source_path = _confined_path(
self.store.source_root(project_id),
logical_path,
)
try:
source_stat = source_path.stat()
except OSError:
source_stat = None
if (
source_stat is not None
and source_path.is_file()
and not source_path.is_symlink()
and source_stat.st_size == byte_length
):
source_path.unlink()
available_bytes = shutil.disk_usage(artifacts_root).free
if available_bytes < required_bytes:
raise SimulationProjectError(
"Недостаточно места для импорта Gaussian-мира: "
f"нужно {_human_bytes(required_bytes)}, "
f"доступно {_human_bytes(available_bytes)}."
)
def delete(self, project_id: str) -> None:
project = self.store.get(project_id)
with self._condition:
@@ -857,6 +908,36 @@ def _artifact_descriptors(value: object) -> list[dict[str, Any]]:
return descriptors
def _remaining_import_bytes(
artifacts_root: Path,
artifacts: list[dict[str, Any]],
) -> int:
missing_bytes = 0
replacement_scratch_bytes = 0
for descriptor in artifacts:
logical_path = str(descriptor["logical_path"])
expected_bytes = int(descriptor["byte_length"])
target = _confined_path(artifacts_root, logical_path)
try:
current = target.stat()
except OSError:
current = None
if (
current is not None
and target.is_file()
and not target.is_symlink()
and current.st_size == expected_bytes
):
replacement_scratch_bytes = max(replacement_scratch_bytes, expected_bytes)
else:
missing_bytes += expected_bytes
return missing_bytes + replacement_scratch_bytes + IMPORT_DISK_RESERVE_BYTES
def _human_bytes(value: int) -> str:
return f"{value / (1024**3):.2f} ГиБ"
def _world_manifest(project_id: str, artifacts: list[dict[str, Any]]) -> dict[str, Any]:
def url_for(role: str) -> str | None:
descriptor = next((item for item in artifacts if item["role"] == role), None)
+57
View File
@@ -611,6 +611,63 @@ def test_failed_project_retries_from_retained_source_and_releases_old_job(tmp_pa
assert queued["source"]["uploaded_byte_length"] == queued["source"]["total_byte_length"]
def test_import_evicts_only_worker_backed_archive_staging_when_disk_is_low(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
store = SimulationProjectStore(tmp_path)
project = store.create(
name="Worker-backed archive",
scene_type="outdoor",
source_kind="archive",
files=[{"logical_path": "scene.rar", "byte_length": 6}],
)
source_file = project["source"]["files"][0]
store.append_upload(
project["project_id"],
source_file["file_id"],
offset=0,
payload=b"source",
)
store.begin_build(project["project_id"])
store.update_processing(
project["project_id"],
status="processing",
provider_job_id="gsp-20260826000000-deadbeef",
provider_state="ready",
)
service = SimulationProjectService(store, provider_factory=lambda: None)
source_path = store.source_root(project["project_id"]) / "scene.rar"
class _DiskUsage:
def __init__(self, free: int) -> None:
self.free = free
monkeypatch.setattr(
"k1link.simulation.projects.shutil.disk_usage",
lambda _path: _DiskUsage(0 if source_path.exists() else 10 * 1024**3),
)
artifacts = [
{
"role": "preview",
"logical_path": "preview.sog",
"media_type": "application/octet-stream",
"sha256": "a" * 64,
"byte_length": 1024,
}
]
service._ensure_import_capacity(
project["project_id"],
artifacts,
store.artifacts_root(project["project_id"]),
)
assert not source_path.exists()
retained_metadata = store.get(project["project_id"])["source"]
assert retained_metadata["uploaded_byte_length"] == retained_metadata["total_byte_length"]
def test_failed_local_project_reattaches_to_live_provider_job_without_rebuild(
tmp_path: Path,
) -> None: