"""Archive full-resolution chunks from an authorized public streamed-SOG scene.""" import argparse import gzip import hashlib import json import re import subprocess import sys import urllib.request from pathlib import Path from urllib.parse import urljoin parser = argparse.ArgumentParser() parser.add_argument("--metadata-url", required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--author", required=True) parser.add_argument("--license", required=True) parser.add_argument("--source-url", required=True) args = parser.parse_args() if not args.metadata_url.startswith("https://") or not args.metadata_url.endswith("/lod-meta.json"): parser.error("Use an observed public HTTPS lod-meta.json URL") args.output.mkdir(parents=True, exist_ok=False) with urllib.request.urlopen(args.metadata_url, timeout=30) as response: raw = response.read(1024**2 + 1) if len(raw) > 1024**2: raise ValueError("LOD manifest too large") if raw[:2] == b"\x1f\x8b": raw = gzip.decompress(raw) if len(raw) > 1024**2: raise ValueError("LOD manifest too large") meta = json.loads(raw) if meta["version"] != 1 or not 1 <= meta["counts"][0] <= 20_000_000: raise ValueError("Unsupported scene size or LOD version") indices = set() def visit(node): if "0" in node.get("lods", {}): indices.add(node["lods"]["0"]["file"]) for child in node.get("children", []): visit(child) visit(meta["tree"]) if not 1 <= len(indices) <= 100: raise ValueError("Unexpected full-resolution chunk count") (args.output / "lod-meta.json").write_bytes(raw) for index in sorted(indices): name = meta["filenames"][index] if not re.fullmatch(r"[a-zA-Z0-9_-]+/meta\.json", name): raise ValueError("Unexpected chunk reference") command = [ sys.executable, str(Path(__file__).with_name("fetch_sog.py")), "--metadata-url", urljoin(args.metadata_url, name), "--output", str(args.output / Path(name).parent), "--author", args.author, "--license", args.license, "--source-url", args.source_url, ] subprocess.run(command, check=True) manifest = { "source_url": args.source_url, "metadata_url": args.metadata_url, "author": args.author, "license": args.license, "lod": 0, "splat_count": meta["counts"][0], "files": [ { "path": p.relative_to(args.output).as_posix(), "sha256": hashlib.sha256(p.read_bytes()).hexdigest(), "byte_length": p.stat().st_size, } for p in sorted(args.output.rglob("*")) if p.is_file() ], } (args.output / "source-manifest.json").write_text(json.dumps(manifest, indent=2)) print(json.dumps({"count": meta["counts"][0], "chunks": len(indices)}))