3701 lines
146 KiB
Python
3701 lines
146 KiB
Python
#!/usr/bin/env python3
|
|
import csv
|
|
import hashlib
|
|
import importlib.machinery
|
|
import importlib.util
|
|
import inspect
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
|
BUILDER_PATH = SCRIPT_DIR / "build-gitea-incident-salvage-artifact.mjs"
|
|
FIXTURE_ROOT = SCRIPT_DIR / "fixtures" / "gitea-salvage"
|
|
COMPOSE_PATH = FIXTURE_ROOT / "docker-compose.gitea.yml"
|
|
DESCRIPTOR_PATH = (
|
|
FIXTURE_ROOT / "deployment" / "gitea-incident-salvage-v3.json"
|
|
)
|
|
LEGACY_DESCRIPTOR_PATH = (
|
|
FIXTURE_ROOT / "deployment" / "gitea-incident-salvage-v1.json"
|
|
)
|
|
TOPICS_DESCRIPTOR_PATH = (
|
|
FIXTURE_ROOT / "deployment" / "gitea-incident-salvage-v2.json"
|
|
)
|
|
DISPOSITION_PATH = (
|
|
FIXTURE_ROOT
|
|
/ "deployment"
|
|
/ "gitea-incident-salvage"
|
|
/ "confirmed-disposition-v1.json"
|
|
)
|
|
CLOSURE_DISPOSITION_PATH = (
|
|
FIXTURE_ROOT
|
|
/ "deployment"
|
|
/ "gitea-incident-salvage"
|
|
/ "confirmed-closure-disposition-v1.json"
|
|
)
|
|
DECISION_ROOT = (
|
|
SCRIPT_DIR.parent.parent.parent
|
|
/ "security-incidents"
|
|
/ "gitea-20260814"
|
|
/ "confirmed-decisions-v2"
|
|
)
|
|
|
|
|
|
def load_runner():
|
|
loader = importlib.machinery.SourceFileLoader(
|
|
"nodedc_gitea_salvage_under_test",
|
|
str(RUNNER_PATH),
|
|
)
|
|
spec = importlib.util.spec_from_loader(loader.name, loader)
|
|
module = importlib.util.module_from_spec(spec)
|
|
loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
RUNNER = load_runner()
|
|
|
|
|
|
def completed(returncode=0, stdout="", stderr=""):
|
|
return subprocess.CompletedProcess([], returncode, stdout, stderr)
|
|
|
|
|
|
def inventory_repository(path, relative_path, trusted_device=None, mountpoints=None):
|
|
if trusted_device is None:
|
|
trusted_device = path.lstat().st_dev
|
|
if mountpoints is None:
|
|
mountpoints = set()
|
|
return RUNNER.inventory_gitea_salvage_bare_repository(
|
|
path,
|
|
relative_path,
|
|
trusted_device,
|
|
mountpoints,
|
|
)
|
|
|
|
|
|
def unsupported_state_test_connection():
|
|
connection = sqlite3.connect(":memory:")
|
|
table_columns = {}
|
|
for _label, table, column in RUNNER.GITEA_SALVAGE_UNSUPPORTED_REPOSITORY_TABLES:
|
|
table_columns.setdefault(table, set()).add(column)
|
|
for table, columns in table_columns.items():
|
|
if table in {"attachment", "lfs_meta_object"}:
|
|
continue
|
|
fields = ", ".join(f'"{column}" INTEGER' for column in sorted(columns))
|
|
extras = ", secret_sentinel TEXT" if table == "secret" else ""
|
|
connection.execute(
|
|
f'CREATE TABLE "{table}" (id INTEGER PRIMARY KEY, {fields}{extras})'
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE lfs_meta_object (id INTEGER PRIMARY KEY, oid TEXT, "
|
|
"size INTEGER, repository_id INTEGER)"
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE attachment (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"issue_id INTEGER, release_id INTEGER, comment_id INTEGER, size INTEGER, "
|
|
"content_sentinel TEXT)"
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE repo_unit (repo_id INTEGER, type INTEGER, config TEXT)"
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE repository (id INTEGER PRIMARY KEY, description TEXT, "
|
|
"website TEXT, original_url TEXT, topics TEXT, avatar TEXT, "
|
|
"num_watches INTEGER, num_stars INTEGER, num_issues INTEGER, "
|
|
"num_pulls INTEGER, num_milestones INTEGER, num_projects INTEGER, "
|
|
"num_action_runs INTEGER, lfs_size INTEGER)"
|
|
)
|
|
return connection
|
|
|
|
|
|
def disposition_evidence_fixture():
|
|
disposition = json.loads(DISPOSITION_PATH.read_text(encoding="utf-8"))
|
|
grouped = {}
|
|
for decision in disposition["referencePolicy"]["exactDecisions"]:
|
|
key = (
|
|
decision["oldRepositoryId"],
|
|
decision["repositoryPath"],
|
|
decision["wiki"],
|
|
)
|
|
grouped.setdefault(key, []).append(
|
|
{"name": decision["name"], "oid": decision["oid"]}
|
|
)
|
|
repositories = []
|
|
for (repo_id, relative_path, wiki), refs_for_store in sorted(grouped.items()):
|
|
live_heads = [
|
|
ref["name"]
|
|
for ref in refs_for_store
|
|
if ref["name"].startswith("refs/heads/")
|
|
]
|
|
repositories.append(
|
|
{
|
|
"head": live_heads[0],
|
|
"object_format": "sha1",
|
|
"old_repo_id": repo_id,
|
|
"relative_path": relative_path,
|
|
"refs": refs_for_store,
|
|
"wiki": wiki,
|
|
}
|
|
)
|
|
for missing in disposition["referencePolicy"]["headInvariants"][
|
|
"allowedMissingTargets"
|
|
]:
|
|
repositories.append(
|
|
{
|
|
"head": missing["head"],
|
|
"object_format": "sha1",
|
|
"old_repo_id": missing["oldRepositoryId"],
|
|
"relative_path": missing["repositoryPath"],
|
|
"refs": [],
|
|
"wiki": missing["wiki"],
|
|
}
|
|
)
|
|
repositories.sort(
|
|
key=lambda row: (
|
|
row["old_repo_id"],
|
|
int(row["wiki"]),
|
|
row["relative_path"],
|
|
)
|
|
)
|
|
refs = {
|
|
"bytes": disposition["sourceEvidence"]["referenceManifestBytes"],
|
|
"manifest": {"repositories": repositories},
|
|
"refs": 105,
|
|
"sha256": RUNNER.GITEA_SALVAGE_DISPOSITION_REFERENCE_MANIFEST_SHA256,
|
|
}
|
|
|
|
direct_counts = {
|
|
row["label"]: row["sourceCount"]
|
|
for row in disposition["repositoryStatePolicy"]["directRelations"]
|
|
}
|
|
report = {
|
|
"aggregates": {
|
|
"direct_relation_counts": direct_counts,
|
|
"repository_metadata_presence": {
|
|
row["name"]: row["sourceRepositories"]
|
|
for row in disposition["repositoryStatePolicy"]["textMetadata"]
|
|
},
|
|
},
|
|
"anomalies": [],
|
|
"coverage": {
|
|
"schema_only_unreviewed_tables": sorted(
|
|
table
|
|
for group in disposition["repositoryStatePolicy"][
|
|
"schemaOnlyDependencyGroups"
|
|
]
|
|
for table in group["tables"]
|
|
),
|
|
},
|
|
"database_sha256": RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_SHA256,
|
|
"decision_manifest_sha256": RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_SHA256,
|
|
"kept_repository_ids": list(range(1, 46)),
|
|
"nonzero_categories": disposition["repositoryStatePolicy"][
|
|
"sourceNonzeroCategories"
|
|
],
|
|
"per_repository": [
|
|
{
|
|
"old_repo_id": repo_id,
|
|
"repo_unit_types": {
|
|
"1": 1,
|
|
"2": 1,
|
|
"3": 1,
|
|
"4": 1,
|
|
"5": 1,
|
|
"8": 1,
|
|
"9": 1,
|
|
},
|
|
}
|
|
for repo_id in range(1, 46)
|
|
],
|
|
"schema_catalog_sha256": (
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_UNSUPPORTED_SCHEMA_SHA256
|
|
),
|
|
"schema_mismatch": [],
|
|
"schema_missing": [],
|
|
"snapshot_uuid": RUNNER.GITEA_SALVAGE_SNAPSHOT_UUID,
|
|
}
|
|
unsupported = {
|
|
"bytes": disposition["sourceEvidence"][
|
|
"unsupportedRepositoryReportBytes"
|
|
],
|
|
"report": report,
|
|
"sha256": RUNNER.GITEA_SALVAGE_EXPECTED_UNSUPPORTED_REPORT_SHA256,
|
|
}
|
|
topic_evidence = {
|
|
"database_sha256": RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_SHA256,
|
|
"decision_manifest_sha256": RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_SHA256,
|
|
"material_repositories": 0,
|
|
"repositories": [
|
|
{
|
|
"encoding": "json-null",
|
|
"material": False,
|
|
"old_repo_id": repo_id,
|
|
"topic_count": 0,
|
|
}
|
|
for repo_id in range(1, 46)
|
|
],
|
|
"schema": "nodedc.gitea.salvage-semantic-topics/v2",
|
|
"serialized_arrays": 0,
|
|
"serialized_nulls": 45,
|
|
"snapshot_uuid": RUNNER.GITEA_SALVAGE_SNAPSHOT_UUID,
|
|
"topics": 0,
|
|
}
|
|
topics = {
|
|
"bytes": 1,
|
|
"evidence": topic_evidence,
|
|
"json": "{}",
|
|
"sha256": "0" * 64,
|
|
}
|
|
return disposition, refs, unsupported, topics
|
|
|
|
|
|
def closure_decisions_fixture():
|
|
with (DECISION_ROOT / "users.decisions.csv").open(
|
|
encoding="utf-8",
|
|
newline="",
|
|
) as handle:
|
|
users = list(csv.DictReader(handle))
|
|
with (DECISION_ROOT / "repositories.decisions.csv").open(
|
|
encoding="utf-8",
|
|
newline="",
|
|
) as handle:
|
|
repositories = list(csv.DictReader(handle))
|
|
return {
|
|
"users": users,
|
|
"repositories": repositories,
|
|
"kept_repositories": [
|
|
row for row in repositories if row["decision"] == "KEEP"
|
|
],
|
|
}
|
|
|
|
|
|
def closure_state_test_connection():
|
|
connection = sqlite3.connect(":memory:")
|
|
connection.row_factory = sqlite3.Row
|
|
statements = (
|
|
"CREATE TABLE access (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"user_id INTEGER, mode INTEGER)",
|
|
"CREATE TABLE collaboration (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"user_id INTEGER, mode INTEGER)",
|
|
"CREATE TABLE issue (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"poster_id INTEGER, name TEXT, content TEXT, is_pull INTEGER, "
|
|
"is_closed INTEGER, milestone_id INTEGER)",
|
|
"CREATE TABLE pull_request (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"base_repo_id INTEGER, head_repo_id INTEGER, has_merged INTEGER, "
|
|
"merger_id INTEGER)",
|
|
"CREATE TABLE comment (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"poster_id INTEGER, original_author TEXT, original_author_id INTEGER, "
|
|
"label_id INTEGER, old_project_id INTEGER, project_id INTEGER, "
|
|
"old_milestone_id INTEGER, milestone_id INTEGER, time_id INTEGER, "
|
|
"assignee_id INTEGER, assignee_team_id INTEGER NOT NULL DEFAULT 0, "
|
|
"resolve_doer_id INTEGER, dependent_issue_id INTEGER, review_id INTEGER, "
|
|
"ref_repo_id INTEGER, ref_issue_id INTEGER, ref_comment_id INTEGER, "
|
|
"ref_action INTEGER, ref_is_pull INTEGER, content TEXT, patch TEXT, "
|
|
"old_title TEXT, new_title TEXT, old_ref TEXT, new_ref TEXT)",
|
|
"CREATE TABLE issue_assignees (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"assignee_id INTEGER)",
|
|
"CREATE TABLE issue_content_history (id INTEGER PRIMARY KEY, "
|
|
"issue_id INTEGER, comment_id INTEGER, poster_id INTEGER, "
|
|
"content_text TEXT)",
|
|
"CREATE TABLE issue_label (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"label_id INTEGER)",
|
|
"CREATE TABLE issue_user (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"uid INTEGER)",
|
|
"CREATE TABLE issue_watch (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"user_id INTEGER)",
|
|
"CREATE TABLE review (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"reviewer_id INTEGER, reviewer_team_id INTEGER NOT NULL DEFAULT 0, "
|
|
"original_author TEXT, original_author_id INTEGER, content TEXT)",
|
|
"CREATE TABLE stopwatch (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"user_id INTEGER)",
|
|
"CREATE TABLE tracked_time (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"user_id INTEGER, time INTEGER)",
|
|
"CREATE TABLE reaction (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"comment_id INTEGER, user_id INTEGER)",
|
|
"CREATE TABLE review_state (id INTEGER PRIMARY KEY, pull_id INTEGER, "
|
|
"user_id INTEGER, updated_files TEXT)",
|
|
"CREATE TABLE issue_dependency (id INTEGER PRIMARY KEY, user_id INTEGER, "
|
|
"issue_id INTEGER, dependency_id INTEGER)",
|
|
"CREATE TABLE label (id INTEGER PRIMARY KEY, repo_id INTEGER, name TEXT, "
|
|
"description TEXT, color TEXT)",
|
|
"CREATE TABLE milestone (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"name TEXT, content TEXT)",
|
|
"CREATE TABLE project (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"creator_id INTEGER, title TEXT, description TEXT)",
|
|
"CREATE TABLE release (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"publisher_id INTEGER, tag_name TEXT, target TEXT, title TEXT, note TEXT)",
|
|
"CREATE TABLE project_board (id INTEGER PRIMARY KEY, project_id INTEGER, "
|
|
"title TEXT, color TEXT)",
|
|
"CREATE TABLE project_issue (id INTEGER PRIMARY KEY, issue_id INTEGER, "
|
|
"project_id INTEGER, project_board_id INTEGER)",
|
|
"CREATE TABLE team (id INTEGER PRIMARY KEY, org_id INTEGER NOT NULL)",
|
|
"CREATE TABLE pull_auto_merge (id INTEGER PRIMARY KEY, pull_id INTEGER, "
|
|
"doer_id INTEGER, merge_style TEXT, message TEXT)",
|
|
"CREATE TABLE notification (id INTEGER PRIMARY KEY, user_id INTEGER, "
|
|
"repo_id INTEGER, issue_id INTEGER, comment_id INTEGER)",
|
|
"CREATE TABLE repo_unit (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"type INTEGER, config TEXT)",
|
|
"CREATE TABLE attachment (id INTEGER PRIMARY KEY, uuid UUID, "
|
|
"repo_id INTEGER, issue_id INTEGER, release_id INTEGER, "
|
|
"uploader_id INTEGER, comment_id INTEGER, size INTEGER, name TEXT)",
|
|
"CREATE TABLE package (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"owner_id INTEGER, name TEXT)",
|
|
"CREATE TABLE package_version (id INTEGER PRIMARY KEY, "
|
|
"package_id INTEGER, creator_id INTEGER, version TEXT, metadata_json TEXT)",
|
|
"CREATE TABLE package_file (id INTEGER PRIMARY KEY, version_id INTEGER, "
|
|
"blob_id INTEGER, name TEXT)",
|
|
"CREATE TABLE package_blob (id INTEGER PRIMARY KEY, size INTEGER, "
|
|
"hash_sha256 TEXT)",
|
|
"CREATE TABLE package_property (id INTEGER PRIMARY KEY, ref_type INTEGER, "
|
|
"ref_id INTEGER, name TEXT, value TEXT)",
|
|
"CREATE TABLE action_run (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"trigger_user_id INTEGER, event_payload TEXT)",
|
|
"CREATE TABLE action_schedule (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"trigger_user_id INTEGER, content BLOB)",
|
|
"CREATE TABLE action_runner (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"token_hash TEXT, token_salt TEXT)",
|
|
"CREATE TABLE action_variable (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"data TEXT)",
|
|
"CREATE TABLE secret (id INTEGER PRIMARY KEY, repo_id INTEGER, data TEXT)",
|
|
"CREATE TABLE action_artifact (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"run_id INTEGER, file_size INTEGER, file_compressed_size INTEGER, "
|
|
"storage_path TEXT)",
|
|
"CREATE TABLE action_run_job (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"run_id INTEGER, workflow_payload BLOB)",
|
|
"CREATE TABLE action_task (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"job_id INTEGER, log_length INTEGER, log_size INTEGER, token_hash TEXT, "
|
|
"log_filename TEXT)",
|
|
"CREATE TABLE action_run_index (group_id INTEGER PRIMARY KEY, "
|
|
"max_index INTEGER)",
|
|
)
|
|
for statement in statements:
|
|
connection.execute(statement)
|
|
return connection
|
|
|
|
|
|
def populated_closure_evidence_fixture():
|
|
connection = closure_state_test_connection()
|
|
decisions = closure_decisions_fixture()
|
|
kept_ids = sorted(
|
|
int(row["repo_id"]) for row in decisions["kept_repositories"]
|
|
)
|
|
kept_user = next(
|
|
int(row["user_id"])
|
|
for row in decisions["users"]
|
|
if row["decision"] == "KEEP_ACTIVE"
|
|
)
|
|
deleted_user = next(
|
|
int(row["user_id"])
|
|
for row in decisions["users"]
|
|
if row["decision"] == "DELETE"
|
|
)
|
|
repo_id, second_repo_id = kept_ids[:2]
|
|
connection.executemany(
|
|
"INSERT INTO access VALUES (?,?,?,?)",
|
|
((1, repo_id, kept_user, 2), (2, repo_id, deleted_user, 1)),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO collaboration VALUES (?,?,?,?)",
|
|
((3, repo_id, kept_user, 2), (4, repo_id, deleted_user, 1)),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO issue VALUES (?,?,?,?,?,?,?,?)",
|
|
(
|
|
(
|
|
100,
|
|
repo_id,
|
|
kept_user,
|
|
"SECRET_ISSUE_TITLE",
|
|
"SECRET_ISSUE_BODY",
|
|
0,
|
|
0,
|
|
401,
|
|
),
|
|
(
|
|
101,
|
|
repo_id,
|
|
deleted_user,
|
|
"SECRET_PULL_TITLE",
|
|
"SECRET_PULL_BODY",
|
|
1,
|
|
1,
|
|
0,
|
|
),
|
|
),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO pull_request "
|
|
"(id,issue_id,base_repo_id,head_repo_id,has_merged,merger_id) "
|
|
"VALUES (200,101,?,?,0,0)",
|
|
(repo_id, second_repo_id),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO comment "
|
|
"(id,issue_id,poster_id,original_author,original_author_id,label_id,"
|
|
"old_project_id,project_id,old_milestone_id,milestone_id,time_id,"
|
|
"assignee_id,assignee_team_id,resolve_doer_id,dependent_issue_id,"
|
|
"review_id,ref_repo_id,ref_issue_id,ref_comment_id,ref_action,"
|
|
"ref_is_pull,content,patch,old_title,new_title,old_ref,new_ref) "
|
|
"VALUES (300,100,?,'SECRET_EXTERNAL_AUTHOR',987654,400,500,500,"
|
|
"401,401,308,?,0,?,101,0,?,100,0,1,0,'SECRET_COMMENT',"
|
|
"'SECRET_PATCH','SECRET_OLD_TITLE','SECRET_NEW_TITLE',"
|
|
"'SECRET_OLD_REF','SECRET_NEW_REF')",
|
|
(kept_user, kept_user, deleted_user, repo_id),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO issue_assignees VALUES (301,100,?)",
|
|
(deleted_user,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO issue_content_history "
|
|
"(id,issue_id,comment_id,poster_id,content_text) "
|
|
"VALUES (302,100,300,?,'SECRET_HISTORY')",
|
|
(kept_user,),
|
|
)
|
|
connection.execute("INSERT INTO issue_label VALUES (303,100,400)")
|
|
connection.execute("INSERT INTO issue_user VALUES (304,100,?)", (kept_user,))
|
|
connection.execute("INSERT INTO issue_watch VALUES (305,100,?)", (deleted_user,))
|
|
connection.execute(
|
|
"INSERT INTO review "
|
|
"(id,issue_id,reviewer_id,reviewer_team_id,original_author,"
|
|
"original_author_id,content) "
|
|
"VALUES (306,101,?,0,'SECRET_REVIEW_EXTERNAL_AUTHOR',456789,"
|
|
"'SECRET_REVIEW')",
|
|
(kept_user,),
|
|
)
|
|
connection.execute("INSERT INTO stopwatch VALUES (307,100,?)", (kept_user,))
|
|
connection.execute("INSERT INTO tracked_time VALUES (308,100,?,15)", (kept_user,))
|
|
connection.execute("INSERT INTO reaction VALUES (309,100,0,?)", (kept_user,))
|
|
connection.execute(
|
|
"INSERT INTO review_state VALUES (310,200,?,'SECRET_UPDATED_FILES')",
|
|
(kept_user,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO issue_dependency VALUES (311,?,100,101)",
|
|
(kept_user,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO label VALUES (400,?,'SECRET_LABEL','SECRET_LABEL_DESC','abcdef')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO milestone VALUES (401,?,'SECRET_MILESTONE','SECRET_MILESTONE_BODY')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO project VALUES (500,?,?,'SECRET_PROJECT','SECRET_PROJECT_BODY')",
|
|
(repo_id, kept_user),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO release VALUES (600,?,?,'v1','main','SECRET_RELEASE','SECRET_NOTE')",
|
|
(repo_id, kept_user),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO project_board VALUES (501,500,'SECRET_BOARD','aabbcc')"
|
|
)
|
|
connection.execute("INSERT INTO project_issue VALUES (502,100,500,501)")
|
|
connection.execute(
|
|
"INSERT INTO pull_auto_merge VALUES (503,200,?,'merge','SECRET_MERGE')",
|
|
(kept_user,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO notification VALUES (504,?,?,100,300)",
|
|
(kept_user, repo_id),
|
|
)
|
|
unit_rows = []
|
|
unit_id = 505
|
|
for unit_repo_id in kept_ids:
|
|
for unit_type in (1, 2, 3, 4, 5, 8, 9):
|
|
unit_rows.append(
|
|
(unit_id, unit_repo_id, unit_type, "SECRET_UNIT_CONFIG")
|
|
)
|
|
unit_id += 1
|
|
connection.executemany(
|
|
"INSERT INTO repo_unit VALUES (?,?,?,?)",
|
|
unit_rows,
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO attachment VALUES (700,'12345678-1234-1234-1234-123456789abc',"
|
|
"?,100,0,?,300,123,'SECRET_ATTACHMENT_NAME')",
|
|
(repo_id, kept_user),
|
|
)
|
|
|
|
schema = RUNNER.gitea_salvage_unsupported_schema_catalog(connection)
|
|
direct_counts = {
|
|
label: 0
|
|
for label, _table, _column in (
|
|
RUNNER.GITEA_SALVAGE_UNSUPPORTED_REPOSITORY_TABLES
|
|
)
|
|
}
|
|
direct_counts.update(
|
|
{
|
|
"access_grants": 2,
|
|
"attachments": 1,
|
|
"collaborators": 2,
|
|
"issues": 2,
|
|
"labels": 1,
|
|
"milestones": 1,
|
|
"pull_requests_base": 1,
|
|
"pull_requests_head": 1,
|
|
"releases": 1,
|
|
}
|
|
)
|
|
report = {
|
|
"aggregates": {
|
|
"attachments": {
|
|
"association_rows": 1,
|
|
"logical_bytes": 123,
|
|
},
|
|
"direct_relation_counts": direct_counts,
|
|
},
|
|
"anomalies": [],
|
|
"database_sha256": RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_SHA256,
|
|
"decision_manifest_sha256": RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_SHA256,
|
|
"kept_repository_ids": kept_ids,
|
|
"schema_catalog": schema["catalog"],
|
|
"schema_catalog_sha256": schema["sha256"],
|
|
"schema_mismatch": [],
|
|
"schema_missing": [],
|
|
"snapshot_uuid": RUNNER.GITEA_SALVAGE_SNAPSHOT_UUID,
|
|
}
|
|
unsupported = {
|
|
"report": report,
|
|
**RUNNER.canonical_gitea_salvage_evidence(
|
|
report,
|
|
"closure test predecessor report",
|
|
),
|
|
}
|
|
topics = {"sha256": "1" * 64}
|
|
return connection, decisions, unsupported, topics, repo_id, deleted_user
|
|
|
|
|
|
def run_closure_inventory_fixture(connection, decisions, unsupported, topics):
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"GITEA_SALVAGE_EXPECTED_UNSUPPORTED_REPORT_SHA256",
|
|
unsupported["sha256"],
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"GITEA_SALVAGE_DISPOSITION_UNSUPPORTED_SCHEMA_SHA256",
|
|
unsupported["report"]["schema_catalog_sha256"],
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"GITEA_SALVAGE_DISPOSITION_TOPICS_SHA256",
|
|
topics["sha256"],
|
|
),
|
|
):
|
|
return RUNNER.gitea_salvage_incident_closure_inventory(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
|
|
class GiteaIncidentSalvageArtifactTest(unittest.TestCase):
|
|
def test_fixture_is_exact_rootless_1272_clean_state_boundary(self):
|
|
compose = COMPOSE_PATH.read_text(encoding="utf-8")
|
|
descriptor = json.loads(DESCRIPTOR_PATH.read_text(encoding="utf-8"))
|
|
self.assertEqual(
|
|
hashlib.sha256(COMPOSE_PATH.read_bytes()).hexdigest(),
|
|
RUNNER.GITEA_SALVAGE_COMPOSE_SHA256,
|
|
)
|
|
self.assertEqual(
|
|
hashlib.sha256(DESCRIPTOR_PATH.read_bytes()).hexdigest(),
|
|
RUNNER.GITEA_SALVAGE_DESCRIPTOR_SHA256,
|
|
)
|
|
self.assertEqual(
|
|
hashlib.sha256(DISPOSITION_PATH.read_bytes()).hexdigest(),
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_SHA256,
|
|
)
|
|
self.assertEqual(
|
|
hashlib.sha256(CLOSURE_DISPOSITION_PATH.read_bytes()).hexdigest(),
|
|
RUNNER.GITEA_SALVAGE_CLOSURE_DISPOSITION_SHA256,
|
|
)
|
|
self.assertEqual(
|
|
hashlib.sha256(LEGACY_DESCRIPTOR_PATH.read_bytes()).hexdigest(),
|
|
"d0a0734e65ad182cbf17e83179a9e19058a1fc2e60c2905aff545483661f55dd",
|
|
)
|
|
self.assertEqual(
|
|
json.loads(LEGACY_DESCRIPTOR_PATH.read_text(encoding="utf-8"))[
|
|
"schemaVersion"
|
|
],
|
|
"nodedc.gitea.incident-salvage.v1",
|
|
)
|
|
self.assertEqual(
|
|
hashlib.sha256(TOPICS_DESCRIPTOR_PATH.read_bytes()).hexdigest(),
|
|
"879f4d761a062bbf967d79329436d4ae8ee1655bb63d0b93562d75146b0feaf3",
|
|
)
|
|
self.assertEqual(
|
|
json.loads(TOPICS_DESCRIPTOR_PATH.read_text(encoding="utf-8"))[
|
|
"schemaVersion"
|
|
],
|
|
"nodedc.gitea.incident-salvage.v2",
|
|
)
|
|
disposition = json.loads(DISPOSITION_PATH.read_text(encoding="utf-8"))
|
|
self.assertEqual(
|
|
descriptor["disposition"],
|
|
{
|
|
"schema": "nodedc.gitea.incident-disposition.v1",
|
|
"file": RUNNER.GITEA_SALVAGE_DISPOSITION_REL,
|
|
"sha256": RUNNER.GITEA_SALVAGE_DISPOSITION_SHA256,
|
|
},
|
|
)
|
|
self.assertTrue(disposition["activation"]["applyFrozen"])
|
|
self.assertEqual(
|
|
descriptor["closureDisposition"],
|
|
{
|
|
"file": RUNNER.GITEA_SALVAGE_CLOSURE_DISPOSITION_REL,
|
|
"predecessorArtifactSha256": (
|
|
RUNNER.GITEA_SALVAGE_CLOSURE_PREDECESSOR_ARTIFACT_SHA256
|
|
),
|
|
"schema": "nodedc.gitea.incident-closure-disposition.v1",
|
|
"sha256": RUNNER.GITEA_SALVAGE_CLOSURE_DISPOSITION_SHA256,
|
|
},
|
|
)
|
|
self.assertEqual(
|
|
disposition["remainingBlockers"],
|
|
list(RUNNER.GITEA_SALVAGE_DISPOSITION_REMAINING_BLOCKERS),
|
|
)
|
|
self.assertEqual(descriptor["runtime"]["image"], RUNNER.GITEA_SALVAGE_IMAGE)
|
|
self.assertEqual(
|
|
descriptor["runtime"]["imageId"],
|
|
"sha256:272085a806e6d182352cdb011c0ebab1d2efc7ec45247de84de5659c7bc5c4c6",
|
|
)
|
|
self.assertEqual(
|
|
descriptor["runtime"]["repoDigest"],
|
|
RUNNER.GITEA_SALVAGE_REPO_DIGEST,
|
|
)
|
|
self.assertFalse(descriptor["identity"]["preserveNumericUserIds"])
|
|
self.assertFalse(descriptor["identity"]["preserveNumericRepositoryIds"])
|
|
self.assertIn("oldToNewIdMapping", descriptor["identity"])
|
|
for required in (
|
|
"network_mode: none",
|
|
'restart: "no"',
|
|
'user: "1000:1000"',
|
|
"- /usr/local/bin/gitea",
|
|
"- /etc/gitea/app.ini",
|
|
"read_only: true",
|
|
"- ALL",
|
|
"no-new-privileges:true",
|
|
"source: /volume1/docker/nodedc-gitea/data",
|
|
"target: /data",
|
|
"source: /volume1/docker/nodedc-gitea/config",
|
|
"target: /etc/gitea",
|
|
"source: /volume1/docker/nodedc-gitea/socket",
|
|
"target: /run/gitea",
|
|
"create_host_path: false",
|
|
):
|
|
self.assertIn(required, compose)
|
|
for forbidden in (
|
|
"ports:",
|
|
"networks:",
|
|
"/volume1/docker/gitea",
|
|
"/var/run/docker.sock",
|
|
"TWO_FACTOR_AUTH",
|
|
"LFS_JWT_SECRET",
|
|
"restart: unless-stopped",
|
|
):
|
|
self.assertNotIn(forbidden, compose)
|
|
|
|
def test_compose_schema_is_valid_without_pull_or_start(self):
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--project-name",
|
|
"nodedc-gitea-salvage-fixture-test",
|
|
"--file",
|
|
str(COMPOSE_PATH),
|
|
"config",
|
|
"--quiet",
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode == 127:
|
|
self.skipTest("docker compose is unavailable")
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
|
|
def test_decision_v2_bundle_is_exact_and_partitioned(self):
|
|
expected = {
|
|
"confirmed-decision.json": RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_SHA256,
|
|
"users.decisions.csv": RUNNER.GITEA_SALVAGE_USERS_SHA256,
|
|
"repositories.decisions.csv": RUNNER.GITEA_SALVAGE_REPOSITORIES_SHA256,
|
|
}
|
|
for name, digest in expected.items():
|
|
self.assertEqual(
|
|
hashlib.sha256((DECISION_ROOT / name).read_bytes()).hexdigest(),
|
|
digest,
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-decision-") as directory:
|
|
payload = Path(directory)
|
|
for relative, source in (
|
|
(
|
|
RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_REL,
|
|
DECISION_ROOT / "confirmed-decision.json",
|
|
),
|
|
(RUNNER.GITEA_SALVAGE_USERS_REL, DECISION_ROOT / "users.decisions.csv"),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_REPOSITORIES_REL,
|
|
DECISION_ROOT / "repositories.decisions.csv",
|
|
),
|
|
):
|
|
target = payload / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(source.read_bytes())
|
|
decisions = RUNNER.validate_gitea_salvage_decision_bundle(payload)
|
|
self.assertEqual(len(decisions["users"]), 972)
|
|
self.assertEqual(len(decisions["kept_users"]), 10)
|
|
self.assertEqual(len(decisions["repositories"]), 2058)
|
|
self.assertEqual(len(decisions["kept_repositories"]), 45)
|
|
self.assertEqual(
|
|
{row["owner"] for row in decisions["kept_repositories"]},
|
|
{"dctouch", "SILVER"},
|
|
)
|
|
|
|
def test_incident_disposition_is_exact_hash_bound_plan_only_data(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-disposition-") as directory:
|
|
payload = Path(directory)
|
|
target = payload / RUNNER.GITEA_SALVAGE_DISPOSITION_REL
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(DISPOSITION_PATH.read_bytes())
|
|
disposition = RUNNER.validate_gitea_salvage_incident_disposition(payload)
|
|
self.assertEqual(
|
|
disposition["sourceEvidence"]["unsupportedRepositoryReportSha256"],
|
|
RUNNER.GITEA_SALVAGE_EXPECTED_UNSUPPORTED_REPORT_SHA256,
|
|
)
|
|
self.assertEqual(
|
|
disposition["referencePolicy"]["liveRestore"]["totalRefs"],
|
|
93,
|
|
)
|
|
self.assertEqual(
|
|
disposition["referencePolicy"]["archiveOnly"]["totalRefs"],
|
|
12,
|
|
)
|
|
self.assertEqual(
|
|
len(disposition["referencePolicy"]["exactDecisions"]),
|
|
105,
|
|
)
|
|
unit_rows = disposition["repositoryStatePolicy"]["units"]["rows"]
|
|
self.assertEqual([row["type"] for row in unit_rows], list(range(1, 11)))
|
|
self.assertEqual(
|
|
[row["type"] for row in unit_rows if row["targetCount"]],
|
|
[1, 2, 3, 4, 5, 8],
|
|
)
|
|
canonical = json.dumps(
|
|
disposition,
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8") + b"\n"
|
|
self.assertEqual(DISPOSITION_PATH.read_bytes(), canonical)
|
|
target.write_bytes(DISPOSITION_PATH.read_bytes() + b"\n")
|
|
with self.assertRaisesRegex(RUNNER.DeployError, "digest mismatch"):
|
|
RUNNER.validate_gitea_salvage_incident_disposition(payload)
|
|
|
|
def test_closure_disposition_is_additive_hash_bound_and_apply_frozen(self):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-closure-disposition-"
|
|
) as directory:
|
|
payload = Path(directory)
|
|
target = payload / RUNNER.GITEA_SALVAGE_CLOSURE_DISPOSITION_REL
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(CLOSURE_DISPOSITION_PATH.read_bytes())
|
|
disposition = RUNNER.validate_gitea_salvage_closure_disposition(
|
|
payload
|
|
)
|
|
self.assertEqual(
|
|
disposition["sourceEvidence"],
|
|
{
|
|
"databaseSha256": (
|
|
RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_SHA256
|
|
),
|
|
"identityDecisionManifestSha256": (
|
|
RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_SHA256
|
|
),
|
|
"referenceManifestSha256": (
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_REFERENCE_MANIFEST_SHA256
|
|
),
|
|
"semanticTopicsSha256": (
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_TOPICS_SHA256
|
|
),
|
|
"snapshotUuid": RUNNER.GITEA_SALVAGE_SNAPSHOT_UUID,
|
|
"unsupportedRepositoryReportSha256": (
|
|
RUNNER.GITEA_SALVAGE_EXPECTED_UNSUPPORTED_REPORT_SHA256
|
|
),
|
|
"unsupportedSchemaCatalogSha256": (
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_UNSUPPORTED_SCHEMA_SHA256
|
|
),
|
|
},
|
|
)
|
|
self.assertTrue(disposition["activation"]["applyFrozen"])
|
|
self.assertIsNone(disposition["closureReport"]["expectedSha256"])
|
|
self.assertIn(
|
|
"closure-report-review-pin-pending",
|
|
disposition["remainingBlockers"],
|
|
)
|
|
self.assertNotIn(
|
|
"issue-pr-polymorphic-subrelation-verifier-pending",
|
|
disposition["remainingBlockers"],
|
|
)
|
|
self.assertEqual(
|
|
disposition["policies"]["issuesPullRequestsMetadata"][
|
|
"subrelationClosure"
|
|
],
|
|
{
|
|
"commentHistoryMerger": (
|
|
"SCHEMA_BOUND_EXACT_RELATION_COUNTS_AND_CLASSES"
|
|
),
|
|
"externalAuthors": (
|
|
"PRESENCE_AND_NAME_BYTE_LENGTHS_ONLY_NO_LOCAL_USER_MAPPING"
|
|
),
|
|
"legacyRowsImported": False,
|
|
"teamRelations": (
|
|
"EXACT_ROW_TEAM_ORG_IDS_SEALED_HOLD_AND_BLOCK_IF_PRESENT"
|
|
),
|
|
},
|
|
)
|
|
target.write_bytes(CLOSURE_DISPOSITION_PATH.read_bytes() + b"\n")
|
|
with self.assertRaisesRegex(RUNNER.DeployError, "identity is unsafe"):
|
|
RUNNER.validate_gitea_salvage_closure_disposition(payload)
|
|
|
|
def test_payload_validator_accepts_only_exact_files(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-payload-") as directory:
|
|
payload = Path(directory)
|
|
for relative, source in (
|
|
(RUNNER.GITEA_COMPOSE_REL, COMPOSE_PATH),
|
|
(RUNNER.GITEA_SALVAGE_DESCRIPTOR_REL, DESCRIPTOR_PATH),
|
|
(RUNNER.GITEA_SALVAGE_DISPOSITION_REL, DISPOSITION_PATH),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_CLOSURE_DISPOSITION_REL,
|
|
CLOSURE_DISPOSITION_PATH,
|
|
),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_REL,
|
|
DECISION_ROOT / "confirmed-decision.json",
|
|
),
|
|
(RUNNER.GITEA_SALVAGE_USERS_REL, DECISION_ROOT / "users.decisions.csv"),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_REPOSITORIES_REL,
|
|
DECISION_ROOT / "repositories.decisions.csv",
|
|
),
|
|
):
|
|
target = payload / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(source.read_bytes())
|
|
result = RUNNER.validate_gitea_incident_salvage_payload(
|
|
payload,
|
|
RUNNER.GITEA_SALVAGE_ENTRIES,
|
|
)
|
|
self.assertEqual(len(result["kept_users"]), 10)
|
|
self.assertEqual(len(result["kept_repositories"]), 45)
|
|
self.assertEqual(
|
|
result["disposition"]["referencePolicy"]["liveRestore"]["totalRefs"],
|
|
93,
|
|
)
|
|
(payload / RUNNER.GITEA_SALVAGE_USERS_REL).write_bytes(
|
|
(DECISION_ROOT / "users.decisions.csv").read_bytes() + b"\n"
|
|
)
|
|
with self.assertRaisesRegex(RUNNER.DeployError, "digest mismatch"):
|
|
RUNNER.validate_gitea_incident_salvage_payload(
|
|
payload,
|
|
RUNNER.GITEA_SALVAGE_ENTRIES,
|
|
)
|
|
|
|
def test_builder_is_deterministic_data_only_and_refuses_overwrite(self):
|
|
with (
|
|
tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-builder-a-"
|
|
) as first_directory,
|
|
tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-builder-b-"
|
|
) as second_directory,
|
|
):
|
|
command = ["node", str(BUILDER_PATH), "gitea-salvage-test-002"]
|
|
results = []
|
|
for directory in (first_directory, second_directory):
|
|
environment = dict(os.environ)
|
|
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = directory
|
|
results.append(
|
|
subprocess.run(
|
|
command,
|
|
env=environment,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
)
|
|
first_result = json.loads(results[0].stdout)
|
|
second_result = json.loads(results[1].stdout)
|
|
first_artifact = Path(first_result["artifact"])
|
|
second_artifact = Path(second_result["artifact"])
|
|
first_sha = hashlib.sha256(first_artifact.read_bytes()).hexdigest()
|
|
second_sha = hashlib.sha256(second_artifact.read_bytes()).hexdigest()
|
|
self.assertEqual(first_sha, second_sha)
|
|
self.assertEqual(first_sha, first_result["sha256"])
|
|
self.assertEqual(second_sha, second_result["sha256"])
|
|
self.assertEqual(
|
|
first_result["stagePolicy"],
|
|
"after-exact-runner-promotion-plan-only",
|
|
)
|
|
self.assertEqual(
|
|
first_result["applyPolicy"],
|
|
"hard-frozen-before-root-creation",
|
|
)
|
|
|
|
overwrite_environment = dict(os.environ)
|
|
overwrite_environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = first_directory
|
|
overwrite = subprocess.run(
|
|
command,
|
|
env=overwrite_environment,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
self.assertNotEqual(overwrite.returncode, 0)
|
|
self.assertIn(
|
|
"gitea_salvage_artifact_target_already_exists",
|
|
overwrite.stderr,
|
|
)
|
|
self.assertEqual(
|
|
hashlib.sha256(first_artifact.read_bytes()).hexdigest(),
|
|
first_sha,
|
|
)
|
|
self.assertEqual(
|
|
list(Path(first_directory).glob(".*.tmp")),
|
|
[],
|
|
)
|
|
|
|
with tarfile.open(first_artifact, "r:gz") as archive:
|
|
names = archive.getnames()
|
|
self.assertNotIn("payload/config", names)
|
|
self.assertNotIn("payload/data", names)
|
|
self.assertNotIn("payload/secrets", names)
|
|
self.assertIn(
|
|
"payload/deployment/gitea-incident-salvage/repositories.decisions.csv",
|
|
names,
|
|
)
|
|
self.assertIn(
|
|
"payload/deployment/gitea-incident-salvage/confirmed-disposition-v1.json",
|
|
names,
|
|
)
|
|
self.assertIn(
|
|
"payload/deployment/gitea-incident-salvage/confirmed-closure-disposition-v1.json",
|
|
names,
|
|
)
|
|
self.assertIn(
|
|
"payload/deployment/gitea-incident-salvage-v3.json",
|
|
names,
|
|
)
|
|
self.assertEqual(
|
|
archive.extractfile("files.txt").read().decode(),
|
|
"\n".join(RUNNER.GITEA_SALVAGE_ENTRIES) + "\n",
|
|
)
|
|
disposition_bytes = archive.extractfile(
|
|
"payload/deployment/gitea-incident-salvage/"
|
|
"confirmed-disposition-v1.json"
|
|
).read()
|
|
self.assertEqual(disposition_bytes, DISPOSITION_PATH.read_bytes())
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-load-artifact-"
|
|
) as work_directory:
|
|
manifest, entries, _payload = RUNNER.load_artifact(
|
|
first_artifact,
|
|
Path(work_directory),
|
|
)
|
|
self.assertEqual(manifest["component"], "gitea")
|
|
self.assertEqual(tuple(entries), RUNNER.GITEA_SALVAGE_ENTRIES)
|
|
|
|
|
|
class GiteaIncidentSalvageRunnerTest(unittest.TestCase):
|
|
def test_slice_is_additive_and_payload_boundary_is_exact(self):
|
|
self.assertTrue(
|
|
RUNNER.is_gitea_incident_salvage_slice(
|
|
"gitea",
|
|
RUNNER.GITEA_SALVAGE_ENTRIES,
|
|
)
|
|
)
|
|
self.assertFalse(
|
|
RUNNER.is_gitea_fresh_install_slice(
|
|
"gitea",
|
|
RUNNER.GITEA_SALVAGE_ENTRIES,
|
|
)
|
|
)
|
|
for relative in RUNNER.GITEA_SALVAGE_ENTRIES:
|
|
self.assertTrue(RUNNER.allowed_payload_path("gitea", relative))
|
|
for rejected in (
|
|
"data/gitea/gitea.db",
|
|
"config/app.ini",
|
|
"secrets/secret-key",
|
|
"deployment/gitea-incident-salvage/refs.decisions.csv",
|
|
"repositories/dctouch/repo.git/objects/aa/object",
|
|
):
|
|
with self.assertRaises(RUNNER.DeployError):
|
|
RUNNER.allowed_payload_path("gitea", rejected)
|
|
|
|
def test_apply_gates_before_root_creation(self):
|
|
source = inspect.getsource(RUNNER.apply_artifact)
|
|
preflight = source.index("preflight_gitea_incident_salvage(")
|
|
bootstrap = source.index("if not root.is_dir()")
|
|
mutation = source.index("apply_started = True")
|
|
self.assertLess(preflight, bootstrap)
|
|
self.assertLess(bootstrap, mutation)
|
|
self.assertIn("enforce_apply=True", source)
|
|
self.assertIn("defer_bootstrap_root = is_gitea_bootstrap_slice", source)
|
|
|
|
def test_executed_apply_preflight_freezes_before_candidate_root_creation(self):
|
|
disposition, refs, unsupported, topics = disposition_evidence_fixture()
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-apply-freeze-"
|
|
) as directory:
|
|
root = Path(directory)
|
|
payload = root / "payload"
|
|
candidate = root / "candidate"
|
|
for relative, source in (
|
|
(RUNNER.GITEA_COMPOSE_REL, COMPOSE_PATH),
|
|
(RUNNER.GITEA_SALVAGE_DESCRIPTOR_REL, DESCRIPTOR_PATH),
|
|
(RUNNER.GITEA_SALVAGE_DISPOSITION_REL, DISPOSITION_PATH),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_CLOSURE_DISPOSITION_REL,
|
|
CLOSURE_DISPOSITION_PATH,
|
|
),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_DECISION_MANIFEST_REL,
|
|
DECISION_ROOT / "confirmed-decision.json",
|
|
),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_USERS_REL,
|
|
DECISION_ROOT / "users.decisions.csv",
|
|
),
|
|
(
|
|
RUNNER.GITEA_SALVAGE_REPOSITORIES_REL,
|
|
DECISION_ROOT / "repositories.decisions.csv",
|
|
),
|
|
):
|
|
target = payload / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(source.read_bytes())
|
|
with (
|
|
mock.patch.object(RUNNER, "GITEA_ROOT", candidate),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_compose_project_container_ids",
|
|
return_value=[],
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"docker_named_container_inspect_fail_closed",
|
|
return_value=None,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"validate_gitea_salvage_snapshot_boundary",
|
|
return_value={
|
|
"database_sha256": (
|
|
RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_SHA256
|
|
),
|
|
"readonly": True,
|
|
"uuid": RUNNER.GITEA_SALVAGE_SNAPSHOT_UUID,
|
|
},
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"bind_gitea_salvage_decisions_to_snapshot",
|
|
return_value={
|
|
"closure": {},
|
|
"topics": topics,
|
|
"unsupported": unsupported,
|
|
},
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"validate_gitea_salvage_closure_evidence",
|
|
return_value={
|
|
"blockers": list(
|
|
RUNNER.GITEA_SALVAGE_CLOSURE_REMAINING_BLOCKERS
|
|
),
|
|
"bytes": 2,
|
|
"json": "{}",
|
|
"sha256": "1" * 64,
|
|
},
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"inventory_gitea_salvage_repository_refs",
|
|
return_value=refs,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"validate_gitea_salvage_legacy_container",
|
|
return_value={"state": "stopped-restart-no"},
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"activation is frozen before candidate root creation",
|
|
),
|
|
):
|
|
RUNNER.preflight_gitea_incident_salvage(
|
|
payload,
|
|
enforce_apply=True,
|
|
)
|
|
self.assertFalse(candidate.exists())
|
|
self.assertEqual(disposition["activation"]["applyFrozen"], True)
|
|
|
|
def test_reviewed_legacy_and_disposition_evidence_pins_are_exact(self):
|
|
self.assertEqual(
|
|
RUNNER.GITEA_SALVAGE_EXPECTED_LEGACY_IMAGE,
|
|
"gitea/gitea:latest",
|
|
)
|
|
self.assertEqual(
|
|
RUNNER.GITEA_SALVAGE_EXPECTED_LEGACY_IMAGE_ID,
|
|
"sha256:bf95d9a45ce4fe38b027d051cdc4a4bc531513489fa6244af4074efbb1c376d6",
|
|
)
|
|
self.assertIsNone(RUNNER.GITEA_SALVAGE_EXPECTED_REF_MANIFEST_SHA256)
|
|
self.assertEqual(
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_REFERENCE_MANIFEST_SHA256,
|
|
"9cddaf0e4d4cf22dd264a6ae589ccc50d29e07f85c55e9d34b14627cecb8a311",
|
|
)
|
|
self.assertEqual(
|
|
RUNNER.GITEA_SALVAGE_EXPECTED_UNSUPPORTED_REPORT_SHA256,
|
|
"4b2cecf88c62fc5c4a43419885e88a01c9f9aac03133afb19dae0a7caef106ac",
|
|
)
|
|
self.assertEqual(
|
|
RUNNER.GITEA_SALVAGE_DISPOSITION_UNSUPPORTED_SCHEMA_SHA256,
|
|
"b5e3b6776926c4f1627fafd882362ed0ef986bfc86fc6ac6507a43976531b6db",
|
|
)
|
|
# The source report itself remains byte-for-byte historical evidence;
|
|
# review closure is represented by the successor disposition contract.
|
|
self.assertIsNone(RUNNER.GITEA_SALVAGE_EXPECTED_UNSUPPORTED_SCHEMA_SHA256)
|
|
container = {
|
|
"Config": {"Image": "gitea/gitea:latest"},
|
|
"HostConfig": {"RestartPolicy": {"Name": "no"}},
|
|
"Image": RUNNER.GITEA_SALVAGE_EXPECTED_LEGACY_IMAGE_ID,
|
|
"Mounts": [
|
|
{
|
|
"Destination": "/data",
|
|
"RW": True,
|
|
"Source": "/volume1/docker/gitea",
|
|
"Type": "bind",
|
|
}
|
|
],
|
|
"Name": "/gitea",
|
|
"State": {"Running": False},
|
|
}
|
|
with mock.patch.object(
|
|
RUNNER,
|
|
"docker_named_container_inspect_fail_closed",
|
|
return_value=container,
|
|
):
|
|
result = RUNNER.validate_gitea_salvage_legacy_container()
|
|
self.assertEqual(result["state"], "stopped-restart-no")
|
|
preflight_source = inspect.getsource(RUNNER.preflight_gitea_incident_salvage)
|
|
self.assertLess(
|
|
preflight_source.index("validate_gitea_salvage_legacy_container()"),
|
|
preflight_source.index("if not enforce_apply"),
|
|
)
|
|
for field, drift in (
|
|
("Name", "other"),
|
|
("Image", "sha256:" + "0" * 64),
|
|
("State", {"Running": True}),
|
|
("Mounts", []),
|
|
):
|
|
candidate = json.loads(json.dumps(container))
|
|
candidate[field] = drift
|
|
with (
|
|
self.subTest(field=field),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"docker_named_container_inspect_fail_closed",
|
|
return_value=candidate,
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"identity/isolation mismatch",
|
|
),
|
|
):
|
|
RUNNER.validate_gitea_salvage_legacy_container()
|
|
|
|
def test_confirmed_disposition_matches_exact_observed_evidence(self):
|
|
disposition, refs, unsupported, topics = disposition_evidence_fixture()
|
|
result = RUNNER.validate_gitea_salvage_disposition_evidence(
|
|
disposition,
|
|
refs,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
self.assertEqual(result["forensic_refs"], 105)
|
|
self.assertEqual(result["live_refs"], 93)
|
|
self.assertEqual(result["archive_only_refs"], 12)
|
|
self.assertEqual(
|
|
result["blockers"],
|
|
list(RUNNER.GITEA_SALVAGE_DISPOSITION_REMAINING_BLOCKERS),
|
|
)
|
|
self.assertIn("unsupported-schema-catalog-verifier-pending", result["blockers"])
|
|
self.assertIn("reference-manifest-fsck-reachability-verifier-pending", result["blockers"])
|
|
|
|
drift_cases = []
|
|
drift_refs = json.loads(json.dumps(refs))
|
|
drift_refs["manifest"]["repositories"][0]["refs"][0]["oid"] = "f" * 40
|
|
drift_cases.append((drift_refs, unsupported, topics, "reference"))
|
|
drift_unsupported = json.loads(json.dumps(unsupported))
|
|
drift_unsupported["report"]["aggregates"]["direct_relation_counts"][
|
|
"issues"
|
|
] += 1
|
|
drift_cases.append((refs, drift_unsupported, topics, "direct-relation"))
|
|
drift_coverage = json.loads(json.dumps(unsupported))
|
|
drift_coverage["report"]["coverage"][
|
|
"schema_only_unreviewed_tables"
|
|
].pop()
|
|
drift_cases.append((refs, drift_coverage, topics, "schema-only"))
|
|
drift_units = json.loads(json.dumps(unsupported))
|
|
drift_units["report"]["per_repository"][0]["repo_unit_types"].pop("9")
|
|
drift_cases.append((refs, drift_units, topics, "repo-unit"))
|
|
drift_topics = json.loads(json.dumps(topics))
|
|
drift_topics["evidence"]["serialized_nulls"] = 44
|
|
drift_cases.append((refs, unsupported, drift_topics, "semantic-topics"))
|
|
drift_topic_row = json.loads(json.dumps(topics))
|
|
drift_topic_row["evidence"]["repositories"][0]["encoding"] = (
|
|
"json-array"
|
|
)
|
|
drift_cases.append(
|
|
(refs, unsupported, drift_topic_row, "semantic-topics-row")
|
|
)
|
|
for drift_refs, drift_report, drift_topic, label in drift_cases:
|
|
with self.subTest(label=label), self.assertRaises(RUNNER.DeployError):
|
|
RUNNER.validate_gitea_salvage_disposition_evidence(
|
|
disposition,
|
|
drift_refs,
|
|
drift_report,
|
|
drift_topic,
|
|
)
|
|
|
|
def test_semantic_topics_parser_accepts_only_canonical_gitea_topics(self):
|
|
self.assertEqual(RUNNER.gitea_salvage_parse_semantic_topics("null"), ())
|
|
self.assertEqual(RUNNER.gitea_salvage_parse_semantic_topics("[]"), ())
|
|
self.assertEqual(
|
|
RUNNER.gitea_salvage_parse_semantic_topics('["alpha","beta-2"]'),
|
|
("alpha", "beta-2"),
|
|
)
|
|
for value in (
|
|
None,
|
|
"",
|
|
" null ",
|
|
"NULL",
|
|
'"null"',
|
|
"{}",
|
|
"0",
|
|
"true",
|
|
"false",
|
|
" [] ",
|
|
'["Beta"]',
|
|
'["beta","alpha"]',
|
|
'["alpha","alpha"]',
|
|
'["unsafe/topic"]',
|
|
'["' + "a" * 36 + '"]',
|
|
):
|
|
with self.subTest(value=value), self.assertRaises(RUNNER.DeployError):
|
|
RUNNER.gitea_salvage_parse_semantic_topics(value)
|
|
|
|
connection = sqlite3.connect(":memory:")
|
|
try:
|
|
connection.execute("CREATE TABLE repository (id INTEGER, topics TEXT)")
|
|
connection.executemany(
|
|
"INSERT INTO repository (id,topics) VALUES (?,?)",
|
|
((7, "null"), (9, "null")),
|
|
)
|
|
evidence = RUNNER.gitea_salvage_semantic_topics_inventory(
|
|
connection,
|
|
({"repo_id": 7}, {"repo_id": 9}),
|
|
)["evidence"]
|
|
finally:
|
|
connection.close()
|
|
self.assertEqual(evidence["serialized_arrays"], 0)
|
|
self.assertEqual(evidence["serialized_nulls"], 2)
|
|
self.assertEqual(
|
|
[row["encoding"] for row in evidence["repositories"]],
|
|
["json-null", "json-null"],
|
|
)
|
|
self.assertEqual(evidence["material_repositories"], 0)
|
|
self.assertEqual(evidence["topics"], 0)
|
|
|
|
def test_closure_inventory_is_per_repo_deterministic_and_payload_free(self):
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
try:
|
|
first = run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
second = run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
self.assertEqual(first["sha256"], second["sha256"])
|
|
self.assertEqual(first["json"], second["json"])
|
|
report = first["report"]
|
|
self.assertEqual(report["scope"]["kept_users"], 10)
|
|
self.assertEqual(report["scope"]["deleted_users"], 962)
|
|
self.assertEqual(len(report["per_repository"]), 45)
|
|
per_repo = {
|
|
row["old_repo_id"]: row for row in report["per_repository"]
|
|
}
|
|
self.assertEqual(
|
|
per_repo[repo_id]["closure"]["issues_ordinary"]["rows"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
per_repo[repo_id]["closure"]["pull_request_wrappers"]["rows"],
|
|
1,
|
|
)
|
|
self.assertEqual(report["aggregates"]["comments"]["rows"], 1)
|
|
self.assertEqual(
|
|
report["issue_state_totals"],
|
|
{
|
|
"ordinary_closed": 0,
|
|
"ordinary_open": 1,
|
|
"pull_wrapper_closed": 1,
|
|
"pull_wrapper_open": 0,
|
|
},
|
|
)
|
|
self.assertEqual(report["pull_state_totals"], {"merged": 0, "unmerged": 1})
|
|
self.assertEqual(
|
|
report["aggregates"]["comments"]["text_bytes"]["content"],
|
|
len("SECRET_COMMENT"),
|
|
)
|
|
self.assertEqual(report["aggregates"]["packages"]["rows"], 0)
|
|
self.assertEqual(report["aggregates"]["action_runs"]["rows"], 0)
|
|
self.assertEqual(
|
|
report["actor_relation_counts"]["collaborations"],
|
|
{"deleted": 1, "kept": 1, "rows": 2},
|
|
)
|
|
self.assertEqual(
|
|
{
|
|
row["actor_class"]: row["disposition"]
|
|
for row in report["actor_relations"]
|
|
if row["relation"] == "collaborations"
|
|
},
|
|
{
|
|
"deleted": "DROP_DELETED_ACTOR",
|
|
"kept": "RECREATE_KEPT_ACTOR_AFTER_ID_MAP",
|
|
},
|
|
)
|
|
self.assertEqual(report["attachment_summary"]["rows"], 1)
|
|
self.assertEqual(
|
|
report["attachment_manifest"][0]["link_class"],
|
|
"multi-link",
|
|
)
|
|
self.assertEqual(
|
|
report["attachment_manifest"][0]["content_hash"],
|
|
"unavailable-in-schema",
|
|
)
|
|
for sentinel in (
|
|
"SECRET_ISSUE_TITLE",
|
|
"SECRET_ISSUE_BODY",
|
|
"SECRET_COMMENT",
|
|
"SECRET_PATCH",
|
|
"SECRET_EXTERNAL_AUTHOR",
|
|
"SECRET_REVIEW_EXTERNAL_AUTHOR",
|
|
"SECRET_ATTACHMENT_NAME",
|
|
"SECRET_UNIT_CONFIG",
|
|
):
|
|
self.assertNotIn(sentinel, first["json"])
|
|
scalar_values = []
|
|
|
|
def collect_scalar_values(value):
|
|
if isinstance(value, dict):
|
|
for child in value.values():
|
|
collect_scalar_values(child)
|
|
return
|
|
if isinstance(value, list):
|
|
for child in value:
|
|
collect_scalar_values(child)
|
|
return
|
|
scalar_values.append(value)
|
|
|
|
collect_scalar_values(report)
|
|
self.assertNotIn(987654, scalar_values)
|
|
self.assertNotIn(456789, scalar_values)
|
|
self.assertFalse(report["privacy_contract"]["payload_values_selected"])
|
|
schema_coverage = {
|
|
row["table"]: row["columns"]
|
|
for row in report["schema_coverage"]
|
|
}
|
|
self.assertEqual(
|
|
schema_coverage["secret"],
|
|
["id", "repo_id"],
|
|
)
|
|
self.assertEqual(
|
|
schema_coverage["package_property"],
|
|
["id", "ref_id", "ref_type"],
|
|
)
|
|
self.assertNotIn("token_hash", schema_coverage["action_runner"])
|
|
self.assertNotIn("name", schema_coverage["attachment"])
|
|
self.assertIn(
|
|
"closure-report-review-pin-pending",
|
|
report["remaining_blockers"],
|
|
)
|
|
self.assertNotIn(
|
|
"issue-pr-polymorphic-subrelation-verifier-pending",
|
|
report["remaining_blockers"],
|
|
)
|
|
subrelations = report["issue_pr_subrelations"]
|
|
self.assertEqual(subrelations["conditional_hold_blockers"], [])
|
|
self.assertEqual(subrelations["holds"], [])
|
|
self.assertEqual(
|
|
subrelations["aggregates"]["counts"]["comment_label"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
subrelations["aggregates"]["counts"]["comment_cross_reference"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
subrelations["aggregates"]["counts"]["content_history_comment"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
subrelations["aggregates"]["external_author_provenance"][
|
|
"comment"
|
|
]["rows_with_id"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
subrelations["extra_schema_coverage"][0]["table"],
|
|
"team",
|
|
)
|
|
|
|
def test_closure_evidence_validator_binds_policy_and_rejects_drift(self):
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
_repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
try:
|
|
closure = run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
disposition = json.loads(
|
|
CLOSURE_DISPOSITION_PATH.read_text(encoding="utf-8")
|
|
)
|
|
disposition["sourceEvidence"]["semanticTopicsSha256"] = topics[
|
|
"sha256"
|
|
]
|
|
disposition["sourceEvidence"]["unsupportedRepositoryReportSha256"] = (
|
|
unsupported["sha256"]
|
|
)
|
|
disposition["sourceEvidence"]["unsupportedSchemaCatalogSha256"] = (
|
|
unsupported["report"]["schema_catalog_sha256"]
|
|
)
|
|
result = RUNNER.validate_gitea_salvage_closure_evidence(
|
|
disposition,
|
|
closure,
|
|
)
|
|
self.assertEqual(result["sha256"], closure["sha256"])
|
|
self.assertIn(
|
|
"closure-report-review-pin-pending",
|
|
result["blockers"],
|
|
)
|
|
drift = json.loads(json.dumps(closure))
|
|
deleted = next(
|
|
row
|
|
for row in drift["report"]["actor_relations"]
|
|
if row["relation"] == "collaborations"
|
|
and row["actor_class"] == "deleted"
|
|
)
|
|
deleted["disposition"] = "RECREATE_KEPT_ACTOR_AFTER_ID_MAP"
|
|
with self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"actor-relation disposition is invalid",
|
|
):
|
|
RUNNER.validate_gitea_salvage_closure_evidence(
|
|
disposition,
|
|
drift,
|
|
)
|
|
|
|
def test_closure_inventory_fails_closed_on_actor_and_join_drift(self):
|
|
cases = (
|
|
(
|
|
"orphan-actor",
|
|
"UPDATE comment SET poster_id=999999 WHERE id=300",
|
|
"actor is outside user decisions",
|
|
),
|
|
(
|
|
"unsafe-attachment-uuid",
|
|
"UPDATE attachment SET uuid='12345678-1234-1234-1234-123456789ABC' "
|
|
"WHERE id=700",
|
|
"attachment manifest row is invalid",
|
|
),
|
|
(
|
|
"cross-repository-label",
|
|
"UPDATE label SET repo_id=3 WHERE id=400",
|
|
"issue-label repository relation is invalid",
|
|
),
|
|
(
|
|
"orphan-issue-dependency-target",
|
|
"UPDATE issue_dependency SET dependency_id=999999 WHERE id=311",
|
|
"issue-dependency row is invalid",
|
|
),
|
|
(
|
|
"orphan-project-link-target",
|
|
"UPDATE project_issue SET project_id=999999 WHERE id=502",
|
|
"project-issue relation is invalid",
|
|
),
|
|
)
|
|
for label, mutation, message in cases:
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
_repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
try:
|
|
connection.execute(mutation)
|
|
with (
|
|
self.subTest(label=label),
|
|
self.assertRaisesRegex(RUNNER.DeployError, message),
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
|
|
def test_closure_inventory_polymorphic_subrelations_fail_closed(self):
|
|
cases = (
|
|
(
|
|
"orphan-comment-label",
|
|
("UPDATE comment SET label_id=999999 WHERE id=300",),
|
|
"comment_label relation is invalid",
|
|
),
|
|
(
|
|
"cross-repository-comment-project",
|
|
(
|
|
"INSERT INTO project "
|
|
"(id,repo_id,creator_id,title,description) "
|
|
"VALUES (510,{second_repo_id},{kept_user},'SECRET','SECRET')",
|
|
"UPDATE comment SET project_id=510 WHERE id=300",
|
|
),
|
|
"comment_current_project relation is invalid",
|
|
),
|
|
(
|
|
"wrong-issue-tracked-time",
|
|
(
|
|
"INSERT INTO tracked_time VALUES (313,101,{kept_user},15)",
|
|
"UPDATE comment SET time_id=313 WHERE id=300",
|
|
),
|
|
"comment tracked-time relation is invalid",
|
|
),
|
|
(
|
|
"orphan-dependent-issue",
|
|
(
|
|
"UPDATE comment SET dependent_issue_id=999999 WHERE id=300",
|
|
),
|
|
"comment dependent-issue relation is invalid",
|
|
),
|
|
(
|
|
"wrong-issue-review",
|
|
("UPDATE comment SET review_id=306 WHERE id=300",),
|
|
"comment review relation is invalid",
|
|
),
|
|
(
|
|
"cross-reference-repository-mismatch",
|
|
(
|
|
"UPDATE comment SET ref_repo_id={second_repo_id} WHERE id=300",
|
|
),
|
|
"comment cross-reference relation is invalid",
|
|
),
|
|
(
|
|
"cross-reference-comment-mismatch",
|
|
(
|
|
"INSERT INTO comment "
|
|
"(id,issue_id,poster_id,assignee_team_id,content) "
|
|
"VALUES (320,101,{kept_user},0,'SECRET_REF_COMMENT')",
|
|
"UPDATE comment SET ref_comment_id=320 WHERE id=300",
|
|
),
|
|
"comment cross-reference comment is invalid",
|
|
),
|
|
(
|
|
"unknown-comment-assignee",
|
|
("UPDATE comment SET assignee_id=999999 WHERE id=300",),
|
|
"actor is outside user decisions",
|
|
),
|
|
(
|
|
"orphan-history-comment",
|
|
(
|
|
"UPDATE issue_content_history SET comment_id=999999 "
|
|
"WHERE id=302",
|
|
),
|
|
"content-history comment relation is invalid",
|
|
),
|
|
(
|
|
"unmerged-pull-with-merger",
|
|
(
|
|
"UPDATE pull_request SET merger_id={kept_user} WHERE id=200",
|
|
),
|
|
"pull merger identity is invalid",
|
|
),
|
|
(
|
|
"merged-pull-unknown-merger",
|
|
(
|
|
"UPDATE pull_request SET has_merged=1,merger_id=999999 "
|
|
"WHERE id=200",
|
|
),
|
|
"actor is outside user decisions",
|
|
),
|
|
(
|
|
"orphan-comment-team",
|
|
(
|
|
"UPDATE comment SET assignee_id=0,assignee_team_id=999999 "
|
|
"WHERE id=300",
|
|
),
|
|
"comment assignee-team relation is orphaned",
|
|
),
|
|
(
|
|
"orphan-review-team",
|
|
(
|
|
"UPDATE review SET reviewer_id=0,reviewer_team_id=999999 "
|
|
"WHERE id=306",
|
|
),
|
|
"review reviewer-team relation is orphaned",
|
|
),
|
|
(
|
|
"conflicting-comment-assignee-identities",
|
|
(
|
|
"INSERT INTO team VALUES (10,{kept_user})",
|
|
"UPDATE comment SET assignee_team_id=10 WHERE id=300",
|
|
),
|
|
"comment assignee identities conflict",
|
|
),
|
|
(
|
|
"conflicting-reviewer-identities",
|
|
(
|
|
"INSERT INTO team VALUES (10,{kept_user})",
|
|
"UPDATE review SET reviewer_team_id=10 WHERE id=306",
|
|
),
|
|
"review subrelation source is invalid",
|
|
),
|
|
(
|
|
"team-organization-outside-decisions",
|
|
(
|
|
"INSERT INTO team VALUES (10,999999)",
|
|
"UPDATE comment SET assignee_id=0,assignee_team_id=10 "
|
|
"WHERE id=300",
|
|
),
|
|
"actor is outside user decisions",
|
|
),
|
|
(
|
|
"invalid-external-author-id",
|
|
(
|
|
"UPDATE comment SET original_author_id='invalid' WHERE id=300",
|
|
),
|
|
"comment subrelation value is invalid",
|
|
),
|
|
(
|
|
"invalid-cross-reference-action",
|
|
("UPDATE comment SET ref_action=9 WHERE id=300",),
|
|
"comment cross-reference state is invalid",
|
|
),
|
|
)
|
|
for label, statements, message in cases:
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
kept_ids = sorted(
|
|
int(row["repo_id"])
|
|
for row in decisions["kept_repositories"]
|
|
)
|
|
kept_user = next(
|
|
int(row["user_id"])
|
|
for row in decisions["users"]
|
|
if row["decision"] == "KEEP_ACTIVE"
|
|
)
|
|
try:
|
|
for statement in statements:
|
|
connection.execute(
|
|
statement.format(
|
|
kept_user=kept_user,
|
|
repo_id=repo_id,
|
|
second_repo_id=kept_ids[1],
|
|
)
|
|
)
|
|
with (
|
|
self.subTest(label=label),
|
|
self.assertRaisesRegex(RUNNER.DeployError, message),
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
|
|
def test_closure_inventory_seals_exact_team_mapping_holds(self):
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
repo_id,
|
|
deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
try:
|
|
connection.execute("INSERT INTO team VALUES (10,?)", (deleted_user,))
|
|
connection.execute(
|
|
"UPDATE comment SET assignee_id=0,assignee_team_id=10 "
|
|
"WHERE id=300"
|
|
)
|
|
connection.execute(
|
|
"UPDATE review SET reviewer_id=0,reviewer_team_id=10 "
|
|
"WHERE id=306"
|
|
)
|
|
closure = run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
subrelations = closure["report"]["issue_pr_subrelations"]
|
|
self.assertEqual(
|
|
subrelations["conditional_hold_blockers"],
|
|
["issue-pr-team-mapping-hold"],
|
|
)
|
|
self.assertEqual(
|
|
[row["kind"] for row in subrelations["holds"]],
|
|
[
|
|
"comment-assignee-team-mapping",
|
|
"review-reviewer-team-mapping",
|
|
],
|
|
)
|
|
for row in subrelations["holds"]:
|
|
self.assertEqual(row["old_org_id"], deleted_user)
|
|
self.assertEqual(row["old_org_identity_class"], "deleted")
|
|
self.assertEqual(row["old_repo_id"], repo_id)
|
|
self.assertEqual(row["old_team_id"], 10)
|
|
self.assertIn(
|
|
"issue-pr-team-mapping-hold",
|
|
closure["report"]["remaining_blockers"],
|
|
)
|
|
disposition = json.loads(
|
|
CLOSURE_DISPOSITION_PATH.read_text(encoding="utf-8")
|
|
)
|
|
disposition["sourceEvidence"]["semanticTopicsSha256"] = topics[
|
|
"sha256"
|
|
]
|
|
disposition["sourceEvidence"][
|
|
"unsupportedRepositoryReportSha256"
|
|
] = unsupported["sha256"]
|
|
disposition["sourceEvidence"]["unsupportedSchemaCatalogSha256"] = (
|
|
unsupported["report"]["schema_catalog_sha256"]
|
|
)
|
|
validated = RUNNER.validate_gitea_salvage_closure_evidence(
|
|
disposition,
|
|
closure,
|
|
)
|
|
self.assertIn("issue-pr-team-mapping-hold", validated["blockers"])
|
|
drift = json.loads(json.dumps(closure))
|
|
drift["report"]["issue_pr_subrelations"]["holds"][0][
|
|
"kind"
|
|
] = "unknown-team-mapping"
|
|
with self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"hold row is invalid",
|
|
):
|
|
RUNNER.validate_gitea_salvage_closure_evidence(
|
|
disposition,
|
|
drift,
|
|
)
|
|
|
|
def test_closure_inventory_attests_nullable_subrelation_encodings(self):
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
_repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
try:
|
|
connection.execute(
|
|
"UPDATE comment SET original_author=NULL,original_author_id=NULL "
|
|
"WHERE id=300"
|
|
)
|
|
connection.execute(
|
|
"UPDATE review SET original_author=NULL,original_author_id=NULL "
|
|
"WHERE id=306"
|
|
)
|
|
connection.execute(
|
|
"UPDATE issue_content_history SET comment_id=NULL WHERE id=302"
|
|
)
|
|
connection.execute(
|
|
"UPDATE pull_request SET merger_id=NULL WHERE id=200"
|
|
)
|
|
report = run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)["report"]
|
|
finally:
|
|
connection.close()
|
|
aggregates = report["issue_pr_subrelations"]["aggregates"]
|
|
self.assertEqual(aggregates["counts"]["content_history_comment"], 0)
|
|
self.assertEqual(aggregates["counts"]["content_history_issue"], 1)
|
|
self.assertEqual(
|
|
aggregates["null_encodings"]["comment.original_author_id"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
aggregates["null_encodings"]["review.original_author_id"],
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
aggregates["null_encodings"]["pull_request.merger_id"],
|
|
1,
|
|
)
|
|
|
|
def test_closure_inventory_team_schema_gate_rejects_absence_and_views(self):
|
|
cases = (
|
|
("missing", ("DROP TABLE team",)),
|
|
(
|
|
"view",
|
|
(
|
|
"DROP TABLE team",
|
|
"CREATE VIEW team AS SELECT 1 AS id,1 AS org_id",
|
|
),
|
|
),
|
|
(
|
|
"missing-org-id",
|
|
(
|
|
"DROP TABLE team",
|
|
"CREATE TABLE team (id INTEGER PRIMARY KEY)",
|
|
),
|
|
),
|
|
)
|
|
for label, statements in cases:
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
_repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
try:
|
|
for statement in statements:
|
|
connection.execute(statement)
|
|
with (
|
|
self.subTest(label=label),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"extra schema is unsafe",
|
|
),
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
|
|
def test_closure_inventory_packages_actions_are_metadata_only(self):
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
kept_user = next(
|
|
int(row["user_id"])
|
|
for row in decisions["users"]
|
|
if row["decision"] == "KEEP_ACTIVE"
|
|
)
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO package VALUES (900,?,?,'SECRET_PACKAGE_NAME')",
|
|
(repo_id, kept_user),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO package_version VALUES "
|
|
"(901,900,?,'SECRET_VERSION','SECRET_PACKAGE_METADATA')",
|
|
(kept_user,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO package_blob VALUES (902,100,'SECRET_BLOB_HASH')"
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO package_file VALUES (903,901,902,'SECRET_FILE_NAME')"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO package_property VALUES (?,?,?,?,?)",
|
|
(
|
|
(904, 2, 900, "SECRET_PROP", "SECRET_VALUE"),
|
|
(905, 0, 901, "SECRET_PROP", "SECRET_VALUE"),
|
|
(906, 1, 903, "SECRET_PROP", "SECRET_VALUE"),
|
|
),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_run VALUES "
|
|
"(1000,?,?,'SECRET_ACTION_EVENT_PAYLOAD')",
|
|
(repo_id, kept_user),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_schedule VALUES "
|
|
"(1001,?,?,CAST('SECRET_WORKFLOW' AS BLOB))",
|
|
(repo_id, kept_user),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_runner VALUES "
|
|
"(1002,?,'SECRET_RUNNER_TOKEN','SECRET_RUNNER_SALT')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_variable VALUES (1003,?,'SECRET_VARIABLE')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO secret VALUES (1004,?,'SECRET_ACTION_SECRET')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_artifact VALUES "
|
|
"(1005,?,1000,20,10,'SECRET_STORAGE_PATH')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_run_job VALUES "
|
|
"(1006,?,1000,CAST('SECRET_JOB_PAYLOAD' AS BLOB))",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_task VALUES "
|
|
"(1007,?,1006,50,40,'SECRET_TASK_TOKEN','SECRET_LOG_PATH')",
|
|
(repo_id,),
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_run_index VALUES (?,1)",
|
|
(repo_id,),
|
|
)
|
|
direct = unsupported["report"]["aggregates"][
|
|
"direct_relation_counts"
|
|
]
|
|
for label in (
|
|
"action_runners",
|
|
"action_runs",
|
|
"action_schedules",
|
|
"action_secrets",
|
|
"action_variables",
|
|
"packages",
|
|
):
|
|
direct[label] = 1
|
|
unsupported.update(
|
|
RUNNER.canonical_gitea_salvage_evidence(
|
|
unsupported["report"],
|
|
"closure package action predecessor report",
|
|
)
|
|
)
|
|
evidence = run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO package_property VALUES "
|
|
"(907,9,900,'SECRET_UNKNOWN_PROP','SECRET_UNKNOWN_VALUE')"
|
|
)
|
|
with self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"package-property type is invalid",
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
connection.execute("DELETE FROM package_property WHERE id=907")
|
|
deleted_repo_id = next(
|
|
int(row["repo_id"])
|
|
for row in decisions["repositories"]
|
|
if row["decision"] == "DELETE"
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO action_artifact VALUES "
|
|
"(1008,?,1000,1,1,'SECRET_INDIRECT_PATH')",
|
|
(deleted_repo_id,),
|
|
)
|
|
with self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"action_artifact indirect relation is invalid",
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
aggregates = evidence["report"]["aggregates"]
|
|
self.assertEqual(aggregates["packages"]["rows"], 1)
|
|
self.assertEqual(aggregates["package_versions"]["rows"], 1)
|
|
self.assertEqual(aggregates["package_files"]["rows"], 1)
|
|
self.assertEqual(aggregates["package_properties"]["rows"], 3)
|
|
self.assertEqual(aggregates["package_blobs"]["logical_bytes"], 100)
|
|
self.assertEqual(aggregates["action_artifacts"]["logical_bytes"], 20)
|
|
self.assertEqual(
|
|
aggregates["action_artifacts"]["numeric_totals"],
|
|
{"file_compressed_size": 10, "file_size": 20},
|
|
)
|
|
self.assertEqual(aggregates["action_tasks"]["logical_bytes"], 40)
|
|
self.assertEqual(
|
|
aggregates["action_tasks"]["numeric_totals"],
|
|
{"log_length": 50, "log_size": 40},
|
|
)
|
|
self.assertEqual(aggregates["action_run_indexes"]["rows"], 1)
|
|
for sentinel in (
|
|
"SECRET_PACKAGE_NAME",
|
|
"SECRET_VERSION",
|
|
"SECRET_PACKAGE_METADATA",
|
|
"SECRET_BLOB_HASH",
|
|
"SECRET_PROP",
|
|
"SECRET_VALUE",
|
|
"SECRET_ACTION_EVENT_PAYLOAD",
|
|
"SECRET_WORKFLOW",
|
|
"SECRET_RUNNER_TOKEN",
|
|
"SECRET_VARIABLE",
|
|
"SECRET_ACTION_SECRET",
|
|
"SECRET_STORAGE_PATH",
|
|
"SECRET_JOB_PAYLOAD",
|
|
"SECRET_TASK_TOKEN",
|
|
"SECRET_LOG_PATH",
|
|
):
|
|
self.assertNotIn(sentinel, evidence["json"])
|
|
|
|
def test_closure_inventory_actions_rejects_null_indirect_repository(self):
|
|
cases = (
|
|
(
|
|
"artifact",
|
|
(
|
|
"INSERT INTO action_artifact VALUES "
|
|
"(1001,NULL,1000,1,1,'SECRET_NULL_ARTIFACT')"
|
|
),
|
|
"action_artifact indirect relation is invalid",
|
|
),
|
|
(
|
|
"job",
|
|
(
|
|
"INSERT INTO action_run_job VALUES "
|
|
"(1001,NULL,1000,CAST('SECRET_NULL_JOB' AS BLOB))"
|
|
),
|
|
"action_run_job indirect relation is invalid",
|
|
),
|
|
(
|
|
"task",
|
|
(
|
|
"INSERT INTO action_run_job VALUES "
|
|
"(1001,{repo_id},1000,CAST('SECRET_JOB' AS BLOB));"
|
|
"INSERT INTO action_task VALUES "
|
|
"(1002,NULL,1001,1,1,'SECRET_NULL_TASK','SECRET_NULL_LOG')"
|
|
),
|
|
"action_task indirect relation is invalid",
|
|
),
|
|
)
|
|
for label, mutation, message in cases:
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
kept_user = next(
|
|
int(row["user_id"])
|
|
for row in decisions["users"]
|
|
if row["decision"] == "KEEP_ACTIVE"
|
|
)
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO action_run VALUES (1000,?,?,?)",
|
|
(repo_id, kept_user, "SECRET_RUN"),
|
|
)
|
|
for statement in mutation.format(repo_id=repo_id).split(";"):
|
|
connection.execute(statement)
|
|
unsupported["report"]["aggregates"]["direct_relation_counts"][
|
|
"action_runs"
|
|
] = 1
|
|
unsupported.update(
|
|
RUNNER.canonical_gitea_salvage_evidence(
|
|
unsupported["report"],
|
|
"closure null indirect repository predecessor report",
|
|
)
|
|
)
|
|
with (
|
|
self.subTest(label=label),
|
|
self.assertRaisesRegex(RUNNER.DeployError, message),
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
|
|
def test_closure_inventory_actions_bounds_every_declared_numeric_value(self):
|
|
cases = (
|
|
(
|
|
"negative-compressed-size",
|
|
"INSERT INTO action_artifact VALUES "
|
|
"(1001,{repo_id},1000,1,-1,'SECRET_ARTIFACT')",
|
|
),
|
|
(
|
|
"oversized-log-length",
|
|
"INSERT INTO action_run_job VALUES "
|
|
"(1001,{repo_id},1000,CAST('SECRET_JOB' AS BLOB));"
|
|
"INSERT INTO action_task VALUES "
|
|
"(1002,{repo_id},1001,{oversized},1,'SECRET_TASK','SECRET_LOG')",
|
|
),
|
|
)
|
|
for label, mutation in cases:
|
|
(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
repo_id,
|
|
_deleted_user,
|
|
) = populated_closure_evidence_fixture()
|
|
kept_user = next(
|
|
int(row["user_id"])
|
|
for row in decisions["users"]
|
|
if row["decision"] == "KEEP_ACTIVE"
|
|
)
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO action_run VALUES (1000,?,?,?)",
|
|
(repo_id, kept_user, "SECRET_RUN"),
|
|
)
|
|
for statement in mutation.format(
|
|
repo_id=repo_id,
|
|
oversized=(
|
|
RUNNER.GITEA_SALVAGE_UNSUPPORTED_SIZE_PER_ROW_MAX_BYTES
|
|
+ 1
|
|
),
|
|
).split(";"):
|
|
connection.execute(statement)
|
|
unsupported["report"]["aggregates"]["direct_relation_counts"][
|
|
"action_runs"
|
|
] = 1
|
|
unsupported.update(
|
|
RUNNER.canonical_gitea_salvage_evidence(
|
|
unsupported["report"],
|
|
"closure numeric bounds predecessor report",
|
|
)
|
|
)
|
|
with (
|
|
self.subTest(label=label),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"declared value is invalid",
|
|
),
|
|
):
|
|
run_closure_inventory_fixture(
|
|
connection,
|
|
decisions,
|
|
unsupported,
|
|
topics,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
|
|
def test_container_absence_parser_rejects_daemon_and_permission_errors(self):
|
|
with mock.patch.object(
|
|
RUNNER.subprocess,
|
|
"run",
|
|
return_value=completed(
|
|
1,
|
|
stderr="Error response from daemon: No such container: candidate",
|
|
),
|
|
):
|
|
self.assertIsNone(
|
|
RUNNER.docker_named_container_inspect_fail_closed(
|
|
"candidate",
|
|
"candidate inspect",
|
|
)
|
|
)
|
|
for error in (
|
|
"permission denied while trying to connect to the Docker daemon",
|
|
"context deadline exceeded",
|
|
"Cannot connect to the Docker daemon",
|
|
):
|
|
with (
|
|
self.subTest(error=error),
|
|
mock.patch.object(
|
|
RUNNER.subprocess,
|
|
"run",
|
|
return_value=completed(1, stdout="[]\n", stderr=error),
|
|
),
|
|
):
|
|
with self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
r"absence is unproven: rc=1 stdout=json-empty-list "
|
|
r"stderr=(?:permission-denied|timeout|daemon-error)",
|
|
):
|
|
RUNNER.docker_named_container_inspect_fail_closed(
|
|
"candidate",
|
|
"candidate inspect",
|
|
)
|
|
|
|
def test_container_absence_parser_accepts_exact_json_empty_list_live_format_only(self):
|
|
for stderr in (
|
|
"Error: No such object: candidate\n",
|
|
"Error response from daemon: No such container: candidate\n",
|
|
):
|
|
with (
|
|
self.subTest(stderr=stderr),
|
|
mock.patch.object(
|
|
RUNNER.subprocess,
|
|
"run",
|
|
return_value=completed(1, stdout="[]\n", stderr=stderr),
|
|
),
|
|
):
|
|
self.assertIsNone(
|
|
RUNNER.docker_named_container_inspect_fail_closed(
|
|
"candidate",
|
|
"candidate inspect",
|
|
)
|
|
)
|
|
|
|
for stdout, stderr, expected_stdout, expected_stderr in (
|
|
(
|
|
"[]\n",
|
|
"Error: No such object: different-name\n",
|
|
"json-empty-list",
|
|
"other",
|
|
),
|
|
(
|
|
"[]\n",
|
|
"Error: No such object: candidate\nextra diagnostic\n",
|
|
"json-empty-list",
|
|
"other",
|
|
),
|
|
(
|
|
"[{}]\n",
|
|
"Error: No such object: candidate\n",
|
|
"json-list",
|
|
"exact-no-such-object",
|
|
),
|
|
(
|
|
"[ ]\n",
|
|
"Error: No such object: candidate\n",
|
|
"json-list",
|
|
"exact-no-such-object",
|
|
),
|
|
(
|
|
"not-json\n",
|
|
"supersecret unexpected failure\n",
|
|
"non-json",
|
|
"other",
|
|
),
|
|
):
|
|
with (
|
|
self.subTest(stdout=stdout, stderr=stderr),
|
|
mock.patch.object(
|
|
RUNNER.subprocess,
|
|
"run",
|
|
return_value=completed(1, stdout=stdout, stderr=stderr),
|
|
),
|
|
):
|
|
with self.assertRaises(RUNNER.DeployError) as raised:
|
|
RUNNER.docker_named_container_inspect_fail_closed(
|
|
"candidate",
|
|
"candidate inspect",
|
|
)
|
|
message = str(raised.exception)
|
|
self.assertIn(f"stdout={expected_stdout}", message)
|
|
self.assertIn(f"stderr={expected_stderr}", message)
|
|
self.assertNotIn("supersecret", message)
|
|
|
|
def test_snapshot_gate_uses_proven_btrfs_property_form_and_accepts_contained_mode(self):
|
|
class FakePath:
|
|
def __init__(self, value, mode, uid=0, gid=0, size=0, parent=None):
|
|
self.value = value
|
|
self._stat = type(
|
|
"FakeStat",
|
|
(),
|
|
{
|
|
"st_mode": mode,
|
|
"st_uid": uid,
|
|
"st_gid": gid,
|
|
"st_size": size,
|
|
},
|
|
)()
|
|
self.parent = parent
|
|
|
|
def lstat(self):
|
|
return self._stat
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
parent = FakePath("/snapshots", stat.S_IFDIR | 0o700)
|
|
root = FakePath("/snapshots/incident", stat.S_IFDIR | 0o755, parent=parent)
|
|
database = FakePath(
|
|
"/snapshots/incident/gitea/gitea/gitea.db",
|
|
stat.S_IFREG | 0o600,
|
|
uid=1000,
|
|
gid=1000,
|
|
size=RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_BYTES,
|
|
)
|
|
repositories = FakePath(
|
|
"/snapshots/incident/gitea/git/repositories",
|
|
stat.S_IFDIR | 0o755,
|
|
uid=1000,
|
|
gid=1000,
|
|
)
|
|
btrfs = mock.MagicMock()
|
|
btrfs.is_file.return_value = True
|
|
btrfs.__str__.return_value = "/usr/sbin/btrfs"
|
|
calls = []
|
|
|
|
def run(command, **_kwargs):
|
|
calls.append(command)
|
|
if command[1:3] == ["subvolume", "show"]:
|
|
return completed(
|
|
0,
|
|
stdout=f"UUID: {RUNNER.GITEA_SALVAGE_SNAPSHOT_UUID}\n",
|
|
)
|
|
if command[1:3] == ["property", "get"]:
|
|
return completed(0, stdout="ro=true\n")
|
|
raise AssertionError(command)
|
|
|
|
with (
|
|
mock.patch.object(RUNNER, "GITEA_SALVAGE_SNAPSHOT_ROOT", root),
|
|
mock.patch.object(RUNNER, "GITEA_SALVAGE_SNAPSHOT_DATABASE", database),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"GITEA_SALVAGE_SNAPSHOT_REPOSITORIES",
|
|
repositories,
|
|
),
|
|
mock.patch.object(RUNNER, "GITEA_SALVAGE_BTRFS", btrfs),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"sha256_file",
|
|
return_value=RUNNER.GITEA_SALVAGE_SNAPSHOT_DATABASE_SHA256,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"validate_gitea_salvage_path_chain",
|
|
side_effect=lambda _root, target, *_args, **_kwargs: target,
|
|
),
|
|
mock.patch.object(RUNNER.subprocess, "run", side_effect=run),
|
|
):
|
|
result = RUNNER.validate_gitea_salvage_snapshot_boundary()
|
|
self.assertTrue(result["readonly"])
|
|
self.assertEqual(
|
|
calls[1],
|
|
["/usr/sbin/btrfs", "property", "get", str(root), "ro"],
|
|
)
|
|
|
|
def test_path_chain_rejects_symlink_parent_and_allows_only_missing_final(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-path-chain-") as directory:
|
|
base = Path(directory)
|
|
root = base / "root"
|
|
outside = base / "outside"
|
|
root.mkdir()
|
|
outside.mkdir()
|
|
(outside / "repo.git").mkdir()
|
|
(root / "owner").symlink_to(outside, target_is_directory=True)
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_mountpoints",
|
|
return_value=set(),
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, "contains a symlink"),
|
|
):
|
|
RUNNER.validate_gitea_salvage_path_chain(
|
|
root,
|
|
root / "owner" / "repo.git",
|
|
stat.S_ISDIR,
|
|
"repository source owner/repo.git",
|
|
)
|
|
|
|
(root / "owner").unlink()
|
|
(root / "owner").mkdir()
|
|
with mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_mountpoints",
|
|
return_value=set(),
|
|
):
|
|
self.assertIsNone(
|
|
RUNNER.validate_gitea_salvage_path_chain(
|
|
root,
|
|
root / "owner" / "missing.wiki.git",
|
|
stat.S_ISDIR,
|
|
"wiki source owner/missing.wiki.git",
|
|
allow_missing_final=True,
|
|
)
|
|
)
|
|
|
|
def test_path_chain_rejects_mount_filesystem_and_nested_subvolume_boundaries(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-boundary-") as directory:
|
|
root = Path(directory) / "root"
|
|
owner = root / "owner"
|
|
repo = owner / "repo.git"
|
|
repo.mkdir(parents=True)
|
|
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_mountpoints",
|
|
return_value={str(owner)},
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, "mount boundary"),
|
|
):
|
|
RUNNER.validate_gitea_salvage_path_chain(
|
|
root,
|
|
repo,
|
|
stat.S_ISDIR,
|
|
"repository source owner/repo.git",
|
|
)
|
|
|
|
original_lstat = Path.lstat
|
|
real_owner_stat = owner.lstat()
|
|
|
|
def lstat_with_owner(**changes):
|
|
values = {
|
|
name: getattr(real_owner_stat, name)
|
|
for name in dir(real_owner_stat)
|
|
if name.startswith("st_")
|
|
}
|
|
values.update(changes)
|
|
return type("BoundaryStat", (), values)()
|
|
|
|
def nested_subvolume_lstat(path):
|
|
if path == owner:
|
|
return lstat_with_owner(st_ino=256)
|
|
return original_lstat(path)
|
|
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_mountpoints",
|
|
return_value=set(),
|
|
),
|
|
mock.patch.object(Path, "lstat", nested_subvolume_lstat),
|
|
self.assertRaisesRegex(RUNNER.DeployError, "nested subvolume boundary"),
|
|
):
|
|
RUNNER.validate_gitea_salvage_path_chain(
|
|
root,
|
|
repo,
|
|
stat.S_ISDIR,
|
|
"repository source owner/repo.git",
|
|
)
|
|
|
|
def foreign_device_lstat(path):
|
|
if path == owner:
|
|
return lstat_with_owner(st_dev=real_owner_stat.st_dev + 1)
|
|
return original_lstat(path)
|
|
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_mountpoints",
|
|
return_value=set(),
|
|
),
|
|
mock.patch.object(Path, "lstat", foreign_device_lstat),
|
|
self.assertRaisesRegex(RUNNER.DeployError, "filesystem boundary"),
|
|
):
|
|
RUNNER.validate_gitea_salvage_path_chain(
|
|
root,
|
|
repo,
|
|
stat.S_ISDIR,
|
|
"repository source owner/repo.git",
|
|
)
|
|
|
|
def test_ref_parser_reconstructs_manifest_and_rejects_alternates(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-repo-") as directory:
|
|
repo = Path(directory) / "owner" / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
ref_file = repo / "refs" / "heads" / "main"
|
|
object_file.parent.mkdir(parents=True)
|
|
ref_file.parent.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
oid = "a" * 40
|
|
ref_file.write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
with mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
):
|
|
inventory = inventory_repository(
|
|
repo,
|
|
"owner/repo.git",
|
|
)
|
|
self.assertEqual(inventory["head"], "refs/heads/main")
|
|
self.assertEqual(inventory["refs"], [{"name": "refs/heads/main", "oid": oid}])
|
|
self.assertEqual(inventory["object_format"], "sha1")
|
|
alternates = repo / "objects" / "info" / "alternates"
|
|
alternates.parent.mkdir()
|
|
alternates.write_text("/evil\n", encoding="ascii")
|
|
with self.assertRaisesRegex(RUNNER.DeployError, "forbidden Git material"):
|
|
inventory_repository(
|
|
repo,
|
|
"owner/repo.git",
|
|
)
|
|
|
|
def test_review_evidence_ascii_escapes_unicode_and_rejects_surrogate_refname(self):
|
|
refname = "refs/heads/ветка-\u202ereview"
|
|
self.assertTrue(RUNNER.gitea_salvage_refname_is_safe(refname))
|
|
evidence = RUNNER.canonical_gitea_salvage_evidence(
|
|
{"ref": refname},
|
|
"test reference manifest",
|
|
)
|
|
evidence["json"].encode("ascii")
|
|
self.assertNotIn("\u202e", evidence["json"])
|
|
self.assertIn("\\u202e", evidence["json"])
|
|
self.assertIn("\\u0432", evidence["json"])
|
|
self.assertFalse(
|
|
RUNNER.gitea_salvage_refname_is_safe("refs/heads/unsafe-\udcff")
|
|
)
|
|
|
|
def test_internal_object_and_ref_walk_boundaries_fail_closed(self):
|
|
cases = (
|
|
("objects-root-device", "objects", "device"),
|
|
("object-directory-subvolume", "objects/aa", "subvolume"),
|
|
("object-file-mount", "objects/aa/" + "b" * 38, "mount"),
|
|
("refs-root-device", "refs", "device"),
|
|
("ref-directory-subvolume", "refs/heads", "subvolume"),
|
|
("ref-file-mount", "refs/heads/main", "mount"),
|
|
(
|
|
"quarantine-subvolume",
|
|
"objects/tmp_objdir-incoming-abc123",
|
|
"subvolume",
|
|
),
|
|
)
|
|
for label, target_relative, boundary in cases:
|
|
with self.subTest(label=label), tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-internal-boundary-"
|
|
) as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
ref_file = repo / "refs" / "heads" / "main"
|
|
object_file.parent.mkdir(parents=True)
|
|
ref_file.parent.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
ref_file.write_text("a" * 40 + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text(
|
|
"ref: refs/heads/main\n",
|
|
encoding="ascii",
|
|
)
|
|
if label == "quarantine-subvolume":
|
|
(repo / target_relative).mkdir()
|
|
target = repo / target_relative
|
|
trusted_device = repo.lstat().st_dev
|
|
mountpoints = {os.path.normpath(str(target))} if boundary == "mount" else set()
|
|
real_lstat = Path.lstat
|
|
|
|
def boundary_lstat(path):
|
|
value = real_lstat(path)
|
|
if path != target or boundary == "mount":
|
|
return value
|
|
result = mock.Mock()
|
|
for attribute in (
|
|
"st_mode",
|
|
"st_uid",
|
|
"st_gid",
|
|
"st_nlink",
|
|
"st_size",
|
|
"st_dev",
|
|
"st_ino",
|
|
):
|
|
setattr(result, attribute, getattr(value, attribute))
|
|
if boundary == "device":
|
|
result.st_dev = trusted_device + 1
|
|
else:
|
|
result.st_ino = 256
|
|
return result
|
|
|
|
expected = {
|
|
"device": "crosses a filesystem boundary",
|
|
"mount": "crosses a mount boundary",
|
|
"subvolume": "crosses a nested subvolume boundary",
|
|
}[boundary]
|
|
with (
|
|
mock.patch.object(Path, "lstat", boundary_lstat),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, expected),
|
|
):
|
|
inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
trusted_device=trusted_device,
|
|
mountpoints=mountpoints,
|
|
)
|
|
|
|
def test_forbidden_path_probes_never_cross_unvalidated_ancestors(self):
|
|
cases = (
|
|
("objects/info", "objects/info/alternates"),
|
|
("info", "info/grafts"),
|
|
("refs", "refs/replace"),
|
|
)
|
|
for boundary_kind in ("symlink", "mount"):
|
|
for ancestor_relative, forbidden_relative in cases:
|
|
with (
|
|
self.subTest(
|
|
boundary_kind=boundary_kind,
|
|
ancestor=ancestor_relative,
|
|
),
|
|
tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-forbidden-ancestor-"
|
|
) as directory,
|
|
):
|
|
base = Path(directory)
|
|
repo = base / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
object_file.parent.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
(repo / "HEAD").write_text(
|
|
"ref: refs/heads/main\n",
|
|
encoding="ascii",
|
|
)
|
|
ancestor = repo / ancestor_relative
|
|
ancestor.parent.mkdir(parents=True, exist_ok=True)
|
|
if boundary_kind == "symlink":
|
|
outside = base / "outside"
|
|
outside.mkdir()
|
|
ancestor.symlink_to(outside, target_is_directory=True)
|
|
mountpoints = set()
|
|
expected = "has an unsafe type"
|
|
else:
|
|
ancestor.mkdir()
|
|
mountpoints = {os.path.normpath(str(ancestor))}
|
|
expected = "crosses a mount boundary"
|
|
forbidden = repo / forbidden_relative
|
|
real_lstat = Path.lstat
|
|
|
|
def reject_nested_probe(path):
|
|
if path == forbidden:
|
|
raise AssertionError(
|
|
f"nested forbidden path was probed: {path}"
|
|
)
|
|
return real_lstat(path)
|
|
|
|
with (
|
|
mock.patch.object(Path, "lstat", reject_nested_probe),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, expected),
|
|
):
|
|
inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
mountpoints=mountpoints,
|
|
)
|
|
|
|
def test_object_boundary_requires_pack_idx_pair_and_rejects_promisor(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-pack-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
pack_root = repo / "objects" / "pack"
|
|
ref_root = repo / "refs" / "heads"
|
|
pack_root.mkdir(parents=True)
|
|
ref_root.mkdir(parents=True)
|
|
digest = "a" * 40
|
|
oid = "b" * 40
|
|
(pack_root / f"pack-{digest}.pack").write_bytes(b"pack")
|
|
(ref_root / "main").write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, "incomplete pack/index pair"),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
(pack_root / f"pack-{digest}.idx").write_bytes(b"idx")
|
|
with mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
):
|
|
inventory = inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
self.assertEqual(inventory["object_files"], 2)
|
|
(pack_root / f"pack-{digest}.promisor").write_bytes(b"promisor")
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
r"promisor object material is forbidden: repo\.git/objects/pack/"
|
|
r"pack-[a-f0-9]+\.promisor",
|
|
),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
def test_object_info_packs_is_validated_excluded_derived_cache(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-info-packs-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
info_packs = repo / "objects" / "info" / "packs"
|
|
ref_file = repo / "refs" / "heads" / "main"
|
|
object_file.parent.mkdir(parents=True)
|
|
info_packs.parent.mkdir(parents=True)
|
|
ref_file.parent.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
info_packs.write_text(
|
|
f"P pack-{'c' * 40}.pack\n\n",
|
|
encoding="ascii",
|
|
)
|
|
oid = "a" * 40
|
|
ref_file.write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
with mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
):
|
|
inventory = inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
self.assertEqual(inventory["object_files"], 1)
|
|
self.assertEqual(inventory["object_bytes"], len(b"object"))
|
|
self.assertEqual(
|
|
inventory["excluded_derived_files"],
|
|
[
|
|
{
|
|
"bytes": len(f"P pack-{'c' * 40}.pack\n\n"),
|
|
"kind": "dumb-http-pack-list",
|
|
"path": "objects/info/packs",
|
|
}
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
inventory["excluded_derived_bytes"],
|
|
len(f"P pack-{'c' * 40}.pack\n\n"),
|
|
)
|
|
|
|
def test_observed_bitmap_and_commit_graph_are_exact_bounded_exclusions(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-derived-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
pack_root = repo / "objects" / "pack"
|
|
info_root = repo / "objects" / "info"
|
|
ref_root = repo / "refs" / "heads"
|
|
pack_root.mkdir(parents=True)
|
|
info_root.mkdir(parents=True)
|
|
ref_root.mkdir(parents=True)
|
|
digest = "a" * 40
|
|
oid = "b" * 40
|
|
pack_bytes = b"pack"
|
|
idx_bytes = b"idx"
|
|
bitmap_bytes = b"bitmap"
|
|
graph_bytes = b"commit-graph"
|
|
info_packs_bytes = b"P pack-derived.pack\n\n"
|
|
(pack_root / f"pack-{digest}.pack").write_bytes(pack_bytes)
|
|
(pack_root / f"pack-{digest}.idx").write_bytes(idx_bytes)
|
|
bitmap = pack_root / f"pack-{digest}.bitmap"
|
|
commit_graph = info_root / "commit-graph"
|
|
info_packs = info_root / "packs"
|
|
bitmap.write_bytes(bitmap_bytes)
|
|
commit_graph.write_bytes(graph_bytes)
|
|
info_packs.write_bytes(info_packs_bytes)
|
|
(ref_root / "main").write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
real_read_bytes = Path.read_bytes
|
|
|
|
def reject_derived_reads(path):
|
|
if path in {bitmap, commit_graph, info_packs}:
|
|
raise AssertionError(f"derived bytes were read: {path}")
|
|
return real_read_bytes(path)
|
|
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
mock.patch.object(Path, "read_bytes", reject_derived_reads),
|
|
):
|
|
inventory = inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
self.assertEqual(inventory["object_files"], 2)
|
|
self.assertEqual(inventory["object_bytes"], len(pack_bytes) + len(idx_bytes))
|
|
self.assertEqual(
|
|
inventory["excluded_derived_files"],
|
|
[
|
|
{
|
|
"bytes": len(graph_bytes),
|
|
"kind": "commit-graph",
|
|
"path": "objects/info/commit-graph",
|
|
},
|
|
{
|
|
"bytes": len(info_packs_bytes),
|
|
"kind": "dumb-http-pack-list",
|
|
"path": "objects/info/packs",
|
|
},
|
|
{
|
|
"bytes": len(bitmap_bytes),
|
|
"kind": "pack-bitmap",
|
|
"path": f"objects/pack/pack-{digest}.bitmap",
|
|
},
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
inventory["excluded_derived_bytes"],
|
|
len(bitmap_bytes) + len(graph_bytes) + len(info_packs_bytes),
|
|
)
|
|
|
|
def test_pack_bitmap_requires_exact_complete_pack_pair(self):
|
|
for present_members in (set(), {"pack"}, {"idx"}):
|
|
with self.subTest(present_members=present_members), tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-orphan-bitmap-"
|
|
) as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
pack_root = repo / "objects" / "pack"
|
|
ref_root = repo / "refs" / "heads"
|
|
pack_root.mkdir(parents=True)
|
|
ref_root.mkdir(parents=True)
|
|
digest = "a" * 40
|
|
oid = "b" * 40
|
|
for member in present_members:
|
|
(pack_root / f"pack-{digest}.{member}").write_bytes(member.encode())
|
|
(pack_root / f"pack-{digest}.bitmap").write_bytes(b"bitmap")
|
|
(ref_root / "main").write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text(
|
|
"ref: refs/heads/main\n",
|
|
encoding="ascii",
|
|
)
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"pack bitmap lacks a complete pack/index pair",
|
|
),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
def test_unobserved_accelerators_and_malformed_temporary_dirs_remain_blocked(self):
|
|
rejected = (
|
|
"pack/pack-" + "a" * 40 + ".rev",
|
|
"pack/pack-" + "a" * 40 + ".mtimes",
|
|
"pack/pack-" + "a" * 40 + ".keep",
|
|
"pack/multi-pack-index",
|
|
)
|
|
for relative in rejected:
|
|
with self.subTest(relative=relative), tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-unobserved-derived-"
|
|
) as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
rejected_path = repo / "objects" / relative
|
|
rejected_path.parent.mkdir(parents=True)
|
|
rejected_path.write_bytes(b"unreviewed")
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"unexpected object material",
|
|
),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-temp-object-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
(repo / "objects" / "tmp_objdir-incoming-incident").mkdir(parents=True)
|
|
with self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
r"unexpected object directory: repo\.git/objects/"
|
|
r"tmp_objdir-incoming-incident",
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
def test_exact_receive_quarantine_is_stat_bound_and_not_traversed(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-quarantine-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
quarantine = repo / "objects" / "tmp_objdir-incoming-aB09zZ"
|
|
ref_file = repo / "refs" / "heads" / "main"
|
|
object_file.parent.mkdir(parents=True)
|
|
quarantine.mkdir(parents=True)
|
|
ref_file.parent.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
# Deliberately invalid main-ODB material proves that os.walk prunes the
|
|
# approved receive-pack quarantine without reading or classifying it.
|
|
(quarantine / "pack").mkdir()
|
|
(quarantine / "pack" / "orphan.pack").write_bytes(b"ignored quarantine")
|
|
oid = "a" * 40
|
|
ref_file.write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
real_lstat = Path.lstat
|
|
|
|
def pinned_owner_lstat(path):
|
|
value = real_lstat(path)
|
|
if path != quarantine:
|
|
return value
|
|
result = mock.Mock()
|
|
result.st_mode = value.st_mode
|
|
result.st_uid = 1000
|
|
result.st_gid = 1000
|
|
result.st_nlink = 1
|
|
result.st_size = value.st_size
|
|
result.st_dev = value.st_dev
|
|
result.st_ino = value.st_ino
|
|
return result
|
|
|
|
with (
|
|
mock.patch.object(Path, "lstat", pinned_owner_lstat),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
):
|
|
inventory = inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
self.assertEqual(inventory["object_files"], 1)
|
|
self.assertEqual(
|
|
inventory["excluded_quarantine_directories"],
|
|
[
|
|
{
|
|
"kind": "receive-pack-quarantine",
|
|
"lstat": {
|
|
"gid": 1000,
|
|
"mode": "0755",
|
|
"nlink": 1,
|
|
"size": real_lstat(quarantine).st_size,
|
|
"uid": 1000,
|
|
},
|
|
"path": "objects/tmp_objdir-incoming-aB09zZ",
|
|
}
|
|
],
|
|
)
|
|
|
|
def test_exact_receive_quarantine_rejects_structural_drift(self):
|
|
for unsafe_kind in ("mode", "nocow"):
|
|
with self.subTest(unsafe_kind=unsafe_kind), tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-quarantine-unsafe-"
|
|
) as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
quarantine = repo / "objects" / "tmp_objdir-incoming-abc123"
|
|
quarantine.mkdir(parents=True)
|
|
if unsafe_kind == "mode":
|
|
quarantine.chmod(0o700)
|
|
real_lstat = Path.lstat
|
|
|
|
def pinned_owner_lstat(path):
|
|
value = real_lstat(path)
|
|
if path != quarantine:
|
|
return value
|
|
result = mock.Mock()
|
|
result.st_mode = value.st_mode
|
|
result.st_uid = 1000
|
|
result.st_gid = 1000
|
|
result.st_nlink = 1
|
|
result.st_size = value.st_size
|
|
result.st_dev = value.st_dev
|
|
result.st_ino = value.st_ino
|
|
return result
|
|
|
|
with (
|
|
mock.patch.object(Path, "lstat", pinned_owner_lstat),
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=unsafe_kind == "nocow",
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"receive quarantine directory is unsafe",
|
|
),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
def test_observed_derived_exclusions_reject_unsafe_metadata(self):
|
|
for cache_kind in ("bitmap", "commit-graph"):
|
|
for unsafe_kind in (
|
|
"directory",
|
|
"symlink",
|
|
"hardlink",
|
|
"fifo",
|
|
"nocow",
|
|
"oversized",
|
|
):
|
|
with (
|
|
self.subTest(cache_kind=cache_kind, unsafe_kind=unsafe_kind),
|
|
tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-derived-unsafe-"
|
|
) as directory,
|
|
):
|
|
base = Path(directory)
|
|
repo = base / "repo.git"
|
|
pack_root = repo / "objects" / "pack"
|
|
info_root = repo / "objects" / "info"
|
|
ref_root = repo / "refs" / "heads"
|
|
pack_root.mkdir(parents=True)
|
|
info_root.mkdir(parents=True)
|
|
ref_root.mkdir(parents=True)
|
|
digest = "a" * 40
|
|
oid = "b" * 40
|
|
(pack_root / f"pack-{digest}.pack").write_bytes(b"pack")
|
|
(pack_root / f"pack-{digest}.idx").write_bytes(b"idx")
|
|
cache = (
|
|
pack_root / f"pack-{digest}.bitmap"
|
|
if cache_kind == "bitmap"
|
|
else info_root / "commit-graph"
|
|
)
|
|
if unsafe_kind == "directory":
|
|
cache.mkdir()
|
|
elif unsafe_kind == "symlink":
|
|
cache.symlink_to("/dev/null")
|
|
elif unsafe_kind == "hardlink":
|
|
source = base / "hardlink-source"
|
|
source.write_bytes(b"derived")
|
|
os.link(source, cache)
|
|
elif unsafe_kind == "fifo":
|
|
os.mkfifo(cache)
|
|
elif unsafe_kind == "oversized":
|
|
limit = (
|
|
RUNNER.GITEA_SALVAGE_DERIVED_PACK_BITMAP_MAX_BYTES
|
|
if cache_kind == "bitmap"
|
|
else RUNNER.GITEA_SALVAGE_DERIVED_COMMIT_GRAPH_MAX_BYTES
|
|
)
|
|
with cache.open("wb") as handle:
|
|
handle.truncate(limit + 1)
|
|
else:
|
|
cache.write_bytes(b"derived")
|
|
(ref_root / "main").write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text(
|
|
"ref: refs/heads/main\n",
|
|
encoding="ascii",
|
|
)
|
|
|
|
def has_nocow(path):
|
|
return unsafe_kind == "nocow" and path == cache
|
|
|
|
expected = (
|
|
"unexpected object directory"
|
|
if unsafe_kind == "directory"
|
|
else "derived object cache is oversized"
|
|
if unsafe_kind == "oversized"
|
|
else "has an unsafe type"
|
|
if unsafe_kind in {"symlink", "fifo"}
|
|
else "object file is unsafe"
|
|
)
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
side_effect=has_nocow,
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, expected),
|
|
):
|
|
inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
|
|
def test_object_info_packs_rejects_unsafe_metadata_variants(self):
|
|
for unsafe_kind in (
|
|
"directory",
|
|
"symlink",
|
|
"hardlink",
|
|
"fifo",
|
|
"nocow",
|
|
"oversized",
|
|
):
|
|
with self.subTest(unsafe_kind=unsafe_kind), tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-info-packs-unsafe-"
|
|
) as directory:
|
|
base = Path(directory)
|
|
repo = base / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
info_packs = repo / "objects" / "info" / "packs"
|
|
ref_file = repo / "refs" / "heads" / "main"
|
|
object_file.parent.mkdir(parents=True)
|
|
info_packs.parent.mkdir(parents=True)
|
|
ref_file.parent.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
if unsafe_kind == "directory":
|
|
info_packs.mkdir()
|
|
elif unsafe_kind == "symlink":
|
|
info_packs.symlink_to("/dev/null")
|
|
elif unsafe_kind == "hardlink":
|
|
hardlink_source = base / "hardlink-source"
|
|
hardlink_source.write_bytes(b"derived")
|
|
os.link(hardlink_source, info_packs)
|
|
elif unsafe_kind == "fifo":
|
|
os.mkfifo(info_packs)
|
|
elif unsafe_kind == "oversized":
|
|
with info_packs.open("wb") as handle:
|
|
handle.truncate(
|
|
RUNNER.GITEA_SALVAGE_DERIVED_INFO_PACKS_MAX_BYTES + 1
|
|
)
|
|
else:
|
|
info_packs.write_bytes(b"derived")
|
|
oid = "a" * 40
|
|
ref_file.write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text(
|
|
"ref: refs/heads/main\n",
|
|
encoding="ascii",
|
|
)
|
|
expected = (
|
|
"unexpected object directory"
|
|
if unsafe_kind == "directory"
|
|
else "derived object cache is oversized"
|
|
if unsafe_kind == "oversized"
|
|
else "has an unsafe type"
|
|
if unsafe_kind in {"symlink", "fifo"}
|
|
else "object file is unsafe"
|
|
)
|
|
|
|
def has_nocow(path):
|
|
return unsafe_kind == "nocow" and path == info_packs
|
|
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
side_effect=has_nocow,
|
|
),
|
|
self.assertRaisesRegex(RUNNER.DeployError, expected),
|
|
):
|
|
inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
|
|
def test_object_walk_error_fails_closed(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-object-walk-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
(repo / "objects").mkdir(parents=True)
|
|
|
|
def failing_walk(
|
|
root,
|
|
topdown=True,
|
|
onerror=None,
|
|
followlinks=False,
|
|
):
|
|
self.assertEqual(Path(root), repo / "objects")
|
|
self.assertTrue(topdown)
|
|
self.assertFalse(followlinks)
|
|
self.assertIsNotNone(onerror)
|
|
onerror(OSError(5, "simulated I/O error", str(root)))
|
|
return ()
|
|
|
|
with (
|
|
mock.patch.object(RUNNER.os, "walk", side_effect=failing_walk),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
r"repository objects repo\.git traversal failed: errno=5",
|
|
),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
def test_loose_ref_walk_error_fails_closed(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-ref-walk-") as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
object_file = repo / "objects" / "aa" / ("b" * 38)
|
|
loose_refs = repo / "refs"
|
|
object_file.parent.mkdir(parents=True)
|
|
loose_refs.mkdir(parents=True)
|
|
object_file.write_bytes(b"object")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
real_walk = RUNNER.os.walk
|
|
|
|
def selective_walk(
|
|
root,
|
|
topdown=True,
|
|
onerror=None,
|
|
followlinks=False,
|
|
):
|
|
if Path(root) == loose_refs:
|
|
self.assertIsNotNone(onerror)
|
|
onerror(PermissionError(13, "simulated denial", str(root)))
|
|
return ()
|
|
return real_walk(
|
|
root,
|
|
topdown=topdown,
|
|
onerror=onerror,
|
|
followlinks=followlinks,
|
|
)
|
|
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=False,
|
|
),
|
|
mock.patch.object(
|
|
RUNNER.os,
|
|
"walk",
|
|
side_effect=selective_walk,
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
r"repository refs repo\.git traversal failed: errno=13",
|
|
),
|
|
):
|
|
inventory_repository(repo, "repo.git")
|
|
|
|
def test_object_boundary_rejects_symlink_special_and_nocow(self):
|
|
for unsafe_kind in ("symlink", "fifo", "nocow"):
|
|
with self.subTest(unsafe_kind=unsafe_kind), tempfile.TemporaryDirectory(
|
|
prefix="gitea-salvage-object-unsafe-"
|
|
) as directory:
|
|
repo = Path(directory) / "repo.git"
|
|
object_root = repo / "objects" / "aa"
|
|
ref_root = repo / "refs" / "heads"
|
|
object_root.mkdir(parents=True)
|
|
ref_root.mkdir(parents=True)
|
|
object_file = object_root / ("b" * 38)
|
|
if unsafe_kind == "symlink":
|
|
object_file.symlink_to("/dev/null")
|
|
elif unsafe_kind == "fifo":
|
|
os.mkfifo(object_file)
|
|
else:
|
|
object_file.write_bytes(b"object")
|
|
oid = "a" * 40
|
|
(ref_root / "main").write_text(oid + "\n", encoding="ascii")
|
|
(repo / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii")
|
|
with (
|
|
mock.patch.object(
|
|
RUNNER,
|
|
"gitea_salvage_file_has_nocow",
|
|
return_value=unsafe_kind == "nocow",
|
|
),
|
|
self.assertRaisesRegex(
|
|
RUNNER.DeployError,
|
|
"has an unsafe type"
|
|
if unsafe_kind in {"symlink", "fifo"}
|
|
else "object file is unsafe",
|
|
),
|
|
):
|
|
inventory_repository(
|
|
repo,
|
|
"repo.git",
|
|
)
|
|
|
|
def test_unsupported_state_report_is_per_repo_deterministic_and_size_bound(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
connection.executemany(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(
|
|
(7, "kept", "", "", "", "", 1, 0, 1, 0, 0, 0, 0, 30),
|
|
(9, "", "", "", "", "", 0, 0, 0, 0, 0, 0, 0, 10),
|
|
(100, "ignored", "", "", "", "", 99, 0, 0, 0, 0, 0, 0, 99),
|
|
),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO issue (id,repo_id) VALUES (?,?)",
|
|
((1, 7), (2, 9), (3, 100)),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO repo_unit (repo_id,type,config) VALUES (?,?,?)",
|
|
((7, 1, "DO-NOT-EXPORT-CONFIG"), (7, 2, "{}"), (100, 1, "{}")),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO lfs_meta_object (id,oid,size,repository_id) VALUES (?,?,?,?)",
|
|
(
|
|
(1, "a" * 64, 10, 7),
|
|
(2, "a" * 64, 10, 9),
|
|
(3, "b" * 64, 20, 9),
|
|
(4, "a" * 64, 10, 100),
|
|
),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO attachment "
|
|
"(id,repo_id,issue_id,release_id,comment_id,size,content_sentinel) "
|
|
"VALUES (?,?,?,?,?,?,?)",
|
|
(
|
|
(1, 7, 11, 0, 21, 5, "DO-NOT-EXPORT-CONTENT"),
|
|
(2, 9, 0, 12, 0, 7, "safe-boundary"),
|
|
(3, 100, 0, 0, 0, 99, "ignored"),
|
|
),
|
|
)
|
|
kept = (
|
|
{"repo_id": 9, "owner": "SILVER", "slug": "nine"},
|
|
{"repo_id": 7, "owner": "dctouch", "slug": "seven"},
|
|
)
|
|
|
|
def authorizer(action, _table, column, _database, _trigger):
|
|
if action == sqlite3.SQLITE_READ and column in {
|
|
"config",
|
|
"content_sentinel",
|
|
"secret_sentinel",
|
|
}:
|
|
return sqlite3.SQLITE_DENY
|
|
return sqlite3.SQLITE_OK
|
|
|
|
connection.set_authorizer(authorizer)
|
|
first = RUNNER.gitea_salvage_unsupported_state_inventory(connection, kept)
|
|
second = RUNNER.gitea_salvage_unsupported_state_inventory(
|
|
connection,
|
|
tuple(reversed(kept)),
|
|
)
|
|
finally:
|
|
connection.close()
|
|
|
|
self.assertEqual(first["json"], second["json"])
|
|
self.assertEqual(first["sha256"], second["sha256"])
|
|
self.assertEqual(json.loads(first["json"]), first["report"])
|
|
report = first["report"]
|
|
self.assertEqual(report["kept_repository_ids"], [7, 9])
|
|
self.assertEqual(
|
|
[row["old_repo_id"] for row in report["per_repository"]],
|
|
[7, 9],
|
|
)
|
|
by_id = {row["old_repo_id"]: row for row in report["per_repository"]}
|
|
self.assertEqual(by_id[7]["counts"]["issues"], 1)
|
|
self.assertEqual(by_id[9]["counts"]["issues"], 1)
|
|
self.assertEqual(by_id[7]["repo_unit_types"], {"1": 1, "2": 1})
|
|
self.assertEqual(by_id[7]["lfs"]["association_logical_bytes"], 10)
|
|
self.assertEqual(by_id[9]["lfs"]["association_logical_bytes"], 30)
|
|
self.assertEqual(report["aggregates"]["lfs"]["association_rows"], 3)
|
|
self.assertEqual(
|
|
report["aggregates"]["lfs"]["association_logical_bytes"],
|
|
40,
|
|
)
|
|
self.assertEqual(report["aggregates"]["lfs"]["distinct_oids"], 2)
|
|
self.assertEqual(report["aggregates"]["lfs"]["unique_logical_bytes"], 30)
|
|
self.assertEqual(report["aggregates"]["lfs"]["non_kept_shared_oids"], 1)
|
|
self.assertEqual(report["aggregates"]["attachments"]["association_rows"], 2)
|
|
self.assertEqual(report["aggregates"]["attachments"]["logical_bytes"], 12)
|
|
self.assertEqual(
|
|
report["aggregates"]["attachments"]["link_splits"]["issue"],
|
|
{"logical_bytes": 5, "rows": 1},
|
|
)
|
|
self.assertEqual(
|
|
report["aggregates"]["attachments"]["link_splits"]["multi_link"],
|
|
{"logical_bytes": 5, "rows": 1},
|
|
)
|
|
self.assertTrue(report["material_present"])
|
|
self.assertNotIn("material_total", first["json"])
|
|
self.assertNotIn("DO-NOT-EXPORT", first["json"])
|
|
self.assertIn("issue-pull-dependent-closure-unreviewed", report["coverage_blockers"])
|
|
self.assertFalse(report["schema_review"]["matches"])
|
|
self.assertIn(
|
|
{"label": "issues", "repository_column": "repo_id", "table": "issue"},
|
|
report["direct_relation_contract"],
|
|
)
|
|
|
|
def test_unsupported_state_report_keeps_zeroes_and_fails_schema_closed(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES "
|
|
"(7,'','','','','',0,0,0,0,0,0,0,0)"
|
|
)
|
|
connection.execute("DROP TABLE issue")
|
|
connection.execute("CREATE TABLE issue (id INTEGER PRIMARY KEY, repo_id TEXT)")
|
|
report = RUNNER.gitea_salvage_unsupported_state_inventory(
|
|
connection,
|
|
({"repo_id": 7, "owner": "dctouch", "slug": "seven"},),
|
|
)["report"]
|
|
finally:
|
|
connection.close()
|
|
record = report["per_repository"][0]
|
|
self.assertIsNone(record["counts"]["issues"])
|
|
self.assertEqual(record["counts"]["releases"], 0)
|
|
self.assertEqual(record["lfs"]["association_rows"], 0)
|
|
self.assertIn("issue.repo_id:integer-affinity", report["schema_mismatch"])
|
|
self.assertFalse(report["material_present"])
|
|
|
|
def test_unsupported_state_report_never_queries_view_as_table(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES "
|
|
"(7,'','','','','',0,0,0,0,0,0,0,0)"
|
|
)
|
|
connection.execute("DROP TABLE issue")
|
|
connection.execute(
|
|
"CREATE TABLE issue_backing (id INTEGER PRIMARY KEY, repo_id INTEGER, "
|
|
"content_sentinel TEXT)"
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO issue_backing VALUES (1,7,'DO-NOT-QUERY-VIEW-CONTENT')"
|
|
)
|
|
connection.execute("CREATE VIEW issue AS SELECT * FROM issue_backing")
|
|
|
|
def authorizer(action, table, column, _database, _trigger):
|
|
if action == sqlite3.SQLITE_READ and table == "issue_backing":
|
|
return sqlite3.SQLITE_DENY
|
|
return sqlite3.SQLITE_OK
|
|
|
|
connection.set_authorizer(authorizer)
|
|
evidence = RUNNER.gitea_salvage_unsupported_state_inventory(
|
|
connection,
|
|
({"repo_id": 7, "owner": "dctouch", "slug": "seven"},),
|
|
)
|
|
finally:
|
|
connection.close()
|
|
report = evidence["report"]
|
|
self.assertIsNone(report["per_repository"][0]["counts"]["issues"])
|
|
self.assertIn("issue:ordinary-main-table", report["schema_mismatch"])
|
|
issue_schema = next(
|
|
item
|
|
for item in report["schema_catalog"]["tables"]
|
|
if item["table"] == "issue"
|
|
)
|
|
self.assertFalse(issue_schema["ordinary_main_table"])
|
|
self.assertEqual(issue_schema["sqlite_schema_kind"], "view")
|
|
self.assertNotIn("DO-NOT-QUERY", evidence["json"])
|
|
|
|
def test_schema_object_gate_has_safe_pre_table_list_fallback(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
table = RUNNER.gitea_salvage_table_schema(
|
|
connection,
|
|
"repository",
|
|
table_list_supported=False,
|
|
)
|
|
connection.execute("DROP TABLE issue")
|
|
connection.execute("CREATE VIEW issue AS SELECT 1 AS id,7 AS repo_id")
|
|
view = RUNNER.gitea_salvage_table_schema(
|
|
connection,
|
|
"issue",
|
|
table_list_supported=False,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
self.assertTrue(table["ordinary_main_table"])
|
|
self.assertIsNone(table["table_list"])
|
|
self.assertEqual(
|
|
table["object_kind_attestation"],
|
|
"sqlite-master-nonvirtual-table",
|
|
)
|
|
self.assertFalse(view["ordinary_main_table"])
|
|
|
|
def test_snapshot_sqlite_connection_reads_back_safety_pragmas(self):
|
|
with tempfile.TemporaryDirectory(prefix="gitea-salvage-sqlite-") as directory:
|
|
database = Path(directory) / "snapshot.db"
|
|
source = sqlite3.connect(database)
|
|
try:
|
|
source.execute("CREATE TABLE evidence (id INTEGER PRIMARY KEY)")
|
|
source.commit()
|
|
finally:
|
|
source.close()
|
|
connection = RUNNER.gitea_salvage_sqlite_connection(database)
|
|
try:
|
|
self.assertEqual(connection.execute("PRAGMA query_only").fetchone()[0], 1)
|
|
self.assertEqual(
|
|
connection.execute("PRAGMA trusted_schema").fetchone()[0],
|
|
0,
|
|
)
|
|
with self.assertRaises(sqlite3.OperationalError):
|
|
connection.execute("INSERT INTO evidence VALUES (1)")
|
|
finally:
|
|
connection.close()
|
|
|
|
def test_unsupported_state_report_rejects_invalid_lfs_sizes_and_oids(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES "
|
|
"(7,'','','','','',0,0,0,0,0,0,0,0)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO lfs_meta_object (id,oid,size,repository_id) VALUES (?,?,?,7)",
|
|
((1, "A" * 64, 1), (2, "b" * 64, -1)),
|
|
)
|
|
report = RUNNER.gitea_salvage_unsupported_state_inventory(
|
|
connection,
|
|
({"repo_id": 7, "owner": "dctouch", "slug": "seven"},),
|
|
)["report"]
|
|
finally:
|
|
connection.close()
|
|
self.assertEqual(report["aggregates"]["lfs"]["invalid_oid_rows"], 1)
|
|
self.assertEqual(report["aggregates"]["lfs"]["invalid_size_rows"], 1)
|
|
self.assertIsNone(report["aggregates"]["lfs"]["association_logical_bytes"])
|
|
self.assertTrue(any("lfs_meta_object.oid" in item for item in report["anomalies"]))
|
|
self.assertTrue(any("lfs_meta_object.size" in item for item in report["anomalies"]))
|
|
|
|
def test_unsupported_state_report_rejects_invalid_related_lfs_size(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES "
|
|
"(7,'','','','','',0,0,0,0,0,0,0,0)"
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES "
|
|
"(100,'','','','','',0,0,0,0,0,0,0,0)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO lfs_meta_object (id,oid,size,repository_id) VALUES (?,?,?,?)",
|
|
((1, "a" * 64, 10, 7), (2, "a" * 64, "invalid", 100)),
|
|
)
|
|
report = RUNNER.gitea_salvage_unsupported_state_inventory(
|
|
connection,
|
|
({"repo_id": 7, "owner": "dctouch", "slug": "seven"},),
|
|
)["report"]
|
|
finally:
|
|
connection.close()
|
|
lfs = report["aggregates"]["lfs"]
|
|
self.assertEqual(lfs["invalid_size_rows"], 0)
|
|
self.assertEqual(lfs["invalid_related_size_rows"], 1)
|
|
self.assertIsNone(lfs["unique_logical_bytes"])
|
|
self.assertTrue(any("invalid_related_rows=1" in item for item in report["anomalies"]))
|
|
|
|
def test_unsupported_state_report_rejects_invalid_and_orphan_lfs_owners(self):
|
|
connection = unsupported_state_test_connection()
|
|
try:
|
|
connection.execute(
|
|
"INSERT INTO repository (id,description,website,original_url,topics,"
|
|
"avatar,num_watches,num_stars,num_issues,num_pulls,num_milestones,"
|
|
"num_projects,num_action_runs,lfs_size) VALUES "
|
|
"(7,'','','','','',0,0,0,0,0,0,0,0)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO lfs_meta_object (id,oid,size,repository_id) VALUES (?,?,?,?)",
|
|
(
|
|
(1, "a" * 64, 10, 7),
|
|
(2, "a" * 64, 10, None),
|
|
(3, "a" * 64, 10, 999),
|
|
),
|
|
)
|
|
report = RUNNER.gitea_salvage_unsupported_state_inventory(
|
|
connection,
|
|
({"repo_id": 7, "owner": "dctouch", "slug": "seven"},),
|
|
)["report"]
|
|
finally:
|
|
connection.close()
|
|
lfs = report["aggregates"]["lfs"]
|
|
self.assertEqual(lfs["invalid_related_repository_rows"], 1)
|
|
self.assertEqual(lfs["orphan_related_repository_rows"], 1)
|
|
self.assertIsNone(lfs["non_kept_shared_oids"])
|
|
self.assertTrue(any("orphan_related_rows=1" in item for item in report["anomalies"]))
|
|
|
|
def test_plan_source_exports_hash_bound_reviewable_evidence(self):
|
|
source = RUNNER_PATH.read_text(encoding="utf-8")
|
|
for required in (
|
|
"gitea_reference_manifest_bytes=",
|
|
"gitea_reference_manifest_json=",
|
|
"gitea_legacy_container_name=",
|
|
"gitea_legacy_container_state=",
|
|
"gitea_legacy_image_ref=",
|
|
"gitea_legacy_image_id=",
|
|
"gitea_legacy_mount=",
|
|
"gitea_unsupported_repository_report_sha256=",
|
|
"gitea_unsupported_repository_report_bytes=",
|
|
"gitea_unsupported_schema_catalog_sha256=",
|
|
"gitea_unsupported_repository_report_json=",
|
|
"gitea_incident_disposition_sha256=",
|
|
"gitea_incident_disposition_refs=",
|
|
"gitea_incident_disposition_remaining_blockers=",
|
|
"gitea_incident_closure_disposition_sha256=",
|
|
"gitea_incident_closure_report_sha256=",
|
|
"gitea_incident_closure_report_bytes=",
|
|
"gitea_incident_closure_report_json=",
|
|
"reference-manifest-fsck-reachability-verifier-pending",
|
|
"unsupported-schema-catalog-verifier-pending",
|
|
"closure-report-review-pin-pending",
|
|
):
|
|
self.assertIn(required, source)
|
|
|
|
def test_salvage_runtime_cannot_activate_while_review_is_open(self):
|
|
with self.assertRaisesRegex(RUNNER.DeployError, "activation is frozen"):
|
|
RUNNER.prepare_component_runtime(
|
|
"gitea",
|
|
RUNNER.GITEA_SALVAGE_ENTRIES,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|