Files
Dreierpack-Manager/tools/public_release/check_public_tree.py
T

317 lines
14 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-only
# Copyright (C) 2026 OpenBMH contributors
"""Check source distributions without printing matched credentials or file contents."""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import json
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
@dataclasses.dataclass(frozen=True, order=True)
class Finding:
path: str
line: int
rule: str
ASSET_SUFFIXES = {
".fbm", ".vga", ".blk", ".col", ".lbm", ".fnt", ".wap", ".ein", ".men", ".dat",
".exe", ".dll", ".so", ".dylib", ".wav", ".mp3", ".ogg", ".flac",
".avi", ".mov", ".mp4", ".png", ".jpg", ".jpeg", ".bmp", ".gif",
".bin", ".zip", ".7z", ".dmg", ".tar", ".gz", ".pdf",
}
PRIVATE_PREFIXES = (
("data", "original"), ("docs", "manuals"), ("docs", "analyses"),
("docs", "agents"), ("src", "ui", "validation"),
("src", "ui", "reference"), ("src", "audio", "validation"),
("re",), ("tests", "fixtures"),
)
PRIVATE_DOCUMENTS = {
"Status.md", "fltd.md", "BUILD_0.1_HANDOFF.md",
"PUBLIC_RELEASE_TODO.md", "chat.txt",
}
SOURCE_SUFFIXES = {".cpp", ".hpp", ".h", ".c", ".inc"}
SKIP_DIRECTORIES = {".git", "__pycache__", ".cache", "_deps", "node_modules"}
# Exact independently generated artwork and synthetic native-editor captures.
# The latter are reviewed against docs/qa/editor-poc/README.md; no game material.
# Keep prior reviewed hashes if an asset is intentionally revised in history.
PROJECT_ARTWORK = {
"docs/qa/editor-poc/clubs-midnight-de.png": {"aa53e9cdd9ac8f6bf18b514862ac704ff9003bd1b62466cad2ebc329b09703dc"},
"docs/qa/editor-poc/gui-search.png": {"48dc00d03c01d5bbec35683ff65672c12af4be6aecc8a84e5fece243c56643f4"},
"docs/qa/editor-poc/image-editor-midnight-de.png": {"eb01bd8fd8eaadfbc8e2b3f634bcdafbbb084dc4592d849e102a6f8c70ab0ad4"},
"docs/qa/editor-poc/options-midnight-de.png": {"7075957db2d5f5a166bb71adb97d8208cb051a72b555202c212fae4f7e5c402c"},
"docs/qa/editor-poc/players-silver-en.png": {"d965a904663f3a30e5410f5d33539f6247737293625039fb9e3028337bbc088c"},
"docs/qa/editor-poc/roster-matchday-de.png": {"937829bb31b7452aa15d4a54cd03ad5de6a42d630b83cce279f22da09dfe180b"},
"assets/launcher/stadium.tga": {
"08f82783a357d5b0722f3737de54844150bc14d98036845f1aaee3211608e1c8",
"fb8e5f1f86afe60de87b70a2f533690d1ef470241fa24ee4384ff000149edaab",
},
}
LEGACY_SYMBOLS = re.compile(
r"(?<![A-Za-z0-9])(?:" + "FUN" + r"_?[0-9a-f]{4,8}|" + "DAT" +
r"_[0-9a-f]{5,8}|source[_ ]*(?=[0-9a-f]*[0-9])[0-9a-f]{4,8}|"
r"(?:raw|heap|saved|expected|sound_cfg)_(?=[ab][0-9a-f]*[0-9])[ab][0-9a-f]{4})(?![A-Za-z0-9])",
re.IGNORECASE,
)
PRIVATE_TEXT = re.compile(
r"(?:(?<![\w}./\\])/" + r"tools/|[A-Z]:[/\\]Users[/\\]|[A-Z]:[/\\]OpenBMH[/\\]|"
r"docs/(?:analyses|agents)/|src/ui/(?:validation|reference)/|"
+ "bmh" + r"-work|matter" + r"most|\bMich" + r"ael\b|\bDIA" +
r"MOND\b|(?<![\w}./\\])/" + r"root/[^\s]+|https?://(?:10\.|192\.168\.|172\.(?:1[6-9]|2[0-9]|3[01])\.)|"
r"github\.com/[^/\s]+/openbmh[_-]dc\b)",
re.IGNORECASE,
)
SECRET_PATTERNS = (
re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"),
re.compile(r"\bgithub_pat_[A-Za-z0-9_]{30,}\b"),
re.compile(r"-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----"),
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]{20,}"),
re.compile(r"(?i)\b[A-Z_]*(?:TOKEN|PASSWORD|SECRET|API_KEY)['\"]?\s*[:=]\s*['\"][A-Za-z0-9_./+-]{20,}['\"]"),
re.compile(r"(?m)^(?:export\s+)?[A-Z_]*(?:TOKEN|PASSWORD|SECRET|API_KEY)\s*=\s*[A-Za-z0-9_./+-]{20,}(?:\s|$)"),
)
EMAIL = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
# Resolution-qualified artwork names are filenames, including in older commits.
SIZED_ASSET_NAME = re.compile(r"[A-Z0-9._+-]+@[1-9][0-9]{0,4}\.(?:png|jpe?g|webp|svg|gif|avif)", re.IGNORECASE)
# Project bot identities and the co-author trailer address are not personal data.
BOT_EMAILS = {"codex@openbmh.invalid", "claude@openbmh.invalid", "noreply@anthropic.com"}
HEX_NUMBER = re.compile(r"\b0x(?P<digits>[0-9a-f]{5,8})(?:u(?:ll?|z)?|ll?u?|z)?\b", re.IGNORECASE)
LAYOUT_BEGIN = 0x50000
LAYOUT_END = 0xE0000
def line_at(text: str, position: int) -> int:
return text.count("\n", 0, position) + 1
def cpp_exclusions(text: str):
"""Yield comment/literal spans, respecting raw strings and numeric separators."""
at = 0
while at < len(text):
if text.startswith("//", at):
end = text.find("\n", at)
end = len(text) if end < 0 else end
yield at, end, "comment"
at = end
elif text.startswith("/*", at):
end = text.find("*/", at + 2)
end = len(text) if end < 0 else end + 2
yield at, end, "comment"
at = end
elif text.startswith('R"', at):
opening = text.find("(", at + 2, at + 19)
if opening < 0:
at += 2
continue
delimiter = text[at + 2:opening]
end = text.find(")" + delimiter + '"', opening + 1)
finish = len(text) if end < 0 else end + len(delimiter) + 2
yield at, finish, "literal"
at = finish
elif text[at] == "'" and at + 1 < len(text) and text[at + 1].isalnum() and re.search(
r"\b[0-9][a-fA-F0-9xXbB']*$", text[max(0, at - 200):at]):
at += 1 # C++ numeric separator, rather than a character literal.
elif text[at] in "\"'":
start = at
quote = text[at]
at += 1
while at < len(text):
if text[at] == "\\":
at += 2
elif text[at] == quote:
at += 1
break
else:
at += 1
yield start, at, "literal"
else:
at += 1
def is_layout_value(value: int) -> bool:
return LAYOUT_BEGIN <= value < LAYOUT_END and value & (value - 1) != 0 and value & (value + 1) != 0
def layout_findings(name: str, text: str):
exclusions = list(cpp_exclusions(text))
code = list(text)
for start, end, kind in exclusions:
if kind == "comment":
for match in HEX_NUMBER.finditer(text, start, end):
if is_layout_value(int(match.group("digits"), 16)):
yield Finding(name, line_at(text, match.start()), "address-in-comment")
for index in range(start, end):
if code[index] != "\n":
code[index] = " "
code = "".join(code)
for match in HEX_NUMBER.finditer(code):
if not is_layout_value(int(match.group("digits"), 16)):
continue
# Keep numeric format knowledge in a named constant, preserving its type.
begin = max(code.rfind(";", 0, match.start()), code.rfind("{", 0, match.start()),
code.rfind("}", 0, match.start())) + 1
prefix = code[begin:match.start()]
declaration = re.search(r"\b(?:constexpr|constinit|const)\b[^;{}]*\b[A-Za-z_]\w*\s*=\s*\(?\s*$", prefix)
if declaration is None:
yield Finding(name, line_at(text, match.start()), "unnamed-layout-address")
def path_findings(name: str) -> list[Finding]:
path = PurePosixPath(name)
parts = path.parts
result = []
if path.is_absolute() or ".." in parts:
result.append(Finding(name, 0, "unsafe-path"))
if path.suffix.lower() in ASSET_SUFFIXES and name not in PROJECT_ARTWORK:
result.append(Finding(name, 0, "binary-or-game-asset"))
if path.name in PRIVATE_DOCUMENTS or any(parts[:len(p)] == p for p in PRIVATE_PREFIXES):
result.append(Finding(name, 0, "private-material-path"))
if any(p.startswith(".claude") or p.startswith(".codex") for p in parts):
result.append(Finding(name, 0, "private-agent-configuration"))
return result
def scan_blob(name: str, blob: bytes) -> list[Finding]:
result = path_findings(name)
if name in PROJECT_ARTWORK:
if hashlib.sha256(blob).hexdigest() not in PROJECT_ARTWORK[name]:
result.append(Finding(name, 0, "unreviewed-project-artwork"))
return result
if len(blob) > 2_000_000:
result.append(Finding(name, 0, "oversized-source-file"))
if b"\0" in blob:
result.append(Finding(name, 0, "binary-content"))
return result
try:
text = blob.decode("utf-8-sig")
except UnicodeDecodeError:
result.append(Finding(name, 0, "non-utf8-source"))
return result
for pattern in SECRET_PATTERNS:
for match in pattern.finditer(text):
result.append(Finding(name, line_at(text, match.start()), "credential-pattern"))
# These exact upstream license/copyright documents must remain unmodified.
upstream_notice = name.startswith(("docs/licenses/", "tools/package/licenses/",
"src/audio/third_party/"))
if upstream_notice:
return result
for pattern, rule in [(LEGACY_SYMBOLS, "legacy-address-symbol"), (PRIVATE_TEXT, "private-provenance")]:
for match in pattern.finditer(text):
result.append(Finding(name, line_at(text, match.start()), rule))
for match in EMAIL.finditer(text):
if match.group().lower() not in BOT_EMAILS and not SIZED_ASSET_NAME.fullmatch(match.group()):
result.append(Finding(name, line_at(text, match.start()), "personal-email"))
if PurePosixPath(name).suffix.lower() in SOURCE_SUFFIXES:
result.extend(layout_findings(name, text))
elif PurePosixPath(name).suffix.lower() == ".md" and not name.startswith("docs/formats/"):
for match in HEX_NUMBER.finditer(text):
if is_layout_value(int(match.group("digits"), 16)):
result.append(Finding(name, line_at(text, match.start()), "address-outside-format-documentation"))
return result
def checked_files(root: Path):
"""Use the Git index when available; otherwise inspect a plain source archive."""
try:
git = subprocess.run(["git", "-C", str(root), "rev-parse", "--show-toplevel"],
capture_output=True, text=True)
except FileNotFoundError:
git = None
if git is not None and git.returncode == 0 and Path(git.stdout.strip()).resolve() == root:
output = subprocess.check_output(["git", "-C", str(root), "ls-files", "--cached",
"--others", "--exclude-standard", "-z"])
return sorted({p.decode("utf-8") for p in output.split(b"\0") if p})
return sorted(str(p.relative_to(root).as_posix()) for p in root.rglob("*")
if p.is_file() and not any(part in SKIP_DIRECTORIES or part.startswith("build")
for part in p.relative_to(root).parts))
def scan_tree(root: Path):
findings = []
names = checked_files(root)
for name in names:
path = root / name
if path.is_symlink():
findings.append(Finding(name, 0, "symlink-not-reviewed"))
elif not path.is_file():
findings.append(Finding(name, 0, "missing-tracked-file"))
else:
findings.extend(scan_blob(name, path.read_bytes()))
return findings, len(names)
def scan_history(root: Path):
findings = []
references = subprocess.check_output(["git", "-C", str(root), "for-each-ref", "--format=%(refname)"])
findings.extend(scan_blob("ref-names", references))
refs = subprocess.check_output(["git", "-C", str(root), "rev-list", "--all"], text=True).splitlines()
seen = set()
checked = 0
for revision in refs:
entries = subprocess.check_output(["git", "-C", str(root), "ls-tree", "-rz", revision])
for entry in entries.split(b"\0"):
if not entry:
continue
metadata, name = entry.split(b"\t", 1)
mode, kind, object_id = metadata.decode().split()
path = name.decode("utf-8")
if kind != "blob":
findings.append(Finding(path, 0, "unreviewed-history-object"))
continue
if mode == "120000":
findings.append(Finding(path, 0, "symlink-in-history"))
key = (object_id, path)
if key in seen:
continue
seen.add(key)
data = subprocess.check_output(["git", "-C", str(root), "cat-file", "blob", object_id])
findings.extend(scan_blob(path, data))
checked += 1
message = subprocess.check_output(["git", "-C", str(root), "show", "-s", "--format=%B", revision])
findings.extend(scan_blob("commit-message", message))
return findings, len(refs), checked
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=Path.cwd())
parser.add_argument("--history", action="store_true", help="also check all reachable commits and their blobs")
parser.add_argument("--json", type=Path, help="write a report containing rule names and locations only")
args = parser.parse_args(argv)
if args.history and shutil.which("git") is None:
parser.error("Git is required for history checks")
root = args.root.resolve(strict=True)
findings, files = scan_tree(root)
commits = blobs = 0
if args.history:
extra, commits, blobs = scan_history(root)
findings.extend(extra)
findings = sorted(set(findings))
def display_path(name):
for pattern in SECRET_PATTERNS:
name = pattern.sub("[redacted]", name)
return name
report = {"ok": not findings, "files_checked": files, "history_checked": args.history,
"commits_checked": commits, "history_blobs_checked": blobs,
"findings": [{**dataclasses.asdict(f), "path": display_path(f.path)} for f in findings]}
if args.json:
args.json.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
for finding in findings:
print(f"{display_path(finding.path)}:{finding.line}: {finding.rule}")
print(f"Public-source check: {files} files, {commits} commits, {len(findings)} findings")
return int(bool(findings))
if __name__ == "__main__":
sys.exit(main())