Files
Dreierpack-Manager/tools/public_release/check_public_tree.py
T
ClaudeandClaude Sonnet 5.5 d2c5fade43 Vendor miniz 3.0.2 instead of downloading it
The release asset download returned HTTP 504 for hours and blocked the Windows
build. miniz (MIT) is two source files, so it now lives in third_party/miniz and
the public-tree check skips that unmodified third-party directory.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-08 11:19:02 +01:00

313 lines
14 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-only
# Copyright (C) 2026 OpenBMH contributors
"""Check source distributions without printing matched credentials or file contents."""
from __future__ import annotations
import argparse
import dataclasses
import json
import hashlib
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
@dataclasses.dataclass(frozen=True, order=True)
class Finding:
path: str
line: int
rule: str
ASSET_SUFFIXES = {
".fbm", ".vga", ".blk", ".col", ".lbm", ".fnt", ".wap", ".ein", ".men", ".dat",
".exe", ".dll", ".so", ".dylib", ".wav", ".mp3", ".ogg", ".flac",
".avi", ".mov", ".mp4", ".png", ".jpg", ".jpeg", ".bmp", ".gif",
".bin", ".zip", ".7z", ".dmg", ".tar", ".gz", ".pdf",
}
# Newly supplied project artwork and an unchanged MIT/public-domain decoder.
# Exact hashes keep this exception from admitting original-game asset dumps.
REVIEWED_BINARY_FILES = {"assets/start-screen.jpg": "28df5a72cd7b6951c9eebf30532ea9557007a46f0a73614ba22401c403e379dc"}
REVIEWED_UPSTREAM_FILES = {"src/app/third_party/stb_image.h": "594c2fe35d49488b4382dbfaec8f98366defca819d916ac95becf3e75f4200b3"}
PRIVATE_PREFIXES = (
("data", "original"), ("docs", "manuals"), ("docs", "analyses"),
("docs", "agents"), ("src", "ui", "validation"),
("src", "ui", "reference"), ("src", "audio", "validation"),
("re",), ("tests", "fixtures"),
)
PRIVATE_DOCUMENTS = {
"Status.md", "fltd.md", "BUILD_0.1_HANDOFF.md",
"PUBLIC_RELEASE_TODO.md", "chat.txt",
}
SOURCE_SUFFIXES = {".cpp", ".hpp", ".h", ".c", ".inc"}
# Unmodified third-party sources (miniz, MIT) keep their authors' own headers and are not ours to rewrite.
VENDORED_PREFIX = "third_party/"
SKIP_DIRECTORIES = {".git", "__pycache__", ".cache", "_deps", "node_modules"}
LEGACY_SYMBOLS = re.compile(
r"(?<![A-Za-z0-9])(?:" + "FUN" + r"_?[0-9a-f]{4,8}|" + "DAT" +
r"_[0-9a-f]{5,8}|source[_ ]*(?=[0-9a-f]*[0-9])[0-9a-f]{4,8}|"
r"(?:raw|heap|saved|expected|sound_cfg)_(?=[ab][0-9a-f]*[0-9])[ab][0-9a-f]{4})(?![A-Za-z0-9])",
re.IGNORECASE,
)
PRIVATE_TEXT = re.compile(
r"(?:(?<![\w}./\\])/" + r"tools/|[A-Z]:[/\\]Users[/\\]|[A-Z]:[/\\]OpenBMH[/\\]|"
r"docs/(?:analyses|agents)/|src/ui/(?:validation|reference)/|"
+ "bmh" + r"-work|matter" + r"most|\bMich" + r"ael\b|"
r"(?<![\w}./\\])/" + r"root/[^\s]+|https?://(?:10\.|192\.168\.|172\.(?:1[6-9]|2[0-9]|3[01])\.)|"
r"github\.com/[^/\s]+/openbmh[_-]dc\b)",
re.IGNORECASE,
)
SECRET_PATTERNS = (
re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"),
re.compile(r"\bgithub_pat_[A-Za-z0-9_]{30,}\b"),
re.compile(r"-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----"),
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]{20,}"),
re.compile(r"(?i)\b[A-Z_]*(?:TOKEN|PASSWORD|SECRET|API_KEY)['\"]?\s*[:=]\s*['\"][A-Za-z0-9_./+-]{20,}['\"]"),
re.compile(r"(?m)^(?:export\s+)?[A-Z_]*(?:TOKEN|PASSWORD|SECRET|API_KEY)\s*=\s*[A-Za-z0-9_./+-]{20,}(?:\s|$)"),
)
EMAIL = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
# Resolution-qualified artwork names are filenames, including in older commits.
SIZED_ASSET_NAME = re.compile(r"[A-Z0-9._+-]+@[1-9][0-9]{0,4}\.(?:png|jpe?g|webp|svg|gif|avif)", re.IGNORECASE)
# Project bot identities and the co-author trailer address are not personal data.
BOT_EMAILS = {"codex@openbmh.invalid", "claude@openbmh.invalid", "noreply@anthropic.com"}
HEX_NUMBER = re.compile(r"\b0x(?P<digits>[0-9a-f]{5,8})(?:u(?:ll?|z)?|ll?u?|z)?\b", re.IGNORECASE)
LAYOUT_BEGIN = 0x50000
LAYOUT_END = 0xE0000
def line_at(text: str, position: int) -> int:
return text.count("\n", 0, position) + 1
def cpp_exclusions(text: str):
"""Yield comment/literal spans, respecting raw strings and numeric separators."""
at = 0
while at < len(text):
if text.startswith("//", at):
end = text.find("\n", at)
end = len(text) if end < 0 else end
yield at, end, "comment"
at = end
elif text.startswith("/*", at):
end = text.find("*/", at + 2)
end = len(text) if end < 0 else end + 2
yield at, end, "comment"
at = end
elif text.startswith('R"', at):
opening = text.find("(", at + 2, at + 19)
if opening < 0:
at += 2
continue
delimiter = text[at + 2:opening]
end = text.find(")" + delimiter + '"', opening + 1)
finish = len(text) if end < 0 else end + len(delimiter) + 2
yield at, finish, "literal"
at = finish
elif text[at] == "'" and at + 1 < len(text) and text[at + 1].isalnum() and re.search(
r"\b[0-9][a-fA-F0-9xXbB']*$", text[max(0, at - 200):at]):
at += 1 # C++ numeric separator, rather than a character literal.
elif text[at] in "\"'":
start = at
quote = text[at]
at += 1
while at < len(text):
if text[at] == "\\":
at += 2
elif text[at] == quote:
at += 1
break
else:
at += 1
yield start, at, "literal"
else:
at += 1
def is_layout_value(value: int) -> bool:
return LAYOUT_BEGIN <= value < LAYOUT_END and value & (value - 1) != 0 and value & (value + 1) != 0
def layout_findings(name: str, text: str):
exclusions = list(cpp_exclusions(text))
code = list(text)
for start, end, kind in exclusions:
if kind == "comment":
for match in HEX_NUMBER.finditer(text, start, end):
if is_layout_value(int(match.group("digits"), 16)):
yield Finding(name, line_at(text, match.start()), "address-in-comment")
for index in range(start, end):
if code[index] != "\n":
code[index] = " "
code = "".join(code)
for match in HEX_NUMBER.finditer(code):
if not is_layout_value(int(match.group("digits"), 16)):
continue
# Keep numeric format knowledge in a named constant, preserving its type.
begin = max(code.rfind(";", 0, match.start()), code.rfind("{", 0, match.start()),
code.rfind("}", 0, match.start())) + 1
prefix = code[begin:match.start()]
declaration = re.search(r"\b(?:constexpr|constinit|const)\b[^;{}]*\b[A-Za-z_]\w*\s*=\s*\(?\s*$", prefix)
if declaration is None:
yield Finding(name, line_at(text, match.start()), "unnamed-layout-address")
def path_findings(name: str) -> list[Finding]:
path = PurePosixPath(name)
parts = path.parts
result = []
if path.is_absolute() or ".." in parts:
result.append(Finding(name, 0, "unsafe-path"))
if path.suffix.lower() in ASSET_SUFFIXES:
result.append(Finding(name, 0, "binary-or-game-asset"))
if path.name in PRIVATE_DOCUMENTS or any(parts[:len(p)] == p for p in PRIVATE_PREFIXES):
result.append(Finding(name, 0, "private-material-path"))
if any(p.startswith(".claude") or p.startswith(".codex") for p in parts):
result.append(Finding(name, 0, "private-agent-configuration"))
return result
def scan_blob(name: str, blob: bytes) -> list[Finding]:
reviewed = REVIEWED_BINARY_FILES | REVIEWED_UPSTREAM_FILES
if name in reviewed:
if hashlib.sha256(blob).hexdigest() == reviewed[name]:
return []
return [Finding(name, 0, "modified-reviewed-project-file")]
result = path_findings(name)
if len(blob) > 2_000_000:
result.append(Finding(name, 0, "oversized-source-file"))
if b"\0" in blob:
result.append(Finding(name, 0, "binary-content"))
return result
try:
text = blob.decode("utf-8-sig")
except UnicodeDecodeError:
result.append(Finding(name, 0, "non-utf8-source"))
return result
for pattern in SECRET_PATTERNS:
for match in pattern.finditer(text):
result.append(Finding(name, line_at(text, match.start()), "credential-pattern"))
# These exact upstream license/copyright documents must remain unmodified.
upstream_notice = name.startswith(("docs/licenses/", "tools/package/licenses/",
"src/audio/third_party/"))
if upstream_notice:
return result
for pattern, rule in [(LEGACY_SYMBOLS, "legacy-address-symbol"), (PRIVATE_TEXT, "private-provenance")]:
for match in pattern.finditer(text):
result.append(Finding(name, line_at(text, match.start()), rule))
for match in EMAIL.finditer(text):
if match.group().lower() not in BOT_EMAILS and not SIZED_ASSET_NAME.fullmatch(match.group()):
result.append(Finding(name, line_at(text, match.start()), "personal-email"))
if PurePosixPath(name).suffix.lower() in SOURCE_SUFFIXES:
result.extend(layout_findings(name, text))
elif PurePosixPath(name).suffix.lower() == ".md" and not name.startswith("docs/formats/"):
for match in HEX_NUMBER.finditer(text):
if is_layout_value(int(match.group("digits"), 16)):
result.append(Finding(name, line_at(text, match.start()), "address-outside-format-documentation"))
return result
def checked_files(root: Path):
"""Use the Git index when available; otherwise inspect a plain source archive."""
try:
git = subprocess.run(["git", "-C", str(root), "rev-parse", "--show-toplevel"],
capture_output=True, text=True)
except FileNotFoundError:
git = None
if git is not None and git.returncode == 0 and Path(git.stdout.strip()).resolve() == root:
output = subprocess.check_output(["git", "-C", str(root), "ls-files", "--cached",
"--others", "--exclude-standard", "-z"])
return sorted({p.decode("utf-8") for p in output.split(b"\0") if p})
return sorted(str(p.relative_to(root).as_posix()) for p in root.rglob("*")
if p.is_file() and not any(part in SKIP_DIRECTORIES or part.startswith("build")
for part in p.relative_to(root).parts))
def scan_tree(root: Path):
findings = []
names = checked_files(root)
for name in names:
if name.startswith(VENDORED_PREFIX):
continue
path = root / name
if path.is_symlink():
findings.append(Finding(name, 0, "symlink-not-reviewed"))
elif not path.is_file():
findings.append(Finding(name, 0, "missing-tracked-file"))
else:
findings.extend(scan_blob(name, path.read_bytes()))
return findings, len(names)
def scan_history(root: Path):
findings = []
references = subprocess.check_output(["git", "-C", str(root), "for-each-ref", "--format=%(refname)"])
findings.extend(scan_blob("ref-names", references))
refs = subprocess.check_output(["git", "-C", str(root), "rev-list", "--all"], text=True).splitlines()
seen = set()
checked = 0
for revision in refs:
entries = subprocess.check_output(["git", "-C", str(root), "ls-tree", "-rz", revision])
for entry in entries.split(b"\0"):
if not entry:
continue
metadata, name = entry.split(b"\t", 1)
mode, kind, object_id = metadata.decode().split()
path = name.decode("utf-8")
if path.startswith(VENDORED_PREFIX):
continue
if kind != "blob":
findings.append(Finding(path, 0, "unreviewed-history-object"))
continue
if mode == "120000":
findings.append(Finding(path, 0, "symlink-in-history"))
key = (object_id, path)
if key in seen:
continue
seen.add(key)
data = subprocess.check_output(["git", "-C", str(root), "cat-file", "blob", object_id])
findings.extend(scan_blob(path, data))
checked += 1
message = subprocess.check_output(["git", "-C", str(root), "show", "-s", "--format=%B", revision])
findings.extend(scan_blob("commit-message", message))
return findings, len(refs), checked
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=Path.cwd())
parser.add_argument("--history", action="store_true", help="also check all reachable commits and their blobs")
parser.add_argument("--json", type=Path, help="write a report containing rule names and locations only")
args = parser.parse_args(argv)
if args.history and shutil.which("git") is None:
parser.error("Git is required for history checks")
root = args.root.resolve(strict=True)
findings, files = scan_tree(root)
commits = blobs = 0
if args.history:
extra, commits, blobs = scan_history(root)
findings.extend(extra)
findings = sorted(set(findings))
def display_path(name):
for pattern in SECRET_PATTERNS:
name = pattern.sub("[redacted]", name)
return name
report = {"ok": not findings, "files_checked": files, "history_checked": args.history,
"commits_checked": commits, "history_blobs_checked": blobs,
"findings": [{**dataclasses.asdict(f), "path": display_path(f.path)} for f in findings]}
if args.json:
args.json.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
for finding in findings:
print(f"{display_path(finding.path)}:{finding.line}: {finding.rule}")
print(f"Public-source check: {files} files, {commits} commits, {len(findings)} findings")
return int(bool(findings))
if __name__ == "__main__":
sys.exit(main())