239 lines
12 KiB
Python
239 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-only
|
|
"""Package the audited public source and the dependency sources used by its build."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zipfile
|
|
|
|
COMPONENTS = {
|
|
'SDL2': ('release-2.32.10', 'https://github.com/libsdl-org/SDL', ['include/SDL.h', 'src/SDL.c', 'LICENSE.txt']),
|
|
'miniz': ('3.0.2', 'https://github.com/richgel999/miniz', ['miniz.c', 'miniz.h', 'LICENSE']),
|
|
'lzma_sdk': ('24.09', 'https://github.com/ip7z/7zip', ['C/7zDec.c', 'C/7z.h', 'DOC/License.txt']),
|
|
'nfd': ('v1.2.1', 'https://github.com/btzy/nativefiledialog-extended', ['CMakeLists.txt', 'src/nfd_cocoa.m', 'LICENSE']),
|
|
'rmlui': ('6.3', 'https://github.com/mikke89/RmlUi', ['CMakeLists.txt', 'Include/RmlUi/Core.h', 'LICENSE.txt']),
|
|
'freetype': ('VER-2-13-3', 'https://github.com/freetype/freetype', ['CMakeLists.txt', 'include/ft2build.h', 'LICENSE.TXT']),
|
|
}
|
|
# Verified against the archive SHA256 pinned in CMakeLists.txt. These are free
|
|
# dependency source files, not game data. Refuse modified archive-cache inputs.
|
|
MINIZ_FILES = {
|
|
'miniz.c': '0fcdc9888cb3a29ca8f176bac087e5fe6c7258a6ab06b1c271c1e109a11d3740',
|
|
'miniz.h': '295d1a0041aea09609598c0f1f35c1977ca05ad662acbadcfdaac44c140af37b',
|
|
'LICENSE': '0115478d567121238cf6cc1c0c361926cf07a49d9e4c9e66da97fac6a01646b3',
|
|
'ChangeLog.md': '261c5cfb87942b5c47735b98ae2fa87ba5f3058d3c755838756b6117f3b0c0dd',
|
|
'readme.md': 'b4ea367be28a36e5386f65118b73962bc659a67936d248be68b7a7c03b8d359b',
|
|
}
|
|
|
|
|
|
def git(directory, *arguments):
|
|
return subprocess.check_output(['git', '-C', str(directory), *arguments])
|
|
|
|
|
|
def unbuilt_submodules(component, staged):
|
|
"""Only FreeType's unused make/autotools logging dependency is omitted.
|
|
|
|
The pinned CMake font-engine target does not compile dlg. Every other
|
|
submodule, or a changed revision, still needs explicit source packaging.
|
|
"""
|
|
excluded = {}
|
|
for record in staged.split(b'\0'):
|
|
if not record.startswith(b'160000 '):
|
|
continue
|
|
metadata, path = record.split(b'\t', 1)
|
|
commit = metadata.split()[1].decode()
|
|
name = path.decode()
|
|
if (component != 'freetype' or name != 'subprojects/dlg' or
|
|
commit != '72dfcc858c040c54a6a0b88fcb7e70ee186d3167'):
|
|
raise RuntimeError('Dependency submodules require explicit packaging: ' + component)
|
|
excluded[name] = commit
|
|
return excluded
|
|
|
|
|
|
def copy_files(source, destination, files, required=False, materialized_links=None):
|
|
"""Copy only the explicit source inventory, materializing internal links.
|
|
|
|
Git dependencies pass their tracked paths here. Directory links expand only
|
|
paths in that inventory, never a recursive filesystem walk or untracked data.
|
|
"""
|
|
source = source.resolve()
|
|
destination = Path(destination)
|
|
manifest = {}
|
|
links = materialized_links if materialized_links is not None else {}
|
|
|
|
def normalized(path):
|
|
path = Path(path)
|
|
if path.is_absolute():
|
|
try:
|
|
path = path.relative_to(source)
|
|
except ValueError as error:
|
|
raise RuntimeError('Source symlink leaves its component') from error
|
|
parts = []
|
|
for part in path.parts:
|
|
if part == '..':
|
|
if not parts:
|
|
raise RuntimeError('Source symlink leaves its component')
|
|
parts.pop()
|
|
elif part != '.':
|
|
parts.append(part)
|
|
return Path(*parts)
|
|
|
|
inventory = {normalized(path) for path in files}
|
|
|
|
def resolve(relative):
|
|
pending = normalized(relative)
|
|
visited = set()
|
|
chain = []
|
|
while True:
|
|
current = Path()
|
|
for index, part in enumerate(pending.parts):
|
|
if part == '..':
|
|
if current == Path():
|
|
raise RuntimeError('Source symlink leaves its component')
|
|
current = current.parent
|
|
continue
|
|
current /= part
|
|
origin = source / current
|
|
if not origin.is_symlink():
|
|
if index + 1 < len(pending.parts) and not origin.is_dir() and (required or chain):
|
|
raise RuntimeError('Source path parent is not a directory')
|
|
continue
|
|
if current not in inventory:
|
|
raise RuntimeError('Source symlink is not in the tracked inventory')
|
|
state = (current, pending.parts[index + 1:])
|
|
if state in visited or len(visited) >= 64:
|
|
raise RuntimeError('Source symlink loop')
|
|
visited.add(state)
|
|
chain.append(current.as_posix())
|
|
target = origin.readlink()
|
|
if not target.is_absolute():
|
|
target = current.parent / target
|
|
else:
|
|
try:
|
|
target = target.relative_to(source)
|
|
except ValueError as error:
|
|
raise RuntimeError('Source symlink leaves its component') from error
|
|
pending = target / Path(*pending.parts[index + 1:])
|
|
break
|
|
else:
|
|
return current, chain
|
|
|
|
def copy(relative, output, ancestors=()):
|
|
resolved, chain = resolve(relative)
|
|
origin = source / resolved
|
|
if origin.is_dir():
|
|
if not chain:
|
|
raise RuntimeError('Tracked source entry is a directory')
|
|
if resolved in ancestors:
|
|
raise RuntimeError('Source directory symlink loop')
|
|
children = sorted(path for path in inventory
|
|
if path != resolved and path.is_relative_to(resolved))
|
|
if not children:
|
|
raise RuntimeError('Source directory target has no tracked contents')
|
|
links[output.as_posix()] = {'kind': 'directory', 'target': resolved.as_posix(), 'via': chain}
|
|
for child in children:
|
|
copy(child, output / child.relative_to(resolved), ancestors + (resolved,))
|
|
return
|
|
if not origin.is_file():
|
|
if required or chain:
|
|
raise RuntimeError('Missing dependency source file: ' + relative.as_posix())
|
|
return
|
|
if resolved not in inventory:
|
|
raise RuntimeError('Source file target is not in the tracked inventory')
|
|
target = destination / output
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(origin, target)
|
|
manifest[output.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest()
|
|
if chain:
|
|
links[output.as_posix()] = {'kind': 'file', 'target': resolved.as_posix(), 'via': chain}
|
|
|
|
for relative in sorted(inventory):
|
|
copy(relative, relative)
|
|
return manifest
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[2])
|
|
parser.add_argument('--build', required=True, type=Path)
|
|
parser.add_argument('--out', required=True, type=Path)
|
|
args = parser.parse_args()
|
|
source = args.source.resolve()
|
|
subprocess.run([sys.executable, str(source / 'tools/public_release/check_public_tree.py'),
|
|
'--root', str(source)], check=True)
|
|
cache = (args.build / 'CMakeCache.txt').read_text()
|
|
import re
|
|
version = re.search(r'^CMAKE_PROJECT_VERSION:STATIC=([^\n]+)$', cache, re.M).group(1)
|
|
if not re.fullmatch(r'\d+\.\d+\.\d+', version):
|
|
raise RuntimeError('Unexpected source version')
|
|
dependencies = json.loads((args.build / 'package-dependencies.json').read_text())
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory(prefix='openbmh-source-', dir=args.out) as temporary:
|
|
stage = Path(temporary)
|
|
project_files = [Path(p.decode('utf-8')) for p in git(source, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split(b'\0') if p]
|
|
project_links = {}
|
|
manifest = {'version': version, 'project_commit': git(source, 'rev-parse', 'HEAD').decode().strip(),
|
|
'project_files': copy_files(source, stage / 'project', project_files,
|
|
materialized_links=project_links),
|
|
'project_materialized_links': project_links, 'dependencies': {}}
|
|
initial_cache = []
|
|
for name, (reference, url, markers) in COMPONENTS.items():
|
|
directory = Path(dependencies.get(name, ''))
|
|
if not dependencies.get(name) or not all((directory / marker).is_file() for marker in markers):
|
|
raise RuntimeError('Actual dependency source unavailable: ' + name)
|
|
commit = None
|
|
excluded = {}
|
|
if (directory / '.git').exists():
|
|
commit = git(directory, 'rev-parse', 'HEAD').decode().strip()
|
|
tag = git(directory, 'rev-parse', reference + '^{commit}').decode().strip()
|
|
if commit != tag or git(directory, 'status', '--porcelain', '--untracked-files=no').strip():
|
|
raise RuntimeError('Dependency differs from its pinned source: ' + name)
|
|
excluded = unbuilt_submodules(name, git(directory, 'ls-files', '--stage', '-z'))
|
|
if commit:
|
|
files = [Path(p.decode('utf-8')) for p in git(directory, 'ls-files', '-z').split(b'\0') if p]
|
|
files = [p for p in files if p.as_posix() not in excluded]
|
|
if name == 'lzma_sdk':
|
|
files = [p for p in files if p.parts[0] == 'C' or p == Path('DOC/License.txt')]
|
|
else:
|
|
# The pinned miniz amalgamation is an archive rather than a Git checkout.
|
|
if name != 'miniz':
|
|
raise RuntimeError('Dependency revision cannot be verified: ' + name)
|
|
files = [Path(p) for p in MINIZ_FILES]
|
|
for filename, expected in MINIZ_FILES.items():
|
|
if hashlib.sha256((directory / filename).read_bytes()).hexdigest() != expected:
|
|
raise RuntimeError('Dependency differs from its pinned archive: ' + name)
|
|
dependency_links = {}
|
|
hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True,
|
|
materialized_links=dependency_links)
|
|
manifest['dependencies'][name] = {'upstream': url, 'reference': reference,
|
|
'commit': commit, 'files': hashes,
|
|
'unbuilt_submodules': excluded,
|
|
'materialized_links': dependency_links}
|
|
initial_cache.append('set(FETCHCONTENT_SOURCE_DIR_' + name.upper() +
|
|
' "${CMAKE_CURRENT_LIST_DIR}/dependencies/' + name + '" CACHE PATH "" FORCE)')
|
|
(stage / 'dependencies.cmake').write_text('\n'.join(initial_cache) + '\n')
|
|
(stage / 'SOURCE_MANIFEST.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
|
(stage / 'BUILD_SOURCE.md').write_text(
|
|
'# Corresponding source\n\nThe project contains its patches, build scripts and vendored Nuked OPL3. '
|
|
'The dependency folders contain the sources used by this build; the LZMA folder contains the C decoder tree and upstream license. '
|
|
'File hashes, upstream revisions and materialized internal links are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n'
|
|
'With the native compiler, CMake and platform development libraries described in project/README.md:\n\n'
|
|
'```sh\ncmake -S project -B build -C dependencies.cmake -DCMAKE_BUILD_TYPE=Release -DBMH_DATA=\n'
|
|
'cmake --build build --config Release --parallel 2\nctest --test-dir build -C Release --output-on-failure\n```\n\n'
|
|
'Modify the library/application sources and rebuild with these scripts to produce a relinked program. '
|
|
'The source package does not supply the compiler, operating-system SDK or system libraries.\n')
|
|
destination = args.out / ('OpenBMH-' + version + '-source.zip')
|
|
with zipfile.ZipFile(destination, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
|
|
for file in sorted(stage.rglob('*')):
|
|
if file.is_file():
|
|
archive.write(file, Path(destination.stem) / file.relative_to(stage))
|
|
print(destination)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|