diff --git a/tests/package_tests.py b/tests/package_tests.py index 194b473..a1c0b71 100644 --- a/tests/package_tests.py +++ b/tests/package_tests.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: GPL-3.0-only """The release allowlist rejects extra game/test files using synthetic payloads.""" import importlib.util +import hashlib +import stat from pathlib import Path import tempfile import unittest @@ -10,6 +12,9 @@ source = Path(__file__).resolve().parents[1] / 'tools' / 'package' / 'make_packa spec = importlib.util.spec_from_file_location('make_package', source) package = importlib.util.module_from_spec(spec) spec.loader.exec_module(package) +source_spec = importlib.util.spec_from_file_location('make_source_package', source.with_name('make_source_package.py')) +source_package = importlib.util.module_from_spec(source_spec) +source_spec.loader.exec_module(source_package) class PackageAllowlistTests(unittest.TestCase): @@ -40,5 +45,110 @@ class PackageAllowlistTests(unittest.TestCase): package.validate(root, False) +class SourceInventoryTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / 'component' + self.source.mkdir() + self.destination = self.root / 'materialized' + + def authored(self, name, text='authored source'): + target = self.source / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + return target + + def copy(self, names, **kwargs): + self.links = {} + return source_package.copy_files(self.source, self.destination, [Path(n) for n in names], + materialized_links=self.links, **kwargs) + + def test_directory_link_copies_only_tracked_contents_and_mode(self): + script = self.authored('real/sub/build.sh', '#!/bin/sh\nexit 0\n') + script.chmod(0o751) + self.authored('real/untracked.txt', 'must never be packaged') + (self.source / 'alias').symlink_to('real', target_is_directory=True) + hashes = self.copy(['alias', 'real/sub/build.sh'], required=True) + copied = self.destination / 'alias/sub/build.sh' + self.assertEqual(copied.read_bytes(), script.read_bytes()) + self.assertFalse((self.destination / 'alias').is_symlink()) + self.assertFalse((self.destination / 'alias/untracked.txt').exists()) + self.assertFalse((self.destination / 'real/untracked.txt').exists()) + self.assertEqual(stat.S_IMODE(copied.stat().st_mode), stat.S_IMODE(script.stat().st_mode)) + self.assertEqual(hashes['alias/sub/build.sh'], hashlib.sha256(script.read_bytes()).hexdigest()) + self.assertEqual(self.links['alias'], {'kind': 'directory', 'target': 'real', 'via': ['alias']}) + + def test_tracked_file_link_chain_is_materialized_and_recorded(self): + target = self.authored('tool.sh', '#!/bin/sh\nexit 0\n') + target.chmod(0o755) + (self.source / 'middle').symlink_to('tool.sh') + (self.source / 'entry').symlink_to('middle') + hashes = self.copy(['entry', 'middle', 'tool.sh'], required=True) + copied = self.destination / 'entry' + self.assertFalse(copied.is_symlink()) + self.assertEqual(copied.read_bytes(), target.read_bytes()) + self.assertEqual(stat.S_IMODE(copied.stat().st_mode), stat.S_IMODE(target.stat().st_mode)) + self.assertEqual(hashes['entry'], hashes['tool.sh']) + self.assertEqual(self.links['entry'], {'kind': 'file', 'target': 'tool.sh', 'via': ['entry', 'middle']}) + + def test_untracked_file_target_is_rejected(self): + self.authored('untracked.txt') + (self.source / 'entry').symlink_to('untracked.txt') + with self.assertRaisesRegex(RuntimeError, 'not in the tracked inventory'): + self.copy(['entry'], required=True) + + def test_directory_without_tracked_targets_is_rejected(self): + self.authored('real/untracked.txt') + (self.source / 'entry').symlink_to('real', target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'no tracked contents'): + self.copy(['entry'], required=True) + + def test_untracked_intermediate_link_is_rejected(self): + self.authored('tracked.txt') + (self.source / 'middle').symlink_to('tracked.txt') + (self.source / 'entry').symlink_to('middle') + with self.assertRaisesRegex(RuntimeError, 'not in the tracked inventory'): + self.copy(['entry', 'tracked.txt'], required=True) + + def test_external_file_directory_and_returning_link_are_rejected(self): + outside = self.root / 'outside' + outside.mkdir() + (outside / 'file.txt').write_text('external source') + self.authored('tracked.txt') + (outside / 'returning').symlink_to(self.source / 'tracked.txt') + for name, target, directory in [('file', outside / 'file.txt', False), + ('directory', outside, True), + ('returning', outside / 'returning', False)]: + with self.subTest(name=name): + (self.source / name).symlink_to(target, target_is_directory=directory) + with self.assertRaisesRegex(RuntimeError, 'leaves its component'): + self.copy([name, 'tracked.txt'], required=True) + + def test_file_and_directory_loops_are_rejected(self): + (self.source / 'first').symlink_to('second') + (self.source / 'second').symlink_to('first') + with self.assertRaisesRegex(RuntimeError, 'loop'): + self.copy(['first', 'second'], required=True) + (self.source / 'directory').symlink_to('.', target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'loop'): + self.copy(['directory'], required=True) + + def test_parent_component_after_link_uses_resolved_directory(self): + self.authored('real/sub/keep.txt') + target = self.authored('real/tool.sh', 'resolved parent source') + (self.source / 'jump').symlink_to('real/sub', target_is_directory=True) + (self.source / 'entry').symlink_to('jump/../tool.sh') + self.copy(['entry', 'jump', 'real/sub/keep.txt', 'real/tool.sh'], required=True) + self.assertEqual((self.destination / 'entry').read_bytes(), target.read_bytes()) + self.assertEqual(self.links['entry']['target'], 'real/tool.sh') + + def test_dangling_link_is_rejected_even_for_optional_project_files(self): + (self.source / 'entry').symlink_to('missing.txt') + with self.assertRaisesRegex(RuntimeError, 'Missing dependency source file'): + self.copy(['entry', 'missing.txt']) + + if __name__ == '__main__': unittest.main() diff --git a/tools/package/make_source_package.py b/tools/package/make_source_package.py index b43bfcd..050ca6c 100644 --- a/tools/package/make_source_package.py +++ b/tools/package/make_source_package.py @@ -32,20 +32,105 @@ def git(directory, *arguments): return subprocess.check_output(['git', '-C', str(directory), *arguments]) -def copy_files(source, destination, files, required=False): +def copy_files(source, destination, files, required=False, materialized_links=None): + """Copy only the explicit source inventory, materializing internal links. + + Git dependencies pass their tracked paths here. Directory links expand only + paths in that inventory, never a recursive filesystem walk or untracked data. + """ + source = source.resolve() + destination = Path(destination) manifest = {} - for relative in sorted(set(files)): - origin = source / relative + links = materialized_links if materialized_links is not None else {} + + def normalized(path): + path = Path(path) + if path.is_absolute(): + try: + path = path.relative_to(source) + except ValueError as error: + raise RuntimeError('Source symlink leaves its component') from error + parts = [] + for part in path.parts: + if part == '..': + if not parts: + raise RuntimeError('Source symlink leaves its component') + parts.pop() + elif part != '.': + parts.append(part) + return Path(*parts) + + inventory = {normalized(path) for path in files} + + def resolve(relative): + pending = normalized(relative) + visited = set() + chain = [] + while True: + current = Path() + for index, part in enumerate(pending.parts): + if part == '..': + if current == Path(): + raise RuntimeError('Source symlink leaves its component') + current = current.parent + continue + current /= part + origin = source / current + if not origin.is_symlink(): + if index + 1 < len(pending.parts) and not origin.is_dir() and (required or chain): + raise RuntimeError('Source path parent is not a directory') + continue + if current not in inventory: + raise RuntimeError('Source symlink is not in the tracked inventory') + state = (current, pending.parts[index + 1:]) + if state in visited or len(visited) >= 64: + raise RuntimeError('Source symlink loop') + visited.add(state) + chain.append(current.as_posix()) + target = origin.readlink() + if not target.is_absolute(): + target = current.parent / target + else: + try: + target = target.relative_to(source) + except ValueError as error: + raise RuntimeError('Source symlink leaves its component') from error + pending = target / Path(*pending.parts[index + 1:]) + break + else: + return current, chain + + def copy(relative, output, ancestors=()): + resolved, chain = resolve(relative) + origin = source / resolved + if origin.is_dir(): + if not chain: + raise RuntimeError('Tracked source entry is a directory') + if resolved in ancestors: + raise RuntimeError('Source directory symlink loop') + children = sorted(path for path in inventory + if path != resolved and path.is_relative_to(resolved)) + if not children: + raise RuntimeError('Source directory target has no tracked contents') + links[output.as_posix()] = {'kind': 'directory', 'target': resolved.as_posix(), 'via': chain} + for child in children: + copy(child, output / child.relative_to(resolved), ancestors + (resolved,)) + return if not origin.is_file(): - if required: + if required or chain: raise RuntimeError('Missing dependency source file: ' + relative.as_posix()) - continue - if not origin.resolve().is_relative_to(source.resolve()): - raise RuntimeError('Source symlink leaves its component') - target = destination / relative + return + if resolved not in inventory: + raise RuntimeError('Source file target is not in the tracked inventory') + target = destination / output target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(origin, target) - manifest[relative.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest() + manifest[output.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest() + if chain: + links[output.as_posix()] = {'kind': 'file', 'target': resolved.as_posix(), 'via': chain} + + for relative in sorted(inventory): + copy(relative, relative) return manifest @@ -68,8 +153,11 @@ def main(): with tempfile.TemporaryDirectory(prefix='openbmh-source-', dir=args.out) as temporary: stage = Path(temporary) project_files = [Path(p.decode('utf-8')) for p in git(source, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split(b'\0') if p] + project_links = {} manifest = {'version': version, 'project_commit': git(source, 'rev-parse', 'HEAD').decode().strip(), - 'project_files': copy_files(source, stage / 'project', project_files), 'dependencies': {}} + 'project_files': copy_files(source, stage / 'project', project_files, + materialized_links=project_links), + 'project_materialized_links': project_links, 'dependencies': {}} initial_cache = [] for name, (reference, url, markers) in COMPONENTS.items(): directory = Path(dependencies.get(name, '')) @@ -95,9 +183,12 @@ def main(): for filename, expected in MINIZ_FILES.items(): if hashlib.sha256((directory / filename).read_bytes()).hexdigest() != expected: raise RuntimeError('Dependency differs from its pinned archive: ' + name) - hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True) + dependency_links = {} + hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True, + materialized_links=dependency_links) manifest['dependencies'][name] = {'upstream': url, 'reference': reference, - 'commit': commit, 'files': hashes} + 'commit': commit, 'files': hashes, + 'materialized_links': dependency_links} initial_cache.append('set(FETCHCONTENT_SOURCE_DIR_' + name.upper() + ' "${CMAKE_CURRENT_LIST_DIR}/dependencies/' + name + '" CACHE PATH "" FORCE)') (stage / 'dependencies.cmake').write_text('\n'.join(initial_cache) + '\n') @@ -105,7 +196,7 @@ def main(): (stage / 'BUILD_SOURCE.md').write_text( '# Corresponding source\n\nThe project contains its patches, build scripts and vendored Nuked OPL3. ' 'The dependency folders contain the sources used by this build; the LZMA folder contains the C decoder tree and upstream license. ' - 'File hashes and upstream revisions are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n' + 'File hashes, upstream revisions and materialized internal links are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n' 'With the native compiler, CMake and platform development libraries described in project/README.md:\n\n' '```sh\ncmake -S project -B build -C dependencies.cmake -DCMAKE_BUILD_TYPE=Release -DBMH_DATA=\n' 'cmake --build build --config Release --parallel 2\nctest --test-dir build -C Release --output-on-failure\n```\n\n'