Materialize tracked dependency symlinks in source packages (by Codex)

This commit is contained in:
Codex committed 2026-10-16 21:09:11 +02:00
1 parent 9321dff2e9
commit c37a0c4036
2 files changed
+214 -13

No files matched your search

+110
View File
@@ -2,6 +2,8 @@
# SPDX-License-Identifier: GPL-3.0-only
"""The release allowlist rejects extra game/test files using synthetic payloads."""
import importlib.util
import hashlib
import stat
from pathlib import Path
import tempfile
import unittest
@@ -10,6 +12,9 @@ source = Path(__file__).resolve().parents[1] / 'tools' / 'package' / 'make_packa
spec = importlib.util.spec_from_file_location('make_package', source)
package = importlib.util.module_from_spec(spec)
spec.loader.exec_module(package)
source_spec = importlib.util.spec_from_file_location('make_source_package', source.with_name('make_source_package.py'))
source_package = importlib.util.module_from_spec(source_spec)
source_spec.loader.exec_module(source_package)
class PackageAllowlistTests(unittest.TestCase):
@@ -40,5 +45,110 @@ class PackageAllowlistTests(unittest.TestCase):
package.validate(root, False)
class SourceInventoryTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.source = self.root / 'component'
self.source.mkdir()
self.destination = self.root / 'materialized'
def authored(self, name, text='authored source'):
target = self.source / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(text)
return target
def copy(self, names, **kwargs):
self.links = {}
return source_package.copy_files(self.source, self.destination, [Path(n) for n in names],
materialized_links=self.links, **kwargs)
def test_directory_link_copies_only_tracked_contents_and_mode(self):
script = self.authored('real/sub/build.sh', '#!/bin/sh\nexit 0\n')
script.chmod(0o751)
self.authored('real/untracked.txt', 'must never be packaged')
(self.source / 'alias').symlink_to('real', target_is_directory=True)
hashes = self.copy(['alias', 'real/sub/build.sh'], required=True)
copied = self.destination / 'alias/sub/build.sh'
self.assertEqual(copied.read_bytes(), script.read_bytes())
self.assertFalse((self.destination / 'alias').is_symlink())
self.assertFalse((self.destination / 'alias/untracked.txt').exists())
self.assertFalse((self.destination / 'real/untracked.txt').exists())
self.assertEqual(stat.S_IMODE(copied.stat().st_mode), stat.S_IMODE(script.stat().st_mode))
self.assertEqual(hashes['alias/sub/build.sh'], hashlib.sha256(script.read_bytes()).hexdigest())
self.assertEqual(self.links['alias'], {'kind': 'directory', 'target': 'real', 'via': ['alias']})
def test_tracked_file_link_chain_is_materialized_and_recorded(self):
target = self.authored('tool.sh', '#!/bin/sh\nexit 0\n')
target.chmod(0o755)
(self.source / 'middle').symlink_to('tool.sh')
(self.source / 'entry').symlink_to('middle')
hashes = self.copy(['entry', 'middle', 'tool.sh'], required=True)
copied = self.destination / 'entry'
self.assertFalse(copied.is_symlink())
self.assertEqual(copied.read_bytes(), target.read_bytes())
self.assertEqual(stat.S_IMODE(copied.stat().st_mode), stat.S_IMODE(target.stat().st_mode))
self.assertEqual(hashes['entry'], hashes['tool.sh'])
self.assertEqual(self.links['entry'], {'kind': 'file', 'target': 'tool.sh', 'via': ['entry', 'middle']})
def test_untracked_file_target_is_rejected(self):
self.authored('untracked.txt')
(self.source / 'entry').symlink_to('untracked.txt')
with self.assertRaisesRegex(RuntimeError, 'not in the tracked inventory'):
self.copy(['entry'], required=True)
def test_directory_without_tracked_targets_is_rejected(self):
self.authored('real/untracked.txt')
(self.source / 'entry').symlink_to('real', target_is_directory=True)
with self.assertRaisesRegex(RuntimeError, 'no tracked contents'):
self.copy(['entry'], required=True)
def test_untracked_intermediate_link_is_rejected(self):
self.authored('tracked.txt')
(self.source / 'middle').symlink_to('tracked.txt')
(self.source / 'entry').symlink_to('middle')
with self.assertRaisesRegex(RuntimeError, 'not in the tracked inventory'):
self.copy(['entry', 'tracked.txt'], required=True)
def test_external_file_directory_and_returning_link_are_rejected(self):
outside = self.root / 'outside'
outside.mkdir()
(outside / 'file.txt').write_text('external source')
self.authored('tracked.txt')
(outside / 'returning').symlink_to(self.source / 'tracked.txt')
for name, target, directory in [('file', outside / 'file.txt', False),
('directory', outside, True),
('returning', outside / 'returning', False)]:
with self.subTest(name=name):
(self.source / name).symlink_to(target, target_is_directory=directory)
with self.assertRaisesRegex(RuntimeError, 'leaves its component'):
self.copy([name, 'tracked.txt'], required=True)
def test_file_and_directory_loops_are_rejected(self):
(self.source / 'first').symlink_to('second')
(self.source / 'second').symlink_to('first')
with self.assertRaisesRegex(RuntimeError, 'loop'):
self.copy(['first', 'second'], required=True)
(self.source / 'directory').symlink_to('.', target_is_directory=True)
with self.assertRaisesRegex(RuntimeError, 'loop'):
self.copy(['directory'], required=True)
def test_parent_component_after_link_uses_resolved_directory(self):
self.authored('real/sub/keep.txt')
target = self.authored('real/tool.sh', 'resolved parent source')
(self.source / 'jump').symlink_to('real/sub', target_is_directory=True)
(self.source / 'entry').symlink_to('jump/../tool.sh')
self.copy(['entry', 'jump', 'real/sub/keep.txt', 'real/tool.sh'], required=True)
self.assertEqual((self.destination / 'entry').read_bytes(), target.read_bytes())
self.assertEqual(self.links['entry']['target'], 'real/tool.sh')
def test_dangling_link_is_rejected_even_for_optional_project_files(self):
(self.source / 'entry').symlink_to('missing.txt')
with self.assertRaisesRegex(RuntimeError, 'Missing dependency source file'):
self.copy(['entry', 'missing.txt'])
if __name__ == '__main__':
unittest.main()
+104 -13
View File
@@ -32,20 +32,105 @@ def git(directory, *arguments):
return subprocess.check_output(['git', '-C', str(directory), *arguments])
def copy_files(source, destination, files, required=False):
def copy_files(source, destination, files, required=False, materialized_links=None):
"""Copy only the explicit source inventory, materializing internal links.
Git dependencies pass their tracked paths here. Directory links expand only
paths in that inventory, never a recursive filesystem walk or untracked data.
"""
source = source.resolve()
destination = Path(destination)
manifest = {}
for relative in sorted(set(files)):
origin = source / relative
links = materialized_links if materialized_links is not None else {}
def normalized(path):
path = Path(path)
if path.is_absolute():
try:
path = path.relative_to(source)
except ValueError as error:
raise RuntimeError('Source symlink leaves its component') from error
parts = []
for part in path.parts:
if part == '..':
if not parts:
raise RuntimeError('Source symlink leaves its component')
parts.pop()
elif part != '.':
parts.append(part)
return Path(*parts)
inventory = {normalized(path) for path in files}
def resolve(relative):
pending = normalized(relative)
visited = set()
chain = []
while True:
current = Path()
for index, part in enumerate(pending.parts):
if part == '..':
if current == Path():
raise RuntimeError('Source symlink leaves its component')
current = current.parent
continue
current /= part
origin = source / current
if not origin.is_symlink():
if index + 1 < len(pending.parts) and not origin.is_dir() and (required or chain):
raise RuntimeError('Source path parent is not a directory')
continue
if current not in inventory:
raise RuntimeError('Source symlink is not in the tracked inventory')
state = (current, pending.parts[index + 1:])
if state in visited or len(visited) >= 64:
raise RuntimeError('Source symlink loop')
visited.add(state)
chain.append(current.as_posix())
target = origin.readlink()
if not target.is_absolute():
target = current.parent / target
else:
try:
target = target.relative_to(source)
except ValueError as error:
raise RuntimeError('Source symlink leaves its component') from error
pending = target / Path(*pending.parts[index + 1:])
break
else:
return current, chain
def copy(relative, output, ancestors=()):
resolved, chain = resolve(relative)
origin = source / resolved
if origin.is_dir():
if not chain:
raise RuntimeError('Tracked source entry is a directory')
if resolved in ancestors:
raise RuntimeError('Source directory symlink loop')
children = sorted(path for path in inventory
if path != resolved and path.is_relative_to(resolved))
if not children:
raise RuntimeError('Source directory target has no tracked contents')
links[output.as_posix()] = {'kind': 'directory', 'target': resolved.as_posix(), 'via': chain}
for child in children:
copy(child, output / child.relative_to(resolved), ancestors + (resolved,))
return
if not origin.is_file():
if required:
if required or chain:
raise RuntimeError('Missing dependency source file: ' + relative.as_posix())
continue
if not origin.resolve().is_relative_to(source.resolve()):
raise RuntimeError('Source symlink leaves its component')
target = destination / relative
return
if resolved not in inventory:
raise RuntimeError('Source file target is not in the tracked inventory')
target = destination / output
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(origin, target)
manifest[relative.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest()
manifest[output.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest()
if chain:
links[output.as_posix()] = {'kind': 'file', 'target': resolved.as_posix(), 'via': chain}
for relative in sorted(inventory):
copy(relative, relative)
return manifest
@@ -68,8 +153,11 @@ def main():
with tempfile.TemporaryDirectory(prefix='openbmh-source-', dir=args.out) as temporary:
stage = Path(temporary)
project_files = [Path(p.decode('utf-8')) for p in git(source, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split(b'\0') if p]
project_links = {}
manifest = {'version': version, 'project_commit': git(source, 'rev-parse', 'HEAD').decode().strip(),
'project_files': copy_files(source, stage / 'project', project_files), 'dependencies': {}}
'project_files': copy_files(source, stage / 'project', project_files,
materialized_links=project_links),
'project_materialized_links': project_links, 'dependencies': {}}
initial_cache = []
for name, (reference, url, markers) in COMPONENTS.items():
directory = Path(dependencies.get(name, ''))
@@ -95,9 +183,12 @@ def main():
for filename, expected in MINIZ_FILES.items():
if hashlib.sha256((directory / filename).read_bytes()).hexdigest() != expected:
raise RuntimeError('Dependency differs from its pinned archive: ' + name)
hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True)
dependency_links = {}
hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True,
materialized_links=dependency_links)
manifest['dependencies'][name] = {'upstream': url, 'reference': reference,
'commit': commit, 'files': hashes}
'commit': commit, 'files': hashes,
'materialized_links': dependency_links}
initial_cache.append('set(FETCHCONTENT_SOURCE_DIR_' + name.upper() +
' "${CMAKE_CURRENT_LIST_DIR}/dependencies/' + name + '" CACHE PATH "" FORCE)')
(stage / 'dependencies.cmake').write_text('\n'.join(initial_cache) + '\n')
@@ -105,7 +196,7 @@ def main():
(stage / 'BUILD_SOURCE.md').write_text(
'# Corresponding source\n\nThe project contains its patches, build scripts and vendored Nuked OPL3. '
'The dependency folders contain the sources used by this build; the LZMA folder contains the C decoder tree and upstream license. '
'File hashes and upstream revisions are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n'
'File hashes, upstream revisions and materialized internal links are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n'
'With the native compiler, CMake and platform development libraries described in project/README.md:\n\n'
'```sh\ncmake -S project -B build -C dependencies.cmake -DCMAKE_BUILD_TYPE=Release -DBMH_DATA=\n'
'cmake --build build --config Release --parallel 2\nctest --test-dir build -C Release --output-on-failure\n```\n\n'