Materialize tracked dependency symlinks in source packages (by Codex)
This commit is contained in:
1 parent
9321dff2e9
commit
c37a0c4036
2 files changed
+214
-13
No files matched your search
@@ -2,6 +2,8 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
"""The release allowlist rejects extra game/test files using synthetic payloads."""
|
||||
import importlib.util
|
||||
import hashlib
|
||||
import stat
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -10,6 +12,9 @@ source = Path(__file__).resolve().parents[1] / 'tools' / 'package' / 'make_packa
|
||||
spec = importlib.util.spec_from_file_location('make_package', source)
|
||||
package = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(package)
|
||||
source_spec = importlib.util.spec_from_file_location('make_source_package', source.with_name('make_source_package.py'))
|
||||
source_package = importlib.util.module_from_spec(source_spec)
|
||||
source_spec.loader.exec_module(source_package)
|
||||
|
||||
|
||||
class PackageAllowlistTests(unittest.TestCase):
|
||||
@@ -40,5 +45,110 @@ class PackageAllowlistTests(unittest.TestCase):
|
||||
package.validate(root, False)
|
||||
|
||||
|
||||
class SourceInventoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name)
|
||||
self.source = self.root / 'component'
|
||||
self.source.mkdir()
|
||||
self.destination = self.root / 'materialized'
|
||||
|
||||
def authored(self, name, text='authored source'):
|
||||
target = self.source / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(text)
|
||||
return target
|
||||
|
||||
def copy(self, names, **kwargs):
|
||||
self.links = {}
|
||||
return source_package.copy_files(self.source, self.destination, [Path(n) for n in names],
|
||||
materialized_links=self.links, **kwargs)
|
||||
|
||||
def test_directory_link_copies_only_tracked_contents_and_mode(self):
|
||||
script = self.authored('real/sub/build.sh', '#!/bin/sh\nexit 0\n')
|
||||
script.chmod(0o751)
|
||||
self.authored('real/untracked.txt', 'must never be packaged')
|
||||
(self.source / 'alias').symlink_to('real', target_is_directory=True)
|
||||
hashes = self.copy(['alias', 'real/sub/build.sh'], required=True)
|
||||
copied = self.destination / 'alias/sub/build.sh'
|
||||
self.assertEqual(copied.read_bytes(), script.read_bytes())
|
||||
self.assertFalse((self.destination / 'alias').is_symlink())
|
||||
self.assertFalse((self.destination / 'alias/untracked.txt').exists())
|
||||
self.assertFalse((self.destination / 'real/untracked.txt').exists())
|
||||
self.assertEqual(stat.S_IMODE(copied.stat().st_mode), stat.S_IMODE(script.stat().st_mode))
|
||||
self.assertEqual(hashes['alias/sub/build.sh'], hashlib.sha256(script.read_bytes()).hexdigest())
|
||||
self.assertEqual(self.links['alias'], {'kind': 'directory', 'target': 'real', 'via': ['alias']})
|
||||
|
||||
def test_tracked_file_link_chain_is_materialized_and_recorded(self):
|
||||
target = self.authored('tool.sh', '#!/bin/sh\nexit 0\n')
|
||||
target.chmod(0o755)
|
||||
(self.source / 'middle').symlink_to('tool.sh')
|
||||
(self.source / 'entry').symlink_to('middle')
|
||||
hashes = self.copy(['entry', 'middle', 'tool.sh'], required=True)
|
||||
copied = self.destination / 'entry'
|
||||
self.assertFalse(copied.is_symlink())
|
||||
self.assertEqual(copied.read_bytes(), target.read_bytes())
|
||||
self.assertEqual(stat.S_IMODE(copied.stat().st_mode), stat.S_IMODE(target.stat().st_mode))
|
||||
self.assertEqual(hashes['entry'], hashes['tool.sh'])
|
||||
self.assertEqual(self.links['entry'], {'kind': 'file', 'target': 'tool.sh', 'via': ['entry', 'middle']})
|
||||
|
||||
def test_untracked_file_target_is_rejected(self):
|
||||
self.authored('untracked.txt')
|
||||
(self.source / 'entry').symlink_to('untracked.txt')
|
||||
with self.assertRaisesRegex(RuntimeError, 'not in the tracked inventory'):
|
||||
self.copy(['entry'], required=True)
|
||||
|
||||
def test_directory_without_tracked_targets_is_rejected(self):
|
||||
self.authored('real/untracked.txt')
|
||||
(self.source / 'entry').symlink_to('real', target_is_directory=True)
|
||||
with self.assertRaisesRegex(RuntimeError, 'no tracked contents'):
|
||||
self.copy(['entry'], required=True)
|
||||
|
||||
def test_untracked_intermediate_link_is_rejected(self):
|
||||
self.authored('tracked.txt')
|
||||
(self.source / 'middle').symlink_to('tracked.txt')
|
||||
(self.source / 'entry').symlink_to('middle')
|
||||
with self.assertRaisesRegex(RuntimeError, 'not in the tracked inventory'):
|
||||
self.copy(['entry', 'tracked.txt'], required=True)
|
||||
|
||||
def test_external_file_directory_and_returning_link_are_rejected(self):
|
||||
outside = self.root / 'outside'
|
||||
outside.mkdir()
|
||||
(outside / 'file.txt').write_text('external source')
|
||||
self.authored('tracked.txt')
|
||||
(outside / 'returning').symlink_to(self.source / 'tracked.txt')
|
||||
for name, target, directory in [('file', outside / 'file.txt', False),
|
||||
('directory', outside, True),
|
||||
('returning', outside / 'returning', False)]:
|
||||
with self.subTest(name=name):
|
||||
(self.source / name).symlink_to(target, target_is_directory=directory)
|
||||
with self.assertRaisesRegex(RuntimeError, 'leaves its component'):
|
||||
self.copy([name, 'tracked.txt'], required=True)
|
||||
|
||||
def test_file_and_directory_loops_are_rejected(self):
|
||||
(self.source / 'first').symlink_to('second')
|
||||
(self.source / 'second').symlink_to('first')
|
||||
with self.assertRaisesRegex(RuntimeError, 'loop'):
|
||||
self.copy(['first', 'second'], required=True)
|
||||
(self.source / 'directory').symlink_to('.', target_is_directory=True)
|
||||
with self.assertRaisesRegex(RuntimeError, 'loop'):
|
||||
self.copy(['directory'], required=True)
|
||||
|
||||
def test_parent_component_after_link_uses_resolved_directory(self):
|
||||
self.authored('real/sub/keep.txt')
|
||||
target = self.authored('real/tool.sh', 'resolved parent source')
|
||||
(self.source / 'jump').symlink_to('real/sub', target_is_directory=True)
|
||||
(self.source / 'entry').symlink_to('jump/../tool.sh')
|
||||
self.copy(['entry', 'jump', 'real/sub/keep.txt', 'real/tool.sh'], required=True)
|
||||
self.assertEqual((self.destination / 'entry').read_bytes(), target.read_bytes())
|
||||
self.assertEqual(self.links['entry']['target'], 'real/tool.sh')
|
||||
|
||||
def test_dangling_link_is_rejected_even_for_optional_project_files(self):
|
||||
(self.source / 'entry').symlink_to('missing.txt')
|
||||
with self.assertRaisesRegex(RuntimeError, 'Missing dependency source file'):
|
||||
self.copy(['entry', 'missing.txt'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -32,20 +32,105 @@ def git(directory, *arguments):
|
||||
return subprocess.check_output(['git', '-C', str(directory), *arguments])
|
||||
|
||||
|
||||
def copy_files(source, destination, files, required=False):
|
||||
def copy_files(source, destination, files, required=False, materialized_links=None):
|
||||
"""Copy only the explicit source inventory, materializing internal links.
|
||||
|
||||
Git dependencies pass their tracked paths here. Directory links expand only
|
||||
paths in that inventory, never a recursive filesystem walk or untracked data.
|
||||
"""
|
||||
source = source.resolve()
|
||||
destination = Path(destination)
|
||||
manifest = {}
|
||||
for relative in sorted(set(files)):
|
||||
origin = source / relative
|
||||
links = materialized_links if materialized_links is not None else {}
|
||||
|
||||
def normalized(path):
|
||||
path = Path(path)
|
||||
if path.is_absolute():
|
||||
try:
|
||||
path = path.relative_to(source)
|
||||
except ValueError as error:
|
||||
raise RuntimeError('Source symlink leaves its component') from error
|
||||
parts = []
|
||||
for part in path.parts:
|
||||
if part == '..':
|
||||
if not parts:
|
||||
raise RuntimeError('Source symlink leaves its component')
|
||||
parts.pop()
|
||||
elif part != '.':
|
||||
parts.append(part)
|
||||
return Path(*parts)
|
||||
|
||||
inventory = {normalized(path) for path in files}
|
||||
|
||||
def resolve(relative):
|
||||
pending = normalized(relative)
|
||||
visited = set()
|
||||
chain = []
|
||||
while True:
|
||||
current = Path()
|
||||
for index, part in enumerate(pending.parts):
|
||||
if part == '..':
|
||||
if current == Path():
|
||||
raise RuntimeError('Source symlink leaves its component')
|
||||
current = current.parent
|
||||
continue
|
||||
current /= part
|
||||
origin = source / current
|
||||
if not origin.is_symlink():
|
||||
if index + 1 < len(pending.parts) and not origin.is_dir() and (required or chain):
|
||||
raise RuntimeError('Source path parent is not a directory')
|
||||
continue
|
||||
if current not in inventory:
|
||||
raise RuntimeError('Source symlink is not in the tracked inventory')
|
||||
state = (current, pending.parts[index + 1:])
|
||||
if state in visited or len(visited) >= 64:
|
||||
raise RuntimeError('Source symlink loop')
|
||||
visited.add(state)
|
||||
chain.append(current.as_posix())
|
||||
target = origin.readlink()
|
||||
if not target.is_absolute():
|
||||
target = current.parent / target
|
||||
else:
|
||||
try:
|
||||
target = target.relative_to(source)
|
||||
except ValueError as error:
|
||||
raise RuntimeError('Source symlink leaves its component') from error
|
||||
pending = target / Path(*pending.parts[index + 1:])
|
||||
break
|
||||
else:
|
||||
return current, chain
|
||||
|
||||
def copy(relative, output, ancestors=()):
|
||||
resolved, chain = resolve(relative)
|
||||
origin = source / resolved
|
||||
if origin.is_dir():
|
||||
if not chain:
|
||||
raise RuntimeError('Tracked source entry is a directory')
|
||||
if resolved in ancestors:
|
||||
raise RuntimeError('Source directory symlink loop')
|
||||
children = sorted(path for path in inventory
|
||||
if path != resolved and path.is_relative_to(resolved))
|
||||
if not children:
|
||||
raise RuntimeError('Source directory target has no tracked contents')
|
||||
links[output.as_posix()] = {'kind': 'directory', 'target': resolved.as_posix(), 'via': chain}
|
||||
for child in children:
|
||||
copy(child, output / child.relative_to(resolved), ancestors + (resolved,))
|
||||
return
|
||||
if not origin.is_file():
|
||||
if required:
|
||||
if required or chain:
|
||||
raise RuntimeError('Missing dependency source file: ' + relative.as_posix())
|
||||
continue
|
||||
if not origin.resolve().is_relative_to(source.resolve()):
|
||||
raise RuntimeError('Source symlink leaves its component')
|
||||
target = destination / relative
|
||||
return
|
||||
if resolved not in inventory:
|
||||
raise RuntimeError('Source file target is not in the tracked inventory')
|
||||
target = destination / output
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(origin, target)
|
||||
manifest[relative.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest()
|
||||
manifest[output.as_posix()] = hashlib.sha256(target.read_bytes()).hexdigest()
|
||||
if chain:
|
||||
links[output.as_posix()] = {'kind': 'file', 'target': resolved.as_posix(), 'via': chain}
|
||||
|
||||
for relative in sorted(inventory):
|
||||
copy(relative, relative)
|
||||
return manifest
|
||||
|
||||
|
||||
@@ -68,8 +153,11 @@ def main():
|
||||
with tempfile.TemporaryDirectory(prefix='openbmh-source-', dir=args.out) as temporary:
|
||||
stage = Path(temporary)
|
||||
project_files = [Path(p.decode('utf-8')) for p in git(source, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split(b'\0') if p]
|
||||
project_links = {}
|
||||
manifest = {'version': version, 'project_commit': git(source, 'rev-parse', 'HEAD').decode().strip(),
|
||||
'project_files': copy_files(source, stage / 'project', project_files), 'dependencies': {}}
|
||||
'project_files': copy_files(source, stage / 'project', project_files,
|
||||
materialized_links=project_links),
|
||||
'project_materialized_links': project_links, 'dependencies': {}}
|
||||
initial_cache = []
|
||||
for name, (reference, url, markers) in COMPONENTS.items():
|
||||
directory = Path(dependencies.get(name, ''))
|
||||
@@ -95,9 +183,12 @@ def main():
|
||||
for filename, expected in MINIZ_FILES.items():
|
||||
if hashlib.sha256((directory / filename).read_bytes()).hexdigest() != expected:
|
||||
raise RuntimeError('Dependency differs from its pinned archive: ' + name)
|
||||
hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True)
|
||||
dependency_links = {}
|
||||
hashes = copy_files(directory, stage / 'dependencies' / name, files, required=True,
|
||||
materialized_links=dependency_links)
|
||||
manifest['dependencies'][name] = {'upstream': url, 'reference': reference,
|
||||
'commit': commit, 'files': hashes}
|
||||
'commit': commit, 'files': hashes,
|
||||
'materialized_links': dependency_links}
|
||||
initial_cache.append('set(FETCHCONTENT_SOURCE_DIR_' + name.upper() +
|
||||
' "${CMAKE_CURRENT_LIST_DIR}/dependencies/' + name + '" CACHE PATH "" FORCE)')
|
||||
(stage / 'dependencies.cmake').write_text('\n'.join(initial_cache) + '\n')
|
||||
@@ -105,7 +196,7 @@ def main():
|
||||
(stage / 'BUILD_SOURCE.md').write_text(
|
||||
'# Corresponding source\n\nThe project contains its patches, build scripts and vendored Nuked OPL3. '
|
||||
'The dependency folders contain the sources used by this build; the LZMA folder contains the C decoder tree and upstream license. '
|
||||
'File hashes and upstream revisions are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n'
|
||||
'File hashes, upstream revisions and materialized internal links are in SOURCE_MANIFEST.json. No game data or test references are included.\n\n'
|
||||
'With the native compiler, CMake and platform development libraries described in project/README.md:\n\n'
|
||||
'```sh\ncmake -S project -B build -C dependencies.cmake -DCMAKE_BUILD_TYPE=Release -DBMH_DATA=\n'
|
||||
'cmake --build build --config Release --parallel 2\nctest --test-dir build -C Release --output-on-failure\n```\n\n'
|
||||
|
||||
Reference in new issue
Block a user