Complete original edition rules, season safety and verified 1.0 releases (by Codex)
This commit is contained in:
1 parent
e432f20ae7
commit
f0ef79a119
21 files changed
+563
-30
No files matched your search
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
"""Build products and every configured test program, without unused data fixtures."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
TEST = re.compile(r'^add_test\((?:\[=*\[.*?\]=*\]|"[^"]*"|[^ )]+)\s+"((?:\\.|[^"\\])*)"')
|
||||
HOST = re.compile(r'(?:python(?:[0-9.]+)?|cmake|ctest)(?:\.exe)?$', re.I)
|
||||
|
||||
|
||||
def targets_for_tests(build):
|
||||
build = build.resolve(strict=True)
|
||||
targets = {'openbmh_play', 'openbmh_cli'}
|
||||
tests = 0
|
||||
for file in sorted(build.rglob('CTestTestfile.cmake')):
|
||||
for line in file.read_text(encoding='utf-8').splitlines():
|
||||
if not line.startswith('add_test('):
|
||||
continue
|
||||
match = TEST.match(line)
|
||||
if not match:
|
||||
raise RuntimeError('Unrecognized generated test command; refusing to omit a test')
|
||||
tests += 1
|
||||
command = match.group(1).replace('\\"', '"').replace('\\\\', '\\')
|
||||
executable = Path(command).resolve()
|
||||
if executable.is_relative_to(build):
|
||||
name = executable.name
|
||||
if name.lower().endswith('.exe'):
|
||||
name = name[:-4]
|
||||
targets.add('openbmh_play' if name == 'OpenBMH' else name)
|
||||
elif not HOST.fullmatch(executable.name):
|
||||
raise RuntimeError('Unrecognized external test executable; refusing to omit a test')
|
||||
if not tests:
|
||||
raise RuntimeError('No configured tests found')
|
||||
return sorted(targets), tests
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--build', type=Path, required=True)
|
||||
parser.add_argument('--config', default='Release')
|
||||
parser.add_argument('--parallel', type=int, default=2)
|
||||
parser.add_argument('--list', action='store_true')
|
||||
args = parser.parse_args()
|
||||
targets, count = targets_for_tests(args.build)
|
||||
print(f'{count} configured tests; {len(targets)} product/test build targets', flush=True)
|
||||
if args.list:
|
||||
print('\n'.join(targets))
|
||||
else:
|
||||
subprocess.run(['cmake', '--build', str(args.build), '--config', args.config,
|
||||
'--parallel', str(args.parallel), '--target', *targets], check=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from build_runnable_tests import targets_for_tests
|
||||
|
||||
|
||||
class BuildDiscoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix='ci build ')
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.build = Path(self.temporary.name) / 'build'
|
||||
self.build.mkdir()
|
||||
|
||||
def write(self, text):
|
||||
(self.build / 'CTestTestfile.cmake').write_text(text, encoding='utf-8')
|
||||
|
||||
def test_required_programs_and_optional_data_skip(self):
|
||||
self.write(f'add_test([=[unit]=] "{self.build.as_posix()}/unit_suite.exe")\n'
|
||||
f'add_test([=[private]=] "{self.build.as_posix()}/openbmh_test_skip.exe" "missing input")\n'
|
||||
f'add_test([=[python]=] "{Path(sys.executable).as_posix()}" "check.py")\n')
|
||||
targets, count = targets_for_tests(self.build)
|
||||
self.assertEqual(count, 3)
|
||||
self.assertEqual(targets, ['openbmh_cli', 'openbmh_play', 'openbmh_test_skip', 'unit_suite'])
|
||||
|
||||
def test_nested_test_and_product_output_name(self):
|
||||
self.write(f'add_test([=[version]=] "{self.build.as_posix()}/OpenBMH.exe" "--version")\n')
|
||||
nested = self.build / 'src' / 'ui'
|
||||
nested.mkdir(parents=True)
|
||||
(nested / 'CTestTestfile.cmake').write_text(
|
||||
f'add_test([=[ui]=] "{nested.as_posix()}/ui_suite")\n', encoding='utf-8')
|
||||
targets, count = targets_for_tests(self.build)
|
||||
self.assertEqual(count, 2)
|
||||
self.assertEqual(targets, ['openbmh_cli', 'openbmh_play', 'ui_suite'])
|
||||
|
||||
def test_unknown_external_binary_is_not_silently_dropped(self):
|
||||
outside = self.build.parent / 'unknown-suite'
|
||||
self.write(f'add_test([=[outside]=] "{outside.as_posix()}")\n')
|
||||
with self.assertRaisesRegex(RuntimeError, 'external'):
|
||||
targets_for_tests(self.build)
|
||||
|
||||
def test_unrecognized_command_and_empty_configuration_fail(self):
|
||||
self.write('add_test(unsupported-syntax)\n')
|
||||
with self.assertRaises(RuntimeError):
|
||||
targets_for_tests(self.build)
|
||||
self.write('# no tests\n')
|
||||
with self.assertRaises(RuntimeError):
|
||||
targets_for_tests(self.build)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from verify_ci_release import PLATFORMS, verify
|
||||
|
||||
|
||||
class ReleaseVerificationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name)
|
||||
self.dist = self.root / 'dist'; self.dist.mkdir()
|
||||
self.reports = self.root / 'reports'; self.reports.mkdir()
|
||||
self.sha = 'a' * 40
|
||||
for platform in (*PLATFORMS, 'source'):
|
||||
with zipfile.ZipFile(self.dist / f'OpenBMH-1.0.0-{platform}.zip', 'w') as z:
|
||||
if platform == 'source':
|
||||
z.writestr('source/SOURCE_MANIFEST.json', json.dumps({'version': '1.0.0', 'project_commit': self.sha}))
|
||||
else:
|
||||
z.writestr('synthetic.txt', 'synthetic package')
|
||||
for platform in PLATFORMS:
|
||||
folder = self.reports / ('tests-' + platform); folder.mkdir()
|
||||
(folder / 'test-results.xml').write_text('<testsuite tests="4" failures="0" skipped="1"/>')
|
||||
|
||||
def run_verify(self, sha=None):
|
||||
return verify(self.dist, self.reports, 'v1.0.0', sha or self.sha, 123, 'example/project')
|
||||
|
||||
def test_matching_release_records_skips_and_checksums(self):
|
||||
report = self.run_verify()
|
||||
self.assertEqual(report['tests']['linux-x64']['skipped'], 1)
|
||||
self.assertEqual(len(report['archive_sha256']), 4)
|
||||
self.assertEqual(len((self.dist / 'SHA256SUMS').read_text().splitlines()), 5)
|
||||
|
||||
def test_wrong_revision_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, 'different'):
|
||||
self.run_verify('b' * 40)
|
||||
|
||||
def test_missing_platform_archive_rejected(self):
|
||||
(self.dist / 'OpenBMH-1.0.0-windows-x64.zip').unlink()
|
||||
with self.assertRaisesRegex(ValueError, 'archive set'):
|
||||
self.run_verify()
|
||||
|
||||
def test_failed_or_all_skipped_reports_rejected(self):
|
||||
path = self.reports / 'tests-windows-x64' / 'test-results.xml'
|
||||
path.write_text('<testsuite tests="4" failures="0"><testcase><failure/></testcase></testsuite>')
|
||||
with self.assertRaisesRegex(ValueError, 'failed'):
|
||||
self.run_verify()
|
||||
path.write_text('<testsuite tests="4" failures="0" skipped="4"/>')
|
||||
with self.assertRaisesRegex(ValueError, 'execute'):
|
||||
self.run_verify()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
"""Bind release archives to one successful CI revision and its test reports."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
|
||||
PLATFORMS = ('windows-x64', 'macos-arm64', 'linux-x64')
|
||||
|
||||
|
||||
def verify(dist, reports, tag, source_sha, ci_run, repository):
|
||||
match = re.fullmatch(r'v([0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9][A-Za-z0-9.-]*)?)', tag)
|
||||
if not match or not re.fullmatch(r'[0-9a-f]{40}', source_sha) or ci_run <= 0:
|
||||
raise ValueError('Invalid release identity')
|
||||
version = match.group(1)
|
||||
expected = {f'OpenBMH-{version}-{p}.zip' for p in (*PLATFORMS, 'source')}
|
||||
if {p.name for p in dist.glob('*.zip')} != expected:
|
||||
raise ValueError('Release archive set does not match the tag/platforms')
|
||||
with zipfile.ZipFile(dist / f'OpenBMH-{version}-source.zip') as source:
|
||||
manifests = [n for n in source.namelist() if n.endswith('/SOURCE_MANIFEST.json')]
|
||||
if len(manifests) != 1:
|
||||
raise ValueError('Missing or ambiguous corresponding-source manifest')
|
||||
manifest = json.loads(source.read(manifests[0]))
|
||||
if manifest['version'] != version or manifest['project_commit'] != source_sha:
|
||||
raise ValueError('Corresponding source is from a different version or revision')
|
||||
tests = {}
|
||||
for platform in PLATFORMS:
|
||||
root = ET.parse(reports / ('tests-' + platform) / 'test-results.xml').getroot()
|
||||
suites = list(root.iter('testsuite'))
|
||||
if not suites or next(root.iter('failure'), None) is not None or next(root.iter('error'), None) is not None:
|
||||
raise ValueError('Missing or failed platform test results')
|
||||
values = {key: sum(int(s.get(key, '0')) for s in suites)
|
||||
for key in ('tests', 'failures', 'errors', 'skipped')}
|
||||
if values['failures'] or values['errors'] or values['tests'] <= values['skipped']:
|
||||
raise ValueError('Platform test results did not execute passing tests')
|
||||
tests[platform] = values
|
||||
archives = {name: hashlib.sha256((dist / name).read_bytes()).hexdigest() for name in sorted(expected)}
|
||||
report = {'version': version, 'tag': tag, 'source_commit': source_sha,
|
||||
'ci_run': ci_run, 'ci_url': f'https://github.com/{repository}/actions/runs/{ci_run}',
|
||||
'tests': tests, 'archive_sha256': archives}
|
||||
output = dist / 'QA-REPORT.json'
|
||||
output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
checksums = dict(archives)
|
||||
checksums[output.name] = hashlib.sha256(output.read_bytes()).hexdigest()
|
||||
(dist / 'SHA256SUMS').write_text(''.join(f'{digest} {name}\n' for name, digest in sorted(checksums.items())), encoding='utf-8')
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--dist', type=Path, required=True)
|
||||
p.add_argument('--reports', type=Path, required=True)
|
||||
p.add_argument('--tag', required=True)
|
||||
p.add_argument('--source-sha', required=True)
|
||||
p.add_argument('--ci-run', type=int, required=True)
|
||||
p.add_argument('--repository', required=True)
|
||||
a = p.parse_args()
|
||||
verify(a.dist, a.reports, a.tag, a.source_sha, a.ci_run, a.repository)
|
||||
print('Release archives, source revision and all platform reports verified')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user