258 lines
15 KiB
Python
258 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Build a local Ubuntu 24.04 amd64 deb with an explicitly selected Qt SDK."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
|
|
from collect_ubuntu_validation_runtime import Collector, EXECUTABLES, bounded_walk
|
|
from verify_pdfium_candidate import verify_installed as verify_pdfium_installed
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
VERSION = '0.1.0~validation3'
|
|
VERSIONS = (VERSION, '0.1.0~validation4', '0.1.0~validation5')
|
|
MANIFEST = 'opt/docview/share/doc/docview/package-manifest.json'
|
|
SDK_SUPPLEMENT_REPORT_SHA = 'e218e5a24d136dbfe2982e56dbf9bb1e3fd104644c967ca02234b5e6be47096e'
|
|
|
|
|
|
def sha(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def canonical(name):
|
|
if not isinstance(name, str) or '\\' in name or '\x00' in name:
|
|
raise ValueError('Invalid relative path')
|
|
path = PurePosixPath(name)
|
|
if path.is_absolute() or '..' in path.parts or str(path) != name or name in ('', '.'):
|
|
raise ValueError('Invalid relative path')
|
|
return path
|
|
|
|
|
|
def copy_verified(source, target, digest=None, executable=None):
|
|
if not source.is_file() or source.stat().st_size > 512 * 1024**2:
|
|
raise ValueError('Invalid or oversized package input')
|
|
expected = digest or sha(source)
|
|
if sha(source) != expected:
|
|
raise ValueError('Input changed: ' + str(source))
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
if target.exists():
|
|
if sha(target) != expected:
|
|
raise ValueError('Conflicting package destination')
|
|
return
|
|
shutil.copyfile(source, target)
|
|
target.chmod(0o755 if (os.access(source, os.X_OK) if executable is None else executable) else 0o644)
|
|
if sha(target) != expected or sha(source) != expected:
|
|
raise ValueError('Package input changed while copying')
|
|
|
|
|
|
def normalize_modes(directory):
|
|
for path in [directory, *directory.rglob('*')]:
|
|
if path.is_symlink() or (not path.is_dir() and not path.is_file()):
|
|
raise ValueError('Unexpected staged link or special file')
|
|
path.chmod(0o755 if path.is_dir() or path.stat().st_mode & 0o111 else 0o644)
|
|
|
|
|
|
def copy_sdk_supplement(directory, base_report_path, known_runtime, destination):
|
|
directory = directory.resolve(strict=True)
|
|
report_path = directory / 'report.json'
|
|
if report_path.is_symlink() or sha(report_path) != SDK_SUPPLEMENT_REPORT_SHA:
|
|
raise ValueError('Qt SDK supplement report differs from the reviewed fixed input')
|
|
report = json.loads(report_path.read_text())
|
|
if sha(base_report_path) != report['baseNoticeReportSha256']:
|
|
raise ValueError('Qt SDK supplement belongs to a different base notice report')
|
|
for row in report['runtimeComparisons']:
|
|
if known_runtime.get(row['resolvedPath']) != row['sha256']:
|
|
raise ValueError('Qt SDK supplement belongs to a different runtime')
|
|
inputs = []
|
|
for row in report['files']:
|
|
name = canonical(row['file'])
|
|
source = directory / name
|
|
if source.is_symlink() or not source.resolve(strict=True).is_relative_to(directory):
|
|
raise ValueError('Qt SDK supplement source escapes its selected directory')
|
|
if source.stat().st_size != row['bytes'] or sha(source) != row['sha256']:
|
|
raise ValueError('Qt SDK supplement source changed')
|
|
inputs.append((source, destination / name, row['sha256']))
|
|
# Check every input before creating the destination, then verify each copy.
|
|
for source, target, digest in inputs:
|
|
copy_verified(source, target, digest, executable=False)
|
|
copy_verified(report_path, destination / 'report.json', SDK_SUPPLEMENT_REPORT_SHA, executable=False)
|
|
return SDK_SUPPLEMENT_REPORT_SHA
|
|
|
|
|
|
def create(args):
|
|
version = getattr(args, 'version', VERSION)
|
|
if version not in VERSIONS:
|
|
raise ValueError('Unsupported local validation package version')
|
|
if platform.freedesktop_os_release().get('ID') != 'ubuntu' or platform.freedesktop_os_release().get('VERSION_ID') != '24.04':
|
|
raise ValueError('Build this package on Ubuntu 24.04')
|
|
if subprocess.check_output(['dpkg', '--print-architecture'], text=True).strip() != 'amd64':
|
|
raise ValueError('Only amd64 is supported')
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
collector = Collector(args.prefix, args.qtpaths, args.dependency_prefix, output / 'runtime', args.source_root,
|
|
qpdf_source_archive=getattr(args, 'qpdf_source_archive', None))
|
|
runtime = collector.collect(platform.freedesktop_os_release(), args.input_manifest)
|
|
if not runtime['runtimeValidationSuccess'] or runtime['systemOwnershipUnresolved']:
|
|
raise ValueError('Runtime inventory has unresolved dependencies')
|
|
(output / 'runtime/runtime.json').write_text(json.dumps(runtime, indent=2, sort_keys=True) + '\n')
|
|
stage = output / 'stage'; stage.mkdir()
|
|
app = stage / 'opt/docview'
|
|
mapping = {'install': (collector.prefix, app), 'qt-sdk': (collector.sdk, app / 'qt'),
|
|
'dependencies': (collector.deps, app)}
|
|
copied = []
|
|
for row in runtime['files']:
|
|
if not row['path'].startswith(tuple(key + ':' for key in mapping)):
|
|
continue
|
|
kind, relative = row['path'].split(':', 1)
|
|
source_root, destination_root = mapping[kind]
|
|
# The inventory has one harmless bin/../lib alias for PDFium. Normalize
|
|
# the lexical destination while checking the actual source separately.
|
|
normalized = os.path.normpath(relative)
|
|
canonical(normalized)
|
|
source = source_root / normalized
|
|
if not source.resolve(strict=True).is_relative_to(source_root):
|
|
raise ValueError('Runtime source escapes its selected prefix')
|
|
target = destination_root / normalized
|
|
copy_verified(source, target, row['sha256'])
|
|
copied.append({'source': row['path'], 'path': str(target.relative_to(stage)), 'sha256': row['sha256']})
|
|
documentation = app / 'share/doc/docview'
|
|
for path in bounded_walk(collector.prefix / 'share/doc/docview', (collector.prefix,)):
|
|
copy_verified(path, documentation / path.relative_to(collector.prefix / 'share/doc/docview'), executable=False)
|
|
pdfium_correspondence = verify_pdfium_installed(app)
|
|
if pdfium_correspondence != runtime['pdfiumCorrespondence']:
|
|
raise ValueError('PDFium correspondence changed during packaging')
|
|
# Preserve selected runtime notices and SBOMs with the original scope ledger.
|
|
shutil.copytree(output / 'runtime', documentation / 'third-party/runtime')
|
|
supplemental = args.sdk_notices.resolve(strict=True)
|
|
notice_report = json.loads((supplemental / 'report.json').read_text())
|
|
if not notice_report['success'] or len(notice_report['notices']) != 129:
|
|
raise ValueError('Verified Qt SDK notice extraction is required')
|
|
known_runtime = {row['resolvedPath']: row['sha256'] for row in runtime['files']}
|
|
for row in notice_report['testedRuntimeComparisons']:
|
|
if known_runtime.get(row['resolvedPath']) != row['sha256']:
|
|
raise ValueError('Qt SDK notice extraction belongs to a different runtime')
|
|
copy_verified(supplemental / 'report.json', documentation / 'third-party/qt-sdk/source-report.json', executable=False)
|
|
copy_verified(supplemental / 'THIRD_PARTY_NOTICES.sdk-extract.txt', documentation / 'third-party/qt-sdk/NOTICES.txt',
|
|
notice_report['noticeBundleSha256'], executable=False)
|
|
for row in notice_report['notices']:
|
|
name = canonical(row['file'])
|
|
copy_verified(supplemental / name, documentation / 'third-party/qt-sdk' / name, row['sha256'], executable=False)
|
|
supplement_sha = copy_sdk_supplement(args.sdk_supplement, supplemental / 'report.json', known_runtime,
|
|
documentation / 'third-party/qt-sdk-supplement')
|
|
for path in bounded_walk(args.qt_licenses.resolve(strict=True), (args.qt_licenses.resolve(strict=True),)):
|
|
copy_verified(path, documentation / 'third-party/qt-licenses' / path.relative_to(args.qt_licenses.resolve()), executable=False)
|
|
(documentation / 'UBUNTU-PACKAGE.txt').write_text(
|
|
'DocView — Ubuntu 24.04 amd64\n\nStart: docview [document]\n'
|
|
'Remove application: sudo apt remove docview\n'
|
|
'Remove application and its system profile: sudo apt purge docview\n'
|
|
'User documents, preferences and reading history are preserved.\n\n'
|
|
'This local validation package includes a private Qt SDK runtime, qpdf, libzip and PDFium.\n'
|
|
'System dependencies are declared in the Debian control metadata.\n'
|
|
'Third-party notices and source references are under third-party/.\n'
|
|
'The Qt SDK supplement retains WebM/WebP Patent files declared by the fixed upstream metadata.\n'
|
|
'Notice/source completeness and public release conditions have not been finalized.\n')
|
|
for source, target, executable in [
|
|
('docview-launcher', 'usr/bin/docview', True), ('docview.desktop', 'usr/share/applications/docview.desktop', False),
|
|
('docview.apparmor', 'etc/apparmor.d/docview', False), ('deb-postinst', 'DEBIAN/postinst', True),
|
|
('deb-prerm', 'DEBIAN/prerm', True)]:
|
|
copy_verified(ROOT / 'resources/linux' / source, stage / target, executable=executable)
|
|
for target, prefix in [(app / 'bin/qt.conf', '../qt'), (app / 'qt/libexec/qt.conf', '..')]:
|
|
target.write_text('[Paths]\nPrefix=' + prefix + '\nLibraries=lib\nLibraryExecutables=libexec\n'
|
|
'Plugins=plugins\nQmlImports=qml\nTranslations=translations\nData=.\n')
|
|
dependencies = {'apparmor', 'fonts-dejavu-core', 'fonts-noto-cjk'}
|
|
for name, package in runtime['systemPackages'].items():
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9+.-]*(?::amd64)?', name) or not re.fullmatch(r'[0-9A-Za-z.+:~\-]+', package['version']):
|
|
raise ValueError('Invalid Debian dependency identity')
|
|
dependencies.add(name.split(':')[0] + ' (>= ' + package['version'] + ')')
|
|
control = stage / 'DEBIAN/control'
|
|
control.write_text('Package: docview\nVersion: ' + version + '\nArchitecture: amd64\n'
|
|
'Maintainer: DocView contributors\nSection: text\nPriority: optional\n'
|
|
'Depends: ' + ', '.join(sorted(dependencies)) + '\n'
|
|
'Description: Local PDF, HTML and EPUB document viewer\n'
|
|
' A keyboard-oriented Qt Quick viewer with isolated PDF and archive workers.\n')
|
|
(stage / 'DEBIAN/conffiles').write_text('/etc/apparmor.d/docview\n')
|
|
normalize_modes(stage)
|
|
payload = []
|
|
total = 0
|
|
for path in sorted(stage.rglob('*')):
|
|
if path.is_symlink() or (not path.is_dir() and not path.is_file()):
|
|
raise ValueError('Unexpected staged link or special file')
|
|
if not path.is_file(): continue
|
|
total += path.stat().st_size
|
|
if total > 2 * 1024**3 or len(payload) >= 10000:
|
|
raise ValueError('Package exceeds finite payload limits')
|
|
payload.append({'path': str(path.relative_to(stage)), 'size': path.stat().st_size,
|
|
'sha256': sha(path), 'mode': oct(path.stat().st_mode & 0o777)})
|
|
manifest = {'schemaVersion': 1, 'package': 'docview', 'version': version, 'target': 'Ubuntu 24.04 amd64',
|
|
'files': payload, 'excludedFromOwnDigest': [MANIFEST], 'runtimeCopies': copied,
|
|
'sdkSupplementReportSha256': supplement_sha,
|
|
'pdfiumCorrespondence': pdfium_correspondence,
|
|
'qpdfNoticeCorrespondence': runtime['qpdfNoticeCorrespondence'],
|
|
'completeChromiumNotices': False, 'completeCorrespondingSources': False,
|
|
'publicReleaseApproved': False}
|
|
(stage / MANIFEST).write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n')
|
|
(stage / MANIFEST).chmod(0o644)
|
|
# Debian's size field includes the data archive, rounded to KiB per file
|
|
# and directory. Its control row is itself recorded in our manifest, so
|
|
# settle the tiny self-size dependency before building the archive.
|
|
for _ in range(8):
|
|
installed_kib = sum(1 if path.is_dir() else (path.stat().st_size + 1023) // 1024
|
|
for path in stage.rglob('*') if path.relative_to(stage).parts[0] != 'DEBIAN')
|
|
previous = control.read_text()
|
|
updated = re.sub(r'^Installed-Size:.*\n', '', previous, flags=re.M)
|
|
updated += 'Installed-Size: ' + str(installed_kib) + '\n'
|
|
if updated == previous:
|
|
break
|
|
control.write_text(updated)
|
|
row = next(row for row in payload if row['path'] == 'DEBIAN/control')
|
|
row.update(size=control.stat().st_size, sha256=sha(control))
|
|
(stage / MANIFEST).write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n')
|
|
else:
|
|
raise ValueError('Installed size did not converge')
|
|
total = sum(row['size'] for row in payload)
|
|
environment = {**os.environ, 'SOURCE_DATE_EPOCH': '0', 'LC_ALL': 'C'}
|
|
archive = output / ('docview_' + version + '_amd64.deb')
|
|
command = ['dpkg-deb', '--root-owner-group', '-Zxz', '-z6', '--threads-max=2', '--build', str(stage), str(archive)]
|
|
subprocess.run(command, env=environment, check=True, timeout=300)
|
|
report = {'success': True, 'archive': archive.name, 'archiveSha256': sha(archive), 'archiveBytes': archive.stat().st_size,
|
|
'payloadFiles': len(payload) + 1, 'payloadBytes': total + (stage / MANIFEST).stat().st_size,
|
|
'sourceDateEpoch': 0, 'packagerSha256': sha(Path(__file__)),
|
|
'collectorSha256': sha(ROOT / 'tools/collect_ubuntu_validation_runtime.py'),
|
|
'runtimeRecordSha256': sha(output / 'runtime/runtime.json'), 'systemDependencies': sorted(dependencies),
|
|
'sdkSupplementReportSha256': supplement_sha,
|
|
'pdfiumCorrespondence': pdfium_correspondence,
|
|
'qpdfNoticeCorrespondence': runtime['qpdfNoticeCorrespondence'],
|
|
'manifestSha256': sha(stage / MANIFEST), 'installedSmoke': 'not-run-by-packager',
|
|
'installedSizeKiB': installed_kib,
|
|
'executableSha256': {name: sha(app / 'bin' / name) for name in EXECUTABLES},
|
|
'completeChromiumNotices': False, 'completeCorrespondingSources': False, 'publicReleaseApproved': False}
|
|
(output / 'report.json').write_text(json.dumps(report, indent=2, sort_keys=True) + '\n')
|
|
print(json.dumps({k: report[k] for k in ('success', 'archiveBytes', 'payloadFiles', 'payloadBytes')}))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--prefix', type=Path, required=True)
|
|
parser.add_argument('--qtpaths', type=Path, required=True)
|
|
parser.add_argument('--dependency-prefix', type=Path, required=True)
|
|
parser.add_argument('--source-root', type=Path, action='append', default=[])
|
|
parser.add_argument('--input-manifest', type=Path, required=True)
|
|
parser.add_argument('--sdk-notices', type=Path, required=True)
|
|
parser.add_argument('--sdk-supplement', type=Path, required=True)
|
|
parser.add_argument('--qt-licenses', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--version', choices=VERSIONS, default=VERSION)
|
|
parser.add_argument('--qpdf-source-archive', type=Path,
|
|
help='Pinned qpdf source archive required when bundling the known qpdf runtime')
|
|
create(parser.parse_args())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|