224 lines
11 KiB
Python
224 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Build a deterministic, local Arch development tar.gz from a trusted install.
|
|
|
|
No system runtime binaries or user configuration/history are copied. This is not
|
|
a self-contained Linux release. The same install, host inventory, packaging code,
|
|
Python/zlib and SOURCE_DATE_EPOCH produce the same archive bytes.
|
|
"""
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import zlib
|
|
|
|
from collect_linux_dependencies import EXECUTABLES, ROOT, inventory, sha256
|
|
|
|
PACKAGE_NAME = 'docview-0.1.0-arch-x86_64-development'
|
|
MAX_PAYLOAD_BYTES = 512 * 1024 * 1024
|
|
MANIFEST = 'share/doc/docview/package-manifest.json'
|
|
|
|
|
|
def payload_files(directory):
|
|
result, size = [], 0
|
|
for path in sorted(directory.rglob('*')):
|
|
info = path.lstat()
|
|
if stat.S_ISLNK(info.st_mode) or not (stat.S_ISREG(info.st_mode) or stat.S_ISDIR(info.st_mode)):
|
|
raise ValueError('Package contains a link or special file: ' + str(path))
|
|
if stat.S_ISDIR(info.st_mode):
|
|
continue
|
|
if info.st_nlink != 1:
|
|
raise ValueError('Package contains a multiply-linked file: ' + str(path))
|
|
size += info.st_size
|
|
if size > MAX_PAYLOAD_BYTES or len(result) >= 10000:
|
|
raise ValueError('Package payload exceeds its finite limit')
|
|
result.append(path)
|
|
return result
|
|
|
|
|
|
def copy_install(prefix, destination):
|
|
allowed = {Path('bin') / name for name in EXECUTABLES} | {Path('lib/libpdfium.so')}
|
|
files = payload_files(prefix)
|
|
present = {path.relative_to(prefix) for path in files}
|
|
if not allowed <= present:
|
|
raise ValueError('Install is missing an application executable or PDFium')
|
|
initial_hashes = {str(path): sha256(prefix / path) for path in allowed}
|
|
for path in files:
|
|
relative = path.relative_to(prefix)
|
|
if relative not in allowed and not relative.is_relative_to('share/doc/docview'):
|
|
raise ValueError('Unexpected file in install payload: ' + str(relative))
|
|
target = destination / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(path, target)
|
|
target.chmod(0o755 if relative in allowed and relative.parts[0] == 'bin' else 0o644)
|
|
if initial_hashes != {str(path): sha256(prefix / path) for path in allowed} or initial_hashes != {
|
|
str(path): sha256(destination / path) for path in allowed}:
|
|
raise ValueError('Installed binaries changed while copying package input')
|
|
|
|
|
|
def describe_payload(directory, epoch):
|
|
files = []
|
|
for path in payload_files(directory):
|
|
relative = path.relative_to(directory).as_posix()
|
|
if relative == MANIFEST:
|
|
continue
|
|
files.append({'path': relative, 'size': path.stat().st_size, 'sha256': sha256(path),
|
|
'mode': '0755' if os.access(path, os.X_OK) else '0644'})
|
|
return {'schemaVersion': 1, 'name': PACKAGE_NAME, 'sourceDateEpoch': epoch,
|
|
'scope': 'Local Arch Linux development payload; not a clean Ubuntu/Windows package',
|
|
'docviewLicense': 'unspecified', 'dependencySourceFulfillment': 'not-complete',
|
|
'excludedFromOwnDigest': [MANIFEST], 'files': files}
|
|
|
|
|
|
def write_archive(directory, archive, epoch):
|
|
if not 0 <= epoch <= 0xffffffff:
|
|
raise ValueError('SOURCE_DATE_EPOCH must fit a gzip timestamp')
|
|
files = payload_files(directory)
|
|
with archive.open('xb') as raw:
|
|
with gzip.GzipFile(filename='', mode='wb', fileobj=raw, compresslevel=9, mtime=epoch) as zipped:
|
|
with tarfile.open(fileobj=zipped, mode='w', format=tarfile.PAX_FORMAT) as tar:
|
|
for path in files:
|
|
relative = path.relative_to(directory).as_posix()
|
|
info = tarfile.TarInfo(PACKAGE_NAME + '/' + relative)
|
|
info.size = path.stat().st_size
|
|
info.mode = 0o755 if os.access(path, os.X_OK) else 0o644
|
|
info.uid = info.gid = 0
|
|
info.uname = info.gname = ''
|
|
info.mtime = epoch
|
|
with path.open('rb') as source:
|
|
tar.addfile(info, source)
|
|
|
|
|
|
def verify_and_extract(archive, destination):
|
|
"""Reject links, traversal, duplicate members and hash mismatches before use."""
|
|
if destination.exists() and any(destination.iterdir()):
|
|
raise ValueError('Extraction destination must be empty')
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
seen, total = set(), 0
|
|
with tarfile.open(archive, 'r:gz') as tar:
|
|
members = tar.getmembers()
|
|
if not members or len(members) > 10000:
|
|
raise ValueError('Invalid archive member count')
|
|
for member in members:
|
|
path = PurePosixPath(member.name)
|
|
if (not member.isfile() or path.is_absolute() or '..' in path.parts or
|
|
len(path.parts) < 2 or path.parts[0] != PACKAGE_NAME or
|
|
str(path) != member.name or member.name in seen or member.mode not in (0o644, 0o755)):
|
|
raise ValueError('Unsafe or duplicate archive member: ' + member.name)
|
|
seen.add(member.name)
|
|
total += member.size
|
|
if total > MAX_PAYLOAD_BYTES:
|
|
raise ValueError('Archive exceeds unpacked size limit')
|
|
manifest_member = tar.getmember(PACKAGE_NAME + '/' + MANIFEST)
|
|
if manifest_member.size > 8 * 1024 * 1024:
|
|
raise ValueError('Oversized payload manifest')
|
|
with tar.extractfile(manifest_member) as source:
|
|
manifest = json.load(source)
|
|
if manifest.get('schemaVersion') != 1 or manifest.get('name') != PACKAGE_NAME:
|
|
raise ValueError('Invalid payload manifest')
|
|
expected = {}
|
|
for row in manifest['files']:
|
|
key = PACKAGE_NAME + '/' + row['path']
|
|
if key in expected or key not in seen or not re.fullmatch('[0-9a-f]{64}', row['sha256']):
|
|
raise ValueError('Invalid manifest file entry')
|
|
expected[key] = row
|
|
if set(expected) | {manifest_member.name} != seen:
|
|
raise ValueError('Manifest and archive entries differ')
|
|
for member in members:
|
|
data = tar.extractfile(member)
|
|
target = destination / member.name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
digest, written = hashlib.sha256(), 0
|
|
with data, target.open('xb') as output:
|
|
for chunk in iter(lambda: data.read(1024 * 1024), b''):
|
|
written += len(chunk)
|
|
digest.update(chunk)
|
|
output.write(chunk)
|
|
if member.name in expected:
|
|
row = expected[member.name]
|
|
if (row['size'] != written or row['sha256'] != digest.hexdigest() or
|
|
int(row['mode'], 8) != member.mode):
|
|
raise ValueError('Payload integrity failure: ' + member.name)
|
|
target.chmod(member.mode)
|
|
return destination / PACKAGE_NAME
|
|
|
|
|
|
def create_package(prefix, output, epoch=0, qtpaths='/usr/lib/qt6/bin/qtpaths'):
|
|
prefix, output = prefix.resolve(strict=True), output.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
archive = output / (PACKAGE_NAME + '.tar.gz')
|
|
report_path = output / (PACKAGE_NAME + '.json')
|
|
if archive.exists() or report_path.exists():
|
|
raise ValueError('Package output already exists; select a new output directory')
|
|
with tempfile.TemporaryDirectory(prefix='docview-package-') as temporary:
|
|
staging = Path(temporary) / 'payload'
|
|
staging.mkdir()
|
|
copy_install(prefix, staging)
|
|
documentation = staging / 'share/doc/docview'
|
|
for source, name in [(ROOT / 'docs/LINUX-DEVELOPMENT-PACKAGE.md', 'LINUX-DEVELOPMENT-PACKAGE.md'),
|
|
(ROOT / 'resources/licenses/README.md', 'THIRD-PARTY-SCOPE.md')]:
|
|
shutil.copyfile(source, documentation / name)
|
|
notices = documentation / 'third-party'
|
|
if notices.exists():
|
|
raise ValueError('Install already contains generated third-party inventory; use a fresh install')
|
|
dependencies = inventory(staging, notices, qtpaths)
|
|
manifest = describe_payload(staging, epoch)
|
|
(staging / MANIFEST).write_text(json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + '\n')
|
|
temporary_archive = Path(temporary) / 'package.tar.gz'
|
|
write_archive(staging, temporary_archive, epoch)
|
|
extracted = verify_and_extract(temporary_archive, Path(temporary) / 'verified')
|
|
# A second archive from the verified extraction establishes deterministic
|
|
# metadata/order/compression without relying on filesystem mtimes.
|
|
repeated = Path(temporary) / 'repeated.tar.gz'
|
|
write_archive(extracted, repeated, epoch)
|
|
if sha256(temporary_archive) != sha256(repeated):
|
|
raise ValueError('Package is not byte reproducible after extraction')
|
|
report = {'schemaVersion': 1, 'name': PACKAGE_NAME, 'archive': archive.name,
|
|
'archiveSha256': sha256(temporary_archive), 'archiveBytes': temporary_archive.stat().st_size,
|
|
'unpackedFileBytes': sum(p.stat().st_size for p in payload_files(staging)),
|
|
'payloadFileCount': len(manifest['files']) + 1, 'sourceDateEpoch': epoch,
|
|
'pythonVersion': platform_version(), 'zlibVersion': zlib.ZLIB_RUNTIME_VERSION,
|
|
'executableSha256': dependencies['executableSha256'],
|
|
'systemRuntimeCandidateBytesNotBundled': dependencies['systemRuntimeCandidateBytes'],
|
|
'systemComponentCount': len(dependencies['components']) - 1,
|
|
'archiveRoundTripSha256Matches': True, 'guiExtractionSmoke': 'not-run-by-packager',
|
|
'scope': dependencies['scope'], 'dependencySources': 'not-collected',
|
|
'completeChromiumNotices': False, 'cleanOsAcceptance': False}
|
|
shutil.copyfile(temporary_archive, archive)
|
|
report_path.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + '\n')
|
|
return report
|
|
|
|
|
|
def platform_version():
|
|
import platform
|
|
return platform.python_version()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
source = parser.add_mutually_exclusive_group(required=True)
|
|
source.add_argument('--prefix', type=Path, help='Trusted, fresh cmake install tree')
|
|
source.add_argument('--build-dir', type=Path, help='Run cmake --install into a private temporary tree; no build')
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--epoch', type=int, default=int(os.environ.get('SOURCE_DATE_EPOCH', '0')))
|
|
parser.add_argument('--qtpaths', default='/usr/lib/qt6/bin/qtpaths')
|
|
args = parser.parse_args()
|
|
if args.build_dir:
|
|
with tempfile.TemporaryDirectory(prefix='docview-install-') as directory:
|
|
subprocess.run(['cmake', '--install', str(args.build_dir.resolve()), '--prefix', directory], check=True)
|
|
result = create_package(Path(directory), args.output, args.epoch, args.qtpaths)
|
|
else:
|
|
result = create_package(args.prefix, args.output, args.epoch, args.qtpaths)
|
|
print(json.dumps(result, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|