initial commit
This commit is contained in:
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inventory a trusted DocView install and its Arch Linux system dependencies.
|
||||
|
||||
This does not download sources, infer a license choice, or certify redistribution.
|
||||
Only use ldd on executables built by this project, never on an input document.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EXECUTABLES = ('docview', 'docview-pdf-worker', 'docview-archive-worker')
|
||||
MAX_FILES = 10000
|
||||
MAX_COMPONENTS = 256
|
||||
MAX_LICENSE_BYTES = 32 * 1024 * 1024
|
||||
QT_EMBEDDED_NOTICE_METADATA_SHA256 = '8d90e93aa487eddef4b81722a53b89953cbb07b45d02488e8ea0413f300fd723'
|
||||
|
||||
|
||||
def sha256(path):
|
||||
with Path(path).open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def run(args):
|
||||
environment = {**os.environ, 'LC_ALL': 'C'}
|
||||
for key in ('LD_PRELOAD', 'LD_LIBRARY_PATH', 'LD_AUDIT'):
|
||||
environment.pop(key, None)
|
||||
result = subprocess.run(args, check=True, text=True, capture_output=True, timeout=30,
|
||||
env=environment)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def pacman_fields(text):
|
||||
result, key = {}, None
|
||||
for line in text.splitlines():
|
||||
if line.startswith('%') and line.endswith('%'):
|
||||
key = line[1:-1]
|
||||
result[key] = []
|
||||
elif line and key:
|
||||
result[key].append(line)
|
||||
return result
|
||||
|
||||
|
||||
class PackageDatabase:
|
||||
def __init__(self, root=Path('/var/lib/pacman/local')):
|
||||
self.packages, self.owners = {}, {}
|
||||
for directory in sorted(root.iterdir()):
|
||||
if not directory.is_dir() or not (directory / 'desc').is_file():
|
||||
continue
|
||||
desc = pacman_fields((directory / 'desc').read_text())
|
||||
name = desc['NAME'][0]
|
||||
self.packages[name] = desc
|
||||
for item in pacman_fields((directory / 'files').read_text()).get('FILES', []):
|
||||
if not item.endswith('/'):
|
||||
self.owners['/' + item] = name
|
||||
|
||||
def owner(self, path):
|
||||
value = self.owners.get(str(path)) or self.owners.get(str(path.resolve()))
|
||||
if not value:
|
||||
raise ValueError(f'No installed Arch package owns runtime file: {path}')
|
||||
return value
|
||||
|
||||
|
||||
def ldd_paths(output):
|
||||
paths = set()
|
||||
for line in output.splitlines():
|
||||
if 'not found' in line:
|
||||
raise ValueError('Unresolved ELF dependency: ' + line.strip())
|
||||
value = line.strip()
|
||||
if not value or value.startswith('linux-vdso'):
|
||||
continue
|
||||
match = re.fullmatch(r'(?:\S+ => )?(/.+?) \(0x[0-9a-fA-F]+\)', value)
|
||||
if not match:
|
||||
raise ValueError('Unrecognized ldd dependency: ' + value)
|
||||
paths.add(Path(match[1]))
|
||||
return paths
|
||||
|
||||
|
||||
def elf(path):
|
||||
with path.open('rb') as stream:
|
||||
return stream.read(4) == b'\x7fELF'
|
||||
|
||||
|
||||
def qt_runtime_files(query):
|
||||
"""Finite candidate modules; not a claim that every style/plugin was loaded."""
|
||||
files = {Path(query['QT_INSTALL_LIBEXECS']) / 'QtWebEngineProcess'}
|
||||
qml = Path(query['QT_INSTALL_QML'])
|
||||
for relative in ('QtQml', 'QtQuick/Controls', 'QtQuick/Dialogs', 'QtQuick/Layouts',
|
||||
'QtQuick/Templates', 'QtQuick/Window', 'QtQuick/NativeStyle',
|
||||
'QtQuick/Effects', 'QtWebEngine'):
|
||||
directory = qml / relative
|
||||
if directory.is_dir():
|
||||
files.update(p for p in directory.rglob('*') if p.is_file())
|
||||
files.update(p for p in (qml / 'QtQuick').glob('*') if p.is_file())
|
||||
plugins = Path(query['QT_INSTALL_PLUGINS'])
|
||||
for relative in ('platforms/libqxcb.so', 'platforms/libqwayland-generic.so',
|
||||
'platforms/libqwayland-egl.so'):
|
||||
path = plugins / relative
|
||||
if path.is_file():
|
||||
files.add(path)
|
||||
for filename in ('libqjpeg.so', 'libqgif.so', 'libqico.so', 'libqsvg.so', 'libqwebp.so'):
|
||||
path = plugins / 'imageformats' / filename
|
||||
if path.is_file():
|
||||
files.add(path)
|
||||
for relative in ('xcbglintegrations', 'wayland-graphics-integration-client'):
|
||||
directory = plugins / relative
|
||||
if directory.is_dir():
|
||||
files.update(p for p in directory.rglob('*') if p.is_file())
|
||||
resources = Path(query['QT_INSTALL_DATA']) / 'resources'
|
||||
files.update(p for p in resources.glob('*') if p.is_file() and
|
||||
(p.name.startswith('qtwebengine') or p.name in ('icudtl.dat', 'v8_context_snapshot.bin')))
|
||||
translations = Path(query['QT_INSTALL_TRANSLATIONS'])
|
||||
for name in ('qtbase_ja.qm', 'qtdeclarative_ja.qm'):
|
||||
path = translations / name
|
||||
if not path.is_file():
|
||||
raise ValueError('Required Japanese Qt translation is missing: ' + name)
|
||||
files.add(path)
|
||||
files.update(p for p in (translations / 'qtwebengine_locales').glob('*') if p.is_file())
|
||||
for pattern in ('qtbase_*.qm', 'qtdeclarative_*.qm', 'qtwebengine_*.qm'):
|
||||
files.update(p for p in translations.glob(pattern) if p.is_file())
|
||||
if not files or len(files) > MAX_FILES or not all(p.is_file() for p in files):
|
||||
raise ValueError('Missing or oversized Qt runtime inventory')
|
||||
return files
|
||||
|
||||
|
||||
def license_candidates(name, desc, base=Path('/usr/share/licenses')):
|
||||
result = set()
|
||||
directory = base / name
|
||||
if directory.is_dir():
|
||||
result.update(p for p in directory.rglob('*') if p.is_file())
|
||||
# Include the generic text when Arch uses a shared SPDX license directory.
|
||||
for label in desc.get('LICENSE', []):
|
||||
for token in re.findall(r'[A-Za-z0-9][A-Za-z0-9.+-]*', label):
|
||||
path = base / 'spdx' / (token + '.txt')
|
||||
if path.is_file():
|
||||
result.add(path)
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def license_supplements(name, version, base=ROOT / 'resources/licenses/linux-supplemental'):
|
||||
result = []
|
||||
upstream_version = version.rsplit('-', 1)[0].split(':')[-1]
|
||||
for row in json.loads((base / 'sources.json').read_text()):
|
||||
if row['package'] != name:
|
||||
continue
|
||||
if row['version'] != upstream_version:
|
||||
raise ValueError('Supplemental license version needs review: ' + name)
|
||||
for filename, expected in sorted(row['files'].items()):
|
||||
if Path(filename).name != filename:
|
||||
raise ValueError('Unsafe supplemental license filename')
|
||||
path = base / name / filename
|
||||
if sha256(path) != expected:
|
||||
raise ValueError('Supplemental license digest mismatch: ' + str(path))
|
||||
result.append((path, row))
|
||||
return result
|
||||
|
||||
|
||||
def qt_embedded_notices(qt_version, package_version, system_files,
|
||||
base=ROOT / 'resources/licenses/qt-embedded-notices'):
|
||||
"""Bind the reviewed partial notice set to the exact system DataPack variant."""
|
||||
source = base / 'sources.json'
|
||||
if not source.is_file() or source.stat().st_size > 65536:
|
||||
raise ValueError('Qt embedded notice metadata missing or oversized')
|
||||
metadata_bytes = source.read_bytes()
|
||||
if hashlib.sha256(metadata_bytes).hexdigest() != QT_EMBEDDED_NOTICE_METADATA_SHA256:
|
||||
raise ValueError('Qt embedded notice metadata changed; review required')
|
||||
value = json.loads(metadata_bytes)
|
||||
if (value['schemaVersion'] != 1 or value['qtVersion'] != qt_version or
|
||||
value['packageVersion'] != package_version or value['package'] != 'qt6-webengine' or
|
||||
value['completeChromiumNotices'] is not False or value['runtimeDistribution'] != 'system-not-bundled'):
|
||||
raise ValueError('Qt embedded notice target version or scope needs review')
|
||||
for expected in value['dataPacks']:
|
||||
matches = [row for row in system_files if row['path'] == expected['path']]
|
||||
if (len(matches) != 1 or any(matches[0].get(key) != expected[key] for key in ('package', 'size', 'sha256')) or
|
||||
expected['packageVersion'] != package_version):
|
||||
raise ValueError('Qt embedded notice DataPack inventory mismatch; review required')
|
||||
for row in value['files']:
|
||||
if Path(row['name']).name != row['name']:
|
||||
raise ValueError('Unsafe Qt embedded notice filename')
|
||||
path = base / row['name']
|
||||
if (not path.is_file() or path.stat().st_size != row['size'] or sha256(path) != row['sha256']):
|
||||
raise ValueError('Qt embedded notice text missing or digest/size mismatch')
|
||||
return source, value
|
||||
|
||||
|
||||
def pdfium_supplemental_notices(prefix, locked):
|
||||
relative = Path('share/doc/docview/pdfium/supplemental')
|
||||
source = prefix / relative / 'sources.json'
|
||||
if not source.is_file() or source.stat().st_size > 65536:
|
||||
raise ValueError('PDFium supplemental notice metadata missing or oversized')
|
||||
value = json.loads(source.read_text())
|
||||
if (value.get('schemaVersion') != 1 or value.get('pdfiumVersion') != locked['version'] or
|
||||
value.get('pdfiumUpstreamCommit') != locked['upstreamCommit'] or
|
||||
value.get('pdfiumLibrarySha256') != sha256(prefix / 'lib/libpdfium.so')):
|
||||
raise ValueError('PDFium supplemental notice source or binary needs review')
|
||||
rows = value.get('files')
|
||||
if (not isinstance(rows, list) or len(rows) != 2 or
|
||||
any(not isinstance(r, dict) for r in rows) or {r.get('name') for r in rows} != {
|
||||
'libcxx-LICENSE.txt', 'libcxxabi-LICENSE.txt'}):
|
||||
raise ValueError('Unexpected PDFium supplemental notice set')
|
||||
files = []
|
||||
for row in rows:
|
||||
path = prefix / relative / row['name']
|
||||
if (not path.is_file() or path.stat().st_size > 1024 * 1024 or
|
||||
path.stat().st_size != row['size'] or sha256(path) != row['sha256']):
|
||||
raise ValueError('PDFium supplemental notice digest or size mismatch')
|
||||
files.append({**row, 'path': (relative / row['name']).as_posix()})
|
||||
return {'metadataPath': (relative / 'sources.json').as_posix(), 'metadataSha256': sha256(source),
|
||||
'scope': value['scope'], 'providerRecipeCommit': value['providerRecipeCommit'],
|
||||
'pdfiumLibrarySha256': value['pdfiumLibrarySha256'], 'files': files}
|
||||
|
||||
|
||||
def inventory(prefix, output, qtpaths='/usr/lib/qt6/bin/qtpaths'):
|
||||
prefix, output = prefix.resolve(strict=True), output.resolve()
|
||||
if sys.platform != 'linux' or platform.machine() != 'x86_64':
|
||||
raise ValueError('This inventory targets Arch Linux x86_64 only')
|
||||
os_release = platform.freedesktop_os_release()
|
||||
if os_release.get('ID') != 'arch':
|
||||
raise ValueError('An Arch package database is required; this is not an Ubuntu package')
|
||||
if output.exists() and any(output.iterdir()):
|
||||
raise ValueError('Inventory output directory must be empty')
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
query = dict(line.split(':', 1) for line in run([qtpaths, '--query']).splitlines() if ':' in line)
|
||||
if query.get('QT_VERSION') != '6.11.2':
|
||||
raise ValueError('Expected the verified Qt 6.11.2 runtime')
|
||||
database = PackageDatabase()
|
||||
inputs = [prefix / 'bin' / name for name in EXECUTABLES]
|
||||
before = {str(p.relative_to(prefix)): sha256(p) for p in inputs}
|
||||
bundled = prefix / 'lib/libpdfium.so'
|
||||
expected_pdfium = ROOT / '.deps/pdfium/lib/libpdfium.so'
|
||||
if not bundled.is_file() or sha256(bundled) != sha256(expected_pdfium):
|
||||
raise ValueError('Installed PDFium differs from the pinned local archive')
|
||||
candidates = qt_runtime_files(query)
|
||||
system = {p.resolve(): {'qt-runtime-candidate'} for p in candidates}
|
||||
edges = []
|
||||
for path in inputs + sorted(candidates):
|
||||
if not elf(path):
|
||||
continue
|
||||
label = str(path.relative_to(prefix)) if path.is_relative_to(prefix) else str(path)
|
||||
resolved = []
|
||||
for dependency in sorted(ldd_paths(run(['ldd', str(path)]))):
|
||||
real = dependency.resolve(strict=True)
|
||||
if real.is_relative_to(prefix):
|
||||
if real != bundled:
|
||||
raise ValueError('Unexpected bundled shared library: ' + str(real))
|
||||
resolved.append('bundle:lib/libpdfium.so')
|
||||
else:
|
||||
system.setdefault(real, set()).add('elf-dependency')
|
||||
resolved.append(str(real))
|
||||
edges.append({'elf': label, 'resolved': sorted(resolved)})
|
||||
if len(system) > MAX_FILES:
|
||||
raise ValueError('Runtime inventory exceeds file limit')
|
||||
components, system_files = {}, []
|
||||
# toml++ is compiled into DocView as well as potentially dynamically linked.
|
||||
names = {'tomlplusplus', 'qt6-base', 'qt6-declarative', 'qt6-webengine'}
|
||||
for path, roles in sorted(system.items()):
|
||||
owner = database.owner(path)
|
||||
names.add(owner)
|
||||
system_files.append({'path': str(path), 'package': owner, 'size': path.stat().st_size,
|
||||
'sha256': sha256(path), 'roles': sorted(roles)})
|
||||
if len(names) > MAX_COMPONENTS:
|
||||
raise ValueError('Runtime inventory exceeds package limit')
|
||||
total_license_bytes = 0
|
||||
for name in sorted(names):
|
||||
desc = database.packages[name]
|
||||
version = desc['VERSION'][0]
|
||||
base = desc.get('BASE', [name])[0]
|
||||
licenses = []
|
||||
embedded_notices = None
|
||||
for source in license_candidates(name, desc):
|
||||
if source.stat().st_size > 8 * 1024 * 1024:
|
||||
raise ValueError('License file exceeds limit: ' + str(source))
|
||||
total_license_bytes += source.stat().st_size
|
||||
if total_license_bytes > MAX_LICENSE_BYTES:
|
||||
raise ValueError('License collection exceeds limit')
|
||||
relative = Path('licenses') / name / source.relative_to('/usr/share/licenses')
|
||||
target = output / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, target)
|
||||
licenses.append({'path': relative.as_posix(), 'sourcePath': str(source),
|
||||
'sha256': sha256(target), 'size': target.stat().st_size})
|
||||
for source, provenance in license_supplements(name, version):
|
||||
total_license_bytes += source.stat().st_size
|
||||
if total_license_bytes > MAX_LICENSE_BYTES:
|
||||
raise ValueError('License collection exceeds limit')
|
||||
relative = Path('licenses') / name / 'upstream' / source.name
|
||||
target = output / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, target)
|
||||
licenses.append({'path': relative.as_posix(), 'sourceUrl': provenance['url'],
|
||||
'upstreamVersion': provenance['version'],
|
||||
'downloadSha256': provenance['downloadSha256'],
|
||||
'collectionScope': 'Upstream notice text only; not a complete source collection',
|
||||
'sha256': sha256(target), 'size': target.stat().st_size})
|
||||
if name == 'qt6-webengine':
|
||||
metadata_path, reviewed = qt_embedded_notices(query['QT_VERSION'], version, system_files)
|
||||
folder = Path('licenses') / name / 'embedded-resource-notices'
|
||||
metadata_relative = folder / 'sources.json'
|
||||
(output / folder).mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(metadata_path, output / metadata_relative)
|
||||
if sha256(output / metadata_relative) != QT_EMBEDDED_NOTICE_METADATA_SHA256:
|
||||
raise ValueError('Qt embedded notice metadata changed while copying')
|
||||
total_license_bytes += (output / metadata_relative).stat().st_size
|
||||
for row in reviewed['files']:
|
||||
relative = folder / row['name']
|
||||
target = output / relative
|
||||
shutil.copyfile(metadata_path.parent / row['name'], target)
|
||||
if target.stat().st_size != row['size'] or sha256(target) != row['sha256']:
|
||||
raise ValueError('Qt embedded notice text changed while copying')
|
||||
total_license_bytes += row['size']
|
||||
licenses.append({'path': relative.as_posix(), 'size': row['size'], 'sha256': row['sha256'],
|
||||
'origin': 'reviewed-installed-DataPack-resource-comment',
|
||||
'sourceResources': row['sourceResources'],
|
||||
'collectionScope': reviewed['scope']})
|
||||
if total_license_bytes > MAX_LICENSE_BYTES:
|
||||
raise ValueError('License collection exceeds limit')
|
||||
embedded_notices = {'status': 'collected-reviewed-resource-subset',
|
||||
'metadataPath': metadata_relative.as_posix(),
|
||||
'metadataSha256': QT_EMBEDDED_NOTICE_METADATA_SHA256,
|
||||
'noticeCount': len(reviewed['files']),
|
||||
'sourceResourceCount': sum(len(row['sourceResources']) for row in reviewed['files']),
|
||||
'runtimeDistribution': reviewed['runtimeDistribution'],
|
||||
'completeChromiumNotices': False, 'scope': reviewed['scope']}
|
||||
components[name] = {
|
||||
'version': version, 'architecture': desc.get('ARCH', ['unknown'])[0],
|
||||
'distribution': 'system-not-bundled',
|
||||
'upstreamHome': desc.get('URL', [''])[0],
|
||||
'licenseLabelsFromPackage': desc.get('LICENSE', []),
|
||||
'licenseTexts': licenses,
|
||||
'noticeStatus': 'collected-package-or-upstream-texts-not-a-compliance-verdict' if licenses else 'text-not-found',
|
||||
'source': {'status': 'not-collected', 'packageBase': base, 'packageVersion': version,
|
||||
'recipeRepository': 'https://gitlab.archlinux.org/archlinux/packaging/packages/' + quote(base),
|
||||
'recipeCommit': None,
|
||||
'note': 'This package contains source pointers, not complete corresponding source. Selected exact recipes and patches are recorded separately in the repository source-correspondence evidence.'}}
|
||||
if name == 'tomlplusplus':
|
||||
components[name]['usage'] = ['system runtime library', 'headers compiled into DocView']
|
||||
if embedded_notices:
|
||||
components[name]['embeddedResourceNotices'] = embedded_notices
|
||||
locked = json.loads((ROOT / 'cmake/pdfium.lock.json').read_text())
|
||||
pdfium_supplements = pdfium_supplemental_notices(prefix, locked)
|
||||
pdfium_licenses = []
|
||||
for path in sorted((prefix / 'share/doc/docview/pdfium').rglob('*')):
|
||||
if path.is_file() and path.name != 'sources.json':
|
||||
pdfium_licenses.append({'path': path.relative_to(prefix).as_posix(), 'sha256': sha256(path),
|
||||
'size': path.stat().st_size})
|
||||
if not pdfium_licenses:
|
||||
raise ValueError('PDFium archive license texts missing from install')
|
||||
components['PDFium-independent-worker'] = {
|
||||
'version': locked['version'], 'distribution': 'bundled', 'path': 'lib/libpdfium.so',
|
||||
'sha256': sha256(bundled), 'size': bundled.stat().st_size,
|
||||
'licenseTexts': pdfium_licenses, 'upstreamHome': locked['upstream'],
|
||||
'supplementalNotices': pdfium_supplements,
|
||||
'source': {'status': 'not-collected', 'upstreamCommit': locked['upstreamCommit'],
|
||||
'sourceUrl': locked['upstream'], 'binaryProvider': locked['binaryProvider'],
|
||||
'binaryArchive': locked['linuxX64Archive'], 'archiveSha256': locked['linuxX64Sha256'],
|
||||
'buildOptions': locked['buildOptions'],
|
||||
'note': 'Upstream revision and provider archive are pinned; corresponding source tree and all dependency sources are not included.'}}
|
||||
versions = {}
|
||||
for path in inputs + [bundled]:
|
||||
values = sorted(set(re.findall(r'Name: ((?:GLIBC|GLIBCXX|CXXABI)_[A-Za-z0-9_.]+)',
|
||||
run(['readelf', '--version-info', str(path)]))))
|
||||
versions[path.relative_to(prefix).as_posix()] = values
|
||||
if before != {str(p.relative_to(prefix)): sha256(p) for p in inputs}:
|
||||
raise ValueError('Installed executables changed during inventory')
|
||||
result = {'schemaVersion': 1,
|
||||
'scope': 'Local Arch Linux x86_64 development package; not a clean-OS or redistribution acceptance',
|
||||
'host': {'id': os_release['ID'], 'prettyName': os_release.get('PRETTY_NAME', ''),
|
||||
'architecture': platform.machine(), 'qtVersion': query['QT_VERSION'],
|
||||
'glibcPackage': database.packages['glibc']['VERSION'][0]},
|
||||
'executableSha256': before, 'requiredSymbolVersions': versions,
|
||||
'components': components, 'systemFiles': system_files, 'elfResolution': edges,
|
||||
'systemRuntimeCandidateBytes': sum(row['size'] for row in system_files),
|
||||
'scopeLimits': [
|
||||
'Qt libraries, selected QML/style/plugin candidates, WebEngine helper/resources/locales and ELF closure are system dependencies, not copied runtime binaries.',
|
||||
'Candidate styles/plugins are a bounded inventory, not a trace proving every file was used. Other platform, IME, theme, GPU and font providers may load additional files.',
|
||||
'System fonts and their licenses are not bundled; FontBroker uses locally installed fonts.',
|
||||
'Arch package license labels can list alternatives. This record does not choose a license for DocView or resolve every component condition.',
|
||||
'Qt WebEngine includes Chromium third parties. Seven reviewed notice texts from 13 system DataPack resources are included, but neither these nor the top-level package license are a complete build-specific Chromium attribution collection.',
|
||||
'Complete corresponding dependency sources are not included in this package. Selected source-correspondence evidence is maintained separately; URLs and license texts alone do not establish source fulfillment.',
|
||||
'Ubuntu 24.04, Windows, clean installation and signed distribution remain unverified.']}
|
||||
(output / 'dependency-manifest.json').write_text(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + '\n')
|
||||
lines = ['DocView Linux development package: third-party notices', '',
|
||||
'Local Arch Linux x86_64 artifact. Not a complete redistribution approval.',
|
||||
'DocView public license: unspecified. This inventory does not license DocView.',
|
||||
'Only the independent PDFium binary is bundled; the other runtime libraries use the host system.',
|
||||
'License texts below are copied verbatim from installed packages, recorded versioned upstream sources, or reviewed system DataPack resource comments. Package labels may be alternatives.',
|
||||
'Seven notice texts extracted from 13 fixed QtWebEngine DataPack resources are included; the Qt runtime remains system-only and complete Chromium attribution remains uncollected.',
|
||||
'Complete corresponding dependency sources and build materials are NOT included in this package.',
|
||||
'Qt WebEngine Chromium build-specific notices remain to be completed before a release.',
|
||||
'See dependency-manifest.json for exact versions, hashes, source pointers and limitations.', '',
|
||||
'References:', 'https://doc.qt.io/qt-6/qtwebengine-licensing.html',
|
||||
'https://doc.qt.io/qt-6/linux-deployment.html', '']
|
||||
for name, item in sorted(components.items()):
|
||||
lines += [f'{name} {item["version"]} [{item["distribution"]}]',
|
||||
' Upstream: ' + item['upstreamHome'],
|
||||
' Package license labels: ' + '; '.join(item.get('licenseLabelsFromPackage', ['See bundled PDFium notices'])),
|
||||
' Source status: ' + item['source']['status']]
|
||||
lines += [' License text: ' + text['path'] for text in item['licenseTexts']]
|
||||
if not item['licenseTexts']:
|
||||
lines.append(' License text: NOT COLLECTED from the installed package')
|
||||
lines.append('')
|
||||
(output / 'NOTICE.txt').write_text('\n'.join(lines), encoding='utf-8')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--prefix', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--qtpaths', default='/usr/lib/qt6/bin/qtpaths')
|
||||
args = parser.parse_args()
|
||||
result = inventory(args.prefix, args.output, args.qtpaths)
|
||||
print(f'{len(result["components"])} components; {len(result["systemFiles"])} system runtime candidates; dependency sources not collected')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,531 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect a trusted Ubuntu install with an explicitly selected Qt SDK.
|
||||
|
||||
This writes a bounded runtime inventory and partial notice ledger, never a tar.
|
||||
Only use on DocView/SDK binaries from trusted builds: ldd executes loader code.
|
||||
No Arch notice pins or claims of complete source/license fulfillment are reused.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
from verify_pdfium_candidate import verify_installed as verify_pdfium_installed
|
||||
|
||||
MAX_FILES = 10000
|
||||
MAX_FILE_BYTES = 512 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024
|
||||
MAX_NOTICES = 2048
|
||||
MAX_NOTICE_BYTES = 32 * 1024 * 1024
|
||||
MAX_SDK_METADATA_BYTES = 64 * 1024 * 1024
|
||||
MAX_SDK_METADATA_FILE_BYTES = 16 * 1024 * 1024
|
||||
MAX_COMMAND_BYTES = 4 * 1024 * 1024
|
||||
MAX_COMPONENTS = 256
|
||||
EXECUTABLES = ('docview', 'docview-pdf-worker', 'docview-archive-worker')
|
||||
QML_MODULES = ('QtQml', 'QtQuick', 'QtQuick/Controls', 'QtQuick/Dialogs',
|
||||
'QtQuick/Layouts', 'QtQuick/Templates', 'QtQuick/Window', 'QtWebEngine')
|
||||
LICENSE_NAMES = ('LICENSE', 'LICENSE.txt', 'LICENSE.md', 'LICENSE-MIT', 'COPYING',
|
||||
'COPYING.txt', 'COPYRIGHT', 'copyright', 'NOTICE', 'NOTICE.txt', 'NOTICE.md')
|
||||
|
||||
# The Ubuntu validation dependency was built from this archived release. These
|
||||
# reviewed identities are code-owned, never taken from a caller's manifest.
|
||||
QPDF_NOTICE_PIN = {
|
||||
'version': '12.4.1',
|
||||
'archive': {'name': 'qpdf-12.4.1.tar.gz', 'size': 19713921,
|
||||
'sha256': 'f045aa277be2356ff53a89a8622945958291177d2483afc20ede7c8a8cd3873c'},
|
||||
'archiveRoot': 'qpdf-12.4.1',
|
||||
'sourceFiles': 2900, 'sourceBytes': 65617659,
|
||||
'sourceInventorySha256': 'b0c7f66f3a3ea35afb5db36716b2f7d6adfc25d54d408af088c3e3b60f321423',
|
||||
'library': {'sha256': '215d435d9636075495df489058ef3ed5ce2a00c7f39164d62cebc3ba5b4cfe9d',
|
||||
'size': 4648400},
|
||||
'notices': {
|
||||
'LICENSE.txt': {'size': 11358, 'sha256': 'cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30'},
|
||||
'NOTICE.md': {'size': 2729, 'sha256': 'b207f65a9e5491195ded63b2941199b19a4d30148871f2742c88eae7bfc513a6'},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def beneath(path, roots):
|
||||
return any(path.is_relative_to(root) for root in roots)
|
||||
|
||||
|
||||
def checked_file(path, roots, maximum=MAX_FILE_BYTES):
|
||||
path = Path(path).absolute()
|
||||
if not beneath(path, roots):
|
||||
raise ValueError('File request is outside allowed roots')
|
||||
target = path.resolve(strict=True)
|
||||
if not beneath(target, roots):
|
||||
raise ValueError('Symlink target is outside allowed roots')
|
||||
before = target.stat()
|
||||
if not stat.S_ISREG(before.st_mode) or not 0 <= before.st_size <= maximum:
|
||||
raise ValueError('File is special or exceeds size limit')
|
||||
with target.open('rb') as source:
|
||||
actual = hashlib.file_digest(source, 'sha256').hexdigest()
|
||||
after = target.stat()
|
||||
if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
|
||||
after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns):
|
||||
raise ValueError('File changed during inventory')
|
||||
return {'path': str(path), 'resolvedPath': str(target), 'size': before.st_size,
|
||||
'sha256': actual, 'symlinkResolution': str(path) != str(target)}
|
||||
|
||||
|
||||
def bounded_walk(directory, roots, limit=MAX_FILES):
|
||||
"""Follow file symlinks with identity checks; never recurse through dir links."""
|
||||
directory = Path(directory)
|
||||
if not beneath(directory.absolute(), roots) or not beneath(directory.resolve(), roots):
|
||||
raise ValueError('Directory is outside allowed roots')
|
||||
if directory.is_symlink():
|
||||
raise ValueError('Directory symlink is not traversed')
|
||||
pending, result, visited = [directory], [], 0
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
with os.scandir(current) as entries:
|
||||
for entry in entries:
|
||||
visited += 1
|
||||
if visited > limit:
|
||||
raise ValueError('Directory entry limit exceeded')
|
||||
path = Path(entry.path)
|
||||
if entry.is_symlink():
|
||||
if path.is_dir():
|
||||
raise ValueError('Directory symlink is not traversed')
|
||||
# Validate now, before any later command/file read.
|
||||
target = path.resolve(strict=True)
|
||||
if not beneath(target, roots) or not target.is_file():
|
||||
raise ValueError('Symlink target is outside allowed roots or special')
|
||||
result.append(path)
|
||||
elif entry.is_dir(follow_symlinks=False):
|
||||
pending.append(path)
|
||||
elif entry.is_file(follow_symlinks=False):
|
||||
result.append(path)
|
||||
else:
|
||||
raise ValueError('Special file in selected directory')
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def run_command(arguments, environment):
|
||||
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
|
||||
p = subprocess.run(arguments, env=environment, stdout=stdout, stderr=stderr, timeout=30)
|
||||
if stdout.tell() > MAX_COMMAND_BYTES or stderr.tell() > MAX_COMMAND_BYTES:
|
||||
raise ValueError('Command output exceeds limit')
|
||||
stdout.seek(0); stderr.seek(0)
|
||||
return p.returncode, stdout.read().decode('utf-8', 'strict'), stderr.read().decode('utf-8', 'replace')
|
||||
|
||||
|
||||
def parse_ldd(text):
|
||||
paths, missing = set(), []
|
||||
for line in text.splitlines():
|
||||
value = line.strip()
|
||||
if not value or value.startswith(('linux-vdso.', 'linux-gate.')):
|
||||
continue
|
||||
if re.fullmatch(r'\S+ => not found', value):
|
||||
missing.append(value.split()[0]); continue
|
||||
match = re.fullmatch(r'(?:\S+ => )?(/.+?) \(0x[0-9a-fA-F]+\)', value)
|
||||
if not match:
|
||||
raise ValueError('Unrecognized ldd result')
|
||||
paths.add(Path(match[1]))
|
||||
return sorted(paths), sorted(missing)
|
||||
|
||||
|
||||
def validate_query(text, sdk):
|
||||
result = {}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
raise ValueError('Invalid qtpaths output')
|
||||
key, value = line.split(':', 1)
|
||||
if key in result:
|
||||
raise ValueError('Duplicate qtpaths key')
|
||||
result[key] = value
|
||||
if result.get('QT_VERSION') != '6.11.2':
|
||||
raise ValueError('Expected Qt SDK 6.11.2')
|
||||
for key in ('QT_INSTALL_PREFIX', 'QT_INSTALL_LIBS', 'QT_INSTALL_LIBEXECS', 'QT_INSTALL_QML',
|
||||
'QT_INSTALL_PLUGINS', 'QT_INSTALL_DATA', 'QT_INSTALL_TRANSLATIONS'):
|
||||
path = Path(result.get(key, ''))
|
||||
if not path.is_absolute() or '..' in path.parts or not path.is_dir():
|
||||
raise ValueError('Missing or invalid Qt runtime directory: ' + key)
|
||||
if not path.resolve().is_relative_to(sdk):
|
||||
raise ValueError('Qt runtime directory escaped selected SDK')
|
||||
result[key] = str(path)
|
||||
if Path(result['QT_INSTALL_PREFIX']).resolve() != sdk:
|
||||
raise ValueError('qtpaths prefix differs from selected SDK')
|
||||
return result
|
||||
|
||||
|
||||
def runtime_candidates(prefix, query):
|
||||
required = {prefix / 'bin' / name for name in EXECUTABLES} | {prefix / 'lib/libpdfium.so'}
|
||||
qml, plugins = Path(query['QT_INSTALL_QML']), Path(query['QT_INSTALL_PLUGINS'])
|
||||
required.add(Path(query['QT_INSTALL_LIBEXECS']) / 'QtWebEngineProcess')
|
||||
required.add(plugins / 'platforms/libqxcb.so')
|
||||
required.update(qml / module / 'qmldir' for module in QML_MODULES)
|
||||
resource = Path(query['QT_INSTALL_DATA']) / 'resources'
|
||||
required.update(resource / name for name in ('qtwebengine_resources.pak',
|
||||
'qtwebengine_resources_100p.pak', 'qtwebengine_resources_200p.pak', 'icudtl.dat'))
|
||||
translation = Path(query['QT_INSTALL_TRANSLATIONS'])
|
||||
required.update(translation / name for name in ('qtbase_ja.qm', 'qtdeclarative_ja.qm'))
|
||||
required.update(translation / 'qtwebengine_locales' / name for name in ('en-US.pak', 'ja.pak'))
|
||||
missing = [str(path) for path in sorted(required) if not path.is_file()]
|
||||
wayland = [plugins / 'platforms' / name for name in ('libqwayland-generic.so', 'libqwayland.so')]
|
||||
if not any(p.is_file() for p in wayland):
|
||||
missing.append(str(plugins / 'platforms') + '/{libqwayland-generic.so|libqwayland.so}')
|
||||
roots = (Path(query['QT_INSTALL_PREFIX']).resolve(), prefix)
|
||||
files = {p for p in required if p.is_file()} | {p for p in wayland if p.is_file()}
|
||||
for relative in ('QtQml', 'QtQuick', 'QtWebEngine'):
|
||||
path = qml / relative
|
||||
if path.is_dir(): files.update(bounded_walk(path, roots))
|
||||
for relative in ('platforms', 'imageformats', 'xcbglintegrations', 'wayland-graphics-integration-client',
|
||||
'platforminputcontexts', 'platformthemes', 'wayland-shell-integration',
|
||||
'wayland-decoration-client', 'iconengines', 'tls', 'networkinformation'):
|
||||
path = plugins / relative
|
||||
if path.is_dir(): files.update(bounded_walk(path, roots))
|
||||
for directory in (resource, translation / 'qtwebengine_locales'):
|
||||
if directory.is_dir(): files.update(bounded_walk(directory, roots))
|
||||
return sorted(files), missing
|
||||
|
||||
|
||||
def manifest_record(path):
|
||||
path = Path(path)
|
||||
if not path.is_file() or path.stat().st_size > 1024 * 1024:
|
||||
raise ValueError('Input manifest missing or oversized')
|
||||
raw = path.read_bytes(); items = json.loads(raw)
|
||||
if not isinstance(items, list) or len(items) > 256:
|
||||
raise ValueError('Expected bounded SDK/source input manifest list')
|
||||
result = []
|
||||
for row in items:
|
||||
if not isinstance(row, dict): raise ValueError('Invalid manifest row')
|
||||
name, sha, size, url = (row.get(k) for k in ('name', 'sha256', 'size', 'url'))
|
||||
if (not isinstance(name, str) or Path(name).name != name or len(name) > 256 or
|
||||
not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or type(size) is not int or size < 0 or
|
||||
not isinstance(url, str) or not url.startswith('https://') or len(url) > 4096):
|
||||
raise ValueError('Invalid input manifest identity')
|
||||
result.append({'name': name, 'sha256': sha, 'size': size, 'url': url,
|
||||
'kind': row.get('kind') if row.get('kind') in ('qt', 'source') else 'unspecified'})
|
||||
return {'manifestSha256': digest(raw), 'inputs': result,
|
||||
'scope': 'Provided download identities; archives and extracted tree correspondence not reverified here'}
|
||||
|
||||
|
||||
class Collector:
|
||||
def __init__(self, prefix, qtpaths, dependency_prefix, output, sources=(), runner=run_command,
|
||||
system_roots=None, system_doc=Path('/usr/share/doc'), qpdf_source_archive=None):
|
||||
self.prefix = Path(prefix).resolve(strict=True)
|
||||
self.qtpaths = Path(qtpaths).absolute()
|
||||
self.sdk = self.qtpaths.parent.parent.resolve(strict=True)
|
||||
self.deps = Path(dependency_prefix).resolve(strict=True)
|
||||
if self.qtpaths.parent.name != 'bin' or self.qtpaths.name not in ('qtpaths', 'qtpaths6'):
|
||||
raise ValueError('Select SDK/bin/qtpaths explicitly')
|
||||
self.output, self.sources, self.runner = Path(output), [Path(p).resolve(strict=True) for p in sources], runner
|
||||
if len(self.sources) > 8:
|
||||
raise ValueError('At most eight selected source roots are allowed')
|
||||
self.system_roots = tuple(Path(p).absolute() for p in (system_roots or ('/lib', '/lib64', '/usr/lib', '/usr/lib64')))
|
||||
self.allowed = (self.prefix, self.sdk, self.deps) + self.system_roots
|
||||
for root in (self.prefix, self.sdk, self.deps, *self.sources):
|
||||
if not root.is_dir() or root in (Path('/'), Path('/usr'), Path('/opt'), Path('/home'), Path('/tmp'), Path.home()):
|
||||
raise ValueError('A specific installed/source root is required')
|
||||
if any(self.output.resolve().is_relative_to(p) for p in (self.prefix, self.sdk, self.deps, *self.sources)):
|
||||
raise ValueError('Output must be outside inspected roots')
|
||||
self.system_doc = Path(system_doc)
|
||||
self.environment = {**os.environ, 'LC_ALL': 'C',
|
||||
'LD_LIBRARY_PATH': str(self.deps / 'lib') + ':' + str(self.sdk / 'lib')}
|
||||
for key in ('LD_PRELOAD', 'LD_AUDIT'):
|
||||
self.environment.pop(key, None)
|
||||
self.rows, self.notices, self.missing_notices, self.total, self.notice_total = {}, [], [], 0, 0
|
||||
self.sdk_metadata_total = 0
|
||||
self.qpdf_source_archive = Path(qpdf_source_archive).absolute() if qpdf_source_archive else None
|
||||
|
||||
def qpdf_correspondence(self, input_manifest):
|
||||
# Detect the dependency by its actual path, regardless of its digest;
|
||||
# a modified library must not evade verification by becoming unknown.
|
||||
libraries = {}
|
||||
for row in self.rows.values():
|
||||
paths = (Path(row['path']), Path(row['resolvedPath']))
|
||||
if any(re.fullmatch(r'libqpdf\.so(?:\.[0-9]+)*', p.name) for p in paths):
|
||||
libraries[row['resolvedPath']] = row
|
||||
if not libraries:
|
||||
if self.qpdf_source_archive:
|
||||
raise ValueError('qpdf source archive supplied without a qpdf runtime dependency')
|
||||
return {'status': 'not-applicable'}
|
||||
if len(libraries) != 1 or self.qpdf_source_archive is None:
|
||||
raise ValueError('Exactly one qpdf runtime and its fixed source archive are required')
|
||||
pin = QPDF_NOTICE_PIN
|
||||
library = next(iter(libraries.values()))
|
||||
real = Path(library['resolvedPath'])
|
||||
if not real.is_relative_to(self.deps / 'lib'):
|
||||
raise ValueError('qpdf runtime must belong to the selected dependency prefix')
|
||||
current = checked_file(real, (self.deps,))
|
||||
if any(current[k] != pin['library'][k] or library[k] != current[k] for k in ('sha256', 'size')):
|
||||
raise ValueError('qpdf runtime differs from the fixed notice correspondence')
|
||||
archive = self.qpdf_source_archive
|
||||
if archive.is_symlink():
|
||||
raise ValueError('qpdf source archive may not be a symlink')
|
||||
identity = checked_file(archive, (archive.parent,), 32 * 1024 * 1024)
|
||||
if archive.name != pin['archive']['name'] or any(identity[k] != pin['archive'][k] for k in ('sha256', 'size')):
|
||||
raise ValueError('qpdf source archive differs from the fixed release')
|
||||
if input_manifest.get('inputs') is not None:
|
||||
rows = [r for r in input_manifest['inputs'] if r['name'] == pin['archive']['name']]
|
||||
if len(rows) != 1 or rows[0]['kind'] != 'source' or any(rows[0][k] != pin['archive'][k] for k in ('sha256', 'size')):
|
||||
raise ValueError('qpdf declared input manifest differs from the fixed release')
|
||||
raw = archive.read_bytes()
|
||||
if digest(raw) != identity['sha256']:
|
||||
raise ValueError('qpdf source archive changed while reading')
|
||||
inventory, seen, total = [], set(), 0
|
||||
with tarfile.open(fileobj=io.BytesIO(raw), mode='r:gz') as source:
|
||||
for member in source:
|
||||
name = member.name.rstrip('/')
|
||||
parts = name.split('/')
|
||||
if (name in seen or len(seen) >= 10000 or parts[0] != pin['archiveRoot'] or
|
||||
any(p in ('', '.', '..') for p in parts) or '\\' in name):
|
||||
raise ValueError('Invalid qpdf archive path or entry count')
|
||||
seen.add(name)
|
||||
if member.isdir():
|
||||
continue
|
||||
if not member.isfile() or len(parts) < 2 or not 0 <= member.size <= 32 * 1024 * 1024:
|
||||
raise ValueError('Invalid qpdf archive member')
|
||||
total += member.size
|
||||
if total > 128 * 1024 * 1024:
|
||||
raise ValueError('qpdf expanded source exceeds limit')
|
||||
data = source.extractfile(member).read()
|
||||
inventory.append({'path': '/'.join(parts[1:]), 'size': len(data), 'sha256': digest(data)})
|
||||
inventory.sort(key=lambda row: row['path'])
|
||||
inventory_sha = digest(json.dumps(inventory, sort_keys=True, separators=(',', ':')).encode())
|
||||
if (len(inventory) != pin['sourceFiles'] or total != pin['sourceBytes'] or
|
||||
inventory_sha != pin['sourceInventorySha256']):
|
||||
raise ValueError('qpdf source inventory differs from the fixed release')
|
||||
candidates = [p for p in self.sources if all((p / name).is_file() for name in pin['notices'])]
|
||||
candidates = [p for p in candidates if all(
|
||||
checked_file(p / name, (p,), 64 * 1024)['sha256'] == expected['sha256']
|
||||
for name, expected in pin['notices'].items())]
|
||||
if len(candidates) != 1:
|
||||
raise ValueError('Exactly one matching qpdf source root with LICENSE and NOTICE is required')
|
||||
root = candidates[0]
|
||||
actual = bounded_walk(root, (root,))
|
||||
if any(p.is_symlink() for p in actual) or {p.relative_to(root).as_posix() for p in actual} != {r['path'] for r in inventory}:
|
||||
raise ValueError('qpdf selected source root differs from archive entries')
|
||||
for entry in inventory:
|
||||
row = checked_file(root / entry['path'], (root,), 32 * 1024 * 1024)
|
||||
if any(row[k] != entry[k] for k in ('sha256', 'size')):
|
||||
raise ValueError('qpdf selected source file differs from archive: ' + entry['path'])
|
||||
return {'status': 'verified', 'version': pin['version'], 'archive': dict(pin['archive']),
|
||||
'sourceRoot': 'source-' + str(self.sources.index(root)) + ':', 'sourceFiles': len(inventory), 'sourceBytes': total,
|
||||
'sourceInventorySha256': inventory_sha,
|
||||
'library': {'path': self.label(Path(library['path'])), 'sha256': current['sha256'], 'size': current['size']},
|
||||
'notices': [{'source': name, **expected, 'copiedPath': 'notices/' + expected['sha256'] + '.txt'}
|
||||
for name, expected in pin['notices'].items()]}
|
||||
|
||||
def label(self, path):
|
||||
path = Path(path)
|
||||
for name, root in [('install', self.prefix), ('qt-sdk', self.sdk), ('dependencies', self.deps),
|
||||
*[(f'source-{i}', p) for i, p in enumerate(self.sources)]]:
|
||||
if path.is_relative_to(root): return name + ':' + path.relative_to(root).as_posix()
|
||||
home = str(Path.home())
|
||||
return str(path).replace(home + '/', '$HOME/', 1) if str(path).startswith(home + '/') else str(path)
|
||||
|
||||
def command(self, args):
|
||||
return self.runner([str(a) for a in args], self.environment)
|
||||
|
||||
def file(self, path, role):
|
||||
key = str(Path(path).absolute())
|
||||
if key not in self.rows:
|
||||
row = checked_file(Path(key), self.allowed)
|
||||
self.total += row['size']
|
||||
if len(self.rows) >= MAX_FILES or self.total > MAX_TOTAL_BYTES:
|
||||
raise ValueError('Runtime inventory limit exceeded')
|
||||
row['roles'] = []; self.rows[key] = row
|
||||
row = self.rows[key]
|
||||
if role not in row['roles']: row['roles'].append(role)
|
||||
return row
|
||||
|
||||
def notice(self, path, roots, origin, category='notice'):
|
||||
sdk_metadata = category == 'sdk-sbom-metadata-not-license-text'
|
||||
row = checked_file(path, roots, MAX_SDK_METADATA_FILE_BYTES if sdk_metadata else 8 * 1024 * 1024)
|
||||
if sdk_metadata:
|
||||
self.sdk_metadata_total += row['size']
|
||||
else:
|
||||
self.notice_total += row['size']
|
||||
if (len(self.notices) >= MAX_NOTICES or self.notice_total > MAX_NOTICE_BYTES or
|
||||
self.sdk_metadata_total > MAX_SDK_METADATA_BYTES):
|
||||
raise ValueError('Notice collection limit exceeded')
|
||||
relative = Path('notices' if category == 'notice' else 'sdk-metadata') / (row['sha256'] + '.txt')
|
||||
target = self.output / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
raw = Path(row['resolvedPath']).read_bytes()
|
||||
if digest(raw) != row['sha256']: raise ValueError('Notice changed while copying')
|
||||
target.write_bytes(raw)
|
||||
row.update({'path': self.label(path), 'resolvedPath': self.label(Path(row['resolvedPath'])),
|
||||
'copiedPath': str(relative), 'origin': origin, 'category': category})
|
||||
self.notices.append(row)
|
||||
|
||||
def source_notices(self, directory, origin, sdk=False):
|
||||
found = set()
|
||||
for name in LICENSE_NAMES:
|
||||
p = directory / name
|
||||
if p.exists(): found.add(p)
|
||||
for name in ('LICENSES', 'licenses', 'Licenses'):
|
||||
p = directory / name
|
||||
if p.is_dir(): found.update(bounded_walk(p, (directory,), MAX_NOTICES))
|
||||
if not found: self.missing_notices.append({'origin': origin, 'status': 'no-text-found-in-selected-locations'})
|
||||
for path in sorted(found): self.notice(path, (directory,), origin)
|
||||
sbom = directory / 'sbom'
|
||||
if sdk and sbom.is_dir():
|
||||
for path in bounded_walk(sbom, (directory,), MAX_NOTICES):
|
||||
self.notice(path, (directory,), origin, 'sdk-sbom-metadata-not-license-text')
|
||||
|
||||
def collect(self, os_release, input_manifest=None):
|
||||
if os_release.get('ID') != 'ubuntu' or os_release.get('VERSION_ID') != '24.04':
|
||||
raise ValueError('This collector requires an Ubuntu 24.04 guest')
|
||||
# A candidate must carry its reviewed build/source/notice correspondence;
|
||||
# removing that metadata cannot fall back to an arbitrary provider binary.
|
||||
pdfium_correspondence = verify_pdfium_installed(self.prefix)
|
||||
input_record = manifest_record(input_manifest) if input_manifest else {'status': 'not-provided'}
|
||||
self.output.mkdir(parents=True, exist_ok=False)
|
||||
checked_file(self.qtpaths, (self.sdk,))
|
||||
if not os.access(self.qtpaths, os.X_OK):
|
||||
raise ValueError('Selected qtpaths is not executable')
|
||||
code, text, _ = self.command([self.qtpaths, '--query'])
|
||||
if code: raise ValueError('qtpaths query failed')
|
||||
query = validate_query(text, self.sdk)
|
||||
candidates, missing = runtime_candidates(self.prefix, query)
|
||||
edges, runtime_failures = [], list(missing)
|
||||
for path in candidates: self.file(path, 'required-or-selected-runtime')
|
||||
for path in ([self.prefix / 'bin' / name for name in EXECUTABLES] +
|
||||
[self.prefix / 'lib/libpdfium.so', Path(query['QT_INSTALL_LIBEXECS']) / 'QtWebEngineProcess']):
|
||||
if path.is_file():
|
||||
with path.open('rb') as f:
|
||||
if f.read(4) != b'\x7fELF': runtime_failures.append(self.label(path) + ': required ELF header missing')
|
||||
if path.name != 'libpdfium.so' and not os.access(path, os.X_OK):
|
||||
runtime_failures.append(self.label(path) + ': executable permission missing')
|
||||
# Every selected ELF (including QML/plugins/helper) receives ldd; ldd already
|
||||
# reports its transitive ELF dependencies, which are additionally hashed.
|
||||
for path in candidates:
|
||||
with path.open('rb') as f: is_elf = f.read(4) == b'\x7fELF'
|
||||
if not is_elf: continue
|
||||
code, text, _ = self.command(['ldd', path])
|
||||
dependencies, unresolved = parse_ldd(text)
|
||||
if code: runtime_failures.append(self.label(path) + ': ldd returned ' + str(code))
|
||||
runtime_failures.extend(self.label(path) + ': missing ' + item for item in unresolved)
|
||||
for dependency in dependencies: self.file(dependency, 'elf-dependency')
|
||||
edges.append({'elf': self.label(path), 'dependencies': [self.label(p) for p in dependencies],
|
||||
'unresolved': unresolved, 'exitCode': code})
|
||||
rpaths = []
|
||||
for path in [self.prefix / 'bin' / name for name in EXECUTABLES]:
|
||||
if not path.is_file(): continue
|
||||
code, text, _ = self.command(['readelf', '-d', path])
|
||||
if code: runtime_failures.append(self.label(path) + ': readelf failed')
|
||||
rpaths.append({'elf': self.label(path), 'exitCode': code,
|
||||
'dynamicPathEntries': [line.strip() for line in text.splitlines() if re.search(r'\((?:RUNPATH|RPATH)\)', line)]})
|
||||
qpdf_correspondence = self.qpdf_correspondence(input_record)
|
||||
packages, unowned = {}, []
|
||||
for key, row in sorted(self.rows.items()):
|
||||
real = Path(row['resolvedPath'])
|
||||
if not beneath(real, self.system_roots): continue
|
||||
owners = set()
|
||||
paths = {key, str(real)}
|
||||
# dpkg can retain the pre-usrmerge pathname while ldd resolves the
|
||||
# canonical /usr/lib object. Check only these equivalent aliases.
|
||||
for value in list(paths):
|
||||
for short, long in (('/lib/', '/usr/lib/'), ('/lib64/', '/usr/lib64/')):
|
||||
if value.startswith(long): paths.add(short + value[len(long):])
|
||||
if value.startswith(short): paths.add(long + value[len(short):])
|
||||
for candidate in sorted(paths):
|
||||
if not Path(candidate).exists() or Path(candidate).resolve() != real: continue
|
||||
code, text, _ = self.command(['dpkg-query', '--search', candidate])
|
||||
for line in text.splitlines() if code == 0 else []:
|
||||
if ': ' not in line: continue
|
||||
owner, filename = line.rsplit(': ', 1)
|
||||
if filename != candidate: continue
|
||||
for name in owner.split(', '):
|
||||
if re.fullmatch(r'[a-z0-9][a-z0-9+.-]*(?::[a-z0-9][a-z0-9-]*)?', name): owners.add(name)
|
||||
if owners: break
|
||||
row['systemOwners'] = sorted(owners)
|
||||
if not owners: unowned.append(self.label(real))
|
||||
for owner in owners:
|
||||
if owner in packages: continue
|
||||
if len(packages) >= MAX_COMPONENTS: raise ValueError('System package limit exceeded')
|
||||
code, text, _ = self.command(['dpkg-query', '--show', '--showformat=${binary:Package}\t${Version}\t${Architecture}\n', owner])
|
||||
fields = text.strip().split('\t')
|
||||
if code or len(fields) != 3 or fields[0] != owner: raise ValueError('Invalid dpkg package identity')
|
||||
packages[owner] = {'version': fields[1], 'architecture': fields[2]}
|
||||
copyright_path = self.system_doc / owner.split(':')[0] / 'copyright'
|
||||
if copyright_path.is_file(): self.notice(copyright_path, (self.system_doc,), 'dpkg:' + owner)
|
||||
else: self.missing_notices.append({'origin': 'dpkg:' + owner, 'status': 'copyright-file-missing'})
|
||||
self.source_notices(self.sdk, 'selected-Qt-SDK', sdk=True)
|
||||
for name in ('qpdf', 'libzip'):
|
||||
path = self.deps / 'share/doc' / name
|
||||
if path.is_dir(): self.source_notices(path, 'dependency-prefix:' + name)
|
||||
else: self.missing_notices.append({'origin': 'dependency-prefix:' + name, 'status': 'notice-directory-missing'})
|
||||
for i, source in enumerate(self.sources): self.source_notices(source, 'selected-source-' + str(i))
|
||||
if qpdf_correspondence['status'] == 'verified':
|
||||
for expected in qpdf_correspondence['notices']:
|
||||
source = qpdf_correspondence['sourceRoot'] + expected['source']
|
||||
matching = [n for n in self.notices if n['path'] == source]
|
||||
if len(matching) != 1 or any(matching[0][k] != expected[k] for k in ('sha256', 'size', 'copiedPath')):
|
||||
raise ValueError('qpdf notice changed after source verification')
|
||||
pdfium = self.prefix / 'share/doc/docview/pdfium'
|
||||
if pdfium.is_dir():
|
||||
for path in bounded_walk(pdfium, (self.prefix,), MAX_NOTICES):
|
||||
category = 'source-metadata' if path.name == 'sources.json' or 'candidate' in path.relative_to(pdfium).parts else 'notice'
|
||||
self.notice(path, (self.prefix,), 'installed-PDFium', category)
|
||||
else: self.missing_notices.append({'origin': 'installed-PDFium', 'status': 'notice-directory-missing'})
|
||||
rows = []
|
||||
for row in self.rows.values():
|
||||
rows.append({**row, 'path': self.label(Path(row['path'])),
|
||||
'resolvedPath': self.label(Path(row['resolvedPath'])), 'roles': sorted(row['roles'])})
|
||||
return {'schemaVersion': 1, 'scope': 'Ubuntu guest installed-runtime validation and partial notices only; no distribution package',
|
||||
'runtimeValidationSuccess': not runtime_failures, 'runtimeFailures': runtime_failures,
|
||||
'osRelease': {k: os_release[k] for k in ('ID', 'VERSION_ID', 'PRETTY_NAME') if k in os_release},
|
||||
'architecture': platform.machine(), 'qtVersion': query['QT_VERSION'],
|
||||
'qtPaths': {k: self.label(Path(v)) for k, v in query.items() if k.startswith('QT_INSTALL_') and k in (
|
||||
'QT_INSTALL_PREFIX','QT_INSTALL_LIBS','QT_INSTALL_LIBEXECS','QT_INSTALL_QML','QT_INSTALL_PLUGINS','QT_INSTALL_DATA','QT_INSTALL_TRANSLATIONS')},
|
||||
'loaderEnvironment': {'LD_LIBRARY_PATHEntries': ['dependencies:lib', 'qt-sdk:lib'], 'LD_PRELOAD': 'removed', 'LD_AUDIT': 'removed'},
|
||||
'files': sorted(rows, key=lambda r:r['path']), 'elfResolution': edges, 'installedRpaths': rpaths,
|
||||
'pdfiumCorrespondence': pdfium_correspondence,
|
||||
'qpdfNoticeCorrespondence': qpdf_correspondence,
|
||||
'systemPackages': packages, 'systemOwnershipUnresolved': sorted(set(unowned)),
|
||||
'notices': self.notices, 'noticeGaps': self.missing_notices,
|
||||
'inputManifest': input_record,
|
||||
'completeChromiumNotices': False, 'completeCorrespondingSources': False, 'legalComplianceVerdict': 'not-assessed',
|
||||
'archNoticePinReused': False, 'runtimeBinariesCopied': False,
|
||||
'limitations': ['Successful ldd is not GUI rendering or sandbox execution evidence.',
|
||||
'Selected Qt plugins/QML are candidates; this is not a trace of every runtime-loaded file.',
|
||||
'Explicit /opt SDK and dependency loader environment is required for this validation.',
|
||||
'No whole source-tree, package signature, complete license, or reproducible-build verification.',
|
||||
'Missing notices are reported separately; runtime success does not mean notice completeness.']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--prefix', required=True, type=Path)
|
||||
parser.add_argument('--qtpaths', required=True, type=Path)
|
||||
parser.add_argument('--dependency-prefix', required=True, type=Path)
|
||||
parser.add_argument('--output', required=True, type=Path, help='New output directory; never overwrite earlier evidence')
|
||||
parser.add_argument('--input-manifest', type=Path)
|
||||
parser.add_argument('--source-root', action='append', default=[], type=Path)
|
||||
parser.add_argument('--qpdf-source-archive', type=Path, help='Fixed qpdf release archive required when libqpdf is included')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
collector = Collector(args.prefix, args.qtpaths, args.dependency_prefix, args.output, args.source_root,
|
||||
qpdf_source_archive=args.qpdf_source_archive)
|
||||
result = collector.collect(platform.freedesktop_os_release(), args.input_manifest)
|
||||
(args.output / 'runtime.json').write_text(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + '\n')
|
||||
print(json.dumps({'runtimeValidationSuccess': result['runtimeValidationSuccess'], 'files': len(result['files']),
|
||||
'systemPackages': len(result['systemPackages']), 'notices': len(result['notices']),
|
||||
'noticeGaps': len(result['noticeGaps'])}))
|
||||
return 0 if result['runtimeValidationSuccess'] else 1
|
||||
except (ValueError, OSError, tarfile.TarError, subprocess.TimeoutExpired) as error:
|
||||
# Never preserve an earlier success: --output is exclusive and exceptions
|
||||
# produce no runtime.json. Partial copied notices alone prove no success.
|
||||
print('Validation failed: ' + str(error))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a deterministic, local Arch development tar.gz from a trusted install.
|
||||
|
||||
No system runtime binaries or user configuration/history are copied. This is not
|
||||
a self-contained Linux release. The same install, host inventory, packaging code,
|
||||
Python/zlib and SOURCE_DATE_EPOCH produce the same archive bytes.
|
||||
"""
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zlib
|
||||
|
||||
from collect_linux_dependencies import EXECUTABLES, ROOT, inventory, sha256
|
||||
|
||||
PACKAGE_NAME = 'docview-0.1.0-arch-x86_64-development'
|
||||
MAX_PAYLOAD_BYTES = 512 * 1024 * 1024
|
||||
MANIFEST = 'share/doc/docview/package-manifest.json'
|
||||
|
||||
|
||||
def payload_files(directory):
|
||||
result, size = [], 0
|
||||
for path in sorted(directory.rglob('*')):
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not (stat.S_ISREG(info.st_mode) or stat.S_ISDIR(info.st_mode)):
|
||||
raise ValueError('Package contains a link or special file: ' + str(path))
|
||||
if stat.S_ISDIR(info.st_mode):
|
||||
continue
|
||||
if info.st_nlink != 1:
|
||||
raise ValueError('Package contains a multiply-linked file: ' + str(path))
|
||||
size += info.st_size
|
||||
if size > MAX_PAYLOAD_BYTES or len(result) >= 10000:
|
||||
raise ValueError('Package payload exceeds its finite limit')
|
||||
result.append(path)
|
||||
return result
|
||||
|
||||
|
||||
def copy_install(prefix, destination):
|
||||
allowed = {Path('bin') / name for name in EXECUTABLES} | {Path('lib/libpdfium.so')}
|
||||
files = payload_files(prefix)
|
||||
present = {path.relative_to(prefix) for path in files}
|
||||
if not allowed <= present:
|
||||
raise ValueError('Install is missing an application executable or PDFium')
|
||||
initial_hashes = {str(path): sha256(prefix / path) for path in allowed}
|
||||
for path in files:
|
||||
relative = path.relative_to(prefix)
|
||||
if relative not in allowed and not relative.is_relative_to('share/doc/docview'):
|
||||
raise ValueError('Unexpected file in install payload: ' + str(relative))
|
||||
target = destination / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(path, target)
|
||||
target.chmod(0o755 if relative in allowed and relative.parts[0] == 'bin' else 0o644)
|
||||
if initial_hashes != {str(path): sha256(prefix / path) for path in allowed} or initial_hashes != {
|
||||
str(path): sha256(destination / path) for path in allowed}:
|
||||
raise ValueError('Installed binaries changed while copying package input')
|
||||
|
||||
|
||||
def describe_payload(directory, epoch):
|
||||
files = []
|
||||
for path in payload_files(directory):
|
||||
relative = path.relative_to(directory).as_posix()
|
||||
if relative == MANIFEST:
|
||||
continue
|
||||
files.append({'path': relative, 'size': path.stat().st_size, 'sha256': sha256(path),
|
||||
'mode': '0755' if os.access(path, os.X_OK) else '0644'})
|
||||
return {'schemaVersion': 1, 'name': PACKAGE_NAME, 'sourceDateEpoch': epoch,
|
||||
'scope': 'Local Arch Linux development payload; not a clean Ubuntu/Windows package',
|
||||
'docviewLicense': 'unspecified', 'dependencySourceFulfillment': 'not-complete',
|
||||
'excludedFromOwnDigest': [MANIFEST], 'files': files}
|
||||
|
||||
|
||||
def write_archive(directory, archive, epoch):
|
||||
if not 0 <= epoch <= 0xffffffff:
|
||||
raise ValueError('SOURCE_DATE_EPOCH must fit a gzip timestamp')
|
||||
files = payload_files(directory)
|
||||
with archive.open('xb') as raw:
|
||||
with gzip.GzipFile(filename='', mode='wb', fileobj=raw, compresslevel=9, mtime=epoch) as zipped:
|
||||
with tarfile.open(fileobj=zipped, mode='w', format=tarfile.PAX_FORMAT) as tar:
|
||||
for path in files:
|
||||
relative = path.relative_to(directory).as_posix()
|
||||
info = tarfile.TarInfo(PACKAGE_NAME + '/' + relative)
|
||||
info.size = path.stat().st_size
|
||||
info.mode = 0o755 if os.access(path, os.X_OK) else 0o644
|
||||
info.uid = info.gid = 0
|
||||
info.uname = info.gname = ''
|
||||
info.mtime = epoch
|
||||
with path.open('rb') as source:
|
||||
tar.addfile(info, source)
|
||||
|
||||
|
||||
def verify_and_extract(archive, destination):
|
||||
"""Reject links, traversal, duplicate members and hash mismatches before use."""
|
||||
if destination.exists() and any(destination.iterdir()):
|
||||
raise ValueError('Extraction destination must be empty')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
seen, total = set(), 0
|
||||
with tarfile.open(archive, 'r:gz') as tar:
|
||||
members = tar.getmembers()
|
||||
if not members or len(members) > 10000:
|
||||
raise ValueError('Invalid archive member count')
|
||||
for member in members:
|
||||
path = PurePosixPath(member.name)
|
||||
if (not member.isfile() or path.is_absolute() or '..' in path.parts or
|
||||
len(path.parts) < 2 or path.parts[0] != PACKAGE_NAME or
|
||||
str(path) != member.name or member.name in seen or member.mode not in (0o644, 0o755)):
|
||||
raise ValueError('Unsafe or duplicate archive member: ' + member.name)
|
||||
seen.add(member.name)
|
||||
total += member.size
|
||||
if total > MAX_PAYLOAD_BYTES:
|
||||
raise ValueError('Archive exceeds unpacked size limit')
|
||||
manifest_member = tar.getmember(PACKAGE_NAME + '/' + MANIFEST)
|
||||
if manifest_member.size > 8 * 1024 * 1024:
|
||||
raise ValueError('Oversized payload manifest')
|
||||
with tar.extractfile(manifest_member) as source:
|
||||
manifest = json.load(source)
|
||||
if manifest.get('schemaVersion') != 1 or manifest.get('name') != PACKAGE_NAME:
|
||||
raise ValueError('Invalid payload manifest')
|
||||
expected = {}
|
||||
for row in manifest['files']:
|
||||
key = PACKAGE_NAME + '/' + row['path']
|
||||
if key in expected or key not in seen or not re.fullmatch('[0-9a-f]{64}', row['sha256']):
|
||||
raise ValueError('Invalid manifest file entry')
|
||||
expected[key] = row
|
||||
if set(expected) | {manifest_member.name} != seen:
|
||||
raise ValueError('Manifest and archive entries differ')
|
||||
for member in members:
|
||||
data = tar.extractfile(member)
|
||||
target = destination / member.name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
digest, written = hashlib.sha256(), 0
|
||||
with data, target.open('xb') as output:
|
||||
for chunk in iter(lambda: data.read(1024 * 1024), b''):
|
||||
written += len(chunk)
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
if member.name in expected:
|
||||
row = expected[member.name]
|
||||
if (row['size'] != written or row['sha256'] != digest.hexdigest() or
|
||||
int(row['mode'], 8) != member.mode):
|
||||
raise ValueError('Payload integrity failure: ' + member.name)
|
||||
target.chmod(member.mode)
|
||||
return destination / PACKAGE_NAME
|
||||
|
||||
|
||||
def create_package(prefix, output, epoch=0, qtpaths='/usr/lib/qt6/bin/qtpaths'):
|
||||
prefix, output = prefix.resolve(strict=True), output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
archive = output / (PACKAGE_NAME + '.tar.gz')
|
||||
report_path = output / (PACKAGE_NAME + '.json')
|
||||
if archive.exists() or report_path.exists():
|
||||
raise ValueError('Package output already exists; select a new output directory')
|
||||
with tempfile.TemporaryDirectory(prefix='docview-package-') as temporary:
|
||||
staging = Path(temporary) / 'payload'
|
||||
staging.mkdir()
|
||||
copy_install(prefix, staging)
|
||||
documentation = staging / 'share/doc/docview'
|
||||
for source, name in [(ROOT / 'docs/LINUX-DEVELOPMENT-PACKAGE.md', 'LINUX-DEVELOPMENT-PACKAGE.md'),
|
||||
(ROOT / 'resources/licenses/README.md', 'THIRD-PARTY-SCOPE.md')]:
|
||||
shutil.copyfile(source, documentation / name)
|
||||
notices = documentation / 'third-party'
|
||||
if notices.exists():
|
||||
raise ValueError('Install already contains generated third-party inventory; use a fresh install')
|
||||
dependencies = inventory(staging, notices, qtpaths)
|
||||
manifest = describe_payload(staging, epoch)
|
||||
(staging / MANIFEST).write_text(json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + '\n')
|
||||
temporary_archive = Path(temporary) / 'package.tar.gz'
|
||||
write_archive(staging, temporary_archive, epoch)
|
||||
extracted = verify_and_extract(temporary_archive, Path(temporary) / 'verified')
|
||||
# A second archive from the verified extraction establishes deterministic
|
||||
# metadata/order/compression without relying on filesystem mtimes.
|
||||
repeated = Path(temporary) / 'repeated.tar.gz'
|
||||
write_archive(extracted, repeated, epoch)
|
||||
if sha256(temporary_archive) != sha256(repeated):
|
||||
raise ValueError('Package is not byte reproducible after extraction')
|
||||
report = {'schemaVersion': 1, 'name': PACKAGE_NAME, 'archive': archive.name,
|
||||
'archiveSha256': sha256(temporary_archive), 'archiveBytes': temporary_archive.stat().st_size,
|
||||
'unpackedFileBytes': sum(p.stat().st_size for p in payload_files(staging)),
|
||||
'payloadFileCount': len(manifest['files']) + 1, 'sourceDateEpoch': epoch,
|
||||
'pythonVersion': platform_version(), 'zlibVersion': zlib.ZLIB_RUNTIME_VERSION,
|
||||
'executableSha256': dependencies['executableSha256'],
|
||||
'systemRuntimeCandidateBytesNotBundled': dependencies['systemRuntimeCandidateBytes'],
|
||||
'systemComponentCount': len(dependencies['components']) - 1,
|
||||
'archiveRoundTripSha256Matches': True, 'guiExtractionSmoke': 'not-run-by-packager',
|
||||
'scope': dependencies['scope'], 'dependencySources': 'not-collected',
|
||||
'completeChromiumNotices': False, 'cleanOsAcceptance': False}
|
||||
shutil.copyfile(temporary_archive, archive)
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + '\n')
|
||||
return report
|
||||
|
||||
|
||||
def platform_version():
|
||||
import platform
|
||||
return platform.python_version()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument('--prefix', type=Path, help='Trusted, fresh cmake install tree')
|
||||
source.add_argument('--build-dir', type=Path, help='Run cmake --install into a private temporary tree; no build')
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--epoch', type=int, default=int(os.environ.get('SOURCE_DATE_EPOCH', '0')))
|
||||
parser.add_argument('--qtpaths', default='/usr/lib/qt6/bin/qtpaths')
|
||||
args = parser.parse_args()
|
||||
if args.build_dir:
|
||||
with tempfile.TemporaryDirectory(prefix='docview-install-') as directory:
|
||||
subprocess.run(['cmake', '--install', str(args.build_dir.resolve()), '--prefix', directory], check=True)
|
||||
result = create_package(Path(directory), args.output, args.epoch, args.qtpaths)
|
||||
else:
|
||||
result = create_package(args.prefix, args.output, args.epoch, args.qtpaths)
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a local Ubuntu 24.04 amd64 deb with an explicitly selected Qt SDK."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from collect_ubuntu_validation_runtime import Collector, EXECUTABLES, bounded_walk
|
||||
from verify_pdfium_candidate import verify_installed as verify_pdfium_installed
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
VERSION = '0.1.0~validation3'
|
||||
VERSIONS = (VERSION, '0.1.0~validation4', '0.1.0~validation5')
|
||||
MANIFEST = 'opt/docview/share/doc/docview/package-manifest.json'
|
||||
SDK_SUPPLEMENT_REPORT_SHA = 'e218e5a24d136dbfe2982e56dbf9bb1e3fd104644c967ca02234b5e6be47096e'
|
||||
|
||||
|
||||
def sha(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def canonical(name):
|
||||
if not isinstance(name, str) or '\\' in name or '\x00' in name:
|
||||
raise ValueError('Invalid relative path')
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or '..' in path.parts or str(path) != name or name in ('', '.'):
|
||||
raise ValueError('Invalid relative path')
|
||||
return path
|
||||
|
||||
|
||||
def copy_verified(source, target, digest=None, executable=None):
|
||||
if not source.is_file() or source.stat().st_size > 512 * 1024**2:
|
||||
raise ValueError('Invalid or oversized package input')
|
||||
expected = digest or sha(source)
|
||||
if sha(source) != expected:
|
||||
raise ValueError('Input changed: ' + str(source))
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists():
|
||||
if sha(target) != expected:
|
||||
raise ValueError('Conflicting package destination')
|
||||
return
|
||||
shutil.copyfile(source, target)
|
||||
target.chmod(0o755 if (os.access(source, os.X_OK) if executable is None else executable) else 0o644)
|
||||
if sha(target) != expected or sha(source) != expected:
|
||||
raise ValueError('Package input changed while copying')
|
||||
|
||||
|
||||
def normalize_modes(directory):
|
||||
for path in [directory, *directory.rglob('*')]:
|
||||
if path.is_symlink() or (not path.is_dir() and not path.is_file()):
|
||||
raise ValueError('Unexpected staged link or special file')
|
||||
path.chmod(0o755 if path.is_dir() or path.stat().st_mode & 0o111 else 0o644)
|
||||
|
||||
|
||||
def copy_sdk_supplement(directory, base_report_path, known_runtime, destination):
|
||||
directory = directory.resolve(strict=True)
|
||||
report_path = directory / 'report.json'
|
||||
if report_path.is_symlink() or sha(report_path) != SDK_SUPPLEMENT_REPORT_SHA:
|
||||
raise ValueError('Qt SDK supplement report differs from the reviewed fixed input')
|
||||
report = json.loads(report_path.read_text())
|
||||
if sha(base_report_path) != report['baseNoticeReportSha256']:
|
||||
raise ValueError('Qt SDK supplement belongs to a different base notice report')
|
||||
for row in report['runtimeComparisons']:
|
||||
if known_runtime.get(row['resolvedPath']) != row['sha256']:
|
||||
raise ValueError('Qt SDK supplement belongs to a different runtime')
|
||||
inputs = []
|
||||
for row in report['files']:
|
||||
name = canonical(row['file'])
|
||||
source = directory / name
|
||||
if source.is_symlink() or not source.resolve(strict=True).is_relative_to(directory):
|
||||
raise ValueError('Qt SDK supplement source escapes its selected directory')
|
||||
if source.stat().st_size != row['bytes'] or sha(source) != row['sha256']:
|
||||
raise ValueError('Qt SDK supplement source changed')
|
||||
inputs.append((source, destination / name, row['sha256']))
|
||||
# Check every input before creating the destination, then verify each copy.
|
||||
for source, target, digest in inputs:
|
||||
copy_verified(source, target, digest, executable=False)
|
||||
copy_verified(report_path, destination / 'report.json', SDK_SUPPLEMENT_REPORT_SHA, executable=False)
|
||||
return SDK_SUPPLEMENT_REPORT_SHA
|
||||
|
||||
|
||||
def create(args):
|
||||
version = getattr(args, 'version', VERSION)
|
||||
if version not in VERSIONS:
|
||||
raise ValueError('Unsupported local validation package version')
|
||||
if platform.freedesktop_os_release().get('ID') != 'ubuntu' or platform.freedesktop_os_release().get('VERSION_ID') != '24.04':
|
||||
raise ValueError('Build this package on Ubuntu 24.04')
|
||||
if subprocess.check_output(['dpkg', '--print-architecture'], text=True).strip() != 'amd64':
|
||||
raise ValueError('Only amd64 is supported')
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
collector = Collector(args.prefix, args.qtpaths, args.dependency_prefix, output / 'runtime', args.source_root,
|
||||
qpdf_source_archive=getattr(args, 'qpdf_source_archive', None))
|
||||
runtime = collector.collect(platform.freedesktop_os_release(), args.input_manifest)
|
||||
if not runtime['runtimeValidationSuccess'] or runtime['systemOwnershipUnresolved']:
|
||||
raise ValueError('Runtime inventory has unresolved dependencies')
|
||||
(output / 'runtime/runtime.json').write_text(json.dumps(runtime, indent=2, sort_keys=True) + '\n')
|
||||
stage = output / 'stage'; stage.mkdir()
|
||||
app = stage / 'opt/docview'
|
||||
mapping = {'install': (collector.prefix, app), 'qt-sdk': (collector.sdk, app / 'qt'),
|
||||
'dependencies': (collector.deps, app)}
|
||||
copied = []
|
||||
for row in runtime['files']:
|
||||
if not row['path'].startswith(tuple(key + ':' for key in mapping)):
|
||||
continue
|
||||
kind, relative = row['path'].split(':', 1)
|
||||
source_root, destination_root = mapping[kind]
|
||||
# The inventory has one harmless bin/../lib alias for PDFium. Normalize
|
||||
# the lexical destination while checking the actual source separately.
|
||||
normalized = os.path.normpath(relative)
|
||||
canonical(normalized)
|
||||
source = source_root / normalized
|
||||
if not source.resolve(strict=True).is_relative_to(source_root):
|
||||
raise ValueError('Runtime source escapes its selected prefix')
|
||||
target = destination_root / normalized
|
||||
copy_verified(source, target, row['sha256'])
|
||||
copied.append({'source': row['path'], 'path': str(target.relative_to(stage)), 'sha256': row['sha256']})
|
||||
documentation = app / 'share/doc/docview'
|
||||
for path in bounded_walk(collector.prefix / 'share/doc/docview', (collector.prefix,)):
|
||||
copy_verified(path, documentation / path.relative_to(collector.prefix / 'share/doc/docview'), executable=False)
|
||||
pdfium_correspondence = verify_pdfium_installed(app)
|
||||
if pdfium_correspondence != runtime['pdfiumCorrespondence']:
|
||||
raise ValueError('PDFium correspondence changed during packaging')
|
||||
# Preserve selected runtime notices and SBOMs with the original scope ledger.
|
||||
shutil.copytree(output / 'runtime', documentation / 'third-party/runtime')
|
||||
supplemental = args.sdk_notices.resolve(strict=True)
|
||||
notice_report = json.loads((supplemental / 'report.json').read_text())
|
||||
if not notice_report['success'] or len(notice_report['notices']) != 129:
|
||||
raise ValueError('Verified Qt SDK notice extraction is required')
|
||||
known_runtime = {row['resolvedPath']: row['sha256'] for row in runtime['files']}
|
||||
for row in notice_report['testedRuntimeComparisons']:
|
||||
if known_runtime.get(row['resolvedPath']) != row['sha256']:
|
||||
raise ValueError('Qt SDK notice extraction belongs to a different runtime')
|
||||
copy_verified(supplemental / 'report.json', documentation / 'third-party/qt-sdk/source-report.json', executable=False)
|
||||
copy_verified(supplemental / 'THIRD_PARTY_NOTICES.sdk-extract.txt', documentation / 'third-party/qt-sdk/NOTICES.txt',
|
||||
notice_report['noticeBundleSha256'], executable=False)
|
||||
for row in notice_report['notices']:
|
||||
name = canonical(row['file'])
|
||||
copy_verified(supplemental / name, documentation / 'third-party/qt-sdk' / name, row['sha256'], executable=False)
|
||||
supplement_sha = copy_sdk_supplement(args.sdk_supplement, supplemental / 'report.json', known_runtime,
|
||||
documentation / 'third-party/qt-sdk-supplement')
|
||||
for path in bounded_walk(args.qt_licenses.resolve(strict=True), (args.qt_licenses.resolve(strict=True),)):
|
||||
copy_verified(path, documentation / 'third-party/qt-licenses' / path.relative_to(args.qt_licenses.resolve()), executable=False)
|
||||
(documentation / 'UBUNTU-PACKAGE.txt').write_text(
|
||||
'DocView — Ubuntu 24.04 amd64\n\nStart: docview [document]\n'
|
||||
'Remove application: sudo apt remove docview\n'
|
||||
'Remove application and its system profile: sudo apt purge docview\n'
|
||||
'User documents, preferences and reading history are preserved.\n\n'
|
||||
'This local validation package includes a private Qt SDK runtime, qpdf, libzip and PDFium.\n'
|
||||
'System dependencies are declared in the Debian control metadata.\n'
|
||||
'Third-party notices and source references are under third-party/.\n'
|
||||
'The Qt SDK supplement retains WebM/WebP Patent files declared by the fixed upstream metadata.\n'
|
||||
'Notice/source completeness and public release conditions have not been finalized.\n')
|
||||
for source, target, executable in [
|
||||
('docview-launcher', 'usr/bin/docview', True), ('docview.desktop', 'usr/share/applications/docview.desktop', False),
|
||||
('docview.apparmor', 'etc/apparmor.d/docview', False), ('deb-postinst', 'DEBIAN/postinst', True),
|
||||
('deb-prerm', 'DEBIAN/prerm', True)]:
|
||||
copy_verified(ROOT / 'resources/linux' / source, stage / target, executable=executable)
|
||||
for target, prefix in [(app / 'bin/qt.conf', '../qt'), (app / 'qt/libexec/qt.conf', '..')]:
|
||||
target.write_text('[Paths]\nPrefix=' + prefix + '\nLibraries=lib\nLibraryExecutables=libexec\n'
|
||||
'Plugins=plugins\nQmlImports=qml\nTranslations=translations\nData=.\n')
|
||||
dependencies = {'apparmor', 'fonts-dejavu-core', 'fonts-noto-cjk'}
|
||||
for name, package in runtime['systemPackages'].items():
|
||||
if not re.fullmatch(r'[a-z0-9][a-z0-9+.-]*(?::amd64)?', name) or not re.fullmatch(r'[0-9A-Za-z.+:~\-]+', package['version']):
|
||||
raise ValueError('Invalid Debian dependency identity')
|
||||
dependencies.add(name.split(':')[0] + ' (>= ' + package['version'] + ')')
|
||||
control = stage / 'DEBIAN/control'
|
||||
control.write_text('Package: docview\nVersion: ' + version + '\nArchitecture: amd64\n'
|
||||
'Maintainer: DocView contributors\nSection: text\nPriority: optional\n'
|
||||
'Depends: ' + ', '.join(sorted(dependencies)) + '\n'
|
||||
'Description: Local PDF, HTML and EPUB document viewer\n'
|
||||
' A keyboard-oriented Qt Quick viewer with isolated PDF and archive workers.\n')
|
||||
(stage / 'DEBIAN/conffiles').write_text('/etc/apparmor.d/docview\n')
|
||||
normalize_modes(stage)
|
||||
payload = []
|
||||
total = 0
|
||||
for path in sorted(stage.rglob('*')):
|
||||
if path.is_symlink() or (not path.is_dir() and not path.is_file()):
|
||||
raise ValueError('Unexpected staged link or special file')
|
||||
if not path.is_file(): continue
|
||||
total += path.stat().st_size
|
||||
if total > 2 * 1024**3 or len(payload) >= 10000:
|
||||
raise ValueError('Package exceeds finite payload limits')
|
||||
payload.append({'path': str(path.relative_to(stage)), 'size': path.stat().st_size,
|
||||
'sha256': sha(path), 'mode': oct(path.stat().st_mode & 0o777)})
|
||||
manifest = {'schemaVersion': 1, 'package': 'docview', 'version': version, 'target': 'Ubuntu 24.04 amd64',
|
||||
'files': payload, 'excludedFromOwnDigest': [MANIFEST], 'runtimeCopies': copied,
|
||||
'sdkSupplementReportSha256': supplement_sha,
|
||||
'pdfiumCorrespondence': pdfium_correspondence,
|
||||
'qpdfNoticeCorrespondence': runtime['qpdfNoticeCorrespondence'],
|
||||
'completeChromiumNotices': False, 'completeCorrespondingSources': False,
|
||||
'publicReleaseApproved': False}
|
||||
(stage / MANIFEST).write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n')
|
||||
(stage / MANIFEST).chmod(0o644)
|
||||
# Debian's size field includes the data archive, rounded to KiB per file
|
||||
# and directory. Its control row is itself recorded in our manifest, so
|
||||
# settle the tiny self-size dependency before building the archive.
|
||||
for _ in range(8):
|
||||
installed_kib = sum(1 if path.is_dir() else (path.stat().st_size + 1023) // 1024
|
||||
for path in stage.rglob('*') if path.relative_to(stage).parts[0] != 'DEBIAN')
|
||||
previous = control.read_text()
|
||||
updated = re.sub(r'^Installed-Size:.*\n', '', previous, flags=re.M)
|
||||
updated += 'Installed-Size: ' + str(installed_kib) + '\n'
|
||||
if updated == previous:
|
||||
break
|
||||
control.write_text(updated)
|
||||
row = next(row for row in payload if row['path'] == 'DEBIAN/control')
|
||||
row.update(size=control.stat().st_size, sha256=sha(control))
|
||||
(stage / MANIFEST).write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n')
|
||||
else:
|
||||
raise ValueError('Installed size did not converge')
|
||||
total = sum(row['size'] for row in payload)
|
||||
environment = {**os.environ, 'SOURCE_DATE_EPOCH': '0', 'LC_ALL': 'C'}
|
||||
archive = output / ('docview_' + version + '_amd64.deb')
|
||||
command = ['dpkg-deb', '--root-owner-group', '-Zxz', '-z6', '--threads-max=2', '--build', str(stage), str(archive)]
|
||||
subprocess.run(command, env=environment, check=True, timeout=300)
|
||||
report = {'success': True, 'archive': archive.name, 'archiveSha256': sha(archive), 'archiveBytes': archive.stat().st_size,
|
||||
'payloadFiles': len(payload) + 1, 'payloadBytes': total + (stage / MANIFEST).stat().st_size,
|
||||
'sourceDateEpoch': 0, 'packagerSha256': sha(Path(__file__)),
|
||||
'collectorSha256': sha(ROOT / 'tools/collect_ubuntu_validation_runtime.py'),
|
||||
'runtimeRecordSha256': sha(output / 'runtime/runtime.json'), 'systemDependencies': sorted(dependencies),
|
||||
'sdkSupplementReportSha256': supplement_sha,
|
||||
'pdfiumCorrespondence': pdfium_correspondence,
|
||||
'qpdfNoticeCorrespondence': runtime['qpdfNoticeCorrespondence'],
|
||||
'manifestSha256': sha(stage / MANIFEST), 'installedSmoke': 'not-run-by-packager',
|
||||
'installedSizeKiB': installed_kib,
|
||||
'executableSha256': {name: sha(app / 'bin' / name) for name in EXECUTABLES},
|
||||
'completeChromiumNotices': False, 'completeCorrespondingSources': False, 'publicReleaseApproved': False}
|
||||
(output / 'report.json').write_text(json.dumps(report, indent=2, sort_keys=True) + '\n')
|
||||
print(json.dumps({k: report[k] for k in ('success', 'archiveBytes', 'payloadFiles', 'payloadBytes')}))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--prefix', type=Path, required=True)
|
||||
parser.add_argument('--qtpaths', type=Path, required=True)
|
||||
parser.add_argument('--dependency-prefix', type=Path, required=True)
|
||||
parser.add_argument('--source-root', type=Path, action='append', default=[])
|
||||
parser.add_argument('--input-manifest', type=Path, required=True)
|
||||
parser.add_argument('--sdk-notices', type=Path, required=True)
|
||||
parser.add_argument('--sdk-supplement', type=Path, required=True)
|
||||
parser.add_argument('--qt-licenses', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--version', choices=VERSIONS, default=VERSION)
|
||||
parser.add_argument('--qpdf-source-archive', type=Path,
|
||||
help='Pinned qpdf source archive required when bundling the known qpdf runtime')
|
||||
create(parser.parse_args())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Record bounded local provenance; never download, build, or execute DocView.
|
||||
|
||||
This is an evidence ledger, not a license-compliance assessment. Package cache
|
||||
metadata is matched to the bundled inventory, not assumed to authenticate the
|
||||
installed files. Raw packager/build-directory/host inventory fields are omitted.
|
||||
Python 3.14+ supplies the streaming zstd tar reader used for Arch cache files.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
from package_linux_development import MANIFEST, ROOT, verify_and_extract
|
||||
from collect_linux_dependencies import qt_embedded_notices
|
||||
|
||||
DEPENDENCIES = 'share/doc/docview/third-party/dependency-manifest.json'
|
||||
PRIMARY_CACHE_HASHES = {'qt6-base', 'qt6-declarative', 'qt6-webengine', 'tomlplusplus'}
|
||||
PKG_FIELDS = {'pkgname', 'pkgbase', 'pkgver', 'arch', 'license', 'builddate'}
|
||||
BUILD_FIELDS = {'format', 'pkgname', 'pkgbase', 'pkgver', 'pkgarch',
|
||||
'pkgbuild_sha256sum', 'builddate', 'buildtool', 'buildtoolver'}
|
||||
MULTI_FIELDS = {'pkgname', 'license'}
|
||||
MAX_METADATA = 1024 * 1024
|
||||
MAX_PREFIX = 8 * 1024 * 1024
|
||||
MAX_ARCHIVE_HASH = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def file_digest(path):
|
||||
result = hashlib.sha256()
|
||||
with path.open('rb') as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b''):
|
||||
result.update(chunk)
|
||||
return result.hexdigest()
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode()
|
||||
|
||||
|
||||
def bounded_bytes(path, maximum=MAX_METADATA):
|
||||
with path.open('rb') as source:
|
||||
data = source.read(maximum + 1)
|
||||
if len(data) > maximum:
|
||||
raise ValueError('Input exceeds its finite metadata bound')
|
||||
return data
|
||||
|
||||
|
||||
def parse_metadata(data, fields):
|
||||
"""Discard sensitive fields before validating/exporting the allowed subset."""
|
||||
if len(data) > MAX_METADATA or b'\0' in data:
|
||||
raise ValueError('Invalid package metadata size or encoding')
|
||||
result = {}
|
||||
for line in data.decode('utf-8', errors='strict').splitlines():
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
key, separator, value = line.partition(' = ')
|
||||
if not separator:
|
||||
raise ValueError('Malformed package metadata record')
|
||||
if key not in fields:
|
||||
continue
|
||||
# The selected identity/license/build-tool fields never need paths,
|
||||
# email addresses, control characters, or freeform descriptions.
|
||||
if not value or len(value) > 512 or not re.fullmatch(r'[A-Za-z0-9_:.,+() /&*~=-]+', value):
|
||||
raise ValueError('Invalid selected package metadata value')
|
||||
if '/' in value or '@' in value:
|
||||
raise ValueError('Selected package metadata contains a path or address')
|
||||
if key in result and key not in MULTI_FIELDS:
|
||||
raise ValueError('Duplicate scalar package metadata')
|
||||
result.setdefault(key, []).append(value)
|
||||
return {key: sorted(values) if key in MULTI_FIELDS else values[0]
|
||||
for key, values in sorted(result.items())}
|
||||
|
||||
|
||||
def cache_metadata(path, expected):
|
||||
"""Read only the bounded metadata prefix; do not unpack package payloads."""
|
||||
raw = {}
|
||||
with tarfile.open(path, mode='r|zst') as archive:
|
||||
for index, member in enumerate(archive):
|
||||
if index >= 32 or member.offset_data + member.size > MAX_PREFIX:
|
||||
raise ValueError('Package metadata is outside the bounded prefix')
|
||||
if member.name in ('.PKGINFO', '.BUILDINFO'):
|
||||
if member.name in raw or not member.isfile() or member.size > MAX_METADATA:
|
||||
raise ValueError('Invalid package metadata member')
|
||||
raw[member.name] = archive.extractfile(member).read(MAX_METADATA + 1)
|
||||
if len(raw) == 2:
|
||||
break
|
||||
if set(raw) != {'.PKGINFO', '.BUILDINFO'}:
|
||||
raise ValueError('Package cache lacks required metadata')
|
||||
package = parse_metadata(raw['.PKGINFO'], PKG_FIELDS)
|
||||
build = parse_metadata(raw['.BUILDINFO'], BUILD_FIELDS)
|
||||
name, version, arch, base = expected
|
||||
for fields, arch_key in ((package, 'arch'), (build, 'pkgarch')):
|
||||
if (name not in fields.get('pkgname', []) or fields.get('pkgver') != version or
|
||||
fields.get(arch_key) != arch or fields.get('pkgbase', name) != base):
|
||||
raise ValueError('Cached package identity does not match the final inventory')
|
||||
if (build.get('format') != '2' or
|
||||
not re.fullmatch('[0-9a-f]{64}', build.get('pkgbuild_sha256sum', ''))):
|
||||
raise ValueError('Missing supported build format or PKGBUILD digest')
|
||||
return {'status': 'identity-matched-metadata', 'file': path.name,
|
||||
'compressedBytes': path.stat().st_size,
|
||||
'packageInfo': {'rawSha256': digest(raw['.PKGINFO']), 'selectedFields': package},
|
||||
'buildInfo': {'rawSha256': digest(raw['.BUILDINFO']), 'selectedFields': build},
|
||||
'signatureVerification': 'not-performed',
|
||||
'installedFileToCachedPayloadComparison': 'not-performed',
|
||||
'recipeCommit': None,
|
||||
'recipeCommitStatus': 'PKGBUILD hash observed; Git revision not resolved by this collector; '
|
||||
'see separate source-correspondence records'}
|
||||
|
||||
|
||||
def cache_record(cache, name, component):
|
||||
version, arch = component['version'], component['architecture']
|
||||
base = component['source']['packageBase']
|
||||
if not all(re.fullmatch('[A-Za-z0-9_.+:~-]+', v) for v in (name, version, arch, base)):
|
||||
raise ValueError('Unsafe component identity in dependency manifest')
|
||||
path = cache / f'{name}-{version}-{arch}.pkg.tar.zst'
|
||||
if not path.is_file():
|
||||
return {'status': 'not-present-in-selected-cache'}
|
||||
record = cache_metadata(path, (name, version, arch, base))
|
||||
if name in PRIMARY_CACHE_HASHES:
|
||||
if path.stat().st_size > MAX_ARCHIVE_HASH:
|
||||
raise ValueError('Selected package exceeds bounded archive hashing limit')
|
||||
record['archiveSha256'] = file_digest(path)
|
||||
else:
|
||||
record['archiveHashStatus'] = 'not-computed; raw metadata digests recorded'
|
||||
return record
|
||||
|
||||
|
||||
def source_evidence(path, label, needles):
|
||||
data = bounded_bytes(path)
|
||||
lines = data.decode('utf-8').splitlines()
|
||||
return {'path': label, 'sha256': digest(data),
|
||||
'selectedLines': [{'line': i, 'text': line.strip()}
|
||||
for i, line in enumerate(lines, 1)
|
||||
if any(needle in line for needle in needles)]}
|
||||
|
||||
|
||||
def inspect_elf(tool, arguments, executable, maximum=32 * 1024 * 1024):
|
||||
# readelf/nm inspect bytes. Neither the input executable nor ldd is run.
|
||||
with tempfile.TemporaryFile() as output:
|
||||
result = subprocess.run([tool, *arguments, str(executable)], stdout=output,
|
||||
stderr=subprocess.DEVNULL, timeout=20, check=False)
|
||||
if result.returncode or output.tell() > maximum:
|
||||
raise ValueError('Bounded ELF inspection failed')
|
||||
output.seek(0)
|
||||
return output.read(maximum + 1).decode('utf-8', errors='strict')
|
||||
|
||||
|
||||
def license_evidence(payload, row, bundled=False):
|
||||
relative = row['path'] if bundled else 'share/doc/docview/third-party/' + row['path']
|
||||
path = PurePosixPath(relative)
|
||||
if path.is_absolute() or '..' in path.parts or str(path) != relative:
|
||||
raise ValueError('Unsafe license evidence path')
|
||||
data = bounded_bytes(payload / relative, 8 * MAX_METADATA)
|
||||
if len(data) != row['size'] or digest(data) != row['sha256']:
|
||||
raise ValueError('License text disagrees with packaged inventory')
|
||||
return {'payloadPath': relative, 'size': len(data), 'sha256': digest(data)}
|
||||
|
||||
|
||||
def qt_embedded_notice_record(payload, manifest, licenses, root):
|
||||
component = manifest['components']['qt6-webengine']
|
||||
relative = 'licenses/qt6-webengine/embedded-resource-notices'
|
||||
full_relative = 'share/doc/docview/third-party/' + relative
|
||||
directory = payload / full_relative
|
||||
inventory = [row for row in licenses if row['payloadPath'].startswith(full_relative + '/')]
|
||||
supplied = component.get('embeddedResourceNotices')
|
||||
if supplied is None:
|
||||
if inventory or directory.exists():
|
||||
raise ValueError('Qt embedded notice payload lacks manifest metadata')
|
||||
return {'status': 'not-in-this-package', 'completeChromiumNotices': False,
|
||||
'scope': 'Legacy package; no embedded Qt resource notice claim'}
|
||||
original_path, metadata = qt_embedded_notices(
|
||||
manifest['host']['qtVersion'], component['version'], manifest['systemFiles'],
|
||||
root / 'resources/licenses/qt-embedded-notices')
|
||||
original = bounded_bytes(original_path, 65536)
|
||||
packaged = bounded_bytes(directory / 'sources.json', 65536)
|
||||
if original != packaged:
|
||||
raise ValueError('Qt embedded notice metadata differs from repository original')
|
||||
names = {row['name'] for row in metadata['files']}
|
||||
if {path.name for path in directory.iterdir()} != names | {'sources.json'}:
|
||||
raise ValueError('Unexpected Qt embedded notice payload contents')
|
||||
if component['source']['status'] != 'not-collected':
|
||||
raise ValueError('Partial Qt notices cannot claim complete source collection')
|
||||
evidence, expected_rows = [], []
|
||||
for row in metadata['files']:
|
||||
data = bounded_bytes(directory / row['name'])
|
||||
source = bounded_bytes(original_path.parent / row['name'])
|
||||
if data != source or len(data) != row['size'] or digest(data) != row['sha256']:
|
||||
raise ValueError('Qt embedded notice differs from original or metadata')
|
||||
evidence.append({'payloadPath': full_relative + '/' + row['name'],
|
||||
'size': len(data), 'sha256': digest(data)})
|
||||
expected_rows.append({'path': relative + '/' + row['name'], 'size': row['size'], 'sha256': row['sha256'],
|
||||
'origin': 'reviewed-installed-DataPack-resource-comment',
|
||||
'sourceResources': row['sourceResources'], 'collectionScope': metadata['scope']})
|
||||
if sorted(inventory, key=lambda row: row['payloadPath']) != sorted(evidence, key=lambda row: row['payloadPath']):
|
||||
raise ValueError('Qt embedded notice license inventory differs from metadata')
|
||||
recorded_rows = [row for row in component['licenseTexts'] if row['path'].startswith(relative + '/')]
|
||||
if sorted(recorded_rows, key=lambda row: row['path']) != sorted(expected_rows, key=lambda row: row['path']):
|
||||
raise ValueError('Qt embedded notice resource provenance differs from metadata')
|
||||
expected = {'status': 'collected-reviewed-resource-subset',
|
||||
'metadataPath': relative + '/sources.json', 'metadataSha256': digest(original),
|
||||
'noticeCount': len(metadata['files']),
|
||||
'sourceResourceCount': sum(len(row['sourceResources']) for row in metadata['files']),
|
||||
'runtimeDistribution': 'system-not-bundled', 'completeChromiumNotices': False,
|
||||
'scope': metadata['scope']}
|
||||
if supplied != expected:
|
||||
raise ValueError('Qt embedded notice manifest metadata mismatch')
|
||||
return {'status': 'repository-original-and-package-matched',
|
||||
'metadata': {'payloadPath': full_relative + '/sources.json', 'sha256': digest(original)},
|
||||
'repositoryMetadata': {'path': 'resources/licenses/qt-embedded-notices/sources.json',
|
||||
'sha256': digest(original)},
|
||||
'noticeCount': expected['noticeCount'], 'sourceResourceCount': expected['sourceResourceCount'],
|
||||
'completeChromiumNotices': False, 'runtimeDistribution': 'system-not-bundled',
|
||||
'sourceCollectionStatus': 'not-collected', 'dataPacks': metadata['dataPacks'],
|
||||
'files': expected_rows, 'scope': metadata['scope'],
|
||||
'scopeLimit': 'Repository originals and packaged notice bytes matched to the recorded system DataPack inventory; '
|
||||
'host DataPacks were not rehashed here. No complete Chromium attribution, source collection, or legal assessment.'}
|
||||
|
||||
|
||||
def pdfium_supplemental_record(payload, component, licenses, lock, library_hash, root):
|
||||
relative = 'share/doc/docview/pdfium/supplemental'
|
||||
inventory = [row for row in licenses if row['payloadPath'].startswith(relative + '/')]
|
||||
supplied = component.get('supplementalNotices')
|
||||
directory = payload / relative
|
||||
if supplied is None:
|
||||
if inventory or directory.exists():
|
||||
raise ValueError('PDFium supplemental payload lacks manifest metadata')
|
||||
return {'status': 'not-in-this-package',
|
||||
'scope': 'Legacy provider-only notices; no supplemental notice claim'}
|
||||
original_root = root / 'resources/licenses/pdfium-supplemental'
|
||||
original = bounded_bytes(original_root / 'sources.json', 65536)
|
||||
packaged = bounded_bytes(directory / 'sources.json', 65536)
|
||||
if original != packaged:
|
||||
raise ValueError('PDFium supplemental metadata differs from repository original')
|
||||
metadata = json.loads(original)
|
||||
if (type(metadata.get('schemaVersion')) is not int or metadata['schemaVersion'] != 1 or
|
||||
metadata.get('pdfiumVersion') != lock['version'] or
|
||||
metadata.get('pdfiumUpstreamCommit') != lock['upstreamCommit'] or
|
||||
metadata.get('pdfiumLibrarySha256') != library_hash):
|
||||
raise ValueError('PDFium supplemental source pin or library hash mismatch')
|
||||
rows = metadata.get('files')
|
||||
names = {'libcxx-LICENSE.txt', 'libcxxabi-LICENSE.txt'}
|
||||
if (not isinstance(rows, list) or len(rows) != 2 or
|
||||
not all(isinstance(row, dict) for row in rows) or
|
||||
{row.get('name') for row in rows} != names):
|
||||
raise ValueError('Unexpected PDFium supplemental notice set')
|
||||
if (not re.fullmatch('[0-9a-f]{40}', metadata.get('providerRecipeCommit', '')) or
|
||||
{path.name for path in directory.iterdir()} != names | {'sources.json'}):
|
||||
raise ValueError('Invalid PDFium supplemental recipe or payload contents')
|
||||
files, evidence = [], []
|
||||
for row in rows:
|
||||
if (type(row.get('size')) is not int or not 0 < row['size'] <= MAX_METADATA or
|
||||
not re.fullmatch('[0-9a-f]{64}', row.get('sha256', '')) or
|
||||
not re.fullmatch('[0-9a-f]{40}', row.get('sourceRevision', '')) or
|
||||
not isinstance(row.get('sourceUrl'), str) or
|
||||
not row['sourceUrl'].startswith('https://chromium.googlesource.com/') or
|
||||
'/+/' + row['sourceRevision'] + '/' not in row['sourceUrl']):
|
||||
raise ValueError('Invalid PDFium supplemental fixed-source notice metadata')
|
||||
source = bounded_bytes(original_root / row['name'])
|
||||
data = bounded_bytes(directory / row['name'])
|
||||
if (source != data or len(source) != row['size'] or digest(source) != row['sha256']):
|
||||
raise ValueError('PDFium supplemental notice differs from original or metadata')
|
||||
path = relative + '/' + row['name']
|
||||
files.append({**row, 'path': path})
|
||||
evidence.append({'payloadPath': path, 'size': len(data), 'sha256': digest(data)})
|
||||
if sorted(inventory, key=lambda row: row['payloadPath']) != sorted(evidence, key=lambda row: row['payloadPath']):
|
||||
raise ValueError('PDFium supplemental license inventory differs from metadata')
|
||||
expected = {'metadataPath': relative + '/sources.json', 'metadataSha256': digest(original),
|
||||
'scope': metadata['scope'], 'providerRecipeCommit': metadata['providerRecipeCommit'],
|
||||
'pdfiumLibrarySha256': library_hash, 'files': files}
|
||||
if supplied != expected:
|
||||
raise ValueError('PDFium supplemental manifest metadata mismatch')
|
||||
return {'status': 'repository-original-and-package-matched', **expected,
|
||||
'repositoryMetadata': {'path': 'resources/licenses/pdfium-supplemental/sources.json',
|
||||
'sha256': digest(original)},
|
||||
'scopeLimit': 'Fixed-source notice bytes and recorded pins matched; '
|
||||
'no new source download, build reproduction or legal assessment'}
|
||||
|
||||
|
||||
def pdfium_record(payload, manifest, pdfium_root, root):
|
||||
component = manifest['components']['PDFium-independent-worker']
|
||||
lock_data = bounded_bytes(root / 'cmake/pdfium.lock.json')
|
||||
lock = json.loads(lock_data)
|
||||
version_data = bounded_bytes(pdfium_root / 'VERSION')
|
||||
values = dict(re.findall(r'^(MAJOR|MINOR|BUILD|PATCH)=(\d+)$', version_data.decode(), re.M))
|
||||
if set(values) != {'MAJOR', 'MINOR', 'BUILD', 'PATCH'}:
|
||||
raise ValueError('Invalid PDFium VERSION metadata')
|
||||
version = '.'.join(values[key] for key in ('MAJOR', 'MINOR', 'BUILD', 'PATCH'))
|
||||
args_data = bounded_bytes(pdfium_root / 'args.gn')
|
||||
args = {}
|
||||
for line in args_data.decode().splitlines():
|
||||
match = re.fullmatch(r'([a-z_][a-z_0-9]*)\s*=\s*(true|false|"[A-Za-z0-9_-]+")', line.strip())
|
||||
if not match or match[1] in args:
|
||||
raise ValueError('Unsupported or duplicate PDFium build argument')
|
||||
args[match[1]] = json.loads(match[2])
|
||||
if (version != component['version'] or version != lock['version'] or
|
||||
lock['upstreamCommit'] != component['source']['upstreamCommit'] or
|
||||
lock['linuxX64Sha256'] != component['source']['archiveSha256'] or
|
||||
any(args.get(key) != value for key, value in lock['buildOptions'].items())):
|
||||
raise ValueError('Local PDFium build metadata disagrees with final inventory/lock')
|
||||
packaged_library = payload / component['path']
|
||||
library_hash = file_digest(packaged_library)
|
||||
if (library_hash != component['sha256'] or
|
||||
library_hash != file_digest(pdfium_root / 'lib/libpdfium.so')):
|
||||
raise ValueError('Local PDFium library differs from packaged library')
|
||||
licenses = [license_evidence(payload, row, True) for row in component['licenseTexts']]
|
||||
if len({row['payloadPath'] for row in licenses}) != len(licenses):
|
||||
raise ValueError('Duplicate PDFium license inventory entry')
|
||||
for row in licenses:
|
||||
if not row['payloadPath'].startswith('share/doc/docview/pdfium/'):
|
||||
raise ValueError('Unexpected PDFium license prefix')
|
||||
suffix = row['payloadPath'].removeprefix('share/doc/docview/pdfium/')
|
||||
if suffix.startswith('supplemental/'):
|
||||
continue
|
||||
if file_digest(pdfium_root / suffix) != row['sha256']:
|
||||
raise ValueError('Local PDFium license differs from package')
|
||||
supplements = pdfium_supplemental_record(payload, component, licenses, lock, library_hash, root)
|
||||
return {'version': version, 'library': {'payloadPath': component['path'],
|
||||
'sha256': library_hash, 'size': packaged_library.stat().st_size},
|
||||
'localLibraryMatchesPayload': True, 'licenseTexts': licenses,
|
||||
'supplementalNotices': supplements,
|
||||
'providerLicense': 'MIT (top-level LICENSE; PDFium upstream license is separate)',
|
||||
'pdfiumUpstreamLicense': 'BSD-3-Clause text in licenses/pdfium.txt; see full file',
|
||||
'buildMetadata': {
|
||||
'VERSION': {'sha256': digest(version_data), 'values': values},
|
||||
'args.gn': {'sha256': digest(args_data), 'values': args},
|
||||
'lock': {'path': 'cmake/pdfium.lock.json', 'sha256': digest(lock_data)}},
|
||||
'pinnedProviderArchive': {'url': lock['linuxX64Archive'],
|
||||
'sha256FromLock': lock['linuxX64Sha256'],
|
||||
'archiveReverifiedThisRun': False},
|
||||
'upstreamCommit': lock['upstreamCommit'], 'upstream': lock['upstream'],
|
||||
'binaryProvider': lock['binaryProvider'],
|
||||
'remaining': ['Provider recipe/patch correspondence is not independently verified by this collector; '
|
||||
'see tests/results/source-correspondence/pdfium records',
|
||||
'Transitive source correspondence is not independently verified by this collector; '
|
||||
'see separate source-correspondence records',
|
||||
'Completeness of corresponding source and applicable obligations is not assessed here']}
|
||||
|
||||
|
||||
def make_record(archive, cache, pdfium_root, root=ROOT):
|
||||
with tempfile.TemporaryDirectory(prefix='docview-provenance-') as temporary:
|
||||
payload = verify_and_extract(archive, Path(temporary) / 'verified')
|
||||
manifest_bytes = bounded_bytes(payload / DEPENDENCIES, 16 * MAX_METADATA)
|
||||
manifest = json.loads(manifest_bytes)
|
||||
package_manifest_bytes = bounded_bytes(payload / MANIFEST, 8 * MAX_METADATA)
|
||||
package_manifest = json.loads(package_manifest_bytes)
|
||||
payload_rows = package_manifest['files']
|
||||
runtime_rows = [row for row in payload_rows if row['path'].startswith(('bin/', 'lib/'))]
|
||||
if {row['path'] for row in runtime_rows} != {
|
||||
'bin/docview', 'bin/docview-pdf-worker', 'bin/docview-archive-worker', 'lib/libpdfium.so'}:
|
||||
raise ValueError('Unexpected runtime payload; distribution scope must be reviewed')
|
||||
components = {}
|
||||
for name, item in sorted(manifest['components'].items()):
|
||||
if name == 'PDFium-independent-worker':
|
||||
continue
|
||||
if item['distribution'] != 'system-not-bundled':
|
||||
raise ValueError('Unreviewed dependency distribution category')
|
||||
files = [row for row in manifest['systemFiles'] if row['package'] == name]
|
||||
components[name] = {
|
||||
'version': item['version'], 'architecture': item['architecture'],
|
||||
'runtimeBinaryDistribution': 'system-only; not in this tar',
|
||||
'headerCodePresence': ('defined toml symbols observed in packaged DocView'
|
||||
if name == 'tomlplusplus' else 'not-audited'),
|
||||
'usageFromManifest': item.get('usage', []),
|
||||
'licenseLabelsFromPackage': item['licenseLabelsFromPackage'],
|
||||
'licenseTexts': [license_evidence(payload, row) for row in item['licenseTexts']],
|
||||
'sourceCollectionStatus': item['source']['status'],
|
||||
'sourceCollectionStatusScope': 'Status copied from this packaged inventory; '
|
||||
'separate source-correspondence records may contain later collection evidence',
|
||||
'obligationAssessment': 'not-performed',
|
||||
'recipeRepository': item['source']['recipeRepository'],
|
||||
'recordedSystemFiles': {'count': len(files),
|
||||
'canonicalRowsSha256': digest(canonical(files)),
|
||||
'evidence': 'packaged dependency-manifest.json#/systemFiles',
|
||||
'hostFilesRehashedThisRun': False},
|
||||
'cachedPackage': cache_record(cache, name, item)}
|
||||
if name == 'qt6-webengine':
|
||||
components[name]['embeddedResourceNotices'] = qt_embedded_notice_record(
|
||||
payload, manifest, components[name]['licenseTexts'], root)
|
||||
dynamic, embedded = [], {}
|
||||
for relative in ('bin/docview', 'bin/docview-pdf-worker', 'bin/docview-archive-worker'):
|
||||
text = inspect_elf('readelf', ['--dynamic', '--wide'], payload / relative)
|
||||
needed = sorted(re.findall(r'\(NEEDED\).*?\[([^\]]+)\]', text))
|
||||
if not needed or not any(name.startswith('libQt6') for name in needed):
|
||||
raise ValueError('Expected dynamic Qt dependency was not found')
|
||||
dynamic.append({'payloadPath': relative, 'directNeeded': needed})
|
||||
text = inspect_elf('nm', ['--defined-only', '--demangle'], payload / 'bin/docview')
|
||||
symbols = sorted({line.split(' ', 2)[-1] for line in text.splitlines()
|
||||
if re.match(r'^[0-9a-fA-F]+ [A-Za-z] toml::', line)})
|
||||
if not symbols:
|
||||
raise ValueError('No defined toml symbols; header-code inference needs review')
|
||||
embedded = {'component': 'tomlplusplus', 'payloadPath': 'bin/docview',
|
||||
'definedSymbolCount': len(symbols), 'definedSymbols': symbols,
|
||||
'interpretation': 'Header-derived code is present in this executable; '
|
||||
'the toml++ shared-library binary is still system-only.',
|
||||
'sourceEvidence': source_evidence(root / 'src/core/config.cpp',
|
||||
'src/core/config.cpp', ['#include <toml++/toml.h>'])}
|
||||
qt_files = [row for row in manifest['systemFiles']
|
||||
if row['package'].startswith('qt6-') and
|
||||
re.fullmatch(r'/usr/lib/libQt6[^/]+\.so(?:\.[0-9]+)+', row['path'])]
|
||||
pdfium = pdfium_record(payload, manifest, pdfium_root, root)
|
||||
return {'schemaVersion': 1, 'scope': 'Local final Arch development package evidence',
|
||||
'legalComplianceVerdict': 'not-assessed', 'docviewLicense': 'unspecified',
|
||||
'sourceCollectionIsNotObligationAssessment': True,
|
||||
'archive': {'file': archive.name, 'sha256': file_digest(archive),
|
||||
'compressedBytes': archive.stat().st_size,
|
||||
'payloadFileCount': len(payload_rows) + 1,
|
||||
'payloadBytes': sum(row['size'] for row in payload_rows) + len(package_manifest_bytes),
|
||||
'payloadIntegrity': 'all members verified against packaged payload manifest',
|
||||
'manifestExcludesOwnDigest': True,
|
||||
'packageManifestSha256': digest(package_manifest_bytes),
|
||||
'dependencyManifestSha256': digest(manifest_bytes), 'runtimeFiles': runtime_rows},
|
||||
'tools': {name: inspect_elf(name, ['--version'], Path('/dev/null'), MAX_METADATA)
|
||||
.splitlines()[0] for name in ('nm', 'readelf')},
|
||||
'pdfium': pdfium, 'systemComponents': components,
|
||||
'dynamicLinkEvidence': {'method': 'readelf DT_NEEDED on verified packaged executables',
|
||||
'executables': dynamic, 'qtLibraryRowsFromPackagedInventory': qt_files,
|
||||
'qtAndWebEngineRuntimeBinariesInPayload': False,
|
||||
'scope': 'Direct ELF links and recorded system resolutions; no claim that '
|
||||
'all Qt header-derived machine code is absent'},
|
||||
'embeddedHeaderEvidence': embedded,
|
||||
'privacy': {'omitted': ['home paths', 'user names', 'packager identities and emails',
|
||||
'build directories', 'complete builder installed-package inventory'],
|
||||
'rawCacheMetadataCopied': False},
|
||||
'limits': ['Cache metadata is identity-matched, not signature-authenticated',
|
||||
'Cache metadata is read only from its bounded prefix; the rest of each cache tar is not validated',
|
||||
'Only four primary cache archives receive a whole-archive SHA-256',
|
||||
'PKGBUILD SHA-256 is evidence of a recipe digest, not a recovered recipe or Git commit',
|
||||
'Recorded installed file hashes are from the final package manifest, not remeasured here',
|
||||
'This collector does not independently collect or verify complete source trees, recipe patches, '
|
||||
'or WebEngine build-specific full Chromium notices; see separate source-correspondence records',
|
||||
'This is not a reproducible-build, clean-OS, target-OS, or license-compliance result']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--archive', type=Path, default=ROOT / 'tests/results/linux-development-package/final/docview-0.1.0-arch-x86_64-development.tar.gz')
|
||||
parser.add_argument('--package-cache', type=Path, default=Path('/var/cache/pacman/pkg'))
|
||||
parser.add_argument('--pdfium-root', type=Path, default=ROOT / '.deps/pdfium')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'tests/results/dependency-provenance/local-final')
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
target = args.output / 'provenance.json'
|
||||
if target.exists():
|
||||
raise ValueError('Output already exists; choose a new evidence directory')
|
||||
record = make_record(args.archive, args.package_cache, args.pdfium_root)
|
||||
data = canonical(record)
|
||||
target.write_bytes(data)
|
||||
matched = sum(c['cachedPackage']['status'] == 'identity-matched-metadata'
|
||||
for c in record['systemComponents'].values())
|
||||
print(json.dumps({'recordSha256': digest(data), 'payloadFileCount': record['archive']['payloadFileCount'],
|
||||
'cacheMetadataMatched': matched, 'systemComponents': len(record['systemComponents'])}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,470 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and stage the fixed Linux PDFium candidate into a new CMake prefix."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MASTER = 'tests/results/pdfium-intent-build/record.json'
|
||||
MASTER_SHA256 = '47b4dc2efbf8e671f904610b3047400dbd3ceb0fa10d6bd53fb04f3727943e7e'
|
||||
LIBRARY_SHA256 = '0c968f503ce549bad5301af2cc3fd0b83c37437315ac0e62dff439d6a1c6cc4d'
|
||||
REVISION = 'a5a7089234f121990b336b3841008009dca143bf'
|
||||
SOURCE = 'build-pdfium-intent/source'
|
||||
LIBRARY = SOURCE + '/out/patched/libpdfium.so'
|
||||
GN_ARGS = SOURCE + '/out/patched/args.gn'
|
||||
GN_ARGS_SHA256 = '5a2e04e1c0bb3eb05dc415dfa005a917804be8f63ade2365016c148e8efd8197'
|
||||
LICENSE_RECORD = 'tests/results/source-correspondence/pdfium/license-correspondence.json'
|
||||
LICENSE_RECORD_SHA256 = 'e579762985e5d828c413bc1ba7cb9e2551a83a0ec695f99cf0603be341ad928c'
|
||||
VERSION_SHA256 = '7124a23c02e7383b7d80cc2cbc593fd8162ee536ca4ea8a85f0442f0c95c197a'
|
||||
MAX_FILE = 256 * 1024 * 1024
|
||||
MAX_OUTPUT = 128 * 1024 * 1024
|
||||
|
||||
# Internal fixed choices only. The context candidate stays unavailable until its
|
||||
# immutable anchor is supplied; neither the CLI nor callers can supply pin paths.
|
||||
V2_PIN = {
|
||||
'master': 'tests/results/pdfium-intent-context/record.json',
|
||||
'masterSha256': '80af72b06d5c9a6a3ab8b9311373e6a23e8df28668ce575e1124350e69aa160b',
|
||||
'parent': {'path': MASTER, 'sha256': MASTER_SHA256},
|
||||
'source': 'build-pdfium-intent-context/source',
|
||||
'library': 'build-pdfium-intent-context/source/out/patched/libpdfium.so',
|
||||
'librarySha256': 'cb049fe434f4c911b5eb047c0aca572d4dd9f5e405dde329b76c05a6aa771403',
|
||||
'gnArgs': 'build-pdfium-intent-context/source/out/patched/args.gn',
|
||||
'gnArgsSha256': GN_ARGS_SHA256,
|
||||
'schemaVersion': 6,
|
||||
'anchorName': 'master6',
|
||||
'recordPaths': {
|
||||
'candidate-patch': 'tests/results/pdfium-intent-context/candidate-patch.json',
|
||||
'source-materialization': 'tests/results/pdfium-intent-build/source-materialization.json',
|
||||
'tool-materialization': 'tests/results/pdfium-intent-build/tool-materialization.json',
|
||||
'provider-patches': 'tests/results/pdfium-intent-build/provider-patches.json',
|
||||
'patched-final-build': 'tests/results/pdfium-intent-context/build-2.json',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class StageError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise StageError(message)
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def relative(name):
|
||||
require(isinstance(name, str) and len(name) <= 1024, 'Invalid relative path')
|
||||
p = PurePosixPath(name)
|
||||
require(name and not p.is_absolute() and '\\' not in name and '\0' not in name
|
||||
and all(x not in ('', '.', '..') for x in name.split('/')),
|
||||
'Unsafe relative path: ' + name)
|
||||
return p.parts
|
||||
|
||||
|
||||
def candidate_pin(candidate):
|
||||
if candidate == 'v1':
|
||||
# Read the original constants at call time, preserving the existing API
|
||||
# and the synthetic fixture tests' independently substituted anchors.
|
||||
pin = {
|
||||
'master': MASTER, 'masterSha256': MASTER_SHA256, 'parent': None,
|
||||
'source': SOURCE, 'library': LIBRARY, 'librarySha256': LIBRARY_SHA256,
|
||||
'gnArgs': GN_ARGS, 'gnArgsSha256': GN_ARGS_SHA256,
|
||||
'schemaVersion': 5, 'anchorName': 'master5',
|
||||
'recordPaths': {name: 'tests/results/pdfium-intent-build/' + name + '.json'
|
||||
for name in ('candidate-patch', 'source-materialization',
|
||||
'tool-materialization', 'provider-patches',
|
||||
'patched-final-build')},
|
||||
}
|
||||
elif candidate == 'v2':
|
||||
pin = dict(V2_PIN)
|
||||
else:
|
||||
raise StageError('Unknown fixed candidate selector')
|
||||
required = {'master', 'masterSha256', 'parent', 'source', 'library',
|
||||
'librarySha256', 'gnArgs', 'gnArgsSha256', 'schemaVersion',
|
||||
'anchorName', 'recordPaths'}
|
||||
require(set(pin) == required, 'Incomplete fixed candidate pin')
|
||||
for key in ('masterSha256', 'librarySha256', 'gnArgsSha256'):
|
||||
require(isinstance(pin[key], str) and re.fullmatch('[0-9a-f]{64}', pin[key]),
|
||||
'Fixed candidate pin is not ready: ' + key)
|
||||
for key in ('master', 'source', 'library', 'gnArgs'):
|
||||
relative(pin[key])
|
||||
schema = 5 if candidate == 'v1' else 6
|
||||
require(type(pin['schemaVersion']) is int and pin['schemaVersion'] == schema
|
||||
and pin['anchorName'] == 'master' + str(schema), 'Invalid fixed anchor identity')
|
||||
require(pin['library'] == pin['source'] + '/out/patched/libpdfium.so'
|
||||
and pin['gnArgs'] == pin['source'] + '/out/patched/args.gn',
|
||||
'Inconsistent fixed candidate source paths')
|
||||
names = {'candidate-patch', 'source-materialization', 'tool-materialization',
|
||||
'provider-patches', 'patched-final-build'}
|
||||
require(isinstance(pin['recordPaths'], dict) and set(pin['recordPaths']) == names,
|
||||
'Incomplete fixed candidate record paths')
|
||||
for path in pin['recordPaths'].values():
|
||||
relative(path)
|
||||
if candidate == 'v2':
|
||||
parent = pin['parent']
|
||||
require(isinstance(parent, dict) and set(parent) == {'path', 'sha256'},
|
||||
'Missing fixed parent anchor')
|
||||
relative(parent['path'])
|
||||
require(isinstance(parent['sha256'], str)
|
||||
and re.fullmatch('[0-9a-f]{64}', parent['sha256']),
|
||||
'Fixed parent anchor is not ready')
|
||||
return pin
|
||||
|
||||
|
||||
def open_directory(path):
|
||||
"""Open each component without following symlinks, including parent paths."""
|
||||
path = Path(os.path.abspath(path))
|
||||
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
for part in path.parts[1:]:
|
||||
nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
||||
os.close(fd)
|
||||
fd = nxt
|
||||
return fd
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
raise
|
||||
|
||||
|
||||
class Tree:
|
||||
def __init__(self, path):
|
||||
self.fd = open_directory(path)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
os.close(self.fd)
|
||||
|
||||
def read(self, name, expected=None, limit=MAX_FILE):
|
||||
parts = relative(name)
|
||||
fd = os.dup(self.fd)
|
||||
try:
|
||||
for part in parts[:-1]:
|
||||
nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
||||
os.close(fd)
|
||||
fd = nxt
|
||||
file_fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK,
|
||||
dir_fd=fd)
|
||||
with os.fdopen(file_fd, 'rb') as stream:
|
||||
info = os.fstat(stream.fileno())
|
||||
require(stat.S_ISREG(info.st_mode) and info.st_size <= limit,
|
||||
'Not a bounded regular file: ' + name)
|
||||
data = stream.read(limit + 1)
|
||||
require(len(data) <= limit, 'File exceeds limit: ' + name)
|
||||
if expected is not None:
|
||||
require(re.fullmatch('[0-9a-f]{64}', expected) is not None,
|
||||
'Invalid SHA-256: ' + name)
|
||||
require(digest(data) == expected, 'SHA-256 mismatch: ' + name)
|
||||
return data
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def json_bytes(data):
|
||||
def unique(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
require(key not in result, 'Duplicate JSON key: ' + key)
|
||||
result[key] = value
|
||||
return result
|
||||
return json.loads(data, object_pairs_hook=unique)
|
||||
|
||||
|
||||
def undo_header_patch(data, patch, header):
|
||||
"""Reverse only exact header hunks; never execute a patch or its filenames."""
|
||||
target = 'public/' + header
|
||||
hunks = []
|
||||
active = False
|
||||
old = new = None
|
||||
for line in patch.decode('utf-8').splitlines(keepends=True):
|
||||
if line.startswith('diff --git '):
|
||||
if old is not None:
|
||||
hunks.append((old, new))
|
||||
old = new = None
|
||||
active = False
|
||||
elif line.startswith('+++ b/'):
|
||||
active = line.removeprefix('+++ b/').rstrip('\n') == target
|
||||
elif active and line.startswith('@@ '):
|
||||
if old is not None:
|
||||
hunks.append((old, new))
|
||||
old, new = [], []
|
||||
elif active and old is not None:
|
||||
if line.startswith((' ', '-')):
|
||||
old.append(line[1:].encode())
|
||||
if line.startswith((' ', '+')):
|
||||
new.append(line[1:].encode())
|
||||
if old is not None:
|
||||
hunks.append((old, new))
|
||||
lines = data.splitlines(keepends=True)
|
||||
for old, new in reversed(hunks):
|
||||
require(new, 'Empty reverse patch hunk')
|
||||
positions = [i for i in range(len(lines) - len(new) + 1)
|
||||
if lines[i:i + len(new)] == new]
|
||||
require(len(positions) == 1, 'Header patch context mismatch: ' + header)
|
||||
i = positions[0]
|
||||
lines[i:i + len(new)] = old
|
||||
return b''.join(lines)
|
||||
|
||||
|
||||
def verify(repo=ROOT, candidate='v1'):
|
||||
"""Return fully verified, bounded output bytes. This performs no writes."""
|
||||
pin = candidate_pin(candidate)
|
||||
output = {}
|
||||
metadata = {}
|
||||
total = 0
|
||||
with Tree(repo) as tree:
|
||||
def add(destination, origin, expected, category, **details):
|
||||
nonlocal total
|
||||
relative(destination)
|
||||
require(destination not in output, 'Duplicate output: ' + destination)
|
||||
data = tree.read(origin, expected)
|
||||
total += len(data)
|
||||
require(len(output) < 256 and total <= MAX_OUTPUT, 'Output limit exceeded')
|
||||
output[destination] = data
|
||||
metadata[destination] = {'source': origin, 'sha256': digest(data),
|
||||
'bytes': len(data), 'category': category, **details}
|
||||
return data
|
||||
|
||||
master_bytes = add('provenance/' + pin['anchorName'] + '.json',
|
||||
pin['master'], pin['masterSha256'], 'evidence')
|
||||
master = json_bytes(master_bytes)
|
||||
require(master['schemaVersion'] == pin['schemaVersion'],
|
||||
'Not the fixed ' + pin['anchorName'] + ' record')
|
||||
if pin['parent']:
|
||||
require(master.get('parent') == pin['parent'], 'Parent anchor identity mismatch')
|
||||
add('provenance/parent-master5.json', pin['parent']['path'],
|
||||
pin['parent']['sha256'], 'evidence')
|
||||
candidate = master['pdfiumRenderingIntentCandidate']
|
||||
require(candidate['sourceRevision'] == REVISION
|
||||
and candidate['librarySha256'] == pin['librarySha256']
|
||||
and candidate['productionDependencyReplaced'] is False,
|
||||
'Candidate does not match the independently fixed anchor')
|
||||
# The old record contains an absolute source path. It is checked as data,
|
||||
# never used to choose which library the tool copies.
|
||||
require(candidate['library'].endswith('/' + pin['library']), 'Unexpected recorded library')
|
||||
|
||||
def record(name):
|
||||
path = pin['recordPaths'][name]
|
||||
return json_bytes(add('provenance/' + name + '.json', path,
|
||||
master['evidenceSha256'][path], 'evidence'))
|
||||
|
||||
patch = record('candidate-patch')
|
||||
material = record('source-materialization')
|
||||
tools = record('tool-materialization')
|
||||
provider = record('provider-patches')
|
||||
build = record('patched-final-build')
|
||||
require(patch['success'] is True and material['success'] is True
|
||||
and tools['success'] is True and build['exitCode'] == 0,
|
||||
'Incomplete candidate build evidence')
|
||||
require(patch['baseRevision'] == REVISION
|
||||
and patch['patchSha256'] == candidate['patchSha256']
|
||||
and patch['patchSnapshot'] == candidate['patchSnapshot'],
|
||||
'Patch evidence mismatch')
|
||||
add('provenance/rendering-intent.patch', candidate['patchSnapshot'],
|
||||
candidate['patchSha256'], 'patch')
|
||||
tree.read(candidate['patch'], candidate['patchSha256'])
|
||||
add('provenance/candidate-report.json', candidate['report'],
|
||||
candidate['reportSha256'], 'evidence')
|
||||
for row in patch['files']:
|
||||
tree.read(pin['source'] + '/' + '/'.join(relative(row['path'])), row['afterSha256'])
|
||||
require(0 < len(patch['files']) <= 128, 'Invalid changed-source count')
|
||||
|
||||
repositories = {row['path']: row for row in material['repositories']}
|
||||
upstream = repositories['.']
|
||||
require(upstream['revision'] == REVISION
|
||||
and upstream['archiveSha256'] == patch['archiveSha256'],
|
||||
'Source revision/archive mismatch')
|
||||
inventory_data = tree.read(upstream['inventory'], upstream['inventorySha256'], 16 * 1024**2)
|
||||
inventory = {}
|
||||
for line in inventory_data.splitlines():
|
||||
row = json_bytes(line)
|
||||
require(row['path'] not in inventory, 'Duplicate source inventory path')
|
||||
inventory[row['path']] = row
|
||||
require(len(inventory) <= 100000, 'Source inventory limit exceeded')
|
||||
for tool in tools['tools']:
|
||||
tree.read(tool['archive'], tool['sha256'])
|
||||
require(set(tools['versions']) == {'gn', 'clang', 'lld'}, 'Missing tool versions')
|
||||
args = add('args.gn', pin['gnArgs'], pin['gnArgsSha256'], 'build-options')
|
||||
add('lib/libpdfium.so', pin['library'], pin['librarySha256'], 'library')
|
||||
add('VERSION', '.deps/pdfium/VERSION', VERSION_SHA256, 'version')
|
||||
|
||||
patches = []
|
||||
for row in provider:
|
||||
require(row['exitCode'] == 0, 'Provider patch did not apply')
|
||||
data = add('provenance/provider-' + PurePosixPath(row['patch']).name,
|
||||
row['patch'], row['sha256'], 'patch')
|
||||
patches.append(data)
|
||||
headers = sorted(k for k, r in inventory.items()
|
||||
if k.startswith('public/') and k.endswith('.h')
|
||||
and r['archived'] and r['mode'] == '100644')
|
||||
require(0 < len(headers) <= 128 and 'public/fpdfview.h' in headers,
|
||||
'Missing public header inventory')
|
||||
for public in headers:
|
||||
header = public.removeprefix('public/')
|
||||
current = tree.read(pin['source'] + '/' + public, limit=4 * 1024**2)
|
||||
original = current
|
||||
for data in reversed(patches):
|
||||
original = undo_header_patch(original, data, header)
|
||||
require(digest(original) == inventory[public]['sha256'],
|
||||
'Header differs from fixed source and provider patches: ' + header)
|
||||
add('include/' + header, '.deps/pdfium/include/' + header,
|
||||
digest(current), 'header', upstreamSha256=digest(original))
|
||||
|
||||
licenses = json_bytes(add('provenance/license-correspondence.json', LICENSE_RECORD,
|
||||
LICENSE_RECORD_SHA256, 'evidence'))
|
||||
require(len(licenses) == 15, 'Unexpected provider notice count')
|
||||
for row in licenses:
|
||||
require(row['exactMatch'] is True
|
||||
and row['packagedSha256'] == row['transformedSha256'],
|
||||
'Notice correspondence not established')
|
||||
tree.read('tests/results/source-correspondence/pdfium/' + row['sourceFile'],
|
||||
row['sourceSha256'])
|
||||
if row['sourceFile'].startswith('upstream/'):
|
||||
source_path = row['sourceFile'].removeprefix('upstream/')
|
||||
elif row['sourceFile'].startswith('dependencies/'):
|
||||
source_path = row['sourceFile'].removeprefix('dependencies/')
|
||||
else:
|
||||
source_path = None # Provider packaging notice, not PDFium source.
|
||||
if source_path:
|
||||
tree.read(pin['source'] + '/' + source_path, row['sourceSha256'])
|
||||
add(row['packagedFile'], '.deps/pdfium/' + row['packagedFile'],
|
||||
row['packagedSha256'], 'license', correspondence=row)
|
||||
|
||||
supplement_path = 'resources/licenses/pdfium-supplemental/sources.json'
|
||||
supplement = json_bytes(add('provenance/supplemental-notices.json', supplement_path,
|
||||
master['sourceSha256'][supplement_path], 'evidence'))
|
||||
require(supplement['pdfiumUpstreamCommit'] == REVISION, 'Supplement revision mismatch')
|
||||
components = {'libc++': 'third_party/libc++/src', 'libc++abi': 'third_party/libc++abi/src'}
|
||||
require(len(supplement['files']) == len(components), 'Supplement count mismatch')
|
||||
for row in supplement['files']:
|
||||
component = components[row['component']]
|
||||
require(repositories[component]['revision'] == row['sourceRevision'],
|
||||
'Supplement dependency revision mismatch')
|
||||
tree.read(pin['source'] + '/' + component + '/LICENSE.TXT', row['sha256'])
|
||||
path = 'resources/licenses/pdfium-supplemental/' + row['name']
|
||||
require(master['sourceSha256'][path] == row['sha256'], 'Supplement anchor mismatch')
|
||||
add('licenses/' + row['name'], path, row['sha256'], 'license',
|
||||
correspondence={'sourceRevision': row['sourceRevision'], 'sourceUrl': row['sourceUrl']})
|
||||
|
||||
info = {'schemaVersion': 1, 'candidateOnly': True, 'productionDependencyReplaced': False,
|
||||
'sourceRevision': REVISION,
|
||||
pin['anchorName']: {'path': pin['master'], 'sha256': pin['masterSha256']},
|
||||
'librarySha256': pin['librarySha256'], 'patchSnapshot': candidate['patchSnapshot'],
|
||||
'patchSha256': candidate['patchSha256'], 'sourceRepositories': material['repositories'],
|
||||
'changedSources': patch['files'], 'gnArgs': args.decode('utf-8'),
|
||||
'gnArgsSha256': pin['gnArgsSha256'], 'toolVersions': tools['versions'],
|
||||
'toolArchives': tools['tools'], 'files': metadata,
|
||||
'stagerSha256': digest(Path(__file__).read_bytes()),
|
||||
'scope': 'Fixed Linux x64 CMake candidate prefix only. No build, install, publication, '
|
||||
'human rendering acceptance, or complete license compliance is asserted.'}
|
||||
if pin['parent']:
|
||||
info['parent'] = dict(pin['parent'])
|
||||
output['BUILDINFO.json'] = (json.dumps(info, ensure_ascii=False, indent=2) + '\n').encode()
|
||||
return output, info
|
||||
|
||||
|
||||
def write_files(fd, files):
|
||||
for name, data in sorted(files.items()):
|
||||
parts = relative(name)
|
||||
parent = os.dup(fd)
|
||||
try:
|
||||
for part in parts[:-1]:
|
||||
try:
|
||||
os.mkdir(part, 0o755, dir_fd=parent)
|
||||
except FileExistsError:
|
||||
pass
|
||||
nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)
|
||||
os.close(parent)
|
||||
parent = nxt
|
||||
mode = 0o755 if name == 'lib/libpdfium.so' else 0o644
|
||||
file_fd = os.open(parts[-1], os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
||||
mode, dir_fd=parent)
|
||||
with os.fdopen(file_fd, 'wb') as stream:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
finally:
|
||||
os.close(parent)
|
||||
|
||||
|
||||
def publish(parent_fd, temporary, destination):
|
||||
"""Linux atomic no-replace publication: even an empty existing dir is protected."""
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
rename = getattr(libc, 'renameat2', None)
|
||||
require(rename is not None, 'Atomic no-replace rename is unavailable')
|
||||
rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
||||
rename.restype = ctypes.c_int
|
||||
if rename(parent_fd, os.fsencode(temporary), parent_fd, os.fsencode(destination), 1):
|
||||
error = ctypes.get_errno()
|
||||
raise OSError(error, os.strerror(error), destination)
|
||||
|
||||
|
||||
def stage(output, repo=ROOT, candidate='v1'):
|
||||
require(sys.platform.startswith('linux'), 'This fixed candidate is Linux-only')
|
||||
output = Path(os.path.abspath(output))
|
||||
parent_fd = open_directory(output.parent)
|
||||
temporary = '.pdfium-candidate-' + secrets.token_hex(12)
|
||||
created = False
|
||||
try:
|
||||
try:
|
||||
os.stat(output.name, dir_fd=parent_fd, follow_symlinks=False)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
raise StageError('Output already exists: ' + str(output))
|
||||
files, info = verify(repo, candidate=candidate)
|
||||
os.mkdir(temporary, 0o700, dir_fd=parent_fd)
|
||||
created = True
|
||||
fd = os.open(temporary, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd)
|
||||
try:
|
||||
write_files(fd, files)
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
publish(parent_fd, temporary, output.name)
|
||||
created = False
|
||||
os.fsync(parent_fd)
|
||||
return info
|
||||
finally:
|
||||
if created:
|
||||
shutil.rmtree(temporary, dir_fd=parent_fd)
|
||||
os.close(parent_fd)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, help='New prefix; its parent must already exist')
|
||||
parser.add_argument('--verify-only', action='store_true', help='Check inputs without writing a prefix')
|
||||
parser.add_argument('--candidate', choices=('v1', 'v2'), default='v1',
|
||||
help='Select a fixed internal candidate pin (default: v1)')
|
||||
args = parser.parse_args()
|
||||
if args.verify_only == bool(args.output):
|
||||
parser.error('Choose exactly one of --output or --verify-only')
|
||||
try:
|
||||
info = (verify(candidate=args.candidate)[1] if args.verify_only
|
||||
else stage(args.output, candidate=args.candidate))
|
||||
except (StageError, OSError, KeyError, TypeError, json.JSONDecodeError) as error:
|
||||
print('PDFium candidate rejected: ' + str(error), file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({'verified': True, 'staged': not args.verify_only,
|
||||
'librarySha256': info['librarySha256'],
|
||||
'files': len(info['files']), 'candidateOnly': True}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify the one reviewed Linux candidate, or the unchanged provider install.
|
||||
|
||||
The reviewed-v2 lock is a repository input, never supplied by the package being
|
||||
checked. An absent lock deliberately prevents candidate configuration/packaging.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
from stage_pdfium_candidate import Tree, digest, json_bytes, relative, require
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCK = 'cmake/pdfium-candidate-v2.lock.json'
|
||||
DOC = 'share/doc/docview/pdfium/'
|
||||
CANDIDATE = DOC + 'candidate/'
|
||||
SUPPLEMENT = DOC + 'supplemental/'
|
||||
MAX_METADATA = 4 * 1024 * 1024
|
||||
MAX_TOTAL = 128 * 1024 * 1024
|
||||
METADATA_PATHS = (CANDIDATE + 'BUILDINFO.json', SUPPLEMENT + 'sources.json')
|
||||
|
||||
|
||||
def root_path(root):
|
||||
return Path(root) if root is not None else ROOT
|
||||
|
||||
|
||||
def source_metadata(root=None):
|
||||
with Tree(root_path(root)) as tree:
|
||||
raw = tree.read('resources/licenses/pdfium-supplemental/sources.json', limit=65536)
|
||||
source = json_bytes(raw)
|
||||
upstream = json_bytes(tree.read('cmake/pdfium.lock.json', limit=65536))
|
||||
require(source['pdfiumVersion'] == upstream['version'] and
|
||||
source['pdfiumUpstreamCommit'] == upstream['upstreamCommit'],
|
||||
'Provider source pin differs from supplemental notices')
|
||||
return raw, source
|
||||
|
||||
|
||||
def reviewed_lock(root=None):
|
||||
with Tree(root_path(root)) as tree:
|
||||
try:
|
||||
raw = tree.read(LOCK, limit=65536)
|
||||
except FileNotFoundError as error:
|
||||
raise ValueError('The reviewed-v2 PDFium lock is not available; candidate rejected') from error
|
||||
lock = json_bytes(raw)
|
||||
hashes = ('librarySha256', 'buildInfoSha256', 'anchorRecordSha256', 'patchSha256',
|
||||
'gnArgsSha256', 'supplementalSourceSha256')
|
||||
require(set(lock) == {'schemaVersion', 'candidateId', 'sourceRevision', 'anchorRecordPath', *hashes}
|
||||
and lock['schemaVersion'] == 1 and lock['candidateId'] == 'reviewed-v2',
|
||||
'Unexpected reviewed PDFium candidate lock')
|
||||
for field in hashes:
|
||||
require(isinstance(lock[field], str) and re.fullmatch('[0-9a-f]{64}', lock[field]),
|
||||
'Invalid candidate digest: ' + field)
|
||||
require(isinstance(lock['sourceRevision'], str) and
|
||||
re.fullmatch('[0-9a-f]{40}', lock['sourceRevision']), 'Invalid candidate revision')
|
||||
relative(lock['anchorRecordPath'])
|
||||
require(re.fullmatch(r'provenance/[A-Za-z0-9_-]+[.]json', lock['anchorRecordPath']),
|
||||
'Invalid candidate anchor path')
|
||||
base_raw, base = source_metadata(root)
|
||||
require(lock['sourceRevision'] == base['pdfiumUpstreamCommit'] and
|
||||
lock['supplementalSourceSha256'] == digest(base_raw),
|
||||
'Candidate notices do not match the reviewed source pin')
|
||||
return lock
|
||||
|
||||
|
||||
def buildinfo(raw, lock):
|
||||
require(len(raw) <= MAX_METADATA and digest(raw) == lock['buildInfoSha256'],
|
||||
'Candidate BUILDINFO differs from reviewed-v2')
|
||||
info = json_bytes(raw)
|
||||
require(info['schemaVersion'] == 1 and info['candidateOnly'] is True and
|
||||
info['sourceRevision'] == lock['sourceRevision'] and
|
||||
info['librarySha256'] == lock['librarySha256'] and
|
||||
info['patchSha256'] == lock['patchSha256'] and
|
||||
info['gnArgsSha256'] == lock['gnArgsSha256'], 'Candidate BUILDINFO identity mismatch')
|
||||
files = info['files']
|
||||
require(isinstance(files, dict) and 1 <= len(files) <= 256, 'Invalid candidate file list')
|
||||
total = 0
|
||||
for name, row in files.items():
|
||||
relative(name)
|
||||
require(re.fullmatch(r'[A-Za-z0-9_./+-]+', name) and name != 'BUILDINFO.json' and
|
||||
(name in ('lib/libpdfium.so', 'LICENSE', 'VERSION', 'args.gn') or
|
||||
name.startswith(('include/', 'licenses/', 'provenance/'))),
|
||||
'Unexpected candidate file path')
|
||||
require(isinstance(row, dict) and isinstance(row.get('sha256'), str) and
|
||||
re.fullmatch('[0-9a-f]{64}', row['sha256']) and type(row.get('bytes')) is int and
|
||||
0 <= row['bytes'] <= MAX_TOTAL, 'Invalid candidate file identity')
|
||||
total += row['bytes']
|
||||
require(total <= MAX_TOTAL and 'include/fpdfview.h' in files and 'LICENSE' in files and
|
||||
'VERSION' in files, 'Incomplete or oversized candidate prefix')
|
||||
for path, expected in {
|
||||
'lib/libpdfium.so': lock['librarySha256'], 'args.gn': lock['gnArgsSha256'],
|
||||
lock['anchorRecordPath']: lock['anchorRecordSha256'],
|
||||
'provenance/rendering-intent.patch': lock['patchSha256'],
|
||||
'provenance/supplemental-notices.json': lock['supplementalSourceSha256']}.items():
|
||||
require(files.get(path, {}).get('sha256') == expected, 'Missing candidate correspondence: ' + path)
|
||||
return info
|
||||
|
||||
|
||||
def installed_path(name):
|
||||
relative(name)
|
||||
if name == 'lib/libpdfium.so':
|
||||
return name
|
||||
if name == 'LICENSE' or name.startswith('licenses/'):
|
||||
return DOC + name
|
||||
return CANDIDATE + name
|
||||
|
||||
|
||||
def candidate_supplement(lock, base_raw):
|
||||
value = json_bytes(base_raw)
|
||||
value['pdfiumLibrarySha256'] = lock['librarySha256']
|
||||
value['scope'] = ('Fixed-source supplemental notices for the reviewed-v2 PDFium candidate. '
|
||||
'The provider source notice record is retained unchanged in candidate/provenance. '
|
||||
'This is not human rendering approval or a complete license assessment.')
|
||||
value['candidateCorrespondence'] = {key: lock[key] for key in
|
||||
('candidateId', 'buildInfoSha256', 'anchorRecordSha256', 'patchSha256', 'supplementalSourceSha256')}
|
||||
return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode()
|
||||
|
||||
|
||||
def verify_rows(raw, rows, lock, installed=False):
|
||||
info = buildinfo(raw, lock)
|
||||
expected = {}
|
||||
for name, entry in info['files'].items():
|
||||
path = installed_path(name) if installed else name
|
||||
expected[path] = {'sha256': entry['sha256'], 'size': entry['bytes']}
|
||||
expected[(CANDIDATE if installed else '') + 'BUILDINFO.json'] = {
|
||||
'sha256': lock['buildInfoSha256'], 'size': len(raw)}
|
||||
for name, entry in expected.items():
|
||||
actual = rows.get(name, {})
|
||||
require(all(actual.get(field) == value for field, value in entry.items()),
|
||||
'Candidate file missing or changed: ' + name)
|
||||
if installed:
|
||||
require({p for p in rows if p.startswith(CANDIDATE)} ==
|
||||
{p for p in expected if p.startswith(CANDIDATE)}, 'Unregistered installed candidate metadata')
|
||||
else:
|
||||
require(set(rows) == set(expected), 'Unregistered candidate prefix file')
|
||||
return info
|
||||
|
||||
|
||||
def verify_payload(rows, contents, root=None):
|
||||
"""Check hashed files plus bounded metadata read from an install or a deb."""
|
||||
base_raw, base = source_metadata(root)
|
||||
library_hash = rows.get('lib/libpdfium.so', {}).get('sha256')
|
||||
is_candidate = any(name.startswith(CANDIDATE) for name in rows)
|
||||
if is_candidate:
|
||||
lock = reviewed_lock(root)
|
||||
verify_rows(contents.get(CANDIDATE + 'BUILDINFO.json', b''), rows, lock, installed=True)
|
||||
require(library_hash == lock['librarySha256'], 'Installed candidate library differs from reviewed-v2')
|
||||
expected_raw = candidate_supplement(lock, base_raw)
|
||||
identity = {'kind': 'reviewed-candidate', 'candidateId': lock['candidateId'],
|
||||
'buildInfoSha256': lock['buildInfoSha256'],
|
||||
'anchorRecordSha256': lock['anchorRecordSha256'], 'patchSha256': lock['patchSha256']}
|
||||
else:
|
||||
require(library_hash == base['pdfiumLibrarySha256'],
|
||||
'Unknown PDFium library or missing candidate provenance')
|
||||
expected_raw = base_raw
|
||||
identity = {'kind': 'original-provider'}
|
||||
require(contents.get(SUPPLEMENT + 'sources.json') == expected_raw,
|
||||
'Installed supplemental notice correspondence differs from reviewed input')
|
||||
require(len(base['files']) == 2 and {row['name'] for row in base['files']} ==
|
||||
{'libcxx-LICENSE.txt', 'libcxxabi-LICENSE.txt'}, 'Unexpected supplemental notice set')
|
||||
for row in base['files']:
|
||||
item = rows.get(SUPPLEMENT + row['name'], {})
|
||||
require(item.get('sha256') == row['sha256'] and item.get('size') == row['size'],
|
||||
'Installed supplemental notice text differs from reviewed input')
|
||||
if is_candidate:
|
||||
candidate_item = rows.get(DOC + 'licenses/' + row['name'], {})
|
||||
require(candidate_item.get('sha256') == row['sha256'] and candidate_item.get('size') == row['size'],
|
||||
'Candidate notice copy differs from reviewed source')
|
||||
return {**identity, 'librarySha256': library_hash, 'sourceRevision': base['pdfiumUpstreamCommit'],
|
||||
'supplementalManifestSha256': digest(expected_raw)}
|
||||
|
||||
|
||||
def files_beneath(path):
|
||||
"""Finite list without following any directory or file links."""
|
||||
pending, files, count = [Path(path)], [], 0
|
||||
while pending:
|
||||
directory = pending.pop()
|
||||
with os.scandir(directory) as entries:
|
||||
for entry in entries:
|
||||
count += 1
|
||||
require(count <= 512, 'Candidate file count exceeds limit')
|
||||
require(not entry.is_symlink(), 'Candidate links are not accepted')
|
||||
if entry.is_dir(follow_symlinks=False): pending.append(Path(entry.path))
|
||||
else:
|
||||
require(entry.is_file(follow_symlinks=False), 'Candidate special file rejected')
|
||||
files.append(Path(entry.path).relative_to(path).as_posix())
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def verify_prefix(prefix, library, include_dir, root=None):
|
||||
prefix = Path(os.path.abspath(prefix))
|
||||
require(Path(os.path.abspath(library)) == prefix / 'lib/libpdfium.so' and
|
||||
Path(os.path.abspath(include_dir)) == prefix / 'include',
|
||||
'CMake PDFium library/headers differ from the selected candidate prefix')
|
||||
lock = reviewed_lock(root)
|
||||
rows = {}
|
||||
with Tree(prefix) as tree:
|
||||
raw = tree.read('BUILDINFO.json', limit=MAX_METADATA)
|
||||
info = buildinfo(raw, lock)
|
||||
names = files_beneath(prefix)
|
||||
require(set(names) == set(info['files']) | {'BUILDINFO.json'}, 'Unregistered candidate prefix file')
|
||||
for name in names:
|
||||
data = tree.read(name, limit=MAX_TOTAL)
|
||||
rows[name] = {'sha256': digest(data), 'size': len(data)}
|
||||
verify_rows(raw, rows, lock)
|
||||
base_raw, base = source_metadata(root)
|
||||
for row in base['files']:
|
||||
require(rows.get('licenses/' + row['name']) == {'sha256': row['sha256'], 'size': row['size']},
|
||||
'Candidate supplemental notice differs from fixed source')
|
||||
return {'candidateId': lock['candidateId'], 'librarySha256': lock['librarySha256'],
|
||||
'buildInfoSha256': lock['buildInfoSha256'],
|
||||
'installFiles': [{'source': name, 'destination': installed_path(name)}
|
||||
for name in sorted(info['files'])
|
||||
if name != 'lib/libpdfium.so' and name != 'LICENSE' and not name.startswith('licenses/')]
|
||||
+ [{'source': 'BUILDINFO.json', 'destination': CANDIDATE + 'BUILDINFO.json'}]}, \
|
||||
candidate_supplement(lock, base_raw)
|
||||
|
||||
|
||||
def verify_installed(prefix, root=None):
|
||||
prefix = Path(os.path.abspath(prefix))
|
||||
rows, contents, total = {}, {}, 0
|
||||
with Tree(prefix) as tree:
|
||||
paths = ['lib/libpdfium.so'] + [DOC + name for name in files_beneath(prefix / DOC)]
|
||||
for name in paths:
|
||||
data = tree.read(name, limit=MAX_TOTAL)
|
||||
total += len(data)
|
||||
require(total <= MAX_TOTAL, 'Installed PDFium correspondence exceeds limit')
|
||||
rows[name] = {'sha256': digest(data), 'size': len(data)}
|
||||
if name in METADATA_PATHS:
|
||||
require(len(data) <= MAX_METADATA, 'Installed PDFium metadata exceeds limit')
|
||||
contents[name] = data
|
||||
return verify_payload(rows, contents, root)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--prefix', type=Path, required=True)
|
||||
parser.add_argument('--library', type=Path, required=True)
|
||||
parser.add_argument('--include-dir', type=Path, required=True)
|
||||
parser.add_argument('--notice-output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
report, supplemental = verify_prefix(args.prefix, args.library, args.include_dir)
|
||||
# Only CMake's own generated metadata is written, after all inputs pass.
|
||||
args.notice_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.notice_output.write_bytes(supplemental)
|
||||
print(json.dumps(report))
|
||||
return 0
|
||||
except (ValueError, OSError, KeyError, TypeError) as error:
|
||||
print('PDFium candidate rejected: ' + str(error), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read a local DocView deb and verify every data/control payload before installation."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tarfile
|
||||
import threading
|
||||
|
||||
from package_ubuntu import MANIFEST, canonical, sha
|
||||
from verify_pdfium_candidate import METADATA_PATHS as PDFIUM_METADATA_PATHS, MAX_METADATA, verify_payload as verify_pdfium_payload
|
||||
|
||||
RUNTIME_RECORD = 'share/doc/docview/third-party/runtime/runtime.json'
|
||||
METADATA_PATHS = (*PDFIUM_METADATA_PATHS, RUNTIME_RECORD)
|
||||
|
||||
|
||||
def inspect(archive, flag):
|
||||
rows, manifest, total, metadata = {}, None, 0, {}
|
||||
process = subprocess.Popen(['dpkg-deb', flag, str(archive)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
timer = threading.Timer(180, process.kill); timer.start()
|
||||
try:
|
||||
with tarfile.open(fileobj=process.stdout, mode='r|') as tar:
|
||||
for member in tar:
|
||||
if member.name == '.' and member.isdir(): continue
|
||||
name = str(canonical(member.name.removeprefix('./').rstrip('/')))
|
||||
if member.uid or member.gid:
|
||||
raise ValueError('Non-root archive ownership')
|
||||
if member.isdir(): continue
|
||||
if name in rows or len(rows) >= 10000 or not member.isfile() or member.mode not in (0o644, 0o755):
|
||||
raise ValueError('Unsafe or duplicate deb payload entry: ' + name + ' mode=' + oct(member.mode))
|
||||
total += member.size
|
||||
if member.size > 512 * 1024**2 or total > 2 * 1024**3:
|
||||
raise ValueError('Deb payload exceeds inspection limits')
|
||||
digest = hashlib.sha256(); chunks = []
|
||||
with tar.extractfile(member) as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
digest.update(chunk)
|
||||
if name == MANIFEST or name.removeprefix('opt/docview/') in METADATA_PATHS or (flag == '--ctrl-tarfile' and name == 'control'):
|
||||
maximum = 8 * 1024**2 if name == MANIFEST else (65536 if name == 'control' else MAX_METADATA)
|
||||
if member.size > maximum: raise ValueError('Oversized manifest or PDFium correspondence')
|
||||
chunks.append(chunk)
|
||||
rows[name] = {'path': name, 'size': member.size, 'sha256': digest.hexdigest(), 'mode': oct(member.mode)}
|
||||
if name == MANIFEST:
|
||||
manifest = json.loads(b''.join(chunks))
|
||||
elif flag == '--ctrl-tarfile' and name == 'control':
|
||||
metadata['control'] = b''.join(chunks)
|
||||
elif name.removeprefix('opt/docview/') in METADATA_PATHS:
|
||||
metadata[name.removeprefix('opt/docview/')] = b''.join(chunks)
|
||||
if process.wait(timeout=15):
|
||||
raise ValueError('dpkg-deb failed reading package')
|
||||
finally:
|
||||
timer.cancel()
|
||||
if process.poll() is None: process.kill(); process.wait()
|
||||
process.stdout.close()
|
||||
process.stderr.close()
|
||||
return rows, manifest, metadata
|
||||
|
||||
|
||||
|
||||
def verify_qpdf_payload(data, metadata, manifest):
|
||||
"""Bind qpdf notice claims to fixed source identities and actual bytes.
|
||||
|
||||
Earlier deficient archives remain historical evidence. Raw inventory can
|
||||
compare them, but this current verifier does not certify their omission.
|
||||
"""
|
||||
summary = manifest.get('qpdfNoticeCorrespondence')
|
||||
from collect_ubuntu_validation_runtime import QPDF_NOTICE_PIN as pin
|
||||
if not isinstance(summary, dict) or summary.get('status') != 'verified':
|
||||
raise ValueError('Missing verified qpdf notice correspondence')
|
||||
try:
|
||||
runtime = json.loads(metadata[RUNTIME_RECORD])
|
||||
except (KeyError, ValueError, TypeError) as error:
|
||||
raise ValueError('Missing or invalid packaged runtime record') from error
|
||||
if runtime.get('qpdfNoticeCorrespondence') != summary:
|
||||
raise ValueError('qpdf correspondence differs between manifest and runtime record')
|
||||
for key in ('version', 'archive', 'sourceFiles', 'sourceBytes', 'sourceInventorySha256'):
|
||||
if summary.get(key) != pin[key]:
|
||||
raise ValueError('qpdf fixed source correspondence differs: ' + key)
|
||||
source_root = summary.get('sourceRoot')
|
||||
if not isinstance(source_root, str) or not re.fullmatch(r'source-[0-9]{1,3}:', source_root):
|
||||
raise ValueError('Invalid qpdf source root identity')
|
||||
library = summary.get('library')
|
||||
if not isinstance(library, dict):
|
||||
raise ValueError('Missing qpdf library identity')
|
||||
path = library.get('path', '')
|
||||
if not isinstance(path, str) or not re.fullmatch(r'dependencies:lib/libqpdf[.]so(?:[.][0-9]+)*', path):
|
||||
raise ValueError('Invalid qpdf runtime path')
|
||||
library_name = 'opt/docview/' + path.split(':', 1)[1]
|
||||
def matches(row, expected):
|
||||
return isinstance(row, dict) and all(row.get(key) == expected[key] for key in ('sha256', 'size'))
|
||||
if not matches(library, pin['library']) or not matches(data.get(library_name), pin['library']):
|
||||
raise ValueError('Packaged qpdf library differs from the reviewed runtime')
|
||||
# The inspected executables load SONAME libqpdf.so.30. A correct, unused
|
||||
# alias cannot authorize a different library at that actual loader path.
|
||||
if not matches(data.get('opt/docview/lib/libqpdf.so.30'), pin['library']):
|
||||
raise ValueError('Packaged qpdf SONAME library differs from the reviewed runtime')
|
||||
for name, row in data.items():
|
||||
if re.fullmatch(r'opt/docview/lib/libqpdf[.]so(?:[.][0-9]+)*', name) and not matches(row, pin['library']):
|
||||
raise ValueError('Conflicting packaged qpdf runtime alias')
|
||||
notices = summary.get('notices')
|
||||
if not isinstance(notices, list) or len(notices) != len(pin['notices']):
|
||||
raise ValueError('Missing qpdf source notices')
|
||||
by_name = {row.get('source'): row for row in notices if isinstance(row, dict)}
|
||||
if len(by_name) != len(notices):
|
||||
raise ValueError('Invalid or duplicate qpdf source notice')
|
||||
for source, expected in pin['notices'].items():
|
||||
row = by_name.get(source)
|
||||
expected_path = 'notices/' + expected['sha256'] + '.txt'
|
||||
if not matches(row, expected) or row.get('copiedPath') != expected_path:
|
||||
raise ValueError('qpdf source notice correspondence differs')
|
||||
name = 'opt/docview/share/doc/docview/third-party/runtime/' + expected_path
|
||||
if not matches(data.get(name), expected):
|
||||
raise ValueError('Packaged qpdf source notice is absent or changed')
|
||||
return summary
|
||||
|
||||
|
||||
def verify(archive):
|
||||
data, manifest, metadata = inspect(archive, '--fsys-tarfile')
|
||||
controls, _, control_metadata = inspect(archive, '--ctrl-tarfile')
|
||||
actual = dict(data)
|
||||
actual.pop(MANIFEST)
|
||||
actual.update({'DEBIAN/' + name: dict(row, path='DEBIAN/' + name) for name, row in controls.items()})
|
||||
if not manifest or manifest['schemaVersion'] != 1 or manifest['package'] != 'docview':
|
||||
raise ValueError('Missing or invalid DocView manifest')
|
||||
control_text = control_metadata.get('control', b'').decode('utf-8', 'strict')
|
||||
for field, expected_value in [('Package', 'docview'), ('Version', manifest.get('version'))]:
|
||||
values = re.findall(r'^' + field + r': ([^\r\n]+)$', control_text, re.M)
|
||||
if not isinstance(expected_value, str) or values != [expected_value]:
|
||||
raise ValueError('Deb control identity differs from manifest: ' + field)
|
||||
expected = {str(canonical(row['path'])): row for row in manifest['files']}
|
||||
if len(expected) != len(manifest['files']) or actual != expected:
|
||||
raise ValueError('Deb payload and manifest differ')
|
||||
pdfium = verify_pdfium_payload({name.removeprefix('opt/docview/'): row for name, row in data.items()
|
||||
if name.startswith('opt/docview/')}, metadata)
|
||||
# Legacy provider packages predate this field; they still undergo the same
|
||||
# pinned library/notice check. A candidate never gets that legacy exception.
|
||||
if (pdfium['kind'] == 'reviewed-candidate' or 'pdfiumCorrespondence' in manifest) and manifest.get('pdfiumCorrespondence') != pdfium:
|
||||
raise ValueError('Deb PDFium correspondence summary differs from reviewed payload')
|
||||
archive_sha = sha(archive)
|
||||
qpdf = verify_qpdf_payload(data, metadata, manifest)
|
||||
return {'success': True, 'archiveSha256': archive_sha, 'archiveBytes': archive.stat().st_size,
|
||||
'filesVerified': len(actual) + 1, 'dataFiles': len(data), 'controlFiles': len(controls),
|
||||
'packageManifestSha256': data[MANIFEST]['sha256'], 'rootOwnership': True,
|
||||
'linksOrSpecialFiles': False, 'allSizesModesAndHashesMatch': True,
|
||||
'pdfiumCorrespondence': pdfium, 'qpdfNoticeCorrespondence': qpdf}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--archive', required=True, type=Path)
|
||||
parser.add_argument('--output', required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
report = verify(args.archive.resolve(strict=True))
|
||||
with args.output.open('x') as stream:
|
||||
json.dump(report, stream, indent=2); stream.write('\n')
|
||||
print(json.dumps(report))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user