#!/usr/bin/env python3 """Extract the tested Ubuntu Qt SDK's embedded Chromium SBOM notice texts.""" import argparse import ast import hashlib import json from pathlib import Path, PurePosixPath import subprocess import tarfile import threading ROOT = Path(__file__).resolve().parents[2] RUNTIME = ROOT / 'tests/results/ui-final/ubuntu/installed-runtime-final' QT_SOURCE = ROOT / '.deps/source-archives/qt-6.11.2-regular-files' ARCHIVE_NAME = '6.11.2-0-202608131118qtwebengine-Linux-RHEL_9_6-GCC-Linux-RHEL_9_6-X86_64.7z' ARCHIVE_SHA256 = '9cbfd85900e85b95fa11926b41818addfc2a96361f62af567be430607ba6b67e' METADATA_NAMES = ['qtwebengine-6.11.2.spdx.json', 'qtwebengine-chromium-webengine-6.11.2.spdx.json', 'qtwebengine-chromium-webengine-6.11.2.spdx'] def sha(path): with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() def write(path, data): path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + '\n') def relative(name): path = PurePosixPath(name) if path.is_absolute() or '..' in path.parts or '\\' in name: raise ValueError('Unsafe input path') return str(path) def inspect_sdk_archive(archive, output): """Stream archive conversion, never extracting links or executing payloads.""" rows, names, total = [], set(), 0 with (output / 'archive-reader.log').open('wb') as error_log: process = subprocess.Popen(['bsdtar', '-cf', '-', '--format=pax', '@' + str(archive)], stdout=subprocess.PIPE, stderr=error_log) timer = threading.Timer(240, process.kill) timer.start() try: with tarfile.open(fileobj=process.stdout, mode='r|') as tar: for member in tar: name = relative(member.name) if name in names or len(names) >= 10000: raise ValueError('Duplicate/excessive SDK entries') names.add(name) if member.isdir(): continue item = {'path': name, 'mode': member.mode} if member.issym() or member.islnk(): item.update(type='symlink' if member.issym() else 'hardlink', target=member.linkname) elif member.isfile(): total += member.size if member.size > 512 * 1024**2 or total > 4 * 1024**3: raise ValueError('SDK exceeds inspection size limits') hashes = {key: hashlib.new(key) for key in ('sha1', 'sha256')} size = 0 with tar.extractfile(member) as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b''): size += len(chunk) for digest in hashes.values(): digest.update(chunk) if size != member.size: raise ValueError('Short archive member') item.update(type='file', bytes=size, **{key: value.hexdigest() for key, value in hashes.items()}) else: raise ValueError('Unexpected SDK archive entry type') rows.append(item) if process.wait(timeout=30): raise ValueError('SDK archive inspection failed') finally: timer.cancel() if process.poll() is None: process.kill(); process.wait() rows.sort(key=lambda x: x['path']) write(output / 'sdk-archive-inventory.json', rows) return {row['path']: row for row in rows}, total def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() output = args.output.resolve() if not output.is_relative_to(ROOT): parser.error('Output must be inside the workspace') output.mkdir(parents=True, exist_ok=False) (output / 'notices').mkdir() runtime_path = RUNTIME / 'runtime.json' runtime = json.loads(runtime_path.read_text()) metadata = {} for name in METADATA_NAMES: entry = next(e for e in runtime['notices'] if e['path'] == 'qt-sdk:sbom/' + name) path = RUNTIME / entry['copiedPath'] if path.stat().st_size != entry['size'] or sha(path) != entry['sha256']: raise ValueError('Saved SDK metadata changed') metadata[name] = dict(entry, localPath=str(path.relative_to(ROOT))) main_sbom = json.loads((ROOT / metadata[METADATA_NAMES[0]]['localPath']).read_text()) chromium = json.loads((ROOT / metadata[METADATA_NAMES[1]]['localPath']).read_text()) external = next(e for e in main_sbom['externalDocumentRefs'] if 'qtwebengine-chromium-webengine' in e['externalDocumentId']) external_bytes = (ROOT / metadata[METADATA_NAMES[2]]['localPath']).read_bytes() if (external['checksum']['algorithm'] != 'SHA1' or hashlib.sha1(external_bytes).hexdigest() != external['checksum']['checksumValue'] or external['spdxDocument'] != chromium['documentNamespace']): raise ValueError('Qt module SBOM does not bind the Chromium SBOM') archive = ROOT / 'build-ubuntu-vm/sdk-downloads' / ARCHIVE_NAME if sha(archive) != ARCHIVE_SHA256: raise ValueError('Qt SDK archive differs from fixed downloaded input') inventory, total_bytes = inspect_sdk_archive(archive, output) for name, entry in metadata.items(): if inventory['sbom/' + name]['sha256'] != entry['sha256']: raise ValueError('SDK metadata differs from original archive') file_checks = [] for file in main_sbom['files']: name = relative(file['fileName']) entry = inventory.get(name) if not entry or entry['type'] != 'file': raise ValueError('SBOM file absent from original archive: ' + name) for checksum in file['checksums']: algorithm = checksum['algorithm'].lower() if algorithm not in {'sha1', 'sha256'} or entry[algorithm] != checksum['checksumValue']: raise ValueError('SBOM file checksum differs: ' + name) file_checks.append({'path': name, 'spdxId': file['SPDXID'], 'sha1': entry['sha1'], 'sha256': entry['sha256']}) runtime_checks = [] for entry in runtime['files']: resolved = entry['resolvedPath'] if not resolved.startswith('qt-sdk:'): continue name = relative(resolved.removeprefix('qt-sdk:')) if name not in inventory: continue original = inventory[name] if original.get('sha256') != entry['sha256'] or original.get('bytes') != entry['size']: raise ValueError('SDK archive differs from tested Ubuntu runtime: ' + name) runtime_checks.append({'path': entry['path'], 'resolvedPath': resolved, 'sha256': entry['sha256']}) if not any(e['resolvedPath'] == 'qt-sdk:lib/libQt6WebEngineCore.so.6.11.2' for e in runtime_checks): raise ValueError('QtWebEngineCore runtime binding is required') packages = {p['SPDXID']: p for p in chromium['packages']} licenses = {p['licenseId']: p for p in chromium['hasExtractedLicensingInfos']} if len(packages) != len(chromium['packages']) or len(licenses) != len(chromium['hasExtractedLicensingInfos']): raise ValueError('Duplicate SPDX IDs') # The SDK's document checksum/namespace can match even when its package # relationship names a nonexistent ID. Preserve that upstream inconsistency. external_package_refs = [] for relationship in main_sbom['relationships']: prefix = external['externalDocumentId'] + ':' target = relationship['relatedSpdxElement'] if target.startswith(prefix): target_id = target[len(prefix):] external_package_refs.append(dict(relationship, targetIdExists=target_id in packages)) for relationship in chromium['relationships']: if (relationship['spdxElementId'] not in packages or relationship['relatedSpdxElement'] not in packages or relationship['relationshipType'] != 'CONTAINS'): raise ValueError('Unreviewed SPDX relationship') without_text = [] for package in packages.values(): license_id = package['licenseConcluded'] if license_id not in licenses: if license_id not in {'BSD-3-Clause', 'MIT'}: raise ValueError('Unresolved SPDX license reference') without_text.append(package) source_report = json.loads((ROOT / 'tests/results/source-archives/qt-inspection/report.json').read_text()) source_inventory = ROOT / 'tests/results/source-archives/qt-inspection/files.jsonl' if sha(source_inventory) != source_report['inventorySha256']: raise ValueError('Qt source inventory changed') helpers = ['qtwebengine/src/3rdparty/chromium/tools/licenses/sbom.py', 'qtwebengine/cmake/QtWebEngineSbomHelpers.cmake'] source_names = set(helpers) for license in licenses.values(): refs = license['crossRefs'] if len(refs) != 1 or not refs[0]['url'].startswith(('/chromium/', '/gn/')): raise ValueError('Unexpected source license reference') source_names.add('qtwebengine/src/3rdparty/' + relative(refs[0]['url'][1:])) sources = {} for line in source_inventory.open(): entry = json.loads(line) if entry['path'] in source_names: if sha(QT_SOURCE / entry['path']) != entry['sha256']: raise ValueError('Qt source notice/helper changed') sources[entry['path']] = entry if sources.keys() != source_names: raise ValueError('Notice source missing from verified source archive') rows, combined = [], ['Qt WebEngine 6.11.2 — Chromium notices contained in the Linux Qt SDK SBOM\n'] for license_id, license in sorted(licenses.items()): text = license['extractedText'].encode('utf-8') source = 'qtwebengine/src/3rdparty/' + license['crossRefs'][0]['url'][1:] if not text or text != (QT_SOURCE / source).read_bytes(): raise ValueError('SBOM license text differs from verified source archive') digest = hashlib.sha256(text).hexdigest() target = output / 'notices' / (digest + '.txt') if not target.exists(): target.write_bytes(text) consumers = [p['SPDXID'] for p in packages.values() if p['licenseConcluded'] == license_id] rows.append({'licenseId': license_id, 'source': source, 'sha256': digest, 'bytes': len(text), 'file': str(target.relative_to(output)), 'packageIds': consumers}) combined.extend(['\n' + '=' * 72 + '\n' + license['name'] + '\n', 'Source: ' + license['crossRefs'][0]['url'] + '\n\n', license['extractedText']]) bundle = output / 'THIRD_PARTY_NOTICES.sdk-extract.txt' bundle.write_text(''.join(combined), encoding='utf-8') # Preserve the generator's explicit limitations as data without importing it. syntax = ast.parse((QT_SOURCE / helpers[0]).read_text()) skip_names = {'DIRECTORIES_TO_SKIP_BECAUSE_THEY_HAVE_VARIOUS_PARSING_ISSUES', 'PACKAGES_TO_OVERRIDE_LICENSE_FILE_WITH_ID'} generator_limits = {} for node in syntax.body: if not isinstance(node, ast.Assign) or not isinstance(node.targets[0], ast.Name): continue name = node.targets[0].id if name not in skip_names: continue values = [] for value in node.value.elts: if isinstance(value, ast.Constant): values.append(value.value) elif isinstance(value, ast.Call) and ast.unparse(value.func) == 'os.path.join': values.append('/'.join(ast.literal_eval(v) for v in value.args)) else: raise ValueError('Unexpected generator limitation declaration') generator_limits[name] = values arch_sbom = Path('/usr/lib/qt6/sbom/qtwebengine-6.11.2.spdx') arch_incomplete = 'not listing all of its consumed 3rd party dependencies' in arch_sbom.read_text() if not arch_incomplete: raise ValueError('Arch SBOM scope changed; reassess separately') (output / 'arch-qtwebengine.spdx').write_bytes(arch_sbom.read_bytes()) report = {'success': True, 'collectorSha256': sha(Path(__file__)), 'runtimeRecordSha256': sha(runtime_path), 'sdkArchive': {'path': str(archive.relative_to(ROOT)), 'sha256': ARCHIVE_SHA256, 'bytes': archive.stat().st_size, 'regularPayloadBytes': total_bytes, 'entries': len(inventory)}, 'sdkArchiveInventorySha256': sha(output / 'sdk-archive-inventory.json'), 'metadata': metadata, 'externalSbomReference': external, 'sbomFileComparisons': file_checks, 'externalPackageReferences': external_package_refs, 'externalPackageReferencesValid': bool(external_package_refs) and all( e['targetIdExists'] for e in external_package_refs), 'testedRuntimeComparisons': runtime_checks, 'packages': chromium['packages'], 'relationships': chromium['relationships'], 'notices': rows, 'noticeBundleSha256': sha(bundle), 'uniqueNoticeFiles': len(list((output / 'notices').glob('*.txt'))), 'packagesWithoutEmbeddedNoticeText': without_text, 'generatorLimits': generator_limits, 'generatorSources': {name: sources[name]['sha256'] for name in helpers}, 'qtSourceArchiveSha256': source_report['archiveSha256'], 'archSbom': {'path': str(arch_sbom), 'sha256': sha(arch_sbom), 'explicitIncompleteNotice': True}, 'completeChromiumNotices': False, 'completeCorrespondingSources': False, 'scope': 'All embedded notice texts in the tested Ubuntu Qt SDK Chromium WebEngine SBOM, ' 'bound to its original archive, runtime fingerprint and Qt source archive. ' 'Not the Arch configuration or a complete license/linked-component verdict. ' 'Unresolved external package IDs, generator skips, identifier-only entries ' 'and omitted patent files remain distinct.'} write(output / 'report.json', report) print(json.dumps({'success': True, 'noticeEntries': len(rows), 'uniqueNoticeFiles': report['uniqueNoticeFiles'], 'sbomFilesMatched': len(file_checks), 'testedRuntimeFilesMatched': len(runtime_checks), 'packagesWithoutEmbeddedText': len(without_text)})) if __name__ == '__main__': main()