261 lines
14 KiB
Python
261 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract the tested Ubuntu Qt SDK's embedded Chromium SBOM notice texts."""
|
|
import argparse
|
|
import ast
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import subprocess
|
|
import tarfile
|
|
import threading
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RUNTIME = ROOT / 'tests/results/ui-final/ubuntu/installed-runtime-final'
|
|
QT_SOURCE = ROOT / '.deps/source-archives/qt-6.11.2-regular-files'
|
|
ARCHIVE_NAME = '6.11.2-0-202608131118qtwebengine-Linux-RHEL_9_6-GCC-Linux-RHEL_9_6-X86_64.7z'
|
|
ARCHIVE_SHA256 = '9cbfd85900e85b95fa11926b41818addfc2a96361f62af567be430607ba6b67e'
|
|
METADATA_NAMES = ['qtwebengine-6.11.2.spdx.json', 'qtwebengine-chromium-webengine-6.11.2.spdx.json',
|
|
'qtwebengine-chromium-webengine-6.11.2.spdx']
|
|
|
|
|
|
def sha(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def write(path, data):
|
|
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + '\n')
|
|
|
|
|
|
def relative(name):
|
|
path = PurePosixPath(name)
|
|
if path.is_absolute() or '..' in path.parts or '\\' in name:
|
|
raise ValueError('Unsafe input path')
|
|
return str(path)
|
|
|
|
|
|
def inspect_sdk_archive(archive, output):
|
|
"""Stream archive conversion, never extracting links or executing payloads."""
|
|
rows, names, total = [], set(), 0
|
|
with (output / 'archive-reader.log').open('wb') as error_log:
|
|
process = subprocess.Popen(['bsdtar', '-cf', '-', '--format=pax', '@' + str(archive)],
|
|
stdout=subprocess.PIPE, stderr=error_log)
|
|
timer = threading.Timer(240, process.kill)
|
|
timer.start()
|
|
try:
|
|
with tarfile.open(fileobj=process.stdout, mode='r|') as tar:
|
|
for member in tar:
|
|
name = relative(member.name)
|
|
if name in names or len(names) >= 10000:
|
|
raise ValueError('Duplicate/excessive SDK entries')
|
|
names.add(name)
|
|
if member.isdir():
|
|
continue
|
|
item = {'path': name, 'mode': member.mode}
|
|
if member.issym() or member.islnk():
|
|
item.update(type='symlink' if member.issym() else 'hardlink', target=member.linkname)
|
|
elif member.isfile():
|
|
total += member.size
|
|
if member.size > 512 * 1024**2 or total > 4 * 1024**3:
|
|
raise ValueError('SDK exceeds inspection size limits')
|
|
hashes = {key: hashlib.new(key) for key in ('sha1', 'sha256')}
|
|
size = 0
|
|
with tar.extractfile(member) as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
size += len(chunk)
|
|
for digest in hashes.values():
|
|
digest.update(chunk)
|
|
if size != member.size:
|
|
raise ValueError('Short archive member')
|
|
item.update(type='file', bytes=size, **{key: value.hexdigest() for key, value in hashes.items()})
|
|
else:
|
|
raise ValueError('Unexpected SDK archive entry type')
|
|
rows.append(item)
|
|
if process.wait(timeout=30):
|
|
raise ValueError('SDK archive inspection failed')
|
|
finally:
|
|
timer.cancel()
|
|
if process.poll() is None:
|
|
process.kill(); process.wait()
|
|
rows.sort(key=lambda x: x['path'])
|
|
write(output / 'sdk-archive-inventory.json', rows)
|
|
return {row['path']: row for row in rows}, total
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
if not output.is_relative_to(ROOT):
|
|
parser.error('Output must be inside the workspace')
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
(output / 'notices').mkdir()
|
|
runtime_path = RUNTIME / 'runtime.json'
|
|
runtime = json.loads(runtime_path.read_text())
|
|
metadata = {}
|
|
for name in METADATA_NAMES:
|
|
entry = next(e for e in runtime['notices'] if e['path'] == 'qt-sdk:sbom/' + name)
|
|
path = RUNTIME / entry['copiedPath']
|
|
if path.stat().st_size != entry['size'] or sha(path) != entry['sha256']:
|
|
raise ValueError('Saved SDK metadata changed')
|
|
metadata[name] = dict(entry, localPath=str(path.relative_to(ROOT)))
|
|
main_sbom = json.loads((ROOT / metadata[METADATA_NAMES[0]]['localPath']).read_text())
|
|
chromium = json.loads((ROOT / metadata[METADATA_NAMES[1]]['localPath']).read_text())
|
|
external = next(e for e in main_sbom['externalDocumentRefs'] if 'qtwebengine-chromium-webengine' in e['externalDocumentId'])
|
|
external_bytes = (ROOT / metadata[METADATA_NAMES[2]]['localPath']).read_bytes()
|
|
if (external['checksum']['algorithm'] != 'SHA1'
|
|
or hashlib.sha1(external_bytes).hexdigest() != external['checksum']['checksumValue']
|
|
or external['spdxDocument'] != chromium['documentNamespace']):
|
|
raise ValueError('Qt module SBOM does not bind the Chromium SBOM')
|
|
archive = ROOT / 'build-ubuntu-vm/sdk-downloads' / ARCHIVE_NAME
|
|
if sha(archive) != ARCHIVE_SHA256:
|
|
raise ValueError('Qt SDK archive differs from fixed downloaded input')
|
|
inventory, total_bytes = inspect_sdk_archive(archive, output)
|
|
for name, entry in metadata.items():
|
|
if inventory['sbom/' + name]['sha256'] != entry['sha256']:
|
|
raise ValueError('SDK metadata differs from original archive')
|
|
file_checks = []
|
|
for file in main_sbom['files']:
|
|
name = relative(file['fileName'])
|
|
entry = inventory.get(name)
|
|
if not entry or entry['type'] != 'file':
|
|
raise ValueError('SBOM file absent from original archive: ' + name)
|
|
for checksum in file['checksums']:
|
|
algorithm = checksum['algorithm'].lower()
|
|
if algorithm not in {'sha1', 'sha256'} or entry[algorithm] != checksum['checksumValue']:
|
|
raise ValueError('SBOM file checksum differs: ' + name)
|
|
file_checks.append({'path': name, 'spdxId': file['SPDXID'], 'sha1': entry['sha1'], 'sha256': entry['sha256']})
|
|
runtime_checks = []
|
|
for entry in runtime['files']:
|
|
resolved = entry['resolvedPath']
|
|
if not resolved.startswith('qt-sdk:'):
|
|
continue
|
|
name = relative(resolved.removeprefix('qt-sdk:'))
|
|
if name not in inventory:
|
|
continue
|
|
original = inventory[name]
|
|
if original.get('sha256') != entry['sha256'] or original.get('bytes') != entry['size']:
|
|
raise ValueError('SDK archive differs from tested Ubuntu runtime: ' + name)
|
|
runtime_checks.append({'path': entry['path'], 'resolvedPath': resolved, 'sha256': entry['sha256']})
|
|
if not any(e['resolvedPath'] == 'qt-sdk:lib/libQt6WebEngineCore.so.6.11.2' for e in runtime_checks):
|
|
raise ValueError('QtWebEngineCore runtime binding is required')
|
|
packages = {p['SPDXID']: p for p in chromium['packages']}
|
|
licenses = {p['licenseId']: p for p in chromium['hasExtractedLicensingInfos']}
|
|
if len(packages) != len(chromium['packages']) or len(licenses) != len(chromium['hasExtractedLicensingInfos']):
|
|
raise ValueError('Duplicate SPDX IDs')
|
|
# The SDK's document checksum/namespace can match even when its package
|
|
# relationship names a nonexistent ID. Preserve that upstream inconsistency.
|
|
external_package_refs = []
|
|
for relationship in main_sbom['relationships']:
|
|
prefix = external['externalDocumentId'] + ':'
|
|
target = relationship['relatedSpdxElement']
|
|
if target.startswith(prefix):
|
|
target_id = target[len(prefix):]
|
|
external_package_refs.append(dict(relationship, targetIdExists=target_id in packages))
|
|
for relationship in chromium['relationships']:
|
|
if (relationship['spdxElementId'] not in packages
|
|
or relationship['relatedSpdxElement'] not in packages
|
|
or relationship['relationshipType'] != 'CONTAINS'):
|
|
raise ValueError('Unreviewed SPDX relationship')
|
|
without_text = []
|
|
for package in packages.values():
|
|
license_id = package['licenseConcluded']
|
|
if license_id not in licenses:
|
|
if license_id not in {'BSD-3-Clause', 'MIT'}:
|
|
raise ValueError('Unresolved SPDX license reference')
|
|
without_text.append(package)
|
|
source_report = json.loads((ROOT / 'tests/results/source-archives/qt-inspection/report.json').read_text())
|
|
source_inventory = ROOT / 'tests/results/source-archives/qt-inspection/files.jsonl'
|
|
if sha(source_inventory) != source_report['inventorySha256']:
|
|
raise ValueError('Qt source inventory changed')
|
|
helpers = ['qtwebengine/src/3rdparty/chromium/tools/licenses/sbom.py',
|
|
'qtwebengine/cmake/QtWebEngineSbomHelpers.cmake']
|
|
source_names = set(helpers)
|
|
for license in licenses.values():
|
|
refs = license['crossRefs']
|
|
if len(refs) != 1 or not refs[0]['url'].startswith(('/chromium/', '/gn/')):
|
|
raise ValueError('Unexpected source license reference')
|
|
source_names.add('qtwebengine/src/3rdparty/' + relative(refs[0]['url'][1:]))
|
|
sources = {}
|
|
for line in source_inventory.open():
|
|
entry = json.loads(line)
|
|
if entry['path'] in source_names:
|
|
if sha(QT_SOURCE / entry['path']) != entry['sha256']:
|
|
raise ValueError('Qt source notice/helper changed')
|
|
sources[entry['path']] = entry
|
|
if sources.keys() != source_names:
|
|
raise ValueError('Notice source missing from verified source archive')
|
|
rows, combined = [], ['Qt WebEngine 6.11.2 — Chromium notices contained in the Linux Qt SDK SBOM\n']
|
|
for license_id, license in sorted(licenses.items()):
|
|
text = license['extractedText'].encode('utf-8')
|
|
source = 'qtwebengine/src/3rdparty/' + license['crossRefs'][0]['url'][1:]
|
|
if not text or text != (QT_SOURCE / source).read_bytes():
|
|
raise ValueError('SBOM license text differs from verified source archive')
|
|
digest = hashlib.sha256(text).hexdigest()
|
|
target = output / 'notices' / (digest + '.txt')
|
|
if not target.exists():
|
|
target.write_bytes(text)
|
|
consumers = [p['SPDXID'] for p in packages.values() if p['licenseConcluded'] == license_id]
|
|
rows.append({'licenseId': license_id, 'source': source, 'sha256': digest, 'bytes': len(text),
|
|
'file': str(target.relative_to(output)), 'packageIds': consumers})
|
|
combined.extend(['\n' + '=' * 72 + '\n' + license['name'] + '\n',
|
|
'Source: ' + license['crossRefs'][0]['url'] + '\n\n', license['extractedText']])
|
|
bundle = output / 'THIRD_PARTY_NOTICES.sdk-extract.txt'
|
|
bundle.write_text(''.join(combined), encoding='utf-8')
|
|
# Preserve the generator's explicit limitations as data without importing it.
|
|
syntax = ast.parse((QT_SOURCE / helpers[0]).read_text())
|
|
skip_names = {'DIRECTORIES_TO_SKIP_BECAUSE_THEY_HAVE_VARIOUS_PARSING_ISSUES',
|
|
'PACKAGES_TO_OVERRIDE_LICENSE_FILE_WITH_ID'}
|
|
generator_limits = {}
|
|
for node in syntax.body:
|
|
if not isinstance(node, ast.Assign) or not isinstance(node.targets[0], ast.Name):
|
|
continue
|
|
name = node.targets[0].id
|
|
if name not in skip_names:
|
|
continue
|
|
values = []
|
|
for value in node.value.elts:
|
|
if isinstance(value, ast.Constant):
|
|
values.append(value.value)
|
|
elif isinstance(value, ast.Call) and ast.unparse(value.func) == 'os.path.join':
|
|
values.append('/'.join(ast.literal_eval(v) for v in value.args))
|
|
else:
|
|
raise ValueError('Unexpected generator limitation declaration')
|
|
generator_limits[name] = values
|
|
arch_sbom = Path('/usr/lib/qt6/sbom/qtwebengine-6.11.2.spdx')
|
|
arch_incomplete = 'not listing all of its consumed 3rd party dependencies' in arch_sbom.read_text()
|
|
if not arch_incomplete:
|
|
raise ValueError('Arch SBOM scope changed; reassess separately')
|
|
(output / 'arch-qtwebengine.spdx').write_bytes(arch_sbom.read_bytes())
|
|
report = {'success': True, 'collectorSha256': sha(Path(__file__)), 'runtimeRecordSha256': sha(runtime_path),
|
|
'sdkArchive': {'path': str(archive.relative_to(ROOT)), 'sha256': ARCHIVE_SHA256,
|
|
'bytes': archive.stat().st_size, 'regularPayloadBytes': total_bytes,
|
|
'entries': len(inventory)},
|
|
'sdkArchiveInventorySha256': sha(output / 'sdk-archive-inventory.json'),
|
|
'metadata': metadata, 'externalSbomReference': external, 'sbomFileComparisons': file_checks,
|
|
'externalPackageReferences': external_package_refs,
|
|
'externalPackageReferencesValid': bool(external_package_refs) and all(
|
|
e['targetIdExists'] for e in external_package_refs),
|
|
'testedRuntimeComparisons': runtime_checks, 'packages': chromium['packages'],
|
|
'relationships': chromium['relationships'], 'notices': rows,
|
|
'noticeBundleSha256': sha(bundle), 'uniqueNoticeFiles': len(list((output / 'notices').glob('*.txt'))),
|
|
'packagesWithoutEmbeddedNoticeText': without_text, 'generatorLimits': generator_limits,
|
|
'generatorSources': {name: sources[name]['sha256'] for name in helpers},
|
|
'qtSourceArchiveSha256': source_report['archiveSha256'],
|
|
'archSbom': {'path': str(arch_sbom), 'sha256': sha(arch_sbom), 'explicitIncompleteNotice': True},
|
|
'completeChromiumNotices': False, 'completeCorrespondingSources': False,
|
|
'scope': 'All embedded notice texts in the tested Ubuntu Qt SDK Chromium WebEngine SBOM, '
|
|
'bound to its original archive, runtime fingerprint and Qt source archive. '
|
|
'Not the Arch configuration or a complete license/linked-component verdict. '
|
|
'Unresolved external package IDs, generator skips, identifier-only entries '
|
|
'and omitted patent files remain distinct.'}
|
|
write(output / 'report.json', report)
|
|
print(json.dumps({'success': True, 'noticeEntries': len(rows), 'uniqueNoticeFiles': report['uniqueNoticeFiles'],
|
|
'sbomFilesMatched': len(file_checks), 'testedRuntimeFilesMatched': len(runtime_checks),
|
|
'packagesWithoutEmbeddedText': len(without_text)}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|