initial commit
This commit is contained in:
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract the tested Ubuntu Qt SDK's embedded Chromium SBOM notice texts."""
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import subprocess
|
||||
import tarfile
|
||||
import threading
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RUNTIME = ROOT / 'tests/results/ui-final/ubuntu/installed-runtime-final'
|
||||
QT_SOURCE = ROOT / '.deps/source-archives/qt-6.11.2-regular-files'
|
||||
ARCHIVE_NAME = '6.11.2-0-202608131118qtwebengine-Linux-RHEL_9_6-GCC-Linux-RHEL_9_6-X86_64.7z'
|
||||
ARCHIVE_SHA256 = '9cbfd85900e85b95fa11926b41818addfc2a96361f62af567be430607ba6b67e'
|
||||
METADATA_NAMES = ['qtwebengine-6.11.2.spdx.json', 'qtwebengine-chromium-webengine-6.11.2.spdx.json',
|
||||
'qtwebengine-chromium-webengine-6.11.2.spdx']
|
||||
|
||||
|
||||
def sha(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def write(path, data):
|
||||
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
def relative(name):
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or '..' in path.parts or '\\' in name:
|
||||
raise ValueError('Unsafe input path')
|
||||
return str(path)
|
||||
|
||||
|
||||
def inspect_sdk_archive(archive, output):
|
||||
"""Stream archive conversion, never extracting links or executing payloads."""
|
||||
rows, names, total = [], set(), 0
|
||||
with (output / 'archive-reader.log').open('wb') as error_log:
|
||||
process = subprocess.Popen(['bsdtar', '-cf', '-', '--format=pax', '@' + str(archive)],
|
||||
stdout=subprocess.PIPE, stderr=error_log)
|
||||
timer = threading.Timer(240, process.kill)
|
||||
timer.start()
|
||||
try:
|
||||
with tarfile.open(fileobj=process.stdout, mode='r|') as tar:
|
||||
for member in tar:
|
||||
name = relative(member.name)
|
||||
if name in names or len(names) >= 10000:
|
||||
raise ValueError('Duplicate/excessive SDK entries')
|
||||
names.add(name)
|
||||
if member.isdir():
|
||||
continue
|
||||
item = {'path': name, 'mode': member.mode}
|
||||
if member.issym() or member.islnk():
|
||||
item.update(type='symlink' if member.issym() else 'hardlink', target=member.linkname)
|
||||
elif member.isfile():
|
||||
total += member.size
|
||||
if member.size > 512 * 1024**2 or total > 4 * 1024**3:
|
||||
raise ValueError('SDK exceeds inspection size limits')
|
||||
hashes = {key: hashlib.new(key) for key in ('sha1', 'sha256')}
|
||||
size = 0
|
||||
with tar.extractfile(member) as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
size += len(chunk)
|
||||
for digest in hashes.values():
|
||||
digest.update(chunk)
|
||||
if size != member.size:
|
||||
raise ValueError('Short archive member')
|
||||
item.update(type='file', bytes=size, **{key: value.hexdigest() for key, value in hashes.items()})
|
||||
else:
|
||||
raise ValueError('Unexpected SDK archive entry type')
|
||||
rows.append(item)
|
||||
if process.wait(timeout=30):
|
||||
raise ValueError('SDK archive inspection failed')
|
||||
finally:
|
||||
timer.cancel()
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait()
|
||||
rows.sort(key=lambda x: x['path'])
|
||||
write(output / 'sdk-archive-inventory.json', rows)
|
||||
return {row['path']: row for row in rows}, total
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
if not output.is_relative_to(ROOT):
|
||||
parser.error('Output must be inside the workspace')
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
(output / 'notices').mkdir()
|
||||
runtime_path = RUNTIME / 'runtime.json'
|
||||
runtime = json.loads(runtime_path.read_text())
|
||||
metadata = {}
|
||||
for name in METADATA_NAMES:
|
||||
entry = next(e for e in runtime['notices'] if e['path'] == 'qt-sdk:sbom/' + name)
|
||||
path = RUNTIME / entry['copiedPath']
|
||||
if path.stat().st_size != entry['size'] or sha(path) != entry['sha256']:
|
||||
raise ValueError('Saved SDK metadata changed')
|
||||
metadata[name] = dict(entry, localPath=str(path.relative_to(ROOT)))
|
||||
main_sbom = json.loads((ROOT / metadata[METADATA_NAMES[0]]['localPath']).read_text())
|
||||
chromium = json.loads((ROOT / metadata[METADATA_NAMES[1]]['localPath']).read_text())
|
||||
external = next(e for e in main_sbom['externalDocumentRefs'] if 'qtwebengine-chromium-webengine' in e['externalDocumentId'])
|
||||
external_bytes = (ROOT / metadata[METADATA_NAMES[2]]['localPath']).read_bytes()
|
||||
if (external['checksum']['algorithm'] != 'SHA1'
|
||||
or hashlib.sha1(external_bytes).hexdigest() != external['checksum']['checksumValue']
|
||||
or external['spdxDocument'] != chromium['documentNamespace']):
|
||||
raise ValueError('Qt module SBOM does not bind the Chromium SBOM')
|
||||
archive = ROOT / 'build-ubuntu-vm/sdk-downloads' / ARCHIVE_NAME
|
||||
if sha(archive) != ARCHIVE_SHA256:
|
||||
raise ValueError('Qt SDK archive differs from fixed downloaded input')
|
||||
inventory, total_bytes = inspect_sdk_archive(archive, output)
|
||||
for name, entry in metadata.items():
|
||||
if inventory['sbom/' + name]['sha256'] != entry['sha256']:
|
||||
raise ValueError('SDK metadata differs from original archive')
|
||||
file_checks = []
|
||||
for file in main_sbom['files']:
|
||||
name = relative(file['fileName'])
|
||||
entry = inventory.get(name)
|
||||
if not entry or entry['type'] != 'file':
|
||||
raise ValueError('SBOM file absent from original archive: ' + name)
|
||||
for checksum in file['checksums']:
|
||||
algorithm = checksum['algorithm'].lower()
|
||||
if algorithm not in {'sha1', 'sha256'} or entry[algorithm] != checksum['checksumValue']:
|
||||
raise ValueError('SBOM file checksum differs: ' + name)
|
||||
file_checks.append({'path': name, 'spdxId': file['SPDXID'], 'sha1': entry['sha1'], 'sha256': entry['sha256']})
|
||||
runtime_checks = []
|
||||
for entry in runtime['files']:
|
||||
resolved = entry['resolvedPath']
|
||||
if not resolved.startswith('qt-sdk:'):
|
||||
continue
|
||||
name = relative(resolved.removeprefix('qt-sdk:'))
|
||||
if name not in inventory:
|
||||
continue
|
||||
original = inventory[name]
|
||||
if original.get('sha256') != entry['sha256'] or original.get('bytes') != entry['size']:
|
||||
raise ValueError('SDK archive differs from tested Ubuntu runtime: ' + name)
|
||||
runtime_checks.append({'path': entry['path'], 'resolvedPath': resolved, 'sha256': entry['sha256']})
|
||||
if not any(e['resolvedPath'] == 'qt-sdk:lib/libQt6WebEngineCore.so.6.11.2' for e in runtime_checks):
|
||||
raise ValueError('QtWebEngineCore runtime binding is required')
|
||||
packages = {p['SPDXID']: p for p in chromium['packages']}
|
||||
licenses = {p['licenseId']: p for p in chromium['hasExtractedLicensingInfos']}
|
||||
if len(packages) != len(chromium['packages']) or len(licenses) != len(chromium['hasExtractedLicensingInfos']):
|
||||
raise ValueError('Duplicate SPDX IDs')
|
||||
# The SDK's document checksum/namespace can match even when its package
|
||||
# relationship names a nonexistent ID. Preserve that upstream inconsistency.
|
||||
external_package_refs = []
|
||||
for relationship in main_sbom['relationships']:
|
||||
prefix = external['externalDocumentId'] + ':'
|
||||
target = relationship['relatedSpdxElement']
|
||||
if target.startswith(prefix):
|
||||
target_id = target[len(prefix):]
|
||||
external_package_refs.append(dict(relationship, targetIdExists=target_id in packages))
|
||||
for relationship in chromium['relationships']:
|
||||
if (relationship['spdxElementId'] not in packages
|
||||
or relationship['relatedSpdxElement'] not in packages
|
||||
or relationship['relationshipType'] != 'CONTAINS'):
|
||||
raise ValueError('Unreviewed SPDX relationship')
|
||||
without_text = []
|
||||
for package in packages.values():
|
||||
license_id = package['licenseConcluded']
|
||||
if license_id not in licenses:
|
||||
if license_id not in {'BSD-3-Clause', 'MIT'}:
|
||||
raise ValueError('Unresolved SPDX license reference')
|
||||
without_text.append(package)
|
||||
source_report = json.loads((ROOT / 'tests/results/source-archives/qt-inspection/report.json').read_text())
|
||||
source_inventory = ROOT / 'tests/results/source-archives/qt-inspection/files.jsonl'
|
||||
if sha(source_inventory) != source_report['inventorySha256']:
|
||||
raise ValueError('Qt source inventory changed')
|
||||
helpers = ['qtwebengine/src/3rdparty/chromium/tools/licenses/sbom.py',
|
||||
'qtwebengine/cmake/QtWebEngineSbomHelpers.cmake']
|
||||
source_names = set(helpers)
|
||||
for license in licenses.values():
|
||||
refs = license['crossRefs']
|
||||
if len(refs) != 1 or not refs[0]['url'].startswith(('/chromium/', '/gn/')):
|
||||
raise ValueError('Unexpected source license reference')
|
||||
source_names.add('qtwebengine/src/3rdparty/' + relative(refs[0]['url'][1:]))
|
||||
sources = {}
|
||||
for line in source_inventory.open():
|
||||
entry = json.loads(line)
|
||||
if entry['path'] in source_names:
|
||||
if sha(QT_SOURCE / entry['path']) != entry['sha256']:
|
||||
raise ValueError('Qt source notice/helper changed')
|
||||
sources[entry['path']] = entry
|
||||
if sources.keys() != source_names:
|
||||
raise ValueError('Notice source missing from verified source archive')
|
||||
rows, combined = [], ['Qt WebEngine 6.11.2 — Chromium notices contained in the Linux Qt SDK SBOM\n']
|
||||
for license_id, license in sorted(licenses.items()):
|
||||
text = license['extractedText'].encode('utf-8')
|
||||
source = 'qtwebengine/src/3rdparty/' + license['crossRefs'][0]['url'][1:]
|
||||
if not text or text != (QT_SOURCE / source).read_bytes():
|
||||
raise ValueError('SBOM license text differs from verified source archive')
|
||||
digest = hashlib.sha256(text).hexdigest()
|
||||
target = output / 'notices' / (digest + '.txt')
|
||||
if not target.exists():
|
||||
target.write_bytes(text)
|
||||
consumers = [p['SPDXID'] for p in packages.values() if p['licenseConcluded'] == license_id]
|
||||
rows.append({'licenseId': license_id, 'source': source, 'sha256': digest, 'bytes': len(text),
|
||||
'file': str(target.relative_to(output)), 'packageIds': consumers})
|
||||
combined.extend(['\n' + '=' * 72 + '\n' + license['name'] + '\n',
|
||||
'Source: ' + license['crossRefs'][0]['url'] + '\n\n', license['extractedText']])
|
||||
bundle = output / 'THIRD_PARTY_NOTICES.sdk-extract.txt'
|
||||
bundle.write_text(''.join(combined), encoding='utf-8')
|
||||
# Preserve the generator's explicit limitations as data without importing it.
|
||||
syntax = ast.parse((QT_SOURCE / helpers[0]).read_text())
|
||||
skip_names = {'DIRECTORIES_TO_SKIP_BECAUSE_THEY_HAVE_VARIOUS_PARSING_ISSUES',
|
||||
'PACKAGES_TO_OVERRIDE_LICENSE_FILE_WITH_ID'}
|
||||
generator_limits = {}
|
||||
for node in syntax.body:
|
||||
if not isinstance(node, ast.Assign) or not isinstance(node.targets[0], ast.Name):
|
||||
continue
|
||||
name = node.targets[0].id
|
||||
if name not in skip_names:
|
||||
continue
|
||||
values = []
|
||||
for value in node.value.elts:
|
||||
if isinstance(value, ast.Constant):
|
||||
values.append(value.value)
|
||||
elif isinstance(value, ast.Call) and ast.unparse(value.func) == 'os.path.join':
|
||||
values.append('/'.join(ast.literal_eval(v) for v in value.args))
|
||||
else:
|
||||
raise ValueError('Unexpected generator limitation declaration')
|
||||
generator_limits[name] = values
|
||||
arch_sbom = Path('/usr/lib/qt6/sbom/qtwebengine-6.11.2.spdx')
|
||||
arch_incomplete = 'not listing all of its consumed 3rd party dependencies' in arch_sbom.read_text()
|
||||
if not arch_incomplete:
|
||||
raise ValueError('Arch SBOM scope changed; reassess separately')
|
||||
(output / 'arch-qtwebengine.spdx').write_bytes(arch_sbom.read_bytes())
|
||||
report = {'success': True, 'collectorSha256': sha(Path(__file__)), 'runtimeRecordSha256': sha(runtime_path),
|
||||
'sdkArchive': {'path': str(archive.relative_to(ROOT)), 'sha256': ARCHIVE_SHA256,
|
||||
'bytes': archive.stat().st_size, 'regularPayloadBytes': total_bytes,
|
||||
'entries': len(inventory)},
|
||||
'sdkArchiveInventorySha256': sha(output / 'sdk-archive-inventory.json'),
|
||||
'metadata': metadata, 'externalSbomReference': external, 'sbomFileComparisons': file_checks,
|
||||
'externalPackageReferences': external_package_refs,
|
||||
'externalPackageReferencesValid': bool(external_package_refs) and all(
|
||||
e['targetIdExists'] for e in external_package_refs),
|
||||
'testedRuntimeComparisons': runtime_checks, 'packages': chromium['packages'],
|
||||
'relationships': chromium['relationships'], 'notices': rows,
|
||||
'noticeBundleSha256': sha(bundle), 'uniqueNoticeFiles': len(list((output / 'notices').glob('*.txt'))),
|
||||
'packagesWithoutEmbeddedNoticeText': without_text, 'generatorLimits': generator_limits,
|
||||
'generatorSources': {name: sources[name]['sha256'] for name in helpers},
|
||||
'qtSourceArchiveSha256': source_report['archiveSha256'],
|
||||
'archSbom': {'path': str(arch_sbom), 'sha256': sha(arch_sbom), 'explicitIncompleteNotice': True},
|
||||
'completeChromiumNotices': False, 'completeCorrespondingSources': False,
|
||||
'scope': 'All embedded notice texts in the tested Ubuntu Qt SDK Chromium WebEngine SBOM, '
|
||||
'bound to its original archive, runtime fingerprint and Qt source archive. '
|
||||
'Not the Arch configuration or a complete license/linked-component verdict. '
|
||||
'Unresolved external package IDs, generator skips, identifier-only entries '
|
||||
'and omitted patent files remain distinct.'}
|
||||
write(output / 'report.json', report)
|
||||
print(json.dumps({'success': True, 'noticeEntries': len(rows), 'uniqueNoticeFiles': report['uniqueNoticeFiles'],
|
||||
'sbomFilesMatched': len(file_checks), 'testedRuntimeFilesMatched': len(runtime_checks),
|
||||
'packagesWithoutEmbeddedText': len(without_text)}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user