131 lines
7.7 KiB
Python
131 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Inspect the verified Qt release archive and retain regular source files safely."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import tarfile
|
|
import time
|
|
import urllib.parse
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def sha(path):
|
|
with path.open('rb') as f:
|
|
return hashlib.file_digest(f, 'sha256').hexdigest()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--archive', type=Path, required=True)
|
|
parser.add_argument('--tree', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
archive, tree, out = args.archive.resolve(), args.tree.resolve(), args.output.resolve()
|
|
if any(not path.is_relative_to(ROOT) for path in [archive, tree, out]):
|
|
parser.error('All paths must remain in this workspace')
|
|
pin = json.loads(Path(__file__).with_name('pins.json').read_text())['qt-everywhere']
|
|
if archive.stat().st_size != pin['bytes'] or sha(archive) != pin['hashes']['sha256']:
|
|
parser.error('Archive does not match the verified release pin')
|
|
tree.mkdir(parents=True, exist_ok=False); out.mkdir(parents=True, exist_ok=False)
|
|
comparison_root = ROOT / 'tests/results/source-correspondence/arch/upstream-metadata'
|
|
expected = {}
|
|
for row in json.loads((comparison_root / 'version-sources.json').read_text()):
|
|
url = urllib.parse.urlparse(row['url']).path
|
|
if '/qtwebengine.git/plain/' in url:
|
|
relative = 'qtwebengine/' + url.split('/plain/', 1)[1]
|
|
elif '/qtwebengine-chromium.git/plain/' in url:
|
|
relative = 'qtwebengine/src/3rdparty/' + url.split('/plain/', 1)[1]
|
|
else:
|
|
continue
|
|
if sha(comparison_root / row['file']) != row['sha256']:
|
|
raise ValueError('Stored fixed-revision comparison input changed')
|
|
expected[relative] = row
|
|
binaries = {name: sha(ROOT / 'build' / name) for name in ['docview', 'docview-pdf-worker', 'docview-archive-worker']}
|
|
report = {'success': False, 'archive': str(archive.relative_to(ROOT)), 'archiveSha256': pin['hashes']['sha256'],
|
|
'archiveBytes': pin['bytes'], 'sourceTree': str(tree.relative_to(ROOT)),
|
|
'scope': 'Official Qt 6.11.2 release sources and selected metadata/notices; not complete linked-component attribution, a rebuild, license acceptance or legal compliance certification'}
|
|
files = size = members = notice_bytes = 0
|
|
notices, links, comparisons, module_counts, versions = [], [], {}, {}, {}
|
|
seen = set(); last = time.monotonic()
|
|
try:
|
|
with tarfile.open(archive, 'r|xz') as stream, (out / 'files.jsonl').open('w') as inventory:
|
|
for member in stream:
|
|
members += 1
|
|
if members > 600000 or member.size > 1024**3 or member.size < 0:
|
|
raise ValueError('Archive exceeded finite member limits')
|
|
path = PurePosixPath(member.name)
|
|
if path.is_absolute() or '..' in path.parts or not path.parts or path.parts[0] != 'qt-everywhere-src-6.11.2' or '\\' in member.name:
|
|
raise ValueError('Unexpected source member path')
|
|
relative = PurePosixPath(*path.parts[1:])
|
|
if member.isdir():
|
|
continue
|
|
name = str(relative)
|
|
if name in seen or name == '.':
|
|
raise ValueError('Duplicate or empty source file path')
|
|
seen.add(name)
|
|
if member.issym() or member.islnk():
|
|
links.append({'path': name, 'target': member.linkname, 'kind': 'symlink' if member.issym() else 'hardlink'})
|
|
continue # Keep links in the original archive; never follow them.
|
|
if not member.isfile():
|
|
raise ValueError('Non-file source entry')
|
|
size += member.size
|
|
if size > 20 * 1024**3:
|
|
raise ValueError('Source tree exceeded 20 GiB limit')
|
|
destination = tree / name
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256(); count = 0
|
|
with stream.extractfile(member) as source, destination.open('xb') as target:
|
|
while block := source.read(1024 * 1024):
|
|
target.write(block); digest.update(block); count += len(block)
|
|
if count != member.size:
|
|
raise ValueError('Source member was truncated')
|
|
row = {'path': name, 'bytes': count, 'sha256': digest.hexdigest(), 'archiveMode': member.mode}
|
|
inventory.write(json.dumps(row) + '\n'); files += 1
|
|
module = relative.parts[0] if len(relative.parts) > 1 else '(root)'
|
|
module_counts[module] = module_counts.get(module, 0) + 1
|
|
if name in expected:
|
|
comparisons[name] = {**row, 'expectedSha256': expected[name]['sha256'],
|
|
'matches': row['sha256'] == expected[name]['sha256'], 'sourceUrl': expected[name]['url']}
|
|
if relative.name == '.cmake.conf' and len(relative.parts) == 2:
|
|
text = destination.read_text(errors='replace')
|
|
versions[module] = re.findall(r'QT_REPO_MODULE_VERSION\s+"([^"]+)"', text)
|
|
# Keep a named-file subset; README.chromium and attribution
|
|
# metadata may refer to further texts. Do not call this complete.
|
|
if (re.fullmatch(r'(licen[cs]e|copying|copyright|notice)([._-].*)?', relative.name, re.I)
|
|
or relative.name in ('README.chromium', 'qt_attributions.json', 'REUSE.toml')
|
|
or 'LICENSES' in relative.parts):
|
|
if count > 8 * 1024**2 or notice_bytes + count > 128 * 1024**2:
|
|
raise ValueError('Notice metadata exceeded finite limits')
|
|
notice_bytes += count; notices.append(row)
|
|
if time.monotonic() - last > 15:
|
|
print(f'Qt source: {files} files / {size / 1048576:.1f} MiB inspected', flush=True)
|
|
last = time.monotonic()
|
|
report['fixedSourceComparisons'] = comparisons
|
|
missing = sorted(set(expected) - set(comparisons))
|
|
report['missingComparisons'] = missing
|
|
if missing or not all(row['matches'] for row in comparisons.values()):
|
|
raise ValueError('Release source differs from a fixed-revision comparison input')
|
|
report['success'] = True
|
|
except Exception as error:
|
|
report['error'] = str(error)
|
|
finally:
|
|
(out / 'notices-index.json').write_text(json.dumps(notices, indent=2) + '\n')
|
|
report.update(regularFiles=files, regularBytes=size, members=members, archivedLinksNotMaterialized=links,
|
|
moduleFileCounts=module_counts, moduleVersionDeclarations=versions,
|
|
noticeAndMetadataFiles=len(notices), noticeAndMetadataBytes=notice_bytes,
|
|
completeChromiumNotices=False, completeCorrespondingSources=False,
|
|
inventorySha256=sha(out / 'files.jsonl'), noticesIndexSha256=sha(out / 'notices-index.json'),
|
|
productionBinaries=binaries,
|
|
productionBinariesUnchanged=all(sha(ROOT / 'build' / name) == digest for name, digest in binaries.items()))
|
|
report['success'] &= report['productionBinariesUnchanged']
|
|
(out / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
print(json.dumps({key: report.get(key) for key in ['success', 'regularFiles', 'regularBytes', 'noticeAndMetadataFiles', 'error']}))
|
|
return 0 if report['success'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|