initial commit
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect the verified Qt release archive and retain regular source files safely."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def sha(path):
|
||||
with path.open('rb') as f:
|
||||
return hashlib.file_digest(f, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--archive', type=Path, required=True)
|
||||
parser.add_argument('--tree', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
archive, tree, out = args.archive.resolve(), args.tree.resolve(), args.output.resolve()
|
||||
if any(not path.is_relative_to(ROOT) for path in [archive, tree, out]):
|
||||
parser.error('All paths must remain in this workspace')
|
||||
pin = json.loads(Path(__file__).with_name('pins.json').read_text())['qt-everywhere']
|
||||
if archive.stat().st_size != pin['bytes'] or sha(archive) != pin['hashes']['sha256']:
|
||||
parser.error('Archive does not match the verified release pin')
|
||||
tree.mkdir(parents=True, exist_ok=False); out.mkdir(parents=True, exist_ok=False)
|
||||
comparison_root = ROOT / 'tests/results/source-correspondence/arch/upstream-metadata'
|
||||
expected = {}
|
||||
for row in json.loads((comparison_root / 'version-sources.json').read_text()):
|
||||
url = urllib.parse.urlparse(row['url']).path
|
||||
if '/qtwebengine.git/plain/' in url:
|
||||
relative = 'qtwebengine/' + url.split('/plain/', 1)[1]
|
||||
elif '/qtwebengine-chromium.git/plain/' in url:
|
||||
relative = 'qtwebengine/src/3rdparty/' + url.split('/plain/', 1)[1]
|
||||
else:
|
||||
continue
|
||||
if sha(comparison_root / row['file']) != row['sha256']:
|
||||
raise ValueError('Stored fixed-revision comparison input changed')
|
||||
expected[relative] = row
|
||||
binaries = {name: sha(ROOT / 'build' / name) for name in ['docview', 'docview-pdf-worker', 'docview-archive-worker']}
|
||||
report = {'success': False, 'archive': str(archive.relative_to(ROOT)), 'archiveSha256': pin['hashes']['sha256'],
|
||||
'archiveBytes': pin['bytes'], 'sourceTree': str(tree.relative_to(ROOT)),
|
||||
'scope': 'Official Qt 6.11.2 release sources and selected metadata/notices; not complete linked-component attribution, a rebuild, license acceptance or legal compliance certification'}
|
||||
files = size = members = notice_bytes = 0
|
||||
notices, links, comparisons, module_counts, versions = [], [], {}, {}, {}
|
||||
seen = set(); last = time.monotonic()
|
||||
try:
|
||||
with tarfile.open(archive, 'r|xz') as stream, (out / 'files.jsonl').open('w') as inventory:
|
||||
for member in stream:
|
||||
members += 1
|
||||
if members > 600000 or member.size > 1024**3 or member.size < 0:
|
||||
raise ValueError('Archive exceeded finite member limits')
|
||||
path = PurePosixPath(member.name)
|
||||
if path.is_absolute() or '..' in path.parts or not path.parts or path.parts[0] != 'qt-everywhere-src-6.11.2' or '\\' in member.name:
|
||||
raise ValueError('Unexpected source member path')
|
||||
relative = PurePosixPath(*path.parts[1:])
|
||||
if member.isdir():
|
||||
continue
|
||||
name = str(relative)
|
||||
if name in seen or name == '.':
|
||||
raise ValueError('Duplicate or empty source file path')
|
||||
seen.add(name)
|
||||
if member.issym() or member.islnk():
|
||||
links.append({'path': name, 'target': member.linkname, 'kind': 'symlink' if member.issym() else 'hardlink'})
|
||||
continue # Keep links in the original archive; never follow them.
|
||||
if not member.isfile():
|
||||
raise ValueError('Non-file source entry')
|
||||
size += member.size
|
||||
if size > 20 * 1024**3:
|
||||
raise ValueError('Source tree exceeded 20 GiB limit')
|
||||
destination = tree / name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
digest = hashlib.sha256(); count = 0
|
||||
with stream.extractfile(member) as source, destination.open('xb') as target:
|
||||
while block := source.read(1024 * 1024):
|
||||
target.write(block); digest.update(block); count += len(block)
|
||||
if count != member.size:
|
||||
raise ValueError('Source member was truncated')
|
||||
row = {'path': name, 'bytes': count, 'sha256': digest.hexdigest(), 'archiveMode': member.mode}
|
||||
inventory.write(json.dumps(row) + '\n'); files += 1
|
||||
module = relative.parts[0] if len(relative.parts) > 1 else '(root)'
|
||||
module_counts[module] = module_counts.get(module, 0) + 1
|
||||
if name in expected:
|
||||
comparisons[name] = {**row, 'expectedSha256': expected[name]['sha256'],
|
||||
'matches': row['sha256'] == expected[name]['sha256'], 'sourceUrl': expected[name]['url']}
|
||||
if relative.name == '.cmake.conf' and len(relative.parts) == 2:
|
||||
text = destination.read_text(errors='replace')
|
||||
versions[module] = re.findall(r'QT_REPO_MODULE_VERSION\s+"([^"]+)"', text)
|
||||
# Keep a named-file subset; README.chromium and attribution
|
||||
# metadata may refer to further texts. Do not call this complete.
|
||||
if (re.fullmatch(r'(licen[cs]e|copying|copyright|notice)([._-].*)?', relative.name, re.I)
|
||||
or relative.name in ('README.chromium', 'qt_attributions.json', 'REUSE.toml')
|
||||
or 'LICENSES' in relative.parts):
|
||||
if count > 8 * 1024**2 or notice_bytes + count > 128 * 1024**2:
|
||||
raise ValueError('Notice metadata exceeded finite limits')
|
||||
notice_bytes += count; notices.append(row)
|
||||
if time.monotonic() - last > 15:
|
||||
print(f'Qt source: {files} files / {size / 1048576:.1f} MiB inspected', flush=True)
|
||||
last = time.monotonic()
|
||||
report['fixedSourceComparisons'] = comparisons
|
||||
missing = sorted(set(expected) - set(comparisons))
|
||||
report['missingComparisons'] = missing
|
||||
if missing or not all(row['matches'] for row in comparisons.values()):
|
||||
raise ValueError('Release source differs from a fixed-revision comparison input')
|
||||
report['success'] = True
|
||||
except Exception as error:
|
||||
report['error'] = str(error)
|
||||
finally:
|
||||
(out / 'notices-index.json').write_text(json.dumps(notices, indent=2) + '\n')
|
||||
report.update(regularFiles=files, regularBytes=size, members=members, archivedLinksNotMaterialized=links,
|
||||
moduleFileCounts=module_counts, moduleVersionDeclarations=versions,
|
||||
noticeAndMetadataFiles=len(notices), noticeAndMetadataBytes=notice_bytes,
|
||||
completeChromiumNotices=False, completeCorrespondingSources=False,
|
||||
inventorySha256=sha(out / 'files.jsonl'), noticesIndexSha256=sha(out / 'notices-index.json'),
|
||||
productionBinaries=binaries,
|
||||
productionBinariesUnchanged=all(sha(ROOT / 'build' / name) == digest for name, digest in binaries.items()))
|
||||
report['success'] &= report['productionBinariesUnchanged']
|
||||
(out / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
print(json.dumps({key: report.get(key) for key in ['success', 'regularFiles', 'regularBytes', 'noticeAndMetadataFiles', 'error']}))
|
||||
return 0 if report['success'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user