initial commit
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download the fixed SDK/source inputs into a disposable VM workspace.
|
||||
|
||||
Qt's public repository provides SHA-1 sidecars; record independent SHA-256
|
||||
digests as well. No archives are executed/extracted by this script.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--work', type=Path, default=Path('build-ubuntu-vm'))
|
||||
args = parser.parse_args()
|
||||
root = args.work.resolve()
|
||||
downloads = root / 'sdk-downloads'
|
||||
downloads.mkdir(parents=True, exist_ok=True)
|
||||
base = 'https://download.qt.io/online/qtsdkrepository/'
|
||||
repositories = [
|
||||
('qt-base-Updates.xml', 'linux_x64/desktop/qt6_6112/qt6_6112/'),
|
||||
('qt-webengine-Updates.xml', 'linux_x64/extensions/qtwebengine/6112/x86_64/'),
|
||||
]
|
||||
selected = {
|
||||
'qt.qt6.6112.linux_gcc_64',
|
||||
'qt.qt6.6112.addons.qtwebchannel.linux_gcc_64',
|
||||
'qt.qt6.6112.addons.qtpositioning.linux_gcc_64',
|
||||
'extensions.qtwebengine.6112.linux_gcc_64',
|
||||
}
|
||||
inputs = []
|
||||
for metadata, repository in repositories:
|
||||
for package in ET.fromstring((root / 'downloads' / metadata).read_bytes()).findall('PackageUpdate'):
|
||||
name = package.findtext('Name')
|
||||
if name not in selected:
|
||||
continue
|
||||
for archive in package.findtext('DownloadableArchives').split(','):
|
||||
filename = package.findtext('Version') + archive.strip()
|
||||
url = base + repository + name + '/' + filename
|
||||
inputs.append({'name': filename, 'url': url, 'kind': 'qt',
|
||||
'checksumUrl': url + '.sha1', 'checksumAlgorithm': 'sha1'})
|
||||
inputs += [
|
||||
{'name': 'qpdf-12.4.1.tar.gz', 'kind': 'source',
|
||||
'url': 'https://github.com/qpdf/qpdf/releases/download/v12.4.1/qpdf-12.4.1.tar.gz',
|
||||
'checksumAlgorithm': 'sha256',
|
||||
'expected': 'f045aa277be2356ff53a89a8622945958291177d2483afc20ede7c8a8cd3873c',
|
||||
'checksumSource': 'https://api.github.com/repos/qpdf/qpdf/releases/tags/v12.4.1'},
|
||||
{'name': 'libzip-1.11.4.tar.xz', 'kind': 'source',
|
||||
'url': 'https://libzip.org/download/libzip-1.11.4.tar.xz',
|
||||
'checksumAlgorithm': 'sha256',
|
||||
'expected': '8a247f57d1e3e6f6d11413b12a6f28a9d388de110adc0ec608d893180ed7097b',
|
||||
'checksumSource': 'https://github.com/flathub/org.kde.ark/blob/master/org.kde.ark.json'},
|
||||
]
|
||||
|
||||
|
||||
def fetch(item):
|
||||
item = dict(item)
|
||||
target = downloads / item['name']
|
||||
if 'checksumUrl' in item:
|
||||
with urllib.request.urlopen(item['checksumUrl'], timeout=45) as response:
|
||||
checksum = response.read(512).decode().strip().split()[0]
|
||||
if len(checksum) != 40 or any(c not in '0123456789abcdef' for c in checksum):
|
||||
raise ValueError('invalid SHA-1 sidecar')
|
||||
item['expected'] = checksum
|
||||
(downloads / (item['name'] + '.sha1')).write_text(checksum + '\n')
|
||||
if not target.exists():
|
||||
partial = target.with_suffix(target.suffix + '.part')
|
||||
count = 0
|
||||
with urllib.request.urlopen(item['url'], timeout=90) as response, partial.open('wb') as stream:
|
||||
item['finalUrl'] = response.url
|
||||
while block := response.read(1024 * 1024):
|
||||
count += len(block)
|
||||
if count > 250 * 1024 * 1024:
|
||||
raise ValueError('archive exceeds per-file download bound')
|
||||
stream.write(block)
|
||||
partial.replace(target)
|
||||
with target.open('rb') as stream:
|
||||
actual = hashlib.file_digest(stream, item['checksumAlgorithm']).hexdigest()
|
||||
if actual != item['expected']:
|
||||
raise ValueError('checksum mismatch: ' + item['name'])
|
||||
with target.open('rb') as stream:
|
||||
item['sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
item['size'] = target.stat().st_size
|
||||
item['checksumMatched'] = True
|
||||
print(item['name'], item['size'], 'verified', flush=True)
|
||||
return item
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
||||
records = list(pool.map(fetch, inputs))
|
||||
(root / 'metadata' / 'sdk-downloads.json').write_text(json.dumps(records, indent=2) + '\n')
|
||||
print('Total verified bytes:', sum(item['size'] for item in records), flush=True)
|
||||
Reference in New Issue
Block a user