247 lines
12 KiB
Python
247 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Collect fixed PDFium Git snapshots without executing checkout/build scripts."""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timezone
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import subprocess
|
|
import tarfile
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
EVIDENCE = ROOT / 'tests/results/source-correspondence/pdfium'
|
|
PINS = EVIDENCE / 'dependency-pins.json'
|
|
ENV = dict(os.environ, GIT_CONFIG_GLOBAL='/dev/null', GIT_CONFIG_NOSYSTEM='1',
|
|
GIT_TERMINAL_PROMPT='0', GIT_CONFIG_COUNT='0', LC_ALL='C')
|
|
GIT = ['git', '-c', 'core.hooksPath=/dev/null', '-c', 'credential.helper=',
|
|
'-c', 'protocol.file.allow=never', '-c', 'transfer.fsckObjects=true',
|
|
'-c', 'fetch.fsckObjects=true', '-c', 'gc.auto=0']
|
|
|
|
|
|
def sha(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def save(path, value):
|
|
path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
|
|
|
|
|
|
def run(repo, arguments, timeout=120):
|
|
command = GIT + ['--git-dir=' + str(repo), *arguments]
|
|
return subprocess.run(command, env=ENV, capture_output=True, timeout=timeout)
|
|
|
|
|
|
def plan():
|
|
data = json.loads(PINS.read_text())
|
|
root = json.loads((EVIDENCE / 'upstream/commit.json').read_text().removeprefix(")]}'\n"))
|
|
if root['commit'] != data['rootRevision']:
|
|
raise ValueError('Root commit evidence differs')
|
|
selected = [{'path': '.', 'url': 'https://pdfium.googlesource.com/pdfium.git',
|
|
'revision': data['rootRevision'], 'expectedTree': root['tree'], 'condition': None}]
|
|
excluded, packages = [], []
|
|
# Explicit union of Linux x64 and Windows x64 minimal, V8/Rust/Skia-off
|
|
# checkouts. Refuse new predicates instead of executing any DEPS code.
|
|
conditions = {'checkout_libpng': True, 'checkout_win': True,
|
|
'checkout_testing_corpus': False, 'checkout_rust': False,
|
|
'checkout_android': False, 'checkout_v8': False, 'checkout_skia': False}
|
|
for name, value in data['root']['deps'].items():
|
|
if isinstance(value, str):
|
|
url, condition = value, None
|
|
elif 'url' in value:
|
|
url, condition = value['url'], value.get('condition')
|
|
else:
|
|
packages.append({'path': name, 'pin': value})
|
|
continue
|
|
if condition is not None and condition not in conditions:
|
|
raise ValueError('Unreviewed Git condition: ' + condition)
|
|
url, revision = url.rsplit('@', 1)
|
|
parsed = urlsplit(url)
|
|
if (parsed.scheme != 'https' or parsed.netloc not in {
|
|
'pdfium.googlesource.com', 'chromium.googlesource.com', 'skia.googlesource.com'}
|
|
or not re.fullmatch('[0-9a-f]{40}', revision)):
|
|
raise ValueError('Unexpected source URL or revision')
|
|
item = {'path': name, 'url': url, 'revision': revision, 'condition': condition}
|
|
(selected if condition is None or conditions[condition] else excluded).append(item)
|
|
for name, recursive in data['recursive'].items():
|
|
for subpath, value in recursive['deps'].items():
|
|
if isinstance(value, str) or 'url' in value:
|
|
raise ValueError('Unreviewed recursive Git source')
|
|
packages.append({'path': name + '/' + subpath, 'pin': value})
|
|
return {'selectedGit': selected, 'excludedGit': excluded, 'uncollectedPackages': packages,
|
|
'conditionValues': conditions,
|
|
'scope': 'Union of Linux/Windows x64 minimal Git snapshots; not a linked-target list. '
|
|
'No hooks, CIPD/GCS packages, compiler/sysroot downloads or build executed.'}
|
|
|
|
|
|
def export(repo, item, archive, inventory):
|
|
listing = run(repo, ['ls-tree', '-rlz', item['revision']])
|
|
listing.check_returncode()
|
|
entries, total = [], 0
|
|
for raw in listing.stdout.split(b'\0'):
|
|
if not raw:
|
|
continue
|
|
metadata, name = raw.split(b'\t', 1)
|
|
mode, kind, oid, size = metadata.decode('ascii').split()
|
|
name = name.decode('utf-8')
|
|
path = PurePosixPath(name)
|
|
if path.is_absolute() or '..' in path.parts or '\\' in name:
|
|
raise ValueError('Unsafe Git path')
|
|
size = int(size) if size != '-' else 0
|
|
if size > 512 * 1024**2 or mode not in {'100644', '100755', '120000', '160000'}:
|
|
raise ValueError('Unexpected source entry')
|
|
total += size
|
|
entries.append({'path': name, 'mode': mode, 'type': kind, 'gitObject': oid, 'bytes': size})
|
|
if len(entries) > 300000 or total > 4 * 1024**3:
|
|
raise ValueError('Source snapshot exceeds collection limits')
|
|
prefix = 'pdfium/' + (item['path'] + '/' if item['path'] != '.' else '')
|
|
child = subprocess.Popen(GIT + ['--git-dir=' + str(repo), 'cat-file', '--batch'],
|
|
env=ENV, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
try:
|
|
with archive.open('xb') as raw, gzip.GzipFile(filename='', mode='wb', fileobj=raw, mtime=0) as zipped:
|
|
with tarfile.open(fileobj=zipped, mode='w|', format=tarfile.PAX_FORMAT) as tar:
|
|
for entry in entries:
|
|
if entry['type'] == 'commit':
|
|
entry['archived'] = False
|
|
continue
|
|
child.stdin.write((entry['gitObject'] + '\n').encode()); child.stdin.flush()
|
|
header = child.stdout.readline().decode().split()
|
|
if header != [entry['gitObject'], 'blob', str(entry['bytes'])]:
|
|
raise ValueError('Git blob header differs')
|
|
blob = child.stdout.read(entry['bytes'])
|
|
if len(blob) != entry['bytes'] or child.stdout.read(1) != b'\n':
|
|
raise ValueError('Short Git blob read')
|
|
object_hash = hashlib.sha1(b'blob ' + str(len(blob)).encode() + b'\0' + blob).hexdigest()
|
|
if object_hash != entry['gitObject']:
|
|
raise ValueError('Git object content differs')
|
|
entry['sha256'] = hashlib.sha256(blob).hexdigest()
|
|
entry['archived'] = True
|
|
info = tarfile.TarInfo(prefix + entry['path'])
|
|
info.mtime = 0
|
|
info.mode = int(entry['mode'][-3:], 8)
|
|
if entry['mode'] == '120000':
|
|
info.type = tarfile.SYMTYPE
|
|
info.linkname = blob.decode('utf-8')
|
|
entry['linkTarget'] = info.linkname
|
|
tar.addfile(info)
|
|
else:
|
|
info.size = len(blob)
|
|
tar.addfile(info, io.BytesIO(blob))
|
|
child.stdin.close()
|
|
if child.wait(timeout=60) != 0:
|
|
raise ValueError('Git cat-file failed')
|
|
finally:
|
|
if child.poll() is None:
|
|
child.kill(); child.wait()
|
|
with inventory.open('x') as stream:
|
|
for entry in entries:
|
|
stream.write(json.dumps(entry, ensure_ascii=False, sort_keys=True) + '\n')
|
|
# Read the entire resulting archive and independently compare payloads.
|
|
by_name = {prefix + e['path']: e for e in entries if e['archived']}
|
|
with tarfile.open(archive, 'r:gz') as tar:
|
|
for member in tar:
|
|
entry = by_name.pop(member.name)
|
|
if member.issym():
|
|
if member.linkname != entry['linkTarget']:
|
|
raise ValueError('Archived symlink differs')
|
|
else:
|
|
with tar.extractfile(member) as stream:
|
|
digest = hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
if digest != entry['sha256'] or member.size != entry['bytes']:
|
|
raise ValueError('Archived blob differs')
|
|
if by_name:
|
|
raise ValueError('Archive omitted entries')
|
|
return {'entries': len(entries), 'sourceBytes': total,
|
|
'regularFiles': sum(e['type'] == 'blob' and e['mode'] != '120000' for e in entries),
|
|
'symlinks': sum(e['mode'] == '120000' for e in entries),
|
|
'gitlinks': [e for e in entries if e['type'] == 'commit'],
|
|
'archiveVerifiedAgainstGitBlobs': True}
|
|
|
|
|
|
def collect(item, cache, output):
|
|
key = 'root' if item['path'] == '.' else item['path'].replace('/', '--')
|
|
directory = output / key
|
|
directory.mkdir()
|
|
repo = cache / (key + '.git')
|
|
# Reuse the previously fetched, verified root; no source checkout is made.
|
|
probe = ROOT / '.deps/source-archives/pdfium-probe/repository.git'
|
|
if key == 'root' and probe.is_dir():
|
|
repo = probe
|
|
report = dict(item, repository=str(repo.relative_to(ROOT)), success=False)
|
|
try:
|
|
if not repo.exists():
|
|
subprocess.run(GIT + ['init', '--bare', '--template=', str(repo)], env=ENV,
|
|
capture_output=True, check=True, timeout=20)
|
|
exists = run(repo, ['cat-file', '-e', item['revision'] + '^{commit}'])
|
|
if exists.returncode:
|
|
fetched = run(repo, ['fetch', '--depth=1', '--no-tags', item['url'], item['revision']], timeout=900)
|
|
(directory / 'fetch.log').write_bytes(fetched.stdout + fetched.stderr)
|
|
fetched.check_returncode()
|
|
report['fetched'] = True
|
|
else:
|
|
report['fetched'] = False
|
|
commit = run(repo, ['rev-parse', item['revision'] + '^{commit}'])
|
|
commit.check_returncode()
|
|
if commit.stdout.decode().strip() != item['revision']:
|
|
raise ValueError('Commit mismatch')
|
|
tree = run(repo, ['rev-parse', item['revision'] + '^{tree}'])
|
|
tree.check_returncode()
|
|
report['tree'] = tree.stdout.decode().strip()
|
|
if item.get('expectedTree') and report['tree'] != item['expectedTree']:
|
|
raise ValueError('Root tree differs from saved Gitiles metadata')
|
|
checked = run(repo, ['fsck', '--full', '--strict', '--no-reflogs', item['revision']], timeout=300)
|
|
(directory / 'fsck.log').write_bytes(checked.stdout + checked.stderr)
|
|
checked.check_returncode()
|
|
report['gitFsckPassed'] = True
|
|
archive = cache / (key + '-' + item['revision'] + '.tar.gz')
|
|
inventory = directory / 'files.jsonl'
|
|
report.update(export(repo, item, archive, inventory))
|
|
report.update(archive=str(archive.relative_to(ROOT)), archiveBytes=archive.stat().st_size,
|
|
archiveSha256=sha(archive), inventory=str(inventory.relative_to(ROOT)),
|
|
inventorySha256=sha(inventory), success=True)
|
|
except Exception as error:
|
|
report['error'] = type(error).__name__ + ': ' + str(error)
|
|
save(directory / 'report.json', report)
|
|
print(json.dumps({'path': item['path'], 'success': report['success'],
|
|
'error': report.get('error'), 'bytes': report.get('archiveBytes')}), flush=True)
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--cache', type=Path, default=ROOT / '.deps/source-archives/pdfium')
|
|
args = parser.parse_args()
|
|
output, cache = args.output.resolve(), args.cache.resolve()
|
|
if not output.is_relative_to(ROOT) or not cache.is_relative_to(ROOT):
|
|
parser.error('Output/cache must be inside the workspace')
|
|
selected = plan()
|
|
output.mkdir(parents=True, exist_ok=False); cache.mkdir(parents=True, exist_ok=True)
|
|
save(output / 'plan.json', selected)
|
|
report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'pinSha256': sha(PINS),
|
|
'collectorSha256': sha(Path(__file__)), 'planSha256': sha(output / 'plan.json'),
|
|
'gitVersion': subprocess.check_output(['git', '--version'], env=ENV, text=True).strip()}
|
|
values = []
|
|
with ThreadPoolExecutor(max_workers=3) as pool:
|
|
futures = [pool.submit(collect, item, cache, output) for item in selected['selectedGit']]
|
|
for future in as_completed(futures):
|
|
values.append(future.result())
|
|
report.update(success=all(x['success'] for x in values), repositories=sorted(values, key=lambda x: x['path']),
|
|
finishedAt=datetime.now(timezone.utc).isoformat(),
|
|
completeCorrespondingSources=False, buildReproduced=False,
|
|
scope=selected['scope'])
|
|
save(output / 'report.json', report)
|
|
print(json.dumps({'success': report['success'], 'repositories': len(values)}))
|
|
return 0 if report['success'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|