#!/usr/bin/env python3 """Collect fixed PDFium Git snapshots without executing checkout/build scripts.""" import argparse from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone import gzip import hashlib import io import json import os from pathlib import Path, PurePosixPath import re import subprocess import tarfile from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[2] EVIDENCE = ROOT / 'tests/results/source-correspondence/pdfium' PINS = EVIDENCE / 'dependency-pins.json' ENV = dict(os.environ, GIT_CONFIG_GLOBAL='/dev/null', GIT_CONFIG_NOSYSTEM='1', GIT_TERMINAL_PROMPT='0', GIT_CONFIG_COUNT='0', LC_ALL='C') GIT = ['git', '-c', 'core.hooksPath=/dev/null', '-c', 'credential.helper=', '-c', 'protocol.file.allow=never', '-c', 'transfer.fsckObjects=true', '-c', 'fetch.fsckObjects=true', '-c', 'gc.auto=0'] def sha(path): with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() def save(path, value): path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n') def run(repo, arguments, timeout=120): command = GIT + ['--git-dir=' + str(repo), *arguments] return subprocess.run(command, env=ENV, capture_output=True, timeout=timeout) def plan(): data = json.loads(PINS.read_text()) root = json.loads((EVIDENCE / 'upstream/commit.json').read_text().removeprefix(")]}'\n")) if root['commit'] != data['rootRevision']: raise ValueError('Root commit evidence differs') selected = [{'path': '.', 'url': 'https://pdfium.googlesource.com/pdfium.git', 'revision': data['rootRevision'], 'expectedTree': root['tree'], 'condition': None}] excluded, packages = [], [] # Explicit union of Linux x64 and Windows x64 minimal, V8/Rust/Skia-off # checkouts. Refuse new predicates instead of executing any DEPS code. conditions = {'checkout_libpng': True, 'checkout_win': True, 'checkout_testing_corpus': False, 'checkout_rust': False, 'checkout_android': False, 'checkout_v8': False, 'checkout_skia': False} for name, value in data['root']['deps'].items(): if isinstance(value, str): url, condition = value, None elif 'url' in value: url, condition = value['url'], value.get('condition') else: packages.append({'path': name, 'pin': value}) continue if condition is not None and condition not in conditions: raise ValueError('Unreviewed Git condition: ' + condition) url, revision = url.rsplit('@', 1) parsed = urlsplit(url) if (parsed.scheme != 'https' or parsed.netloc not in { 'pdfium.googlesource.com', 'chromium.googlesource.com', 'skia.googlesource.com'} or not re.fullmatch('[0-9a-f]{40}', revision)): raise ValueError('Unexpected source URL or revision') item = {'path': name, 'url': url, 'revision': revision, 'condition': condition} (selected if condition is None or conditions[condition] else excluded).append(item) for name, recursive in data['recursive'].items(): for subpath, value in recursive['deps'].items(): if isinstance(value, str) or 'url' in value: raise ValueError('Unreviewed recursive Git source') packages.append({'path': name + '/' + subpath, 'pin': value}) return {'selectedGit': selected, 'excludedGit': excluded, 'uncollectedPackages': packages, 'conditionValues': conditions, 'scope': 'Union of Linux/Windows x64 minimal Git snapshots; not a linked-target list. ' 'No hooks, CIPD/GCS packages, compiler/sysroot downloads or build executed.'} def export(repo, item, archive, inventory): listing = run(repo, ['ls-tree', '-rlz', item['revision']]) listing.check_returncode() entries, total = [], 0 for raw in listing.stdout.split(b'\0'): if not raw: continue metadata, name = raw.split(b'\t', 1) mode, kind, oid, size = metadata.decode('ascii').split() name = name.decode('utf-8') path = PurePosixPath(name) if path.is_absolute() or '..' in path.parts or '\\' in name: raise ValueError('Unsafe Git path') size = int(size) if size != '-' else 0 if size > 512 * 1024**2 or mode not in {'100644', '100755', '120000', '160000'}: raise ValueError('Unexpected source entry') total += size entries.append({'path': name, 'mode': mode, 'type': kind, 'gitObject': oid, 'bytes': size}) if len(entries) > 300000 or total > 4 * 1024**3: raise ValueError('Source snapshot exceeds collection limits') prefix = 'pdfium/' + (item['path'] + '/' if item['path'] != '.' else '') child = subprocess.Popen(GIT + ['--git-dir=' + str(repo), 'cat-file', '--batch'], env=ENV, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: with archive.open('xb') as raw, gzip.GzipFile(filename='', mode='wb', fileobj=raw, mtime=0) as zipped: with tarfile.open(fileobj=zipped, mode='w|', format=tarfile.PAX_FORMAT) as tar: for entry in entries: if entry['type'] == 'commit': entry['archived'] = False continue child.stdin.write((entry['gitObject'] + '\n').encode()); child.stdin.flush() header = child.stdout.readline().decode().split() if header != [entry['gitObject'], 'blob', str(entry['bytes'])]: raise ValueError('Git blob header differs') blob = child.stdout.read(entry['bytes']) if len(blob) != entry['bytes'] or child.stdout.read(1) != b'\n': raise ValueError('Short Git blob read') object_hash = hashlib.sha1(b'blob ' + str(len(blob)).encode() + b'\0' + blob).hexdigest() if object_hash != entry['gitObject']: raise ValueError('Git object content differs') entry['sha256'] = hashlib.sha256(blob).hexdigest() entry['archived'] = True info = tarfile.TarInfo(prefix + entry['path']) info.mtime = 0 info.mode = int(entry['mode'][-3:], 8) if entry['mode'] == '120000': info.type = tarfile.SYMTYPE info.linkname = blob.decode('utf-8') entry['linkTarget'] = info.linkname tar.addfile(info) else: info.size = len(blob) tar.addfile(info, io.BytesIO(blob)) child.stdin.close() if child.wait(timeout=60) != 0: raise ValueError('Git cat-file failed') finally: if child.poll() is None: child.kill(); child.wait() with inventory.open('x') as stream: for entry in entries: stream.write(json.dumps(entry, ensure_ascii=False, sort_keys=True) + '\n') # Read the entire resulting archive and independently compare payloads. by_name = {prefix + e['path']: e for e in entries if e['archived']} with tarfile.open(archive, 'r:gz') as tar: for member in tar: entry = by_name.pop(member.name) if member.issym(): if member.linkname != entry['linkTarget']: raise ValueError('Archived symlink differs') else: with tar.extractfile(member) as stream: digest = hashlib.file_digest(stream, 'sha256').hexdigest() if digest != entry['sha256'] or member.size != entry['bytes']: raise ValueError('Archived blob differs') if by_name: raise ValueError('Archive omitted entries') return {'entries': len(entries), 'sourceBytes': total, 'regularFiles': sum(e['type'] == 'blob' and e['mode'] != '120000' for e in entries), 'symlinks': sum(e['mode'] == '120000' for e in entries), 'gitlinks': [e for e in entries if e['type'] == 'commit'], 'archiveVerifiedAgainstGitBlobs': True} def collect(item, cache, output): key = 'root' if item['path'] == '.' else item['path'].replace('/', '--') directory = output / key directory.mkdir() repo = cache / (key + '.git') # Reuse the previously fetched, verified root; no source checkout is made. probe = ROOT / '.deps/source-archives/pdfium-probe/repository.git' if key == 'root' and probe.is_dir(): repo = probe report = dict(item, repository=str(repo.relative_to(ROOT)), success=False) try: if not repo.exists(): subprocess.run(GIT + ['init', '--bare', '--template=', str(repo)], env=ENV, capture_output=True, check=True, timeout=20) exists = run(repo, ['cat-file', '-e', item['revision'] + '^{commit}']) if exists.returncode: fetched = run(repo, ['fetch', '--depth=1', '--no-tags', item['url'], item['revision']], timeout=900) (directory / 'fetch.log').write_bytes(fetched.stdout + fetched.stderr) fetched.check_returncode() report['fetched'] = True else: report['fetched'] = False commit = run(repo, ['rev-parse', item['revision'] + '^{commit}']) commit.check_returncode() if commit.stdout.decode().strip() != item['revision']: raise ValueError('Commit mismatch') tree = run(repo, ['rev-parse', item['revision'] + '^{tree}']) tree.check_returncode() report['tree'] = tree.stdout.decode().strip() if item.get('expectedTree') and report['tree'] != item['expectedTree']: raise ValueError('Root tree differs from saved Gitiles metadata') checked = run(repo, ['fsck', '--full', '--strict', '--no-reflogs', item['revision']], timeout=300) (directory / 'fsck.log').write_bytes(checked.stdout + checked.stderr) checked.check_returncode() report['gitFsckPassed'] = True archive = cache / (key + '-' + item['revision'] + '.tar.gz') inventory = directory / 'files.jsonl' report.update(export(repo, item, archive, inventory)) report.update(archive=str(archive.relative_to(ROOT)), archiveBytes=archive.stat().st_size, archiveSha256=sha(archive), inventory=str(inventory.relative_to(ROOT)), inventorySha256=sha(inventory), success=True) except Exception as error: report['error'] = type(error).__name__ + ': ' + str(error) save(directory / 'report.json', report) print(json.dumps({'path': item['path'], 'success': report['success'], 'error': report.get('error'), 'bytes': report.get('archiveBytes')}), flush=True) return report def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--cache', type=Path, default=ROOT / '.deps/source-archives/pdfium') args = parser.parse_args() output, cache = args.output.resolve(), args.cache.resolve() if not output.is_relative_to(ROOT) or not cache.is_relative_to(ROOT): parser.error('Output/cache must be inside the workspace') selected = plan() output.mkdir(parents=True, exist_ok=False); cache.mkdir(parents=True, exist_ok=True) save(output / 'plan.json', selected) report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'pinSha256': sha(PINS), 'collectorSha256': sha(Path(__file__)), 'planSha256': sha(output / 'plan.json'), 'gitVersion': subprocess.check_output(['git', '--version'], env=ENV, text=True).strip()} values = [] with ThreadPoolExecutor(max_workers=3) as pool: futures = [pool.submit(collect, item, cache, output) for item in selected['selectedGit']] for future in as_completed(futures): values.append(future.result()) report.update(success=all(x['success'] for x in values), repositories=sorted(values, key=lambda x: x['path']), finishedAt=datetime.now(timezone.utc).isoformat(), completeCorrespondingSources=False, buildReproduced=False, scope=selected['scope']) save(output / 'report.json', report) print(json.dumps({'success': report['success'], 'repositories': len(values)})) return 0 if report['success'] else 1 if __name__ == '__main__': raise SystemExit(main())