initial commit
This commit is contained in:
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect fixed PDFium Git snapshots without executing checkout/build scripts."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
EVIDENCE = ROOT / 'tests/results/source-correspondence/pdfium'
|
||||
PINS = EVIDENCE / 'dependency-pins.json'
|
||||
ENV = dict(os.environ, GIT_CONFIG_GLOBAL='/dev/null', GIT_CONFIG_NOSYSTEM='1',
|
||||
GIT_TERMINAL_PROMPT='0', GIT_CONFIG_COUNT='0', LC_ALL='C')
|
||||
GIT = ['git', '-c', 'core.hooksPath=/dev/null', '-c', 'credential.helper=',
|
||||
'-c', 'protocol.file.allow=never', '-c', 'transfer.fsckObjects=true',
|
||||
'-c', 'fetch.fsckObjects=true', '-c', 'gc.auto=0']
|
||||
|
||||
|
||||
def sha(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def save(path, value):
|
||||
path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
def run(repo, arguments, timeout=120):
|
||||
command = GIT + ['--git-dir=' + str(repo), *arguments]
|
||||
return subprocess.run(command, env=ENV, capture_output=True, timeout=timeout)
|
||||
|
||||
|
||||
def plan():
|
||||
data = json.loads(PINS.read_text())
|
||||
root = json.loads((EVIDENCE / 'upstream/commit.json').read_text().removeprefix(")]}'\n"))
|
||||
if root['commit'] != data['rootRevision']:
|
||||
raise ValueError('Root commit evidence differs')
|
||||
selected = [{'path': '.', 'url': 'https://pdfium.googlesource.com/pdfium.git',
|
||||
'revision': data['rootRevision'], 'expectedTree': root['tree'], 'condition': None}]
|
||||
excluded, packages = [], []
|
||||
# Explicit union of Linux x64 and Windows x64 minimal, V8/Rust/Skia-off
|
||||
# checkouts. Refuse new predicates instead of executing any DEPS code.
|
||||
conditions = {'checkout_libpng': True, 'checkout_win': True,
|
||||
'checkout_testing_corpus': False, 'checkout_rust': False,
|
||||
'checkout_android': False, 'checkout_v8': False, 'checkout_skia': False}
|
||||
for name, value in data['root']['deps'].items():
|
||||
if isinstance(value, str):
|
||||
url, condition = value, None
|
||||
elif 'url' in value:
|
||||
url, condition = value['url'], value.get('condition')
|
||||
else:
|
||||
packages.append({'path': name, 'pin': value})
|
||||
continue
|
||||
if condition is not None and condition not in conditions:
|
||||
raise ValueError('Unreviewed Git condition: ' + condition)
|
||||
url, revision = url.rsplit('@', 1)
|
||||
parsed = urlsplit(url)
|
||||
if (parsed.scheme != 'https' or parsed.netloc not in {
|
||||
'pdfium.googlesource.com', 'chromium.googlesource.com', 'skia.googlesource.com'}
|
||||
or not re.fullmatch('[0-9a-f]{40}', revision)):
|
||||
raise ValueError('Unexpected source URL or revision')
|
||||
item = {'path': name, 'url': url, 'revision': revision, 'condition': condition}
|
||||
(selected if condition is None or conditions[condition] else excluded).append(item)
|
||||
for name, recursive in data['recursive'].items():
|
||||
for subpath, value in recursive['deps'].items():
|
||||
if isinstance(value, str) or 'url' in value:
|
||||
raise ValueError('Unreviewed recursive Git source')
|
||||
packages.append({'path': name + '/' + subpath, 'pin': value})
|
||||
return {'selectedGit': selected, 'excludedGit': excluded, 'uncollectedPackages': packages,
|
||||
'conditionValues': conditions,
|
||||
'scope': 'Union of Linux/Windows x64 minimal Git snapshots; not a linked-target list. '
|
||||
'No hooks, CIPD/GCS packages, compiler/sysroot downloads or build executed.'}
|
||||
|
||||
|
||||
def export(repo, item, archive, inventory):
|
||||
listing = run(repo, ['ls-tree', '-rlz', item['revision']])
|
||||
listing.check_returncode()
|
||||
entries, total = [], 0
|
||||
for raw in listing.stdout.split(b'\0'):
|
||||
if not raw:
|
||||
continue
|
||||
metadata, name = raw.split(b'\t', 1)
|
||||
mode, kind, oid, size = metadata.decode('ascii').split()
|
||||
name = name.decode('utf-8')
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or '..' in path.parts or '\\' in name:
|
||||
raise ValueError('Unsafe Git path')
|
||||
size = int(size) if size != '-' else 0
|
||||
if size > 512 * 1024**2 or mode not in {'100644', '100755', '120000', '160000'}:
|
||||
raise ValueError('Unexpected source entry')
|
||||
total += size
|
||||
entries.append({'path': name, 'mode': mode, 'type': kind, 'gitObject': oid, 'bytes': size})
|
||||
if len(entries) > 300000 or total > 4 * 1024**3:
|
||||
raise ValueError('Source snapshot exceeds collection limits')
|
||||
prefix = 'pdfium/' + (item['path'] + '/' if item['path'] != '.' else '')
|
||||
child = subprocess.Popen(GIT + ['--git-dir=' + str(repo), 'cat-file', '--batch'],
|
||||
env=ENV, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
try:
|
||||
with archive.open('xb') as raw, gzip.GzipFile(filename='', mode='wb', fileobj=raw, mtime=0) as zipped:
|
||||
with tarfile.open(fileobj=zipped, mode='w|', format=tarfile.PAX_FORMAT) as tar:
|
||||
for entry in entries:
|
||||
if entry['type'] == 'commit':
|
||||
entry['archived'] = False
|
||||
continue
|
||||
child.stdin.write((entry['gitObject'] + '\n').encode()); child.stdin.flush()
|
||||
header = child.stdout.readline().decode().split()
|
||||
if header != [entry['gitObject'], 'blob', str(entry['bytes'])]:
|
||||
raise ValueError('Git blob header differs')
|
||||
blob = child.stdout.read(entry['bytes'])
|
||||
if len(blob) != entry['bytes'] or child.stdout.read(1) != b'\n':
|
||||
raise ValueError('Short Git blob read')
|
||||
object_hash = hashlib.sha1(b'blob ' + str(len(blob)).encode() + b'\0' + blob).hexdigest()
|
||||
if object_hash != entry['gitObject']:
|
||||
raise ValueError('Git object content differs')
|
||||
entry['sha256'] = hashlib.sha256(blob).hexdigest()
|
||||
entry['archived'] = True
|
||||
info = tarfile.TarInfo(prefix + entry['path'])
|
||||
info.mtime = 0
|
||||
info.mode = int(entry['mode'][-3:], 8)
|
||||
if entry['mode'] == '120000':
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkname = blob.decode('utf-8')
|
||||
entry['linkTarget'] = info.linkname
|
||||
tar.addfile(info)
|
||||
else:
|
||||
info.size = len(blob)
|
||||
tar.addfile(info, io.BytesIO(blob))
|
||||
child.stdin.close()
|
||||
if child.wait(timeout=60) != 0:
|
||||
raise ValueError('Git cat-file failed')
|
||||
finally:
|
||||
if child.poll() is None:
|
||||
child.kill(); child.wait()
|
||||
with inventory.open('x') as stream:
|
||||
for entry in entries:
|
||||
stream.write(json.dumps(entry, ensure_ascii=False, sort_keys=True) + '\n')
|
||||
# Read the entire resulting archive and independently compare payloads.
|
||||
by_name = {prefix + e['path']: e for e in entries if e['archived']}
|
||||
with tarfile.open(archive, 'r:gz') as tar:
|
||||
for member in tar:
|
||||
entry = by_name.pop(member.name)
|
||||
if member.issym():
|
||||
if member.linkname != entry['linkTarget']:
|
||||
raise ValueError('Archived symlink differs')
|
||||
else:
|
||||
with tar.extractfile(member) as stream:
|
||||
digest = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
if digest != entry['sha256'] or member.size != entry['bytes']:
|
||||
raise ValueError('Archived blob differs')
|
||||
if by_name:
|
||||
raise ValueError('Archive omitted entries')
|
||||
return {'entries': len(entries), 'sourceBytes': total,
|
||||
'regularFiles': sum(e['type'] == 'blob' and e['mode'] != '120000' for e in entries),
|
||||
'symlinks': sum(e['mode'] == '120000' for e in entries),
|
||||
'gitlinks': [e for e in entries if e['type'] == 'commit'],
|
||||
'archiveVerifiedAgainstGitBlobs': True}
|
||||
|
||||
|
||||
def collect(item, cache, output):
|
||||
key = 'root' if item['path'] == '.' else item['path'].replace('/', '--')
|
||||
directory = output / key
|
||||
directory.mkdir()
|
||||
repo = cache / (key + '.git')
|
||||
# Reuse the previously fetched, verified root; no source checkout is made.
|
||||
probe = ROOT / '.deps/source-archives/pdfium-probe/repository.git'
|
||||
if key == 'root' and probe.is_dir():
|
||||
repo = probe
|
||||
report = dict(item, repository=str(repo.relative_to(ROOT)), success=False)
|
||||
try:
|
||||
if not repo.exists():
|
||||
subprocess.run(GIT + ['init', '--bare', '--template=', str(repo)], env=ENV,
|
||||
capture_output=True, check=True, timeout=20)
|
||||
exists = run(repo, ['cat-file', '-e', item['revision'] + '^{commit}'])
|
||||
if exists.returncode:
|
||||
fetched = run(repo, ['fetch', '--depth=1', '--no-tags', item['url'], item['revision']], timeout=900)
|
||||
(directory / 'fetch.log').write_bytes(fetched.stdout + fetched.stderr)
|
||||
fetched.check_returncode()
|
||||
report['fetched'] = True
|
||||
else:
|
||||
report['fetched'] = False
|
||||
commit = run(repo, ['rev-parse', item['revision'] + '^{commit}'])
|
||||
commit.check_returncode()
|
||||
if commit.stdout.decode().strip() != item['revision']:
|
||||
raise ValueError('Commit mismatch')
|
||||
tree = run(repo, ['rev-parse', item['revision'] + '^{tree}'])
|
||||
tree.check_returncode()
|
||||
report['tree'] = tree.stdout.decode().strip()
|
||||
if item.get('expectedTree') and report['tree'] != item['expectedTree']:
|
||||
raise ValueError('Root tree differs from saved Gitiles metadata')
|
||||
checked = run(repo, ['fsck', '--full', '--strict', '--no-reflogs', item['revision']], timeout=300)
|
||||
(directory / 'fsck.log').write_bytes(checked.stdout + checked.stderr)
|
||||
checked.check_returncode()
|
||||
report['gitFsckPassed'] = True
|
||||
archive = cache / (key + '-' + item['revision'] + '.tar.gz')
|
||||
inventory = directory / 'files.jsonl'
|
||||
report.update(export(repo, item, archive, inventory))
|
||||
report.update(archive=str(archive.relative_to(ROOT)), archiveBytes=archive.stat().st_size,
|
||||
archiveSha256=sha(archive), inventory=str(inventory.relative_to(ROOT)),
|
||||
inventorySha256=sha(inventory), success=True)
|
||||
except Exception as error:
|
||||
report['error'] = type(error).__name__ + ': ' + str(error)
|
||||
save(directory / 'report.json', report)
|
||||
print(json.dumps({'path': item['path'], 'success': report['success'],
|
||||
'error': report.get('error'), 'bytes': report.get('archiveBytes')}), flush=True)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--cache', type=Path, default=ROOT / '.deps/source-archives/pdfium')
|
||||
args = parser.parse_args()
|
||||
output, cache = args.output.resolve(), args.cache.resolve()
|
||||
if not output.is_relative_to(ROOT) or not cache.is_relative_to(ROOT):
|
||||
parser.error('Output/cache must be inside the workspace')
|
||||
selected = plan()
|
||||
output.mkdir(parents=True, exist_ok=False); cache.mkdir(parents=True, exist_ok=True)
|
||||
save(output / 'plan.json', selected)
|
||||
report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'pinSha256': sha(PINS),
|
||||
'collectorSha256': sha(Path(__file__)), 'planSha256': sha(output / 'plan.json'),
|
||||
'gitVersion': subprocess.check_output(['git', '--version'], env=ENV, text=True).strip()}
|
||||
values = []
|
||||
with ThreadPoolExecutor(max_workers=3) as pool:
|
||||
futures = [pool.submit(collect, item, cache, output) for item in selected['selectedGit']]
|
||||
for future in as_completed(futures):
|
||||
values.append(future.result())
|
||||
report.update(success=all(x['success'] for x in values), repositories=sorted(values, key=lambda x: x['path']),
|
||||
finishedAt=datetime.now(timezone.utc).isoformat(),
|
||||
completeCorrespondingSources=False, buildReproduced=False,
|
||||
scope=selected['scope'])
|
||||
save(output / 'report.json', report)
|
||||
print(json.dumps({'success': report['success'], 'repositories': len(values)}))
|
||||
return 0 if report['success'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user