#!/usr/bin/env python3 """Compare collected PDFium source, provider patches and installed public headers.""" import argparse import hashlib import importlib.util import json from pathlib import Path, PurePosixPath import subprocess from collect_pdfium import ROOT, EVIDENCE, sha, run, save def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--collection', type=Path, default=ROOT / 'tests/results/source-archives/pdfium-git') parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() output = args.output.resolve() if not output.is_relative_to(ROOT): parser.error('Use an output directory inside the workspace') collection = args.collection.resolve() report_path = collection / 'report.json' collected = json.loads(report_path.read_text()) if not collected['success']: raise ValueError('Complete successful selected-Git collection required') repositories, inventories = {}, {} for item in collected['repositories']: if sha(ROOT / item['archive']) != item['archiveSha256'] or sha(ROOT / item['inventory']) != item['inventorySha256']: raise ValueError('Source archive or inventory changed') repositories[item['path']] = item inventories[item['path']] = {e['path']: e for line in (ROOT / item['inventory']).open() if (e := json.loads(line))} def blob(repository, path): item = repositories[repository] expected = inventories[repository][path] if expected['mode'] not in {'100644', '100755'}: raise ValueError('Expected a regular source file') result = run(ROOT / item['repository'], ['cat-file', 'blob', expected['gitObject']]) result.check_returncode() if hashlib.sha256(result.stdout).hexdigest() != expected['sha256']: raise ValueError('Source blob differs from inventory') return result.stdout def locate(saved): if saved.startswith('upstream/'): return '.', saved.removeprefix('upstream/') if saved.startswith('dependencies/'): relative = saved.removeprefix('dependencies/') key = next(k for k in sorted(repositories, key=len, reverse=True) if relative.startswith(k + '/')) return key, relative[len(key) + 1:] raise ValueError('Unexpected saved source') output.mkdir(parents=True, exist_ok=False) # Tie earlier Gitiles bytes to the full fetched snapshots, not just filenames. correspondence = [] for folder in ('upstream', 'dependencies'): for sidecar in sorted((EVIDENCE / folder).rglob('*.retrieval.json')): retrieval = json.loads(sidecar.read_text()) if retrieval['transform'] != 'base64 decoded Gitiles response': continue saved = retrieval['file'] repository, path = locate(saved) content = blob(repository, path) digest = hashlib.sha256(content).hexdigest() if digest != retrieval['sha256'] or content != (EVIDENCE / saved).read_bytes(): raise ValueError('Saved Gitiles source differs: ' + saved) correspondence.append({'saved': saved, 'repository': repository, 'path': path, 'revision': repositories[repository]['revision'], 'sha256': digest}) provider = EVIDENCE / 'provider/recipe' provider_report = json.loads((EVIDENCE / 'report.json').read_text()) recipe_hashes = {e['file']: e['sha256'] for e in provider_report['recipeFilesVerifiedAgainstGitTree']} for name in ('steps/03-patch.sh', 'steps/07-stage.sh', 'steps/08-licenses.sh'): if sha(provider / name) != recipe_hashes[name]: raise ValueError('Provider recipe changed') selections = [('patches/shared_library.patch', '.'), ('patches/public_headers.patch', '.'), ('patches/clang_rt.patch', 'build'), ('patches/win/build.patch', 'build')] binaries = {name: sha(ROOT / 'build' / name) for name in ['docview', 'docview-pdf-worker', 'docview-archive-worker']} platforms = {} for platform, count in [('linux', 3), ('win', 4)]: tree = output / platform / 'tree' before = {} # Include the entire public directory so the package header inventory is # compared in both directions after applying the provider's text patch. for path, entry in inventories['.'].items(): if path.startswith('public/') and entry['mode'] in {'100644', '100755'}: target = tree / path target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(blob('.', path)) before[path] = entry['sha256'] steps = [] for name, repository in selections[:count]: patch = provider / name if sha(patch) != recipe_hashes[name]: raise ValueError('Provider patch changed') work = tree if repository == '.' else tree / repository for line in patch.read_text().splitlines(): if not line.startswith('+++ b/'): continue path = line[len('+++ b/'):].split('\t')[0] safe = PurePosixPath(path) if safe.is_absolute() or '..' in safe.parts: raise ValueError('Unsafe patch path') target = work / path target.parent.mkdir(parents=True, exist_ok=True) full_path = str(target.relative_to(tree)) if full_path not in before: target.write_bytes(blob(repository, path)) before[full_path] = sha(target) step = {'patch': str(patch.relative_to(ROOT)), 'sha256': sha(patch), 'directory': repository, 'fuzzLimit': 2} for phase, extra in [('dryRun', ['--dry-run']), ('apply', [])]: applied = subprocess.run(['patch', '--batch', '--forward', '--verbose', '--fuzz=2', '-p1', '-d', str(work), '-i', str(patch), *extra], capture_output=True, text=True, timeout=20) step[phase] = {'exitCode': applied.returncode, 'stdout': applied.stdout, 'stderr': applied.stderr} if applied.returncode: save(output / 'failure.json', step) raise ValueError('Recorded provider patch failed: ' + name) steps.append(step) expected_headers = {str(p.relative_to(tree / 'public')): p for p in (tree / 'public').rglob('*.h')} packaged_headers = {str(p.relative_to(ROOT / '.deps/pdfium/include')): p for p in (ROOT / '.deps/pdfium/include').rglob('*.h')} if expected_headers.keys() != packaged_headers.keys(): raise ValueError('Public header inventory differs') header_checks = [{'path': name, 'sourceSha256': sha(path), 'packagedSha256': sha(packaged_headers[name]), 'match': path.read_bytes() == packaged_headers[name].read_bytes()} for name, path in sorted(expected_headers.items())] if not all(e['match'] for e in header_checks): raise ValueError('Patched public headers differ') # Provider GNU patch can retain an original file after an offset/fuzzy # application. Keep the observed .orig file separate from API headers. extras = [] for path in (ROOT / '.deps/pdfium/include').rglob('*'): if not path.is_file() or path.suffix == '.h': continue relative = str(path.relative_to(ROOT / '.deps/pdfium/include')) matches = relative.endswith('.orig') and path.read_bytes() == blob('.', 'public/' + relative[:-5]) extras.append({'path': relative, 'sha256': sha(path), 'matchesPristineSource': matches}) if not matches: raise ValueError('Unexplained packaged extra header file') platforms[platform] = {'steps': steps, 'beforeSha256': before, 'afterSha256': {p: sha(tree / p) for p in before}, 'linuxPackagedHeaderComparisons': header_checks, 'extraPackagedFiles': extras, 'windowsResourceGenerated': False, 'compiled': False} spec = importlib.util.spec_from_file_location('notice_correspondence', EVIDENCE / 'check_correspondence.py') notices = importlib.util.module_from_spec(spec); spec.loader.exec_module(notices) notice_checks = [] for packaged, (source, mode) in notices.MAPPING.items(): if source.startswith('provider/'): content = (EVIDENCE / source).read_bytes() if sha(EVIDENCE / source) != recipe_hashes['LICENSE']: raise ValueError('Provider license changed') else: content = blob(*locate(source)) transformed = notices.transform(content, mode) actual = ROOT / '.deps/pdfium' / packaged match = transformed == actual.read_bytes() notice_checks.append({'path': packaged, 'sha256': sha(actual), 'match': match, 'transform': mode}) if not match: raise ValueError('Notice content differs') after_binaries = {name: sha(ROOT / 'build' / name) for name in binaries} if binaries != after_binaries: raise ValueError('Production binaries changed') report = {'success': True, 'collectionReportSha256': sha(report_path), 'checkerSha256': sha(Path(__file__)), 'gitilesSourceComparisons': correspondence, 'platforms': platforms, 'noticeComparisons': notice_checks, 'productionBinaries': binaries, 'productionBinariesUnchanged': True, 'scope': 'Source text, selected Linux/Windows provider patches and installed Linux public headers/notices only. ' 'No build scripts, Windows compiler/resource generation or reproducible binary build executed.'} save(output / 'report.json', report) print(json.dumps({'success': True, 'gitilesComparisons': len(correspondence), 'publicHeaders': len(expected_headers), 'notices': len(notice_checks), 'patchApplications': sum(len(p['steps']) for p in platforms.values())})) if __name__ == '__main__': main()