177 lines
10 KiB
Python
177 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare collected PDFium source, provider patches and installed public headers."""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import subprocess
|
|
|
|
from collect_pdfium import ROOT, EVIDENCE, sha, run, save
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--collection', type=Path, default=ROOT / 'tests/results/source-archives/pdfium-git')
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
if not output.is_relative_to(ROOT):
|
|
parser.error('Use an output directory inside the workspace')
|
|
collection = args.collection.resolve()
|
|
report_path = collection / 'report.json'
|
|
collected = json.loads(report_path.read_text())
|
|
if not collected['success']:
|
|
raise ValueError('Complete successful selected-Git collection required')
|
|
repositories, inventories = {}, {}
|
|
for item in collected['repositories']:
|
|
if sha(ROOT / item['archive']) != item['archiveSha256'] or sha(ROOT / item['inventory']) != item['inventorySha256']:
|
|
raise ValueError('Source archive or inventory changed')
|
|
repositories[item['path']] = item
|
|
inventories[item['path']] = {e['path']: e for line in (ROOT / item['inventory']).open() if (e := json.loads(line))}
|
|
|
|
def blob(repository, path):
|
|
item = repositories[repository]
|
|
expected = inventories[repository][path]
|
|
if expected['mode'] not in {'100644', '100755'}:
|
|
raise ValueError('Expected a regular source file')
|
|
result = run(ROOT / item['repository'], ['cat-file', 'blob', expected['gitObject']])
|
|
result.check_returncode()
|
|
if hashlib.sha256(result.stdout).hexdigest() != expected['sha256']:
|
|
raise ValueError('Source blob differs from inventory')
|
|
return result.stdout
|
|
|
|
def locate(saved):
|
|
if saved.startswith('upstream/'):
|
|
return '.', saved.removeprefix('upstream/')
|
|
if saved.startswith('dependencies/'):
|
|
relative = saved.removeprefix('dependencies/')
|
|
key = next(k for k in sorted(repositories, key=len, reverse=True) if relative.startswith(k + '/'))
|
|
return key, relative[len(key) + 1:]
|
|
raise ValueError('Unexpected saved source')
|
|
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
# Tie earlier Gitiles bytes to the full fetched snapshots, not just filenames.
|
|
correspondence = []
|
|
for folder in ('upstream', 'dependencies'):
|
|
for sidecar in sorted((EVIDENCE / folder).rglob('*.retrieval.json')):
|
|
retrieval = json.loads(sidecar.read_text())
|
|
if retrieval['transform'] != 'base64 decoded Gitiles response':
|
|
continue
|
|
saved = retrieval['file']
|
|
repository, path = locate(saved)
|
|
content = blob(repository, path)
|
|
digest = hashlib.sha256(content).hexdigest()
|
|
if digest != retrieval['sha256'] or content != (EVIDENCE / saved).read_bytes():
|
|
raise ValueError('Saved Gitiles source differs: ' + saved)
|
|
correspondence.append({'saved': saved, 'repository': repository, 'path': path,
|
|
'revision': repositories[repository]['revision'], 'sha256': digest})
|
|
provider = EVIDENCE / 'provider/recipe'
|
|
provider_report = json.loads((EVIDENCE / 'report.json').read_text())
|
|
recipe_hashes = {e['file']: e['sha256'] for e in provider_report['recipeFilesVerifiedAgainstGitTree']}
|
|
for name in ('steps/03-patch.sh', 'steps/07-stage.sh', 'steps/08-licenses.sh'):
|
|
if sha(provider / name) != recipe_hashes[name]:
|
|
raise ValueError('Provider recipe changed')
|
|
selections = [('patches/shared_library.patch', '.'), ('patches/public_headers.patch', '.'),
|
|
('patches/clang_rt.patch', 'build'), ('patches/win/build.patch', 'build')]
|
|
binaries = {name: sha(ROOT / 'build' / name) for name in ['docview', 'docview-pdf-worker', 'docview-archive-worker']}
|
|
platforms = {}
|
|
for platform, count in [('linux', 3), ('win', 4)]:
|
|
tree = output / platform / 'tree'
|
|
before = {}
|
|
# Include the entire public directory so the package header inventory is
|
|
# compared in both directions after applying the provider's text patch.
|
|
for path, entry in inventories['.'].items():
|
|
if path.startswith('public/') and entry['mode'] in {'100644', '100755'}:
|
|
target = tree / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(blob('.', path))
|
|
before[path] = entry['sha256']
|
|
steps = []
|
|
for name, repository in selections[:count]:
|
|
patch = provider / name
|
|
if sha(patch) != recipe_hashes[name]:
|
|
raise ValueError('Provider patch changed')
|
|
work = tree if repository == '.' else tree / repository
|
|
for line in patch.read_text().splitlines():
|
|
if not line.startswith('+++ b/'):
|
|
continue
|
|
path = line[len('+++ b/'):].split('\t')[0]
|
|
safe = PurePosixPath(path)
|
|
if safe.is_absolute() or '..' in safe.parts:
|
|
raise ValueError('Unsafe patch path')
|
|
target = work / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
full_path = str(target.relative_to(tree))
|
|
if full_path not in before:
|
|
target.write_bytes(blob(repository, path))
|
|
before[full_path] = sha(target)
|
|
step = {'patch': str(patch.relative_to(ROOT)), 'sha256': sha(patch), 'directory': repository, 'fuzzLimit': 2}
|
|
for phase, extra in [('dryRun', ['--dry-run']), ('apply', [])]:
|
|
applied = subprocess.run(['patch', '--batch', '--forward', '--verbose', '--fuzz=2', '-p1',
|
|
'-d', str(work), '-i', str(patch), *extra],
|
|
capture_output=True, text=True, timeout=20)
|
|
step[phase] = {'exitCode': applied.returncode, 'stdout': applied.stdout, 'stderr': applied.stderr}
|
|
if applied.returncode:
|
|
save(output / 'failure.json', step)
|
|
raise ValueError('Recorded provider patch failed: ' + name)
|
|
steps.append(step)
|
|
expected_headers = {str(p.relative_to(tree / 'public')): p for p in (tree / 'public').rglob('*.h')}
|
|
packaged_headers = {str(p.relative_to(ROOT / '.deps/pdfium/include')): p
|
|
for p in (ROOT / '.deps/pdfium/include').rglob('*.h')}
|
|
if expected_headers.keys() != packaged_headers.keys():
|
|
raise ValueError('Public header inventory differs')
|
|
header_checks = [{'path': name, 'sourceSha256': sha(path), 'packagedSha256': sha(packaged_headers[name]),
|
|
'match': path.read_bytes() == packaged_headers[name].read_bytes()}
|
|
for name, path in sorted(expected_headers.items())]
|
|
if not all(e['match'] for e in header_checks):
|
|
raise ValueError('Patched public headers differ')
|
|
# Provider GNU patch can retain an original file after an offset/fuzzy
|
|
# application. Keep the observed .orig file separate from API headers.
|
|
extras = []
|
|
for path in (ROOT / '.deps/pdfium/include').rglob('*'):
|
|
if not path.is_file() or path.suffix == '.h':
|
|
continue
|
|
relative = str(path.relative_to(ROOT / '.deps/pdfium/include'))
|
|
matches = relative.endswith('.orig') and path.read_bytes() == blob('.', 'public/' + relative[:-5])
|
|
extras.append({'path': relative, 'sha256': sha(path), 'matchesPristineSource': matches})
|
|
if not matches:
|
|
raise ValueError('Unexplained packaged extra header file')
|
|
platforms[platform] = {'steps': steps, 'beforeSha256': before,
|
|
'afterSha256': {p: sha(tree / p) for p in before},
|
|
'linuxPackagedHeaderComparisons': header_checks, 'extraPackagedFiles': extras,
|
|
'windowsResourceGenerated': False, 'compiled': False}
|
|
spec = importlib.util.spec_from_file_location('notice_correspondence', EVIDENCE / 'check_correspondence.py')
|
|
notices = importlib.util.module_from_spec(spec); spec.loader.exec_module(notices)
|
|
notice_checks = []
|
|
for packaged, (source, mode) in notices.MAPPING.items():
|
|
if source.startswith('provider/'):
|
|
content = (EVIDENCE / source).read_bytes()
|
|
if sha(EVIDENCE / source) != recipe_hashes['LICENSE']:
|
|
raise ValueError('Provider license changed')
|
|
else:
|
|
content = blob(*locate(source))
|
|
transformed = notices.transform(content, mode)
|
|
actual = ROOT / '.deps/pdfium' / packaged
|
|
match = transformed == actual.read_bytes()
|
|
notice_checks.append({'path': packaged, 'sha256': sha(actual), 'match': match, 'transform': mode})
|
|
if not match:
|
|
raise ValueError('Notice content differs')
|
|
after_binaries = {name: sha(ROOT / 'build' / name) for name in binaries}
|
|
if binaries != after_binaries:
|
|
raise ValueError('Production binaries changed')
|
|
report = {'success': True, 'collectionReportSha256': sha(report_path),
|
|
'checkerSha256': sha(Path(__file__)), 'gitilesSourceComparisons': correspondence,
|
|
'platforms': platforms, 'noticeComparisons': notice_checks,
|
|
'productionBinaries': binaries, 'productionBinariesUnchanged': True,
|
|
'scope': 'Source text, selected Linux/Windows provider patches and installed Linux public headers/notices only. '
|
|
'No build scripts, Windows compiler/resource generation or reproducible binary build executed.'}
|
|
save(output / 'report.json', report)
|
|
print(json.dumps({'success': True, 'gitilesComparisons': len(correspondence),
|
|
'publicHeaders': len(expected_headers), 'notices': len(notice_checks),
|
|
'patchApplications': sum(len(p['steps']) for p in platforms.values())}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|