initial commit
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare collected PDFium source, provider patches and installed public headers."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import subprocess
|
||||
|
||||
from collect_pdfium import ROOT, EVIDENCE, sha, run, save
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--collection', type=Path, default=ROOT / 'tests/results/source-archives/pdfium-git')
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
if not output.is_relative_to(ROOT):
|
||||
parser.error('Use an output directory inside the workspace')
|
||||
collection = args.collection.resolve()
|
||||
report_path = collection / 'report.json'
|
||||
collected = json.loads(report_path.read_text())
|
||||
if not collected['success']:
|
||||
raise ValueError('Complete successful selected-Git collection required')
|
||||
repositories, inventories = {}, {}
|
||||
for item in collected['repositories']:
|
||||
if sha(ROOT / item['archive']) != item['archiveSha256'] or sha(ROOT / item['inventory']) != item['inventorySha256']:
|
||||
raise ValueError('Source archive or inventory changed')
|
||||
repositories[item['path']] = item
|
||||
inventories[item['path']] = {e['path']: e for line in (ROOT / item['inventory']).open() if (e := json.loads(line))}
|
||||
|
||||
def blob(repository, path):
|
||||
item = repositories[repository]
|
||||
expected = inventories[repository][path]
|
||||
if expected['mode'] not in {'100644', '100755'}:
|
||||
raise ValueError('Expected a regular source file')
|
||||
result = run(ROOT / item['repository'], ['cat-file', 'blob', expected['gitObject']])
|
||||
result.check_returncode()
|
||||
if hashlib.sha256(result.stdout).hexdigest() != expected['sha256']:
|
||||
raise ValueError('Source blob differs from inventory')
|
||||
return result.stdout
|
||||
|
||||
def locate(saved):
|
||||
if saved.startswith('upstream/'):
|
||||
return '.', saved.removeprefix('upstream/')
|
||||
if saved.startswith('dependencies/'):
|
||||
relative = saved.removeprefix('dependencies/')
|
||||
key = next(k for k in sorted(repositories, key=len, reverse=True) if relative.startswith(k + '/'))
|
||||
return key, relative[len(key) + 1:]
|
||||
raise ValueError('Unexpected saved source')
|
||||
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
# Tie earlier Gitiles bytes to the full fetched snapshots, not just filenames.
|
||||
correspondence = []
|
||||
for folder in ('upstream', 'dependencies'):
|
||||
for sidecar in sorted((EVIDENCE / folder).rglob('*.retrieval.json')):
|
||||
retrieval = json.loads(sidecar.read_text())
|
||||
if retrieval['transform'] != 'base64 decoded Gitiles response':
|
||||
continue
|
||||
saved = retrieval['file']
|
||||
repository, path = locate(saved)
|
||||
content = blob(repository, path)
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
if digest != retrieval['sha256'] or content != (EVIDENCE / saved).read_bytes():
|
||||
raise ValueError('Saved Gitiles source differs: ' + saved)
|
||||
correspondence.append({'saved': saved, 'repository': repository, 'path': path,
|
||||
'revision': repositories[repository]['revision'], 'sha256': digest})
|
||||
provider = EVIDENCE / 'provider/recipe'
|
||||
provider_report = json.loads((EVIDENCE / 'report.json').read_text())
|
||||
recipe_hashes = {e['file']: e['sha256'] for e in provider_report['recipeFilesVerifiedAgainstGitTree']}
|
||||
for name in ('steps/03-patch.sh', 'steps/07-stage.sh', 'steps/08-licenses.sh'):
|
||||
if sha(provider / name) != recipe_hashes[name]:
|
||||
raise ValueError('Provider recipe changed')
|
||||
selections = [('patches/shared_library.patch', '.'), ('patches/public_headers.patch', '.'),
|
||||
('patches/clang_rt.patch', 'build'), ('patches/win/build.patch', 'build')]
|
||||
binaries = {name: sha(ROOT / 'build' / name) for name in ['docview', 'docview-pdf-worker', 'docview-archive-worker']}
|
||||
platforms = {}
|
||||
for platform, count in [('linux', 3), ('win', 4)]:
|
||||
tree = output / platform / 'tree'
|
||||
before = {}
|
||||
# Include the entire public directory so the package header inventory is
|
||||
# compared in both directions after applying the provider's text patch.
|
||||
for path, entry in inventories['.'].items():
|
||||
if path.startswith('public/') and entry['mode'] in {'100644', '100755'}:
|
||||
target = tree / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(blob('.', path))
|
||||
before[path] = entry['sha256']
|
||||
steps = []
|
||||
for name, repository in selections[:count]:
|
||||
patch = provider / name
|
||||
if sha(patch) != recipe_hashes[name]:
|
||||
raise ValueError('Provider patch changed')
|
||||
work = tree if repository == '.' else tree / repository
|
||||
for line in patch.read_text().splitlines():
|
||||
if not line.startswith('+++ b/'):
|
||||
continue
|
||||
path = line[len('+++ b/'):].split('\t')[0]
|
||||
safe = PurePosixPath(path)
|
||||
if safe.is_absolute() or '..' in safe.parts:
|
||||
raise ValueError('Unsafe patch path')
|
||||
target = work / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path = str(target.relative_to(tree))
|
||||
if full_path not in before:
|
||||
target.write_bytes(blob(repository, path))
|
||||
before[full_path] = sha(target)
|
||||
step = {'patch': str(patch.relative_to(ROOT)), 'sha256': sha(patch), 'directory': repository, 'fuzzLimit': 2}
|
||||
for phase, extra in [('dryRun', ['--dry-run']), ('apply', [])]:
|
||||
applied = subprocess.run(['patch', '--batch', '--forward', '--verbose', '--fuzz=2', '-p1',
|
||||
'-d', str(work), '-i', str(patch), *extra],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
step[phase] = {'exitCode': applied.returncode, 'stdout': applied.stdout, 'stderr': applied.stderr}
|
||||
if applied.returncode:
|
||||
save(output / 'failure.json', step)
|
||||
raise ValueError('Recorded provider patch failed: ' + name)
|
||||
steps.append(step)
|
||||
expected_headers = {str(p.relative_to(tree / 'public')): p for p in (tree / 'public').rglob('*.h')}
|
||||
packaged_headers = {str(p.relative_to(ROOT / '.deps/pdfium/include')): p
|
||||
for p in (ROOT / '.deps/pdfium/include').rglob('*.h')}
|
||||
if expected_headers.keys() != packaged_headers.keys():
|
||||
raise ValueError('Public header inventory differs')
|
||||
header_checks = [{'path': name, 'sourceSha256': sha(path), 'packagedSha256': sha(packaged_headers[name]),
|
||||
'match': path.read_bytes() == packaged_headers[name].read_bytes()}
|
||||
for name, path in sorted(expected_headers.items())]
|
||||
if not all(e['match'] for e in header_checks):
|
||||
raise ValueError('Patched public headers differ')
|
||||
# Provider GNU patch can retain an original file after an offset/fuzzy
|
||||
# application. Keep the observed .orig file separate from API headers.
|
||||
extras = []
|
||||
for path in (ROOT / '.deps/pdfium/include').rglob('*'):
|
||||
if not path.is_file() or path.suffix == '.h':
|
||||
continue
|
||||
relative = str(path.relative_to(ROOT / '.deps/pdfium/include'))
|
||||
matches = relative.endswith('.orig') and path.read_bytes() == blob('.', 'public/' + relative[:-5])
|
||||
extras.append({'path': relative, 'sha256': sha(path), 'matchesPristineSource': matches})
|
||||
if not matches:
|
||||
raise ValueError('Unexplained packaged extra header file')
|
||||
platforms[platform] = {'steps': steps, 'beforeSha256': before,
|
||||
'afterSha256': {p: sha(tree / p) for p in before},
|
||||
'linuxPackagedHeaderComparisons': header_checks, 'extraPackagedFiles': extras,
|
||||
'windowsResourceGenerated': False, 'compiled': False}
|
||||
spec = importlib.util.spec_from_file_location('notice_correspondence', EVIDENCE / 'check_correspondence.py')
|
||||
notices = importlib.util.module_from_spec(spec); spec.loader.exec_module(notices)
|
||||
notice_checks = []
|
||||
for packaged, (source, mode) in notices.MAPPING.items():
|
||||
if source.startswith('provider/'):
|
||||
content = (EVIDENCE / source).read_bytes()
|
||||
if sha(EVIDENCE / source) != recipe_hashes['LICENSE']:
|
||||
raise ValueError('Provider license changed')
|
||||
else:
|
||||
content = blob(*locate(source))
|
||||
transformed = notices.transform(content, mode)
|
||||
actual = ROOT / '.deps/pdfium' / packaged
|
||||
match = transformed == actual.read_bytes()
|
||||
notice_checks.append({'path': packaged, 'sha256': sha(actual), 'match': match, 'transform': mode})
|
||||
if not match:
|
||||
raise ValueError('Notice content differs')
|
||||
after_binaries = {name: sha(ROOT / 'build' / name) for name in binaries}
|
||||
if binaries != after_binaries:
|
||||
raise ValueError('Production binaries changed')
|
||||
report = {'success': True, 'collectionReportSha256': sha(report_path),
|
||||
'checkerSha256': sha(Path(__file__)), 'gitilesSourceComparisons': correspondence,
|
||||
'platforms': platforms, 'noticeComparisons': notice_checks,
|
||||
'productionBinaries': binaries, 'productionBinariesUnchanged': True,
|
||||
'scope': 'Source text, selected Linux/Windows provider patches and installed Linux public headers/notices only. '
|
||||
'No build scripts, Windows compiler/resource generation or reproducible binary build executed.'}
|
||||
save(output / 'report.json', report)
|
||||
print(json.dumps({'success': True, 'gitilesComparisons': len(correspondence),
|
||||
'publicHeaders': len(expected_headers), 'notices': len(notice_checks),
|
||||
'patchApplications': sum(len(p['steps']) for p in platforms.values())}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user