initial commit

This commit is contained in:
2026-09-21 13:41:40 +09:00
commit 855c7328df
411 changed files with 85352 additions and 0 deletions
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Preserve the original PDF bytes while correcting pattern entry-state expectations."""
import argparse
import hashlib
import json
from pathlib import Path
import shutil
ROOT = Path(__file__).resolve().parents[2]
ORIGINAL = ROOT / 'tests/fixtures/pdf/rendering-intents'
MANIFEST_SHA = 'b0db4ee3e57658b09d0ae10fcf8175986c3ab80ccd64442e3718802257ecf543'
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
assert sha(ORIGINAL / 'manifest.json') == MANIFEST_SHA
spec = json.loads((ORIGINAL / 'manifest.json').read_text())
files = {spec['file']: spec['sha256'], **{
row['file']: row['sha256'] for row in spec['profiles'].values()}}
for name, expected in files.items():
assert sha(ORIGINAL / name) == expected
spec['oracleRevision'] = 2
spec['oracleCorrection'] = {
'sourceManifestSha256': MANIFEST_SHA,
'correctorSha256': sha(Path(__file__)),
'case': 'shading-pattern-inherit',
'affectedProbesPerScale': 8,
'reason': 'A shading pattern without an ExtGState RI uses the graphics state at entry to its defining content stream, not the paint-time RI. These patterns are defined in page streams whose entry RI is RelativeColorimetric.',
'reference': 'https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/pdfreference1.4.pdf',
'section': '4.6.3 Table 4.23, printed page 231',
'pdfAndProfilesByteIdentical': True,
}
changed = []
for index, probe in enumerate(spec['probes']):
if probe['case'] == 'shading-pattern-inherit':
changed.append({'probeIndex': index, 'previousIntentIndex': probe['expectedIntentIndex']})
probe['expectedIntentIndex'] = 1
probe['expectedIntent'] = 'RelativeColorimetric'
assert len(changed) == 8
spec['oracleCorrection']['rows'] = changed
args.output.mkdir(parents=True, exist_ok=False)
for name in files:
shutil.copyfile(ORIGINAL / name, args.output / name)
(args.output / 'manifest.json').write_text(json.dumps(spec, indent=2) + '\n')
print(json.dumps({'probes': len(spec['probes']), 'correctedRows': len(changed),
'manifestSha256': sha(args.output / 'manifest.json')}))
if __name__ == '__main__':
main()
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Explain the retained failing CMYK comparison without relaxing its criterion."""
from io import BytesIO
import hashlib
import json
from pathlib import Path
from PIL import Image, ImageCms
from pypdf import PdfReader
ROOT = Path(__file__).resolve().parents[2]
RESULTS = ROOT / 'tests/results/cmyk-lut'
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
corpus=ROOT/'tests/fixtures/pdf/cmyk-lut'
spec=json.loads((corpus/'manifest.json').read_text())
pdf=PdfReader(corpus/spec['file'])
assert b'/RelativeColorimetric ri' in pdf.pages[0].get_contents().get_data()
assert pdf.pages[0]['/Resources']['/XObject']['/Sample']['/Intent']=='/RelativeColorimetric'
source=json.loads((RESULTS/'pdfium-source/record.json').read_text())
for row in source['files']:
assert sha(RESULTS/'pdfium-source'/row['path'])==row['sha256']
transform_source=(RESULTS/'pdfium-source/core/fxcodec/icc/icc_transform.cpp').read_text()
assert 'TYPE_BGR_8, INTENT_PERCEPTUAL, /*dwFlags=*/0' in transform_source
profile=ImageCms.ImageCmsProfile(BytesIO((corpus/spec['profileFile']).read_bytes()))
transformations={intent:ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),
'CMYK','RGB',renderingIntent=value) for intent,value in [('perceptual',0),('relative',1)]}
colors={}
for probe in spec['probes']:
if probe['kind']!='icc-image': continue
pixel=Image.new('CMYK',(1,1),tuple(round(c*255) for c in probe['components']))
colors[probe['sample']]={name:list(ImageCms.applyTransform(pixel,t).getpixel((0,0))) for name,t in transformations.items()}
rows=[]
for os_name in ('arch','ubuntu'):
report=json.loads((RESULTS/os_name/'report.json').read_text())
assert not report['success'] and report['fixture']['sha256']==spec['sha256']
assert report['originalUnchanged'] and report['binariesUnchanged']
assert report['runnerSha256']==sha(ROOT/'tests/cmyk_lut/run.py')
for name,digest in report['evidenceSha256'].items():
assert sha(RESULTS/os_name/name)==digest
failed=0; matches=[]
for scale in report['scales']:
failed+=sum(not p['success'] for p in scale['probes'])
for probe in scale['probes']:
if probe['kind']!='icc-image': continue
pair=colors[probe['sample']]
errors={name:max(abs(a-b) for a,b in zip(probe['actualRgb'][name],pair[intent]))
for name,intent in [('pdfium','perceptual'),('poppler','relative')]}
matches.append({'scale':scale['scale'],'sample':probe['sample'],'actualRgb':probe['actualRgb'],
'cmmIntentColors':pair,'errorsAgainstRespectiveIntent':errors})
failed_by_renderer={name:sum(p['maxChannelError'].get(name,0)>spec['tolerance8Bit']
for scale in report['scales'] for p in scale['probes'])
for name in ('pdfium','poppler')}
maximum=max(p['maxChannelError'].get('pdfium',0) for scale in report['scales'] for p in scale['probes'])
assert failed==failed_by_renderer['pdfium']==30 and failed_by_renderer['poppler']==0 and maximum==74
assert all(max(m['errorsAgainstRespectiveIntent'].values())<=1 for m in matches)
rows.append({'os':os_name,'comparisonReportSha256':sha(RESULTS/os_name/'report.json'),
'analyticProbes':48,'failedPdfiumAnalyticProbes':failed,'failedPopplerAnalyticProbes':failed_by_renderer['poppler'],
'maximumPdfiumAnalyticChannelDifference':maximum,'intentMatchedImageProbes':matches})
result={'diagnosisVerified':True,'renderingAcceptance':False,'relativeIntentPreserved':False,
'toleranceUnchanged':spec['tolerance8Bit'],'profileSha256':spec['profileSha256'],
'sourceRevision':source['revision'],'sourceEvidenceSha256':sha(RESULTS/'pdfium-source/record.json'),
'diagnoserSha256':sha(Path(__file__)),'cmmVersion':ImageCms.core.littlecms_version,'results':rows,
'finding':'At this fixed PDFium revision, ICC conversion selects INTENT_PERCEPTUAL regardless of the '
'relative-colorimetric intent in the test PDF. On both OSs, all eight image swatches at all '
'three scales match LittleCMS perceptual output within one channel level. Poppler matches '
'relative-colorimetric output. This explains, but does not accept or repair, the difference.',
'remaining':'G-RENDER requires review and resolution/explicit compatibility judgment. No renderer '
'patch, engine substitution or user-visible detection was implemented in this comparison.'}
(RESULTS/'diagnosis.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps({'diagnosisVerified':True,'renderingAcceptance':False,'failedAnalyticProbes':60,
'intentMatchedImageProbes':48}))
if __name__=='__main__': main()
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Export the second PDFium candidate, preserving the first candidate evidence."""
import difflib
import hashlib
import json
from pathlib import Path
import subprocess
import tarfile
import tempfile
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
results = ROOT / 'tests/results/pdfium-intent-context'
materialization = json.loads((ROOT / 'tests/results/pdfium-intent-build/source-materialization.json').read_text())
repository = next(row for row in materialization['repositories'] if row['path'] == '.')
archive = ROOT / repository['archive']
inventory = ROOT / repository['inventory']
assert sha(archive) == repository['archiveSha256']
assert sha(inventory) == repository['inventorySha256']
source = ROOT / 'build-pdfium-intent-context/source'
originals = {row['path']: row for line in inventory.open()
if (row := json.loads(line))['archived'] and row['path'].startswith('core/')}
changed = {}
for name, row in originals.items():
assert row['mode'] == '100644'
if sha(source / name) != row['sha256']:
changed[name] = row
added = {str(path.relative_to(source)) for path in (source / 'core').rglob('*')
if path.is_file() and str(path.relative_to(source)) not in originals}
assert all(Path(name).suffix in ('.h', '.cpp') for name in added)
chunks = []
records = []
with tarfile.open(archive) as tar:
for name in sorted(changed.keys() | added):
before = b'' if name in added else tar.extractfile('pdfium/' + name).read()
if name in changed:
assert hashlib.sha256(before).hexdigest() == changed[name]['sha256']
after = (source / name).read_bytes()
diff = ''.join(difflib.unified_diff(
before.decode().splitlines(keepends=True),
after.decode().splitlines(keepends=True),
fromfile='/dev/null' if name in added else 'a/' + name,
tofile='b/' + name))
assert diff
chunks.append('diff --git a/' + name + ' b/' + name + '\n' + diff)
records.append({'path': name, 'added': name in added,
'beforeSha256': None if name in added else hashlib.sha256(before).hexdigest(),
'afterSha256': hashlib.sha256(after).hexdigest()})
destination = ROOT / 'cmake/patches/pdfium-rendering-intent-context.patch'
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(''.join(chunks))
snapshot = results / 'patches' / (sha(destination) + '.patch')
snapshot.parent.mkdir(exist_ok=True)
if snapshot.exists():
assert snapshot.read_bytes() == destination.read_bytes()
else:
snapshot.write_bytes(destination.read_bytes())
# Check that this reviewable patch, rather than an unrecorded local change,
# reproduces every modified core source without fuzzy matching.
with tempfile.TemporaryDirectory(prefix='docview-pdfium-patch-') as folder:
check_root = Path(folder)
with tarfile.open(archive) as tar:
for name in changed:
target = check_root / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(tar.extractfile('pdfium/' + name).read())
result = subprocess.run(['patch', '-p1', '--batch', '--fuzz=0', '-i', str(destination)],
cwd=check_root, capture_output=True, text=True)
assert result.returncode == 0, result.stdout + result.stderr
for row in records:
assert sha(check_root / row['path']) == row['afterSha256']
record = {'success': True, 'exporterSha256': sha(Path(__file__)),
'source': str(source.relative_to(ROOT)),
'parentRecord': 'tests/results/pdfium-intent-build/record.json',
'parentRecordSha256': sha(ROOT / 'tests/results/pdfium-intent-build/record.json'),
'baseRevision': repository['revision'], 'archiveSha256': sha(archive),
'patch': str(destination.relative_to(ROOT)), 'patchSha256': sha(destination),
'patchSnapshot': str(snapshot.relative_to(ROOT)),
'appliesWithoutFuzz': True, 'appliedSourcesByteIdentical': True,
'files': records,
'scope': 'Candidate core patch only. Provider shared-library/public-header patches are recorded separately. Production dependency is not replaced by this exporter.'}
(results / 'candidate-patch.json').write_text(json.dumps(record, indent=2) + '\n')
print(json.dumps({'files': len(records), 'patchBytes': destination.stat().st_size,
'patchSha256': record['patchSha256']}))
if __name__ == '__main__':
main()
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Export the isolated PDFium core changes against the verified fixed archive."""
import difflib
import hashlib
import json
from pathlib import Path
import subprocess
import tarfile
import tempfile
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
results = ROOT / 'tests/results/pdfium-intent-build'
materialization = json.loads((results / 'source-materialization.json').read_text())
repository = next(row for row in materialization['repositories'] if row['path'] == '.')
archive = ROOT / repository['archive']
inventory = ROOT / repository['inventory']
assert sha(archive) == repository['archiveSha256']
assert sha(inventory) == repository['inventorySha256']
source = ROOT / materialization['source']
originals = {row['path']: row for line in inventory.open()
if (row := json.loads(line))['archived'] and row['path'].startswith('core/')}
changed = {}
for name, row in originals.items():
assert row['mode'] == '100644'
if sha(source / name) != row['sha256']:
changed[name] = row
added = {str(path.relative_to(source)) for path in (source / 'core').rglob('*')
if path.is_file() and str(path.relative_to(source)) not in originals}
assert all(Path(name).suffix in ('.h', '.cpp') for name in added)
chunks = []
records = []
with tarfile.open(archive) as tar:
for name in sorted(changed.keys() | added):
before = b'' if name in added else tar.extractfile('pdfium/' + name).read()
if name in changed:
assert hashlib.sha256(before).hexdigest() == changed[name]['sha256']
after = (source / name).read_bytes()
diff = ''.join(difflib.unified_diff(
before.decode().splitlines(keepends=True),
after.decode().splitlines(keepends=True),
fromfile='/dev/null' if name in added else 'a/' + name,
tofile='b/' + name))
assert diff
chunks.append('diff --git a/' + name + ' b/' + name + '\n' + diff)
records.append({'path': name, 'added': name in added,
'beforeSha256': None if name in added else hashlib.sha256(before).hexdigest(),
'afterSha256': hashlib.sha256(after).hexdigest()})
destination = ROOT / 'cmake/patches/pdfium-rendering-intent.patch'
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(''.join(chunks))
snapshot = results / 'patches' / (sha(destination) + '.patch')
snapshot.parent.mkdir(exist_ok=True)
if snapshot.exists():
assert snapshot.read_bytes() == destination.read_bytes()
else:
snapshot.write_bytes(destination.read_bytes())
# Check that this reviewable patch, rather than an unrecorded local change,
# reproduces every modified core source without fuzzy matching.
with tempfile.TemporaryDirectory(prefix='docview-pdfium-patch-') as folder:
check_root = Path(folder)
with tarfile.open(archive) as tar:
for name in changed:
target = check_root / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(tar.extractfile('pdfium/' + name).read())
result = subprocess.run(['patch', '-p1', '--batch', '--fuzz=0', '-i', str(destination)],
cwd=check_root, capture_output=True, text=True)
assert result.returncode == 0, result.stdout + result.stderr
for row in records:
assert sha(check_root / row['path']) == row['afterSha256']
record = {'success': True, 'exporterSha256': sha(Path(__file__)),
'baseRevision': repository['revision'], 'archiveSha256': sha(archive),
'patch': str(destination.relative_to(ROOT)), 'patchSha256': sha(destination),
'patchSnapshot': str(snapshot.relative_to(ROOT)),
'appliesWithoutFuzz': True, 'appliedSourcesByteIdentical': True,
'files': records,
'scope': 'Candidate core patch only. Provider shared-library/public-header patches are recorded separately. Production dependency is not replaced by this exporter.'}
(results / 'candidate-patch.json').write_text(json.dumps(record, indent=2) + '\n')
print(json.dumps({'files': len(records), 'patchBytes': destination.stat().st_size,
'patchSha256': record['patchSha256']}))
if __name__ == '__main__':
main()
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Fetch only the fixed Linux GN, Clang and sysroot inputs for the isolated build."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import time
import urllib.request
ROOT=Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream: return hashlib.file_digest(stream,'sha256').hexdigest()
def main():
pins=json.loads((ROOT/'tests/results/source-correspondence/pdfium/dependency-pins.json').read_text())
source=ROOT/'build-pdfium-intent/source'
clang=next(row for row in pins['root']['deps']['third_party/llvm-build/Release+Asserts']['objects']
if row['object_name'].startswith('Linux_x64/clang-llvmorg-'))
sysroot=json.loads((source/'build/linux/sysroot_scripts/sysroots.json').read_text())['bullseye_amd64']
gn_version=pins['root']['vars']['gn_version']
inputs=[{'name':'clang.tar.xz','url':'https://commondatastorage.googleapis.com/chromium-browser-clang/'+clang['object_name']+'?generation='+str(clang['generation']),
'expectedSha256':clang['sha256sum'],'expectedBytes':clang['size_bytes']},
{'name':'sysroot.tar.xz','url':sysroot['URL']+'/'+sysroot['Sha256Sum'],'expectedSha256':sysroot['Sha256Sum']},
{'name':'gn.zip','url':'https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-amd64/+/'+gn_version,
'cipdPackage':'gn/gn/linux-amd64','cipdVersion':gn_version}]
cache=ROOT/'build-pdfium-intent/downloads'; cache.mkdir(parents=True,exist_ok=False)
def fetch(row):
path=cache/row['name']; total=0; started=time.monotonic()
with urllib.request.urlopen(row['url'],timeout=60) as response, path.open('xb') as stream:
content_length=response.headers.get('Content-Length')
if content_length: assert int(content_length)<=512*1024**2
for chunk in iter(lambda:response.read(1024*1024),b''):
total+=len(chunk)
assert total<=512*1024**2 and time.monotonic()-started<=600
stream.write(chunk)
digest=sha(path)
if 'expectedSha256' in row: assert row['expectedSha256']==digest
if 'expectedBytes' in row: assert row['expectedBytes']==total
print(row['name']+' '+str(total)+' bytes; SHA-256 '+digest,flush=True)
return {**row,'path':str(path.relative_to(ROOT)),'bytes':total,'sha256':digest}
with ThreadPoolExecutor(max_workers=3) as pool: rows=list(pool.map(fetch,inputs))
record={'success':True,'fetcherSha256':sha(Path(__file__)),'inputs':rows,
'scope':'Clang and sysroot checked against source-pinned SHA-256. GN fetched from the official CIPD fixed Git-revision selector; archive SHA-256 recorded. No downloaded executable or installer has been run.'}
(ROOT/'tests/results/pdfium-intent-build/tool-downloads.json').write_text(json.dumps(record,indent=2)+'\n')
if __name__=='__main__':main()
+307
View File
@@ -0,0 +1,307 @@
#!/usr/bin/env python3
"""Strict ICC rendering-intent probes against direct CMM, PDFium and Poppler.
The direct oracle never reads rendered pixels. It uses the embedded profile and
an independently selected expected intent. It may share LittleCMS algorithms
with the renderers, and is not claimed to be an independent CMM implementation.
"""
import argparse
import ctypes as C
import ctypes.util
from io import BytesIO
import hashlib
import itertools
import json
import math
import os
from pathlib import Path
import platform
import re
import shutil
import struct
import subprocess
from PIL import Image, ImageChops, ImageCms, ImageDraw
from pypdf import PdfReader
ROOT=Path(__file__).resolve().parents[2]
CORPUS=ROOT/'tests/fixtures/pdf/rendering-intents'
INTENTS=('Perceptual','RelativeColorimetric','Saturation','AbsoluteColorimetric')
def sha(path):
with path.open('rb') as source:return hashlib.file_digest(source,'sha256').hexdigest()
class Cmm:
"""Public LittleCMS API, normalized CMYK doubles -> unsigned 8-bit sRGB."""
def __init__(self,profiles):
library=ctypes.util.find_library('lcms2')
if not library:raise RuntimeError('LittleCMS2 shared library is required')
self.lib=C.CDLL(library);self.transforms={};self.profiles=[];self.buffers=[]
declarations={
'cmsOpenProfileFromMem':([C.c_void_p,C.c_uint32],C.c_void_p),
'cmsCreate_sRGBProfile':([],C.c_void_p),
'cmsCreateTransform':([C.c_void_p,C.c_uint32,C.c_void_p,C.c_uint32,C.c_uint32,C.c_uint32],C.c_void_p),
'cmsDoTransform':([C.c_void_p,C.c_void_p,C.c_void_p,C.c_uint32],None),
'cmsDeleteTransform':([C.c_void_p],None),'cmsCloseProfile':([C.c_void_p],C.c_int),
'cmsGetEncodedCMMversion':([],C.c_uint32)}
for name,(args,result) in declarations.items():
function=getattr(self.lib,name);function.argtypes=args;function.restype=result
output=self.lib.cmsCreate_sRGBProfile();assert output;self.profiles.append(output)
for identifier,data in profiles.items():
buffer=C.create_string_buffer(data)
self.buffers.append(buffer)
profile=self.lib.cmsOpenProfileFromMem(buffer,len(data));assert profile;self.profiles.append(profile)
for intent in range(4):
# lcms2.h public pixel-format macros: FLOAT_SH(1), PT_CMYK=6,
# PT_RGB=4, CHANNELS_SH(n), BYTES_SH(n). Double CMYK uses %.
transform=self.lib.cmsCreateTransform(profile,(1<<22)|(6<<16)|(4<<3),
output,(4<<16)|(3<<3)|1,intent,0)
assert transform;self.transforms[identifier,intent]=transform
self.version=self.lib.cmsGetEncodedCMMversion()
self.libraryPaths=[]
maps=Path('/proc/self/maps')
if maps.exists():
paths={line.split()[-1] for line in maps.read_text().splitlines()
if '/' in line and 'liblcms2.so' in line}
self.libraryPaths=[{'file':Path(p).name,'sha256':sha(Path(p))} for p in sorted(paths) if Path(p).is_file()]
def color(self,profile,intent,values):
assert len(values)==4 and all(math.isfinite(x) and 0<=x<=1 for x in values)
source=(C.c_double*4)(*(x*100 for x in values));target=(C.c_ubyte*3)()
self.lib.cmsDoTransform(self.transforms[profile,intent],source,target,1)
return list(target)
def close(self):
for transform in self.transforms.values():self.lib.cmsDeleteTransform(transform)
for profile in self.profiles:self.lib.cmsCloseProfile(profile)
self.transforms.clear();self.profiles.clear();self.buffers.clear()
def tags_from(profile):
assert len(profile)==struct.unpack_from('>I',profile)[0]
assert profile[8:24]==bytes.fromhex('02100000')+b'scnrCMYKLab ' and profile[36:40]==b'acsp'
tags={}
for index in range(struct.unpack_from('>I',profile,128)[0]):
signature,offset,size=struct.unpack_from('>4sII',profile,132+12*index)
assert offset%4==0 and offset>=132 and offset+size<=len(profile) and signature not in tags
tags[signature]=profile[offset:offset+size]
return tags
def verify_fixture(corpus,spec):
source=corpus/spec['file'];assert sha(source)==spec['sha256']
assert sha(ROOT/'tests/fixtures/pdf/generate_rendering_intents.py')==spec['generatorSha256']
corrected=spec.get('oracleRevision')==2
if corrected:
correction=spec['oracleCorrection']
assert correction['correctorSha256']==sha(ROOT/'tests/cmyk_lut/correct_intent_oracle.py')
assert correction['sourceManifestSha256']=='b0db4ee3e57658b09d0ae10fcf8175986c3ab80ccd64442e3718802257ecf543'
original=ROOT/'tests/fixtures/pdf/cmyk-lut/synthetic-cmyk-lab.icc'
assert sha(original)==spec['baseProfileSha256']
profiles={identifier:(corpus/item['file']).read_bytes() for identifier,item in spec['profiles'].items()}
assert profiles['original']==original.read_bytes()
vertex_count=0
# Independent coefficient representation of the written linear CLUTs.
coefficients={b'A2B0':[(100,(-12,-15,-10,-35)),(0,(-15,18,0,0)),(0,(-12,0,20,0))],
b'A2B1':[(100,(-20,-20,-20,-40)),(0,(-20,40,0,-20)),(0,(-20,0,40,-20))],
b'A2B2':[(100,(-25,-25,-20,-30)),(0,(20,-40,0,20)),(0,(0,-20,35,-15))]}
for identifier,data in profiles.items():
item=spec['profiles'][identifier]
assert hashlib.sha256(data).hexdigest()==item['sha256'] and len(data)==item['size']
tags=tags_from(data)
assert set(tags)=={b'desc',b'cprt',b'wtpt',*(t.encode() for t in item['tables'])}
white=struct.unpack_from('>3i',tags[b'wtpt'],8)
assert all(abs(actual/65536-expected)<=1/65536 for actual,expected in zip(white,item['mediaWhitePoint']))
for tag in item['tables']:
lut=tags[tag.encode()]
assert len(lut)==176 and lut[:12]==b'mft2'+bytes(4)+bytes((4,3,2,0))
assert struct.unpack_from('>HH',lut,48)==(2,2)
assert lut[52:68]==struct.pack('>HH',0,65535)*4 and lut[164:176]==struct.pack('>HH',0,65535)*3
for i,values in enumerate(itertools.product((0,1),repeat=4)):
encoded=struct.unpack_from('>3H',lut,68+i*6)
actual=(encoded[0]*100/65280,encoded[1]/256-128,encoded[2]/256-128)
expected=[offset+sum(a*b for a,b in zip(vector,values)) for offset,vector in coefficients[tag.encode()]]
assert max(abs(a-b) for a,b in zip(actual,expected))<.002
vertex_count+=1
pdf=PdfReader(source);assert len(pdf.pages)==spec['pageCount']
identities={}
for index,page in enumerate(pdf.pages,1):
assert list(page.mediabox)==[0,0,*spec['pageSizePt']]
probes=[p for p in spec['probes'] if p['page']==index];assert probes
identifier=probes[0]['profile'];resources=page['/Resources'];space=resources['/ColorSpace']['/CS']
# pypdf 4 leaves array members indirect; dictionary lookup alone dereferences.
embedded_profile=space[1].get_object()
assert space[0]=='/ICCBased' and embedded_profile['/N']==4 and embedded_profile.get_data()==profiles[identifier]
assert resources['/ColorSpace']['/IX'][0]=='/Indexed'
lookup=resources['/ColorSpace']['/IX'][3]
assert (lookup.original_bytes if isinstance(lookup,str) else bytes(lookup))==bytes(spec['sampleBytes'])
xobjects=resources['/XObject'];shared=xobjects.raw_get('/Shared')
identities.setdefault(identifier,shared.idnum);assert identities[identifier]==shared.idnum
assert '/Intent' not in xobjects['/Shared'] and xobjects['/Shared'].get_data()==bytes(spec['sampleBytes'])
assert xobjects['/Indexed'].get_data()==b'\0'
for font_name in ('T3Rect','T3Image',*(f'T3Override{i}' for i in range(4))):
glyph_font=resources['/Font']['/'+font_name]
assert glyph_font['/Subtype']=='/Type3' and glyph_font['/FirstChar']==glyph_font['/LastChar']==65
glyph=glyph_font['/CharProcs']['/A'].get_data();assert glyph.startswith(b'1000 0 d0\n')
if font_name=='T3Image':assert b'/Shared Do' in glyph
elif font_name.startswith('T3Override'):assert ('/'+INTENTS[int(font_name[-1])]+' ri').encode() in glyph
else:assert b' ri' not in glyph
assert '/RelativeColorimetric ri' not in xobjects['/Inherited'].get_data().decode()
for i,intent in enumerate(INTENTS):
assert resources['/ExtGState']['/I'+str(i)]['/RI']=='/'+intent
assert xobjects['/Image'+str(i)]['/Intent']=='/'+intent
assert ('/'+intent+' ri').encode() in xobjects['/Form'+str(i)].get_data()
assert ('/I'+str(i)+' gs').encode() in xobjects['/FormGs'+str(i)].get_data()
assert resources['/Pattern']['/P'+str(i)]['/ExtGState']['/RI']=='/'+intent
content=page.get_contents().get_data()
for probe in probes:
assert ('\n'.join(probe['commandSequence'])+'\n').encode() in content
expected=probe['expectedIntentIndex'];assert INTENTS[expected]==probe['expectedIntent']
current=1 if probe['graphicsStateIntent']=='default' else INTENTS.index(probe['graphicsStateIntent'])
if corrected and probe['case']=='shading-pattern-inherit':
assert expected==1
elif probe['case'] in ('nested-q-inner','form-ri-override','form-gs-override','image-intent-override','shading-pattern-RI-override','type3-ri-override'):
assert expected==(current+1)%4
else:assert expected==current
assert len(spec['probes'])==len(spec['cases'])*4*2
return profiles,{'pagesVerified':len(pdf.pages),'probesVerified':len(spec['probes']),
'clutVerticesVerified':vertex_count,'sharedImageObjectIds':identities,
'originalProfileByteIdentical':True,'oracleRevision':2 if corrected else 1,
'knownIncorrectPatternOracle':not corrected}
def cmm_checks(profiles,spec,cmm):
sample=tuple(spec['sampleBytes']);checks=[];colors={};bpc_checks=[]
for identifier,data in profiles.items():
profile=ImageCms.ImageCmsProfile(BytesIO(data));colors[identifier]=[]
for intent in range(4):
expected=cmm.color(identifier,intent,[x/255 for x in sample]);colors[identifier].append(expected)
transform=ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=intent)
actual=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),transform).getpixel((0,0)))
error=max(abs(a-b) for a,b in zip(actual,expected))
checks.append({'profile':identifier,'intent':INTENTS[intent],'directCmmRgb':expected,
'pillow8BitRgb':actual,'maxChannelError':error,'success':error<=1})
if identifier=='distinct':
# Pillow 10.2 (Ubuntu 24.04) exposes the same public flag in
# FLAGS; later releases expose the Flags enum instead.
bpc_flag=(ImageCms.Flags.BLACKPOINTCOMPENSATION if hasattr(ImageCms,'Flags')
else ImageCms.FLAGS['BLACKPOINTCOMPENSATION'])
assert int(bpc_flag)==8192
with_bpc=ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),'CMYK','RGB',
renderingIntent=intent,flags=bpc_flag)
bpc=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),with_bpc).getpixel((0,0)))
assert max(abs(a-b) for a,b in zip(bpc,expected))<=1
bpc_checks.append({'intent':INTENTS[intent],'rgb':bpc,'flags':8192,'matchesWithoutBpc':True})
assert all(c['success'] for c in checks)
separation=[max(abs(a-b) for a,b in zip(colors['distinct'][i],colors['distinct'][j])) for i,j in itertools.combinations(range(4),2)]
assert min(separation)>2*spec['tolerance8Bit'],'Derived profile must distinguish every intent beyond tolerance'
return {'checks':checks,'distinctProfileBlackPointCompensationChecks':bpc_checks,
'distinctProfileMinimumPairwiseMaxChannelSeparation':min(separation),
'originalProfileIntentEquivalence':'Relative/Saturation/Absolute may coincide; distinct profile covers their dispatch.'}
def loaded_pdfium(worker,environment,requested):
text=subprocess.run(['ldd',str(worker)],capture_output=True,text=True,env=environment,check=True,timeout=30).stdout
match=re.search(r'^\s*libpdfium\.so\s+=>\s+(/.+?)\s+\(0x',text,re.M)
assert match,'ldd did not resolve worker libpdfium.so'
path=Path(match[1]).resolve(strict=True)
if requested:assert path==requested.resolve(strict=True),'Requested library lost to loader/RPATH precedence'
return {'path':str(path),'sha256':sha(path),'ldd':text,'selectionVerified':True}
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--build-dir',type=Path)
parser.add_argument('--pdfium-library',type=Path,help='Prepend this library directory to LD_LIBRARY_PATH and verify ldd resolves it')
parser.add_argument('--corpus',type=Path,default=CORPUS)
parser.add_argument('--output',type=Path,required=True)
parser.add_argument('--self-check',action='store_true',help='Validate fixture/CMM only; not rendering acceptance')
parser.add_argument('--scales',type=float,nargs='+',default=[.5,1.,2.])
args=parser.parse_args()
if not args.self_check and not args.build_dir:parser.error('--build-dir is required unless --self-check')
if not 1<=len(args.scales)<=4 or any(not math.isfinite(s) or s<.25 or s>4 for s in args.scales):parser.error('scales must be 1..4 finite values in .25..4')
if len(set(args.scales))!=len(args.scales):parser.error('scales must be unique')
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
spec=json.loads((args.corpus/'manifest.json').read_text());source=(args.corpus/spec['file']).resolve()
report={'schemaVersion':1,'success':False,'fixtureValidationSuccess':False,'renderingAcceptance':False,
'mode':'fixture-self-check' if args.self_check else 'renderer-comparison','commands':[],'scales':[],
'runnerSha256':sha(Path(__file__)),'fixtureManifestSha256':sha(args.corpus/'manifest.json'),
'fixtureSha256':sha(source),'pythonVersion':platform.python_version(),'pillowLcmsVersion':ImageCms.core.littlecms_version,
'osRelease':platform.freedesktop_os_release(),'tolerance8Bit':spec['tolerance8Bit'],
'oracleRevision':spec.get('oracleRevision',1),'oracleKnownIssue':spec.get('oracleRevision')!=2,
'scope':'Direct explicit-intent CMM expected colors; production isolated PDF worker and independent Poppler. '
'CMM may share LittleCMS code with PDF renderers. Synthetic profiles only; no physical display/press, '
'human-approved golden or general ICC conformance assertion.'}
cmm=None;binaries={};environment=os.environ.copy()
try:
profiles,verified=verify_fixture(args.corpus,spec);report['fixtureVerification']=verified
cmm=Cmm(profiles);report['directCmm']={'encodedVersion':cmm.version,'libraries':cmm.libraryPaths,
'inputFormat':'TYPE_CMYK_DBL, 0..100 percent','outputFormat':'TYPE_RGB_8, cmsCreate_sRGBProfile','flags':0}
report['cmmValidation']=cmm_checks(profiles,spec,cmm);report['fixtureValidationSuccess']=True
if not args.self_check:
build=args.build_dir.resolve();binaries={name:sha(build/name) for name in ('pdf-worker-evidence','docview-pdf-worker')}
report['binaries']=binaries
if args.pdfium_library:
environment['LD_LIBRARY_PATH']=str(args.pdfium_library.resolve().parent)+(':'+environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '')
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT'),'Loader injection would invalidate evidence'
report['pdfiumLibrary']=loaded_pdfium(build/'docview-pdf-worker',environment,args.pdfium_library)
for program in ('qpdf','pdftoppm'):assert shutil.which(program),program+' required'
report['popplerVersion']=subprocess.run(['pdftoppm','-v'],capture_output=True,text=True,check=True).stderr.splitlines()[0]
def run(label,command,env=None):
log=output/(label+'.log')
with log.open('w') as f:result=subprocess.run(command,stdout=f,stderr=subprocess.STDOUT,env=env,timeout=300)
report['commands'].append({'name':label,'command':command,'exitCode':result.returncode,'logSha256':sha(log)})
assert result.returncode==0,label+' failed'
run('qpdf-check',['qpdf','--check',str(source)])
for scale in args.scales:
label='scale-'+str(scale).replace('.','_');folder=output/label;(folder/'pdfium').mkdir(parents=True)
run(label+'-pdfium',[str(build/'pdf-worker-evidence'),str(source),str(folder/'pdfium'),str(scale)],environment)
run(label+'-poppler',['pdftoppm','-cropbox','-r',str(72*scale),'-png',str(source),str(folder/'poppler')])
metadata=json.loads((folder/'pdfium/metadata.json').read_text())
assert metadata['pageCount']==spec['pageCount'] and metadata['renderScale']==scale and 'sandbox enabled' in metadata['transport']
row={'scale':scale,'probes':[],'pdfiumSuccess':True,'popplerSuccess':True,'pages':[]}
for page in range(1,spec['pageCount']+1):
paths={'pdfium':folder/'pdfium'/f'page-{page}.png','poppler':folder/f'poppler-{page}.png'}
images={key:Image.open(path).convert('RGB') for key,path in paths.items()}
dimensions=[math.ceil(v*scale) for v in spec['pageSizePt']]
assert all(list(image.size)==dimensions for image in images.values())
for probe in (p for p in spec['probes'] if p['page']==page):
x,y=probe['pointPt'];pixel=(round(x*scale),round((spec['pageSizePt'][1]-y)*scale))
components=probe['components']
if 'shading' in probe:
shading=probe['shading'];t=((pixel[0]+.5)/scale-shading['xStart'])/(shading['xEnd']-shading['xStart'])
components=[a+(b-a)*t for a,b in zip(shading['c0'],shading['c1'])]
expected=cmm.color(probe['profile'],probe['expectedIntentIndex'],components)
actual={key:list(image.getpixel(pixel)) for key,image in images.items()}
errors={key:max(abs(a-b) for a,b in zip(color,expected)) for key,color in actual.items()}
matches={key:error<=spec['tolerance8Bit'] for key,error in errors.items()}
row['probes'].append({**probe,'pointPx':pixel,'sampledComponents':components,'expectedRgb':expected,
'actualRgb':actual,'maxChannelError':errors,'matches':matches,'success':all(matches.values())})
for key,matched in matches.items():row[key+'Success'] &= matched
diff=ImageChops.difference(images['pdfium'],images['poppler']);diff.save(folder/f'difference-{page}.png')
sheet=Image.new('RGB',(1260,420),'#eeeeee');draw=ImageDraw.Draw(sheet)
for column,(label2,image) in enumerate([*images.items(),('difference',diff)]):
draw.text((column*420+8,7),f'{scale:g}x page {page}: {label2}',fill='black')
thumb=image.copy();thumb.thumbnail((410,380));sheet.paste(thumb,(column*420+5,30))
sheet.save(folder/f'comparison-{page}.png')
row['pages'].append({'page':page,'images':{key:sha(path) for key,path in paths.items()}})
row['success']=row['pdfiumSuccess'] and row['popplerSuccess'];report['scales'].append(row)
report['originalUnchanged']=sha(source)==spec['sha256'] and all(sha(args.corpus/item['file'])==item['sha256'] for item in spec['profiles'].values())
report['binariesUnchanged']=all(sha(build/name)==value for name,value in binaries.items()) and sha(Path(report['pdfiumLibrary']['path']))==report['pdfiumLibrary']['sha256']
report['pdfiumSuccess']=all(row['pdfiumSuccess'] for row in report['scales'])
report['popplerSuccess']=all(row['popplerSuccess'] for row in report['scales'])
report['renderingAcceptance']=report['success']=report['pdfiumSuccess'] and report['popplerSuccess'] and report['originalUnchanged'] and report['binariesUnchanged'] and not report['oracleKnownIssue']
except Exception as error:
report['error']=type(error).__name__+': '+str(error)
raise
finally:
if cmm:cmm.close()
report['evidenceSha256']={str(p.relative_to(output)):sha(p) for p in sorted(output.rglob('*')) if p.is_file()}
(output/'report.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps({key:report.get(key) for key in ('mode','fixtureValidationSuccess','success','pdfiumSuccess','popplerSuccess')}))
if not args.self_check and not report['success']:raise SystemExit('Intent validation failed; retained report preserves every failed probe')
if __name__=='__main__':main()
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Materialize previously verified fixed Git archives for an isolated PDFium build."""
import hashlib
import json
from pathlib import Path, PurePosixPath
import tarfile
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream: return hashlib.file_digest(stream,'sha256').hexdigest()
def main():
reports=[ROOT/'tests/results/source-archives/pdfium-git/report.json',
ROOT/'tests/results/source-archives/pdfium-gitlink/report.json']
repositories=[]
for path in reports:
report=json.loads(path.read_text()); assert report['success']
repositories.extend(report['repositories'])
source=ROOT/'build-pdfium-intent/source'
source.mkdir(parents=True,exist_ok=False)
output=ROOT/'tests/results/pdfium-intent-build'; output.mkdir(parents=True,exist_ok=False)
materialized=[]; links=[]; total=0; count=0
for repository in repositories:
archive=ROOT/repository['archive']; inventory=ROOT/repository['inventory']
assert sha(archive)==repository['archiveSha256'] and sha(inventory)==repository['inventorySha256']
expected={r['path']:r for line in inventory.open() if (r:=json.loads(line))['archived']}
prefix='pdfium/'+('' if repository['path']=='.' else repository['path']+'/')
found=set()
with tarfile.open(archive) as stream:
for member in stream:
assert member.name.startswith(prefix)
relative=member.name.removeprefix(prefix)
if member.isdir(): continue
assert relative in expected and relative not in found
found.add(relative); row=expected[relative]
name=PurePosixPath(member.name.removeprefix('pdfium/'))
assert not name.is_absolute() and '..' not in name.parts and '\\' not in str(name)
path=source/name; assert path.resolve().is_relative_to(source) and not path.exists()
path.parent.mkdir(parents=True,exist_ok=True)
if member.issym():
assert row['mode']=='120000' and member.linkname==row['linkTarget']
target=PurePosixPath(member.linkname)
assert not target.is_absolute() and (path.parent/target).resolve().is_relative_to(source)
assert hashlib.sha256(member.linkname.encode()).hexdigest()==row['sha256']
links.append((path,member.linkname))
continue
assert member.isfile() and row['mode'] in ('100644','100755') and member.size==row['bytes']
total+=member.size; count+=1
assert member.size<=256*1024**2 and total<=2*1024**3 and count<=100000
digest=hashlib.sha256()
with stream.extractfile(member) as incoming, path.open('xb') as outgoing:
for chunk in iter(lambda:incoming.read(1024*1024),b''):
digest.update(chunk); outgoing.write(chunk)
assert digest.hexdigest()==row['sha256']
path.chmod(0o755 if row['mode']=='100755' else 0o644)
assert found==expected.keys()
materialized.append({k:repository[k] for k in ('path','revision','archive','archiveSha256','inventory','inventorySha256')})
for path,target in links:
assert (path.parent/target).resolve().is_file()
path.symlink_to(target)
record={'success':True,'preparerSha256':sha(Path(__file__)),'source':str(source.relative_to(ROOT)),
'repositories':materialized,'regularFiles':count,'regularBytes':total,'symlinks':len(links),
'sourceReports':{str(p.relative_to(ROOT)):sha(p) for p in reports},
'scope':'Fixed sources materialized for a separate local build. No source hooks, downloads or build executed by this preparer. Production dependency remains unchanged.'}
(output/'source-materialization.json').write_text(json.dumps(record,indent=2)+'\n')
print(json.dumps({'repositories':len(materialized),'regularFiles':count,'regularBytes':total,'symlinks':len(links)}))
if __name__=='__main__':main()
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Materialize the three verified, pinned tools without running download hooks."""
import hashlib
import json
from pathlib import Path, PurePosixPath
import subprocess
import tarfile
import zipfile
ROOT = Path(__file__).resolve().parents[2]
SOURCE = ROOT / 'build-pdfium-intent/source'
RESULTS = ROOT / 'tests/results/pdfium-intent-build'
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
downloads = json.loads((RESULTS / 'tool-downloads.json').read_text())
destinations = {
'clang.tar.xz': 'third_party/llvm-build/Release+Asserts',
'sysroot.tar.xz': 'build/linux/debian_bullseye_amd64-sysroot',
'gn.zip': 'buildtools/linux64',
}
records = []
for row in downloads['inputs']:
archive = ROOT / row['path']
assert sha(archive) == row['sha256']
destination = SOURCE / destinations[row['name']]
already_extracted = destination.exists()
destination.mkdir(parents=True, exist_ok=True)
if row['name'].endswith('.tar.xz'):
with tarfile.open(archive) as tar:
members = tar.getmembers()
assert len(members) < 30000
assert sum(member.size for member in members) < 1024**3
for member in members:
assert member.isfile() or member.isdir() or member.issym() or member.islnk()
assert not PurePosixPath(member.name).is_absolute()
assert '..' not in PurePosixPath(member.name).parts
# Python's data filter also rejects links that escape the destination.
tarfile.data_filter(member, str(destination))
if not already_extracted:
tar.extractall(destination, members=members, filter='data')
for member in members:
target = destination / member.name
if member.isfile() or member.islnk():
with tar.extractfile(member) as stream:
assert sha(target) == hashlib.file_digest(stream, 'sha256').hexdigest()
elif member.issym():
assert target.is_symlink() and str(target.readlink()) == member.linkname
else:
with zipfile.ZipFile(archive) as zip_file:
assert set(zip_file.namelist()) == {'gn', '.cipdpkg/manifest.json'}
manifest = json.loads(zip_file.read('.cipdpkg/manifest.json'))
assert manifest['package_name'] == row['cipdPackage']
for member in zip_file.infolist():
assert member.file_size < 16 * 1024**2
target = destination / member.filename
target.parent.mkdir(parents=True, exist_ok=True)
if not already_extracted:
target.write_bytes(zip_file.read(member))
target.chmod((member.external_attr >> 16) & 0o777)
assert target.read_bytes() == zip_file.read(member)
records.append({'archive': row['path'], 'sha256': row['sha256'],
'destination': str(destination.relative_to(ROOT))})
stamp = SOURCE / 'third_party/llvm-build/Release+Asserts/cr_build_revision'
stamp.write_text('llvmorg-24-init-3796-g20e97c4b-4\n')
gclient = SOURCE / 'build/config/gclient_args.gni'
gclient_text = ('android_ndk_version = "[email protected]"\n'
'build_with_chromium = false\ncheckout_android = false\n'
'checkout_libpng = true\ncheckout_skia = false\n')
if gclient.exists():
assert gclient.read_text() == gclient_text
else:
gclient.write_text(gclient_text)
versions = {}
for name, relative in [('gn', 'buildtools/linux64/gn'),
('clang', 'third_party/llvm-build/Release+Asserts/bin/clang'),
('lld', 'third_party/llvm-build/Release+Asserts/bin/ld.lld')]:
result = subprocess.run([str(SOURCE / relative), '--version'], check=True,
capture_output=True, text=True)
versions[name] = result.stdout.strip()
assert '150a9d6ba0aa' in versions['gn']
assert 'clang version 24.0.0git' in versions['clang']
record = {'success': True, 'preparerSha256': sha(Path(__file__)),
'tools': records, 'versions': versions,
'gclientArgsSha256': sha(gclient),
'scope': 'Pinned tools extracted; version commands only. No system installation or source download hooks.'}
(RESULTS / 'tool-materialization.json').write_text(json.dumps(record, indent=2) + '\n')
print(json.dumps(record, indent=2))
if __name__ == '__main__':
main()
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env python3
"""Prepare an isolated, pinned LittleCMS reference for the dedicated Ubuntu VM.
No package installation, system linker change, PDFium replacement, or downloads.
The PDFium allocator integration is reversed only in the dedicated build copy.
Other vendored changes remain and are explicitly identified in the build record.
"""
import argparse
import hashlib
import io
import json
import os
from pathlib import Path
import platform
import re
import shutil
import subprocess
import sys
import tarfile
ROOT=Path(__file__).resolve().parents[2]
REVISION='b76633e60c8387a77268fb3359277ca25b5fd75c'
def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path,value): path.write_text(json.dumps(value,indent=2)+'\n')
def json_hash(value): return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':')).encode()).hexdigest()
def pack(args):
source=args.source.resolve(strict=True);output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
readme=(source/'README.pdfium').read_text()
assert 'Version: 2.19\n' in readme and 'Revision: '+REVISION+'\n' in readme
assert re.search(r'#define\s+LCMS_VERSION\s+2190\b',(source/'include/lcms2.h').read_text())
files={}
for p in sorted(source.rglob('*')):
if p.is_dir():continue
assert p.is_file() and not p.is_symlink() and p.resolve().is_relative_to(source)
relative=p.relative_to(source)
allowed=(len(relative.parts)==1 and (p.name in ('LICENSE','README.pdfium') or p.suffix=='.patch')) or (relative.parts[0] in ('src','include') and p.suffix in ('.c','.h'))
assert allowed and p.stat().st_size<=2*1024*1024
files['third_party/lcms/'+str(relative)]=p.read_bytes()
assert len(files)<100 and sum(map(len,files.values()))<16*1024*1024
reference=json.loads(args.reference_report.read_text());other=json.loads(args.comparison_report.read_text())
assert reference['directCmm']['encodedVersion']==2190 and other['directCmm']['encodedVersion']==2140
rows=[];comparisons=[]
for sr,so in zip(reference['scales'],other['scales'],strict=True):
assert sr['scale']==so['scale']
for a,b in zip(sr['probes'],so['probes'],strict=True):
fields=('page','profile','case','column','expectedIntentIndex','sampledComponents')
assert all(a[k]==b[k] for k in fields)
row={k:a[k] for k in fields};row.update(scale=sr['scale'],expectedRgb=a['expectedRgb'])
rows.append(row)
comparisons.append({'scale':sr['scale'],'page':a['page'],'case':a['case'],'column':a['column'],'profile':a['profile'],
'referenceExpectedRgb':a['expectedRgb'],'system214ExpectedRgb':b['expectedRgb'],
'pdfiumRgbEqual':a['actualRgb']['pdfium']==b['actualRgb']['pdfium'],
'pdfiumRgb':a['actualRgb']['pdfium'],'expectedRgbEqual':a['expectedRgb']==b['expectedRgb']})
assert len(rows)==600
profiles={}
corpus=ROOT/'tests/fixtures/pdf/rendering-intents'
manifest=json.loads((corpus/'manifest.json').read_text())
for key,item in manifest['profiles'].items():
path=corpus/item['file'];assert sha(path)==item['sha256']
files['profiles/'+path.name]=path.read_bytes();profiles[key]={'file':'profiles/'+path.name,'sha256':sha(path)}
expectations={'referenceReportSha256':sha(args.reference_report),'referenceDirectCmm':reference['directCmm'],
'comparisonReportSha256':sha(args.comparison_report),'profiles':profiles,'probes':rows,
'scope':'CMM numerical reproducibility only; the older shading-pattern-inherit state expectation is not adopted as PDF specification acceptance.'}
files['expected.json']=(json.dumps(expectations,indent=2)+'\n').encode()
files['intents.py']=Path(__file__).with_name('intents.py').read_bytes()
files['prepare_reference_cmm.py']=Path(__file__).read_bytes()
archive=output/'reference-lcms219.tar'
with tarfile.open(archive,'w',format=tarfile.PAX_FORMAT) as tar:
for name,data in sorted(files.items()):
info=tarfile.TarInfo(name);info.size=len(data);info.mode=0o644;info.mtime=0
tar.addfile(info,io.BytesIO(data))
inventory=[{'path':name,'size':len(data),'sha256':hashlib.sha256(data).hexdigest()} for name,data in sorted(files.items())]
record={'schemaVersion':1,'upstreamVersion':'2.19','upstreamRevision':REVISION,
'sourceScope':'Fixed PDFium-vendored source; allocator integration will be reversed in build copy only.',
'archive':archive.name,'archiveSha256':sha(archive),'files':inventory,
'sourceInventorySha256':json_hash([r for r in inventory if r['path'].startswith('third_party/')]),
'referenceReportSha256':sha(args.reference_report),'comparisonReportSha256':sha(args.comparison_report)}
write_json(output/'bundle.json',record)
write_json(output/'cross-os-before.json',{'scope':expectations['scope'],'rows':comparisons,
'total':len(comparisons),'allPdfiumRgbEqual':all(r['pdfiumRgbEqual'] for r in comparisons),
'differentExpectedRgb':sum(not r['expectedRgbEqual'] for r in comparisons)})
print(json.dumps({'archive':str(archive),'sha256':record['archiveSha256'],'files':len(inventory)}))
def build(args):
bundle=args.bundle.resolve(strict=True);record=json.loads(bundle.read_text());archive=bundle.parent/record['archive']
assert sha(archive)==record['archiveSha256']
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
source=output/'source';source.mkdir()
rows={r['path']:r for r in record['files']};assert len(rows)==len(record['files'])<=100
with tarfile.open(archive,'r:') as tar:
entries=tar.getmembers();assert len(entries)==len(rows)
seen=set()
for entry in entries:
name=entry.name;relative=Path(name)
assert name in rows and name not in seen and entry.isfile() and not relative.is_absolute() and '..' not in relative.parts
assert entry.size==rows[name]['size']<=2*1024*1024
seen.add(name);data=tar.extractfile(entry).read()
assert hashlib.sha256(data).hexdigest()==rows[name]['sha256']
path=source/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(data)
lcms=source/'third_party/lcms';cmserr=lcms/'src/cmserr.c';before=sha(cmserr)
patch_source=lcms/'0000-cmserr-changes.patch'
# This file first includes a historical patch-file diff. Select only its
# final, actual cmserr.c delta, never execute arbitrary packaged hooks.
marker='diff --git a/third_party/lcms/src/cmserr.c b/third_party/lcms/src/cmserr.c\n'
patch_text=patch_source.read_text();assert patch_text.count('\n'+marker)==1
delta=patch_text[patch_text.index('\n'+marker)+1:]
assert delta.count('\ndiff --git ')==0
patch_file=output/'restore-standard-allocator.patch';patch_file.write_text(delta)
commands=[]
def run(label,command,**kwargs):
result=subprocess.run(command,capture_output=True,text=True,timeout=300,**kwargs)
log=output/(label+'.log');log.write_text(result.stdout+result.stderr)
commands.append({'command':command,'exitCode':result.returncode,'log':log.name,'logSha256':sha(log)})
assert result.returncode==0,label+' failed; see '+str(log)
return result.stdout
run('allocator-restore',['patch','--batch','--fuzz=0','--reverse','-p1','-i',str(patch_file)],cwd=source)
assert 'FXMEM_' not in cmserr.read_text() and '#include "core/' not in cmserr.read_text()
changes=[]
for row in record['files']:
now=sha(source/row['path'])
if now!=row['sha256']:changes.append({'path':row['path'],'before':row['sha256'],'after':now})
assert len(changes)==1 and changes[0]['path']=='third_party/lcms/src/cmserr.c' and changes[0]['before']==before
compiler=shutil.which('gcc');assert compiler
version=run('compiler-version',[compiler,'--version']);target=run('compiler-target',[compiler,'-dumpmachine']).strip()
library_dir=output/'lib';library_dir.mkdir();library=library_dir/'liblcms2.so.2.0.19'
inputs=[str(p) for p in sorted((lcms/'src').glob('*.c'))];assert len(inputs)==26
compile_command=[compiler,'-std=c99','-O2','-fPIC','-D_POSIX_C_SOURCE=200809L','-shared',
'-Wl,-soname,liblcms2.so.2','-Wl,-z,defs','-I'+str(source),'-I'+str(lcms/'include'),
*inputs,'-lm','-lpthread','-o',str(library)]
run('compile',compile_command)
(library_dir/'liblcms2.so.2').symlink_to(library.name);(library_dir/'liblcms2.so').symlink_to(library.name)
dependencies=run('ldd',['ldd',str(library)]);assert 'not found' not in dependencies
dynamic=run('readelf',['readelf','-d',str(library)])
environment=os.environ.copy();environment['LD_LIBRARY_PATH']=str(library_dir)
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT')
run('runtime-version',[sys.executable,'-c',
'import ctypes; from PIL import ImageCms; c=ctypes.CDLL("liblcms2.so.2"); print(c.cmsGetEncodedCMMversion(), ImageCms.core.littlecms_version); assert c.cmsGetEncodedCMMversion()==2190; assert ImageCms.core.littlecms_version=="2.19"'],env=environment)
result={'schemaVersion':1,'success':True,'bundleSha256':sha(bundle),'archiveSha256':record['archiveSha256'],
'sourceInventorySha256':record['sourceInventorySha256'],'upstreamVersion':'2.19','upstreamRevision':REVISION,
'derivedSourceChanges':changes,'allocatorPatchSha256':sha(patch_file),
'remainingVendorPatches':'Retained; this is a pinned PDFium-vendored reference build with only its allocator integration reversed.',
'compiler':{'path':compiler,'sha256':sha(Path(compiler).resolve()),'version':version,'target':target},
'library':{'path':str(library),'size':library.stat().st_size,'sha256':sha(library),'soname':'liblcms2.so.2'},
'dependencies':dependencies,'dynamicSection':dynamic,'commands':commands,
'osRelease':platform.freedesktop_os_release(),'pythonVersion':platform.python_version(),
'scope':'Isolated reference library only; no system installation or loader configuration changes.'}
write_json(output/'build.json',result)
print(json.dumps({'success':True,'library':str(library),'sha256':sha(library)}))
def proof(args):
# Must start a fresh Python interpreter with the desired LD_LIBRARY_PATH;
# changing it after import cannot select an already loaded native library.
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
data=json.loads(args.expected.read_text());source=args.expected.resolve().parent
sys.path.insert(0,str(source))
from intents import Cmm
from PIL import Image,ImageCms
profiles={key:(source/row['file']).read_bytes() for key,row in data['profiles'].items()}
assert all(sha(source/row['file'])==row['sha256'] for row in data['profiles'].values())
cmm=Cmm(profiles)
try:
assert cmm.version==args.version
paths={line.split()[-1] for line in Path('/proc/self/maps').read_text().splitlines() if 'liblcms2.so' in line and '/' in line}
assert len(paths)==1
actual=Path(next(iter(paths))).resolve()
if args.library:assert actual==args.library.resolve(strict=True)
assert ImageCms.core.littlecms_version==f'{args.version//1000}.{(args.version%1000)//10}'
module=Path(ImageCms.core.__file__)
ldd=subprocess.run(['ldd',str(module)],capture_output=True,text=True,check=True,timeout=20).stdout
match=re.search(r'liblcms2\.so\.2 => (\S+)',ldd);assert match and Path(match[1]).resolve()==actual
pillow={}
for key,profile in profiles.items():
for i in range(4):
pillow[key,i]=ImageCms.buildTransformFromOpenProfiles(ImageCms.ImageCmsProfile(io.BytesIO(profile)),ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=i)
rows=[];quantized=[]
for p in data['probes']:
color=cmm.color(p['profile'],p['expectedIntentIndex'],p['sampledComponents'])
rows.append({**p,'actualRgb':color,'equal':color==p['expectedRgb'],
'maxChannelError':max(abs(a-b) for a,b in zip(color,p['expectedRgb']))})
for key in profiles:
for i in range(4):
pixel=(51,128,204,77)
direct=cmm.color(key,i,[v/255 for v in pixel])
pil=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),pixel),pillow[key,i]).getpixel((0,0)))
quantized.append({'profile':key,'intent':i,'directRgb':direct,'pillowRgb':pil,
'maxChannelError':max(abs(a-b) for a,b in zip(direct,pil))})
report={'schemaVersion':1,'success':all(p['equal'] for p in rows) and all(p['maxChannelError']<=1 for p in quantized),
'referenceReportSha256':data['referenceReportSha256'],'expectedFileSha256':sha(args.expected),
'expectedScope':data['scope'],'pythonVersion':platform.python_version(),'encodedVersion':cmm.version,
'pillowVersion':__import__('PIL').__version__,'pillowLcmsVersion':ImageCms.core.littlecms_version,
'loadedLibrary':{'path':str(actual),'sha256':sha(actual)},'mappedLibraries':sorted(paths),
'pillowExtension':{'path':str(module),'sha256':sha(module),'ldd':ldd},
'total':len(rows),'identical':sum(p['equal'] for p in rows),'different':sum(not p['equal'] for p in rows),
'pillowChecks':quantized,'probes':rows,
'limitations':'Numerical matching at the declared probes, not independent-CMM or PDF semantic correctness.'}
write_json(output/'proof.json',report)
print(json.dumps({k:report[k] for k in ('success','encodedVersion','pillowLcmsVersion','total','identical','different')}))
if args.version==2190:assert report['success'],'Pinned CMM differs from host reference'
finally:cmm.close()
def main():
parser=argparse.ArgumentParser(description=__doc__);commands=parser.add_subparsers(dest='command',required=True)
p=commands.add_parser('pack');p.add_argument('--source',type=Path,required=True);p.add_argument('--reference-report',type=Path,required=True);p.add_argument('--comparison-report',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=pack)
p=commands.add_parser('build');p.add_argument('--bundle',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=build)
p=commands.add_parser('proof');p.add_argument('--expected',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--version',type=int,choices=(2140,2190),required=True);p.add_argument('--library',type=Path);p.set_defaults(function=proof)
args=parser.parse_args();args.function(args)
if __name__=='__main__':main()
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Publish failing color evidence without converting diagnosis into acceptance."""
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
ROOT=Path(__file__).resolve().parents[2]
RESULTS=ROOT/'tests/results/cmyk-lut'
def sha(path):
with path.open('rb') as stream: return hashlib.file_digest(stream,'sha256').hexdigest()
def main():
prior=json.loads((RESULTS/'prior-validation-record.json').read_text())
assert sha(RESULTS/'prior-validation-record.json')=='c80457d32c246e40743f59f5e602ff0f71d59d496b33a72618a97475659c9013'
for name,digest in prior['evidenceSha256'].items(): assert sha(ROOT/name)==digest,name
sources={str(p.relative_to(ROOT)):sha(p) for directory in ('src','qml','cmake','resources')
for p in (ROOT/directory).rglob('*') if p.is_file() and '__pycache__' not in p.parts}
sources.update({name:sha(ROOT/name) for name in ('CMakeLists.txt','resources.qrc')})
assert sources==prior['sourceSha256']
for name,digest in prior['builds']['arch']['binaries'].items(): assert sha(ROOT/'build'/name)==digest
package=prior['ubuntuPackage']
assert sha(ROOT/'build-ubuntu-vm/packages'/package['archive'])==package['archiveSha256']
diagnosis=json.loads((RESULTS/'diagnosis.json').read_text())
assert diagnosis['diagnosisVerified'] and not diagnosis['renderingAcceptance']
assert not diagnosis['relativeIntentPreserved'] and diagnosis['toleranceUnchanged']==4
assert diagnosis['diagnoserSha256']==sha(ROOT/'tests/cmyk_lut/diagnose.py')
reports={}
for os_name in ('arch','ubuntu'):
report=json.loads((RESULTS/os_name/'report.json').read_text())
assert not report['success'] and report['originalUnchanged'] and report['binariesUnchanged']
assert len(report['scales'])==3
for name,digest in report['evidenceSha256'].items(): assert sha(RESULTS/os_name/name)==digest
assert report['binaries']['docview-pdf-worker']==prior['builds'][os_name]['binaries']['docview-pdf-worker']
assert report['runnerSha256']==sha(ROOT/'tests/cmyk_lut/run.py')
entry=next(r for r in diagnosis['results'] if r['os']==os_name)
assert entry['comparisonReportSha256']==sha(RESULTS/os_name/'report.json')
assert entry['failedPdfiumAnalyticProbes']==30 and entry['failedPopplerAnalyticProbes']==0
reports[os_name]=report
assert sha(ROOT/'build/pdf-worker-evidence')==reports['arch']['binaries']['pdf-worker-evidence']
assert sha(ROOT/'.deps/pdfium/lib/libpdfium.so')=='08f6ea53a64f6fbb9f5ba8d9c02e0051987c6a9175f3fb5ba25f219174731aaa'
reproduced=json.loads((RESULTS/'reproduction.json').read_text())
for name,digest in reproduced['filesByteIdentical'].items():
assert sha(ROOT/'tests/fixtures/pdf/cmyk-lut'/name)==digest==sha(RESULTS/'reproduction'/name)
shutdown=json.loads((RESULTS/'shutdown.json').read_text())
assert shutdown['qemuExitCode']==0 and shutdown['guestPowerDownObserved']
assert sha(RESULTS/'shutdown-tail.txt')==shutdown['tailSha256']
gallery=['''<!doctype html><html lang="ja"><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"><title>ICC CMYK 描画比較</title>
<style>body{margin:0;background:#f5f6f8;color:#202734;font:16px/1.7 system-ui,sans-serif}main{max-width:1500px;margin:auto;padding:36px 24px}h1{font-size:28px;line-height:1.4}h2{margin-top:40px}h3{margin:20px 0 8px}.status{background:#fff0ee;border-left:4px solid #bb3b30;padding:16px 20px}a{color:#1459a0}.row{display:grid;grid-template-columns:repeat(3,minmax(260px,1fr));gap:16px;overflow:auto}figure{margin:0;background:white;padding:12px}figcaption{font-weight:600;margin-bottom:8px}img{display:block;width:100%;height:auto}.hint{color:#596476;font-size:14px}</style>
<main><h1>ICC CMYK の追加描画比較</h1>
<p class="status"><strong>色の一致は未達です。</strong> Arch・Ubuntu各48点中30点が許容差を超えました。原因の照合は完了していますが、描画の受入や人による承認は完了していません。</p>
<p>相対的な測色を指定した資料に対して、固定PDFiumは知覚的なICC変換を使用します。上2段の右端の暗色に明瞭な差があります。下段はプロファイルを適用しないDeviceCMYKの対照です。</p>
<p><a href="README.md">試験条件・原因・未完了事項</a> / <a href="../../fixtures/pdf/cmyk-lut/icc-cmyk-lut.pdf">生成PDF</a> / <a href="diagnosis.json">数値による原因照合</a></p>
<p class="hint">各画像を開くと元の解像度で確認できます。差分画像は絶対RGB差で、色差の許容性や印刷色を判定する画像ではありません。</p>''']
for os_name,label in [('arch','Arch Linux'),('ubuntu','Ubuntu 24.04')]:
gallery.append('<h2>'+label+'</h2>')
for scale in reports[os_name]['scales']:
folder=os_name+'/scale-'+str(scale['scale']).replace('.','_')
gallery.append(f'<h3>{scale["scale"]:g}倍 / {scale["dpi"]:g} dpi</h3><div class="row">')
for title,name in [('PDFium / 本番隔離ワーカー','pdfium/page-1.png'),('Poppler / 独立した比較','poppler.png'),('絶対RGB差','difference.png')]:
path=folder+'/'+name
assert (RESULTS/path).is_file()
gallery.append(f'<figure><figcaption>{title}</figcaption><a href="{path}"><img src="{path}" alt="{title}の描画比較" loading="lazy"></a></figure>')
gallery.append('</div>')
gallery.append('</main></html>')
(RESULTS/'index.html').write_text('\n'.join(gallery))
tools=['tests/cmyk_lut/run.py','tests/cmyk_lut/diagnose.py','tests/cmyk_lut/record.py','tests/fixtures/pdf/generate_cmyk_lut.py','tests/render_pdf_worker.cpp']
new_requirement='Resolve or explicitly review the fixed PDFium ICC rendering-intent discrepancy demonstrated by the CMYK CLUT fixture; runtime detection/notice is not yet implemented'
record={**prior,'schemaVersion':4,'recordedAtUtc':datetime.now(timezone.utc).isoformat(),
'scope':'Unchanged validation3 product plus a new, failing CMYK ICC rendering comparison on Arch and Ubuntu. '
'Prior passing functional/package tests remain scoped to their original runs; color acceptance is not achieved.',
'goalComplete':False,'colorRenderingFinding':diagnosis,
'colorRenderingReports':{name:{'path':'tests/results/cmyk-lut/'+name+'/report.json','sha256':sha(RESULTS/name/'report.json')} for name in reports},
'previousValidation':{'record':'tests/results/cmyk-lut/prior-validation-record.json','sha256':sha(RESULTS/'prior-validation-record.json'),
'scope':'Validation3 package, all prior test executions and their exact product identities. No product bytes changed in the ICC investigation.'},
'remainingAcceptance':prior['remainingAcceptance']+[new_requirement],
'validationToolsSha256':{**prior['validationToolsSha256'],**{name:sha(ROOT/name) for name in tools}},
'evidenceSha256':{**prior['evidenceSha256'],**{str(p.relative_to(ROOT)):sha(p) for folder in (RESULTS,ROOT/'tests/fixtures/pdf/cmyk-lut')
for p in sorted(folder.rglob('*')) if p.is_file() and p.suffix!='.pyc' and p!=RESULTS/'record.json'}}}
encoded=json.dumps(record,ensure_ascii=False,indent=2)+'\n'
(RESULTS/'record.json').write_text(encoded); (ROOT/'docs/validation-record.json').write_text(encoded)
print(json.dumps({'colorRenderingAcceptance':False,'failedAnalyticProbes':60,'productUnchanged':True,'goalComplete':False}))
if __name__=='__main__': main()
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env python3
"""Add isolated-candidate evidence while preserving production failure history."""
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import xml.etree.ElementTree as ET
ROOT = Path(__file__).resolve().parents[2]
RESULTS = ROOT / 'tests/results/pdfium-intent-build'
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def read(relative):
return json.loads((RESULTS / relative).read_text())
def main():
previous = RESULTS / 'prior-validation-record.json'
assert sha(previous) == 'e171862993beb61a9786ffee1f6a4a869ba30e3c3bd03223918a20d8e41f297b'
prior = json.loads(previous.read_text())
for name, digest in prior['evidenceSha256'].items():
assert sha(ROOT / name) == digest, name
for name, digest in prior['sourceSha256'].items():
assert sha(ROOT / name) == digest, name
for name, digest in prior['validationToolsSha256'].items():
assert sha(ROOT / name) == digest, name
for name, digest in prior['builds']['arch']['binaries'].items():
assert sha(ROOT / 'build' / name) == digest, name
package = prior['ubuntuPackage']
assert sha(ROOT / 'build-ubuntu-vm/packages' / package['archive']) == package['archiveSha256']
assert sha(ROOT / '.deps/pdfium/lib/libpdfium.so') == '08f6ea53a64f6fbb9f5ba8d9c02e0051987c6a9175f3fb5ba25f219174731aaa'
candidate = read('candidate-final/report.json')
assert candidate['upstreamTestsPass'] and candidate['candidatePdfiumProbesPass']
assert candidate['allBinaryInputsUnchanged'] and not candidate['productionDependencyReplaced']
assert not candidate['success'] and not candidate['intentPopplerSuccess']
assert [(r['tests'], r['failures']) for r in candidate['upstreamTests']] == [(1104, 0), (872, 0)]
for name, digest in candidate['evidenceSha256'].items():
assert sha(RESULTS / 'candidate-final' / name) == digest, name
for name, digest in candidate['inputs'].items():
assert sha(Path(name)) == digest, name
intents = read('candidate-final/intents/report.json')
probes = [p for scale in intents['scales'] for p in scale['probes']]
assert len(probes) == 600 and all(p['matches']['pdfium'] for p in probes)
comparison_failures = [p for p in probes if not p['matches']['poppler']]
assert len(comparison_failures) == 18
assert {p['case'] for p in comparison_failures} == {'shading-pattern-RI-override'}
baseline = read('intents-baseline/report.json')
assert sum(not p['matches']['pdfium'] for s in baseline['scales'] for p in s['probes']) == 150
ctest = read('docview-ctest/report.json')
assert ctest['exitCode'] == 0 and ctest['libraryUnchanged']
assert ctest['librarySha256'] == intents['pdfiumLibrary']['sha256']
suite = ET.parse(RESULTS / 'docview-ctest/tests.xml').getroot()
assert suite.tag == 'testsuite' and suite.attrib['tests'] == '22' and suite.attrib['failures'] == '0'
patch = read('candidate-patch.json')
assert patch['success'] and patch['appliesWithoutFuzz'] and patch['appliedSourcesByteIdentical']
assert sha(ROOT / patch['patch']) == patch['patchSha256']
assert sha(ROOT / patch['patchSnapshot']) == patch['patchSha256']
for row in patch['files']:
assert sha(ROOT / 'build-pdfium-intent/source' / row['path']) == row['afterSha256']
reproduced = read('fixture-reproduction.json')
assert reproduced['success'] and len(reproduced['filesByteIdentical']) == 4
for name, digest in reproduced['filesByteIdentical'].items():
assert sha(RESULTS / 'reproduced-fixture' / name) == digest
assert sha(ROOT / 'tests/fixtures/pdf/rendering-intents' / name) == digest
tools = ['tests/cmyk_lut/' + name + '.py' for name in
('prepare_pdfium_source', 'fetch_pdfium_build_tools', 'prepare_pdfium_tools',
'export_pdfium_patch', 'validate_pdfium_candidate', 'record_candidate', 'intents')]
tools.append('tests/fixtures/pdf/generate_rendering_intents.py')
candidate_summary = {
'productionDependencyReplaced': False, 'candidatePdfiumProbesPass': True,
'humanRenderingAcceptance': False, 'crossRendererComparisonAccepted': False,
'sourceRevision': patch['baseRevision'], 'patch': patch['patch'],
'patchSha256': patch['patchSha256'], 'changedCoreFiles': len(patch['files']),
'patchSnapshot': patch['patchSnapshot'],
'library': intents['pdfiumLibrary']['path'], 'librarySha256': intents['pdfiumLibrary']['sha256'],
'upstreamTests': candidate['upstreamTests'], 'docviewCtestGroups': 22,
'cmykAnalyticProbesPassed': 48, 'intentProbesPassed': 600,
'popplerFailedProbes': 18, 'popplerFailureCase': 'shading-pattern-RI-override',
'report': 'tests/results/pdfium-intent-build/candidate-final/report.json',
'reportSha256': sha(RESULTS / 'candidate-final/report.json'),
'scope': 'Arch isolated candidate with existing application/worker binaries; no adoption into the Ubuntu package, Windows execution, calibrated-output or human golden acceptance.'}
evidence = {str(path.relative_to(ROOT)): sha(path)
for folder in (RESULTS, ROOT / 'tests/fixtures/pdf/rendering-intents')
for path in sorted(folder.rglob('*')) if path.is_file()
and '__pycache__' not in path.parts and path != RESULTS / 'record.json'}
record = {**prior, 'schemaVersion': 5, 'recordedAtUtc': datetime.now(timezone.utc).isoformat(),
'scope': 'Unchanged validation3 product plus an isolated source-built PDFium rendering-intent candidate. Candidate numeric/upstream/application tests pass; comparator differences and final adoption/acceptance remain.',
'goalComplete': False, 'pdfiumRenderingIntentCandidate': candidate_summary,
'previousValidation': {'record': str(previous.relative_to(ROOT)), 'sha256': sha(previous),
'scope': 'Unchanged product and original failing ICC evidence; retained without rewriting its verdict.'},
'sourceSha256': {**prior['sourceSha256'], patch['patch']: patch['patchSha256']},
'validationToolsSha256': {**prior['validationToolsSha256'], **{name: sha(ROOT / name) for name in tools}},
'evidenceSha256': {**prior['evidenceSha256'], **evidence},
'remainingAcceptance': [*prior['remainingAcceptance'],
'Review and adopt the source-built rendering-intent patch; validate Ubuntu packaging and remaining rendering boundaries (including dedicated soft-mask/tiling probes). Independent Poppler shading-pattern intent differences remain recorded.']}
encoded = json.dumps(record, ensure_ascii=False, indent=2) + '\n'
(RESULTS / 'record.json').write_text(encoded)
(ROOT / 'docs/validation-record.json').write_text(encoded)
print(json.dumps({'goalComplete': False, 'candidateProbesPassed': 648,
'upstreamTestsPassed': 1976, 'docviewGroupsPassed': 22,
'evidenceFiles': len(record['evidenceSha256'])}))
if __name__ == '__main__':
main()
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""Freeze a build/test anchor for the corrected PDFium candidate, before staging."""
import hashlib
import json
from pathlib import Path
import xml.etree.ElementTree as ET
ROOT = Path(__file__).resolve().parents[2]
RESULTS = ROOT / 'tests/results/pdfium-intent-context'
PARENT = ROOT / 'tests/results/pdfium-intent-build/record.json'
PARENT_SHA = '47b4dc2efbf8e671f904610b3047400dbd3ceb0fa10d6bd53fb04f3727943e7e'
LIBRARY_SHA = 'cb049fe434f4c911b5eb047c0aca572d4dd9f5e405dde329b76c05a6aa771403'
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def rel(path):
return str(path.relative_to(ROOT))
def check_nested(path):
data = json.loads(path.read_text())
for name, digest in data.get('evidenceSha256', {}).items():
assert sha(path.parent / name) == digest, name
return data
def main():
destination = RESULTS / 'record.json'
assert not destination.exists(), 'The build anchor is immutable; create another version for changed inputs.'
assert sha(PARENT) == PARENT_SHA
parent = json.loads(PARENT.read_text())
source = ROOT / 'build-pdfium-intent-context/source'
library = source / 'out/patched/libpdfium.so'
assert sha(library) == LIBRARY_SHA
patch = json.loads((RESULTS / 'candidate-patch.json').read_text())
assert patch['success'] and patch['appliesWithoutFuzz'] and patch['appliedSourcesByteIdentical']
assert patch['parentRecordSha256'] == PARENT_SHA
assert sha(ROOT / patch['patch']) == sha(ROOT / patch['patchSnapshot']) == patch['patchSha256']
for row in patch['files']:
assert sha(source / row['path']) == row['afterSha256'], row['path']
build = json.loads((RESULTS / 'build-2.json').read_text())
assert build['exitCode'] == 0
upstream = json.loads((RESULTS / 'upstream-tests-execution.json').read_text())
assert upstream['success'] and upstream['librarySha256'] == LIBRARY_SHA
tests = 0
for row in upstream['commands']:
assert row['exitCode'] == 0
data = row['results']
assert data['failures'] == data['errors'] == data['disabled'] == 0
assert sha(Path(row['command'][0])) == row['executableSha256']
tests += data['tests']
assert tests == 1979
arch = check_nested(RESULTS / 'arch/report.json')
assert arch['candidateTestsPass'] and arch['allBinaryInputsUnchanged']
assert arch['inputs'][str(library)] == LIBRARY_SHA
assert arch['intents']['probeCount'] == 600 and arch['transparency']['probeCount'] == 324
for name in ('cmyk', 'intents', 'transparency'):
check_nested(RESULTS / 'arch' / name / 'report.json')
junit = ET.parse(RESULTS / 'arch/tests.xml').getroot()
cases = list(junit.iter('testcase'))
assert len(cases) == 22 and not list(junit.iter('failure')) and not list(junit.iter('error'))
evidence = {rel(path): sha(path) for path in sorted(RESULTS.rglob('*'))
if path.is_file() and path != destination}
for name in ('source-materialization', 'tool-materialization', 'provider-patches'):
path = ROOT / ('tests/results/pdfium-intent-build/' + name + '.json')
assert sha(path) == parent['evidenceSha256'][rel(path)]
evidence[rel(path)] = sha(path)
evidence[rel(PARENT)] = PARENT_SHA
sources = {name: value for name, value in parent['sourceSha256'].items()
if name.startswith('resources/licenses/pdfium-supplemental/')}
for name, value in sources.items():
assert sha(ROOT / name) == value
tools = [Path(__file__), ROOT / 'tests/cmyk_lut/export_pdfium_context_patch.py',
ROOT / 'tests/cmyk_lut/intents.py', ROOT / 'tests/cmyk_lut/transparency.py',
ROOT / 'tests/cmyk_lut/correct_intent_oracle.py', ROOT / 'tests/cmyk_lut/validate_context_candidate.py',
ROOT / 'tests/fixtures/pdf/generate_intent_transparency.py']
sources.update({rel(path): sha(path) for path in tools})
for folder in ('rendering-intents-context', 'intent-transparency'):
sources.update({rel(path): sha(path) for path in (ROOT / 'tests/fixtures/pdf' / folder).iterdir() if path.is_file()})
record = {'schemaVersion': 6, 'goalComplete': False,
'parent': {'path': rel(PARENT), 'sha256': PARENT_SHA},
'scope': 'Immutable Linux candidate build and Arch test anchor before staging. Ubuntu, package adoption and release acceptance are recorded separately.',
'pdfiumRenderingIntentCandidate': {
'sourceRevision': patch['baseRevision'], 'source': rel(source),
'library': str(library), 'librarySha256': LIBRARY_SHA,
'patch': patch['patch'], 'patchSha256': patch['patchSha256'],
'patchSnapshot': patch['patchSnapshot'], 'report': rel(RESULTS / 'arch/report.json'),
'reportSha256': sha(RESULTS / 'arch/report.json'),
'productionDependencyReplaced': False, 'comparisonAccepted': False,
'candidateProbesPass': True, 'candidateProbeCount': 972,
'upstreamTestCount': tests, 'docviewCTestGroups': 22,
'oracleRevision': 2,
'supersedes': 'The old shading-pattern-inherit expectation was incorrect. The historical v1 pass remains a numerical result against that old expectation, not pattern conformance evidence.'},
'evidenceSha256': evidence, 'sourceSha256': sources}
destination.write_text(json.dumps(record, indent=2) + '\n')
print(json.dumps({'recordSha256': sha(destination), 'evidenceFiles': len(evidence),
'sourceFiles': len(sources), 'candidateProbeCount': 972}))
if __name__ == '__main__':
main()
@@ -0,0 +1,121 @@
#!/usr/bin/env python3
"""Record the tested v2 deliverables without rewriting their fixed build anchor."""
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
OUT = ROOT / 'tests/results/pdfium-intent-context'
PARENT = ROOT / 'tests/results/pdfium-intent-build/record.json'
PARENT_SHA = '47b4dc2efbf8e671f904610b3047400dbd3ceb0fa10d6bd53fb04f3727943e7e'
ANCHOR_SHA = '80af72b06d5c9a6a3ab8b9311373e6a23e8df28668ce575e1124350e69aa160b'
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def relative(path):
return str(path.relative_to(ROOT))
def reference(path):
return {'path': relative(path), 'sha256': sha(path)}
def main():
current = ROOT / 'docs/validation-record.json'
destination = OUT / 'integration-record.json'
assert not destination.exists(), 'Integration records are immutable.'
assert sha(current) == sha(PARENT) == PARENT_SHA
assert sha(OUT / 'record.json') == ANCHOR_SHA
parent = json.loads(PARENT.read_text())
anchor = json.loads((OUT / 'record.json').read_text())
required = {
'archCandidate': OUT / 'arch/report.json',
'ubuntuCandidate': OUT / 'ubuntu/report.json',
'archNewBuild': OUT / 'arch-integration/report.json',
'ubuntuNewBuildAndPackage': OUT / 'ubuntu-package/report.json',
'sdkFreeUbuntu': OUT / 'ubuntu-package/clean-vm/report.json',
'performance': OUT / 'performance/report.json',
'renderReview': OUT / 'render-review/report.json',
}
reports = {key: json.loads(path.read_text()) for key, path in required.items()}
for key, data in reports.items():
assert data.get('candidateTestsPass', data.get('success')) is True, key
assert reports['renderReview']['humanApproved'] is False
arch = reports['archNewBuild']
for name, digest in arch['binaries'].items():
assert sha(ROOT / 'build-context' / name) == digest, name
library = ROOT / '.deps/pdfium-context/lib/libpdfium.so'
assert sha(library) == anchor['pdfiumRenderingIntentCandidate']['librarySha256']
assert sha(ROOT / '.deps/pdfium/lib/libpdfium.so') == '08f6ea53a64f6fbb9f5ba8d9c02e0051987c6a9175f3fb5ba25f219174731aaa'
source_paths = {ROOT / name for name in parent['sourceSha256']}
source_paths.update(ROOT / name for name in parent['validationToolsSha256'])
for folder in ('tools', 'cmake', 'tests/cmyk_lut'):
source_paths.update(path for path in (ROOT / folder).rglob('*')
if path.is_file() and '__pycache__' not in path.parts)
required_sources = tuple(ROOT / name for name in (
'CMakeLists.txt', 'tests/test_stage_pdfium_candidate.py',
'tests/test_pdfium_candidate_integration.py', 'tests/ubuntu_vm/context_package.py',
'tests/ubuntu_vm/package_smoke.py', 'tests/ubuntu_vm/clean_package.py',
'tests/compare_pdf_extended.py', 'tests/generate_render_review.py'))
for path in required_sources:
assert path.is_file(), relative(path)
source_paths.update(required_sources)
sources = {relative(path): sha(path) for path in sorted(source_paths) if path.is_file()}
changed = [{'path': name, 'previousSha256': digest, 'currentSha256': sources[name]}
for name, digest in {**parent['sourceSha256'], **parent['validationToolsSha256']}.items()
if name in sources and sources[name] != digest]
evidence = {}
for folder in (OUT, ROOT / 'tests/results/pdfium-intent-transparency',
ROOT / 'tests/results/pdfium-intent-ubuntu'):
for path in sorted(folder.rglob('*')):
if path.is_file() and path != destination and '__pycache__' not in path.parts:
evidence[relative(path)] = sha(path)
record = {
'schemaVersion': 7, 'recordedAtUtc': datetime.now(timezone.utc).isoformat(),
'goalComplete': False,
'scope': 'Linux v2 candidate integrated into dedicated Arch and Ubuntu builds and a local validation deb. Earlier evidence is retained by immutable parent references. Target Windows/physical/human/release gates remain incomplete.',
'previousValidation': reference(PARENT),
'candidateBuildAnchor': reference(OUT / 'record.json'),
'pdfiumRenderingIntentCandidate': anchor['pdfiumRenderingIntentCandidate'],
'candidateIntegration': {
'selectedLibrary': reference(library),
'originalProviderPreserved': True,
'buildInfo': reference(ROOT / '.deps/pdfium-context/BUILDINFO.json'),
'reviewedLock': reference(ROOT / 'cmake/pdfium-candidate-v2.lock.json'),
'archExecutable': 'build-context/docview',
'archInstalledExecutable': 'build-context/install/bin/docview',
'ubuntuPackageRecord': reference(required['ubuntuNewBuildAndPackage']),
},
'executables': arch['binaries'],
'validationReports': {key: reference(path) for key, path in required.items()},
'oracleCorrection': reference(ROOT / 'tests/fixtures/pdf/rendering-intents-context/manifest.json'),
'historicalOracleLimitation': 'The v1 shading-pattern-inherit expectations selected paint-time RI incorrectly. Its numerical pass is preserved as historical evidence, not pattern conformance. Revision 2 keeps the original PDF/profile bytes and corrects eight rows.',
'referenceCmm': reference(ROOT / 'tests/results/pdfium-intent-ubuntu/reference-cmm/verification.json'),
'remainingAcceptance': [
'Windows 11 native build, sandbox/pipe/IME execution and distribution tests',
'Target-OS physical GPU/display/input, reference hardware and long-duration acceptance',
'Human approval of G-RENDER reference images and documented comparison differences',
'Final public distribution/license decision and complete corresponding-source/notice assessment',
],
'sourceSha256': sources, 'changedSincePreviousRecord': changed,
'documentationSha256': {name: sha(ROOT / name) for name in (
'README.md', 'docs/PDFIUM-CANDIDATE.md', 'docs/VALIDATION.md',
'docs/ACCEPTANCE-AUDIT.md', 'docs/UBUNTU-PACKAGE.md',
'tests/PDF-VALIDATION.md')},
'evidenceSha256': evidence,
}
encoded = json.dumps(record, indent=2) + '\n'
destination.write_text(encoded)
current.write_text(encoded)
print(json.dumps({'schemaVersion': 7, 'sha256': sha(current),
'evidenceFiles': len(evidence), 'sourceFiles': len(sources),
'goalComplete': False}))
if __name__ == '__main__':
main()
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Validate a known CMYK CLUT through the production isolated worker and Poppler."""
import argparse
import hashlib
from io import BytesIO
import json
import math
import os
from pathlib import Path
import platform
import struct
import subprocess
from PIL import Image, ImageChops, ImageCms, ImageDraw, ImageStat
from pypdf import PdfReader
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--build-dir', required=True, type=Path)
parser.add_argument('--output', required=True, type=Path)
args = parser.parse_args()
build = args.build_dir.resolve(); output = args.output.resolve()
output.mkdir(parents=True, exist_ok=False)
corpus = ROOT / 'tests/fixtures/pdf/cmyk-lut'
spec = json.loads((corpus / 'manifest.json').read_text())
source = corpus / spec['file']; icc_file = corpus / spec['profileFile']
assert sha(source) == spec['sha256'] and sha(icc_file) == spec['profileSha256']
assert sha(ROOT / 'tests/fixtures/pdf/generate_cmyk_lut.py') == spec['generatorSha256']
binaries = {name: sha(build / name) for name in ('pdf-worker-evidence', 'docview-pdf-worker')}
profile = icc_file.read_bytes()
assert len(profile) == spec['profileBytes'] == struct.unpack_from('>I', profile)[0]
assert profile[8:24] == bytes.fromhex('02100000') + b'scnrCMYKLab ' and profile[36:40] == b'acsp'
tags = {}
for i in range(struct.unpack_from('>I', profile, 128)[0]):
sig, offset, size = struct.unpack_from('>4sII', profile, 132+12*i)
assert offset % 4 == 0 and offset+size <= len(profile) and sig not in tags
tags[sig] = profile[offset:offset+size]
assert set(tags) == {b'desc', b'cprt', b'wtpt', b'A2B0'}
lut = tags[b'A2B0']
assert len(lut) == 176 and lut[:12] == b'mft2' + bytes(4) + bytes((4,3,2,0))
assert struct.unpack_from('>HH', lut, 48) == (2,2)
# Decode and check all 16 four-dimensional CLUT vertices independently.
for index in range(16):
c, m, y, k = [(index >> shift) & 1 for shift in (3,2,1,0)]
encoded = struct.unpack_from('>3H', lut, 68+index*6)
lab = [encoded[0]*100/65280, encoded[1]/256-128, encoded[2]/256-128]
expected = [100-12*c-15*m-10*y-35*k, 18*m-15*c, 20*y-12*c]
assert max(abs(a-b) for a,b in zip(lab,expected)) < .002
pdf = PdfReader(source)
assert len(pdf.pages) == 1 and list(pdf.pages[0].mediabox)[2:] == spec['pageSizePt']
resources = pdf.pages[0]['/Resources']
vector_space = resources['/ColorSpace']['/TestCMYK']
image_object = resources['/XObject']['/Sample'].get_object()
for space in (vector_space, image_object['/ColorSpace']):
assert space[0] == '/ICCBased' and space[1].get_object()['/N'] == 4
assert space[1].get_object()['/Alternate'] == '/DeviceCMYK'
assert space[1].get_object().get_data() == profile
assert image_object['/Width'] == 8 and image_object['/Height'] == 1 and image_object['/BitsPerComponent'] == 8
pixel_values = bytes(round(c*255) for p in spec['probes'] if p['kind'] == 'icc-image' for c in p['components'])
assert image_object.get_data() == pixel_values
icc = ImageCms.ImageCmsProfile(BytesIO(profile))
transform = ImageCms.buildTransformFromOpenProfiles(icc, ImageCms.createProfile('sRGB'), 'CMYK', 'RGB', renderingIntent=1)
cmm_probes = []
for probe in spec['probes']:
if probe['kind'] != 'icc-image': continue
pixel = Image.new('CMYK',(1,1),tuple(round(c*255) for c in probe['components']))
actual = ImageCms.applyTransform(pixel, transform).getpixel((0,0))
error = max(abs(a-b) for a,b in zip(actual,probe['expectedRgb']))
cmm_probes.append({'sample':probe['sample'], 'expectedRgb':probe['expectedRgb'], 'actualRgb':actual,
'maxChannelError':error, 'success':error <= spec['tolerance8Bit']})
report = {'success':False, 'osRelease':platform.freedesktop_os_release(), 'fixture':spec,
'python':platform.python_version(), 'pillowLcmsVersion':ImageCms.core.littlecms_version,
'runnerSha256':sha(Path(__file__)), 'binaries':binaries,
'rendererVersion':subprocess.run(['/usr/bin/pdftoppm','-v'],capture_output=True,text=True,check=True).stderr.splitlines()[0],
'embeddedProfileAndImageBytesVerified':True, 'clutVerticesVerified':16, 'cmmProbes':cmm_probes,
'scope':'Synthetic ICC v2 CMYK input -> Lab four-dimensional CLUT, vectors and images. '
'Production isolated worker at three scales and independent Poppler. '
'The CMM probe uses Pillow/LittleCMS and is not a third independent PDF renderer. '
'No calibrated press/display, arbitrary ICC conformance or human-approved golden.',
'commands':[], 'scales':[]}
def run(name, command):
with (output / (name+'.log')).open('w') as log:
result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=180)
report['commands'].append({'name':name, 'command':command, 'exitCode':result.returncode,
'logSha256':sha(output / (name+'.log'))})
assert result.returncode == 0, name
try:
assert all(p['success'] for p in cmm_probes), 'Profile differs from the analytic model'
run('qpdf-check',['/usr/bin/qpdf','--check',str(source)])
sheets=[]
for scale in (.5, 1., 4.):
name = 'scale-'+str(scale).replace('.','_'); folder=output/name
(folder/'pdfium').mkdir(parents=True)
run(name+'-pdfium',[str(build/'pdf-worker-evidence'),str(source),str(folder/'pdfium'),str(scale)])
run(name+'-poppler',['/usr/bin/pdftoppm','-singlefile','-cropbox','-r',str(72*scale),'-png',str(source),str(folder/'poppler')])
meta=json.loads((folder/'pdfium/metadata.json').read_text())
assert meta['renderScale']==scale and 'sandbox enabled' in meta['transport']
images={'pdfium':Image.open(folder/'pdfium/page-1.png').convert('RGB'),
'poppler':Image.open(folder/'poppler.png').convert('RGB')}
dimensions=[math.ceil(v*scale) for v in spec['pageSizePt']]
assert all(list(im.size)==dimensions for im in images.values())
row={'scale':scale, 'dpi':72*scale, 'dimensions':dimensions, 'probes':[], 'success':True}
for probe in spec['probes']:
x,y=probe['pointPt']; point=(round(x*scale),round((spec['pageSizePt'][1]-y)*scale))
colors={key:list(im.getpixel(point)) for key,im in images.items()}
errors={key:max(abs(a-b) for a,b in zip(color,probe['expectedRgb'])) for key,color in colors.items()} if 'expectedRgb' in probe else {}
matched=all(error<=spec['tolerance8Bit'] for error in errors.values())
row['probes'].append({**probe,'pointPx':point,'actualRgb':colors,'maxChannelError':errors,
'checkedAgainstAnalyticColor':bool(errors),'success':matched})
row['success'] &= matched
# A profile ignored in favor of its DeviceCMYK Alternate cannot pass.
black={p['kind']:p for p in row['probes'] if p['sample']==4}
differences={key:min(black['icc-vector']['actualRgb'][key][i]-black['device-control']['actualRgb'][key][i]
for i in range(3)) for key in images}
row['blackProfileMinusAlternateMinChannel']=differences
row['profileEffectDetected']=all(value>=80 for value in differences.values())
row['success'] &= row['profileEffectDetected']
diff=ImageChops.difference(images['pdfium'],images['poppler']); diff.save(folder/'difference.png')
row['meanAbsoluteChannelError']=sum(ImageStat.Stat(diff).mean)/3
sheet=Image.new('RGB',(1500,340),'#eeeeee'); draw=ImageDraw.Draw(sheet)
for col,(label,im) in enumerate([('PDFium - isolated worker',images['pdfium']),('Poppler - independent',images['poppler']),('Absolute RGB difference',diff)]):
draw.text((col*500+10,10),f'{scale:g}x / {label}',fill='black')
thumb=im.copy(); thumb.thumbnail((488,295)); sheet.paste(thumb,(col*500+(500-thumb.width)//2,35))
sheet.save(folder/'comparison.png'); sheets.append(sheet); report['scales'].append(row)
overview=Image.new('RGB',(1500,340*len(sheets)),'white')
for i,sheet in enumerate(sheets): overview.paste(sheet,(0,340*i))
overview.save(output/'overview.png')
report['originalUnchanged']=sha(source)==spec['sha256'] and sha(icc_file)==spec['profileSha256']
report['binariesUnchanged']=all(sha(build/name)==digest for name,digest in binaries.items())
report['success']=all(row['success'] for row in report['scales']) and report['originalUnchanged'] and report['binariesUnchanged']
finally:
report['evidenceSha256']={str(path.relative_to(output)):sha(path) for path in sorted(output.rglob('*')) if path.is_file()}
(output/'report.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps({'success':report['success'],'scales':len(report['scales']),
'analyticImageAndVectorProbes':sum(p['checkedAgainstAnalyticColor'] for row in report['scales'] for p in row['probes'])}))
if not report['success']: raise SystemExit('Color validation failed; inspect the retained report')
if __name__=='__main__':
main()
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Validate the fixed rendering-intent candidate in the dedicated Ubuntu guest."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import shutil
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
EXPECTED_LIBRARY = '0c968f503ce549bad5301af2cc3fd0b83c37437315ac0e62dff439d6a1c6cc4d'
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--library', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--build-dir', type=Path, default=Path.home() / 'docview-build')
args = parser.parse_args()
os_release = platform.freedesktop_os_release()
assert os_release['ID'] == 'ubuntu' and os_release['VERSION_ID'] == '24.04'
library = args.library.resolve()
assert sha(library) == EXPECTED_LIBRARY
output = args.output.resolve()
build = args.build_dir.resolve()
inputs = [library, ROOT / '.deps/pdfium/lib/libpdfium.so']
inputs += [build / name for name in ('docview', 'docview-pdf-worker', 'docview-archive-worker', 'pdf-worker-evidence')]
identities = {str(path): sha(path) for path in inputs}
output.mkdir(parents=True, exist_ok=False)
env = os.environ.copy()
assert not env.get('LD_PRELOAD') and not env.get('LD_AUDIT')
assert not env.get('QTWEBENGINE_DISABLE_SANDBOX')
env.pop('WAYLAND_DISPLAY', None)
env.update(LD_LIBRARY_PATH=str(library.parent) + ':/opt/docview-deps/lib:/opt/docview-qt/6.11.2/gcc_64/lib',
QT_QPA_PLATFORM='xcb', QT_QUICK_BACKEND='software',
QTWEBENGINE_CHROMIUM_FLAGS='--disable-gpu', QT_SCALE_FACTOR='1',
QT_AUTO_SCREEN_SCALE_FACTOR='0', XDG_SESSION_TYPE='x11')
report = {'schemaVersion': 1, 'success': False, 'osRelease': os_release,
'platform': platform.platform(), 'pythonVersion': platform.python_version(),
'runnerSha256': sha(Path(__file__)), 'inputs': identities, 'commands': [],
'productionDependencyReplaced': False,
'scope': 'Same fixed Linux candidate library as Arch, used by the existing Ubuntu-built application and workers. Dedicated Ubuntu VM, Xvfb/software, ordinary worker sandbox. No package installation or physical-display acceptance.'}
def run(label, command):
start = time.monotonic()
log = output / (label + '.log')
with log.open('w') as stream:
result = subprocess.run(command, cwd=ROOT, env=env,
stdout=stream, stderr=subprocess.STDOUT, timeout=900)
report['commands'].append({'name': label, 'command': command, 'exitCode': result.returncode,
'seconds': time.monotonic() - start, 'logSha256': sha(log)})
print(label, result.returncode, flush=True)
return result.returncode
try:
linked = subprocess.run(['ldd', str(build / 'docview-pdf-worker')], env=env,
capture_output=True, text=True, check=True).stdout
assert str(library) in linked and 'not found' not in linked
(output / 'worker-ldd.txt').write_text(linked)
run('cmyk', [sys.executable, str(ROOT / 'tests/cmyk_lut/run.py'),
'--build-dir', str(build), '--output', str(output / 'cmyk')])
run('intents', [sys.executable, str(ROOT / 'tests/cmyk_lut/intents.py'),
'--build-dir', str(build), '--pdfium-library', str(library),
'--scales', '.5', '1', '4', '--output', str(output / 'intents')])
ctest_status = run('ctest', ['xvfb-run', '-a', '-s', '-screen 0 1100x760x24',
'dbus-run-session', '--', 'ctest', '--test-dir', str(build),
'--output-on-failure', '--output-junit', str(output / 'tests.xml')])
shutil.copyfile(build / 'Testing/Temporary/LastTest.log', output / 'LastTest.log')
cmyk = json.loads((output / 'cmyk/report.json').read_text())
intents = json.loads((output / 'intents/report.json').read_text())
report['cmykSuccess'] = cmyk['success']
report['intentPdfiumSuccess'] = intents.get('pdfiumSuccess', False)
report['intentPopplerSuccess'] = intents.get('popplerSuccess', False)
report['ctestSuccess'] = ctest_status == 0
report['allBinaryInputsUnchanged'] = all(sha(Path(path)) == digest for path, digest in identities.items())
report['candidateTestsPass'] = (report['cmykSuccess'] and report['intentPdfiumSuccess']
and report['ctestSuccess'] and report['allBinaryInputsUnchanged'])
report['success'] = report['candidateTestsPass'] and intents['renderingAcceptance']
finally:
report['evidenceSha256'] = {str(path.relative_to(output)): sha(path)
for path in sorted(output.rglob('*')) if path.is_file()}
(output / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({key: report.get(key) for key in ('success', 'candidateTestsPass',
'intentPopplerSuccess', 'allBinaryInputsUnchanged')}))
if not report['success']:
raise SystemExit('Comparison not fully accepted; candidate and comparator verdicts are retained separately.')
if __name__ == '__main__':
main()
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/env python3
"""Check RI masks/patterns against declared PDF state rules and a direct CMM.
This runner never treats agreement between renderers as the expected value.
The numerical DeviceRGB mask convention is distinct from intent selection.
"""
import argparse
from io import BytesIO
import itertools
import json
import math
import os
from pathlib import Path
import platform
import shutil
import subprocess
from PIL import Image, ImageChops, ImageCms, ImageDraw
from pypdf import PdfReader
from intents import Cmm, INTENTS, loaded_pdfium, sha
ROOT=Path(__file__).resolve().parents[2]
CORPUS=ROOT/'tests/fixtures/pdf/intent-transparency'
PROFILE_SHA='b9a40bfc18e8280eefe0fedc4e607467e1cc27adeee3beb77f4a2b83fc7e9315'
def preserved_inputs():
paths=[ROOT/'tests/fixtures/pdf/generate_rendering_intents.py',ROOT/'tests/cmyk_lut/intents.py']
paths += sorted((ROOT/'tests/fixtures/pdf/rendering-intents').glob('*'))
return {str(p.relative_to(ROOT)):sha(p) for p in paths if p.is_file()}
def verify(corpus,spec):
assert spec['schemaVersion']==1 and spec['pageCount']==4
assert spec['pageSizePt']==[880,850] and spec['sample']==[0,.75,.25,0]
assert spec['tolerance8Bit']==4 and len(spec['cases'])==27
assert len(spec['probes'])==108
assert sha(corpus/spec['file'])==spec['sha256']
assert sha(ROOT/'tests/fixtures/pdf/generate_intent_transparency.py')==spec['generatorSha256']
profile=(corpus/spec['profile']['file']).read_bytes()
assert len(profile)==984 and sha(corpus/spec['profile']['file'])==PROFILE_SHA==spec['profile']['sha256']
assert profile==(ROOT/spec['profile']['origin']).read_bytes()
pdf=PdfReader(corpus/spec['file']);assert len(pdf.pages)==4
identities={};probe_ids=set()
for page_index,page in enumerate(pdf.pages,1):
assert list(page.mediabox)==[0,0,880,850]
resources=page['/Resources'];patterns=resources['/Pattern'];gs=resources['/ExtGState']
assert resources['/ColorSpace']['/CS'][1].get_object().get_data()==profile
assert resources['/ColorSpace']['/PCS'][0]=='/Pattern'
for key in ('U','ULarge'):
stencil=patterns['/'+key]
assert stencil['/PaintType']==2
assert stencil.get_data().split()==[b'0',b'0',str(stencil['/XStep']).encode(),str(stencil['/YStep']).encode(),b're',b'f']
for key in ('Tile','TileForm','ShadeForm'):
obj=patterns.raw_get('/'+key) if key=='Tile' else resources['/XObject'].raw_get('/'+key)
identities.setdefault(key,obj.idnum);assert identities[key]==obj.idnum
assert b' ri' not in patterns['/Tile'].get_data()
assert '/ExtGState' not in patterns['/Shade']
for label in ('Tile','Shade'):
form=resources['/XObject']['/'+label+'Form']
assert form.get_data().startswith(b'/AbsoluteColorimetric ri\n')
assert form['/Resources']['/Pattern'].raw_get('/Tile').idnum==identities['Tile']
for i,intent in enumerate(INTENTS):
assert ('/'+intent+' ri').encode() in patterns['/TileRi'+str(i)].get_data()
assert patterns['/ShadeRi'+str(i)]['/ExtGState']['/RI']=='/'+intent
for key in ('Mask','OpaqueBC','BC0','BC1','BC2','BC3',*[f'MaskRi{i}' for i in range(4)],
*[f'MaskGs{i}' for i in range(4)],*[f'MaskQ{i}' for i in range(4)]):
mask=gs['/'+key]['/SMask'];group=mask['/G']
assert mask['/S']=='/Luminosity' and mask['/TR']=='/Identity'
assert group['/Group']['/CS']=='/DeviceRGB' and bool(group['/Group']['/I'])
assert group['/Resources']['/ColorSpace']['/CS'][1].get_object().get_data()==profile
if key.startswith('BC'):assert group.get_data().strip()==b''
assert gs['/NoMask']['/SMask']=='/None'
assert gs['/Mask']['/SMask'].raw_get('/G').idnum==gs['/OpaqueBC']['/SMask'].raw_get('/G').idnum
content=page.get_contents().get_data()
assert content.startswith(b'1 g 0 0 880 850 re f\n')
for p in (p for p in spec['probes'] if p['page']==page_index):
assert p['id'] not in probe_ids;probe_ids.add(p['id'])
assert ('\n'.join(p['commandSequence'])+'\n').encode() in content
case=p['case'];column=p['column'];current=(2,0,2,1)[column] if case.endswith('-reuse') else column
assert p['callerIntentIndex']==current and p['callerIntent']==INTENTS[current]
expected=current;oracle='icc'
if case in ('tiling-page-initial','tiling-colored-qQ','shading-page-initial'):expected=1
elif case in ('tiling-colored-ri','tiling-colored-gs','tiling-form-ri','shading-pattern-ri','shading-form-ri',
'mask-internal-ri','mask-internal-gs'):expected=(current+1)%4
if case.startswith('mask-'):
oracle='luminosity-icc'
if case in ('mask-q-inner-none','mask-none'):oracle='black'
elif case=='mask-BC-only':oracle='luminosity-bc'
assert p['expectedIntentIndex']==expected and p['expectedIntent']==INTENTS[expected]
assert p['oracle']==oracle and p['components']==[0,.75,.25,0]
if oracle=='luminosity-bc':assert p['backgroundRgb']==[[1,0,0],[0,1,0],[0,0,1],[0,.5,1]][column]
return profile,{'pages':4,'probes':len(probe_ids),'reusedObjectIds':identities,
'uncoloredStencilsContainOnlyGeometry':True,'profileByteIdentical':True,
'expectedIntentIndependentlyChecked':True}
def oracle(probe,cmm):
kind=probe['oracle']
if kind=='black':return [0,0,0],{'kind':kind}
if kind=='luminosity-bc':rgb=[255*c for c in probe['backgroundRgb']]
else:rgb=cmm.color('distinct',probe['expectedIntentIndex'],probe['components'])
if kind=='icc':return rgb,{'kind':kind,'directCmmRgb8':rgb}
luminance=sum(w*c for w,c in zip((.30,.59,.11),rgb))
return [round(255-luminance)]*3,{'kind':kind,'groupRgb8':rgb,'luminosity8Bit':luminance,
'recommendedDeviceRgbWeights':[.30,.59,.11]}
def cmm_checks(profile,cmm,spec):
colors=[cmm.color('distinct',i,spec['sample']) for i in range(4)]
lumas=[sum(w*c for w,c in zip((.30,.59,.11),rgb)) for rgb in colors]
separation=min(abs(a-b) for a,b in itertools.combinations(lumas,2))
assert separation>2*spec['tolerance8Bit']
source=ImageCms.ImageCmsProfile(BytesIO(profile));sample=tuple(round(v*255) for v in spec['sample'])
checks=[]
for i in range(4):
transform=ImageCms.buildTransformFromOpenProfiles(source,ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=i)
quantized=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),transform).getpixel((0,0)))
assert max(abs(a-b) for a,b in zip(quantized,colors[i]))<=1
checks.append({'intent':INTENTS[i],'directCmmRgb8':colors[i],'pillowQuantizedRgb8':quantized,
'luminosity8Bit':lumas[i],'blackOnWhiteGray8Bit':round(255-lumas[i])})
return {'values':checks,'minimumLuminositySeparation':separation,
'noRenderedPixelsUsedForExpectedValues':True}
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--build-dir',type=Path)
parser.add_argument('--pdfium-library',type=Path)
parser.add_argument('--corpus',type=Path,default=CORPUS)
parser.add_argument('--output',type=Path,required=True)
parser.add_argument('--self-check',action='store_true')
parser.add_argument('--scales',type=float,nargs='+',default=[.5,1.,2.])
args=parser.parse_args()
if not args.self_check and not args.build_dir:parser.error('--build-dir required for rendering')
if not 1<=len(args.scales)<=4 or len(set(args.scales))!=len(args.scales) or any(not math.isfinite(v) or not .25<=v<=4 for v in args.scales):
parser.error('scales must be 1..4 unique finite values in .25..4')
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
old=preserved_inputs();spec=json.loads((args.corpus/'manifest.json').read_text());source=(args.corpus/spec['file']).resolve()
report={'schemaVersion':1,'success':False,'renderingAcceptance':False,'fixtureValidationSuccess':False,
'mode':'fixture-self-check' if args.self_check else 'renderer-comparison',
'runnerSha256':sha(Path(__file__)),'sharedCmmAdapterSha256':sha(Path(__file__).with_name('intents.py')),
'fixtureManifestSha256':sha(args.corpus/'manifest.json'),'fixtureSha256':sha(source),
'pythonVersion':platform.python_version(),'pillowLcmsVersion':ImageCms.core.littlecms_version,
'tolerance8Bit':4,'commands':[],'scales':[],'preservedLegacyInputs':old,
'scope':'Synthetic ICC RI dispatch and state probes. Direct public LittleCMS calls supply colors, not renderer pixels; '
'the renderers may use the same CMM. DeviceRGB luminosity uses the PDF recommended weights, which are device-dependent. '
'No physical-display, press, general ICC, or all-PDF acceptance claim.'}
cmm=None;environment=os.environ.copy();binaries={}
try:
profile,verification=verify(args.corpus,spec);report['fixtureVerification']=verification
cmm=Cmm({'distinct':profile})
report['directCmm']={'encodedVersion':cmm.version,'libraries':cmm.libraryPaths,
'input':'CMYK double percent','output':'8-bit sRGB','flags':0}
report['cmmChecks']=cmm_checks(profile,cmm,spec);report['fixtureValidationSuccess']=True
if not args.self_check:
build=args.build_dir.resolve()
binaries={name:sha(build/name) for name in ('pdf-worker-evidence','docview-pdf-worker')};report['binaries']=binaries
if args.pdfium_library:
environment['LD_LIBRARY_PATH']=str(args.pdfium_library.resolve().parent)+(':'+environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '')
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT')
report['pdfiumLibrary']=loaded_pdfium(build/'docview-pdf-worker',environment,args.pdfium_library)
for program in ('qpdf','pdftoppm'):assert shutil.which(program)
report['popplerVersion']=subprocess.run(['pdftoppm','-v'],capture_output=True,text=True,check=True).stderr.splitlines()[0]
def run(label,command,env=None):
log=output/(label+'.log')
with log.open('w') as f:result=subprocess.run(command,stdout=f,stderr=subprocess.STDOUT,env=env,timeout=300)
report['commands'].append({'name':label,'command':command,'exitCode':result.returncode,'logSha256':sha(log)})
assert result.returncode==0,label+' failed'
run('qpdf-check',['qpdf','--check',str(source)])
for scale in args.scales:
label='scale-'+str(scale).replace('.','_');folder=output/label;(folder/'pdfium').mkdir(parents=True)
run(label+'-pdfium',[str(build/'pdf-worker-evidence'),str(source),str(folder/'pdfium'),str(scale)],environment)
run(label+'-poppler',['pdftoppm','-cropbox','-r',str(72*scale),'-png',str(source),str(folder/'poppler')])
metadata=json.loads((folder/'pdfium/metadata.json').read_text())
assert metadata['pageCount']==spec['pageCount'] and metadata['renderScale']==scale and 'sandbox enabled' in metadata['transport']
row={'scale':scale,'probes':[],'pdfiumSuccess':True,'popplerSuccess':True,'pages':[]}
for page in range(1,5):
paths={'pdfium':folder/'pdfium'/f'page-{page}.png','poppler':folder/f'poppler-{page}.png'}
images={key:Image.open(path).convert('RGB') for key,path in paths.items()}
assert all(list(im.size)==[math.ceil(v*scale) for v in spec['pageSizePt']] for im in images.values())
for probe in (p for p in spec['probes'] if p['page']==page):
x,y=probe['pointPt'];pixel=(round(x*scale),round((850-y)*scale))
expected,details=oracle(probe,cmm)
# Interior 3x3 minimum/maximum detects seams/unstable edge
# samples instead of silently selecting a favorable pixel.
samples={key:[list(im.getpixel((pixel[0]+dx,pixel[1]+dy))) for dy in (-1,0,1) for dx in (-1,0,1)] for key,im in images.items()}
actual={key:values[4] for key,values in samples.items()}
errors={key:max(abs(a-b) for color in values for a,b in zip(color,expected)) for key,values in samples.items()}
matches={key:err<=4 for key,err in errors.items()}
row['probes'].append({**probe,'pointPx':pixel,'oracleDetails':details,'expectedRgb':expected,
'actualRgb':actual,'neighborhoodMinRgb':{k:[min(c[i] for c in v) for i in range(3)] for k,v in samples.items()},
'neighborhoodMaxRgb':{k:[max(c[i] for c in v) for i in range(3)] for k,v in samples.items()},
'maxChannelError':errors,'matches':matches})
for key,match in matches.items():row[key+'Success'] &= match
diff=ImageChops.difference(images['pdfium'],images['poppler']);diff.save(folder/f'difference-{page}.png')
sheet=Image.new('RGB',(1260,440),'#eeeeee');draw=ImageDraw.Draw(sheet)
for col,(key,im) in enumerate([*images.items(),('difference',diff)]):
draw.text((col*420+8,7),f'{scale:g}x page {page}: {key}',fill='black')
thumb=im.copy();thumb.thumbnail((410,400));sheet.paste(thumb,(col*420+5,30))
sheet.save(folder/f'comparison-{page}.png')
row['pages'].append({'page':page,'images':{key:sha(path) for key,path in paths.items()}})
row['failedProbes']={engine:sum(not p['matches'][engine] for p in row['probes']) for engine in ('pdfium','poppler')}
row['success']=row['pdfiumSuccess'] and row['popplerSuccess'];report['scales'].append(row)
report['pdfiumSuccess']=all(r['pdfiumSuccess'] for r in report['scales'])
report['popplerSuccess']=all(r['popplerSuccess'] for r in report['scales'])
report['binariesUnchanged']=all(sha(build/n)==value for n,value in binaries.items()) and sha(Path(report['pdfiumLibrary']['path']))==report['pdfiumLibrary']['sha256']
report['originalUnchanged']=sha(source)==spec['sha256'] and sha(args.corpus/spec['profile']['file'])==PROFILE_SHA
report['legacyInputsUnchanged']=preserved_inputs()==old
if not args.self_check:
report['success']=report['renderingAcceptance']=all(report[k] for k in ('pdfiumSuccess','popplerSuccess','binariesUnchanged','originalUnchanged','legacyInputsUnchanged'))
except Exception as error:
report['error']=type(error).__name__+': '+str(error)
raise
finally:
if cmm:cmm.close()
report['evidenceSha256']={str(p.relative_to(output)):sha(p) for p in sorted(output.rglob('*')) if p.is_file()}
(output/'report.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps({key:report.get(key) for key in ('mode','fixtureValidationSuccess','success','pdfiumSuccess','popplerSuccess','legacyInputsUnchanged')}))
if not args.self_check and not report['success']:raise SystemExit('Pixel criteria failed; every failing probe remains in report.json')
if __name__=='__main__':main()
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Run corrected RI, transparency and original CMYK probes plus DocView CTest."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import shutil
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--library', type=Path, required=True)
parser.add_argument('--build-dir', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
library, build = args.library.resolve(), args.build_dir.resolve()
binaries = [library, ROOT / '.deps/pdfium/lib/libpdfium.so'] + [build / name for name in
('docview', 'docview-pdf-worker', 'docview-archive-worker', 'pdf-worker-evidence')]
identities = {str(path): sha(path) for path in binaries}
environment = os.environ.copy()
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT')
assert not environment.get('QTWEBENGINE_DISABLE_SANDBOX')
environment.pop('WAYLAND_DISPLAY', None)
environment.update(LD_LIBRARY_PATH=str(library.parent) + (
':' + environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else ''),
QT_QPA_PLATFORM='xcb', QT_QUICK_BACKEND='software',
QTWEBENGINE_CHROMIUM_FLAGS='--disable-gpu', QT_SCALE_FACTOR='1',
QT_AUTO_SCREEN_SCALE_FACTOR='0', XDG_SESSION_TYPE='x11')
output = args.output.resolve()
output.mkdir(parents=True, exist_ok=False)
report = {'schemaVersion': 1, 'candidateTestsPass': False, 'acceptanceComplete': False,
'platform': platform.platform(), 'osRelease': platform.freedesktop_os_release(),
'inputs': identities, 'runnerSha256': sha(Path(__file__)), 'commands': [],
'productionDependencyReplaced': False,
'scope': 'Fixed candidate selected only for this test process tree. Generated fixtures and corrected stream-entry RI expectations; no physical display, Windows, human golden or release acceptance.'}
def run(label, command):
start = time.monotonic()
log = output / (label + '.log')
with log.open('w') as stream:
result = subprocess.run(command, cwd=ROOT, env=environment,
stdout=stream, stderr=subprocess.STDOUT, timeout=900)
report['commands'].append({'name': label, 'command': command,
'exitCode': result.returncode, 'seconds': time.monotonic() - start,
'logSha256': sha(log)})
print(label, result.returncode, flush=True)
return result.returncode
try:
linked = subprocess.run(['ldd', str(build / 'docview-pdf-worker')], env=environment,
capture_output=True, text=True, check=True).stdout
assert str(library) in linked and 'not found' not in linked
(output / 'worker-ldd.txt').write_text(linked)
run('cmyk', [sys.executable, str(ROOT / 'tests/cmyk_lut/run.py'),
'--build-dir', str(build), '--output', str(output / 'cmyk')])
run('intents', [sys.executable, str(ROOT / 'tests/cmyk_lut/intents.py'),
'--build-dir', str(build), '--pdfium-library', str(library),
'--corpus', str(ROOT / 'tests/fixtures/pdf/rendering-intents-context'),
'--scales', '.5', '1', '4', '--output', str(output / 'intents')])
run('transparency', [sys.executable, str(ROOT / 'tests/cmyk_lut/transparency.py'),
'--build-dir', str(build), '--pdfium-library', str(library),
'--scales', '.5', '1', '2', '--output', str(output / 'transparency')])
ctest_status = run('ctest', ['xvfb-run', '-a', '-s', '-screen 0 1100x760x24',
'dbus-run-session', '--', 'ctest', '--test-dir', str(build),
'--output-on-failure', '--output-junit', str(output / 'tests.xml')])
shutil.copyfile(build / 'Testing/Temporary/LastTest.log', output / 'LastTest.log')
reports = {name: json.loads((output / name / 'report.json').read_text())
for name in ('cmyk', 'intents', 'transparency')}
report['cmykSuccess'] = reports['cmyk']['success']
report['correctedOracle'] = reports['intents']['oracleRevision'] == 2 and not reports['intents']['oracleKnownIssue']
for name in ('intents', 'transparency'):
data = reports[name]
report[name] = {key: data.get(key, False) for key in
('pdfiumSuccess', 'popplerSuccess', 'fixtureValidationSuccess', 'binariesUnchanged', 'originalUnchanged')}
report[name]['probeCount'] = sum(len(row['probes']) for row in data['scales'])
report['ctestSuccess'] = ctest_status == 0
report['allBinaryInputsUnchanged'] = all(sha(Path(path)) == value for path, value in identities.items())
report['candidateTestsPass'] = all((report['cmykSuccess'], report['correctedOracle'],
report['ctestSuccess'], report['allBinaryInputsUnchanged'], *[
report[name][key] for name in ('intents', 'transparency') for key in
('pdfiumSuccess', 'fixtureValidationSuccess', 'binariesUnchanged', 'originalUnchanged')]))
report['comparisonAgreement'] = all(report[name]['popplerSuccess'] for name in ('intents', 'transparency'))
finally:
report['evidenceSha256'] = {str(path.relative_to(output)): sha(path)
for path in sorted(output.rglob('*')) if path.is_file()}
(output / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({key: report.get(key) for key in
('candidateTestsPass', 'comparisonAgreement', 'allBinaryInputsUnchanged', 'acceptanceComplete')}))
if not report['candidateTestsPass']:
raise SystemExit('Candidate regression failed; all individual outcomes are retained.')
if __name__ == '__main__':
main()
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Run the isolated candidate's upstream tests and both ICC regression corpora."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--build-dir', type=Path, default=ROOT / 'build')
parser.add_argument('--source', type=Path, default=ROOT / 'build-pdfium-intent/source')
parser.add_argument('--variant', choices=('baseline', 'patched'), default='patched')
args = parser.parse_args()
output = args.output.resolve()
output.mkdir(parents=True, exist_ok=False)
source = args.source.resolve()
binaries = source / 'out' / args.variant
library = binaries / 'libpdfium.so'
environment = os.environ.copy()
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT')
environment['LD_LIBRARY_PATH'] = str(binaries) + (
':' + environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '')
environment['LOCPATH'] = str(ROOT / 'build-pdfium-intent/locales')
inputs = [library, binaries / 'pdfium_unittests', binaries / 'pdfium_embeddertests',
args.build_dir / 'pdf-worker-evidence', args.build_dir / 'docview-pdf-worker',
ROOT / '.deps/pdfium/lib/libpdfium.so']
identities = {str(path.resolve()): sha(path) for path in inputs}
report = {'schemaVersion': 1, 'variant': args.variant, 'success': False,
'runnerSha256': sha(Path(__file__)), 'inputs': identities, 'commands': [],
'productionDependencyReplaced': False,
'scope': 'Isolated Linux source-build candidate. No Windows, Ubuntu, calibrated display, human golden, or release acceptance claimed.'}
def run(label, command, cwd=ROOT):
start = time.monotonic()
log = output / (label + '.log')
with log.open('w') as stream:
result = subprocess.run(command, cwd=cwd, env=environment,
stdout=stream, stderr=subprocess.STDOUT, timeout=600)
report['commands'].append({'name': label, 'command': command,
'exitCode': result.returncode,
'seconds': time.monotonic() - start,
'logSha256': sha(log)})
return result.returncode
try:
loader = subprocess.run(['ldd', str(args.build_dir.resolve() / 'docview-pdf-worker')],
env=environment, capture_output=True, text=True, check=True)
assert str(library) in loader.stdout
(output / 'worker-ldd.txt').write_text(loader.stdout)
upstream = []
for target in ('pdfium_unittests', 'pdfium_embeddertests'):
result = output / (target + '.json')
status = run(target, [str(binaries / target), '--gtest_output=json:' + str(result)], source)
data = json.loads(result.read_text()) if result.exists() else {}
upstream.append({'target': target, 'exitCode': status,
**{key: data.get(key) for key in ('tests', 'failures', 'disabled', 'errors', 'time')}})
report['upstreamTests'] = upstream
run('cmyk', [sys.executable, str(ROOT / 'tests/cmyk_lut/run.py'),
'--build-dir', str(args.build_dir.resolve()), '--output', str(output / 'cmyk')])
run('intents', [sys.executable, str(ROOT / 'tests/cmyk_lut/intents.py'),
'--build-dir', str(args.build_dir.resolve()),
'--pdfium-library', str(library), '--scales', '.5', '1', '4',
'--output', str(output / 'intents')])
cmyk = json.loads((output / 'cmyk/report.json').read_text())
intents = json.loads((output / 'intents/report.json').read_text())
report['cmykSuccess'] = cmyk['success']
report['intentPdfiumSuccess'] = intents.get('pdfiumSuccess', False)
report['intentPopplerSuccess'] = intents.get('popplerSuccess', False)
report['intentRenderingAcceptance'] = intents['renderingAcceptance']
report['upstreamTestsPass'] = all(row['exitCode'] == 0 and row['failures'] == 0 and row['errors'] == 0 for row in upstream)
report['allBinaryInputsUnchanged'] = all(sha(Path(path)) == digest for path, digest in identities.items())
report['candidatePdfiumProbesPass'] = cmyk['success'] and report['intentPdfiumSuccess']
report['success'] = (report['upstreamTestsPass'] and report['candidatePdfiumProbesPass']
and report['intentRenderingAcceptance'] and report['allBinaryInputsUnchanged'])
finally:
report['evidenceSha256'] = {str(path.relative_to(output)): sha(path)
for path in sorted(output.rglob('*')) if path.is_file()}
(output / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({key: report.get(key) for key in ('success', 'upstreamTestsPass',
'candidatePdfiumProbesPass', 'intentPopplerSuccess', 'allBinaryInputsUnchanged')}))
if not report['success']:
raise SystemExit('Validation not fully accepted; see separate candidate and comparison results.')
if __name__ == '__main__':
main()