308 lines
21 KiB
Python
308 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Strict ICC rendering-intent probes against direct CMM, PDFium and Poppler.
|
|
|
|
The direct oracle never reads rendered pixels. It uses the embedded profile and
|
|
an independently selected expected intent. It may share LittleCMS algorithms
|
|
with the renderers, and is not claimed to be an independent CMM implementation.
|
|
"""
|
|
import argparse
|
|
import ctypes as C
|
|
import ctypes.util
|
|
from io import BytesIO
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
|
|
from PIL import Image, ImageChops, ImageCms, ImageDraw
|
|
from pypdf import PdfReader
|
|
|
|
ROOT=Path(__file__).resolve().parents[2]
|
|
CORPUS=ROOT/'tests/fixtures/pdf/rendering-intents'
|
|
INTENTS=('Perceptual','RelativeColorimetric','Saturation','AbsoluteColorimetric')
|
|
|
|
|
|
def sha(path):
|
|
with path.open('rb') as source:return hashlib.file_digest(source,'sha256').hexdigest()
|
|
|
|
|
|
class Cmm:
|
|
"""Public LittleCMS API, normalized CMYK doubles -> unsigned 8-bit sRGB."""
|
|
def __init__(self,profiles):
|
|
library=ctypes.util.find_library('lcms2')
|
|
if not library:raise RuntimeError('LittleCMS2 shared library is required')
|
|
self.lib=C.CDLL(library);self.transforms={};self.profiles=[];self.buffers=[]
|
|
declarations={
|
|
'cmsOpenProfileFromMem':([C.c_void_p,C.c_uint32],C.c_void_p),
|
|
'cmsCreate_sRGBProfile':([],C.c_void_p),
|
|
'cmsCreateTransform':([C.c_void_p,C.c_uint32,C.c_void_p,C.c_uint32,C.c_uint32,C.c_uint32],C.c_void_p),
|
|
'cmsDoTransform':([C.c_void_p,C.c_void_p,C.c_void_p,C.c_uint32],None),
|
|
'cmsDeleteTransform':([C.c_void_p],None),'cmsCloseProfile':([C.c_void_p],C.c_int),
|
|
'cmsGetEncodedCMMversion':([],C.c_uint32)}
|
|
for name,(args,result) in declarations.items():
|
|
function=getattr(self.lib,name);function.argtypes=args;function.restype=result
|
|
output=self.lib.cmsCreate_sRGBProfile();assert output;self.profiles.append(output)
|
|
for identifier,data in profiles.items():
|
|
buffer=C.create_string_buffer(data)
|
|
self.buffers.append(buffer)
|
|
profile=self.lib.cmsOpenProfileFromMem(buffer,len(data));assert profile;self.profiles.append(profile)
|
|
for intent in range(4):
|
|
# lcms2.h public pixel-format macros: FLOAT_SH(1), PT_CMYK=6,
|
|
# PT_RGB=4, CHANNELS_SH(n), BYTES_SH(n). Double CMYK uses %.
|
|
transform=self.lib.cmsCreateTransform(profile,(1<<22)|(6<<16)|(4<<3),
|
|
output,(4<<16)|(3<<3)|1,intent,0)
|
|
assert transform;self.transforms[identifier,intent]=transform
|
|
self.version=self.lib.cmsGetEncodedCMMversion()
|
|
self.libraryPaths=[]
|
|
maps=Path('/proc/self/maps')
|
|
if maps.exists():
|
|
paths={line.split()[-1] for line in maps.read_text().splitlines()
|
|
if '/' in line and 'liblcms2.so' in line}
|
|
self.libraryPaths=[{'file':Path(p).name,'sha256':sha(Path(p))} for p in sorted(paths) if Path(p).is_file()]
|
|
|
|
def color(self,profile,intent,values):
|
|
assert len(values)==4 and all(math.isfinite(x) and 0<=x<=1 for x in values)
|
|
source=(C.c_double*4)(*(x*100 for x in values));target=(C.c_ubyte*3)()
|
|
self.lib.cmsDoTransform(self.transforms[profile,intent],source,target,1)
|
|
return list(target)
|
|
|
|
def close(self):
|
|
for transform in self.transforms.values():self.lib.cmsDeleteTransform(transform)
|
|
for profile in self.profiles:self.lib.cmsCloseProfile(profile)
|
|
self.transforms.clear();self.profiles.clear();self.buffers.clear()
|
|
|
|
|
|
def tags_from(profile):
|
|
assert len(profile)==struct.unpack_from('>I',profile)[0]
|
|
assert profile[8:24]==bytes.fromhex('02100000')+b'scnrCMYKLab ' and profile[36:40]==b'acsp'
|
|
tags={}
|
|
for index in range(struct.unpack_from('>I',profile,128)[0]):
|
|
signature,offset,size=struct.unpack_from('>4sII',profile,132+12*index)
|
|
assert offset%4==0 and offset>=132 and offset+size<=len(profile) and signature not in tags
|
|
tags[signature]=profile[offset:offset+size]
|
|
return tags
|
|
|
|
|
|
def verify_fixture(corpus,spec):
|
|
source=corpus/spec['file'];assert sha(source)==spec['sha256']
|
|
assert sha(ROOT/'tests/fixtures/pdf/generate_rendering_intents.py')==spec['generatorSha256']
|
|
corrected=spec.get('oracleRevision')==2
|
|
if corrected:
|
|
correction=spec['oracleCorrection']
|
|
assert correction['correctorSha256']==sha(ROOT/'tests/cmyk_lut/correct_intent_oracle.py')
|
|
assert correction['sourceManifestSha256']=='b0db4ee3e57658b09d0ae10fcf8175986c3ab80ccd64442e3718802257ecf543'
|
|
original=ROOT/'tests/fixtures/pdf/cmyk-lut/synthetic-cmyk-lab.icc'
|
|
assert sha(original)==spec['baseProfileSha256']
|
|
profiles={identifier:(corpus/item['file']).read_bytes() for identifier,item in spec['profiles'].items()}
|
|
assert profiles['original']==original.read_bytes()
|
|
vertex_count=0
|
|
# Independent coefficient representation of the written linear CLUTs.
|
|
coefficients={b'A2B0':[(100,(-12,-15,-10,-35)),(0,(-15,18,0,0)),(0,(-12,0,20,0))],
|
|
b'A2B1':[(100,(-20,-20,-20,-40)),(0,(-20,40,0,-20)),(0,(-20,0,40,-20))],
|
|
b'A2B2':[(100,(-25,-25,-20,-30)),(0,(20,-40,0,20)),(0,(0,-20,35,-15))]}
|
|
for identifier,data in profiles.items():
|
|
item=spec['profiles'][identifier]
|
|
assert hashlib.sha256(data).hexdigest()==item['sha256'] and len(data)==item['size']
|
|
tags=tags_from(data)
|
|
assert set(tags)=={b'desc',b'cprt',b'wtpt',*(t.encode() for t in item['tables'])}
|
|
white=struct.unpack_from('>3i',tags[b'wtpt'],8)
|
|
assert all(abs(actual/65536-expected)<=1/65536 for actual,expected in zip(white,item['mediaWhitePoint']))
|
|
for tag in item['tables']:
|
|
lut=tags[tag.encode()]
|
|
assert len(lut)==176 and lut[:12]==b'mft2'+bytes(4)+bytes((4,3,2,0))
|
|
assert struct.unpack_from('>HH',lut,48)==(2,2)
|
|
assert lut[52:68]==struct.pack('>HH',0,65535)*4 and lut[164:176]==struct.pack('>HH',0,65535)*3
|
|
for i,values in enumerate(itertools.product((0,1),repeat=4)):
|
|
encoded=struct.unpack_from('>3H',lut,68+i*6)
|
|
actual=(encoded[0]*100/65280,encoded[1]/256-128,encoded[2]/256-128)
|
|
expected=[offset+sum(a*b for a,b in zip(vector,values)) for offset,vector in coefficients[tag.encode()]]
|
|
assert max(abs(a-b) for a,b in zip(actual,expected))<.002
|
|
vertex_count+=1
|
|
pdf=PdfReader(source);assert len(pdf.pages)==spec['pageCount']
|
|
identities={}
|
|
for index,page in enumerate(pdf.pages,1):
|
|
assert list(page.mediabox)==[0,0,*spec['pageSizePt']]
|
|
probes=[p for p in spec['probes'] if p['page']==index];assert probes
|
|
identifier=probes[0]['profile'];resources=page['/Resources'];space=resources['/ColorSpace']['/CS']
|
|
# pypdf 4 leaves array members indirect; dictionary lookup alone dereferences.
|
|
embedded_profile=space[1].get_object()
|
|
assert space[0]=='/ICCBased' and embedded_profile['/N']==4 and embedded_profile.get_data()==profiles[identifier]
|
|
assert resources['/ColorSpace']['/IX'][0]=='/Indexed'
|
|
lookup=resources['/ColorSpace']['/IX'][3]
|
|
assert (lookup.original_bytes if isinstance(lookup,str) else bytes(lookup))==bytes(spec['sampleBytes'])
|
|
xobjects=resources['/XObject'];shared=xobjects.raw_get('/Shared')
|
|
identities.setdefault(identifier,shared.idnum);assert identities[identifier]==shared.idnum
|
|
assert '/Intent' not in xobjects['/Shared'] and xobjects['/Shared'].get_data()==bytes(spec['sampleBytes'])
|
|
assert xobjects['/Indexed'].get_data()==b'\0'
|
|
for font_name in ('T3Rect','T3Image',*(f'T3Override{i}' for i in range(4))):
|
|
glyph_font=resources['/Font']['/'+font_name]
|
|
assert glyph_font['/Subtype']=='/Type3' and glyph_font['/FirstChar']==glyph_font['/LastChar']==65
|
|
glyph=glyph_font['/CharProcs']['/A'].get_data();assert glyph.startswith(b'1000 0 d0\n')
|
|
if font_name=='T3Image':assert b'/Shared Do' in glyph
|
|
elif font_name.startswith('T3Override'):assert ('/'+INTENTS[int(font_name[-1])]+' ri').encode() in glyph
|
|
else:assert b' ri' not in glyph
|
|
assert '/RelativeColorimetric ri' not in xobjects['/Inherited'].get_data().decode()
|
|
for i,intent in enumerate(INTENTS):
|
|
assert resources['/ExtGState']['/I'+str(i)]['/RI']=='/'+intent
|
|
assert xobjects['/Image'+str(i)]['/Intent']=='/'+intent
|
|
assert ('/'+intent+' ri').encode() in xobjects['/Form'+str(i)].get_data()
|
|
assert ('/I'+str(i)+' gs').encode() in xobjects['/FormGs'+str(i)].get_data()
|
|
assert resources['/Pattern']['/P'+str(i)]['/ExtGState']['/RI']=='/'+intent
|
|
content=page.get_contents().get_data()
|
|
for probe in probes:
|
|
assert ('\n'.join(probe['commandSequence'])+'\n').encode() in content
|
|
expected=probe['expectedIntentIndex'];assert INTENTS[expected]==probe['expectedIntent']
|
|
current=1 if probe['graphicsStateIntent']=='default' else INTENTS.index(probe['graphicsStateIntent'])
|
|
if corrected and probe['case']=='shading-pattern-inherit':
|
|
assert expected==1
|
|
elif probe['case'] in ('nested-q-inner','form-ri-override','form-gs-override','image-intent-override','shading-pattern-RI-override','type3-ri-override'):
|
|
assert expected==(current+1)%4
|
|
else:assert expected==current
|
|
assert len(spec['probes'])==len(spec['cases'])*4*2
|
|
return profiles,{'pagesVerified':len(pdf.pages),'probesVerified':len(spec['probes']),
|
|
'clutVerticesVerified':vertex_count,'sharedImageObjectIds':identities,
|
|
'originalProfileByteIdentical':True,'oracleRevision':2 if corrected else 1,
|
|
'knownIncorrectPatternOracle':not corrected}
|
|
|
|
|
|
def cmm_checks(profiles,spec,cmm):
|
|
sample=tuple(spec['sampleBytes']);checks=[];colors={};bpc_checks=[]
|
|
for identifier,data in profiles.items():
|
|
profile=ImageCms.ImageCmsProfile(BytesIO(data));colors[identifier]=[]
|
|
for intent in range(4):
|
|
expected=cmm.color(identifier,intent,[x/255 for x in sample]);colors[identifier].append(expected)
|
|
transform=ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=intent)
|
|
actual=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),transform).getpixel((0,0)))
|
|
error=max(abs(a-b) for a,b in zip(actual,expected))
|
|
checks.append({'profile':identifier,'intent':INTENTS[intent],'directCmmRgb':expected,
|
|
'pillow8BitRgb':actual,'maxChannelError':error,'success':error<=1})
|
|
if identifier=='distinct':
|
|
# Pillow 10.2 (Ubuntu 24.04) exposes the same public flag in
|
|
# FLAGS; later releases expose the Flags enum instead.
|
|
bpc_flag=(ImageCms.Flags.BLACKPOINTCOMPENSATION if hasattr(ImageCms,'Flags')
|
|
else ImageCms.FLAGS['BLACKPOINTCOMPENSATION'])
|
|
assert int(bpc_flag)==8192
|
|
with_bpc=ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),'CMYK','RGB',
|
|
renderingIntent=intent,flags=bpc_flag)
|
|
bpc=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),with_bpc).getpixel((0,0)))
|
|
assert max(abs(a-b) for a,b in zip(bpc,expected))<=1
|
|
bpc_checks.append({'intent':INTENTS[intent],'rgb':bpc,'flags':8192,'matchesWithoutBpc':True})
|
|
assert all(c['success'] for c in checks)
|
|
separation=[max(abs(a-b) for a,b in zip(colors['distinct'][i],colors['distinct'][j])) for i,j in itertools.combinations(range(4),2)]
|
|
assert min(separation)>2*spec['tolerance8Bit'],'Derived profile must distinguish every intent beyond tolerance'
|
|
return {'checks':checks,'distinctProfileBlackPointCompensationChecks':bpc_checks,
|
|
'distinctProfileMinimumPairwiseMaxChannelSeparation':min(separation),
|
|
'originalProfileIntentEquivalence':'Relative/Saturation/Absolute may coincide; distinct profile covers their dispatch.'}
|
|
|
|
|
|
def loaded_pdfium(worker,environment,requested):
|
|
text=subprocess.run(['ldd',str(worker)],capture_output=True,text=True,env=environment,check=True,timeout=30).stdout
|
|
match=re.search(r'^\s*libpdfium\.so\s+=>\s+(/.+?)\s+\(0x',text,re.M)
|
|
assert match,'ldd did not resolve worker libpdfium.so'
|
|
path=Path(match[1]).resolve(strict=True)
|
|
if requested:assert path==requested.resolve(strict=True),'Requested library lost to loader/RPATH precedence'
|
|
return {'path':str(path),'sha256':sha(path),'ldd':text,'selectionVerified':True}
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--build-dir',type=Path)
|
|
parser.add_argument('--pdfium-library',type=Path,help='Prepend this library directory to LD_LIBRARY_PATH and verify ldd resolves it')
|
|
parser.add_argument('--corpus',type=Path,default=CORPUS)
|
|
parser.add_argument('--output',type=Path,required=True)
|
|
parser.add_argument('--self-check',action='store_true',help='Validate fixture/CMM only; not rendering acceptance')
|
|
parser.add_argument('--scales',type=float,nargs='+',default=[.5,1.,2.])
|
|
args=parser.parse_args()
|
|
if not args.self_check and not args.build_dir:parser.error('--build-dir is required unless --self-check')
|
|
if not 1<=len(args.scales)<=4 or any(not math.isfinite(s) or s<.25 or s>4 for s in args.scales):parser.error('scales must be 1..4 finite values in .25..4')
|
|
if len(set(args.scales))!=len(args.scales):parser.error('scales must be unique')
|
|
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
|
|
spec=json.loads((args.corpus/'manifest.json').read_text());source=(args.corpus/spec['file']).resolve()
|
|
report={'schemaVersion':1,'success':False,'fixtureValidationSuccess':False,'renderingAcceptance':False,
|
|
'mode':'fixture-self-check' if args.self_check else 'renderer-comparison','commands':[],'scales':[],
|
|
'runnerSha256':sha(Path(__file__)),'fixtureManifestSha256':sha(args.corpus/'manifest.json'),
|
|
'fixtureSha256':sha(source),'pythonVersion':platform.python_version(),'pillowLcmsVersion':ImageCms.core.littlecms_version,
|
|
'osRelease':platform.freedesktop_os_release(),'tolerance8Bit':spec['tolerance8Bit'],
|
|
'oracleRevision':spec.get('oracleRevision',1),'oracleKnownIssue':spec.get('oracleRevision')!=2,
|
|
'scope':'Direct explicit-intent CMM expected colors; production isolated PDF worker and independent Poppler. '
|
|
'CMM may share LittleCMS code with PDF renderers. Synthetic profiles only; no physical display/press, '
|
|
'human-approved golden or general ICC conformance assertion.'}
|
|
cmm=None;binaries={};environment=os.environ.copy()
|
|
try:
|
|
profiles,verified=verify_fixture(args.corpus,spec);report['fixtureVerification']=verified
|
|
cmm=Cmm(profiles);report['directCmm']={'encodedVersion':cmm.version,'libraries':cmm.libraryPaths,
|
|
'inputFormat':'TYPE_CMYK_DBL, 0..100 percent','outputFormat':'TYPE_RGB_8, cmsCreate_sRGBProfile','flags':0}
|
|
report['cmmValidation']=cmm_checks(profiles,spec,cmm);report['fixtureValidationSuccess']=True
|
|
if not args.self_check:
|
|
build=args.build_dir.resolve();binaries={name:sha(build/name) for name in ('pdf-worker-evidence','docview-pdf-worker')}
|
|
report['binaries']=binaries
|
|
if args.pdfium_library:
|
|
environment['LD_LIBRARY_PATH']=str(args.pdfium_library.resolve().parent)+(':'+environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '')
|
|
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT'),'Loader injection would invalidate evidence'
|
|
report['pdfiumLibrary']=loaded_pdfium(build/'docview-pdf-worker',environment,args.pdfium_library)
|
|
for program in ('qpdf','pdftoppm'):assert shutil.which(program),program+' required'
|
|
report['popplerVersion']=subprocess.run(['pdftoppm','-v'],capture_output=True,text=True,check=True).stderr.splitlines()[0]
|
|
def run(label,command,env=None):
|
|
log=output/(label+'.log')
|
|
with log.open('w') as f:result=subprocess.run(command,stdout=f,stderr=subprocess.STDOUT,env=env,timeout=300)
|
|
report['commands'].append({'name':label,'command':command,'exitCode':result.returncode,'logSha256':sha(log)})
|
|
assert result.returncode==0,label+' failed'
|
|
run('qpdf-check',['qpdf','--check',str(source)])
|
|
for scale in args.scales:
|
|
label='scale-'+str(scale).replace('.','_');folder=output/label;(folder/'pdfium').mkdir(parents=True)
|
|
run(label+'-pdfium',[str(build/'pdf-worker-evidence'),str(source),str(folder/'pdfium'),str(scale)],environment)
|
|
run(label+'-poppler',['pdftoppm','-cropbox','-r',str(72*scale),'-png',str(source),str(folder/'poppler')])
|
|
metadata=json.loads((folder/'pdfium/metadata.json').read_text())
|
|
assert metadata['pageCount']==spec['pageCount'] and metadata['renderScale']==scale and 'sandbox enabled' in metadata['transport']
|
|
row={'scale':scale,'probes':[],'pdfiumSuccess':True,'popplerSuccess':True,'pages':[]}
|
|
for page in range(1,spec['pageCount']+1):
|
|
paths={'pdfium':folder/'pdfium'/f'page-{page}.png','poppler':folder/f'poppler-{page}.png'}
|
|
images={key:Image.open(path).convert('RGB') for key,path in paths.items()}
|
|
dimensions=[math.ceil(v*scale) for v in spec['pageSizePt']]
|
|
assert all(list(image.size)==dimensions for image in images.values())
|
|
for probe in (p for p in spec['probes'] if p['page']==page):
|
|
x,y=probe['pointPt'];pixel=(round(x*scale),round((spec['pageSizePt'][1]-y)*scale))
|
|
components=probe['components']
|
|
if 'shading' in probe:
|
|
shading=probe['shading'];t=((pixel[0]+.5)/scale-shading['xStart'])/(shading['xEnd']-shading['xStart'])
|
|
components=[a+(b-a)*t for a,b in zip(shading['c0'],shading['c1'])]
|
|
expected=cmm.color(probe['profile'],probe['expectedIntentIndex'],components)
|
|
actual={key:list(image.getpixel(pixel)) for key,image in images.items()}
|
|
errors={key:max(abs(a-b) for a,b in zip(color,expected)) for key,color in actual.items()}
|
|
matches={key:error<=spec['tolerance8Bit'] for key,error in errors.items()}
|
|
row['probes'].append({**probe,'pointPx':pixel,'sampledComponents':components,'expectedRgb':expected,
|
|
'actualRgb':actual,'maxChannelError':errors,'matches':matches,'success':all(matches.values())})
|
|
for key,matched in matches.items():row[key+'Success'] &= matched
|
|
diff=ImageChops.difference(images['pdfium'],images['poppler']);diff.save(folder/f'difference-{page}.png')
|
|
sheet=Image.new('RGB',(1260,420),'#eeeeee');draw=ImageDraw.Draw(sheet)
|
|
for column,(label2,image) in enumerate([*images.items(),('difference',diff)]):
|
|
draw.text((column*420+8,7),f'{scale:g}x page {page}: {label2}',fill='black')
|
|
thumb=image.copy();thumb.thumbnail((410,380));sheet.paste(thumb,(column*420+5,30))
|
|
sheet.save(folder/f'comparison-{page}.png')
|
|
row['pages'].append({'page':page,'images':{key:sha(path) for key,path in paths.items()}})
|
|
row['success']=row['pdfiumSuccess'] and row['popplerSuccess'];report['scales'].append(row)
|
|
report['originalUnchanged']=sha(source)==spec['sha256'] and all(sha(args.corpus/item['file'])==item['sha256'] for item in spec['profiles'].values())
|
|
report['binariesUnchanged']=all(sha(build/name)==value for name,value in binaries.items()) and sha(Path(report['pdfiumLibrary']['path']))==report['pdfiumLibrary']['sha256']
|
|
report['pdfiumSuccess']=all(row['pdfiumSuccess'] for row in report['scales'])
|
|
report['popplerSuccess']=all(row['popplerSuccess'] for row in report['scales'])
|
|
report['renderingAcceptance']=report['success']=report['pdfiumSuccess'] and report['popplerSuccess'] and report['originalUnchanged'] and report['binariesUnchanged'] and not report['oracleKnownIssue']
|
|
except Exception as error:
|
|
report['error']=type(error).__name__+': '+str(error)
|
|
raise
|
|
finally:
|
|
if cmm:cmm.close()
|
|
report['evidenceSha256']={str(p.relative_to(output)):sha(p) for p in sorted(output.rglob('*')) if p.is_file()}
|
|
(output/'report.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
print(json.dumps({key:report.get(key) for key in ('mode','fixtureValidationSuccess','success','pdfiumSuccess','popplerSuccess')}))
|
|
if not args.self_check and not report['success']:raise SystemExit('Intent validation failed; retained report preserves every failed probe')
|
|
|
|
|
|
if __name__=='__main__':main()
|