#!/usr/bin/env python3 """Strict ICC rendering-intent probes against direct CMM, PDFium and Poppler. The direct oracle never reads rendered pixels. It uses the embedded profile and an independently selected expected intent. It may share LittleCMS algorithms with the renderers, and is not claimed to be an independent CMM implementation. """ import argparse import ctypes as C import ctypes.util from io import BytesIO import hashlib import itertools import json import math import os from pathlib import Path import platform import re import shutil import struct import subprocess from PIL import Image, ImageChops, ImageCms, ImageDraw from pypdf import PdfReader ROOT=Path(__file__).resolve().parents[2] CORPUS=ROOT/'tests/fixtures/pdf/rendering-intents' INTENTS=('Perceptual','RelativeColorimetric','Saturation','AbsoluteColorimetric') def sha(path): with path.open('rb') as source:return hashlib.file_digest(source,'sha256').hexdigest() class Cmm: """Public LittleCMS API, normalized CMYK doubles -> unsigned 8-bit sRGB.""" def __init__(self,profiles): library=ctypes.util.find_library('lcms2') if not library:raise RuntimeError('LittleCMS2 shared library is required') self.lib=C.CDLL(library);self.transforms={};self.profiles=[];self.buffers=[] declarations={ 'cmsOpenProfileFromMem':([C.c_void_p,C.c_uint32],C.c_void_p), 'cmsCreate_sRGBProfile':([],C.c_void_p), 'cmsCreateTransform':([C.c_void_p,C.c_uint32,C.c_void_p,C.c_uint32,C.c_uint32,C.c_uint32],C.c_void_p), 'cmsDoTransform':([C.c_void_p,C.c_void_p,C.c_void_p,C.c_uint32],None), 'cmsDeleteTransform':([C.c_void_p],None),'cmsCloseProfile':([C.c_void_p],C.c_int), 'cmsGetEncodedCMMversion':([],C.c_uint32)} for name,(args,result) in declarations.items(): function=getattr(self.lib,name);function.argtypes=args;function.restype=result output=self.lib.cmsCreate_sRGBProfile();assert output;self.profiles.append(output) for identifier,data in profiles.items(): buffer=C.create_string_buffer(data) self.buffers.append(buffer) profile=self.lib.cmsOpenProfileFromMem(buffer,len(data));assert profile;self.profiles.append(profile) for intent in range(4): # lcms2.h public pixel-format macros: FLOAT_SH(1), PT_CMYK=6, # PT_RGB=4, CHANNELS_SH(n), BYTES_SH(n). Double CMYK uses %. transform=self.lib.cmsCreateTransform(profile,(1<<22)|(6<<16)|(4<<3), output,(4<<16)|(3<<3)|1,intent,0) assert transform;self.transforms[identifier,intent]=transform self.version=self.lib.cmsGetEncodedCMMversion() self.libraryPaths=[] maps=Path('/proc/self/maps') if maps.exists(): paths={line.split()[-1] for line in maps.read_text().splitlines() if '/' in line and 'liblcms2.so' in line} self.libraryPaths=[{'file':Path(p).name,'sha256':sha(Path(p))} for p in sorted(paths) if Path(p).is_file()] def color(self,profile,intent,values): assert len(values)==4 and all(math.isfinite(x) and 0<=x<=1 for x in values) source=(C.c_double*4)(*(x*100 for x in values));target=(C.c_ubyte*3)() self.lib.cmsDoTransform(self.transforms[profile,intent],source,target,1) return list(target) def close(self): for transform in self.transforms.values():self.lib.cmsDeleteTransform(transform) for profile in self.profiles:self.lib.cmsCloseProfile(profile) self.transforms.clear();self.profiles.clear();self.buffers.clear() def tags_from(profile): assert len(profile)==struct.unpack_from('>I',profile)[0] assert profile[8:24]==bytes.fromhex('02100000')+b'scnrCMYKLab ' and profile[36:40]==b'acsp' tags={} for index in range(struct.unpack_from('>I',profile,128)[0]): signature,offset,size=struct.unpack_from('>4sII',profile,132+12*index) assert offset%4==0 and offset>=132 and offset+size<=len(profile) and signature not in tags tags[signature]=profile[offset:offset+size] return tags def verify_fixture(corpus,spec): source=corpus/spec['file'];assert sha(source)==spec['sha256'] assert sha(ROOT/'tests/fixtures/pdf/generate_rendering_intents.py')==spec['generatorSha256'] corrected=spec.get('oracleRevision')==2 if corrected: correction=spec['oracleCorrection'] assert correction['correctorSha256']==sha(ROOT/'tests/cmyk_lut/correct_intent_oracle.py') assert correction['sourceManifestSha256']=='b0db4ee3e57658b09d0ae10fcf8175986c3ab80ccd64442e3718802257ecf543' original=ROOT/'tests/fixtures/pdf/cmyk-lut/synthetic-cmyk-lab.icc' assert sha(original)==spec['baseProfileSha256'] profiles={identifier:(corpus/item['file']).read_bytes() for identifier,item in spec['profiles'].items()} assert profiles['original']==original.read_bytes() vertex_count=0 # Independent coefficient representation of the written linear CLUTs. coefficients={b'A2B0':[(100,(-12,-15,-10,-35)),(0,(-15,18,0,0)),(0,(-12,0,20,0))], b'A2B1':[(100,(-20,-20,-20,-40)),(0,(-20,40,0,-20)),(0,(-20,0,40,-20))], b'A2B2':[(100,(-25,-25,-20,-30)),(0,(20,-40,0,20)),(0,(0,-20,35,-15))]} for identifier,data in profiles.items(): item=spec['profiles'][identifier] assert hashlib.sha256(data).hexdigest()==item['sha256'] and len(data)==item['size'] tags=tags_from(data) assert set(tags)=={b'desc',b'cprt',b'wtpt',*(t.encode() for t in item['tables'])} white=struct.unpack_from('>3i',tags[b'wtpt'],8) assert all(abs(actual/65536-expected)<=1/65536 for actual,expected in zip(white,item['mediaWhitePoint'])) for tag in item['tables']: lut=tags[tag.encode()] assert len(lut)==176 and lut[:12]==b'mft2'+bytes(4)+bytes((4,3,2,0)) assert struct.unpack_from('>HH',lut,48)==(2,2) assert lut[52:68]==struct.pack('>HH',0,65535)*4 and lut[164:176]==struct.pack('>HH',0,65535)*3 for i,values in enumerate(itertools.product((0,1),repeat=4)): encoded=struct.unpack_from('>3H',lut,68+i*6) actual=(encoded[0]*100/65280,encoded[1]/256-128,encoded[2]/256-128) expected=[offset+sum(a*b for a,b in zip(vector,values)) for offset,vector in coefficients[tag.encode()]] assert max(abs(a-b) for a,b in zip(actual,expected))<.002 vertex_count+=1 pdf=PdfReader(source);assert len(pdf.pages)==spec['pageCount'] identities={} for index,page in enumerate(pdf.pages,1): assert list(page.mediabox)==[0,0,*spec['pageSizePt']] probes=[p for p in spec['probes'] if p['page']==index];assert probes identifier=probes[0]['profile'];resources=page['/Resources'];space=resources['/ColorSpace']['/CS'] # pypdf 4 leaves array members indirect; dictionary lookup alone dereferences. embedded_profile=space[1].get_object() assert space[0]=='/ICCBased' and embedded_profile['/N']==4 and embedded_profile.get_data()==profiles[identifier] assert resources['/ColorSpace']['/IX'][0]=='/Indexed' lookup=resources['/ColorSpace']['/IX'][3] assert (lookup.original_bytes if isinstance(lookup,str) else bytes(lookup))==bytes(spec['sampleBytes']) xobjects=resources['/XObject'];shared=xobjects.raw_get('/Shared') identities.setdefault(identifier,shared.idnum);assert identities[identifier]==shared.idnum assert '/Intent' not in xobjects['/Shared'] and xobjects['/Shared'].get_data()==bytes(spec['sampleBytes']) assert xobjects['/Indexed'].get_data()==b'\0' for font_name in ('T3Rect','T3Image',*(f'T3Override{i}' for i in range(4))): glyph_font=resources['/Font']['/'+font_name] assert glyph_font['/Subtype']=='/Type3' and glyph_font['/FirstChar']==glyph_font['/LastChar']==65 glyph=glyph_font['/CharProcs']['/A'].get_data();assert glyph.startswith(b'1000 0 d0\n') if font_name=='T3Image':assert b'/Shared Do' in glyph elif font_name.startswith('T3Override'):assert ('/'+INTENTS[int(font_name[-1])]+' ri').encode() in glyph else:assert b' ri' not in glyph assert '/RelativeColorimetric ri' not in xobjects['/Inherited'].get_data().decode() for i,intent in enumerate(INTENTS): assert resources['/ExtGState']['/I'+str(i)]['/RI']=='/'+intent assert xobjects['/Image'+str(i)]['/Intent']=='/'+intent assert ('/'+intent+' ri').encode() in xobjects['/Form'+str(i)].get_data() assert ('/I'+str(i)+' gs').encode() in xobjects['/FormGs'+str(i)].get_data() assert resources['/Pattern']['/P'+str(i)]['/ExtGState']['/RI']=='/'+intent content=page.get_contents().get_data() for probe in probes: assert ('\n'.join(probe['commandSequence'])+'\n').encode() in content expected=probe['expectedIntentIndex'];assert INTENTS[expected]==probe['expectedIntent'] current=1 if probe['graphicsStateIntent']=='default' else INTENTS.index(probe['graphicsStateIntent']) if corrected and probe['case']=='shading-pattern-inherit': assert expected==1 elif probe['case'] in ('nested-q-inner','form-ri-override','form-gs-override','image-intent-override','shading-pattern-RI-override','type3-ri-override'): assert expected==(current+1)%4 else:assert expected==current assert len(spec['probes'])==len(spec['cases'])*4*2 return profiles,{'pagesVerified':len(pdf.pages),'probesVerified':len(spec['probes']), 'clutVerticesVerified':vertex_count,'sharedImageObjectIds':identities, 'originalProfileByteIdentical':True,'oracleRevision':2 if corrected else 1, 'knownIncorrectPatternOracle':not corrected} def cmm_checks(profiles,spec,cmm): sample=tuple(spec['sampleBytes']);checks=[];colors={};bpc_checks=[] for identifier,data in profiles.items(): profile=ImageCms.ImageCmsProfile(BytesIO(data));colors[identifier]=[] for intent in range(4): expected=cmm.color(identifier,intent,[x/255 for x in sample]);colors[identifier].append(expected) transform=ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=intent) actual=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),transform).getpixel((0,0))) error=max(abs(a-b) for a,b in zip(actual,expected)) checks.append({'profile':identifier,'intent':INTENTS[intent],'directCmmRgb':expected, 'pillow8BitRgb':actual,'maxChannelError':error,'success':error<=1}) if identifier=='distinct': # Pillow 10.2 (Ubuntu 24.04) exposes the same public flag in # FLAGS; later releases expose the Flags enum instead. bpc_flag=(ImageCms.Flags.BLACKPOINTCOMPENSATION if hasattr(ImageCms,'Flags') else ImageCms.FLAGS['BLACKPOINTCOMPENSATION']) assert int(bpc_flag)==8192 with_bpc=ImageCms.buildTransformFromOpenProfiles(profile,ImageCms.createProfile('sRGB'),'CMYK','RGB', renderingIntent=intent,flags=bpc_flag) bpc=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),with_bpc).getpixel((0,0))) assert max(abs(a-b) for a,b in zip(bpc,expected))<=1 bpc_checks.append({'intent':INTENTS[intent],'rgb':bpc,'flags':8192,'matchesWithoutBpc':True}) assert all(c['success'] for c in checks) separation=[max(abs(a-b) for a,b in zip(colors['distinct'][i],colors['distinct'][j])) for i,j in itertools.combinations(range(4),2)] assert min(separation)>2*spec['tolerance8Bit'],'Derived profile must distinguish every intent beyond tolerance' return {'checks':checks,'distinctProfileBlackPointCompensationChecks':bpc_checks, 'distinctProfileMinimumPairwiseMaxChannelSeparation':min(separation), 'originalProfileIntentEquivalence':'Relative/Saturation/Absolute may coincide; distinct profile covers their dispatch.'} def loaded_pdfium(worker,environment,requested): text=subprocess.run(['ldd',str(worker)],capture_output=True,text=True,env=environment,check=True,timeout=30).stdout match=re.search(r'^\s*libpdfium\.so\s+=>\s+(/.+?)\s+\(0x',text,re.M) assert match,'ldd did not resolve worker libpdfium.so' path=Path(match[1]).resolve(strict=True) if requested:assert path==requested.resolve(strict=True),'Requested library lost to loader/RPATH precedence' return {'path':str(path),'sha256':sha(path),'ldd':text,'selectionVerified':True} def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--build-dir',type=Path) parser.add_argument('--pdfium-library',type=Path,help='Prepend this library directory to LD_LIBRARY_PATH and verify ldd resolves it') parser.add_argument('--corpus',type=Path,default=CORPUS) parser.add_argument('--output',type=Path,required=True) parser.add_argument('--self-check',action='store_true',help='Validate fixture/CMM only; not rendering acceptance') parser.add_argument('--scales',type=float,nargs='+',default=[.5,1.,2.]) args=parser.parse_args() if not args.self_check and not args.build_dir:parser.error('--build-dir is required unless --self-check') if not 1<=len(args.scales)<=4 or any(not math.isfinite(s) or s<.25 or s>4 for s in args.scales):parser.error('scales must be 1..4 finite values in .25..4') if len(set(args.scales))!=len(args.scales):parser.error('scales must be unique') output=args.output.resolve();output.mkdir(parents=True,exist_ok=False) spec=json.loads((args.corpus/'manifest.json').read_text());source=(args.corpus/spec['file']).resolve() report={'schemaVersion':1,'success':False,'fixtureValidationSuccess':False,'renderingAcceptance':False, 'mode':'fixture-self-check' if args.self_check else 'renderer-comparison','commands':[],'scales':[], 'runnerSha256':sha(Path(__file__)),'fixtureManifestSha256':sha(args.corpus/'manifest.json'), 'fixtureSha256':sha(source),'pythonVersion':platform.python_version(),'pillowLcmsVersion':ImageCms.core.littlecms_version, 'osRelease':platform.freedesktop_os_release(),'tolerance8Bit':spec['tolerance8Bit'], 'oracleRevision':spec.get('oracleRevision',1),'oracleKnownIssue':spec.get('oracleRevision')!=2, 'scope':'Direct explicit-intent CMM expected colors; production isolated PDF worker and independent Poppler. ' 'CMM may share LittleCMS code with PDF renderers. Synthetic profiles only; no physical display/press, ' 'human-approved golden or general ICC conformance assertion.'} cmm=None;binaries={};environment=os.environ.copy() try: profiles,verified=verify_fixture(args.corpus,spec);report['fixtureVerification']=verified cmm=Cmm(profiles);report['directCmm']={'encodedVersion':cmm.version,'libraries':cmm.libraryPaths, 'inputFormat':'TYPE_CMYK_DBL, 0..100 percent','outputFormat':'TYPE_RGB_8, cmsCreate_sRGBProfile','flags':0} report['cmmValidation']=cmm_checks(profiles,spec,cmm);report['fixtureValidationSuccess']=True if not args.self_check: build=args.build_dir.resolve();binaries={name:sha(build/name) for name in ('pdf-worker-evidence','docview-pdf-worker')} report['binaries']=binaries if args.pdfium_library: environment['LD_LIBRARY_PATH']=str(args.pdfium_library.resolve().parent)+(':'+environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '') assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT'),'Loader injection would invalidate evidence' report['pdfiumLibrary']=loaded_pdfium(build/'docview-pdf-worker',environment,args.pdfium_library) for program in ('qpdf','pdftoppm'):assert shutil.which(program),program+' required' report['popplerVersion']=subprocess.run(['pdftoppm','-v'],capture_output=True,text=True,check=True).stderr.splitlines()[0] def run(label,command,env=None): log=output/(label+'.log') with log.open('w') as f:result=subprocess.run(command,stdout=f,stderr=subprocess.STDOUT,env=env,timeout=300) report['commands'].append({'name':label,'command':command,'exitCode':result.returncode,'logSha256':sha(log)}) assert result.returncode==0,label+' failed' run('qpdf-check',['qpdf','--check',str(source)]) for scale in args.scales: label='scale-'+str(scale).replace('.','_');folder=output/label;(folder/'pdfium').mkdir(parents=True) run(label+'-pdfium',[str(build/'pdf-worker-evidence'),str(source),str(folder/'pdfium'),str(scale)],environment) run(label+'-poppler',['pdftoppm','-cropbox','-r',str(72*scale),'-png',str(source),str(folder/'poppler')]) metadata=json.loads((folder/'pdfium/metadata.json').read_text()) assert metadata['pageCount']==spec['pageCount'] and metadata['renderScale']==scale and 'sandbox enabled' in metadata['transport'] row={'scale':scale,'probes':[],'pdfiumSuccess':True,'popplerSuccess':True,'pages':[]} for page in range(1,spec['pageCount']+1): paths={'pdfium':folder/'pdfium'/f'page-{page}.png','poppler':folder/f'poppler-{page}.png'} images={key:Image.open(path).convert('RGB') for key,path in paths.items()} dimensions=[math.ceil(v*scale) for v in spec['pageSizePt']] assert all(list(image.size)==dimensions for image in images.values()) for probe in (p for p in spec['probes'] if p['page']==page): x,y=probe['pointPt'];pixel=(round(x*scale),round((spec['pageSizePt'][1]-y)*scale)) components=probe['components'] if 'shading' in probe: shading=probe['shading'];t=((pixel[0]+.5)/scale-shading['xStart'])/(shading['xEnd']-shading['xStart']) components=[a+(b-a)*t for a,b in zip(shading['c0'],shading['c1'])] expected=cmm.color(probe['profile'],probe['expectedIntentIndex'],components) actual={key:list(image.getpixel(pixel)) for key,image in images.items()} errors={key:max(abs(a-b) for a,b in zip(color,expected)) for key,color in actual.items()} matches={key:error<=spec['tolerance8Bit'] for key,error in errors.items()} row['probes'].append({**probe,'pointPx':pixel,'sampledComponents':components,'expectedRgb':expected, 'actualRgb':actual,'maxChannelError':errors,'matches':matches,'success':all(matches.values())}) for key,matched in matches.items():row[key+'Success'] &= matched diff=ImageChops.difference(images['pdfium'],images['poppler']);diff.save(folder/f'difference-{page}.png') sheet=Image.new('RGB',(1260,420),'#eeeeee');draw=ImageDraw.Draw(sheet) for column,(label2,image) in enumerate([*images.items(),('difference',diff)]): draw.text((column*420+8,7),f'{scale:g}x page {page}: {label2}',fill='black') thumb=image.copy();thumb.thumbnail((410,380));sheet.paste(thumb,(column*420+5,30)) sheet.save(folder/f'comparison-{page}.png') row['pages'].append({'page':page,'images':{key:sha(path) for key,path in paths.items()}}) row['success']=row['pdfiumSuccess'] and row['popplerSuccess'];report['scales'].append(row) report['originalUnchanged']=sha(source)==spec['sha256'] and all(sha(args.corpus/item['file'])==item['sha256'] for item in spec['profiles'].values()) report['binariesUnchanged']=all(sha(build/name)==value for name,value in binaries.items()) and sha(Path(report['pdfiumLibrary']['path']))==report['pdfiumLibrary']['sha256'] report['pdfiumSuccess']=all(row['pdfiumSuccess'] for row in report['scales']) report['popplerSuccess']=all(row['popplerSuccess'] for row in report['scales']) report['renderingAcceptance']=report['success']=report['pdfiumSuccess'] and report['popplerSuccess'] and report['originalUnchanged'] and report['binariesUnchanged'] and not report['oracleKnownIssue'] except Exception as error: report['error']=type(error).__name__+': '+str(error) raise finally: if cmm:cmm.close() report['evidenceSha256']={str(p.relative_to(output)):sha(p) for p in sorted(output.rglob('*')) if p.is_file()} (output/'report.json').write_text(json.dumps(report,indent=2)+'\n') print(json.dumps({key:report.get(key) for key in ('mode','fixtureValidationSuccess','success','pdfiumSuccess','popplerSuccess')})) if not args.self_check and not report['success']:raise SystemExit('Intent validation failed; retained report preserves every failed probe') if __name__=='__main__':main()