#!/usr/bin/env python3 """Check RI masks/patterns against declared PDF state rules and a direct CMM. This runner never treats agreement between renderers as the expected value. The numerical DeviceRGB mask convention is distinct from intent selection. """ import argparse from io import BytesIO import itertools import json import math import os from pathlib import Path import platform import shutil import subprocess from PIL import Image, ImageChops, ImageCms, ImageDraw from pypdf import PdfReader from intents import Cmm, INTENTS, loaded_pdfium, sha ROOT=Path(__file__).resolve().parents[2] CORPUS=ROOT/'tests/fixtures/pdf/intent-transparency' PROFILE_SHA='b9a40bfc18e8280eefe0fedc4e607467e1cc27adeee3beb77f4a2b83fc7e9315' def preserved_inputs(): paths=[ROOT/'tests/fixtures/pdf/generate_rendering_intents.py',ROOT/'tests/cmyk_lut/intents.py'] paths += sorted((ROOT/'tests/fixtures/pdf/rendering-intents').glob('*')) return {str(p.relative_to(ROOT)):sha(p) for p in paths if p.is_file()} def verify(corpus,spec): assert spec['schemaVersion']==1 and spec['pageCount']==4 assert spec['pageSizePt']==[880,850] and spec['sample']==[0,.75,.25,0] assert spec['tolerance8Bit']==4 and len(spec['cases'])==27 assert len(spec['probes'])==108 assert sha(corpus/spec['file'])==spec['sha256'] assert sha(ROOT/'tests/fixtures/pdf/generate_intent_transparency.py')==spec['generatorSha256'] profile=(corpus/spec['profile']['file']).read_bytes() assert len(profile)==984 and sha(corpus/spec['profile']['file'])==PROFILE_SHA==spec['profile']['sha256'] assert profile==(ROOT/spec['profile']['origin']).read_bytes() pdf=PdfReader(corpus/spec['file']);assert len(pdf.pages)==4 identities={};probe_ids=set() for page_index,page in enumerate(pdf.pages,1): assert list(page.mediabox)==[0,0,880,850] resources=page['/Resources'];patterns=resources['/Pattern'];gs=resources['/ExtGState'] assert resources['/ColorSpace']['/CS'][1].get_object().get_data()==profile assert resources['/ColorSpace']['/PCS'][0]=='/Pattern' for key in ('U','ULarge'): stencil=patterns['/'+key] assert stencil['/PaintType']==2 assert stencil.get_data().split()==[b'0',b'0',str(stencil['/XStep']).encode(),str(stencil['/YStep']).encode(),b're',b'f'] for key in ('Tile','TileForm','ShadeForm'): obj=patterns.raw_get('/'+key) if key=='Tile' else resources['/XObject'].raw_get('/'+key) identities.setdefault(key,obj.idnum);assert identities[key]==obj.idnum assert b' ri' not in patterns['/Tile'].get_data() assert '/ExtGState' not in patterns['/Shade'] for label in ('Tile','Shade'): form=resources['/XObject']['/'+label+'Form'] assert form.get_data().startswith(b'/AbsoluteColorimetric ri\n') assert form['/Resources']['/Pattern'].raw_get('/Tile').idnum==identities['Tile'] for i,intent in enumerate(INTENTS): assert ('/'+intent+' ri').encode() in patterns['/TileRi'+str(i)].get_data() assert patterns['/ShadeRi'+str(i)]['/ExtGState']['/RI']=='/'+intent for key in ('Mask','OpaqueBC','BC0','BC1','BC2','BC3',*[f'MaskRi{i}' for i in range(4)], *[f'MaskGs{i}' for i in range(4)],*[f'MaskQ{i}' for i in range(4)]): mask=gs['/'+key]['/SMask'];group=mask['/G'] assert mask['/S']=='/Luminosity' and mask['/TR']=='/Identity' assert group['/Group']['/CS']=='/DeviceRGB' and bool(group['/Group']['/I']) assert group['/Resources']['/ColorSpace']['/CS'][1].get_object().get_data()==profile if key.startswith('BC'):assert group.get_data().strip()==b'' assert gs['/NoMask']['/SMask']=='/None' assert gs['/Mask']['/SMask'].raw_get('/G').idnum==gs['/OpaqueBC']['/SMask'].raw_get('/G').idnum content=page.get_contents().get_data() assert content.startswith(b'1 g 0 0 880 850 re f\n') for p in (p for p in spec['probes'] if p['page']==page_index): assert p['id'] not in probe_ids;probe_ids.add(p['id']) assert ('\n'.join(p['commandSequence'])+'\n').encode() in content case=p['case'];column=p['column'];current=(2,0,2,1)[column] if case.endswith('-reuse') else column assert p['callerIntentIndex']==current and p['callerIntent']==INTENTS[current] expected=current;oracle='icc' if case in ('tiling-page-initial','tiling-colored-qQ','shading-page-initial'):expected=1 elif case in ('tiling-colored-ri','tiling-colored-gs','tiling-form-ri','shading-pattern-ri','shading-form-ri', 'mask-internal-ri','mask-internal-gs'):expected=(current+1)%4 if case.startswith('mask-'): oracle='luminosity-icc' if case in ('mask-q-inner-none','mask-none'):oracle='black' elif case=='mask-BC-only':oracle='luminosity-bc' assert p['expectedIntentIndex']==expected and p['expectedIntent']==INTENTS[expected] assert p['oracle']==oracle and p['components']==[0,.75,.25,0] if oracle=='luminosity-bc':assert p['backgroundRgb']==[[1,0,0],[0,1,0],[0,0,1],[0,.5,1]][column] return profile,{'pages':4,'probes':len(probe_ids),'reusedObjectIds':identities, 'uncoloredStencilsContainOnlyGeometry':True,'profileByteIdentical':True, 'expectedIntentIndependentlyChecked':True} def oracle(probe,cmm): kind=probe['oracle'] if kind=='black':return [0,0,0],{'kind':kind} if kind=='luminosity-bc':rgb=[255*c for c in probe['backgroundRgb']] else:rgb=cmm.color('distinct',probe['expectedIntentIndex'],probe['components']) if kind=='icc':return rgb,{'kind':kind,'directCmmRgb8':rgb} luminance=sum(w*c for w,c in zip((.30,.59,.11),rgb)) return [round(255-luminance)]*3,{'kind':kind,'groupRgb8':rgb,'luminosity8Bit':luminance, 'recommendedDeviceRgbWeights':[.30,.59,.11]} def cmm_checks(profile,cmm,spec): colors=[cmm.color('distinct',i,spec['sample']) for i in range(4)] lumas=[sum(w*c for w,c in zip((.30,.59,.11),rgb)) for rgb in colors] separation=min(abs(a-b) for a,b in itertools.combinations(lumas,2)) assert separation>2*spec['tolerance8Bit'] source=ImageCms.ImageCmsProfile(BytesIO(profile));sample=tuple(round(v*255) for v in spec['sample']) checks=[] for i in range(4): transform=ImageCms.buildTransformFromOpenProfiles(source,ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=i) quantized=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),sample),transform).getpixel((0,0))) assert max(abs(a-b) for a,b in zip(quantized,colors[i]))<=1 checks.append({'intent':INTENTS[i],'directCmmRgb8':colors[i],'pillowQuantizedRgb8':quantized, 'luminosity8Bit':lumas[i],'blackOnWhiteGray8Bit':round(255-lumas[i])}) return {'values':checks,'minimumLuminositySeparation':separation, 'noRenderedPixelsUsedForExpectedValues':True} def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--build-dir',type=Path) parser.add_argument('--pdfium-library',type=Path) parser.add_argument('--corpus',type=Path,default=CORPUS) parser.add_argument('--output',type=Path,required=True) parser.add_argument('--self-check',action='store_true') parser.add_argument('--scales',type=float,nargs='+',default=[.5,1.,2.]) args=parser.parse_args() if not args.self_check and not args.build_dir:parser.error('--build-dir required for rendering') if not 1<=len(args.scales)<=4 or len(set(args.scales))!=len(args.scales) or any(not math.isfinite(v) or not .25<=v<=4 for v in args.scales): parser.error('scales must be 1..4 unique finite values in .25..4') output=args.output.resolve();output.mkdir(parents=True,exist_ok=False) old=preserved_inputs();spec=json.loads((args.corpus/'manifest.json').read_text());source=(args.corpus/spec['file']).resolve() report={'schemaVersion':1,'success':False,'renderingAcceptance':False,'fixtureValidationSuccess':False, 'mode':'fixture-self-check' if args.self_check else 'renderer-comparison', 'runnerSha256':sha(Path(__file__)),'sharedCmmAdapterSha256':sha(Path(__file__).with_name('intents.py')), 'fixtureManifestSha256':sha(args.corpus/'manifest.json'),'fixtureSha256':sha(source), 'pythonVersion':platform.python_version(),'pillowLcmsVersion':ImageCms.core.littlecms_version, 'tolerance8Bit':4,'commands':[],'scales':[],'preservedLegacyInputs':old, 'scope':'Synthetic ICC RI dispatch and state probes. Direct public LittleCMS calls supply colors, not renderer pixels; ' 'the renderers may use the same CMM. DeviceRGB luminosity uses the PDF recommended weights, which are device-dependent. ' 'No physical-display, press, general ICC, or all-PDF acceptance claim.'} cmm=None;environment=os.environ.copy();binaries={} try: profile,verification=verify(args.corpus,spec);report['fixtureVerification']=verification cmm=Cmm({'distinct':profile}) report['directCmm']={'encodedVersion':cmm.version,'libraries':cmm.libraryPaths, 'input':'CMYK double percent','output':'8-bit sRGB','flags':0} report['cmmChecks']=cmm_checks(profile,cmm,spec);report['fixtureValidationSuccess']=True if not args.self_check: build=args.build_dir.resolve() binaries={name:sha(build/name) for name in ('pdf-worker-evidence','docview-pdf-worker')};report['binaries']=binaries if args.pdfium_library: environment['LD_LIBRARY_PATH']=str(args.pdfium_library.resolve().parent)+(':'+environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '') assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT') report['pdfiumLibrary']=loaded_pdfium(build/'docview-pdf-worker',environment,args.pdfium_library) for program in ('qpdf','pdftoppm'):assert shutil.which(program) report['popplerVersion']=subprocess.run(['pdftoppm','-v'],capture_output=True,text=True,check=True).stderr.splitlines()[0] def run(label,command,env=None): log=output/(label+'.log') with log.open('w') as f:result=subprocess.run(command,stdout=f,stderr=subprocess.STDOUT,env=env,timeout=300) report['commands'].append({'name':label,'command':command,'exitCode':result.returncode,'logSha256':sha(log)}) assert result.returncode==0,label+' failed' run('qpdf-check',['qpdf','--check',str(source)]) for scale in args.scales: label='scale-'+str(scale).replace('.','_');folder=output/label;(folder/'pdfium').mkdir(parents=True) run(label+'-pdfium',[str(build/'pdf-worker-evidence'),str(source),str(folder/'pdfium'),str(scale)],environment) run(label+'-poppler',['pdftoppm','-cropbox','-r',str(72*scale),'-png',str(source),str(folder/'poppler')]) metadata=json.loads((folder/'pdfium/metadata.json').read_text()) assert metadata['pageCount']==spec['pageCount'] and metadata['renderScale']==scale and 'sandbox enabled' in metadata['transport'] row={'scale':scale,'probes':[],'pdfiumSuccess':True,'popplerSuccess':True,'pages':[]} for page in range(1,5): paths={'pdfium':folder/'pdfium'/f'page-{page}.png','poppler':folder/f'poppler-{page}.png'} images={key:Image.open(path).convert('RGB') for key,path in paths.items()} assert all(list(im.size)==[math.ceil(v*scale) for v in spec['pageSizePt']] for im in images.values()) for probe in (p for p in spec['probes'] if p['page']==page): x,y=probe['pointPt'];pixel=(round(x*scale),round((850-y)*scale)) expected,details=oracle(probe,cmm) # Interior 3x3 minimum/maximum detects seams/unstable edge # samples instead of silently selecting a favorable pixel. samples={key:[list(im.getpixel((pixel[0]+dx,pixel[1]+dy))) for dy in (-1,0,1) for dx in (-1,0,1)] for key,im in images.items()} actual={key:values[4] for key,values in samples.items()} errors={key:max(abs(a-b) for color in values for a,b in zip(color,expected)) for key,values in samples.items()} matches={key:err<=4 for key,err in errors.items()} row['probes'].append({**probe,'pointPx':pixel,'oracleDetails':details,'expectedRgb':expected, 'actualRgb':actual,'neighborhoodMinRgb':{k:[min(c[i] for c in v) for i in range(3)] for k,v in samples.items()}, 'neighborhoodMaxRgb':{k:[max(c[i] for c in v) for i in range(3)] for k,v in samples.items()}, 'maxChannelError':errors,'matches':matches}) for key,match in matches.items():row[key+'Success'] &= match diff=ImageChops.difference(images['pdfium'],images['poppler']);diff.save(folder/f'difference-{page}.png') sheet=Image.new('RGB',(1260,440),'#eeeeee');draw=ImageDraw.Draw(sheet) for col,(key,im) in enumerate([*images.items(),('difference',diff)]): draw.text((col*420+8,7),f'{scale:g}x page {page}: {key}',fill='black') thumb=im.copy();thumb.thumbnail((410,400));sheet.paste(thumb,(col*420+5,30)) sheet.save(folder/f'comparison-{page}.png') row['pages'].append({'page':page,'images':{key:sha(path) for key,path in paths.items()}}) row['failedProbes']={engine:sum(not p['matches'][engine] for p in row['probes']) for engine in ('pdfium','poppler')} row['success']=row['pdfiumSuccess'] and row['popplerSuccess'];report['scales'].append(row) report['pdfiumSuccess']=all(r['pdfiumSuccess'] for r in report['scales']) report['popplerSuccess']=all(r['popplerSuccess'] for r in report['scales']) report['binariesUnchanged']=all(sha(build/n)==value for n,value in binaries.items()) and sha(Path(report['pdfiumLibrary']['path']))==report['pdfiumLibrary']['sha256'] report['originalUnchanged']=sha(source)==spec['sha256'] and sha(args.corpus/spec['profile']['file'])==PROFILE_SHA report['legacyInputsUnchanged']=preserved_inputs()==old if not args.self_check: report['success']=report['renderingAcceptance']=all(report[k] for k in ('pdfiumSuccess','popplerSuccess','binariesUnchanged','originalUnchanged','legacyInputsUnchanged')) except Exception as error: report['error']=type(error).__name__+': '+str(error) raise finally: if cmm:cmm.close() report['evidenceSha256']={str(p.relative_to(output)):sha(p) for p in sorted(output.rglob('*')) if p.is_file()} (output/'report.json').write_text(json.dumps(report,indent=2)+'\n') print(json.dumps({key:report.get(key) for key in ('mode','fixtureValidationSuccess','success','pdfiumSuccess','popplerSuccess','legacyInputsUnchanged')})) if not args.self_check and not report['success']:raise SystemExit('Pixel criteria failed; every failing probe remains in report.json') if __name__=='__main__':main()