#!/usr/bin/env python3 """Prepare an isolated, pinned LittleCMS reference for the dedicated Ubuntu VM. No package installation, system linker change, PDFium replacement, or downloads. The PDFium allocator integration is reversed only in the dedicated build copy. Other vendored changes remain and are explicitly identified in the build record. """ import argparse import hashlib import io import json import os from pathlib import Path import platform import re import shutil import subprocess import sys import tarfile ROOT=Path(__file__).resolve().parents[2] REVISION='b76633e60c8387a77268fb3359277ca25b5fd75c' def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() def write_json(path,value): path.write_text(json.dumps(value,indent=2)+'\n') def json_hash(value): return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':')).encode()).hexdigest() def pack(args): source=args.source.resolve(strict=True);output=args.output.resolve();output.mkdir(parents=True,exist_ok=False) readme=(source/'README.pdfium').read_text() assert 'Version: 2.19\n' in readme and 'Revision: '+REVISION+'\n' in readme assert re.search(r'#define\s+LCMS_VERSION\s+2190\b',(source/'include/lcms2.h').read_text()) files={} for p in sorted(source.rglob('*')): if p.is_dir():continue assert p.is_file() and not p.is_symlink() and p.resolve().is_relative_to(source) relative=p.relative_to(source) allowed=(len(relative.parts)==1 and (p.name in ('LICENSE','README.pdfium') or p.suffix=='.patch')) or (relative.parts[0] in ('src','include') and p.suffix in ('.c','.h')) assert allowed and p.stat().st_size<=2*1024*1024 files['third_party/lcms/'+str(relative)]=p.read_bytes() assert len(files)<100 and sum(map(len,files.values()))<16*1024*1024 reference=json.loads(args.reference_report.read_text());other=json.loads(args.comparison_report.read_text()) assert reference['directCmm']['encodedVersion']==2190 and other['directCmm']['encodedVersion']==2140 rows=[];comparisons=[] for sr,so in zip(reference['scales'],other['scales'],strict=True): assert sr['scale']==so['scale'] for a,b in zip(sr['probes'],so['probes'],strict=True): fields=('page','profile','case','column','expectedIntentIndex','sampledComponents') assert all(a[k]==b[k] for k in fields) row={k:a[k] for k in fields};row.update(scale=sr['scale'],expectedRgb=a['expectedRgb']) rows.append(row) comparisons.append({'scale':sr['scale'],'page':a['page'],'case':a['case'],'column':a['column'],'profile':a['profile'], 'referenceExpectedRgb':a['expectedRgb'],'system214ExpectedRgb':b['expectedRgb'], 'pdfiumRgbEqual':a['actualRgb']['pdfium']==b['actualRgb']['pdfium'], 'pdfiumRgb':a['actualRgb']['pdfium'],'expectedRgbEqual':a['expectedRgb']==b['expectedRgb']}) assert len(rows)==600 profiles={} corpus=ROOT/'tests/fixtures/pdf/rendering-intents' manifest=json.loads((corpus/'manifest.json').read_text()) for key,item in manifest['profiles'].items(): path=corpus/item['file'];assert sha(path)==item['sha256'] files['profiles/'+path.name]=path.read_bytes();profiles[key]={'file':'profiles/'+path.name,'sha256':sha(path)} expectations={'referenceReportSha256':sha(args.reference_report),'referenceDirectCmm':reference['directCmm'], 'comparisonReportSha256':sha(args.comparison_report),'profiles':profiles,'probes':rows, 'scope':'CMM numerical reproducibility only; the older shading-pattern-inherit state expectation is not adopted as PDF specification acceptance.'} files['expected.json']=(json.dumps(expectations,indent=2)+'\n').encode() files['intents.py']=Path(__file__).with_name('intents.py').read_bytes() files['prepare_reference_cmm.py']=Path(__file__).read_bytes() archive=output/'reference-lcms219.tar' with tarfile.open(archive,'w',format=tarfile.PAX_FORMAT) as tar: for name,data in sorted(files.items()): info=tarfile.TarInfo(name);info.size=len(data);info.mode=0o644;info.mtime=0 tar.addfile(info,io.BytesIO(data)) inventory=[{'path':name,'size':len(data),'sha256':hashlib.sha256(data).hexdigest()} for name,data in sorted(files.items())] record={'schemaVersion':1,'upstreamVersion':'2.19','upstreamRevision':REVISION, 'sourceScope':'Fixed PDFium-vendored source; allocator integration will be reversed in build copy only.', 'archive':archive.name,'archiveSha256':sha(archive),'files':inventory, 'sourceInventorySha256':json_hash([r for r in inventory if r['path'].startswith('third_party/')]), 'referenceReportSha256':sha(args.reference_report),'comparisonReportSha256':sha(args.comparison_report)} write_json(output/'bundle.json',record) write_json(output/'cross-os-before.json',{'scope':expectations['scope'],'rows':comparisons, 'total':len(comparisons),'allPdfiumRgbEqual':all(r['pdfiumRgbEqual'] for r in comparisons), 'differentExpectedRgb':sum(not r['expectedRgbEqual'] for r in comparisons)}) print(json.dumps({'archive':str(archive),'sha256':record['archiveSha256'],'files':len(inventory)})) def build(args): bundle=args.bundle.resolve(strict=True);record=json.loads(bundle.read_text());archive=bundle.parent/record['archive'] assert sha(archive)==record['archiveSha256'] output=args.output.resolve();output.mkdir(parents=True,exist_ok=False) source=output/'source';source.mkdir() rows={r['path']:r for r in record['files']};assert len(rows)==len(record['files'])<=100 with tarfile.open(archive,'r:') as tar: entries=tar.getmembers();assert len(entries)==len(rows) seen=set() for entry in entries: name=entry.name;relative=Path(name) assert name in rows and name not in seen and entry.isfile() and not relative.is_absolute() and '..' not in relative.parts assert entry.size==rows[name]['size']<=2*1024*1024 seen.add(name);data=tar.extractfile(entry).read() assert hashlib.sha256(data).hexdigest()==rows[name]['sha256'] path=source/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(data) lcms=source/'third_party/lcms';cmserr=lcms/'src/cmserr.c';before=sha(cmserr) patch_source=lcms/'0000-cmserr-changes.patch' # This file first includes a historical patch-file diff. Select only its # final, actual cmserr.c delta, never execute arbitrary packaged hooks. marker='diff --git a/third_party/lcms/src/cmserr.c b/third_party/lcms/src/cmserr.c\n' patch_text=patch_source.read_text();assert patch_text.count('\n'+marker)==1 delta=patch_text[patch_text.index('\n'+marker)+1:] assert delta.count('\ndiff --git ')==0 patch_file=output/'restore-standard-allocator.patch';patch_file.write_text(delta) commands=[] def run(label,command,**kwargs): result=subprocess.run(command,capture_output=True,text=True,timeout=300,**kwargs) log=output/(label+'.log');log.write_text(result.stdout+result.stderr) commands.append({'command':command,'exitCode':result.returncode,'log':log.name,'logSha256':sha(log)}) assert result.returncode==0,label+' failed; see '+str(log) return result.stdout run('allocator-restore',['patch','--batch','--fuzz=0','--reverse','-p1','-i',str(patch_file)],cwd=source) assert 'FXMEM_' not in cmserr.read_text() and '#include "core/' not in cmserr.read_text() changes=[] for row in record['files']: now=sha(source/row['path']) if now!=row['sha256']:changes.append({'path':row['path'],'before':row['sha256'],'after':now}) assert len(changes)==1 and changes[0]['path']=='third_party/lcms/src/cmserr.c' and changes[0]['before']==before compiler=shutil.which('gcc');assert compiler version=run('compiler-version',[compiler,'--version']);target=run('compiler-target',[compiler,'-dumpmachine']).strip() library_dir=output/'lib';library_dir.mkdir();library=library_dir/'liblcms2.so.2.0.19' inputs=[str(p) for p in sorted((lcms/'src').glob('*.c'))];assert len(inputs)==26 compile_command=[compiler,'-std=c99','-O2','-fPIC','-D_POSIX_C_SOURCE=200809L','-shared', '-Wl,-soname,liblcms2.so.2','-Wl,-z,defs','-I'+str(source),'-I'+str(lcms/'include'), *inputs,'-lm','-lpthread','-o',str(library)] run('compile',compile_command) (library_dir/'liblcms2.so.2').symlink_to(library.name);(library_dir/'liblcms2.so').symlink_to(library.name) dependencies=run('ldd',['ldd',str(library)]);assert 'not found' not in dependencies dynamic=run('readelf',['readelf','-d',str(library)]) environment=os.environ.copy();environment['LD_LIBRARY_PATH']=str(library_dir) assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT') run('runtime-version',[sys.executable,'-c', 'import ctypes; from PIL import ImageCms; c=ctypes.CDLL("liblcms2.so.2"); print(c.cmsGetEncodedCMMversion(), ImageCms.core.littlecms_version); assert c.cmsGetEncodedCMMversion()==2190; assert ImageCms.core.littlecms_version=="2.19"'],env=environment) result={'schemaVersion':1,'success':True,'bundleSha256':sha(bundle),'archiveSha256':record['archiveSha256'], 'sourceInventorySha256':record['sourceInventorySha256'],'upstreamVersion':'2.19','upstreamRevision':REVISION, 'derivedSourceChanges':changes,'allocatorPatchSha256':sha(patch_file), 'remainingVendorPatches':'Retained; this is a pinned PDFium-vendored reference build with only its allocator integration reversed.', 'compiler':{'path':compiler,'sha256':sha(Path(compiler).resolve()),'version':version,'target':target}, 'library':{'path':str(library),'size':library.stat().st_size,'sha256':sha(library),'soname':'liblcms2.so.2'}, 'dependencies':dependencies,'dynamicSection':dynamic,'commands':commands, 'osRelease':platform.freedesktop_os_release(),'pythonVersion':platform.python_version(), 'scope':'Isolated reference library only; no system installation or loader configuration changes.'} write_json(output/'build.json',result) print(json.dumps({'success':True,'library':str(library),'sha256':sha(library)})) def proof(args): # Must start a fresh Python interpreter with the desired LD_LIBRARY_PATH; # changing it after import cannot select an already loaded native library. output=args.output.resolve();output.mkdir(parents=True,exist_ok=False) data=json.loads(args.expected.read_text());source=args.expected.resolve().parent sys.path.insert(0,str(source)) from intents import Cmm from PIL import Image,ImageCms profiles={key:(source/row['file']).read_bytes() for key,row in data['profiles'].items()} assert all(sha(source/row['file'])==row['sha256'] for row in data['profiles'].values()) cmm=Cmm(profiles) try: assert cmm.version==args.version paths={line.split()[-1] for line in Path('/proc/self/maps').read_text().splitlines() if 'liblcms2.so' in line and '/' in line} assert len(paths)==1 actual=Path(next(iter(paths))).resolve() if args.library:assert actual==args.library.resolve(strict=True) assert ImageCms.core.littlecms_version==f'{args.version//1000}.{(args.version%1000)//10}' module=Path(ImageCms.core.__file__) ldd=subprocess.run(['ldd',str(module)],capture_output=True,text=True,check=True,timeout=20).stdout match=re.search(r'liblcms2\.so\.2 => (\S+)',ldd);assert match and Path(match[1]).resolve()==actual pillow={} for key,profile in profiles.items(): for i in range(4): pillow[key,i]=ImageCms.buildTransformFromOpenProfiles(ImageCms.ImageCmsProfile(io.BytesIO(profile)),ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=i) rows=[];quantized=[] for p in data['probes']: color=cmm.color(p['profile'],p['expectedIntentIndex'],p['sampledComponents']) rows.append({**p,'actualRgb':color,'equal':color==p['expectedRgb'], 'maxChannelError':max(abs(a-b) for a,b in zip(color,p['expectedRgb']))}) for key in profiles: for i in range(4): pixel=(51,128,204,77) direct=cmm.color(key,i,[v/255 for v in pixel]) pil=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),pixel),pillow[key,i]).getpixel((0,0))) quantized.append({'profile':key,'intent':i,'directRgb':direct,'pillowRgb':pil, 'maxChannelError':max(abs(a-b) for a,b in zip(direct,pil))}) report={'schemaVersion':1,'success':all(p['equal'] for p in rows) and all(p['maxChannelError']<=1 for p in quantized), 'referenceReportSha256':data['referenceReportSha256'],'expectedFileSha256':sha(args.expected), 'expectedScope':data['scope'],'pythonVersion':platform.python_version(),'encodedVersion':cmm.version, 'pillowVersion':__import__('PIL').__version__,'pillowLcmsVersion':ImageCms.core.littlecms_version, 'loadedLibrary':{'path':str(actual),'sha256':sha(actual)},'mappedLibraries':sorted(paths), 'pillowExtension':{'path':str(module),'sha256':sha(module),'ldd':ldd}, 'total':len(rows),'identical':sum(p['equal'] for p in rows),'different':sum(not p['equal'] for p in rows), 'pillowChecks':quantized,'probes':rows, 'limitations':'Numerical matching at the declared probes, not independent-CMM or PDF semantic correctness.'} write_json(output/'proof.json',report) print(json.dumps({k:report[k] for k in ('success','encodedVersion','pillowLcmsVersion','total','identical','different')})) if args.version==2190:assert report['success'],'Pinned CMM differs from host reference' finally:cmm.close() def main(): parser=argparse.ArgumentParser(description=__doc__);commands=parser.add_subparsers(dest='command',required=True) p=commands.add_parser('pack');p.add_argument('--source',type=Path,required=True);p.add_argument('--reference-report',type=Path,required=True);p.add_argument('--comparison-report',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=pack) p=commands.add_parser('build');p.add_argument('--bundle',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=build) p=commands.add_parser('proof');p.add_argument('--expected',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--version',type=int,choices=(2140,2190),required=True);p.add_argument('--library',type=Path);p.set_defaults(function=proof) args=parser.parse_args();args.function(args) if __name__=='__main__':main()