217 lines
15 KiB
Python
217 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Prepare an isolated, pinned LittleCMS reference for the dedicated Ubuntu VM.
|
|
|
|
No package installation, system linker change, PDFium replacement, or downloads.
|
|
The PDFium allocator integration is reversed only in the dedicated build copy.
|
|
Other vendored changes remain and are explicitly identified in the build record.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
|
|
ROOT=Path(__file__).resolve().parents[2]
|
|
REVISION='b76633e60c8387a77268fb3359277ca25b5fd75c'
|
|
|
|
|
|
def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
def write_json(path,value): path.write_text(json.dumps(value,indent=2)+'\n')
|
|
def json_hash(value): return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':')).encode()).hexdigest()
|
|
|
|
|
|
def pack(args):
|
|
source=args.source.resolve(strict=True);output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
|
|
readme=(source/'README.pdfium').read_text()
|
|
assert 'Version: 2.19\n' in readme and 'Revision: '+REVISION+'\n' in readme
|
|
assert re.search(r'#define\s+LCMS_VERSION\s+2190\b',(source/'include/lcms2.h').read_text())
|
|
files={}
|
|
for p in sorted(source.rglob('*')):
|
|
if p.is_dir():continue
|
|
assert p.is_file() and not p.is_symlink() and p.resolve().is_relative_to(source)
|
|
relative=p.relative_to(source)
|
|
allowed=(len(relative.parts)==1 and (p.name in ('LICENSE','README.pdfium') or p.suffix=='.patch')) or (relative.parts[0] in ('src','include') and p.suffix in ('.c','.h'))
|
|
assert allowed and p.stat().st_size<=2*1024*1024
|
|
files['third_party/lcms/'+str(relative)]=p.read_bytes()
|
|
assert len(files)<100 and sum(map(len,files.values()))<16*1024*1024
|
|
reference=json.loads(args.reference_report.read_text());other=json.loads(args.comparison_report.read_text())
|
|
assert reference['directCmm']['encodedVersion']==2190 and other['directCmm']['encodedVersion']==2140
|
|
rows=[];comparisons=[]
|
|
for sr,so in zip(reference['scales'],other['scales'],strict=True):
|
|
assert sr['scale']==so['scale']
|
|
for a,b in zip(sr['probes'],so['probes'],strict=True):
|
|
fields=('page','profile','case','column','expectedIntentIndex','sampledComponents')
|
|
assert all(a[k]==b[k] for k in fields)
|
|
row={k:a[k] for k in fields};row.update(scale=sr['scale'],expectedRgb=a['expectedRgb'])
|
|
rows.append(row)
|
|
comparisons.append({'scale':sr['scale'],'page':a['page'],'case':a['case'],'column':a['column'],'profile':a['profile'],
|
|
'referenceExpectedRgb':a['expectedRgb'],'system214ExpectedRgb':b['expectedRgb'],
|
|
'pdfiumRgbEqual':a['actualRgb']['pdfium']==b['actualRgb']['pdfium'],
|
|
'pdfiumRgb':a['actualRgb']['pdfium'],'expectedRgbEqual':a['expectedRgb']==b['expectedRgb']})
|
|
assert len(rows)==600
|
|
profiles={}
|
|
corpus=ROOT/'tests/fixtures/pdf/rendering-intents'
|
|
manifest=json.loads((corpus/'manifest.json').read_text())
|
|
for key,item in manifest['profiles'].items():
|
|
path=corpus/item['file'];assert sha(path)==item['sha256']
|
|
files['profiles/'+path.name]=path.read_bytes();profiles[key]={'file':'profiles/'+path.name,'sha256':sha(path)}
|
|
expectations={'referenceReportSha256':sha(args.reference_report),'referenceDirectCmm':reference['directCmm'],
|
|
'comparisonReportSha256':sha(args.comparison_report),'profiles':profiles,'probes':rows,
|
|
'scope':'CMM numerical reproducibility only; the older shading-pattern-inherit state expectation is not adopted as PDF specification acceptance.'}
|
|
files['expected.json']=(json.dumps(expectations,indent=2)+'\n').encode()
|
|
files['intents.py']=Path(__file__).with_name('intents.py').read_bytes()
|
|
files['prepare_reference_cmm.py']=Path(__file__).read_bytes()
|
|
archive=output/'reference-lcms219.tar'
|
|
with tarfile.open(archive,'w',format=tarfile.PAX_FORMAT) as tar:
|
|
for name,data in sorted(files.items()):
|
|
info=tarfile.TarInfo(name);info.size=len(data);info.mode=0o644;info.mtime=0
|
|
tar.addfile(info,io.BytesIO(data))
|
|
inventory=[{'path':name,'size':len(data),'sha256':hashlib.sha256(data).hexdigest()} for name,data in sorted(files.items())]
|
|
record={'schemaVersion':1,'upstreamVersion':'2.19','upstreamRevision':REVISION,
|
|
'sourceScope':'Fixed PDFium-vendored source; allocator integration will be reversed in build copy only.',
|
|
'archive':archive.name,'archiveSha256':sha(archive),'files':inventory,
|
|
'sourceInventorySha256':json_hash([r for r in inventory if r['path'].startswith('third_party/')]),
|
|
'referenceReportSha256':sha(args.reference_report),'comparisonReportSha256':sha(args.comparison_report)}
|
|
write_json(output/'bundle.json',record)
|
|
write_json(output/'cross-os-before.json',{'scope':expectations['scope'],'rows':comparisons,
|
|
'total':len(comparisons),'allPdfiumRgbEqual':all(r['pdfiumRgbEqual'] for r in comparisons),
|
|
'differentExpectedRgb':sum(not r['expectedRgbEqual'] for r in comparisons)})
|
|
print(json.dumps({'archive':str(archive),'sha256':record['archiveSha256'],'files':len(inventory)}))
|
|
|
|
|
|
def build(args):
|
|
bundle=args.bundle.resolve(strict=True);record=json.loads(bundle.read_text());archive=bundle.parent/record['archive']
|
|
assert sha(archive)==record['archiveSha256']
|
|
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
|
|
source=output/'source';source.mkdir()
|
|
rows={r['path']:r for r in record['files']};assert len(rows)==len(record['files'])<=100
|
|
with tarfile.open(archive,'r:') as tar:
|
|
entries=tar.getmembers();assert len(entries)==len(rows)
|
|
seen=set()
|
|
for entry in entries:
|
|
name=entry.name;relative=Path(name)
|
|
assert name in rows and name not in seen and entry.isfile() and not relative.is_absolute() and '..' not in relative.parts
|
|
assert entry.size==rows[name]['size']<=2*1024*1024
|
|
seen.add(name);data=tar.extractfile(entry).read()
|
|
assert hashlib.sha256(data).hexdigest()==rows[name]['sha256']
|
|
path=source/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(data)
|
|
lcms=source/'third_party/lcms';cmserr=lcms/'src/cmserr.c';before=sha(cmserr)
|
|
patch_source=lcms/'0000-cmserr-changes.patch'
|
|
# This file first includes a historical patch-file diff. Select only its
|
|
# final, actual cmserr.c delta, never execute arbitrary packaged hooks.
|
|
marker='diff --git a/third_party/lcms/src/cmserr.c b/third_party/lcms/src/cmserr.c\n'
|
|
patch_text=patch_source.read_text();assert patch_text.count('\n'+marker)==1
|
|
delta=patch_text[patch_text.index('\n'+marker)+1:]
|
|
assert delta.count('\ndiff --git ')==0
|
|
patch_file=output/'restore-standard-allocator.patch';patch_file.write_text(delta)
|
|
commands=[]
|
|
def run(label,command,**kwargs):
|
|
result=subprocess.run(command,capture_output=True,text=True,timeout=300,**kwargs)
|
|
log=output/(label+'.log');log.write_text(result.stdout+result.stderr)
|
|
commands.append({'command':command,'exitCode':result.returncode,'log':log.name,'logSha256':sha(log)})
|
|
assert result.returncode==0,label+' failed; see '+str(log)
|
|
return result.stdout
|
|
run('allocator-restore',['patch','--batch','--fuzz=0','--reverse','-p1','-i',str(patch_file)],cwd=source)
|
|
assert 'FXMEM_' not in cmserr.read_text() and '#include "core/' not in cmserr.read_text()
|
|
changes=[]
|
|
for row in record['files']:
|
|
now=sha(source/row['path'])
|
|
if now!=row['sha256']:changes.append({'path':row['path'],'before':row['sha256'],'after':now})
|
|
assert len(changes)==1 and changes[0]['path']=='third_party/lcms/src/cmserr.c' and changes[0]['before']==before
|
|
compiler=shutil.which('gcc');assert compiler
|
|
version=run('compiler-version',[compiler,'--version']);target=run('compiler-target',[compiler,'-dumpmachine']).strip()
|
|
library_dir=output/'lib';library_dir.mkdir();library=library_dir/'liblcms2.so.2.0.19'
|
|
inputs=[str(p) for p in sorted((lcms/'src').glob('*.c'))];assert len(inputs)==26
|
|
compile_command=[compiler,'-std=c99','-O2','-fPIC','-D_POSIX_C_SOURCE=200809L','-shared',
|
|
'-Wl,-soname,liblcms2.so.2','-Wl,-z,defs','-I'+str(source),'-I'+str(lcms/'include'),
|
|
*inputs,'-lm','-lpthread','-o',str(library)]
|
|
run('compile',compile_command)
|
|
(library_dir/'liblcms2.so.2').symlink_to(library.name);(library_dir/'liblcms2.so').symlink_to(library.name)
|
|
dependencies=run('ldd',['ldd',str(library)]);assert 'not found' not in dependencies
|
|
dynamic=run('readelf',['readelf','-d',str(library)])
|
|
environment=os.environ.copy();environment['LD_LIBRARY_PATH']=str(library_dir)
|
|
assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT')
|
|
run('runtime-version',[sys.executable,'-c',
|
|
'import ctypes; from PIL import ImageCms; c=ctypes.CDLL("liblcms2.so.2"); print(c.cmsGetEncodedCMMversion(), ImageCms.core.littlecms_version); assert c.cmsGetEncodedCMMversion()==2190; assert ImageCms.core.littlecms_version=="2.19"'],env=environment)
|
|
result={'schemaVersion':1,'success':True,'bundleSha256':sha(bundle),'archiveSha256':record['archiveSha256'],
|
|
'sourceInventorySha256':record['sourceInventorySha256'],'upstreamVersion':'2.19','upstreamRevision':REVISION,
|
|
'derivedSourceChanges':changes,'allocatorPatchSha256':sha(patch_file),
|
|
'remainingVendorPatches':'Retained; this is a pinned PDFium-vendored reference build with only its allocator integration reversed.',
|
|
'compiler':{'path':compiler,'sha256':sha(Path(compiler).resolve()),'version':version,'target':target},
|
|
'library':{'path':str(library),'size':library.stat().st_size,'sha256':sha(library),'soname':'liblcms2.so.2'},
|
|
'dependencies':dependencies,'dynamicSection':dynamic,'commands':commands,
|
|
'osRelease':platform.freedesktop_os_release(),'pythonVersion':platform.python_version(),
|
|
'scope':'Isolated reference library only; no system installation or loader configuration changes.'}
|
|
write_json(output/'build.json',result)
|
|
print(json.dumps({'success':True,'library':str(library),'sha256':sha(library)}))
|
|
|
|
|
|
def proof(args):
|
|
# Must start a fresh Python interpreter with the desired LD_LIBRARY_PATH;
|
|
# changing it after import cannot select an already loaded native library.
|
|
output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
|
|
data=json.loads(args.expected.read_text());source=args.expected.resolve().parent
|
|
sys.path.insert(0,str(source))
|
|
from intents import Cmm
|
|
from PIL import Image,ImageCms
|
|
profiles={key:(source/row['file']).read_bytes() for key,row in data['profiles'].items()}
|
|
assert all(sha(source/row['file'])==row['sha256'] for row in data['profiles'].values())
|
|
cmm=Cmm(profiles)
|
|
try:
|
|
assert cmm.version==args.version
|
|
paths={line.split()[-1] for line in Path('/proc/self/maps').read_text().splitlines() if 'liblcms2.so' in line and '/' in line}
|
|
assert len(paths)==1
|
|
actual=Path(next(iter(paths))).resolve()
|
|
if args.library:assert actual==args.library.resolve(strict=True)
|
|
assert ImageCms.core.littlecms_version==f'{args.version//1000}.{(args.version%1000)//10}'
|
|
module=Path(ImageCms.core.__file__)
|
|
ldd=subprocess.run(['ldd',str(module)],capture_output=True,text=True,check=True,timeout=20).stdout
|
|
match=re.search(r'liblcms2\.so\.2 => (\S+)',ldd);assert match and Path(match[1]).resolve()==actual
|
|
pillow={}
|
|
for key,profile in profiles.items():
|
|
for i in range(4):
|
|
pillow[key,i]=ImageCms.buildTransformFromOpenProfiles(ImageCms.ImageCmsProfile(io.BytesIO(profile)),ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=i)
|
|
rows=[];quantized=[]
|
|
for p in data['probes']:
|
|
color=cmm.color(p['profile'],p['expectedIntentIndex'],p['sampledComponents'])
|
|
rows.append({**p,'actualRgb':color,'equal':color==p['expectedRgb'],
|
|
'maxChannelError':max(abs(a-b) for a,b in zip(color,p['expectedRgb']))})
|
|
for key in profiles:
|
|
for i in range(4):
|
|
pixel=(51,128,204,77)
|
|
direct=cmm.color(key,i,[v/255 for v in pixel])
|
|
pil=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),pixel),pillow[key,i]).getpixel((0,0)))
|
|
quantized.append({'profile':key,'intent':i,'directRgb':direct,'pillowRgb':pil,
|
|
'maxChannelError':max(abs(a-b) for a,b in zip(direct,pil))})
|
|
report={'schemaVersion':1,'success':all(p['equal'] for p in rows) and all(p['maxChannelError']<=1 for p in quantized),
|
|
'referenceReportSha256':data['referenceReportSha256'],'expectedFileSha256':sha(args.expected),
|
|
'expectedScope':data['scope'],'pythonVersion':platform.python_version(),'encodedVersion':cmm.version,
|
|
'pillowVersion':__import__('PIL').__version__,'pillowLcmsVersion':ImageCms.core.littlecms_version,
|
|
'loadedLibrary':{'path':str(actual),'sha256':sha(actual)},'mappedLibraries':sorted(paths),
|
|
'pillowExtension':{'path':str(module),'sha256':sha(module),'ldd':ldd},
|
|
'total':len(rows),'identical':sum(p['equal'] for p in rows),'different':sum(not p['equal'] for p in rows),
|
|
'pillowChecks':quantized,'probes':rows,
|
|
'limitations':'Numerical matching at the declared probes, not independent-CMM or PDF semantic correctness.'}
|
|
write_json(output/'proof.json',report)
|
|
print(json.dumps({k:report[k] for k in ('success','encodedVersion','pillowLcmsVersion','total','identical','different')}))
|
|
if args.version==2190:assert report['success'],'Pinned CMM differs from host reference'
|
|
finally:cmm.close()
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__);commands=parser.add_subparsers(dest='command',required=True)
|
|
p=commands.add_parser('pack');p.add_argument('--source',type=Path,required=True);p.add_argument('--reference-report',type=Path,required=True);p.add_argument('--comparison-report',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=pack)
|
|
p=commands.add_parser('build');p.add_argument('--bundle',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=build)
|
|
p=commands.add_parser('proof');p.add_argument('--expected',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--version',type=int,choices=(2140,2190),required=True);p.add_argument('--library',type=Path);p.set_defaults(function=proof)
|
|
args=parser.parse_args();args.function(args)
|
|
|
|
|
|
if __name__=='__main__':main()
|