initial commit

2026-09-21 13:41:40 +09:00
commit 855c7328df
411 changed files with 85352 additions and 0 deletions
@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""Prepare an isolated, pinned LittleCMS reference for the dedicated Ubuntu VM.
+
+No package installation, system linker change, PDFium replacement, or downloads.
+The PDFium allocator integration is reversed only in the dedicated build copy.
+Other vendored changes remain and are explicitly identified in the build record.
+"""
+import argparse
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+
+ROOT=Path(__file__).resolve().parents[2]
+REVISION='b76633e60c8387a77268fb3359277ca25b5fd75c'
+
+
+def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest()
+def write_json(path,value): path.write_text(json.dumps(value,indent=2)+'\n')
+def json_hash(value): return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':')).encode()).hexdigest()
+
+
+def pack(args):
+    source=args.source.resolve(strict=True);output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
+    readme=(source/'README.pdfium').read_text()
+    assert 'Version: 2.19\n' in readme and 'Revision: '+REVISION+'\n' in readme
+    assert re.search(r'#define\s+LCMS_VERSION\s+2190\b',(source/'include/lcms2.h').read_text())
+    files={}
+    for p in sorted(source.rglob('*')):
+        if p.is_dir():continue
+        assert p.is_file() and not p.is_symlink() and p.resolve().is_relative_to(source)
+        relative=p.relative_to(source)
+        allowed=(len(relative.parts)==1 and (p.name in ('LICENSE','README.pdfium') or p.suffix=='.patch')) or (relative.parts[0] in ('src','include') and p.suffix in ('.c','.h'))
+        assert allowed and p.stat().st_size<=2*1024*1024
+        files['third_party/lcms/'+str(relative)]=p.read_bytes()
+    assert len(files)<100 and sum(map(len,files.values()))<16*1024*1024
+    reference=json.loads(args.reference_report.read_text());other=json.loads(args.comparison_report.read_text())
+    assert reference['directCmm']['encodedVersion']==2190 and other['directCmm']['encodedVersion']==2140
+    rows=[];comparisons=[]
+    for sr,so in zip(reference['scales'],other['scales'],strict=True):
+        assert sr['scale']==so['scale']
+        for a,b in zip(sr['probes'],so['probes'],strict=True):
+            fields=('page','profile','case','column','expectedIntentIndex','sampledComponents')
+            assert all(a[k]==b[k] for k in fields)
+            row={k:a[k] for k in fields};row.update(scale=sr['scale'],expectedRgb=a['expectedRgb'])
+            rows.append(row)
+            comparisons.append({'scale':sr['scale'],'page':a['page'],'case':a['case'],'column':a['column'],'profile':a['profile'],
+                'referenceExpectedRgb':a['expectedRgb'],'system214ExpectedRgb':b['expectedRgb'],
+                'pdfiumRgbEqual':a['actualRgb']['pdfium']==b['actualRgb']['pdfium'],
+                'pdfiumRgb':a['actualRgb']['pdfium'],'expectedRgbEqual':a['expectedRgb']==b['expectedRgb']})
+    assert len(rows)==600
+    profiles={}
+    corpus=ROOT/'tests/fixtures/pdf/rendering-intents'
+    manifest=json.loads((corpus/'manifest.json').read_text())
+    for key,item in manifest['profiles'].items():
+        path=corpus/item['file'];assert sha(path)==item['sha256']
+        files['profiles/'+path.name]=path.read_bytes();profiles[key]={'file':'profiles/'+path.name,'sha256':sha(path)}
+    expectations={'referenceReportSha256':sha(args.reference_report),'referenceDirectCmm':reference['directCmm'],
+        'comparisonReportSha256':sha(args.comparison_report),'profiles':profiles,'probes':rows,
+        'scope':'CMM numerical reproducibility only; the older shading-pattern-inherit state expectation is not adopted as PDF specification acceptance.'}
+    files['expected.json']=(json.dumps(expectations,indent=2)+'\n').encode()
+    files['intents.py']=Path(__file__).with_name('intents.py').read_bytes()
+    files['prepare_reference_cmm.py']=Path(__file__).read_bytes()
+    archive=output/'reference-lcms219.tar'
+    with tarfile.open(archive,'w',format=tarfile.PAX_FORMAT) as tar:
+        for name,data in sorted(files.items()):
+            info=tarfile.TarInfo(name);info.size=len(data);info.mode=0o644;info.mtime=0
+            tar.addfile(info,io.BytesIO(data))
+    inventory=[{'path':name,'size':len(data),'sha256':hashlib.sha256(data).hexdigest()} for name,data in sorted(files.items())]
+    record={'schemaVersion':1,'upstreamVersion':'2.19','upstreamRevision':REVISION,
+        'sourceScope':'Fixed PDFium-vendored source; allocator integration will be reversed in build copy only.',
+        'archive':archive.name,'archiveSha256':sha(archive),'files':inventory,
+        'sourceInventorySha256':json_hash([r for r in inventory if r['path'].startswith('third_party/')]),
+        'referenceReportSha256':sha(args.reference_report),'comparisonReportSha256':sha(args.comparison_report)}
+    write_json(output/'bundle.json',record)
+    write_json(output/'cross-os-before.json',{'scope':expectations['scope'],'rows':comparisons,
+        'total':len(comparisons),'allPdfiumRgbEqual':all(r['pdfiumRgbEqual'] for r in comparisons),
+        'differentExpectedRgb':sum(not r['expectedRgbEqual'] for r in comparisons)})
+    print(json.dumps({'archive':str(archive),'sha256':record['archiveSha256'],'files':len(inventory)}))
+
+
+def build(args):
+    bundle=args.bundle.resolve(strict=True);record=json.loads(bundle.read_text());archive=bundle.parent/record['archive']
+    assert sha(archive)==record['archiveSha256']
+    output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
+    source=output/'source';source.mkdir()
+    rows={r['path']:r for r in record['files']};assert len(rows)==len(record['files'])<=100
+    with tarfile.open(archive,'r:') as tar:
+        entries=tar.getmembers();assert len(entries)==len(rows)
+        seen=set()
+        for entry in entries:
+            name=entry.name;relative=Path(name)
+            assert name in rows and name not in seen and entry.isfile() and not relative.is_absolute() and '..' not in relative.parts
+            assert entry.size==rows[name]['size']<=2*1024*1024
+            seen.add(name);data=tar.extractfile(entry).read()
+            assert hashlib.sha256(data).hexdigest()==rows[name]['sha256']
+            path=source/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(data)
+    lcms=source/'third_party/lcms';cmserr=lcms/'src/cmserr.c';before=sha(cmserr)
+    patch_source=lcms/'0000-cmserr-changes.patch'
+    # This file first includes a historical patch-file diff. Select only its
+    # final, actual cmserr.c delta, never execute arbitrary packaged hooks.
+    marker='diff --git a/third_party/lcms/src/cmserr.c b/third_party/lcms/src/cmserr.c\n'
+    patch_text=patch_source.read_text();assert patch_text.count('\n'+marker)==1
+    delta=patch_text[patch_text.index('\n'+marker)+1:]
+    assert delta.count('\ndiff --git ')==0
+    patch_file=output/'restore-standard-allocator.patch';patch_file.write_text(delta)
+    commands=[]
+    def run(label,command,**kwargs):
+        result=subprocess.run(command,capture_output=True,text=True,timeout=300,**kwargs)
+        log=output/(label+'.log');log.write_text(result.stdout+result.stderr)
+        commands.append({'command':command,'exitCode':result.returncode,'log':log.name,'logSha256':sha(log)})
+        assert result.returncode==0,label+' failed; see '+str(log)
+        return result.stdout
+    run('allocator-restore',['patch','--batch','--fuzz=0','--reverse','-p1','-i',str(patch_file)],cwd=source)
+    assert 'FXMEM_' not in cmserr.read_text() and '#include "core/' not in cmserr.read_text()
+    changes=[]
+    for row in record['files']:
+        now=sha(source/row['path'])
+        if now!=row['sha256']:changes.append({'path':row['path'],'before':row['sha256'],'after':now})
+    assert len(changes)==1 and changes[0]['path']=='third_party/lcms/src/cmserr.c' and changes[0]['before']==before
+    compiler=shutil.which('gcc');assert compiler
+    version=run('compiler-version',[compiler,'--version']);target=run('compiler-target',[compiler,'-dumpmachine']).strip()
+    library_dir=output/'lib';library_dir.mkdir();library=library_dir/'liblcms2.so.2.0.19'
+    inputs=[str(p) for p in sorted((lcms/'src').glob('*.c'))];assert len(inputs)==26
+    compile_command=[compiler,'-std=c99','-O2','-fPIC','-D_POSIX_C_SOURCE=200809L','-shared',
+        '-Wl,-soname,liblcms2.so.2','-Wl,-z,defs','-I'+str(source),'-I'+str(lcms/'include'),
+        *inputs,'-lm','-lpthread','-o',str(library)]
+    run('compile',compile_command)
+    (library_dir/'liblcms2.so.2').symlink_to(library.name);(library_dir/'liblcms2.so').symlink_to(library.name)
+    dependencies=run('ldd',['ldd',str(library)]);assert 'not found' not in dependencies
+    dynamic=run('readelf',['readelf','-d',str(library)])
+    environment=os.environ.copy();environment['LD_LIBRARY_PATH']=str(library_dir)
+    assert not environment.get('LD_PRELOAD') and not environment.get('LD_AUDIT')
+    run('runtime-version',[sys.executable,'-c',
+        'import ctypes; from PIL import ImageCms; c=ctypes.CDLL("liblcms2.so.2"); print(c.cmsGetEncodedCMMversion(), ImageCms.core.littlecms_version); assert c.cmsGetEncodedCMMversion()==2190; assert ImageCms.core.littlecms_version=="2.19"'],env=environment)
+    result={'schemaVersion':1,'success':True,'bundleSha256':sha(bundle),'archiveSha256':record['archiveSha256'],
+        'sourceInventorySha256':record['sourceInventorySha256'],'upstreamVersion':'2.19','upstreamRevision':REVISION,
+        'derivedSourceChanges':changes,'allocatorPatchSha256':sha(patch_file),
+        'remainingVendorPatches':'Retained; this is a pinned PDFium-vendored reference build with only its allocator integration reversed.',
+        'compiler':{'path':compiler,'sha256':sha(Path(compiler).resolve()),'version':version,'target':target},
+        'library':{'path':str(library),'size':library.stat().st_size,'sha256':sha(library),'soname':'liblcms2.so.2'},
+        'dependencies':dependencies,'dynamicSection':dynamic,'commands':commands,
+        'osRelease':platform.freedesktop_os_release(),'pythonVersion':platform.python_version(),
+        'scope':'Isolated reference library only; no system installation or loader configuration changes.'}
+    write_json(output/'build.json',result)
+    print(json.dumps({'success':True,'library':str(library),'sha256':sha(library)}))
+
+
+def proof(args):
+    # Must start a fresh Python interpreter with the desired LD_LIBRARY_PATH;
+    # changing it after import cannot select an already loaded native library.
+    output=args.output.resolve();output.mkdir(parents=True,exist_ok=False)
+    data=json.loads(args.expected.read_text());source=args.expected.resolve().parent
+    sys.path.insert(0,str(source))
+    from intents import Cmm
+    from PIL import Image,ImageCms
+    profiles={key:(source/row['file']).read_bytes() for key,row in data['profiles'].items()}
+    assert all(sha(source/row['file'])==row['sha256'] for row in data['profiles'].values())
+    cmm=Cmm(profiles)
+    try:
+        assert cmm.version==args.version
+        paths={line.split()[-1] for line in Path('/proc/self/maps').read_text().splitlines() if 'liblcms2.so' in line and '/' in line}
+        assert len(paths)==1
+        actual=Path(next(iter(paths))).resolve()
+        if args.library:assert actual==args.library.resolve(strict=True)
+        assert ImageCms.core.littlecms_version==f'{args.version//1000}.{(args.version%1000)//10}'
+        module=Path(ImageCms.core.__file__)
+        ldd=subprocess.run(['ldd',str(module)],capture_output=True,text=True,check=True,timeout=20).stdout
+        match=re.search(r'liblcms2\.so\.2 => (\S+)',ldd);assert match and Path(match[1]).resolve()==actual
+        pillow={}
+        for key,profile in profiles.items():
+            for i in range(4):
+                pillow[key,i]=ImageCms.buildTransformFromOpenProfiles(ImageCms.ImageCmsProfile(io.BytesIO(profile)),ImageCms.createProfile('sRGB'),'CMYK','RGB',renderingIntent=i)
+        rows=[];quantized=[]
+        for p in data['probes']:
+            color=cmm.color(p['profile'],p['expectedIntentIndex'],p['sampledComponents'])
+            rows.append({**p,'actualRgb':color,'equal':color==p['expectedRgb'],
+                         'maxChannelError':max(abs(a-b) for a,b in zip(color,p['expectedRgb']))})
+        for key in profiles:
+            for i in range(4):
+                pixel=(51,128,204,77)
+                direct=cmm.color(key,i,[v/255 for v in pixel])
+                pil=list(ImageCms.applyTransform(Image.new('CMYK',(1,1),pixel),pillow[key,i]).getpixel((0,0)))
+                quantized.append({'profile':key,'intent':i,'directRgb':direct,'pillowRgb':pil,
+                                  'maxChannelError':max(abs(a-b) for a,b in zip(direct,pil))})
+        report={'schemaVersion':1,'success':all(p['equal'] for p in rows) and all(p['maxChannelError']<=1 for p in quantized),
+            'referenceReportSha256':data['referenceReportSha256'],'expectedFileSha256':sha(args.expected),
+            'expectedScope':data['scope'],'pythonVersion':platform.python_version(),'encodedVersion':cmm.version,
+            'pillowVersion':__import__('PIL').__version__,'pillowLcmsVersion':ImageCms.core.littlecms_version,
+            'loadedLibrary':{'path':str(actual),'sha256':sha(actual)},'mappedLibraries':sorted(paths),
+            'pillowExtension':{'path':str(module),'sha256':sha(module),'ldd':ldd},
+            'total':len(rows),'identical':sum(p['equal'] for p in rows),'different':sum(not p['equal'] for p in rows),
+            'pillowChecks':quantized,'probes':rows,
+            'limitations':'Numerical matching at the declared probes, not independent-CMM or PDF semantic correctness.'}
+        write_json(output/'proof.json',report)
+        print(json.dumps({k:report[k] for k in ('success','encodedVersion','pillowLcmsVersion','total','identical','different')}))
+        if args.version==2190:assert report['success'],'Pinned CMM differs from host reference'
+    finally:cmm.close()
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__);commands=parser.add_subparsers(dest='command',required=True)
+    p=commands.add_parser('pack');p.add_argument('--source',type=Path,required=True);p.add_argument('--reference-report',type=Path,required=True);p.add_argument('--comparison-report',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=pack)
+    p=commands.add_parser('build');p.add_argument('--bundle',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.set_defaults(function=build)
+    p=commands.add_parser('proof');p.add_argument('--expected',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--version',type=int,choices=(2140,2190),required=True);p.add_argument('--library',type=Path);p.set_defaults(function=proof)
+    args=parser.parse_args();args.function(args)
+
+
+if __name__=='__main__':main()