Files
2026-09-21 13:41:40 +09:00

231 lines
18 KiB
Python

#!/usr/bin/env python3
"""Deterministic, self-authored tagged-PDF boundary corpus (no external tools)."""
from pathlib import Path
import hashlib
import json
import zlib
ROOT = Path(__file__).parent / 'headings'
ROOT.mkdir(exist_ok=True)
def stream(text, extra=''):
data = text.encode('ascii')
return f'<< /Length {len(data)} {extra} >>\nstream\n'.encode() + data + b'\nendstream'
def write(name, objects, expected):
data = bytearray(b'%PDF-1.7\n%\xe2\xe3\xcf\xd3\n')
offsets = [0]
for number, obj in enumerate(objects, 1):
offsets.append(len(data))
data += f'{number} 0 obj\n'.encode() + (obj.encode('ascii') if isinstance(obj, str) else obj) + b'\nendobj\n'
xref = len(data)
data += f'xref\n0 {len(offsets)}\n0000000000 65535 f \n'.encode()
for offset in offsets[1:]:
data += f'{offset:010d} 00000 n \n'.encode()
data += f'trailer\n<< /Size {len(offsets)} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n'.encode()
(ROOT / name).write_bytes(data)
return {'file': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'expected': expected}
font = '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'
records = []
records.append(write('transformed-form.pdf', [
'<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 7 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 260] /StructParents 0 /Resources << /XObject << /Fm 6 0 R >> >> /Contents 4 0 R >>',
stream('/H2 <</MCID 0>> BDC q 1.5 0 0 1.25 55 70 cm /Fm Do Q EMC'),
font,
stream('/H2 <</MCID 4>> BDC BT /F1 16 Tf 10 30 Td (Form heading) Tj ET EMC', '/Type /XObject /Subtype /Form /BBox [0 0 160 80] /StructParents 1 /Resources << /Font << /F1 5 0 R >> >>'),
'<< /Type /StructTreeRoot /K [8 0 R] /ParentTree << /Nums [0 [8 0 R] 1 [null null null null 8 0 R]] >> /ParentTreeNextKey 2 >>',
'<< /Type /StructElem /S /H2 /P 7 0 R /Pg 3 0 R /T (Form heading) /K [0 << /Type /MCR /Pg 3 0 R /Stm 6 0 R /MCID 4 >>] >>'
], {'pages': 1, 'headings': [{'page': 0, 'title': 'Form heading', 'level': 2, 'precision': 'exact', 'text_region_pt': [70, 102, 240, 130]}], 'form_matrix': [1.5, 0, 0, 1.25, 55, 70], 'local_baseline': [10, 30]}))
records.append(write('rotated-tags.pdf', [
'<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 6 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 260] /CropBox [20 30 280 230] /Rotate 90 /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>',
stream('/H3 <</MCID 1>> BDC BT /F1 16 Tf 60 80 Td (Rotated second) Tj ET EMC\n/H1 <</MCID 0>> BDC BT /F1 20 Tf 50 170 Td (Rotated first) Tj ET EMC'),
font,
'<< /Type /StructTreeRoot /K [7 0 R 8 0 R] /ParentTree << /Nums [0 [7 0 R 8 0 R]] >> >>',
'<< /Type /StructElem /S /H1 /P 6 0 R /Pg 3 0 R /K 0 >>',
'<< /Type /StructElem /S /H3 /P 6 0 R /Pg 3 0 R /K 1 >>'
], {'pages': 1, 'crop_box': [20, 30, 280, 230], 'rotation': 90, 'headings': [
{'page': 0, 'title': 'Rotated first', 'level': 1, 'precision': 'exact', 'text_region_pt': [50, 168, 170, 190]},
{'page': 0, 'title': 'Rotated second', 'level': 3, 'precision': 'exact', 'text_region_pt': [60, 78, 180, 96]}
], 'order': 'structure order, intentionally different from content-stream draw order'}))
records.append(write('cross-page-structure.pdf', [
'<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 8 0 R >>',
'<< /Type /Pages /Count 2 /Kids [3 0 R 4 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 260] /StructParents 0 /Resources << /Font << /F1 7 0 R >> >> /Contents 5 0 R >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 260] /StructParents 1 /Resources << /Font << /F1 7 0 R >> >> /Contents 6 0 R >>',
stream('/Span <</MCID 0>> BDC BT /F1 16 Tf 40 60 Td (Across page one) Tj ET EMC\n/H1 <</MCID 1>> BDC BT /F1 16 Tf 170 60 Td ( middle) Tj ET EMC'),
stream('/P <</MCID 2>> BDC BT /F1 16 Tf 40 120 Td (Unrelated paragraph) Tj ET EMC\n/H2 <</MCID 1>> BDC BT /F1 16 Tf 40 160 Td (Following heading) Tj ET EMC\n/Span <</MCID 0>> BDC BT /F1 16 Tf 40 210 Td (Across page two) Tj ET EMC'),
font,
'<< /Type /StructTreeRoot /K [9 0 R] /ParentTree << /Nums [0 [12 0 R 10 0 R] 1 [13 0 R 11 0 R 14 0 R]] >> /ParentTreeNextKey 2 >>',
'<< /Type /StructElem /S /Sect /P 8 0 R /K [10 0 R 11 0 R 14 0 R] >>',
'<< /Type /StructElem /S /H1 /P 9 0 R /T (Across pages) /K [12 0 R << /Type /MCR /Pg 3 0 R /MCID 1 >> 13 0 R] >>',
'<< /Type /StructElem /S /H2 /P 9 0 R /Pg 4 0 R /K 1 >>',
'<< /Type /StructElem /S /Span /P 10 0 R /Pg 3 0 R /K 0 >>',
'<< /Type /StructElem /S /Span /P 10 0 R /Pg 4 0 R /K 0 >>',
'<< /Type /StructElem /S /P /P 9 0 R /Pg 4 0 R /K 2 >>'
], {'pages': 2, 'headings': [
{'page': 0, 'title': 'Across page one middle', 'level': 1, 'precision': 'exact', 'text_region_pt': [40, 55, 230, 78]},
{'page': 1, 'title': 'Across page two', 'level': 1, 'precision': 'exact', 'text_region_pt': [40, 205, 170, 228]},
{'page': 1, 'title': 'Following heading', 'level': 2, 'precision': 'exact', 'text_region_pt': [40, 155, 180, 178]}
], 'scope': 'One heading spans two page-local Span elements with an interleaved direct MCR; MCID 0 is reused independently on both pages. Results are page-local fragments.'}))
records.append(write('vector-heading.pdf', [
'<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 5 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 260] /StructParents 0 /Resources << >> /Contents 4 0 R >>',
stream('/H4 <</MCID 0>> BDC 0 0 1 rg 40 180 150 20 re f EMC'),
'<< /Type /StructTreeRoot /K [6 0 R] /ParentTree << /Nums [0 [6 0 R]] >> >>',
'<< /Type /StructElem /S /H4 /P 5 0 R /Pg 3 0 R /ActualText (Vector section) /K 0 >>'
], {'pages': 1, 'headings': [{'page': 0, 'title': 'Vector section', 'level': 4, 'precision': 'page', 'reason_contains': '座標'}], 'drawn_region_pt': [40, 180, 190, 200], 'fallback': 'No text glyph bounding box is available for the marked vector path.'}))
records.append(write('form-only-structure.pdf', [
'<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 7 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 260] /Resources << /XObject << /Fm 6 0 R >> >> /Contents 4 0 R >>',
stream('q 1 0 0 1 40 100 cm /Fm Do Q'),
font,
stream('/H2 <</MCID 0>> BDC BT /F1 16 Tf 10 30 Td (Form only heading) Tj ET EMC', '/Type /XObject /Subtype /Form /BBox [0 0 200 80] /StructParents 0 /Resources << /Font << /F1 5 0 R >> >>'),
'<< /Type /StructTreeRoot /K [8 0 R] /ParentTree << /Nums [0 [8 0 R]] >> /ParentTreeNextKey 1 >>',
'<< /Type /StructElem /S /H2 /P 7 0 R /Pg 3 0 R /T (Form only heading) /K << /Type /MCR /Pg 3 0 R /Stm 6 0 R /MCID 0 >> >>'
], {'pages': 1, 'headings': [{'page': 0, 'title': 'Form only heading', 'level': 2, 'precision': 'exact', 'text_region_pt': [50, 125, 190, 147]}],
'scope': 'Form StructParents and source-qualified MCR; no page StructParents.'}))
records.append(write('form-vector-mcid-collision.pdf', [
'<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 7 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 260] /StructParents 0 /Resources << /XObject << /Fm 6 0 R >> >> /Contents 4 0 R >>',
stream('/H1 <</MCID 0>> BDC 0 0 1 rg 40 180 180 20 re f EMC q 1 0 0 1 40 100 cm /Fm Do Q'),
font,
stream('/P <</MCID 0>> BDC BT /F1 12 Tf 10 30 Td (Unrelated Form paragraph) Tj ET EMC', '/Type /XObject /Subtype /Form /BBox [0 0 240 80] /StructParents 1 /Resources << /Font << /F1 5 0 R >> >>'),
'<< /Type /StructTreeRoot /K [8 0 R 9 0 R] /ParentTree << /Nums [0 [8 0 R] 1 [9 0 R]] >> /ParentTreeNextKey 2 >>',
'<< /Type /StructElem /S /H1 /P 7 0 R /Pg 3 0 R /T (Page vector heading) /K 0 >>',
'<< /Type /StructElem /S /P /P 7 0 R /Pg 3 0 R /K << /Type /MCR /Pg 3 0 R /Stm 6 0 R /MCID 0 >> >>'
], {'pages': 1, 'headings': [{'page': 0, 'title': 'Page vector heading', 'level': 1, 'precision': 'page', 'reason_contains': 'MCID'}],
'fallback': 'MCID 0 names a page vector and unrelated Form text. Non-text page objects must participate in stream collision detection.'}))
def form_case(name, forms, content, expected, *, rolemap='', root_order=None, mutate=None):
"""Create source-qualified MCRs with an indirect ParentTree Kids node."""
objects = ['<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 6 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>', '', stream(content), font, '']
heading_ids = [7 + len(forms) + i for i, form in enumerate(forms)]
names = ' '.join(f'/Fm{i} {7+i} 0 R' for i in range(len(forms)))
objects[2] = f'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 350] /Resources << /XObject << {names} >> >> /Contents 4 0 R >>'
for i, form in enumerate(forms):
extra = f'/Type /XObject /Subtype /Form /BBox [0 0 260 90] /StructParents {i} /Resources << /Font << /F1 5 0 R >> /XObject << {names} >> /Properties << /HeadingProperty << /MCID 0 >> >> >>'
if 'matrix' in form: extra += f' /Matrix [{form["matrix"]}]'
objects.append(stream(form['content'], extra))
for i, form in enumerate(forms):
title = f'/T ({form["title"]})' if 'title' in form else ''
objects.append(f'<< /Type /StructElem /S /{form.get("role", "H2")} /P 6 0 R /Pg 3 0 R {title} /K << /Type /MCR /Pg 3 0 R /Stm {7+i} 0 R /MCID 0 >> >>')
parent_id = len(objects) + 1
nums = ' '.join(f'{i} [{heading_ids[i]} 0 R]' for i in range(len(forms)))
objects.append(f'<< /Limits [0 {len(forms)-1}] /Nums [{nums}] >>')
order = root_order if root_order is not None else range(len(forms))
children = ' '.join(f'{heading_ids[i]} 0 R' for i in order)
objects[5] = f'<< /Type /StructTreeRoot /K [{children}] /ParentTree << /Kids [{parent_id} 0 R] >> {rolemap} >>'
if mutate: mutate(objects, heading_ids)
records.append(write(name, objects, {'pages': 1, **expected}))
def tagged(label, tag='H2'):
return f'/{tag} <</MCID 0>> BDC BT /F1 12 Tf 10 30 Td ({label}) Tj ET EMC'
def exact(title, region, level=2):
return {'page': 0, 'title': title, 'level': level, 'precision': 'exact', 'text_region_pt': region}
def fallback(title, reason):
return {'page': 0, 'title': title, 'level': 2, 'precision': 'page', 'reason_contains': reason}
form_case('named-property-rolemap.pdf', [{'content': '/Section /HeadingProperty BDC BT /F1 12 Tf 10 30 Td (Named property heading) Tj ET EMC', 'role': 'Section'}],
'q 1 0 0 1 40 100 cm /Fm0 Do Q', {'headings': [exact('Named property heading', [50, 127, 190, 142], 3)]}, rolemap='/RoleMap << /Section /H3 >>')
form_case('nested-form-matrix.pdf', [{'content': 'q 1.5 0 0 1.25 10 20 cm /Fm1 Do Q', 'matrix': '1.2 0 0 0.9 5 7'},
{'content': tagged('Nested heading'), 'matrix': '1 0 0 1 4 6'}],
'q 1 0 0 1 40 100 cm /Fm0 Do Q', {'headings': [exact('Nested heading', [82, 161, 245, 181])]})
form_case('different-form-same-mcid.pdf', [{'content': tagged('First form')}, {'content': tagged('Second form')}],
'q 1 0 0 1 30 180 cm /Fm0 Do Q q 1 0 0 1 80 60 cm /Fm1 Do Q',
{'headings': [exact('Second form', [90, 87, 170, 102]), exact('First form', [40, 207, 105, 222])]}, root_order=[1, 0])
form_case('ambiguous-form-matrix.pdf', [{'content': tagged('Do not borrow first'), 'title': 'First structure'}, {'content': tagged('Do not borrow second'), 'title': 'Second structure'}],
'q 1 0 0 1 40 100 cm /Fm0 Do /Fm1 Do Q',
{'headings': [fallback('First structure', '一意'), fallback('Second structure', '一意')]})
form_case('repeated-form.pdf', [{'content': tagged('Repeated heading')}],
'q 1 0 0 1 30 180 cm /Fm0 Do Q q 1 0 0 1 80 60 cm /Fm0 Do Q',
{'headings': [fallback('Repeated heading', '複数回')]})
form_case('unused-form-resource.pdf', [{'content': tagged('Used heading')}, {'content': tagged('Unused heading')}],
'q 1 0 0 1 40 100 cm /Fm0 Do Q', {'headings': [exact('Used heading', [50, 127, 140, 142])]})
form_case('inline-image-before-form.pdf', [{'content': tagged('After image')}],
'q 1 0 0 1 40 100 cm BI /W 1 /H 1 /BPC 8 /CS /G ID X EI /Fm0 Do Q',
{'headings': [exact('After image', [50, 127, 125, 142])]})
limited = {'headings': [], 'limited': True}
form_case('cyclic-form.pdf', [{'content': '/Fm0 Do'}], '/Fm0 Do', limited)
form_case('cyclic-rolemap.pdf', [{'content': tagged('Hidden by cycle'), 'role': 'Foo'}], '/Fm0 Do', limited, rolemap='/RoleMap << /Foo /Bar /Bar /Foo >>')
form_case('cyclic-parent.pdf', [{'content': tagged('Parent cycle')}], '/Fm0 Do', limited,
mutate=lambda o, ids: o.__setitem__(ids[0]-1, o[ids[0]-1].replace('/P 6 0 R', f'/P {ids[0]} 0 R')))
form_case('cyclic-children.pdf', [{'content': tagged('Child cycle')}], '/Fm0 Do', limited,
mutate=lambda o, ids: o.__setitem__(ids[0]-1, f'<< /Type /StructElem /S /H2 /P 6 0 R /Pg 3 0 R /K {ids[0]} 0 R >>'))
form_case('broken-mcr-page.pdf', [{'content': tagged('Wrong page')}], '/Fm0 Do', limited,
mutate=lambda o, ids: o.__setitem__(ids[0]-1, o[ids[0]-1].replace('/Type /MCR /Pg 3 0 R', '/Type /MCR /Pg 7 0 R')))
form_case('broken-mcr-stream.pdf', [{'content': tagged('Wrong stream')}], '/Fm0 Do', limited,
mutate=lambda o, ids: o.__setitem__(ids[0]-1, o[ids[0]-1].replace('/Stm 7 0 R', '/Stm 3 0 R')))
form_case('annotation-owned-mcr.pdf', [{'content': tagged('Annotation-owned')}], '/Fm0 Do', limited,
mutate=lambda o, ids: o.__setitem__(ids[0]-1, o[ids[0]-1].replace('/Stm 7 0 R', '/Stm 7 0 R /StmOwn 3 0 R')))
form_case('broken-graphics-stack.pdf', [{'content': tagged('Graphics stack')}], 'Q /Fm0 Do', limited)
form_case('cyclic-number-tree.pdf', [{'content': tagged('Number tree')}], '/Fm0 Do', limited,
mutate=lambda o, ids: o.__setitem__(len(o)-1, f'<< /Limits [0 0] /Kids [{len(o)} 0 R] >>'))
form_case('conflicting-parent-tree-owner.pdf', [{'content': tagged('First owner')}, {'content': tagged('Second owner')}],
'q 1 0 0 1 30 180 cm /Fm0 Do Q q 1 0 0 1 80 60 cm /Fm1 Do Q', limited,
mutate=lambda o, ids: o.__setitem__(ids[0]-1, o[ids[0]-1].replace('/Stm 7 0 R', '/Stm 8 0 R')))
def compressed_content(size):
def mutate(objects, ids):
body = ('q 1 0 0 1 40 100 cm /Fm0 Do Q').encode()
data = zlib.compress(b' ' * (size-len(body)) + body)
objects[3] = f'<< /Length {len(data)} /Filter /FlateDecode >>\nstream\n'.encode() + data + b'\nendstream'
return mutate
form_case('stream-quota-at-limit.pdf', [{'content': tagged('At stream limit')}], '',
{'headings': [exact('At stream limit', [50, 127, 140, 142])]}, mutate=compressed_content(16*1024*1024))
form_case('stream-quota-over-limit.pdf', [{'content': tagged('Over stream limit')}], '',
{'headings': [], 'limited': True, 'truncated': True}, mutate=compressed_content(16*1024*1024+1))
long_title = '<feff' + ('あ' * 4090).encode('utf-16-be').hex() + '>'
many = ['<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 5 0 R >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 350] /StructParents 0 /Resources << >> /Contents 4 0 R >>',
stream('\n'.join(f'/H2 <</MCID {i}>> BDC 10 {i*4} 20 2 re f EMC' for i in range(70))), '']
for i in range(70):
many.append(f'<< /Type /StructElem /S /H2 /P 5 0 R /Pg 3 0 R /K {i} /ActualText {long_title} >>')
refs = ' '.join(f'{i+6} 0 R' for i in range(70))
many[4] = f'<< /Type /StructTreeRoot /K [{refs}] /ParentTree << /Nums [0 [{refs}]] >> >>'
records.append(write('bounded-heading-response.pdf', many, {'pages': 1, 'headings': [], 'response_limited': True}))
deep = ['<< /Type /Catalog /Pages 2 0 R >>', '<< /Type /Pages /Count 2 /Kids [3 0 R 70 0 R] >>']
for obj in range(3, 69):
deep.append(f'<< /Type /Pages /Parent {obj-1} 0 R /Count 1 /Kids [{obj+1} 0 R] >>')
deep.append('<< /Type /Page /Parent 68 0 R /MediaBox [0 0 100 100] /Resources << >> >>')
deep.append('<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << >> >>')
context_record = write('context-deep-tree.pdf', deep, {'scope': 'Context-only regression: after a depth failure, the second page must never be assigned index zero.'})
context_records = [context_record]
for count in (32, 33):
context_records.append(write(f'context-warnings-{count}.pdf', [
f'<< /Type /Catalog /Pages 2 0 R /Duplicate 1 /Duplicate 2 /Probe [' + ' '.join(f'{i+4} 0 R' for i in range(count-1)) + '] >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Resources << >> >>'] +
['<< /Duplicate 1 /Duplicate 2 >>'] * (count-1),
{'scope': 'Context-only cumulative diagnostic limit, starting in initial parse; warning bodies must never reach stdout/stderr.', 'accepted': count == 32}))
long_key = '/' + 'A' * (64*1024)
context_records.append(write('context-warning-bytes.pdf', [
f'<< /Type /Catalog /Pages 2 0 R {long_key} 1 {long_key} 2 >>',
'<< /Type /Pages /Count 1 /Kids [3 0 R] >>',
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Resources << >> >>'],
{'scope': 'Context-only diagnostic byte limit during initial parse; no warning text is retained or emitted.', 'accepted': False}))
(ROOT / 'manifest.json').write_text(json.dumps({'origin': 'Self-created deterministic PDF syntax, Helvetica Standard-14 text and vector paths; no third-party document content.', 'generator': '../generate_headings.py', 'documents': records, 'context_documents': context_records}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(f'Generated {len(records)} heading fixtures in {ROOT}')