194 lines
13 KiB
Python
194 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Fixed input sequences and explicit close/reopen memory measurements.
|
|
|
|
Use a dedicated display or Xvfb. Chromium and worker sandboxes stay enabled.
|
|
Runtime completion is distinct from budget observations and target acceptance.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import pathlib
|
|
import statistics
|
|
import subprocess
|
|
import tempfile
|
|
|
|
root = pathlib.Path(__file__).resolve().parents[1]
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--binary', type=pathlib.Path, default=root/'build/docview')
|
|
p.add_argument('--runs', type=int, default=30)
|
|
p.add_argument('--operations', type=int, default=100, help='operations in each applicable navigation series')
|
|
p.add_argument('--close-cycles', type=int, default=10)
|
|
p.add_argument('--scroll-steps', type=int, default=240)
|
|
p.add_argument('--cold-process-runs', type=int, default=0, help='fresh process and fresh XDG app-cache opens; OS file cache retained')
|
|
p.add_argument('--formats', nargs='+', default=['pdf', 'html', 'zip', 'epub'])
|
|
p.add_argument('--corpus', choices=['smoke', 'standard', 'fonts'], default='smoke')
|
|
p.add_argument('--output', type=pathlib.Path, default=root/'tests/results/performance')
|
|
a = p.parse_args()
|
|
if not 0 <= a.cold_process_runs <= 100:
|
|
p.error('--cold-process-runs must be between 0 and 100')
|
|
a.output.mkdir(parents=True, exist_ok=True)
|
|
|
|
def file_hash(path):
|
|
with path.open('rb') as source:
|
|
return hashlib.file_digest(source, 'sha256').hexdigest()
|
|
|
|
def stats(samples):
|
|
if not samples:
|
|
return {'n': 0}
|
|
ordered = sorted(samples)
|
|
return {'n': len(samples), 'p50_ms': statistics.median(samples),
|
|
'p95_ms': ordered[max(0, math.ceil(len(ordered)*.95)-1)],
|
|
'min_ms': ordered[0], 'max_ms': ordered[-1]}
|
|
|
|
def budget(stats_, limit):
|
|
return {'limit_ms': limit, 'observed_met': stats_['p95_ms'] <= limit} if stats_.get('n') else {'status': 'not-measured', 'limit_ms': limit}
|
|
|
|
executables = {name: file_hash(a.binary.resolve().parent / name) for name in
|
|
[a.binary.name, 'docview-pdf-worker', 'docview-archive-worker']}
|
|
runner_hash = file_hash(pathlib.Path(__file__))
|
|
fixtures = {'pdf': 'pdf/navigation.pdf', 'html': 'web/html/index.html', 'zip': 'web/html-book.zip', 'epub': 'web/reflow-rtl.epub'}
|
|
if a.corpus == 'standard':
|
|
fixtures = {'pdf': 'performance/standard-500.pdf', 'html': 'performance/html/index.html',
|
|
'zip': 'performance/standard-html.zip', 'epub': 'performance/standard-500.epub',
|
|
'pdf-headings': 'performance/standard-headings-500.pdf'}
|
|
if 'pdf' in a.formats and 'pdf-headings' not in a.formats:
|
|
a.formats.append('pdf-headings')
|
|
elif a.corpus == 'fonts':
|
|
fixtures = {'pdf': 'pdf/fonts/unembedded-japanese.pdf'}
|
|
a.formats = ['pdf']
|
|
unknown = set(a.formats) - fixtures.keys()
|
|
if unknown:
|
|
p.error('formats unavailable in this corpus: '+', '.join(sorted(unknown)))
|
|
# Clear this invocation's selected evidence before starting any process. A
|
|
# failure or interruption must not leave later selected cases from an old run.
|
|
for kind in a.formats:
|
|
(a.output/(kind+'.json')).unlink(missing_ok=True)
|
|
cold_directory = a.output/'cold'/kind
|
|
for old in cold_directory.glob('[0-9][0-9][0-9].json'):
|
|
old.unlink()
|
|
failures = []
|
|
with tempfile.TemporaryDirectory(prefix='docview-benchmark-') as temporary:
|
|
directory = pathlib.Path(temporary)
|
|
config = directory/'config.toml'
|
|
config.write_text('schema_version=1\n[privacy]\nremember_position=false\nrecent_documents=0\n', encoding='utf-8')
|
|
env = os.environ.copy()
|
|
for name, leaf in [('XDG_CONFIG_HOME', 'config'), ('XDG_STATE_HOME', 'state'), ('XDG_CACHE_HOME', 'cache')]:
|
|
env[name] = str(directory/leaf)
|
|
for kind in a.formats:
|
|
result = a.output/(kind+'.json')
|
|
# Never reuse a previous success when this invocation fails before it
|
|
# produces a record (including a timeout or an executable startup error).
|
|
result.unlink(missing_ok=True)
|
|
source = root/'tests/fixtures'/fixtures[kind]
|
|
source_hash = file_hash(source)
|
|
command = [str(a.binary.resolve()), '--config', str(config), '--benchmark-output', str(result.resolve()),
|
|
'--benchmark-runs', str(a.runs), '--benchmark-operations', str(a.operations),
|
|
'--benchmark-close-cycles', str(a.close_cycles), '--benchmark-scroll-steps', str(a.scroll_steps), str(source)]
|
|
try:
|
|
completed = subprocess.run(command, env=env, check=False, timeout=max(180, a.runs*3+a.operations*6+a.scroll_steps*.05))
|
|
except subprocess.TimeoutExpired:
|
|
failures.append(kind+': process timeout')
|
|
continue
|
|
except OSError:
|
|
failures.append(kind+': executable startup failed')
|
|
continue
|
|
if not result.exists():
|
|
failures.append(kind+': no measurement record')
|
|
continue
|
|
values = json.loads(result.read_text())
|
|
values['harnessReportedSuccess'] = values.get('success', False)
|
|
values['success'] = values.get('success', False) and completed.returncode == 0
|
|
values['processExitCode'] = completed.returncode
|
|
actions = values['operations']
|
|
summary = {'open': stats(values['openFrameMs']),
|
|
'response': stats([v['firstResponseFrameMs'] for v in actions if 'firstResponseFrameMs' in v]),
|
|
'final_quality': stats([v['finalQualityFrameMs'] for v in actions if 'finalQualityFrameMs' in v])}
|
|
summary['cached_pdf_navigation'] = stats([v['finalQualityFrameMs'] for v in actions if v['command'] == 'nav.page' and v.get('cacheHit') is True and v.get('positionChanged') is True])
|
|
summary['uncached_pdf_navigation'] = stats([v['finalQualityFrameMs'] for v in actions if v['command'] == 'nav.page' and v.get('cacheHit') is False and v.get('positionChanged') is True])
|
|
summary['excluded_noop_page_navigation'] = sum(v['command'] == 'nav.page' and v.get('positionChanged') is not True for v in actions)
|
|
summary['first_open_in_process'] = stats(values['openFrameMs'][:1])
|
|
summary['same_process_document_reopen'] = stats(values['openFrameMs'][1:])
|
|
values['openTimingMode'] = 'First sample opens in a fresh application process; subsequent samples explicitly close/reopen the document while retaining process-global/browser/font and OS caches.'
|
|
cold_samples, cold_failures, cold_maximum_pressure = [], 0, 0
|
|
for trial in range(a.cold_process_runs):
|
|
cold_output = a.output/'cold'/kind/f'{trial + 1:03}.json'
|
|
cold_output.parent.mkdir(parents=True, exist_ok=True)
|
|
cold_output.unlink(missing_ok=True)
|
|
with tempfile.TemporaryDirectory(prefix='docview-cold-') as cold:
|
|
cold_env = env.copy()
|
|
for variable, leaf in [('XDG_CONFIG_HOME', 'config'), ('XDG_STATE_HOME', 'state'), ('XDG_CACHE_HOME', 'cache')]:
|
|
cold_env[variable] = str(pathlib.Path(cold)/leaf)
|
|
cold_command = [str(a.binary.resolve()), '--config', str(config), '--benchmark-output', str(cold_output.resolve()),
|
|
'--benchmark-runs', '1', '--benchmark-operations', '0', '--benchmark-close-cycles', '0', '--benchmark-scroll-steps', '0', str(source)]
|
|
try:
|
|
cold_run = subprocess.run(cold_command, env=cold_env, check=False, timeout=45)
|
|
cold_value = json.loads(cold_output.read_text()) if cold_output.exists() else {}
|
|
except (subprocess.TimeoutExpired, json.JSONDecodeError, OSError):
|
|
cold_run, cold_value = None, {}
|
|
pressure = cold_value.get('maximumMemoryPressureLevel')
|
|
if isinstance(pressure, int):
|
|
cold_maximum_pressure = max(cold_maximum_pressure, pressure)
|
|
if cold_run is None or cold_run.returncode or not cold_value.get('success') or pressure != 0 or len(cold_value.get('openFrameMs', [])) != 1:
|
|
cold_failures += 1
|
|
else:
|
|
cold_samples.append(cold_value['openFrameMs'][0])
|
|
summary['fresh_process_fresh_app_cache_open'] = stats(cold_samples)
|
|
values['coldProcessOpen'] = {'requested': a.cold_process_runs, 'failed': cold_failures,
|
|
'maximumMemoryPressureLevel': cold_maximum_pressure,
|
|
'samplesMs': cold_samples, 'status': 'measured' if a.cold_process_runs else 'not-measured',
|
|
'method': 'New application process and empty XDG config/state/cache roots per sample. System fonts and OS file cache retained; not fully OS-cold. Qt startup precedes the timed file-open event.'}
|
|
if cold_failures: failures.append(kind+': cold process open failure')
|
|
summary['navigation_series'] = {name: stats([v['finalQualityFrameMs'] for v in actions if v.get('series') == name])
|
|
for name in ['page-scroll', 'heading', 'chapter']}
|
|
summary['continuous_scroll_frames'] = stats(values.get('continuousScroll', {}).get('frameIntervalMs', []))
|
|
summary['continuous_scroll_input_dispatch'] = stats(values.get('continuousScroll', {}).get('dispatchIntervalMs', []))
|
|
summary['ui_timer_intervals'] = stats(values.get('uiTimerIntervalMs', []))
|
|
expected = values.get('uiTimerExpectedIntervalMs', 16)
|
|
summary['ui_timer_excess_delay_ms'] = max([0]+[max(0, v-expected) for v in values.get('uiTimerIntervalMs', [])])
|
|
cycles = values.get('openCloseCycles', [])
|
|
summary['explicit_close_cycles'] = len(cycles)
|
|
uniform_cycles = [v for v in cycles if v.get('workload') == 'open-first-visible-only']
|
|
summary['uniform_open_close_cycles'] = len(uniform_cycles)
|
|
summary['all_closes_empty'] = bool(cycles) and all(v['stateEmpty'] and v['cacheBytesAfterClose'] == 0 for v in cycles)
|
|
for name in ['applicationBytes', 'processGroupBytes']:
|
|
samples = [v['afterClose'][name] for v in uniform_cycles if v['afterClose'].get('available')]
|
|
if samples:
|
|
summary['post_close_'+name] = {'n': len(samples), 'first': samples[0], 'last': samples[-1],
|
|
'min': min(samples), 'max': max(samples), 'last_minus_first': samples[-1]-samples[0],
|
|
'interpretation': 'Finite repeated-close observation; retained RSS alone is not a leak verdict.'}
|
|
values['summary'] = summary
|
|
values['navigationCoverage'] = {v['series']: {'status': v['status'], 'measured': v['count'],
|
|
'minimum100Operations': v['count'] >= 100 if v['status'] == 'measured' else None}
|
|
for v in values.get('operationSeries', [])}
|
|
values['proposedBudgetObservations'] = {
|
|
'visual_response': budget(summary['response'], 50),
|
|
'app_cache_cold_open': budget(summary['fresh_process_fresh_app_cache_open'], 2000 if kind in ('pdf', 'pdf-headings', 'html') else 5000),
|
|
'cached_pdf_navigation': budget(summary['cached_pdf_navigation'], 100),
|
|
'uncached_pdf_navigation': budget(summary['uncached_pdf_navigation'], 500),
|
|
'continuous_scroll': budget(summary['continuous_scroll_frames'], 33),
|
|
'process_group_memory': {'limit_bytes': 1024**3, 'observed_met': values['processGroupRssPeakBytes'] <= 1024**3} if values['processGroupRssPeakBytes'] >= 0 else {'status': 'not-measured'},
|
|
'tile_cache_within_configured_budget': values['tileCachePeakChargedBytes'] <= values['tileCacheBudgetBytes'],
|
|
'acceptance': 'Informational comparison with original proposed budgets; not reference-hardware/target-OS acceptance.'}
|
|
values['originalUnchanged'] = source_hash == file_hash(source) == values['documentSha256']
|
|
values['corpus'] = a.corpus
|
|
values['executableSha256'] = executables
|
|
values['runnerSha256'] = runner_hash
|
|
checks = {'requested_open_timings': len(values['openFrameMs']) == a.runs,
|
|
'full_resolution_conditions': values.get('maximumMemoryPressureLevel') == 0 and cold_maximum_pressure == 0,
|
|
'uniform_open_close_cycles': len(uniform_cycles) >= a.close_cycles,
|
|
'close_releases_document': summary['all_closes_empty'],
|
|
'source_unchanged': values['originalUnchanged'],
|
|
'navigation_moved': all(v.get('positionChanged') is True for v in actions if v.get('series') in ('heading', 'chapter')),
|
|
'page_navigation_moved': summary['excluded_noop_page_navigation'] == 0,
|
|
'scroll_dispatched': values.get('continuousScroll', {}).get('dispatchedInputs') == a.scroll_steps}
|
|
values['measurementChecks'] = checks
|
|
values['success'] = values['success'] and not cold_failures and all(checks.values())
|
|
if not values['success']:
|
|
failures.append(kind+': measurement failed')
|
|
result.write_text(json.dumps(values, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
|
|
print(kind, json.dumps(summary), flush=True)
|
|
if failures:
|
|
raise SystemExit('; '.join(failures))
|