100 lines
5.4 KiB
Python
100 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Runner failure-evidence regressions; these are not performance measurements."""
|
|
import json
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import runpy
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
RUNNER = Path(__file__).with_name('benchmark.py')
|
|
|
|
class BenchmarkRunnerTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name)
|
|
self.binary = self.root/'docview'
|
|
self.binary.write_text('#!/bin/sh\nexit 7\n')
|
|
self.binary.chmod(0o700)
|
|
for name in ['docview-pdf-worker', 'docview-archive-worker']:
|
|
(self.root/name).write_bytes(b'fixture')
|
|
self.output = self.root/'results'
|
|
self.output.mkdir()
|
|
self.result = self.output/'pdf.json'
|
|
self.result.write_text(json.dumps({'success': True, 'executableSha256': {'docview': 'old'}}))
|
|
self.arguments = [str(RUNNER), '--binary', str(self.binary), '--formats', 'pdf', '--runs', '1',
|
|
'--operations', '0', '--close-cycles', '0', '--scroll-steps', '0', '--output', str(self.output)]
|
|
|
|
@unittest.skipUnless(os.name == 'posix', 'POSIX fixture executable')
|
|
def test_startup_failure_cannot_relabel_previous_success(self):
|
|
result = subprocess.run([sys.executable, *self.arguments], capture_output=True, text=True)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn('no measurement record', result.stderr)
|
|
self.assertFalse(self.result.exists())
|
|
|
|
def test_timeout_cannot_leave_previous_success(self):
|
|
with patch.object(sys, 'argv', self.arguments), patch('subprocess.run', side_effect=subprocess.TimeoutExpired(['fixture'], 1)):
|
|
with self.assertRaises(SystemExit) as failure:
|
|
runpy.run_path(str(RUNNER), run_name='__main__')
|
|
self.assertIn('process timeout', str(failure.exception))
|
|
self.assertFalse(self.result.exists())
|
|
|
|
def test_os_startup_error_clears_all_selected_previous_evidence(self):
|
|
other = self.output/'html.json'
|
|
other.write_text('{"success":true}')
|
|
cold = self.output/'cold/pdf'
|
|
cold.mkdir(parents=True)
|
|
(cold/'030.json').write_text('{"success":true}')
|
|
arguments = self.arguments + ['--formats', 'pdf', 'html']
|
|
with patch.object(sys, 'argv', arguments), patch('subprocess.run', side_effect=OSError('fixture')):
|
|
with self.assertRaises(SystemExit) as failure:
|
|
runpy.run_path(str(RUNNER), run_name='__main__')
|
|
self.assertIn('executable startup failed', str(failure.exception))
|
|
self.assertFalse(self.result.exists())
|
|
self.assertFalse(other.exists())
|
|
self.assertFalse((cold/'030.json').exists())
|
|
|
|
def test_aggregate_success_requires_cold_success_and_valid_measurements(self):
|
|
source = RUNNER.parent/'fixtures/pdf/navigation.pdf'
|
|
source_hash = hashlib.sha256(source.read_bytes()).hexdigest()
|
|
for scenario in ['cold failure', 'measurement failure', 'memory pressure', 'cold memory pressure']:
|
|
with self.subTest(scenario=scenario):
|
|
values = {'success': True, 'operations': [], 'openFrameMs': [1],
|
|
'processGroupRssPeakBytes': 1, 'tileCachePeakChargedBytes': 0, 'tileCacheBudgetBytes': 1,
|
|
'maximumMemoryPressureLevel': 2 if scenario == 'memory pressure' else 0,
|
|
'documentSha256': 'wrong' if scenario == 'measurement failure' else source_hash,
|
|
'continuousScroll': {'dispatchedInputs': 0},
|
|
'openCloseCycles': [{'stateEmpty': True, 'cacheBytesAfterClose': 0,
|
|
'workload': 'navigation-and-scroll', 'afterClose': {'available': False}}]}
|
|
def execute(command, **_):
|
|
output = Path(command[command.index('--benchmark-output')+1])
|
|
if output == self.result:
|
|
output.write_text(json.dumps(values))
|
|
return subprocess.CompletedProcess(command, 0)
|
|
if scenario == 'cold memory pressure':
|
|
output.write_text(json.dumps(dict(values, maximumMemoryPressureLevel=3)))
|
|
return subprocess.CompletedProcess(command, 0)
|
|
return subprocess.CompletedProcess(command, 7)
|
|
arguments = self.arguments + ['--cold-process-runs', '1' if scenario.startswith('cold ') else '0']
|
|
with patch.object(sys, 'argv', arguments), patch('subprocess.run', side_effect=execute), patch('builtins.print'):
|
|
with self.assertRaises(SystemExit):
|
|
runpy.run_path(str(RUNNER), run_name='__main__')
|
|
result = json.loads(self.result.read_text())
|
|
self.assertTrue(result['harnessReportedSuccess'])
|
|
self.assertFalse(result['success'])
|
|
if scenario == 'cold failure':
|
|
self.assertEqual(result['coldProcessOpen']['failed'], 1)
|
|
self.assertTrue(all(result['measurementChecks'].values()))
|
|
elif scenario == 'measurement failure':
|
|
self.assertFalse(result['measurementChecks']['source_unchanged'])
|
|
else:
|
|
self.assertFalse(result['measurementChecks']['full_resolution_conditions'])
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|