initial commit

tests/test_benchmark_runner.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Runner failure-evidence regressions; these are not performance measurements."""
+import json
+import hashlib
+import os
+from pathlib import Path
+import runpy
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+RUNNER = Path(__file__).with_name('benchmark.py')
+
+class BenchmarkRunnerTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.root = Path(self.temporary.name)
+        self.binary = self.root/'docview'
+        self.binary.write_text('#!/bin/sh\nexit 7\n')
+        self.binary.chmod(0o700)
+        for name in ['docview-pdf-worker', 'docview-archive-worker']:
+            (self.root/name).write_bytes(b'fixture')
+        self.output = self.root/'results'
+        self.output.mkdir()
+        self.result = self.output/'pdf.json'
+        self.result.write_text(json.dumps({'success': True, 'executableSha256': {'docview': 'old'}}))
+        self.arguments = [str(RUNNER), '--binary', str(self.binary), '--formats', 'pdf', '--runs', '1',
+            '--operations', '0', '--close-cycles', '0', '--scroll-steps', '0', '--output', str(self.output)]
+
+    @unittest.skipUnless(os.name == 'posix', 'POSIX fixture executable')
+    def test_startup_failure_cannot_relabel_previous_success(self):
+        result = subprocess.run([sys.executable, *self.arguments], capture_output=True, text=True)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn('no measurement record', result.stderr)
+        self.assertFalse(self.result.exists())
+
+    def test_timeout_cannot_leave_previous_success(self):
+        with patch.object(sys, 'argv', self.arguments), patch('subprocess.run', side_effect=subprocess.TimeoutExpired(['fixture'], 1)):
+            with self.assertRaises(SystemExit) as failure:
+                runpy.run_path(str(RUNNER), run_name='__main__')
+        self.assertIn('process timeout', str(failure.exception))
+        self.assertFalse(self.result.exists())
+
+    def test_os_startup_error_clears_all_selected_previous_evidence(self):
+        other = self.output/'html.json'
+        other.write_text('{"success":true}')
+        cold = self.output/'cold/pdf'
+        cold.mkdir(parents=True)
+        (cold/'030.json').write_text('{"success":true}')
+        arguments = self.arguments + ['--formats', 'pdf', 'html']
+        with patch.object(sys, 'argv', arguments), patch('subprocess.run', side_effect=OSError('fixture')):
+            with self.assertRaises(SystemExit) as failure:
+                runpy.run_path(str(RUNNER), run_name='__main__')
+        self.assertIn('executable startup failed', str(failure.exception))
+        self.assertFalse(self.result.exists())
+        self.assertFalse(other.exists())
+        self.assertFalse((cold/'030.json').exists())
+
+    def test_aggregate_success_requires_cold_success_and_valid_measurements(self):
+        source = RUNNER.parent/'fixtures/pdf/navigation.pdf'
+        source_hash = hashlib.sha256(source.read_bytes()).hexdigest()
+        for scenario in ['cold failure', 'measurement failure', 'memory pressure', 'cold memory pressure']:
+            with self.subTest(scenario=scenario):
+                values = {'success': True, 'operations': [], 'openFrameMs': [1],
+                    'processGroupRssPeakBytes': 1, 'tileCachePeakChargedBytes': 0, 'tileCacheBudgetBytes': 1,
+                    'maximumMemoryPressureLevel': 2 if scenario == 'memory pressure' else 0,
+                    'documentSha256': 'wrong' if scenario == 'measurement failure' else source_hash,
+                    'continuousScroll': {'dispatchedInputs': 0},
+                    'openCloseCycles': [{'stateEmpty': True, 'cacheBytesAfterClose': 0,
+                        'workload': 'navigation-and-scroll', 'afterClose': {'available': False}}]}
+                def execute(command, **_):
+                    output = Path(command[command.index('--benchmark-output')+1])
+                    if output == self.result:
+                        output.write_text(json.dumps(values))
+                        return subprocess.CompletedProcess(command, 0)
+                    if scenario == 'cold memory pressure':
+                        output.write_text(json.dumps(dict(values, maximumMemoryPressureLevel=3)))
+                        return subprocess.CompletedProcess(command, 0)
+                    return subprocess.CompletedProcess(command, 7)
+                arguments = self.arguments + ['--cold-process-runs', '1' if scenario.startswith('cold ') else '0']
+                with patch.object(sys, 'argv', arguments), patch('subprocess.run', side_effect=execute), patch('builtins.print'):
+                    with self.assertRaises(SystemExit):
+                        runpy.run_path(str(RUNNER), run_name='__main__')
+                result = json.loads(self.result.read_text())
+                self.assertTrue(result['harnessReportedSuccess'])
+                self.assertFalse(result['success'])
+                if scenario == 'cold failure':
+                    self.assertEqual(result['coldProcessOpen']['failed'], 1)
+                    self.assertTrue(all(result['measurementChecks'].values()))
+                elif scenario == 'measurement failure':
+                    self.assertFalse(result['measurementChecks']['source_unchanged'])
+                else:
+                    self.assertFalse(result['measurementChecks']['full_resolution_conditions'])
+
+if __name__ == '__main__':
+    unittest.main()