001/* 002 * Copyright © 2025 CUI-OpenSource-Software (info@cuioss.de) 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package de.cuioss.benchmarking.common.runner; 017 018import de.cuioss.benchmarking.common.config.BenchmarkConfiguration; 019import de.cuioss.benchmarking.common.metrics.IterationTimestampParser; 020import de.cuioss.benchmarking.common.metrics.PrometheusMetricsManager; 021import de.cuioss.tools.logging.CuiLogger; 022import org.openjdk.jmh.results.RunResult; 023import org.openjdk.jmh.runner.Runner; 024import org.openjdk.jmh.runner.RunnerException; 025import org.openjdk.jmh.runner.options.Options; 026import org.openjdk.jmh.runner.options.OptionsBuilder; 027 028import java.io.IOException; 029import java.nio.file.Files; 030import java.nio.file.Path; 031import java.time.Instant; 032import java.util.Collection; 033import java.util.List; 034import java.util.Map; 035import java.util.Optional; 036import java.util.stream.Collectors; 037 038import static de.cuioss.benchmarking.common.util.BenchmarkingLogMessages.INFO.*; 039import static de.cuioss.benchmarking.common.util.BenchmarkingLogMessages.WARN.*; 040 041/** 042 * Abstract base class for JMH benchmark runners that integrates with CUI benchmarking infrastructure. 043 * <p> 044 * This runner uses the Template Method pattern to orchestrate benchmark execution while 045 * allowing concrete implementations to customize specific steps. 046 * <p> 047 * The template method {@link #runBenchmark()} defines the benchmark execution flow: 048 * <ol> 049 * <li>Validation of configuration</li> 050 * <li>Preparation phase ({@link #prepareBenchmark(BenchmarkConfiguration)})</li> 051 * <li>Benchmark execution ({@link #executeBenchmark(Options)})</li> 052 * <li>Results processing ({@link #processResults(Collection, BenchmarkConfiguration)})</li> 053 * <li>Cleanup phase ({@link #cleanup(BenchmarkConfiguration)})</li> 054 * </ol> 055 * <p> 056 * Features: 057 * <ul> 058 * <li>Smart benchmark type detection (micro vs integration)</li> 059 * <li>Automatic badge generation with performance scoring</li> 060 * <li>Self-contained HTML reports with embedded CSS</li> 061 * <li>GitHub Pages ready deployment structure</li> 062 * <li>Structured metrics in JSON format</li> 063 * </ul> 064 */ 065public abstract class AbstractBenchmarkRunner { 066 067 private static final CuiLogger LOGGER = 068 new CuiLogger(AbstractBenchmarkRunner.class); 069 070 private final PrometheusMetricsManager prometheusMetricsManager; 071 private Instant benchmarkStartTime; 072 private Instant benchmarkEndTime; 073 074 /** 075 * Default constructor that initializes the Prometheus metrics manager. 076 */ 077 protected AbstractBenchmarkRunner() { 078 this.prometheusMetricsManager = new PrometheusMetricsManager(); 079 } 080 081 /** 082 * Creates the benchmark configuration for this runner. 083 * This method must provide a complete configuration including: 084 * - Benchmark type 085 * - Include pattern 086 * - Result file name and directory 087 * - Throughput and latency benchmark names 088 * - Any other specific configuration 089 * 090 * @return the complete benchmark configuration 091 */ 092 protected abstract BenchmarkConfiguration createConfiguration(); 093 094 /** 095 * Prepares the benchmark environment. 096 * This method is called after configuration validation and before benchmark execution. 097 * Use this for initialization tasks like: 098 * - Setting up resources 099 * - Initializing caches 100 * - Configuring logging 101 * 102 * @param config the benchmark configuration 103 * @throws IOException if preparation fails 104 */ 105 protected abstract void prepareBenchmark(BenchmarkConfiguration config) throws IOException; 106 107 /** 108 * Executes the benchmark with the given options. 109 * Default implementation uses JMH Runner, but can be overridden for custom execution. 110 * 111 * @param options the JMH options 112 * @return collection of benchmark results 113 * @throws RunnerException if benchmark execution fails 114 */ 115 protected Collection<RunResult> executeBenchmark(Options options) throws RunnerException { 116 return new Runner(options).run(); 117 } 118 119 /** 120 * Processes the benchmark results. 121 * Default implementation uses BenchmarkResultProcessor and collects Prometheus metrics. 122 * 123 * @param results the benchmark results 124 * @param config the benchmark configuration 125 * @throws IOException if processing fails 126 */ 127 protected void processResults(Collection<RunResult> results, BenchmarkConfiguration config) throws IOException { 128 // Collect Prometheus metrics BEFORE processing results 129 // This ensures metrics exist when GitHubPagesGenerator runs 130 prometheusMetricsManager.collectMetricsForResults(results, config); 131 132 // Process results (generates reports and GitHub Pages) 133 BenchmarkResultProcessor processor = new BenchmarkResultProcessor( 134 config.reportConfig().benchmarkType(), 135 config.reportConfig() 136 ); 137 processor.processResults(results, config.resultsDirectory()); 138 } 139 140 /** 141 * Performs cleanup after benchmark execution. 142 * This method is called after results processing, regardless of success or failure. 143 * Use this for: 144 * - Releasing resources 145 * - Final metrics collection 146 * - Post-processing tasks 147 * 148 * @param config the benchmark configuration 149 * @throws IOException if cleanup fails 150 */ 151 protected abstract void cleanup(BenchmarkConfiguration config) throws IOException; 152 153 /** 154 * Hook method called before benchmark execution starts. 155 * Override to add custom pre-benchmark logic. 156 * Default implementation records benchmark start time for Prometheus metrics. 157 * 158 * @param config the benchmark configuration 159 */ 160 protected void beforeBenchmark(BenchmarkConfiguration config) { 161 // Record benchmark start time for real-time metrics collection 162 benchmarkStartTime = Instant.now(); 163 LOGGER.debug("Benchmark execution started at: %s", benchmarkStartTime); 164 165 // Record start time for each benchmark if we know the names 166 if (config.throughputBenchmarkName() != null) { 167 prometheusMetricsManager.recordBenchmarkStart(config.throughputBenchmarkName()); 168 } 169 if (config.latencyBenchmarkName() != null && 170 !config.latencyBenchmarkName().equals(config.throughputBenchmarkName())) { 171 prometheusMetricsManager.recordBenchmarkStart(config.latencyBenchmarkName()); 172 } 173 } 174 175 /** 176 * Hook method called after benchmark execution completes. 177 * Override to add custom post-benchmark logic. 178 * Default implementation logs completion. 179 * 180 * @param results the benchmark results 181 * @param config the benchmark configuration 182 */ 183 protected void afterBenchmark(Collection<RunResult> results, BenchmarkConfiguration config) { 184 LOGGER.debug("Benchmark execution completed at: %s", benchmarkEndTime); 185 } 186 187 188 /** 189 * Extracts a clean benchmark name from the JMH RunResult. 190 * 191 * @param result the JMH run result 192 * @return a clean benchmark name suitable for file naming 193 */ 194 private String extractBenchmarkName(RunResult result) { 195 String label = result.getPrimaryResult().getLabel(); 196 int lastDot = label.lastIndexOf('.'); 197 if (lastDot >= 0 && lastDot < label.length() - 1) { 198 return label.substring(lastDot + 1); 199 } 200 return label; 201 } 202 203 /** 204 * Processes timestamp data from TimestampProfiler and records it in Prometheus metrics. 205 * Attempts to use precise iteration timestamps when available, falls back to session timestamps. 206 * 207 * @param results the benchmark results 208 * @param outputPath the directory containing the timestamp file 209 */ 210 private void processTimestampData(Collection<RunResult> results, Path outputPath) { 211 Path timestampsFile = outputPath.resolve("jmh-iteration-timestamps.jsonl"); 212 213 if (!Files.exists(timestampsFile)) { 214 LOGGER.debug("No timestamp file found at %s, using session-wide timestamps", timestampsFile); 215 recordSessionTimestamps(results); 216 return; 217 } 218 219 try { 220 processPreciseTimestamps(results, timestampsFile); 221 } catch (IOException e) { 222 LOGGER.warn(FAILED_PARSE_TIMESTAMP_FILE, e.getMessage()); 223 recordSessionTimestamps(results); 224 } 225 } 226 227 /** 228 * Processes precise timestamps from the timestamp file. 229 * 230 * @param results the benchmark results 231 * @param timestampsFile the path to the timestamp file 232 * @throws IOException if reading the file fails 233 */ 234 private void processPreciseTimestamps(Collection<RunResult> results, Path timestampsFile) throws IOException { 235 List<IterationTimestampParser.IterationWindow> allWindows = 236 IterationTimestampParser.parseJsonlFile(timestampsFile); 237 238 Map<String, List<IterationTimestampParser.IterationWindow>> byBenchmark = 239 allWindows.stream().collect(Collectors.groupingBy( 240 IterationTimestampParser.IterationWindow::benchmarkName)); 241 242 for (RunResult result : results) { 243 String benchmarkName = extractBenchmarkName(result); 244 List<IterationTimestampParser.IterationWindow> benchmarkWindows = byBenchmark.get(benchmarkName); 245 246 if (benchmarkWindows == null || benchmarkWindows.isEmpty()) { 247 LOGGER.warn(NO_TIMESTAMP_DATA, benchmarkName); 248 recordBenchmarkTimestamp(benchmarkName); 249 continue; 250 } 251 252 recordPreciseBenchmarkTimestamp(benchmarkName, benchmarkWindows); 253 } 254 } 255 256 /** 257 * Records precise timestamps for a benchmark from its iteration windows. 258 * 259 * @param benchmarkName the name of the benchmark 260 * @param benchmarkWindows the iteration windows for this benchmark 261 */ 262 private void recordPreciseBenchmarkTimestamp(String benchmarkName, 263 List<IterationTimestampParser.IterationWindow> benchmarkWindows) { 264 Optional<IterationTimestampParser.IterationWindow> measurementWindow = 265 benchmarkWindows.stream() 266 .filter(w -> !w.isWarmup()) 267 .findFirst(); 268 269 if (measurementWindow.isPresent()) { 270 IterationTimestampParser.IterationWindow window = measurementWindow.get(); 271 LOGGER.info(USING_PRECISE_TIMESTAMPS, benchmarkName, window.startTime(), window.endTime()); 272 prometheusMetricsManager.recordBenchmarkTimestamps( 273 benchmarkName, window.startTime(), window.endTime()); 274 } else { 275 LOGGER.warn(NO_MEASUREMENT_WINDOWS, benchmarkName); 276 recordBenchmarkTimestamp(benchmarkName); 277 } 278 } 279 280 /** 281 * Records session-wide timestamps for all results. 282 * 283 * @param results the benchmark results 284 */ 285 private void recordSessionTimestamps(Collection<RunResult> results) { 286 for (RunResult result : results) { 287 recordBenchmarkTimestamp(extractBenchmarkName(result)); 288 } 289 } 290 291 /** 292 * Records session-wide timestamps for a single benchmark. 293 * 294 * @param benchmarkName the name of the benchmark 295 */ 296 private void recordBenchmarkTimestamp(String benchmarkName) { 297 prometheusMetricsManager.recordBenchmarkTimestamps( 298 benchmarkName, benchmarkStartTime, benchmarkEndTime); 299 } 300 301 /** 302 * Builds JMH options from the benchmark configuration. 303 * This method extracts common option building logic that can be reused or extended. 304 * 305 * @param config the benchmark configuration 306 * @return JMH options builder with common settings applied 307 */ 308 protected OptionsBuilder buildCommonOptions(BenchmarkConfiguration config) { 309 var builder = new OptionsBuilder(); 310 311 builder.include(config.includePattern()) 312 .resultFormat(config.reportConfig().resultFormat()) 313 .result(config.reportConfig().getOrCreateResultFile()) 314 .forks(config.forks()) 315 .warmupIterations(config.warmupIterations()) 316 .measurementIterations(config.measurementIterations()) 317 .measurementTime(config.measurementTime()) 318 .warmupTime(config.warmupTime()) 319 .threads(config.threads()); 320 321 // Pass all system properties from parent JVM to forked JVMs 322 // This ensures all Maven-provided properties are available in benchmark processes 323 var systemProperties = System.getProperties().entrySet().stream() 324 .map(e -> "-D" + e.getKey() + "=" + e.getValue()) 325 .toArray(String[]::new); 326 327 builder.jvmArgsPrepend(systemProperties); 328 329 return builder; 330 } 331 332 /** 333 * Validates the benchmark configuration. 334 * Throws IllegalArgumentException if configuration is invalid. 335 * 336 * @param config the configuration to validate 337 * @throws IllegalArgumentException if configuration is invalid 338 */ 339 protected void validateConfiguration(BenchmarkConfiguration config) { 340 if (config == null) { 341 throw new IllegalArgumentException("Benchmark configuration cannot be null"); 342 } 343 344 if (config.benchmarkType() == null) { 345 throw new IllegalArgumentException("Benchmark type must be specified"); 346 } 347 348 if (config.includePattern() == null || config.includePattern().isEmpty()) { 349 throw new IllegalArgumentException("Include pattern must be specified"); 350 } 351 352 if (config.resultsDirectory() == null || config.resultsDirectory().isEmpty()) { 353 throw new IllegalArgumentException("Results directory must be specified"); 354 } 355 356 if (config.forks() < 0) { 357 throw new IllegalArgumentException("Forks must be non-negative"); 358 } 359 360 if (config.warmupIterations() < 0) { 361 throw new IllegalArgumentException("Warmup iterations must be non-negative"); 362 } 363 364 if (config.measurementIterations() <= 0) { 365 throw new IllegalArgumentException("Measurement iterations must be positive"); 366 } 367 368 if (config.threads() <= 0) { 369 throw new IllegalArgumentException("Threads must be positive"); 370 } 371 } 372 373 /** 374 * Template method that defines the benchmark execution flow. 375 * This is the main entry point that orchestrates all phases of benchmark execution. 376 * 377 * @throws IOException if I/O operations fail 378 * @throws RunnerException if benchmark execution fails 379 */ 380 public final void runBenchmark() throws IOException, RunnerException { 381 // Step 1: Create and validate configuration 382 BenchmarkConfiguration config = createConfiguration(); 383 validateConfiguration(config); 384 385 // Clear any previous timestamps to ensure fresh metrics collection 386 prometheusMetricsManager.clear(); 387 388 String outputDir = config.resultsDirectory(); 389 LOGGER.info(BENCHMARK_RUNNER_STARTING_WITH_DETAILS, config.benchmarkType(), outputDir); 390 391 // Step 2: Ensure output directory exists 392 Path outputPath = Path.of(outputDir); 393 Files.createDirectories(outputPath); 394 395 try { 396 // Step 3: Preparation phase 397 prepareBenchmark(config); 398 399 // Step 4: Pre-benchmark hook 400 beforeBenchmark(config); 401 402 // Step 5: Build options 403 Options options = buildCommonOptions(config).build(); 404 405 // Step 6: Execute benchmarks 406 Collection<RunResult> results = executeBenchmark(options); 407 408 if (results.isEmpty()) { 409 throw new IllegalStateException("No benchmark results produced"); 410 } 411 412 // Step 7: Use precise iteration timestamps from TimestampProfiler 413 benchmarkEndTime = Instant.now(); 414 processTimestampData(results, outputPath); 415 416 // Step 8: Process results (including Prometheus metrics collection) 417 processResults(results, config); 418 419 // Step 9: Post-benchmark hook 420 afterBenchmark(results, config); 421 422 LOGGER.info(BENCHMARKS_COMPLETED_WITH_ARTIFACTS, results.size(), outputDir); 423 424 } finally { 425 // Step 9: Cleanup (always executed) 426 cleanup(config); 427 } 428 } 429 430}