001/*
002 * Copyright © 2025 CUI-OpenSource-Software (info@cuioss.de)
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package de.cuioss.benchmarking.common.runner;
017
018import de.cuioss.benchmarking.common.config.BenchmarkConfiguration;
019import de.cuioss.benchmarking.common.metrics.IterationTimestampParser;
020import de.cuioss.benchmarking.common.metrics.PrometheusMetricsManager;
021import de.cuioss.tools.logging.CuiLogger;
022import org.openjdk.jmh.results.RunResult;
023import org.openjdk.jmh.runner.Runner;
024import org.openjdk.jmh.runner.RunnerException;
025import org.openjdk.jmh.runner.options.Options;
026import org.openjdk.jmh.runner.options.OptionsBuilder;
027
028import java.io.IOException;
029import java.nio.file.Files;
030import java.nio.file.Path;
031import java.time.Instant;
032import java.util.Collection;
033import java.util.List;
034import java.util.Map;
035import java.util.Optional;
036import java.util.stream.Collectors;
037
038import static de.cuioss.benchmarking.common.util.BenchmarkingLogMessages.INFO.*;
039import static de.cuioss.benchmarking.common.util.BenchmarkingLogMessages.WARN.*;
040
041/**
042 * Abstract base class for JMH benchmark runners that integrates with CUI benchmarking infrastructure.
043 * <p>
044 * This runner uses the Template Method pattern to orchestrate benchmark execution while
045 * allowing concrete implementations to customize specific steps.
046 * <p>
047 * The template method {@link #runBenchmark()} defines the benchmark execution flow:
048 * <ol>
049 *   <li>Validation of configuration</li>
050 *   <li>Preparation phase ({@link #prepareBenchmark(BenchmarkConfiguration)})</li>
051 *   <li>Benchmark execution ({@link #executeBenchmark(Options)})</li>
052 *   <li>Results processing ({@link #processResults(Collection, BenchmarkConfiguration)})</li>
053 *   <li>Cleanup phase ({@link #cleanup(BenchmarkConfiguration)})</li>
054 * </ol>
055 * <p>
056 * Features:
057 * <ul>
058 *   <li>Smart benchmark type detection (micro vs integration)</li>
059 *   <li>Automatic badge generation with performance scoring</li>
060 *   <li>Self-contained HTML reports with embedded CSS</li>
061 *   <li>GitHub Pages ready deployment structure</li>
062 *   <li>Structured metrics in JSON format</li>
063 * </ul>
064 */
065public abstract class AbstractBenchmarkRunner {
066
067    private static final CuiLogger LOGGER =
068            new CuiLogger(AbstractBenchmarkRunner.class);
069
070    private final PrometheusMetricsManager prometheusMetricsManager;
071    private Instant benchmarkStartTime;
072    private Instant benchmarkEndTime;
073
074    /**
075     * Default constructor that initializes the Prometheus metrics manager.
076     */
077    protected AbstractBenchmarkRunner() {
078        this.prometheusMetricsManager = new PrometheusMetricsManager();
079    }
080
081    /**
082     * Creates the benchmark configuration for this runner.
083     * This method must provide a complete configuration including:
084     * - Benchmark type
085     * - Include pattern
086     * - Result file name and directory
087     * - Throughput and latency benchmark names
088     * - Any other specific configuration
089     *
090     * @return the complete benchmark configuration
091     */
092    protected abstract BenchmarkConfiguration createConfiguration();
093
094    /**
095     * Prepares the benchmark environment.
096     * This method is called after configuration validation and before benchmark execution.
097     * Use this for initialization tasks like:
098     * - Setting up resources
099     * - Initializing caches
100     * - Configuring logging
101     *
102     * @param config the benchmark configuration
103     * @throws IOException if preparation fails
104     */
105    protected abstract void prepareBenchmark(BenchmarkConfiguration config) throws IOException;
106
107    /**
108     * Executes the benchmark with the given options.
109     * Default implementation uses JMH Runner, but can be overridden for custom execution.
110     *
111     * @param options the JMH options
112     * @return collection of benchmark results
113     * @throws RunnerException if benchmark execution fails
114     */
115    protected Collection<RunResult> executeBenchmark(Options options) throws RunnerException {
116        return new Runner(options).run();
117    }
118
119    /**
120     * Processes the benchmark results.
121     * Default implementation uses BenchmarkResultProcessor and collects Prometheus metrics.
122     *
123     * @param results the benchmark results
124     * @param config the benchmark configuration
125     * @throws IOException if processing fails
126     */
127    protected void processResults(Collection<RunResult> results, BenchmarkConfiguration config) throws IOException {
128        // Collect Prometheus metrics BEFORE processing results
129        // This ensures metrics exist when GitHubPagesGenerator runs
130        prometheusMetricsManager.collectMetricsForResults(results, config);
131
132        // Process results (generates reports and GitHub Pages)
133        BenchmarkResultProcessor processor = new BenchmarkResultProcessor(
134                config.reportConfig().benchmarkType(),
135                config.reportConfig()
136        );
137        processor.processResults(results, config.resultsDirectory());
138    }
139
140    /**
141     * Performs cleanup after benchmark execution.
142     * This method is called after results processing, regardless of success or failure.
143     * Use this for:
144     * - Releasing resources
145     * - Final metrics collection
146     * - Post-processing tasks
147     *
148     * @param config the benchmark configuration
149     * @throws IOException if cleanup fails
150     */
151    protected abstract void cleanup(BenchmarkConfiguration config) throws IOException;
152
153    /**
154     * Hook method called before benchmark execution starts.
155     * Override to add custom pre-benchmark logic.
156     * Default implementation records benchmark start time for Prometheus metrics.
157     *
158     * @param config the benchmark configuration
159     */
160    protected void beforeBenchmark(BenchmarkConfiguration config) {
161        // Record benchmark start time for real-time metrics collection
162        benchmarkStartTime = Instant.now();
163        LOGGER.debug("Benchmark execution started at: %s", benchmarkStartTime);
164
165        // Record start time for each benchmark if we know the names
166        if (config.throughputBenchmarkName() != null) {
167            prometheusMetricsManager.recordBenchmarkStart(config.throughputBenchmarkName());
168        }
169        if (config.latencyBenchmarkName() != null &&
170                !config.latencyBenchmarkName().equals(config.throughputBenchmarkName())) {
171            prometheusMetricsManager.recordBenchmarkStart(config.latencyBenchmarkName());
172        }
173    }
174
175    /**
176     * Hook method called after benchmark execution completes.
177     * Override to add custom post-benchmark logic.
178     * Default implementation logs completion.
179     *
180     * @param results the benchmark results
181     * @param config the benchmark configuration
182     */
183    protected void afterBenchmark(Collection<RunResult> results, BenchmarkConfiguration config) {
184        LOGGER.debug("Benchmark execution completed at: %s", benchmarkEndTime);
185    }
186
187
188    /**
189     * Extracts a clean benchmark name from the JMH RunResult.
190     *
191     * @param result the JMH run result
192     * @return a clean benchmark name suitable for file naming
193     */
194    private String extractBenchmarkName(RunResult result) {
195        String label = result.getPrimaryResult().getLabel();
196        int lastDot = label.lastIndexOf('.');
197        if (lastDot >= 0 && lastDot < label.length() - 1) {
198            return label.substring(lastDot + 1);
199        }
200        return label;
201    }
202
203    /**
204     * Processes timestamp data from TimestampProfiler and records it in Prometheus metrics.
205     * Attempts to use precise iteration timestamps when available, falls back to session timestamps.
206     *
207     * @param results the benchmark results
208     * @param outputPath the directory containing the timestamp file
209     */
210    private void processTimestampData(Collection<RunResult> results, Path outputPath) {
211        Path timestampsFile = outputPath.resolve("jmh-iteration-timestamps.jsonl");
212
213        if (!Files.exists(timestampsFile)) {
214            LOGGER.debug("No timestamp file found at %s, using session-wide timestamps", timestampsFile);
215            recordSessionTimestamps(results);
216            return;
217        }
218
219        try {
220            processPreciseTimestamps(results, timestampsFile);
221        } catch (IOException e) {
222            LOGGER.warn(FAILED_PARSE_TIMESTAMP_FILE, e.getMessage());
223            recordSessionTimestamps(results);
224        }
225    }
226
227    /**
228     * Processes precise timestamps from the timestamp file.
229     *
230     * @param results the benchmark results
231     * @param timestampsFile the path to the timestamp file
232     * @throws IOException if reading the file fails
233     */
234    private void processPreciseTimestamps(Collection<RunResult> results, Path timestampsFile) throws IOException {
235        List<IterationTimestampParser.IterationWindow> allWindows =
236                IterationTimestampParser.parseJsonlFile(timestampsFile);
237
238        Map<String, List<IterationTimestampParser.IterationWindow>> byBenchmark =
239                allWindows.stream().collect(Collectors.groupingBy(
240                        IterationTimestampParser.IterationWindow::benchmarkName));
241
242        for (RunResult result : results) {
243            String benchmarkName = extractBenchmarkName(result);
244            List<IterationTimestampParser.IterationWindow> benchmarkWindows = byBenchmark.get(benchmarkName);
245
246            if (benchmarkWindows == null || benchmarkWindows.isEmpty()) {
247                LOGGER.warn(NO_TIMESTAMP_DATA, benchmarkName);
248                recordBenchmarkTimestamp(benchmarkName);
249                continue;
250            }
251
252            recordPreciseBenchmarkTimestamp(benchmarkName, benchmarkWindows);
253        }
254    }
255
256    /**
257     * Records precise timestamps for a benchmark from its iteration windows.
258     *
259     * @param benchmarkName the name of the benchmark
260     * @param benchmarkWindows the iteration windows for this benchmark
261     */
262    private void recordPreciseBenchmarkTimestamp(String benchmarkName,
263            List<IterationTimestampParser.IterationWindow> benchmarkWindows) {
264        Optional<IterationTimestampParser.IterationWindow> measurementWindow =
265                benchmarkWindows.stream()
266                        .filter(w -> !w.isWarmup())
267                        .findFirst();
268
269        if (measurementWindow.isPresent()) {
270            IterationTimestampParser.IterationWindow window = measurementWindow.get();
271            LOGGER.info(USING_PRECISE_TIMESTAMPS, benchmarkName, window.startTime(), window.endTime());
272            prometheusMetricsManager.recordBenchmarkTimestamps(
273                    benchmarkName, window.startTime(), window.endTime());
274        } else {
275            LOGGER.warn(NO_MEASUREMENT_WINDOWS, benchmarkName);
276            recordBenchmarkTimestamp(benchmarkName);
277        }
278    }
279
280    /**
281     * Records session-wide timestamps for all results.
282     *
283     * @param results the benchmark results
284     */
285    private void recordSessionTimestamps(Collection<RunResult> results) {
286        for (RunResult result : results) {
287            recordBenchmarkTimestamp(extractBenchmarkName(result));
288        }
289    }
290
291    /**
292     * Records session-wide timestamps for a single benchmark.
293     *
294     * @param benchmarkName the name of the benchmark
295     */
296    private void recordBenchmarkTimestamp(String benchmarkName) {
297        prometheusMetricsManager.recordBenchmarkTimestamps(
298                benchmarkName, benchmarkStartTime, benchmarkEndTime);
299    }
300
301    /**
302     * Builds JMH options from the benchmark configuration.
303     * This method extracts common option building logic that can be reused or extended.
304     *
305     * @param config the benchmark configuration
306     * @return JMH options builder with common settings applied
307     */
308    protected OptionsBuilder buildCommonOptions(BenchmarkConfiguration config) {
309        var builder = new OptionsBuilder();
310
311        builder.include(config.includePattern())
312                .resultFormat(config.reportConfig().resultFormat())
313                .result(config.reportConfig().getOrCreateResultFile())
314                .forks(config.forks())
315                .warmupIterations(config.warmupIterations())
316                .measurementIterations(config.measurementIterations())
317                .measurementTime(config.measurementTime())
318                .warmupTime(config.warmupTime())
319                .threads(config.threads());
320
321        // Pass all system properties from parent JVM to forked JVMs
322        // This ensures all Maven-provided properties are available in benchmark processes
323        var systemProperties = System.getProperties().entrySet().stream()
324                .map(e -> "-D" + e.getKey() + "=" + e.getValue())
325                .toArray(String[]::new);
326
327        builder.jvmArgsPrepend(systemProperties);
328
329        return builder;
330    }
331
332    /**
333     * Validates the benchmark configuration.
334     * Throws IllegalArgumentException if configuration is invalid.
335     *
336     * @param config the configuration to validate
337     * @throws IllegalArgumentException if configuration is invalid
338     */
339    protected void validateConfiguration(BenchmarkConfiguration config) {
340        if (config == null) {
341            throw new IllegalArgumentException("Benchmark configuration cannot be null");
342        }
343
344        if (config.benchmarkType() == null) {
345            throw new IllegalArgumentException("Benchmark type must be specified");
346        }
347
348        if (config.includePattern() == null || config.includePattern().isEmpty()) {
349            throw new IllegalArgumentException("Include pattern must be specified");
350        }
351
352        if (config.resultsDirectory() == null || config.resultsDirectory().isEmpty()) {
353            throw new IllegalArgumentException("Results directory must be specified");
354        }
355
356        if (config.forks() < 0) {
357            throw new IllegalArgumentException("Forks must be non-negative");
358        }
359
360        if (config.warmupIterations() < 0) {
361            throw new IllegalArgumentException("Warmup iterations must be non-negative");
362        }
363
364        if (config.measurementIterations() <= 0) {
365            throw new IllegalArgumentException("Measurement iterations must be positive");
366        }
367
368        if (config.threads() <= 0) {
369            throw new IllegalArgumentException("Threads must be positive");
370        }
371    }
372
373    /**
374     * Template method that defines the benchmark execution flow.
375     * This is the main entry point that orchestrates all phases of benchmark execution.
376     *
377     * @throws IOException if I/O operations fail
378     * @throws RunnerException if benchmark execution fails
379     */
380    public final void runBenchmark() throws IOException, RunnerException {
381        // Step 1: Create and validate configuration
382        BenchmarkConfiguration config = createConfiguration();
383        validateConfiguration(config);
384
385        // Clear any previous timestamps to ensure fresh metrics collection
386        prometheusMetricsManager.clear();
387
388        String outputDir = config.resultsDirectory();
389        LOGGER.info(BENCHMARK_RUNNER_STARTING_WITH_DETAILS, config.benchmarkType(), outputDir);
390
391        // Step 2: Ensure output directory exists
392        Path outputPath = Path.of(outputDir);
393        Files.createDirectories(outputPath);
394
395        try {
396            // Step 3: Preparation phase
397            prepareBenchmark(config);
398
399            // Step 4: Pre-benchmark hook
400            beforeBenchmark(config);
401
402            // Step 5: Build options
403            Options options = buildCommonOptions(config).build();
404
405            // Step 6: Execute benchmarks
406            Collection<RunResult> results = executeBenchmark(options);
407
408            if (results.isEmpty()) {
409                throw new IllegalStateException("No benchmark results produced");
410            }
411
412            // Step 7: Use precise iteration timestamps from TimestampProfiler
413            benchmarkEndTime = Instant.now();
414            processTimestampData(results, outputPath);
415
416            // Step 8: Process results (including Prometheus metrics collection)
417            processResults(results, config);
418
419            // Step 9: Post-benchmark hook
420            afterBenchmark(results, config);
421
422            LOGGER.info(BENCHMARKS_COMPLETED_WITH_ARTIFACTS, results.size(), outputDir);
423
424        } finally {
425            // Step 9: Cleanup (always executed)
426            cleanup(config);
427        }
428    }
429
430}