001/*
002 * Copyright © 2025 CUI-OpenSource-Software (info@cuioss.de)
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package de.cuioss.benchmarking.common.report;
017
018import de.cuioss.benchmarking.common.config.BenchmarkType;
019import de.cuioss.benchmarking.common.constants.BenchmarkConstants;
020import de.cuioss.benchmarking.common.model.BenchmarkData;
021import de.cuioss.benchmarking.common.util.JsonSerializationHelper;
022import de.cuioss.tools.logging.CuiLogger;
023
024import java.io.IOException;
025import java.nio.file.Files;
026import java.nio.file.Path;
027import java.time.Instant;
028import java.time.ZoneOffset;
029import java.time.format.DateTimeFormatter;
030import java.util.*;
031
032import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Files.Data.BENCHMARK_DATA_JSON;
033import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Files.Directories.DATA_DIR;
034import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Metrics.Modes.*;
035import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Metrics.Percentiles.*;
036import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Metrics.Units.OPS;
037import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Metrics.Units.SUFFIX_OP;
038import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Report.Badge.MESSAGE;
039import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Report.DateFormats.DISPLAY_TIMESTAMP_PATTERN;
040import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Report.Grades.CssClasses.GRADE_F;
041import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Report.JsonFields.*;
042import static de.cuioss.benchmarking.common.constants.BenchmarkConstants.Report.Messages.HISTORICAL_DATA_NOT_AVAILABLE;
043import static de.cuioss.benchmarking.common.util.BenchmarkingLogMessages.INFO;
044
045/**
046 * Generates a standardized JSON data file containing all report data
047 * that can be consumed by HTML templates via JavaScript.
048 * <p>
049 * This generator consolidates all benchmark data and metrics into a single
050 * JSON file named "benchmark-data.json" that templates load and process.
051 * <p>
052 * The generated JSON structure includes:
053 * <ul>
054 *   <li>Metadata (timestamp, benchmark type, etc.)</li>
055 *   <li>Overview metrics (totals, averages, grades)</li>
056 *   <li>Detailed benchmark results</li>
057 *   <li>Chart data for visualizations</li>
058 *   <li>Trend data for historical analysis</li>
059 * </ul>
060 */
061public class ReportDataGenerator {
062
063    private static final CuiLogger LOGGER = new CuiLogger(ReportDataGenerator.class);
064    private static final DateTimeFormatter ISO_FORMATTER = DateTimeFormatter.ISO_INSTANT;
065    private static final String DATA_FILE_NAME = BENCHMARK_DATA_JSON;
066
067    private final TrendDataProcessor trendProcessor = new TrendDataProcessor();
068    private final BadgeGenerator badgeGenerator = new BadgeGenerator();
069    private final HistoricalDataManager historyManager = new HistoricalDataManager();
070
071    /**
072     * Generates a comprehensive data file for report templates using BenchmarkData.
073     *
074     * @param benchmarkData the benchmark data to generate reports from
075     * @param type the benchmark type
076     * @param outputDir the output directory
077     * @throws IOException if writing fails
078     */
079    public void generateDataFile(BenchmarkData benchmarkData, BenchmarkType type, String outputDir)
080            throws IOException {
081
082        Map<String, Object> data = new LinkedHashMap<>();
083
084        // Convert BenchmarkData to report format
085        data.put(METADATA, convertMetadata(benchmarkData.getMetadata()));
086        data.put(OVERVIEW, convertOverview(benchmarkData.getOverview()));
087        data.put(BENCHMARKS, convertBenchmarks(benchmarkData.getBenchmarks()));
088        data.put(CHART_DATA, createChartData(benchmarkData.getBenchmarks()));
089
090        // Extract metrics for trend processing
091        BenchmarkMetrics metrics = extractMetrics(benchmarkData.getOverview());
092        data.put(TRENDS, createTrendData(outputDir, metrics));
093
094        // Write data file to data subdirectory
095        Path dataDir = Path.of(outputDir, DATA_DIR);
096        Files.createDirectories(dataDir);
097        Path dataFile = dataDir.resolve(DATA_FILE_NAME);
098        Files.writeString(dataFile, JsonSerializationHelper.toJson(data));
099
100        LOGGER.info(INFO.METRICS_FILE_GENERATED, dataFile);
101
102        // Archive current run to history
103        String commitSha = System.getenv("GITHUB_SHA");
104        if (commitSha == null || commitSha.isEmpty()) {
105            commitSha = "local-run";
106        }
107        historyManager.archiveCurrentRun(data, outputDir, commitSha);
108
109        // Enforce retention policy
110        Path historyDir = Path.of(outputDir, "history");
111        historyManager.enforceRetentionPolicy(historyDir);
112
113        // Generate badges
114        TrendDataProcessor.TrendMetrics trendMetrics = null;
115        if (historyManager.hasHistoricalData(outputDir)) {
116            List<TrendDataProcessor.HistoricalDataPoint> historicalData =
117                    trendProcessor.loadHistoricalData(historyDir);
118            if (!historicalData.isEmpty()) {
119                trendMetrics = trendProcessor.calculateTrends(metrics, historicalData);
120            }
121        }
122        badgeGenerator.writeBadgeFiles(metrics, trendMetrics, type, outputDir);
123    }
124
125    private Map<String, Object> convertMetadata(BenchmarkData.Metadata metadata) {
126        if (metadata == null) {
127            return createDefaultMetadata();
128        }
129        Map<String, Object> result = new LinkedHashMap<>();
130        result.put(TIMESTAMP, metadata.getTimestamp());
131        result.put(DISPLAY_TIMESTAMP, metadata.getDisplayTimestamp());
132        result.put(BENCHMARK_TYPE, metadata.getBenchmarkType());
133        result.put(BenchmarkConstants.Report.JsonFields.REPORT_VERSION, metadata.getReportVersion());
134        result.put("projectName", metadata.getProjectName());
135        return result;
136    }
137
138    private Map<String, Object> createDefaultMetadata() {
139        Map<String, Object> metadata = new LinkedHashMap<>();
140        Instant now = Instant.now();
141        metadata.put(TIMESTAMP, ISO_FORMATTER.format(now.atOffset(ZoneOffset.UTC)));
142        metadata.put(DISPLAY_TIMESTAMP, DateTimeFormatter.ofPattern(DISPLAY_TIMESTAMP_PATTERN)
143                .format(now.atOffset(ZoneOffset.UTC)));
144        metadata.put(BENCHMARK_TYPE, BenchmarkType.MICRO.getDisplayName());
145        metadata.put(BenchmarkConstants.Report.JsonFields.REPORT_VERSION, BenchmarkConstants.Report.Versions.REPORT_VERSION);
146        metadata.put("projectName", null);
147        return metadata;
148    }
149
150    private Map<String, Object> convertOverview(BenchmarkData.Overview overview) {
151        if (overview == null) {
152            return createDefaultOverview();
153        }
154        Map<String, Object> result = new LinkedHashMap<>();
155        result.put(BenchmarkConstants.Report.JsonFields.THROUGHPUT, overview.getThroughput());
156        result.put(LATENCY, overview.getLatency());
157        result.put(THROUGHPUT_BENCHMARK_NAME, overview.getThroughputBenchmarkName());
158        result.put(LATENCY_BENCHMARK_NAME, overview.getLatencyBenchmarkName());
159        result.put(PERFORMANCE_SCORE, overview.getPerformanceScore());
160        result.put(PERFORMANCE_GRADE, overview.getPerformanceGrade());
161        result.put(PERFORMANCE_GRADE_CLASS, overview.getPerformanceGradeClass());
162        return result;
163    }
164
165    private Map<String, Object> createDefaultOverview() {
166        Map<String, Object> overview = new LinkedHashMap<>();
167        overview.put(BenchmarkConstants.Report.JsonFields.THROUGHPUT, "N/A");
168        overview.put(LATENCY, "N/A");
169        overview.put(THROUGHPUT_BENCHMARK_NAME, "");
170        overview.put(LATENCY_BENCHMARK_NAME, "");
171        overview.put(PERFORMANCE_SCORE, 0);
172        overview.put(PERFORMANCE_GRADE, "F");
173        overview.put(PERFORMANCE_GRADE_CLASS, GRADE_F);
174        return overview;
175    }
176
177    private BenchmarkMetrics extractMetrics(BenchmarkData.Overview overview) {
178        if (overview == null) {
179            return new BenchmarkMetrics("N/A", "N/A", 0.0, 0.0, 0, "F");
180        }
181
182        // Use numeric values directly from overview (no parsing needed!)
183        double throughput = overview.getThroughputOpsPerSec() != null ? overview.getThroughputOpsPerSec() : 0.0;
184        double latency = overview.getLatencyMs() != null ? overview.getLatencyMs() : 0.0;
185
186        return new BenchmarkMetrics(
187                overview.getThroughputBenchmarkName() != null ? overview.getThroughputBenchmarkName() : "N/A",
188                overview.getLatencyBenchmarkName() != null ? overview.getLatencyBenchmarkName() : "N/A",
189                throughput,
190                latency,
191                overview.getPerformanceScore(),
192                overview.getPerformanceGrade() != null ? overview.getPerformanceGrade() : "F"
193        );
194    }
195
196
197    private List<Map<String, Object>> convertBenchmarks(List<BenchmarkData.Benchmark> benchmarks) {
198        if (benchmarks == null) {
199            return new ArrayList<>();
200        }
201
202        List<Map<String, Object>> results = new ArrayList<>();
203        for (BenchmarkData.Benchmark benchmark : benchmarks) {
204            Map<String, Object> result = new LinkedHashMap<>();
205
206            result.put(NAME, benchmark.getName());
207            result.put(FULL_NAME, benchmark.getFullName());
208            result.put(MODE, benchmark.getMode());
209            result.put(SCORE, benchmark.getScore());
210            result.put(SCORE_UNIT, benchmark.getScoreUnit());
211
212            if (benchmark.getThroughput() != null) {
213                result.put(BenchmarkConstants.Report.JsonFields.THROUGHPUT, benchmark.getThroughput());
214            }
215            if (benchmark.getLatency() != null) {
216                result.put(BenchmarkConstants.Report.JsonFields.LATENCY, benchmark.getLatency());
217            }
218
219            result.put(ERROR, benchmark.getError());
220            result.put(VARIABILITY_COEFFICIENT, benchmark.getVariabilityCoefficient());
221            result.put(CONFIDENCE_LOW, benchmark.getConfidenceLow());
222            result.put(CONFIDENCE_HIGH, benchmark.getConfidenceHigh());
223
224            if (benchmark.getPercentiles() != null && !benchmark.getPercentiles().isEmpty()) {
225                result.put(PERCENTILES, benchmark.getPercentiles());
226            }
227
228            results.add(result);
229        }
230        return results;
231    }
232
233    private Map<String, Object> createChartData(List<BenchmarkData.Benchmark> benchmarks) {
234        if (benchmarks == null) {
235            benchmarks = new ArrayList<>();
236        }
237
238        Map<String, Object> chartData = new LinkedHashMap<>();
239        List<String> labels = new ArrayList<>();
240        List<Double> throughput = new ArrayList<>();
241        List<Double> latency = new ArrayList<>();
242
243        for (BenchmarkData.Benchmark benchmark : benchmarks) {
244            labels.add(benchmark.getName());
245
246            // Extract throughput: use rawScore if mode is thrpt or scoreUnit contains "ops"
247            Double throughputValue = null;
248            if (BenchmarkConstants.Metrics.Modes.THROUGHPUT.equals(benchmark.getMode()) ||
249                    (benchmark.getScoreUnit() != null && benchmark.getScoreUnit().contains(OPS))) {
250                throughputValue = benchmark.getRawScore();
251            }
252
253            // Extract latency: Only for benchmarks that measure latency
254            // - WRK benchmarks: mode=thrpt but percentiles represent latency (fullName starts with "wrk.")
255            // - JMH latency benchmarks: mode=avgt/sample, use rawScore or percentiles
256            // - JMH throughput benchmarks: mode=thrpt, percentiles are throughput variance (NOT latency!)
257            Double latencyValue = null;
258            if (hasLatencyPercentiles(benchmark)) {
259                // WRK benchmarks or JMH latency benchmarks with percentiles - use P50 as latency
260                latencyValue = benchmark.getPercentiles().get(P_50);
261            } else if (AVERAGE_TIME.equals(benchmark.getMode()) || SAMPLE.equals(benchmark.getMode()) ||
262                    (benchmark.getScoreUnit() != null && benchmark.getScoreUnit().contains(SUFFIX_OP))) {
263                // JMH latency benchmarks without percentiles - use rawScore
264                latencyValue = benchmark.getRawScore();
265            }
266
267            throughput.add(throughputValue);
268            latency.add(latencyValue);
269        }
270
271        chartData.put(LABELS, labels);
272        chartData.put(BenchmarkConstants.Report.JsonFields.THROUGHPUT, throughput);
273        chartData.put(BenchmarkConstants.Report.JsonFields.LATENCY, latency);
274        chartData.put(PERCENTILES_DATA, createPercentilesChartData(benchmarks));
275
276        return chartData;
277    }
278
279    private Map<String, Object> createPercentilesChartData(List<BenchmarkData.Benchmark> benchmarks) {
280        String[] percentileKeys = {P_0, P_50, P_90, P_95, P_99, P_99_9, P_99_99, P_100};
281        List<String> percentileLabels = Arrays.asList(LABEL_P0, LABEL_P50, LABEL_P90, LABEL_P95, LABEL_P99, LABEL_P99_9, LABEL_P99_99, LABEL_P100);
282
283        List<String> benchmarkNames = new ArrayList<>();
284        Map<String, List<Double>> dataByBenchmark = new LinkedHashMap<>();
285
286        collectPercentileData(benchmarks, percentileKeys, benchmarkNames, dataByBenchmark);
287
288        Map<String, Object> percentilesChart = new LinkedHashMap<>();
289        percentilesChart.put(PERCENTILE_LABELS, percentileLabels);
290        percentilesChart.put(BenchmarkConstants.Report.JsonFields.BENCHMARKS, benchmarkNames);
291        percentilesChart.put(DATA, dataByBenchmark);
292        percentilesChart.put(LABELS, benchmarkNames);
293        percentilesChart.put(DATASETS, createPercentileDatasets(percentileKeys, benchmarkNames, dataByBenchmark));
294
295        return percentilesChart;
296    }
297
298    private void collectPercentileData(List<BenchmarkData.Benchmark> benchmarks, String[] percentileKeys,
299            List<String> benchmarkNames, Map<String, List<Double>> dataByBenchmark) {
300        if (benchmarks == null) {
301            return;
302        }
303
304        for (BenchmarkData.Benchmark benchmark : benchmarks) {
305            // Only include benchmarks with LATENCY percentiles (not throughput variance)
306            // - WRK benchmarks: fullName starts with "wrk.", percentiles = latency distribution
307            // - JMH latency benchmarks: mode=avgt/sample, percentiles = latency distribution
308            // - JMH throughput benchmarks: mode=thrpt (non-WRK), percentiles = throughput variance (EXCLUDE)
309            if (hasLatencyPercentiles(benchmark)) {
310                benchmarkNames.add(benchmark.getName());
311                List<Double> benchmarkData = extractPercentileValues(benchmark.getPercentiles(), percentileKeys);
312                dataByBenchmark.put(benchmark.getName(), benchmarkData);
313            }
314        }
315    }
316
317    /**
318     * Checks if a benchmark has latency percentiles (as opposed to throughput variance percentiles).
319     * <p>
320     * WRK benchmarks have mode=thrpt but their percentiles represent latency distribution.
321     * JMH throughput benchmarks have mode=thrpt but their percentiles represent throughput iteration variance.
322     * JMH latency benchmarks have mode=avgt/sample and their percentiles represent latency distribution.
323     *
324     * @param benchmark the benchmark to check
325     * @return true if the benchmark has latency percentiles
326     */
327    private boolean hasLatencyPercentiles(BenchmarkData.Benchmark benchmark) {
328        if (benchmark.getPercentiles() == null || !benchmark.getPercentiles().containsKey(P_50)) {
329            return false;
330        }
331
332        // WRK benchmarks: fullName starts with "wrk." - these have latency percentiles
333        String fullName = benchmark.getFullName();
334        if (fullName != null && fullName.startsWith("wrk.")) {
335            return true;
336        }
337
338        // JMH latency benchmarks: mode is avgt or sample - these have latency percentiles
339        String mode = benchmark.getMode();
340        return AVERAGE_TIME.equals(mode) || SAMPLE.equals(mode);
341    }
342
343    private List<Double> extractPercentileValues(Map<String, Double> percentiles, String[] percentileKeys) {
344        List<Double> benchmarkData = new ArrayList<>();
345
346        if (percentiles != null && !percentiles.isEmpty()) {
347            for (String key : percentileKeys) {
348                benchmarkData.add(percentiles.get(key));
349            }
350        } else {
351            for (int i = 0; i < percentileKeys.length; i++) {
352                benchmarkData.add(null);
353            }
354        }
355
356        return benchmarkData;
357    }
358
359    private Map<String, List<Double>> createPercentileDatasets(String[] percentileKeys,
360            List<String> benchmarkNames,
361            Map<String, List<Double>> dataByBenchmark) {
362        Map<String, List<Double>> datasets = new LinkedHashMap<>();
363
364        for (int i = 0; i < percentileKeys.length; i++) {
365            String percentileLabel = percentileKeys[i] + SUFFIX_TH;
366            List<Double> percentileValues = extractValuesForPercentile(i, benchmarkNames, dataByBenchmark);
367            datasets.put(percentileLabel, percentileValues);
368        }
369
370        return datasets;
371    }
372
373    private List<Double> extractValuesForPercentile(int percentileIndex, List<String> benchmarkNames,
374            Map<String, List<Double>> dataByBenchmark) {
375        List<Double> percentileValues = new ArrayList<>();
376
377        for (String benchmarkName : benchmarkNames) {
378            List<Double> benchmarkData = dataByBenchmark.get(benchmarkName);
379            if (benchmarkData != null && percentileIndex < benchmarkData.size()) {
380                percentileValues.add(benchmarkData.get(percentileIndex));
381            } else {
382                percentileValues.add(null);
383            }
384        }
385
386        return percentileValues;
387    }
388
389    private Map<String, Object> createTrendData(String outputDir, BenchmarkMetrics metrics) {
390        // Check if external history directory is provided via system property
391        String externalHistoryPath = System.getProperty("benchmark.history.dir");
392        Path historyDir;
393
394        if (externalHistoryPath != null && !externalHistoryPath.isEmpty()) {
395            // Use external history directory (e.g., for CI/CD workflows)
396            historyDir = Path.of(externalHistoryPath);
397            LOGGER.info(INFO.USING_EXTERNAL_HISTORY_DIR, historyDir);
398        } else {
399            // Default to output directory/history (for local runs and tests)
400            historyDir = Path.of(outputDir, "history");
401        }
402
403        if (!Files.exists(historyDir)) {
404            // First run, no history available
405            LOGGER.info(INFO.HISTORY_DIR_NOT_FOUND, historyDir);
406            return createNoHistoryResponse();
407        }
408
409        List<TrendDataProcessor.HistoricalDataPoint> historicalData =
410                trendProcessor.loadHistoricalData(historyDir);
411
412        if (historicalData.isEmpty()) {
413            return createNoHistoryResponse();
414        }
415
416        TrendDataProcessor.TrendMetrics trendMetrics =
417                trendProcessor.calculateTrends(metrics, historicalData);
418
419        Map<String, Object> trendData = new LinkedHashMap<>();
420        trendData.put(AVAILABLE, true);
421        trendData.put("direction", trendMetrics.direction());
422        trendData.put("changePercentage", trendMetrics.changePercentage());
423        trendData.put("movingAverage", trendMetrics.movingAverage());
424        trendData.put("throughputTrend", trendMetrics.throughputTrend());
425        trendData.put("latencyTrend", trendMetrics.latencyTrend());
426        trendData.put("chartData", trendProcessor.generateTrendChartData(historicalData, metrics));
427        trendData.put("summary", generateTrendSummary(trendMetrics));
428
429        return trendData;
430    }
431
432    private Map<String, Object> createNoHistoryResponse() {
433        Map<String, Object> trends = new LinkedHashMap<>();
434        trends.put(AVAILABLE, false);
435        trends.put(MESSAGE, HISTORICAL_DATA_NOT_AVAILABLE);
436        return trends;
437    }
438
439    private String generateTrendSummary(TrendDataProcessor.TrendMetrics trendMetrics) {
440        String direction = trendMetrics.direction();
441        double change = Math.abs(trendMetrics.changePercentage());
442
443        if (BenchmarkConstants.Report.Badge.TrendDirection.STABLE.equals(direction)) {
444            return BenchmarkConstants.Report.Messages.PERFORMANCE_STABLE_FORMAT.formatted(change);
445        } else if (BenchmarkConstants.Report.Badge.TrendDirection.UP.equals(direction)) {
446            return BenchmarkConstants.Report.Messages.PERFORMANCE_IMPROVED_FORMAT.formatted(change);
447        } else {
448            return BenchmarkConstants.Report.Messages.PERFORMANCE_DECREASED_FORMAT.formatted(change);
449        }
450    }
451
452}