From a13037b85542fc5dd8f3cf6be9692bc9ce72bea0 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 1 Oct 2026 20:55:04 +0200 Subject: [PATCH 1/4] split results by paramset_name --- src/reporting/get_results/script.R | 43 ++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/reporting/get_results/script.R b/src/reporting/get_results/script.R index ea629784b..1793d78be 100644 --- a/src/reporting/get_results/script.R +++ b/src/reporting/get_results/script.R @@ -124,7 +124,7 @@ scores <- score_entries |> .x[c("dataset_id", "method_id", "metric_ids", "metric_values")] |> tibble::as_tibble() |> - dplyr::mutate(paramset_name = .x$paramset_id %||% NA_character_) + dplyr::mutate(paramset_name = .x$paramset_name %||% NA_character_) }) |> dplyr::rename( dataset_name = dataset_id, @@ -146,7 +146,7 @@ run_index <- score_entries |> tibble::tibble( dataset_name = .x$dataset_id, method_name = .x$method_id, - paramset_name = .x$paramset_id %||% NA_character_, + paramset_name = .x$paramset_name %||% NA_character_, paramset = list(if (is.null(.x$paramset)) NULL else paramset), run_ids = list(as.character(unlist(.x$run_ids) %||% character(0))), metric_component = .x$metric_component %||% NA_character_, @@ -234,14 +234,28 @@ trace <- readr::read_tsv( ) if (!use_run_ids) { - # Split the process tag into dataset, method and metric + # Split the process tag: ".[.][.]" trace <- trace |> tidyr::separate_wider_delim( id, delim = ".", - names = c("dataset_name", "method_name", "metric_component"), + names = c("dataset_name", "method_name", "tag_rest"), too_few = "align_start", - too_many = "drop" + too_many = "merge" + ) |> + dplyr::mutate( + metric_component = stringr::str_extract(tag_rest, "[^.]+$"), + metric_component = dplyr::if_else( + metric_component %in% metric_components, + metric_component, + NA_character_ + ), + paramset_name = dplyr::if_else( + is.na(metric_component), + tag_rest, + stringr::str_remove(tag_rest, "\\.?[^.]+$") + ), + paramset_name = dplyr::na_if(paramset_name, "") ) |> # Only keep method and metric processes dplyr::filter( @@ -252,6 +266,7 @@ if (!use_run_ids) { process, dataset_name, method_name, + paramset_name, metric_component, tidyselect::starts_with("run_") ) @@ -366,11 +381,9 @@ if (use_run_ids) { dplyr::filter( method_name %in% method_names, is.na(metric_component) - ) |> - dplyr::mutate(paramset_name = NA_character_) + ) metric_trace <- trace |> - dplyr::filter(metric_component %in% metric_components) |> - dplyr::mutate(paramset_name = NA_character_) + dplyr::filter(metric_component %in% metric_components) } cat("Extracting method resources...\n", sep = "") @@ -387,9 +400,11 @@ metric_component_names <- purrr::map_chr(metric_info, "component_name") metric_component_map <- purrr::map_chr(metric_info, "name") |> purrr::set_names(metric_component_names) +# Look up parameter sets by method, so that failed runs get their parameters too +paramset_keys <- c("method_name", "paramset_name") paramset_values <- run_index |> - dplyr::select(tidyselect::all_of(result_keys), paramset) |> - dplyr::distinct(dplyr::across(tidyselect::all_of(result_keys)), .keep_all = TRUE) + dplyr::select(tidyselect::all_of(paramset_keys), paramset) |> + dplyr::distinct(dplyr::across(tidyselect::all_of(paramset_keys)), .keep_all = TRUE) results <- scores |> # There shouldn't be any but remove missing/NaN values just in case @@ -428,12 +443,12 @@ results <- scores |> } ) ) |> - dplyr::left_join(paramset_values, by = result_keys) |> + dplyr::left_join(paramset_values, by = paramset_keys) |> dplyr::mutate( metric_names = map_missing_to_empty(metric_names, mode = "character"), metric_values = map_missing_to_empty(metric_values, mode = "numeric"), - # A paramset row has no trace rows of its own in tag-attribution mode, so it - # gets empty run stats rather than nulls the schema rejects. + # A row whose processes are not in the trace under its parameter set gets empty + # run stats rather than nulls the schema rejects. run_exit_code = map_missing_to_empty(run_exit_code, mode = "integer"), run_duration_secs = map_missing_to_empty(run_duration_secs, mode = "numeric"), run_cpu_pct = map_missing_to_empty(run_cpu_pct, mode = "numeric"), From 68f05d35e411c0c3a54db9958496f5446feac812 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 1 Oct 2026 20:55:04 +0200 Subject: [PATCH 2/4] show paramsets as methods in report --- .../render_report/report-functions.R | 55 +++++++++++++++++++ .../render_report/report-template.qmd | 5 ++ 2 files changed, 60 insertions(+) diff --git a/src/reporting/render_report/report-functions.R b/src/reporting/render_report/report-functions.R index 535ba3581..96fe4bce3 100644 --- a/src/reporting/render_report/report-functions.R +++ b/src/reporting/render_report/report-functions.R @@ -569,3 +569,58 @@ aggregate_scores <- function(scores) { mean(scores, na.rm = TRUE) } + +#' Split methods by parameter set +#' +#' @param results Results list from results JSON +#' @param method_info Method info list from results JSON +#' +#' @returns A list with items "results" and "method_info" in which every +#' parameter set of a method is a method of its own +#' +#' @details +#' Results are keyed by parameter set as well as by method but the report is +#' keyed by method alone. A method run with parameter sets is therefore shown as +#' one method per set, named "." as in the benchmark workflow. +split_methods_by_paramset <- function(results, method_info) { + paramsets <- purrr::map_dfr(results, function(.result) { + data.frame( + method = .result$method_name, + paramset = .result$paramset_name %||% NA_character_ + ) + }) |> + dplyr::distinct() + + results <- purrr::map(results, function(.result) { + if (!is.null(.result$paramset_name)) { + .result$method_name <- paste0( + .result$method_name, + ".", + .result$paramset_name + ) + } + + .result + }) + + method_info <- purrr::map(method_info, function(.method) { + method_paramsets <- paramsets$paramset[paramsets$method == .method$name] + + # Methods without parameter sets, or without any results, are kept as they are + if (all(is.na(method_paramsets))) { + return(list(.method)) + } + + purrr::map(method_paramsets, function(.paramset) { + if (!is.na(.paramset)) { + .method$name <- paste0(.method$name, ".", .paramset) + .method$label <- paste0(.method$label, " (", .paramset, ")") + } + + .method + }) + }) |> + purrr::list_flatten() + + list(results = results, method_info = method_info) +} diff --git a/src/reporting/render_report/report-template.qmd b/src/reporting/render_report/report-template.qmd index 886ae98cf..27db33aaa 100644 --- a/src/reporting/render_report/report-template.qmd +++ b/src/reporting/render_report/report-template.qmd @@ -500,6 +500,11 @@ Click the tabs to see the plots for each metric. ```{r} #| label: normalization-controls #| results: asis +# From here on every parameter set of a method is shown as a method of its own +split_results <- split_methods_by_paramset(task_results$results, method_info) +task_results$results <- split_results$results +method_info <- split_results$method_info + dataset_names <- purrr::map_chr(dataset_info, "name") method_names <- purrr::map_chr(method_info, "name") metric_names <- purrr::map_chr(metric_info, "name") From 9bb8481e9fe688fd1ee084db88da6ed1617e9844 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 1 Oct 2026 20:55:04 +0200 Subject: [PATCH 3/4] add paramset trace rows to test data --- scripts/create_resources/task_results_v4.sh | 46 ++++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/scripts/create_resources/task_results_v4.sh b/scripts/create_resources/task_results_v4.sh index a61669903..89e114e70 100755 --- a/scripts/create_resources/task_results_v4.sh +++ b/scripts/create_resources/task_results_v4.sh @@ -33,19 +33,18 @@ echo echo ">>> Adding dummy paramsets if none are present..." # get_results keys its output by parameter set as well as by method, so the test # resources need score entries with a paramset. This run predates paramsets, so -# duplicate one method's scores under two dummy paramsets. The report drops the -# extra rows (they have no metric component runs), so the rest of the processed -# output is unaffected. -python3 - "$OUT_DIR/raw/score_uns.yaml" << 'HERE' +# duplicate one method's scores and trace rows under two dummy paramsets, tagged +# "..[.]" like a run_benchmark workflow would. +python3 - "$OUT_DIR/raw/score_uns.yaml" "$OUT_DIR/raw/trace.txt" << 'HERE' import copy import sys import yaml -path = sys.argv[1] -with open(path) as f: +scores_path, trace_path = sys.argv[1:3] +with open(scores_path) as f: scores = yaml.safe_load(f) -if any("paramset_id" in entry for entry in scores): +if any(entry.get("paramset_name") for entry in scores): print("Scores already contain paramsets; leaving them as-is.") sys.exit(0) @@ -54,21 +53,46 @@ paramsets = { "dummy_paramset_1": {"learning_rate": 0.1, "n_iterations": 100, "mode": "fast"}, "dummy_paramset_2": {"learning_rate": 0.01, "n_iterations": 500, "mode": "accurate"}, } + +# duplicate the scores new_entries = [] for entry in scores: if entry["method_id"] != method_id: continue - for paramset_id, paramset in paramsets.items(): + for paramset_name, paramset in paramsets.items(): dup = copy.deepcopy(entry) - dup["paramset_id"] = paramset_id + dup["paramset_name"] = paramset_name dup["paramset"] = dict(paramset) new_entries.append(dup) scores.extend(new_entries) -with open(path, "w") as f: +with open(scores_path, "w") as f: yaml.safe_dump(scores, f, sort_keys=False) -print(f"Added {len(new_entries)} dummy paramset entries for method '{method_id}'.") +# duplicate the trace rows of the method and of the metrics run on its output +with open(trace_path) as f: + lines = f.read().splitlines() + +name_ix = lines[0].split("\t").index("name") +new_lines = [] +for line in lines[1:]: + fields = line.split("\t") + process, _, tag = fields[name_ix].partition(" (") + tag_parts = tag.rstrip(")").split(".") + if len(tag_parts) < 2 or tag_parts[1] != method_id: + continue + for paramset_name in paramsets: + new_tag = ".".join(tag_parts[:2] + [paramset_name] + tag_parts[2:]) + fields[name_ix] = f"{process} ({new_tag})" + new_lines.append("\t".join(fields)) + +with open(trace_path, "w") as f: + f.write("\n".join(lines + new_lines) + "\n") + +print( + f"Added {len(new_entries)} dummy paramset entries and {len(new_lines)} trace " + f"rows for method '{method_id}'." +) HERE echo From b1c53cf04a14ecfe50d7267b91ea627fa531ecde Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 1 Oct 2026 20:56:46 +0200 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 095576c07..fc8e9db4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ ## Bug fixes +- Split results by `paramset_name` in `get_results` instead of merging all parameter sets of a method, and show each parameter set as a separate method in `render_report` (PR #946). + - Read the task commit from `task_info.yaml` in `get_task_info`, instead of always writing a null (PR #943). - Fix metric direction (`maximize`/`minimize`) handling in `generate_qc` and `render_report` components of the `process_task_results` workflow (PR #936).