Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@

## Bug fixes

- Split results by `paramset_name` in `get_results` instead of merging all parameter sets of a method, and show each parameter set as a separate method in `render_report` (PR #946).

- Read the task commit from `task_info.yaml` in `get_task_info`, instead of always writing a null (PR #943).

- Fix metric direction (`maximize`/`minimize`) handling in `generate_qc` and `render_report` components of the `process_task_results` workflow (PR #936).
Expand Down
46 changes: 35 additions & 11 deletions scripts/create_resources/task_results_v4.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,18 @@ echo
echo ">>> Adding dummy paramsets if none are present..."
# get_results keys its output by parameter set as well as by method, so the test
# resources need score entries with a paramset. This run predates paramsets, so
# duplicate one method's scores under two dummy paramsets. The report drops the
# extra rows (they have no metric component runs), so the rest of the processed
# output is unaffected.
python3 - "$OUT_DIR/raw/score_uns.yaml" << 'HERE'
# duplicate one method's scores and trace rows under two dummy paramsets, tagged
# "<dataset>.<method>.<paramset>[.<metric>]" like a run_benchmark workflow would.
python3 - "$OUT_DIR/raw/score_uns.yaml" "$OUT_DIR/raw/trace.txt" << 'HERE'
import copy
import sys
import yaml

path = sys.argv[1]
with open(path) as f:
scores_path, trace_path = sys.argv[1:3]
with open(scores_path) as f:
scores = yaml.safe_load(f)

if any("paramset_id" in entry for entry in scores):
if any(entry.get("paramset_name") for entry in scores):
print("Scores already contain paramsets; leaving them as-is.")
sys.exit(0)

Expand All @@ -54,21 +53,46 @@ paramsets = {
"dummy_paramset_1": {"learning_rate": 0.1, "n_iterations": 100, "mode": "fast"},
"dummy_paramset_2": {"learning_rate": 0.01, "n_iterations": 500, "mode": "accurate"},
}

# duplicate the scores
new_entries = []
for entry in scores:
if entry["method_id"] != method_id:
continue
for paramset_id, paramset in paramsets.items():
for paramset_name, paramset in paramsets.items():
dup = copy.deepcopy(entry)
dup["paramset_id"] = paramset_id
dup["paramset_name"] = paramset_name
dup["paramset"] = dict(paramset)
new_entries.append(dup)

scores.extend(new_entries)
with open(path, "w") as f:
with open(scores_path, "w") as f:
yaml.safe_dump(scores, f, sort_keys=False)

print(f"Added {len(new_entries)} dummy paramset entries for method '{method_id}'.")
# duplicate the trace rows of the method and of the metrics run on its output
with open(trace_path) as f:
lines = f.read().splitlines()

name_ix = lines[0].split("\t").index("name")
new_lines = []
for line in lines[1:]:
fields = line.split("\t")
process, _, tag = fields[name_ix].partition(" (")
tag_parts = tag.rstrip(")").split(".")
if len(tag_parts) < 2 or tag_parts[1] != method_id:
continue
for paramset_name in paramsets:
new_tag = ".".join(tag_parts[:2] + [paramset_name] + tag_parts[2:])
fields[name_ix] = f"{process} ({new_tag})"
new_lines.append("\t".join(fields))

with open(trace_path, "w") as f:
f.write("\n".join(lines + new_lines) + "\n")

print(
f"Added {len(new_entries)} dummy paramset entries and {len(new_lines)} trace "
f"rows for method '{method_id}'."
)
HERE

echo
Expand Down
43 changes: 29 additions & 14 deletions src/reporting/get_results/script.R
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ scores <- score_entries |>

.x[c("dataset_id", "method_id", "metric_ids", "metric_values")] |>
tibble::as_tibble() |>
dplyr::mutate(paramset_name = .x$paramset_id %||% NA_character_)
dplyr::mutate(paramset_name = .x$paramset_name %||% NA_character_)
}) |>
dplyr::rename(
dataset_name = dataset_id,
Expand All @@ -146,7 +146,7 @@ run_index <- score_entries |>
tibble::tibble(
dataset_name = .x$dataset_id,
method_name = .x$method_id,
paramset_name = .x$paramset_id %||% NA_character_,
paramset_name = .x$paramset_name %||% NA_character_,
paramset = list(if (is.null(.x$paramset)) NULL else paramset),
run_ids = list(as.character(unlist(.x$run_ids) %||% character(0))),
metric_component = .x$metric_component %||% NA_character_,
Expand Down Expand Up @@ -234,14 +234,28 @@ trace <- readr::read_tsv(
)

if (!use_run_ids) {
# Split the process tag into dataset, method and metric
# Split the process tag: "<dataset>.<method>[.<paramset>][.<metric>]"
trace <- trace |>
tidyr::separate_wider_delim(
id,
delim = ".",
names = c("dataset_name", "method_name", "metric_component"),
names = c("dataset_name", "method_name", "tag_rest"),
too_few = "align_start",
too_many = "drop"
too_many = "merge"
) |>
dplyr::mutate(
metric_component = stringr::str_extract(tag_rest, "[^.]+$"),
metric_component = dplyr::if_else(
metric_component %in% metric_components,
metric_component,
NA_character_
),
paramset_name = dplyr::if_else(
is.na(metric_component),
tag_rest,
stringr::str_remove(tag_rest, "\\.?[^.]+$")
),
paramset_name = dplyr::na_if(paramset_name, "")
) |>
# Only keep method and metric processes
dplyr::filter(
Expand All @@ -252,6 +266,7 @@ if (!use_run_ids) {
process,
dataset_name,
method_name,
paramset_name,
metric_component,
tidyselect::starts_with("run_")
)
Expand Down Expand Up @@ -366,11 +381,9 @@ if (use_run_ids) {
dplyr::filter(
method_name %in% method_names,
is.na(metric_component)
) |>
dplyr::mutate(paramset_name = NA_character_)
)
metric_trace <- trace |>
dplyr::filter(metric_component %in% metric_components) |>
dplyr::mutate(paramset_name = NA_character_)
dplyr::filter(metric_component %in% metric_components)
}

cat("Extracting method resources...\n", sep = "")
Expand All @@ -387,9 +400,11 @@ metric_component_names <- purrr::map_chr(metric_info, "component_name")
metric_component_map <- purrr::map_chr(metric_info, "name") |>
purrr::set_names(metric_component_names)

# Look up parameter sets by method, so that failed runs get their parameters too
paramset_keys <- c("method_name", "paramset_name")
paramset_values <- run_index |>
dplyr::select(tidyselect::all_of(result_keys), paramset) |>
dplyr::distinct(dplyr::across(tidyselect::all_of(result_keys)), .keep_all = TRUE)
dplyr::select(tidyselect::all_of(paramset_keys), paramset) |>
dplyr::distinct(dplyr::across(tidyselect::all_of(paramset_keys)), .keep_all = TRUE)

results <- scores |>
# There shouldn't be any but remove missing/NaN values just in case
Expand Down Expand Up @@ -428,12 +443,12 @@ results <- scores |>
}
)
) |>
dplyr::left_join(paramset_values, by = result_keys) |>
dplyr::left_join(paramset_values, by = paramset_keys) |>
dplyr::mutate(
metric_names = map_missing_to_empty(metric_names, mode = "character"),
metric_values = map_missing_to_empty(metric_values, mode = "numeric"),
# A paramset row has no trace rows of its own in tag-attribution mode, so it
# gets empty run stats rather than nulls the schema rejects.
# A row whose processes are not in the trace under its parameter set gets empty
# run stats rather than nulls the schema rejects.
run_exit_code = map_missing_to_empty(run_exit_code, mode = "integer"),
run_duration_secs = map_missing_to_empty(run_duration_secs, mode = "numeric"),
run_cpu_pct = map_missing_to_empty(run_cpu_pct, mode = "numeric"),
Expand Down
55 changes: 55 additions & 0 deletions src/reporting/render_report/report-functions.R
Original file line number Diff line number Diff line change
Expand Up @@ -569,3 +569,58 @@ aggregate_scores <- function(scores) {

mean(scores, na.rm = TRUE)
}

#' Split methods by parameter set
#'
#' @param results Results list from results JSON
#' @param method_info Method info list from results JSON
#'
#' @returns A list with items "results" and "method_info" in which every
#' parameter set of a method is a method of its own
#'
#' @details
#' Results are keyed by parameter set as well as by method but the report is
#' keyed by method alone. A method run with parameter sets is therefore shown as
#' one method per set, named "<method>.<paramset>" as in the benchmark workflow.
split_methods_by_paramset <- function(results, method_info) {
paramsets <- purrr::map_dfr(results, function(.result) {
data.frame(
method = .result$method_name,
paramset = .result$paramset_name %||% NA_character_
)
}) |>
dplyr::distinct()

results <- purrr::map(results, function(.result) {
if (!is.null(.result$paramset_name)) {
.result$method_name <- paste0(
.result$method_name,
".",
.result$paramset_name
)
}

.result
})

method_info <- purrr::map(method_info, function(.method) {
method_paramsets <- paramsets$paramset[paramsets$method == .method$name]

# Methods without parameter sets, or without any results, are kept as they are
if (all(is.na(method_paramsets))) {
return(list(.method))
}

purrr::map(method_paramsets, function(.paramset) {
if (!is.na(.paramset)) {
.method$name <- paste0(.method$name, ".", .paramset)
.method$label <- paste0(.method$label, " (", .paramset, ")")
}

.method
})
}) |>
purrr::list_flatten()

list(results = results, method_info = method_info)
}
5 changes: 5 additions & 0 deletions src/reporting/render_report/report-template.qmd
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,11 @@ Click the tabs to see the plots for each metric.
```{r}
#| label: normalization-controls
#| results: asis
# From here on every parameter set of a method is shown as a method of its own
split_results <- split_methods_by_paramset(task_results$results, method_info)
task_results$results <- split_results$results
method_info <- split_results$method_info

dataset_names <- purrr::map_chr(dataset_info, "name")
method_names <- purrr::map_chr(method_info, "name")
metric_names <- purrr::map_chr(metric_info, "name")
Expand Down
Loading