diff --git a/.github/acton-programs.sh b/.github/acton-programs.sh new file mode 100755 index 00000000..61c51586 --- /dev/null +++ b/.github/acton-programs.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Replace the Acton programs (.act files) in bench/algorithm with the programs +# maintained in test/perf-benchgames of the Acton repository. Each problem gets +# exactly the .act files of its directory there. ACTON_REPO and ACTON_REF select +# the source; the default is the commit of the published Acton tip build. +set -euo pipefail +shopt -s nullglob +cd "$(dirname "$0")/.." +repo="${ACTON_REPO:-https://github.com/actonlang/acton.git}" +ref="${ACTON_REF:-tip}" + +source_dir="$(mktemp -d)" +trap 'rm -rf "$source_dir"' EXIT +git -c advice.detachedHead=false clone --quiet --depth 1 --branch "$ref" --filter=blob:none --sparse "$repo" "$source_dir" +git -C "$source_dir" sparse-checkout set test/perf-benchgames +commit="$(git -C "$source_dir" rev-parse HEAD)" +programs="$source_dir/test/perf-benchgames" + +missing=() +for problem_dir in bench/algorithm/*/; do + problem="$(basename "$problem_dir")" + ours=("$problem_dir"*.act) + theirs=("$programs/$problem/"*.act) + if (( ${#ours[@]} > 0 && ${#theirs[@]} == 0 )); then + missing+=("$problem") + fi +done +if (( ${#missing[@]} )); then + echo "Acton programs missing from $repo at $ref ($commit): ${missing[*]}" >&2 + exit 1 +fi +for problem_dir in "$programs"/*/; do + problem="$(basename "$problem_dir")" + theirs=("$problem_dir"*.act) + (( ${#theirs[@]} )) || continue + if [[ -d "bench/algorithm/$problem" ]]; then + rm -f "bench/algorithm/$problem/"*.act + cp "${theirs[@]}" "bench/algorithm/$problem/" + else + echo "Acton programs without a benchmark problem here: $problem" >&2 + fi +done +echo "Acton programs: $repo $ref $commit" diff --git a/.github/run-language.sh b/.github/run-language.sh index 42f05965..4c524446 100644 --- a/.github/run-language.sh +++ b/.github/run-language.sh @@ -5,6 +5,11 @@ mode="${1:?check or measure}" language="${2:?language}" [[ "$mode" == check || "$mode" == measure ]] toolchain_image="$(python3 -c 'import json,sys; print(json.load(open(".github/languages.json"))[sys.argv[1]].get("image", "mcr.microsoft.com/dotnet/sdk:9.0-noble"))' "$language")" +acton_programs="" +if [[ "$language" == acton || "$language" == acton-tip ]]; then + acton_programs="$(bash .github/acton-programs.sh)" + echo "$acton_programs" +fi mkdir -p "${XDG_STATE_HOME:-$HOME/.local/state}" exec 9>"${XDG_STATE_HOME:-$HOME/.local/state}/acton-perf.lock" @@ -58,3 +63,6 @@ echo "::endgroup::" "${engine[@]}" image inspect "$image" --format '{{.Id}}' \ >> "bench/build/environment-${language}.txt" 9>&- +if [[ -n "$acton_programs" ]]; then + echo "$acton_programs" >> "bench/build/environment-${language}.txt" +fi diff --git a/README.md b/README.md index 6067283c..eb9d199f 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,18 @@ Use `acton-tip` to select the development build in workflow inputs or local commands. Go uses one 1.26 toolchain throughout the job; dependencies cannot silently select another compiler. +The Acton programs are maintained in `test/perf-benchgames` of the +[Acton repository](https://github.com/actonlang/acton). Before building, both +Acton jobs run `.github/acton-programs.sh`. It gives each problem in +`bench/algorithm` exactly the `.act` files of the same problem directory at the +Acton `tip` tag, the commit of the published development build. The `source` +lists in `bench/bench_acton.yaml` and `bench/bench_acton_tip.yaml` still choose +which programs to build. The job records the Acton commit in its environment +file. A program that needs a newer compiler than the latest release fails the +stable Acton job, so that run has no stable Acton results. `ACTON_REPO` and +`ACTON_REF` select another source, for example a branch in a local Acton +checkout: `ACTON_REPO=file:///path/to/acton ACTON_REF=my-branch`. + ## Running locally Install .NET 9 and the compilers or runtimes you want to compare. Run commands @@ -87,6 +99,9 @@ dotnet run --no-launch-profile -c Release --project tool -- --task bench --langs dotnet run --no-launch-profile -c Release --project tool -- --task build --langs python --compilers cpython --problems binarytrees --no-docker ``` +Run `bash .github/acton-programs.sh` first to build the Acton programs from the +Acton repository instead of the copies in `bench/algorithm`. + `bench/bench.yaml` defines the inputs and expected outputs. Language configurations are in `bench/bench_*.yaml`. `--no-docker` uses locally installed compilers instead of the container images in those configurations. On Linux with sudo access, the CI wrapper installs Podman if needed and uses diff --git a/bench/bench_acton.yaml b/bench/bench_acton.yaml index a6c6c921..4713119a 100644 --- a/bench/bench_acton.yaml +++ b/bench/bench_acton.yaml @@ -1,3 +1,6 @@ +# The programs come from test/perf-benchgames in the Acton repository: +# .github/acton-programs.sh copies their .act files into bench/algorithm before +# each run. A program is built only when it is listed under source. lang: acton problems: - name: binarytrees diff --git a/bench/bench_acton_tip.yaml b/bench/bench_acton_tip.yaml index 6e07f196..d5a8d469 100644 --- a/bench/bench_acton_tip.yaml +++ b/bench/bench_acton_tip.yaml @@ -1,3 +1,6 @@ +# The programs come from test/perf-benchgames in the Acton repository: +# .github/acton-programs.sh copies their .act files into bench/algorithm before +# each run. A program is built only when it is listed under source. lang: acton-tip problems: - name: binarytrees