Add JDK 27 to the build matrix & move Dockerfile generation & image smoke-testing to GH Actions

.github/workflows/ci.yml

@@ -1,6 +1,16 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   test:
@@ -13,10 +23,6 @@ jobs:
       with:
         bb: 1.12.217
     - uses: actions/checkout@v4
-    - name: Check for stale Dockerfiles
-      run: |
-        bb run dockerfiles
-        git diff --exit-code
     - name: Run tests
       run: bb run test
 
@@ -32,3 +38,150 @@ jobs:
     - uses: actions/checkout@v4
     - name: Check formatting
       run: bb fmt-check
+
+  dockerfiles:
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      # whether the committed Dockerfiles are out of date
+      stale: ${{ steps.changes.outputs.stale }}
+      # JSON array of {id, dirs} chunks of build dirs to build
+      builds: ${{ steps.changes.outputs.builds }}
+
+    steps:
+    - name: Install babashka
+      uses: DeLaGuardo/[email protected]
+      with:
+        bb: 1.12.217
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Generate Dockerfiles
+      run: |
+        bb run dockerfiles
+        git add --all target
+    - name: Find changed Dockerfiles
+      id: changes
+      env:
+        # Compare against what the change started from so that Dockerfiles
+        # committed by hand get built too.
+        BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}
+        BUILDS_PER_JOB: 8
+      run: |
+        git cat-file -e "${BASE}^{commit}" 2>/dev/null || BASE=HEAD
+
+        if git diff --cached --quiet HEAD -- target; then
+          stale=false
+        else
+          stale=true
+        fi
+        echo "stale=$stale" >> "$GITHUB_OUTPUT"
+
+        build_dirs=$(bb run changed-build-dirs "$BASE")
+        builds=$(jq --raw-input --null-input --compact-output \
+          --argjson n "$BUILDS_PER_JOB" \
+          '[inputs | select(length > 0)]
+           | [range(0; length; $n) as $i
+              | {id: ($i / $n + 1), dirs: (.[$i:$i + $n] | join(" "))}]' \
+          <<< "$build_dirs")
+        echo "builds=$builds" >> "$GITHUB_OUTPUT"
+
+        {
+          echo "## Dockerfiles"
+          echo
+          if [ "$stale" = true ]; then
+            echo "The committed Dockerfiles are out of date. They are" \
+                 "regenerated and committed automatically on \`master\`."
+            echo
+            echo "<details><summary>Changes to be committed</summary>"
+            echo
+            echo '```diff'
+            git diff --cached --stat HEAD -- target
+            echo
+            git diff --cached HEAD -- target | head -c 500000
+            echo '```'
+            echo
+            echo "</details>"
+          else
+            echo "The committed Dockerfiles are up to date."
+          fi
+          echo
+          echo "### Images to build"
+          echo
+          if [ -n "$build_dirs" ]; then
+            echo '```'
+            echo "$build_dirs"
+            echo '```'
+          else
+            echo "None."
+          fi
+        } >> "$GITHUB_STEP_SUMMARY"
+
+  build:
+
+    needs: dockerfiles
+    if: needs.dockerfiles.outputs.builds != '[]'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk: ${{ fromJSON(needs.dockerfiles.outputs.builds) }}
+        platform:
+        - arch: amd64
+          runner: ubuntu-latest
+        - arch: arm64v8
+          runner: ubuntu-24.04-arm
+
+    name: build (${{ matrix.platform.arch }}, ${{ matrix.chunk.id }})
+    runs-on: ${{ matrix.platform.runner }}
+
+    steps:
+    - name: Install babashka
+      uses: DeLaGuardo/[email protected]
+      with:
+        bb: 1.12.217
+    - uses: actions/checkout@v4
+    - name: Build and smoke test images
+      env:
+        ARCH: ${{ matrix.platform.arch }}
+        BUILD_DIRS: ${{ matrix.chunk.dirs }}
+      # BUILD_DIRS is deliberately unquoted: it's a space-separated list
+      run: bb run build-images --architecture "$ARCH" --build-dir $BUILD_DIRS
+
+  commit-dockerfiles:
+
+    needs: [test, format, dockerfiles, build]
+    # build is skipped when there's nothing to build, so don't require it to
+    # have succeeded, only that nothing failed
+    if: >-
+      ${{ !failure() && !cancelled() && github.event_name == 'push'
+          && needs.dockerfiles.outputs.stale == 'true' }}
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+
+    steps:
+    - name: Install babashka
+      uses: DeLaGuardo/[email protected]
+      with:
+        bb: 1.12.217
+    - uses: actions/checkout@v4
+    - name: Regenerate and commit Dockerfiles
+      run: |
+        if [ "$(git ls-remote origin "$GITHUB_REF" | cut -f1)" != "$GITHUB_SHA" ]; then
+          echo "::notice::$GITHUB_REF_NAME has moved on; leaving the Dockerfiles to the run for its latest commit."
+          exit 0
+        fi
+
+        bb run dockerfiles
+        git add --all target
+        git -c user.name="github-actions[bot]" \
+            -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
+            commit -m "Regenerate Dockerfiles"
+        # Pushes made with GITHUB_TOKEN don't trigger workflows, so this
+        # doesn't start another CI run. The official-images PR workflow runs
+        # when this run completes and picks the commit up from there.
+        git push origin "HEAD:$GITHUB_REF"

.github/workflows/official-images-pr.yml

@@ -9,17 +9,38 @@ on:
 jobs:
   generate-docker-manifest:
     runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    # Only for CI runs of pushes to master, not PRs from a fork's master branch
+    if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' }}
+    outputs:
+      fresh: ${{ steps.dockerfiles.outputs.fresh }}
     steps:
       - name: Install babashka
         uses: DeLaGuardo/[email protected]
         with:
           bb: 1.12.217
+      # This checks out the tip of master, which includes the Dockerfiles the
+      # CI run that triggered us committed (if any).
       - name: Checkout code
         uses: actions/checkout@v6
+      # The manifest points official-images at one commit, so that commit's
+      # Dockerfiles have to match its config. They won't if master has moved
+      # on since the CI run; in that case the CI run for the newer commit
+      # regenerates them and triggers this workflow again.
+      - name: Check that the Dockerfiles are up to date
+        id: dockerfiles
+        run: |
+          bb run dockerfiles
+          if [ -z "$(git status --porcelain -- target)" ]; then
+            echo "fresh=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "fresh=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::Dockerfiles at $(git rev-parse --short HEAD) are out of date; skipping the official-images PR until CI regenerates them."
+          fi
       - name: Generate manifest
+        if: steps.dockerfiles.outputs.fresh == 'true'
         run: bb run manifest target/docker.manifest
       - name: Upload manifest artifact
+        if: steps.dockerfiles.outputs.fresh == 'true'
         uses: actions/upload-artifact@v7
         with:
           name: manifest
@@ -28,6 +49,7 @@ jobs:
   compare-manifests:
     runs-on: ubuntu-latest
     needs: [generate-docker-manifest]
+    if: needs.generate-docker-manifest.outputs.fresh == 'true'
     outputs:
       diff: ${{ steps.manifest-diff.outputs.diff }}
     steps:

README.md

@@ -133,6 +133,11 @@ Each of these can be invoked via `bb run [task]` at the command line.
 
 For example, to build all images locally, run `bb run build-images`.
 
+You don't need to regenerate the Dockerfiles in `target` yourself when you
+change the config or templates. CI builds and smoke tests a representative set
+of the images your change affects, and once it's merged to `master` it
+regenerates the Dockerfiles and commits them for you.
+
 ### buildx
 
 Note that you'll need to enable the new `buildx` feature and set as the default

bb.edn

@@ -11,6 +11,7 @@
                 :task    (apply dc/-main "dockerfiles" *command-line-args*)}
   manifest     (apply dc/-main "manifest" *command-line-args*)
   build-images {:task    (apply dc/-main "build-images" *command-line-args*)}
+  changed-build-dirs (apply dc/-main "changed-build-dirs" *command-line-args*)
   test         (clojure '-X:test)
   fmt-check    {:doc "Check formatting with cljfmt"
                 :task (fmt/check {:paths ["src" "test" "bb.edn" "deps.edn" "tests.edn"]})}

src/docker_clojure/changes.clj

@@ -0,0 +1,98 @@
+(ns docker-clojure.changes
+  "Works out which build dirs a change to the generated Dockerfiles touches and
+  which of them are worth building to verify it."
+  (:require [clojure.set :as set]
+            [clojure.string :as str]
+            [docker-clojure.config :as cfg]
+            [docker-clojure.dockerfile :as df]
+            [docker-clojure.util :refer [get-or-default]]))
+
+(defn- parse-file-diff
+  "Parses one file's section of a git diff into its build dir, file name,
+  status and changed lines. Returns nil for files that aren't in a build dir."
+  [[header & lines]]
+  (let [[_ path] (re-matches #"diff --git a/(.+) b/\1" header)
+        parts    (some-> path (str/split #"/"))]
+    (when (and (= 4 (count parts)) (= "target" (first parts)))
+      {:build-dir (str/join "/" (butlast parts))
+       :file      (last parts)
+       :status    (cond
+                    (some #(str/starts-with? % "new file mode") lines) :added
+                    (some #(str/starts-with? % "deleted file mode") lines) :deleted
+                    :else :modified)
+       :lines     (->> lines
+                       (drop-while #(not (str/starts-with? % "@@")))
+                       (filter #(re-find #"^[+-]" %)))})))
+
+(defn parse-diff
+  "Parses `git diff -U0 --no-renames` output into a map of build dir to its
+  change: `:added?` is true for new build dirs and `:signature` is equal for
+  build dirs that received exactly the same change. Deleted build dirs are
+  left out."
+  [diff]
+  (->> (str/split-lines diff)
+       (partition-by #(str/starts-with? % "diff --git "))
+       (partition 2)
+       (keep (fn [[header lines]] (parse-file-diff (concat header lines))))
+       (group-by :build-dir)
+       (keep (fn [[build-dir files]]
+               (let [dockerfile (some #(when (= "Dockerfile" (:file %)) %)
+                                      files)]
+                 (when-not (= :deleted (:status dockerfile))
+                   [build-dir
+                    {:added?    (= :added (:status dockerfile))
+                     :signature (set (map (juxt :file :status :lines)
+                                          files))}]))))
+       (into {})))
+
+(defn- preference
+  "Sort key that puts the default JDK version on its default distro with the
+  default build tool first."
+  [[build-dir {:keys [jdk-version distro build-tool]}]]
+  [(not= cfg/default-jdk-version jdk-version)
+   (not= (get-or-default cfg/default-distros jdk-version) distro)
+   (not= cfg/default-build-tool build-tool)
+   build-dir])
+
+(defn- traits
+  "What building a build dir's variant exercises: its JDK version on its
+  distro (i.e. its base image) and its build tool."
+  [[_build-dir {:keys [jdk-version distro build-tool]}]]
+  #{[:base jdk-version distro] [:build-tool build-tool]})
+
+(defn- covering
+  "Greedily picks from `dir-variants` until every JDK version & distro pair
+  and every build tool among them has been picked at least once."
+  [dir-variants]
+  (loop [candidates (sort-by preference dir-variants)
+         uncovered  (into #{} (mapcat traits) dir-variants)
+         picked     []]
+    (if (empty? uncovered)
+      picked
+      (let [gain #(count (set/intersection uncovered (traits %)))
+            best (first (sort-by (comp - gain) candidates))]
+        (recur (remove #{best} candidates)
+               (set/difference uncovered (traits best))
+               (conj picked best))))))
+
+(defn select-build-dirs
+  "Returns the sorted build dirs worth building to verify `changes` (see
+  `parse-diff`). Of the new build dirs, just enough are built to exercise each
+  of their JDK version & distro pairs and build tools once. Modified build dirs
+  that received exactly the same change are represented by one of them. Both
+  prefer the default JDK version on its default distro."
+  [variants changes]
+  (let [dir->variant (into {} (map (juxt df/build-dir identity)) variants)
+        dir-variants (fn [dir-changes]
+                       (map (fn [[build-dir _]]
+                              [build-dir (dir->variant build-dir)])
+                            dir-changes))
+        known        (filter (comp dir->variant key) changes)
+        added        (filter (comp :added? val) known)
+        modified     (remove (comp :added? val) known)]
+    (->> (group-by (comp :signature val) modified)
+         vals
+         (map #(first (sort-by preference (dir-variants %))))
+         (concat (covering (dir-variants added)))
+         (map first)
+         sort)))

src/docker_clojure/config.clj

@@ -85,7 +85,7 @@
 
 (def git-repo "https://github.com/Quantisan/docker-clojure.git")
 
-(def jdk-versions #{8 11 17 21 25 26})
+(def jdk-versions #{8 11 17 21 25 26 27})
 
 (def base-images
   "Map of JDK version to base image name(s) with :default as a fallback"
@@ -100,7 +100,7 @@
 (def distros
   "Map of base image name to set of distro tags to use, namespaced by Linux
   distro type. :default key is a fallback for base images not o/w specified."
-  {:default #{:alpine/alpine :ubuntu/jammy :ubuntu/noble}
+  {:default #{:alpine/alpine :ubuntu/jammy :ubuntu/noble :ubuntu/resolute}
    "debian" #{:debian-slim/bookworm-slim :debian/bookworm
               :debian-slim/trixie-slim :debian/trixie}})
 
@@ -132,6 +132,12 @@
     ;; No more jammy builds for JDK 23+
     {:jdk-version #(>= % 23)
      :distro      :ubuntu/jammy}
+    ;; No noble builds for JDK 27+
+    {:jdk-version #(>= % 27)
+     :distro      :ubuntu/noble}
+    ;; Only build resolute for JDK 26+
+    {:jdk-version #(< % 26)
+     :distro      :ubuntu/resolute}
     ;; No upstream ARM alpine images available before JDK 21
     {:jdk-version  #(< % 21)
      :architecture "arm64v8"