diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6beaddf..9bf4f557 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,16 @@ name: CI -on: [push, pull_request] +on: + push: + branches: [master] + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: test: @@ -13,10 +23,6 @@ jobs: with: bb: 1.12.217 - uses: actions/checkout@v4 - - name: Check for stale Dockerfiles - run: | - bb run dockerfiles - git diff --exit-code - name: Run tests run: bb run test @@ -32,3 +38,150 @@ jobs: - uses: actions/checkout@v4 - name: Check formatting run: bb fmt-check + + dockerfiles: + + runs-on: ubuntu-latest + + outputs: + # whether the committed Dockerfiles are out of date + stale: ${{ steps.changes.outputs.stale }} + # JSON array of {id, dirs} chunks of build dirs to build + builds: ${{ steps.changes.outputs.builds }} + + steps: + - name: Install babashka + uses: DeLaGuardo/setup-clojure@13.4 + with: + bb: 1.12.217 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Generate Dockerfiles + run: | + bb run dockerfiles + git add --all target + - name: Find changed Dockerfiles + id: changes + env: + # Compare against what the change started from so that Dockerfiles + # committed by hand get built too. + BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }} + BUILDS_PER_JOB: 8 + run: | + git cat-file -e "${BASE}^{commit}" 2>/dev/null || BASE=HEAD + + if git diff --cached --quiet HEAD -- target; then + stale=false + else + stale=true + fi + echo "stale=$stale" >> "$GITHUB_OUTPUT" + + build_dirs=$(bb run changed-build-dirs "$BASE") + builds=$(jq --raw-input --null-input --compact-output \ + --argjson n "$BUILDS_PER_JOB" \ + '[inputs | select(length > 0)] + | [range(0; length; $n) as $i + | {id: ($i / $n + 1), dirs: (.[$i:$i + $n] | join(" "))}]' \ + <<< "$build_dirs") + echo "builds=$builds" >> "$GITHUB_OUTPUT" + + { + echo "## Dockerfiles" + echo + if [ "$stale" = true ]; then + echo "The committed Dockerfiles are out of date. They are" \ + "regenerated and committed automatically on \`master\`." + echo + echo "
Changes to be committed" + echo + echo '```diff' + git diff --cached --stat HEAD -- target + echo + git diff --cached HEAD -- target | head -c 500000 + echo '```' + echo + echo "
" + else + echo "The committed Dockerfiles are up to date." + fi + echo + echo "### Images to build" + echo + if [ -n "$build_dirs" ]; then + echo '```' + echo "$build_dirs" + echo '```' + else + echo "None." + fi + } >> "$GITHUB_STEP_SUMMARY" + + build: + + needs: dockerfiles + if: needs.dockerfiles.outputs.builds != '[]' + + strategy: + fail-fast: false + matrix: + chunk: ${{ fromJSON(needs.dockerfiles.outputs.builds) }} + platform: + - arch: amd64 + runner: ubuntu-latest + - arch: arm64v8 + runner: ubuntu-24.04-arm + + name: build (${{ matrix.platform.arch }}, ${{ matrix.chunk.id }}) + runs-on: ${{ matrix.platform.runner }} + + steps: + - name: Install babashka + uses: DeLaGuardo/setup-clojure@13.4 + with: + bb: 1.12.217 + - uses: actions/checkout@v4 + - name: Build and smoke test images + env: + ARCH: ${{ matrix.platform.arch }} + BUILD_DIRS: ${{ matrix.chunk.dirs }} + # BUILD_DIRS is deliberately unquoted: it's a space-separated list + run: bb run build-images --architecture "$ARCH" --build-dir $BUILD_DIRS + + commit-dockerfiles: + + needs: [test, format, dockerfiles, build] + # build is skipped when there's nothing to build, so don't require it to + # have succeeded, only that nothing failed + if: >- + ${{ !failure() && !cancelled() && github.event_name == 'push' + && needs.dockerfiles.outputs.stale == 'true' }} + + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - name: Install babashka + uses: DeLaGuardo/setup-clojure@13.4 + with: + bb: 1.12.217 + - uses: actions/checkout@v4 + - name: Regenerate and commit Dockerfiles + run: | + if [ "$(git ls-remote origin "$GITHUB_REF" | cut -f1)" != "$GITHUB_SHA" ]; then + echo "::notice::$GITHUB_REF_NAME has moved on; leaving the Dockerfiles to the run for its latest commit." + exit 0 + fi + + bb run dockerfiles + git add --all target + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -m "Regenerate Dockerfiles" + # Pushes made with GITHUB_TOKEN don't trigger workflows, so this + # doesn't start another CI run. The official-images PR workflow runs + # when this run completes and picks the commit up from there. + git push origin "HEAD:$GITHUB_REF" diff --git a/.github/workflows/official-images-pr.yml b/.github/workflows/official-images-pr.yml index dcb2de21..20f64292 100644 --- a/.github/workflows/official-images-pr.yml +++ b/.github/workflows/official-images-pr.yml @@ -9,17 +9,38 @@ on: jobs: generate-docker-manifest: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} + # Only for CI runs of pushes to master, not PRs from a fork's master branch + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' }} + outputs: + fresh: ${{ steps.dockerfiles.outputs.fresh }} steps: - name: Install babashka uses: DeLaGuardo/setup-clojure@13.6.0 with: bb: 1.12.217 + # This checks out the tip of master, which includes the Dockerfiles the + # CI run that triggered us committed (if any). - name: Checkout code uses: actions/checkout@v6 + # The manifest points official-images at one commit, so that commit's + # Dockerfiles have to match its config. They won't if master has moved + # on since the CI run; in that case the CI run for the newer commit + # regenerates them and triggers this workflow again. + - name: Check that the Dockerfiles are up to date + id: dockerfiles + run: | + bb run dockerfiles + if [ -z "$(git status --porcelain -- target)" ]; then + echo "fresh=true" >> "$GITHUB_OUTPUT" + else + echo "fresh=false" >> "$GITHUB_OUTPUT" + echo "::notice::Dockerfiles at $(git rev-parse --short HEAD) are out of date; skipping the official-images PR until CI regenerates them." + fi - name: Generate manifest + if: steps.dockerfiles.outputs.fresh == 'true' run: bb run manifest target/docker.manifest - name: Upload manifest artifact + if: steps.dockerfiles.outputs.fresh == 'true' uses: actions/upload-artifact@v7 with: name: manifest @@ -28,6 +49,7 @@ jobs: compare-manifests: runs-on: ubuntu-latest needs: [generate-docker-manifest] + if: needs.generate-docker-manifest.outputs.fresh == 'true' outputs: diff: ${{ steps.manifest-diff.outputs.diff }} steps: diff --git a/README.md b/README.md index bde77453..436bb516 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,11 @@ Each of these can be invoked via `bb run [task]` at the command line. For example, to build all images locally, run `bb run build-images`. +You don't need to regenerate the Dockerfiles in `target` yourself when you +change the config or templates. CI builds and smoke tests a representative set +of the images your change affects, and once it's merged to `master` it +regenerates the Dockerfiles and commits them for you. + ### buildx Note that you'll need to enable the new `buildx` feature and set as the default diff --git a/bb.edn b/bb.edn index 7173c7cf..30e18e18 100644 --- a/bb.edn +++ b/bb.edn @@ -11,6 +11,7 @@ :task (apply dc/-main "dockerfiles" *command-line-args*)} manifest (apply dc/-main "manifest" *command-line-args*) build-images {:task (apply dc/-main "build-images" *command-line-args*)} + changed-build-dirs (apply dc/-main "changed-build-dirs" *command-line-args*) test (clojure '-X:test) fmt-check {:doc "Check formatting with cljfmt" :task (fmt/check {:paths ["src" "test" "bb.edn" "deps.edn" "tests.edn"]})} diff --git a/src/docker_clojure/changes.clj b/src/docker_clojure/changes.clj new file mode 100644 index 00000000..d9c2b4c7 --- /dev/null +++ b/src/docker_clojure/changes.clj @@ -0,0 +1,98 @@ +(ns docker-clojure.changes + "Works out which build dirs a change to the generated Dockerfiles touches and + which of them are worth building to verify it." + (:require [clojure.set :as set] + [clojure.string :as str] + [docker-clojure.config :as cfg] + [docker-clojure.dockerfile :as df] + [docker-clojure.util :refer [get-or-default]])) + +(defn- parse-file-diff + "Parses one file's section of a git diff into its build dir, file name, + status and changed lines. Returns nil for files that aren't in a build dir." + [[header & lines]] + (let [[_ path] (re-matches #"diff --git a/(.+) b/\1" header) + parts (some-> path (str/split #"/"))] + (when (and (= 4 (count parts)) (= "target" (first parts))) + {:build-dir (str/join "/" (butlast parts)) + :file (last parts) + :status (cond + (some #(str/starts-with? % "new file mode") lines) :added + (some #(str/starts-with? % "deleted file mode") lines) :deleted + :else :modified) + :lines (->> lines + (drop-while #(not (str/starts-with? % "@@"))) + (filter #(re-find #"^[+-]" %)))}))) + +(defn parse-diff + "Parses `git diff -U0 --no-renames` output into a map of build dir to its + change: `:added?` is true for new build dirs and `:signature` is equal for + build dirs that received exactly the same change. Deleted build dirs are + left out." + [diff] + (->> (str/split-lines diff) + (partition-by #(str/starts-with? % "diff --git ")) + (partition 2) + (keep (fn [[header lines]] (parse-file-diff (concat header lines)))) + (group-by :build-dir) + (keep (fn [[build-dir files]] + (let [dockerfile (some #(when (= "Dockerfile" (:file %)) %) + files)] + (when-not (= :deleted (:status dockerfile)) + [build-dir + {:added? (= :added (:status dockerfile)) + :signature (set (map (juxt :file :status :lines) + files))}])))) + (into {}))) + +(defn- preference + "Sort key that puts the default JDK version on its default distro with the + default build tool first." + [[build-dir {:keys [jdk-version distro build-tool]}]] + [(not= cfg/default-jdk-version jdk-version) + (not= (get-or-default cfg/default-distros jdk-version) distro) + (not= cfg/default-build-tool build-tool) + build-dir]) + +(defn- traits + "What building a build dir's variant exercises: its JDK version on its + distro (i.e. its base image) and its build tool." + [[_build-dir {:keys [jdk-version distro build-tool]}]] + #{[:base jdk-version distro] [:build-tool build-tool]}) + +(defn- covering + "Greedily picks from `dir-variants` until every JDK version & distro pair + and every build tool among them has been picked at least once." + [dir-variants] + (loop [candidates (sort-by preference dir-variants) + uncovered (into #{} (mapcat traits) dir-variants) + picked []] + (if (empty? uncovered) + picked + (let [gain #(count (set/intersection uncovered (traits %))) + best (first (sort-by (comp - gain) candidates))] + (recur (remove #{best} candidates) + (set/difference uncovered (traits best)) + (conj picked best)))))) + +(defn select-build-dirs + "Returns the sorted build dirs worth building to verify `changes` (see + `parse-diff`). Of the new build dirs, just enough are built to exercise each + of their JDK version & distro pairs and build tools once. Modified build dirs + that received exactly the same change are represented by one of them. Both + prefer the default JDK version on its default distro." + [variants changes] + (let [dir->variant (into {} (map (juxt df/build-dir identity)) variants) + dir-variants (fn [dir-changes] + (map (fn [[build-dir _]] + [build-dir (dir->variant build-dir)]) + dir-changes)) + known (filter (comp dir->variant key) changes) + added (filter (comp :added? val) known) + modified (remove (comp :added? val) known)] + (->> (group-by (comp :signature val) modified) + vals + (map #(first (sort-by preference (dir-variants %)))) + (concat (covering (dir-variants added))) + (map first) + sort))) diff --git a/src/docker_clojure/config.clj b/src/docker_clojure/config.clj index 257bb323..74107e1e 100644 --- a/src/docker_clojure/config.clj +++ b/src/docker_clojure/config.clj @@ -85,7 +85,7 @@ (def git-repo "https://github.com/Quantisan/docker-clojure.git") -(def jdk-versions #{8 11 17 21 25 26}) +(def jdk-versions #{8 11 17 21 25 26 27}) (def base-images "Map of JDK version to base image name(s) with :default as a fallback" @@ -100,7 +100,7 @@ (def distros "Map of base image name to set of distro tags to use, namespaced by Linux distro type. :default key is a fallback for base images not o/w specified." - {:default #{:alpine/alpine :ubuntu/jammy :ubuntu/noble} + {:default #{:alpine/alpine :ubuntu/jammy :ubuntu/noble :ubuntu/resolute} "debian" #{:debian-slim/bookworm-slim :debian/bookworm :debian-slim/trixie-slim :debian/trixie}}) @@ -132,6 +132,12 @@ ;; No more jammy builds for JDK 23+ {:jdk-version #(>= % 23) :distro :ubuntu/jammy} + ;; No noble builds for JDK 27+ + {:jdk-version #(>= % 27) + :distro :ubuntu/noble} + ;; Only build resolute for JDK 26+ + {:jdk-version #(< % 26) + :distro :ubuntu/resolute} ;; No upstream ARM alpine images available before JDK 21 {:jdk-version #(< % 21) :architecture "arm64v8" diff --git a/src/docker_clojure/core.clj b/src/docker_clojure/core.clj index 6b9e1eb2..9a0a2df4 100644 --- a/src/docker_clojure/core.clj +++ b/src/docker_clojure/core.clj @@ -4,6 +4,7 @@ [clojure.java.io :as io] [clojure.java.shell :refer [sh]] [clojure.spec.alpha :as s] + [docker-clojure.changes :as changes] [docker-clojure.config :as cfg] [docker-clojure.docker :as docker] [docker-clojure.dockerfile :as df] @@ -55,6 +56,7 @@ (apply f args))) (defn build-images + "Builds the images for `variants` and returns the variants that failed." [parallelization installer-hashes variants] (log "Building images" parallelization "at a time") (let [variants-ch (to-chan! variants) @@ -63,7 +65,11 @@ (map (partial rand-delay docker/build-image installer-hashes)) variants-ch)) - (while ( failed + (= ::docker/failed status) (conj variant))) + failed)))) (defn generate-dockerfiles! [installer-hashes variants] (log "Generated" (count variants) "variants") @@ -116,8 +122,27 @@ (defn cmd-build-images [{:keys [opts]}] (let [parallelism (:parallelism opts) - filter-opts (dissoc opts :parallelism)] - (build-images (or parallelism 4) cfg/installer-hashes (generate-variants filter-opts)))) + build-dirs (set (:build-dir opts)) + filter-opts (dissoc opts :parallelism :build-dir) + variants (cond->> (generate-variants filter-opts) + (seq build-dirs) (filter #(build-dirs (df/build-dir %)))) + failed (build-images (or parallelism 4) cfg/installer-hashes + variants)] + (when (seq failed) + (log "Failed to build" (count failed) "of" (count variants) "images:") + (doseq [{:keys [docker-tag architecture]} (variant/sort failed)] + (log (str " clojure:" docker-tag " (" architecture ")"))) + ::failed))) + +(defn cmd-changed-build-dirs [{:keys [opts]}] + (let [base (or (:base opts) "HEAD") + {:keys [exit out err]} + (sh "git" "diff" "--cached" "--no-renames" "-U0" base "--" "target")] + (if (zero? exit) + (run! println (changes/select-build-dirs (valid-variants) + (changes/parse-diff out))) + (do (log "ERROR diffing target against" (str base ":") err) + ::failed)))) (def cmd-table [{:cmds ["clean"] @@ -133,16 +158,25 @@ :spec {:output-file {:desc "Output file (default: stdout)"}}} {:cmds ["build-images"] :fn cmd-build-images - :doc "Build Docker images for matching variants" + :doc "Build and smoke test Docker images for matching variants" :spec {:parallelism {:alias :p :coerce :long :desc "Number of parallel builds" :default 4 - :validate pos?}}}]) + :validate pos?} + :build-dir {:coerce [:string] + :desc "Only build variants in these build dirs"}}} + {:cmds ["changed-build-dirs"] + :fn cmd-changed-build-dirs + :doc "List the build dirs worth building to verify staged target changes" + :args->opts [:base] + :spec {:base {:desc "Git ref to compare staged target against (default: HEAD)"}}}]) (defn -main "Entrypoint for CLI." [& args] (logger/start) - (cli/dispatch cmd-table args {:prog "docker-clojure" :help true}) - (logger/stop)) + (let [result (cli/dispatch cmd-table args {:prog "docker-clojure" :help true})] + (logger/stop) + (when (= ::failed result) + (System/exit 1)))) diff --git a/src/docker_clojure/docker.clj b/src/docker_clojure/docker.clj index 4728cdde..bbdcd804 100644 --- a/src/docker_clojure/docker.clj +++ b/src/docker_clojure/docker.clj @@ -12,7 +12,34 @@ (defn pull-image [image] (sh "docker" "pull" image)) +(defn smoke-test-cmds + "Commands to run in a freshly built image to check its build tool(s) work." + [{:keys [build-tool]}] + (let [cmds {"lein" ["lein" "version"] + "tools-deps" ["clojure" "-M" "-e" "(clojure-version)"]}] + (if (= ::core/all build-tool) + (vals cmds) + [(get cmds build-tool)]))) + +(defn smoke-test + "Runs the variant's smoke test commands in `image-tag`. Returns true if they + all succeed." + [image-tag platform-flag variant] + (every? (fn [cmd] + (let [run-cmd (remove nil? (concat ["docker" "run" "--rm" + platform-flag image-tag] + cmd)) + _ (apply log "Running" run-cmd) + {:keys [out err exit]} (apply sh run-cmd)] + (if (zero? exit) + true + (do (log "ERROR smoke testing" (str image-tag ":") err out) + false)))) + (smoke-test-cmds variant))) + (defn build-image + "Builds and smoke tests the image for `variant`. Returns [::done variant] on + success or [::failed variant] otherwise." [installer-hashes {:keys [docker-tag base-image architecture] :as variant}] (let [image-tag (str "clojure:" docker-tag) _ (log "Pulling base image" base-image) @@ -31,15 +58,17 @@ build-cmd (remove nil? ["docker" "buildx" "build" "--no-cache" "-t" image-tag platform-flag "--load" - "-f" dockerfile "."])] - (apply log "Running" build-cmd) - (let [{:keys [out err exit]} - (with-sh-dir build-dir (apply sh build-cmd))] - (if (zero? exit) - (log "Succeeded building" (str "clojure:" docker-tag)) - (log "ERROR building" (str "clojure:" docker-tag ":") err out)))) - (log) - [::done variant]) + "-f" dockerfile "."]) + _ (apply log "Running" build-cmd) + {:keys [out err exit]} + (with-sh-dir build-dir (apply sh build-cmd)) + built? (zero? exit) + _ (if built? + (log "Succeeded building" image-tag) + (log "ERROR building" (str image-tag ":") err out)) + ok? (and built? (smoke-test image-tag platform-flag variant))] + (log) + [(if ok? ::done ::failed) variant])) (defn base-image-tag [base-image jdk-version distro] diff --git a/src/docker_clojure/log.clj b/src/docker_clojure/log.clj index e321ca09..937bb6aa 100644 --- a/src/docker_clojure/log.clj +++ b/src/docker_clojure/log.clj @@ -4,17 +4,26 @@ ;; init w/ closed chan so that start works the same on first startup as subsequent ones (defonce log-ch (doto (chan) close!)) +;; closes once the logging thread has printed everything it was sent +(defonce drained-ch (doto (chan) close!)) + (defn start [] (when (nil? (