Compress bundle-oci and retire outdated module versions

README.md

@@ -15,11 +15,11 @@ Output might look like:
 {
   "bun-0.5": "/nix/store/l08f5vl1af9rpzd7kvr0l5gx9v7y8p12-replit-module-bun-0.5",
   "c-clang14.0": "/nix/store/n4vzd9rkpjs72xj9yvlakxh3bardvdki-replit-module-c-clang14.0",
-  "clojure-1.11": "/nix/store/giqq76fl3yphzsm6rkl1qxqh4mszknpl-replit-module-clojure-1.11",
+  "clojure-1.12": "/nix/store/69ai122m5grza3cm50l7ivb941av6ppb-replit-module-clojure-1.12",
   "cpp-clang14": "/nix/store/8iv4czda6j5nfhxs80ci625hj91ffpbn-replit-module-cpp-clang14",
-  "dart-2.18": "/nix/store/mhl58f8y3z6jv0javkmx28y5h9aacw39-replit-module-dart-2.18",
-  "dotnet-7.0": "/nix/store/06mjna44a7w9bby6r121a7i9a5027qqn-replit-module-dotnet-7.0",
-  "go-1.19": "/nix/store/y1gb1k8bkyd9jqxi4r1g9qibcqn3c6dm-replit-module-go-1.19",
+  "dart-3.10": "/nix/store/3i16jlnmcwdhmwqg46p2hrkqfzm3zvg4-replit-module-dart-3.10",
+  "dotnet-8.0": "/nix/store/dhrbdryqbkbyyi1wlnpvlwi28ni2hwcj-replit-module-dotnet-8.0",
+  "go-1.25": "/nix/store/l76y8m5y19biid1wmqy03442lx9kairb-replit-module-go-1.25",
   ...
 }
 ```
@@ -31,6 +31,13 @@ nix eval .#modules --json | jq
 ```
 This will take a while.
 
+Retired module IDs remain available through `pkgs/upgrade-map`: they resolve to
+newer modules without keeping the retired implementations in the bundle.
+For example, Haskell 9.0/9.2/9.4 resolve to 9.10, Python 3.8/3.9 to 3.10,
+and Swift 5.6 to 5.8. The unversioned `swift` alias also resolves to 5.8.
+The old Python and Node.js Prybar IDs now use the standard Python 3.10 and
+Node.js 20 file runners, not Prybar. Projects using retired IDs get the newer
+runtime and tools with this bundle and may need code or dependency changes.
 
 ## Building
 
@@ -93,17 +100,20 @@ Local sources without a clean revision omit the revision annotation; sources
 without a last-modified timestamp omit creation metadata. Publish from a
 revision-pinned Git flake when complete provenance is required.
 
-It contains one uncompressed `application/vnd.oci.image.layer.v1.erofs` layer,
-not a tar layer. The consumer must support native EROFS layers; ordinary
-tar-only image unpackers cannot use it. Its diffID equals the layer blob digest.
+It contains one `application/vnd.oci.image.layer.v1.erofs` layer, not a tar layer.
+The whole filesystem uses zstd level 15 with 64 KiB clusters and tail packing,
+without an uncompressed hot set. The consumer must support native EROFS layers
+and guest-kernel zstd decompression; ordinary tar-only image unpackers cannot
+use it. Its diffID equals the layer blob digest.
 
 The filesystem is generated directly from the same pinned full bundle used by
 `disk-script`: its complete store closure and `/etc/nixmodules` metadata, with
 unchanged store paths, file contents, modes and symlink targets. All guest
 UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and
-per-module OCI outputs are unchanged. There is no disk conversion or dependency
-update. Uncompressed EROFS does not by itself guarantee DAX sharing; that also
-depends on the consumer's backing storage and mount configuration.
+per-module OCI builders are unchanged. There is no disk conversion or dependency
+update. Compressed files are not DAX-eligible: each guest decompresses into its
+own page cache, paying decompression CPU and memory rather than sharing pages
+through DAX.
 
 A small fixture exercises the producer without building the full bundle:
 

pkgs/bundle-oci/check.nix

@@ -4,12 +4,14 @@ let
   dependency = pkgs.runCommand "bundle-oci-dependency" { } ''
     mkdir -p "$out"
     printf 'dependency\n' > "$out/data"
+    head -c 131072 /dev/zero >> "$out/data"
     ln "$out/data" "$out/hardlink"
   '';
   module = pkgs.runCommand "bundle-oci-module" { } ''
     mkdir -p "$out/bin"
     printf '%s\n' '${dependency}' > "$out/module.json"
     printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool"
+    head -c 131072 /dev/zero >> "$out/bin/tool"
     chmod 755 "$out/bin/tool"
     ln -s ../module.json "$out/bin/relative"
     ln -s '${dependency}/data' "$out/absolute"

pkgs/bundle-oci/check.py

@@ -71,6 +71,28 @@ def blob(descriptor):
 assert "created" not in local_config
 subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True)
 root = pathlib.Path("extracted")
+
+
+def assert_layout(path, expected):
+    info = subprocess.check_output(
+        ["dump.erofs", f"--path={path}", str(erofs)], text=True
+    )
+    assert re.search(rf"Layout:\s*{expected}\b", info), info
+
+
+assert_layout(f"{(bundle / 'fixture').resolve()}/bin/tool", 3)
+assert_layout(f"{dependency}/data", 3)
+for workers in (1, 2):
+    rebuilt = pathlib.Path(f"workers-{workers}.erofs")
+    subprocess.run([
+        "mkfs.erofs", "-T", "1", "-U", "00000000-0000-0000-0000-000000000000",
+        "--force-uid=11000", "--force-gid=11000",
+        "-zzstd,level=15", "-C65536", "-Eztailpacking",
+        f"--workers={workers}", str(rebuilt), str(root),
+    ], check=True)
+    assert "sha256:" + hashlib.sha256(rebuilt.read_bytes()).hexdigest() == layer["digest"]
+print("Compressed file layouts verified; layer digest identical with 1 and 2 workers")
+
 paths = closure.read_text().splitlines()
 assert str(dependency) in paths
 assert {p.name for p in (root / "nix/store").iterdir()} == {

pkgs/bundle-oci/default.nix

@@ -45,10 +45,11 @@ let
       done < "$bundleClosure/store-paths"
       chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules
 
-      # Match the legacy disk's guest ownership. No compression: consumers may
-      # mount this blob directly; compression must not imply DAX page sharing.
+      # Compress the whole layer instead of preserving DAX-eligible files:
+      # each guest decompresses into its own page cache, without DAX sharing.
       mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \
-        --force-uid=11000 --force-gid=11000 layer.erofs root
+        --force-uid=11000 --force-gid=11000 \
+        -zzstd,level=15 -C65536 -Eztailpacking layer.erofs root
       python3 ${./layout.py} layer.erofs "$out" "$architecture" ${lib.escapeShellArg metadata}
     '';
 in

pkgs/default.nix

@@ -13,11 +13,10 @@ let
     "nodejs-20"
     "nodejs-22"
     "nodejs-24"
-    "go-1.21"
+    "go-1.25"
     "docker"
     "replit"
     "replit-rtld-loader"
-    "ruby"
     "ruby-3.2"
     "postgresql-16"
   ];

pkgs/historical-modules/default.nix

@@ -24,70 +24,13 @@ let
         displayVersion = "1.1.45";
       };
     }
-    {
-      moduleId = "clojure-1.11";
-      commit = "4327245815e8500233ed3af1cbb674bd147f673b";
-      overrides = {
-        displayVersion = "1.11";
-      };
-    }
-    {
-      moduleId = "dart-2.18";
-      commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b";
-      overrides = {
-        # /nix/store/05i6y0nwn8zgm6250my14xbcialzak9b-replit-module-dart-2.18
-        # .runners.dart.displayVersion = "2.18.0";
-        displayVersion = "2";
-      };
-    }
-    {
-      moduleId = "dart-3.0";
-      commit = "0b7a60667d2c29f2686211e952924a9693916a20";
-      overrides = {
-        # /nix/store/0wr2lk1h8vqzs38pfwiil6cf6s5mn54j-replit-module-dart-3.2
-        # .runners.dart.displayVersion = "3.2.4";
-        displayVersion = "3";
-      };
-    }
-    {
-      moduleId = "dart-3.1";
-      commit = "3b22c787fd20b13fe5afb868589c574068303b5e";
-      # displayVersion = "3.2.4";
-      overrides = {
-        displayVersion = "3";
-      };
-    }
-    {
-      moduleId = "dart-3.2";
-      commit = "9e3a33995f94a52e61ae962777e9bcbe6d75885e";
-      # displayVersion = "3.2.4";
-      overrides = {
-        displayVersion = "3";
-      };
-    }
-    {
-      moduleId = "dart-3.3";
-      commit = "54ea6c58e71bf278c3ce8824de3dbbaa49662be4";
-      overrides = {
-        # /nix/store/ampsihgvns831ig1w0warbpq1pyj2cg0-replit-module-dart-3.3
-        # .runners.dart.displayVersion = "3.3.4";
-        displayVersion = "3";
-      };
-    }
     {
       moduleId = "dart-3.8";
       commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24";
       overrides = {
         displayVersion = "3.8";
       };
     }
-    {
-      moduleId = "deno-1";
-      commit = "4327245815e8500233ed3af1cbb674bd147f673b";
-      overrides = {
-        displayVersion = "1";
-      };
-    }
     {
       moduleId = "docker";
       commit = "185d7dc7645178f92fd891f9efa20b3343c0f20d";
@@ -96,163 +39,13 @@ let
         internal = true;
       };
     }
-    {
-      moduleId = "dotnet-7.0";
-      commit = "4327245815e8500233ed3af1cbb674bd147f673b";
-      overrides = {
-        displayVersion = "7.0";
-      };
-    }
-    {
-      moduleId = "go-1.19";
-      commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b";
-      overrides = {
-        # /nix/store/7icmam9gmbfas0zbpm53f3ilw79kx0rj-replit-module-go-1.19
-        # .runners["go-run"].displayVersion = "1.19.3";
-        displayVersion = "1.19";
-      };
-    }
-    {
-      moduleId = "go-1.20";
-      commit = "b5aa5df636c4cd8cd1aea251e8dea4fc0aa51781";
-      overrides = {
-        # /nix/store/q6j9za8smkdbnlybg63qq5lmj7zbambr-replit-module-go-1.20
-        # .runners["go-run"].displayVersion = "1.20.4";
-        displayVersion = "1.20";
-      };
-    }
-    {
-      moduleId = "go-1.21";
-      commit = "76ae6535d7d60e767fe9d54902400229ca0b9448";
-      overrides = {
-        # /nix/store/...-replit-module-go-1.21
-        # .runners["go-run"].displayVersion = "1...";
-        displayVersion = "1.21";
-      };
-    }
-    {
-      moduleId = "haskell-ghc9.0";
-      commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b";
-      overrides = {
-        # /nix/store/h6hp6fd07dr8q9dhlz45nrs17wp2kmp4-replit-module-haskell-ghc9.0
-        # .runners.runghc.displayVersion = "9.0.2";
-        displayVersion = "9";
-      };
-    }
-    {
-      moduleId = "haskell-ghc9.2";
-      commit = "4c6f2315da24b84bd5e9dfedb952e41677724aaa";
-      overrides = {
-        # /nix/store/1r94iizf604jasg72f239fw0if4ldmd5-replit-module-haskell-ghc9.2
-        # .runners.runghc.displayVersion = "9.2.8";
-        displayVersion = "9";
-      };
-    }
-    {
-      moduleId = "haskell-ghc9.4";
-      commit = "4d6f7cd9fd685a3319ac7b6e3fc0789b430d6289";
-      overrides = {
-        # /nix/store/kkaxlri9s0bb7jydb1kpqzi2sjkf4rmj-replit-module-haskell-ghc9.4
-        # .runners.runghc.displayVersion = "9.4.8";
-        displayVersion = "9";
-      };
-    }
-    {
-      moduleId = "elixir-1_15";
-      commit = "4d6f7cd9fd685a3319ac7b6e3fc0789b430d6289";
-      overrides = {
-        # /nix/store/8nggvav01q7f1094m485nnd32094j522-replit-module-elixir-1_15
-        # .runners.elixir.displayVersion = "1.15.7";
-        displayVersion = "1";
-      };
-    }
-    {
-      moduleId = "nodejs-14";
-      commit = "f4cd419a646009297c049a2f1eec434381e08f13";
-      overrides = {
-        name = "Node.js Tools";
-        displayVersion = "14";
-        runners = {
-          nodeJS = {
-            displayVersion = "14.21.1";
-          };
-        };
-      };
-    }
-    {
-      moduleId = "nodejs-16";
-      commit = "f4cd419a646009297c049a2f1eec434381e08f13";
-      overrides = {
-        name = "Node.js Tools";
-        displayVersion = "16";
-        runners = {
-          nodeJS = {
-            displayVersion = "16.18.1";
-          };
-        };
-      };
-    }
-    {
-      moduleId = "nodejs-19";
-      commit = "f4cd419a646009297c049a2f1eec434381e08f13";
-      overrides = {
-        name = "Node.js Tools";
-        displayVersion = "19";
-        runners = {
-          nodeJS = {
-            displayVersion = "19.1.0";
-          };
-        };
-      };
-    }
-    {
-      moduleId = "php-8.1";
-      commit = "0b7a60667d2c29f2686211e952924a9693916a20";
-      overrides = {
-        # /nix/store/xc89rlwnmg5vh66hvrv1saq6md6kp1g5-replit-module-php-8.1
-        # .runners.php.displayVersion = "8.1.20";
-        displayVersion = "8";
-      };
-    }
     {
       moduleId = "php-8.2";
       commit = "4327245815e8500233ed3af1cbb674bd147f673b";
       overrides = {
         displayVersion = "8.2";
       };
     }
-    {
-      moduleId = "python-3.8";
-      commit = "76ae6535d7d60e767fe9d54902400229ca0b9448";
-      overrides = {
-        displayVersion = "3.8";
-      };
-    }
-    {
-      moduleId = "python-3.9";
-      commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24";
-      overrides = {
-        displayVersion = "3.9";
-      };
-    }
-    {
-      moduleId = "r-4.2";
-      commit = "1e1bb663068482cdb7c04bf585daed00205c0140";
-      overrides = {
-        # /nix/store/knizlzgmqgm488llvxxaai6kjb17mhm2-replit-module-r-4.2
-        # .runners.r.displayVersion = "4.2.3";
-        displayVersion = "4";
-      };
-    }
-    {
-      moduleId = "swift-5.6";
-      commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b";
-      overrides = {
-        # /nix/store/f96l8blsvpx1dbgnxw2x611hhzbm199l-replit-module-swift-5.6
-        # .runners.swift.displayVersion = "5.6.2";
-        displayVersion = "5";
-      };
-    }
     {
       moduleId = "zig-0.11";
       commit = "15426ef79793bf7c424eb40865d507eacfdd44e6";
@@ -262,19 +55,10 @@ let
         displayVersion = "0";
       };
     }
-    {
-      moduleId = "python-with-prybar-3.10";
-      commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24";
-      overrides = {
-        displayVersion = "3.10";
-      };
-    }
     {
       moduleId = "go-1.24";
       commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24";
       overrides = {
-        # /nix/store/...-replit-module-go-1.21
-        # .runners["go-run"].displayVersion = "1...";
         displayVersion = "1.24";
       };
     }
@@ -285,13 +69,6 @@ let
         displayVersion = "3.1";
       };
     }
-    {
-      moduleId = "nodejs-with-prybar-18";
-      commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24";
-      overrides = {
-        displayVersion = "18";
-      };
-    }
   ];
 
   moduleFromHistory =