From 5f27897a7e59029c5fe3bd5f3b538f75039e194f Mon Sep 17 00:00:00 2001 From: John Trujillo Date: Fri, 16 Oct 2026 12:46:28 -0500 Subject: [PATCH] fix(addons): skip submodules when staging source tarballs Publish failed copying AI-Agent-Local's llama.cpp gitlink. The notes now pin its commit, and the plugin's BUILDING.md is renamed NATIVE-BUILD.md. Refs: ADFA-6566 --- .../{BUILDING.md => NATIVE-BUILD.md} | 2 +- plugins/AI-Agent-Local/README.md | 2 +- tools/addons/src/addons/tarball.py | 83 +++++++++++++++++-- tools/addons/tests/test_tarball.py | 62 ++++++++++++++ 4 files changed, 141 insertions(+), 8 deletions(-) rename plugins/AI-Agent-Local/{BUILDING.md => NATIVE-BUILD.md} (99%) diff --git a/plugins/AI-Agent-Local/BUILDING.md b/plugins/AI-Agent-Local/NATIVE-BUILD.md similarity index 99% rename from plugins/AI-Agent-Local/BUILDING.md rename to plugins/AI-Agent-Local/NATIVE-BUILD.md index 54d4502e..f5dfff25 100644 --- a/plugins/AI-Agent-Local/BUILDING.md +++ b/plugins/AI-Agent-Local/NATIVE-BUILD.md @@ -404,7 +404,7 @@ When contributing changes to the AI plugins: 2. Verify **ai-core**, **ai-agent-local**, **ai-agent-gemini** and **ai-assistant** work together 3. Ensure **llama.cpp submodule** updates are documented 4. Run ProGuard/R8 release build to catch reflection issues -5. Update this BUILDING.md with any new dependencies or steps +5. Update this guide with any new dependencies or steps --- diff --git a/plugins/AI-Agent-Local/README.md b/plugins/AI-Agent-Local/README.md index 7ac95533..ebeb0eb6 100644 --- a/plugins/AI-Agent-Local/README.md +++ b/plugins/AI-Agent-Local/README.md @@ -34,7 +34,7 @@ checked out (`subprojects/llama.cpp/CMakeLists.txt` exists) — which is exactly the setup a CI checkout does *not* have, so CI builds against the prebuilt AAR. To regenerate the AAR after bumping the llama.cpp fork, see -**[BUILDING.md](BUILDING.md)** and `scripts/rebuild-llama-aar.sh` (requires the +**[NATIVE-BUILD.md](NATIVE-BUILD.md)** and `scripts/rebuild-llama-aar.sh` (requires the submodule + NDK/CMake). Notes on the fork live in [SUBMODULE_NOTES.md](SUBMODULE_NOTES.md). diff --git a/tools/addons/src/addons/tarball.py b/tools/addons/src/addons/tarball.py index 013f7602..71fcfe9f 100644 --- a/tools/addons/src/addons/tarball.py +++ b/tools/addons/src/addons/tarball.py @@ -4,6 +4,7 @@ import subprocess import tarfile from pathlib import Path +from typing import NamedTuple from addons import model @@ -19,7 +20,7 @@ - Author: {author} - Comes from: {source}/tree/main/{directory} - License: {license}. The full text is in LICENSE beside this file. - +{prerequisites} This folder is the project root. Open it in Code on the Go, or build it from a desktop: @@ -54,6 +55,16 @@ ./gradlew downloadAssets """ +# Stated before the build instructions (design §9.4, R46): a submodule is not +# copied, so a reader who needs it has to fetch the exact commit themselves. +SUBMODULE_NOTE = """ +`{path}` is a git submodule and is not in this archive. If you +need it, clone it into place at the commit this source was published with: + + git clone {url} {path} + git -C {path} checkout {commit} +""" + # Every shared jar is referenced as "../libs/x.jar" (or "../../libs/x.jar") # from the addon's place in the repository. Flattening moves the addon to the # archive root, so those references have to lose their parent hops. @@ -64,6 +75,9 @@ # and stay as they are. REWRITTEN = ("build.gradle.kts", "settings.gradle.kts") +# The index mode git gives a submodule: a commit pointer, not a file. +GITLINK = "160000" + def jars_for(addon: Path) -> list[str]: text = "" @@ -74,12 +88,62 @@ def jars_for(addon: Path) -> list[str]: return sorted(set(re.findall(r"\.\./libs/([A-Za-z0-9._-]+\.jar)", text))) -def tracked_files(root: Path, addon: Path) -> list[str]: +class IndexEntry(NamedTuple): + mode: str + object_id: str + path: str + + @property + def is_submodule(self) -> bool: + return self.mode == GITLINK + + +def _index(root: Path, addon: Path) -> list[IndexEntry]: + """Every index entry under the addon, submodules included.""" relative = addon.relative_to(root).as_posix() result = subprocess.run( - ["git", "-C", str(root), "ls-files", "--", relative], + ["git", "-C", str(root), "ls-files", "--stage", "-z", "--", relative], capture_output=True, text=True, check=True) - return [line for line in result.stdout.splitlines() if line] + entries = [] + for entry in result.stdout.split("\0"): + if entry: + info, path = entry.split("\t", 1) + mode, object_id, _ = info.split(" ") + entries.append(IndexEntry(mode, object_id, path)) + return entries + + +def submodule_urls(root: Path) -> dict[str, str]: + """Submodule path -> URL, from .gitmodules (keyed by path, not name).""" + result = subprocess.run( + ["git", "-C", str(root), "config", "-f", ".gitmodules", + "--get-regexp", r"^submodule\..*\.(path|url)$"], + capture_output=True, text=True) + # Exit 1 only means no .gitmodules or no match; anything else is a real error + if result.returncode not in (0, 1): + raise RuntimeError(f"reading .gitmodules failed: {result.stderr.strip()}") + fields: dict[str, dict[str, str]] = {} + for line in result.stdout.splitlines(): + key, value = line.split(" ", 1) + name, field = key.removeprefix("submodule.").rsplit(".", 1) + fields.setdefault(name, {})[field] = value + return {f["path"]: f["url"] for f in fields.values() if "path" in f and "url" in f} + + +def _submodule_notes(root: Path, addon: Path, submodules: list[IndexEntry]) -> str: + """The clone instructions for each submodule, or "" when there are none.""" + if not submodules: + return "" + inside = addon.relative_to(root).as_posix() + urls = submodule_urls(root) + notes = "" + for entry in submodules: + if entry.path not in urls: + raise RuntimeError( + f"{addon.name}: submodule {entry.path} has no url in .gitmodules") + notes += SUBMODULE_NOTE.format(path=entry.path[len(inside) + 1:], + url=urls[entry.path], commit=entry.object_id) + return notes def flatten(text: str) -> str: @@ -107,7 +171,11 @@ def _stage(root: Path, addon: Path, out: Path, meta: dict) -> Path: top.mkdir(parents=True) inside = addon.relative_to(root).as_posix() - files = tracked_files(root, addon) + entries = _index(root, addon) + # A submodule is one entry pointing at a commit, not a file, so there is + # nothing to copy. AI-Agent-Local's llama.cpp is the case in point: its + # build only needs the committed AAR (see its settings.gradle.kts). + files = [entry.path for entry in entries if not entry.is_submodule] if not files: raise RuntimeError(f"{addon.name}: git tracks no file in this directory") # The archive root IS the project root: Code on the Go reads a folder as a @@ -152,9 +220,12 @@ def mentions(relative: str, text: str) -> bool: if mentions("build.gradle.kts", "downloadAssets"): caveats += DOWNLOAD_NOTE + prerequisites = _submodule_notes( + root, addon, [entry for entry in entries if entry.is_submodule]) + author = meta.get("author") or {} _claim(top, "BUILDING.md", addon.name).write_text(NOTES.format( - caveats=caveats, + caveats=caveats, prerequisites=prerequisites, name=model.display_name(addon.name), directory=inside, origin="Community contribution" if meta.get("origin") == "community" else "App Dev for All", diff --git a/tools/addons/tests/test_tarball.py b/tools/addons/tests/test_tarball.py index d7d08f87..ac79fadf 100644 --- a/tools/addons/tests/test_tarball.py +++ b/tools/addons/tests/test_tarball.py @@ -184,6 +184,68 @@ def test_verify_rejects_a_surviving_parent_path(tmp_path): tarball.verify(top, "plugins/Keystore-Generator", ["plugin-api.jar"]) +LLAMA_URL = "https://github.com/appdevforall/llama.cpp.git" + + +def add_submodule(root: Path, addon: Path, url: str | None = LLAMA_URL) -> str: + """Track a nested repo under the addon as a gitlink; returns its commit.""" + nested = addon / "subprojects" / "llama.cpp" + subprocess.run(["git", "init", "-q", str(nested)], check=True) + (nested / "CMakeLists.txt").write_text("project(llama)\n") + commit(nested) + if url: + (root / ".gitmodules").write_text( + '[submodule "llama"]\n' + f"\tpath = {nested.relative_to(root).as_posix()}\n" + f"\turl = {url}\n") + commit(root) + return subprocess.run(["git", "-C", str(nested), "rev-parse", "HEAD"], + capture_output=True, text=True).stdout.strip() + + +def test_a_submodule_is_left_out_of_the_archive(tmp_path): + """AI-Agent-Local's llama.cpp: copying the gitlink raised IsADirectoryError.""" + addon = make_repo(tmp_path) + pinned = add_submodule(tmp_path, addon) + archive = build(tmp_path, addon, META) + top = "keystore-generator-src" + names = names_in(archive) + assert f"{top}/build.gradle.kts" in names + assert not any(n.startswith(f"{top}/subprojects") for n in names) + notes = text_in(archive, f"{top}/BUILDING.md") + assert f"git clone {LLAMA_URL} subprojects/llama.cpp" in notes + assert f"checkout {pinned}" in notes + # R46: the prerequisite comes before the build instructions + assert notes.index("git clone") < notes.index("./gradlew assemblePlugin") + # Blank lines keep the note out of the License bullet and end the code block + assert "this file.\n\n`subprojects/llama.cpp`" in notes + assert f"checkout {pinned}\n\nThis folder" in notes + + +def test_an_addon_without_submodules_gets_no_submodule_note(tmp_path): + archive = build(tmp_path, make_repo(tmp_path), META) + notes = text_in(archive, "keystore-generator-src/BUILDING.md") + assert "submodule" not in notes + assert "this file.\n\nThis folder" in notes + + +def test_a_submodule_missing_from_gitmodules_stops_the_build(tmp_path): + """No clone instructions at all beats ones with a placeholder URL.""" + addon = make_repo(tmp_path) + add_submodule(tmp_path, addon, url=None) + with pytest.raises(RuntimeError, match="has no url in .gitmodules"): + build(tmp_path, addon, META) + + +def test_an_addon_shipping_its_own_building_md_stops_the_build(tmp_path): + """AI-Agent-Local's guide is NATIVE-BUILD.md for this reason (ADFA-6566).""" + addon = make_repo(tmp_path) + (addon / "BUILDING.md").write_text("# The addon's own guide\n") + commit(tmp_path) + with pytest.raises(RuntimeError, match="BUILDING.md is claimed twice"): + build(tmp_path, addon, META) + + def test_a_sibling_directory_is_not_inside_the_archive(tmp_path): # "/out/foo-src-evil" starts with "/out/foo-src" but is not inside it root = tmp_path / "foo-src"