Skip to content

Avoid nvJitLink cache crashes during incremental LTO #3040

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leofang merged 3 commits into from
Nov 1, 2026
Merged

Avoid nvJitLink cache crashes during incremental LTO #3040

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
594e54e
Avoid nvJitLink cache crashes during incremental LTO
GodlyDonuts Oct 7, 2026
ebab5f5
Limit incremental LTO cache workaround to nvJitLink before 13.6
GodlyDonuts Oct 31, 2026
16f89fa
remove redundant defensive line that is problematic in FT Python
leofang Nov 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cuda_core/cuda/core/_linker.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,9 @@ class LinkerOptions:
incremental : bool, optional
Perform an incremental link. The result can be passed
directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer
and is not supported by the driver linker backend.
and is not supported by the driver linker backend. Intermediate
caching is disabled on nvJitLink versions older than 13.6 when combined
with link-time optimization, to work around an nvJitLink bug.
Default: False.
ptx : bool, optional
Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``.
Expand Down Expand Up @@ -198,6 +200,9 @@ class LinkerOptions:
Default: 1.
no_cache : bool, optional
Do not cache the intermediate steps of nvJitLink.
Caching is also disabled automatically for incremental links that use
link-time optimization on nvJitLink versions older than 13.6, to work
around an nvJitLink bug.
Default: False.
numba_debug : bool, optional
Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option;
Expand Down
14 changes: 10 additions & 4 deletions cuda_core/cuda/core/_linker.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,9 @@ class LinkerOptions:
incremental : bool, optional
Perform an incremental link. The result can be passed
directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer
and is not supported by the driver linker backend.
and is not supported by the driver linker backend. Intermediate
caching is disabled on nvJitLink versions older than 13.6 when combined
with link-time optimization, to work around an nvJitLink bug.
Default: False.
ptx : bool, optional
Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``.
Expand Down Expand Up @@ -316,6 +318,9 @@ class LinkerOptions:
Default: 1.
no_cache : bool, optional
Do not cache the intermediate steps of nvJitLink.
Caching is also disabled automatically for incremental links that use
link-time optimization on nvJitLink versions older than 13.6, to work
around an nvJitLink bug.
Default: False.
numba_debug : bool, optional
Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option;
Expand Down Expand Up @@ -431,7 +436,10 @@ class LinkerOptions:
options.append(f"-split-compile={int(self.split_compile)}")
if self.split_compile_extended is not None:
options.append(f"-split-compile-extended={int(self.split_compile_extended)}")
if self.no_cache is True:
# nvbugs 6659652
if self.no_cache is True or (
_nvjitlink_version < (13, 6) and self.incremental and self.link_time_optimization
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
):
options.append("-no-cache")

if as_bytes:
Expand Down Expand Up @@ -782,8 +790,6 @@ def _decide_nvjitlink_or_driver() -> bool:
if _use_nvjitlink_backend is not None:
return not _use_nvjitlink_backend

_nvjitlink_version = None

warn_txt_common = (
"the driver APIs will be used instead, which do not support"
" minor version compatibility or linking LTO IRs."
Expand Down
20 changes: 20 additions & 0 deletions cuda_core/tests/test_linker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,26 @@ def test_linker_options_incremental_as_bytes(value, expected_count):
assert options.as_bytes().count(b"-r") == expected_count


@pytest.mark.agent_authored(model="gpt-6")
@pytest.mark.skipif(is_culink_backend, reason="as_bytes() only supported for nvjitlink backend")
@pytest.mark.parametrize(
"version,needs_workaround", [((13, 4), True), ((13, 5), True), ((13, 6), False), ((14, 0), False)]
)
@pytest.mark.parametrize(
"incremental,link_time_optimization", [(False, False), (False, True), (True, False), (True, True)]
)
@pytest.mark.parametrize("no_cache", [None, False, True])
def test_linker_options_incremental_lto_disables_cache(
monkeypatch, version, needs_workaround, incremental, link_time_optimization, no_cache
):
options = LinkerOptions(
arch="sm_80", incremental=incremental, link_time_optimization=link_time_optimization, no_cache=no_cache
)
monkeypatch.setattr(_linker, "_nvjitlink_version", version)
expected = no_cache is True or (needs_workaround and incremental and link_time_optimization)
assert options.as_bytes().count(b"-no-cache") == int(expected)


@pytest.mark.parametrize("backend", ("invalid", "driver"))
def test_linker_options_as_bytes_invalid_backend(backend):
"""Test LinkerOptions.as_bytes() with invalid backend"""
Expand Down
Loading