From 594e54ee4b9c7604a2c80949b811981e5153bff5 Mon Sep 17 00:00:00 2001 From: Sai Ramineni Date: Wed, 7 Oct 2026 04:53:32 -0400 Subject: [PATCH 1/3] Avoid nvJitLink cache crashes during incremental LTO --- cuda_core/cuda/core/_linker.pyi | 6 +++++- cuda_core/cuda/core/_linker.pyx | 11 +++++++++-- cuda_core/tests/test_linker.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/cuda_core/cuda/core/_linker.pyi b/cuda_core/cuda/core/_linker.pyi index 47eed3dcf65..ddc3a900a9c 100644 --- a/cuda_core/cuda/core/_linker.pyi +++ b/cuda_core/cuda/core/_linker.pyi @@ -153,7 +153,9 @@ class LinkerOptions: incremental : bool, optional Perform an incremental link. The result can be passed directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer - and is not supported by the driver linker backend. + and is not supported by the driver linker backend. Intermediate + caching is disabled automatically when combined with link-time + optimization. Default: False. ptx : bool, optional Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``. @@ -198,6 +200,8 @@ class LinkerOptions: Default: 1. no_cache : bool, optional Do not cache the intermediate steps of nvJitLink. + This is also enabled automatically for incremental links that use + link-time optimization. Default: False. numba_debug : bool, optional Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option; diff --git a/cuda_core/cuda/core/_linker.pyx b/cuda_core/cuda/core/_linker.pyx index 6a077aafa2a..f91b9ad2ff6 100644 --- a/cuda_core/cuda/core/_linker.pyx +++ b/cuda_core/cuda/core/_linker.pyx @@ -271,7 +271,9 @@ class LinkerOptions: incremental : bool, optional Perform an incremental link. The result can be passed directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer - and is not supported by the driver linker backend. + and is not supported by the driver linker backend. Intermediate + caching is disabled automatically when combined with link-time + optimization. Default: False. ptx : bool, optional Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``. @@ -316,6 +318,8 @@ class LinkerOptions: Default: 1. no_cache : bool, optional Do not cache the intermediate steps of nvJitLink. + This is also enabled automatically for incremental links that use + link-time optimization. Default: False. numba_debug : bool, optional Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option; @@ -431,7 +435,10 @@ class LinkerOptions: options.append(f"-split-compile={int(self.split_compile)}") if self.split_compile_extended is not None: options.append(f"-split-compile-extended={int(self.split_compile_extended)}") - if self.no_cache is True: + # Repeated incremental LTO links can crash in nvJitLink's intermediate + # cache (reproduced with 13.4.92 on Windows). Avoid reusing cached + # state for each incremental stage. + if self.no_cache is True or (self.incremental and self.link_time_optimization): options.append("-no-cache") if as_bytes: diff --git a/cuda_core/tests/test_linker.py b/cuda_core/tests/test_linker.py index 5a7994f9110..95e7270efcf 100644 --- a/cuda_core/tests/test_linker.py +++ b/cuda_core/tests/test_linker.py @@ -244,6 +244,19 @@ def test_linker_options_incremental_as_bytes(value, expected_count): assert options.as_bytes().count(b"-r") == expected_count +@pytest.mark.agent_authored(model="gpt-6") +@pytest.mark.skipif(is_culink_backend, reason="as_bytes() only supported for nvjitlink backend") +def test_linker_options_incremental_lto_disables_cache(): + options = LinkerOptions(arch="sm_80", incremental=True, link_time_optimization=True) + assert b"-no-cache" in options.as_bytes() + + options = LinkerOptions(arch="sm_80", incremental=True) + assert b"-no-cache" not in options.as_bytes() + + options = LinkerOptions(arch="sm_80", link_time_optimization=True) + assert b"-no-cache" not in options.as_bytes() + + @pytest.mark.parametrize("backend", ("invalid", "driver")) def test_linker_options_as_bytes_invalid_backend(backend): """Test LinkerOptions.as_bytes() with invalid backend""" From ebab5f57e45f5b4eea2de467b3d6469f402cbf61 Mon Sep 17 00:00:00 2001 From: Sai Ramineni Date: Sat, 31 Oct 2026 10:31:34 -0400 Subject: [PATCH 2/3] Limit incremental LTO cache workaround to nvJitLink before 13.6 --- cuda_core/cuda/core/_linker.pyi | 9 +++++---- cuda_core/cuda/core/_linker.pyx | 17 +++++++++-------- cuda_core/tests/test_linker.py | 25 ++++++++++++++++--------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/cuda_core/cuda/core/_linker.pyi b/cuda_core/cuda/core/_linker.pyi index ddc3a900a9c..61465612a2f 100644 --- a/cuda_core/cuda/core/_linker.pyi +++ b/cuda_core/cuda/core/_linker.pyi @@ -154,8 +154,8 @@ class LinkerOptions: Perform an incremental link. The result can be passed directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer and is not supported by the driver linker backend. Intermediate - caching is disabled automatically when combined with link-time - optimization. + caching is disabled on nvJitLink versions older than 13.6 when combined + with link-time optimization, to work around an nvJitLink bug. Default: False. ptx : bool, optional Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``. @@ -200,8 +200,9 @@ class LinkerOptions: Default: 1. no_cache : bool, optional Do not cache the intermediate steps of nvJitLink. - This is also enabled automatically for incremental links that use - link-time optimization. + Caching is also disabled automatically for incremental links that use + link-time optimization on nvJitLink versions older than 13.6, to work + around an nvJitLink bug. Default: False. numba_debug : bool, optional Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option; diff --git a/cuda_core/cuda/core/_linker.pyx b/cuda_core/cuda/core/_linker.pyx index f91b9ad2ff6..a45969eb24d 100644 --- a/cuda_core/cuda/core/_linker.pyx +++ b/cuda_core/cuda/core/_linker.pyx @@ -272,8 +272,8 @@ class LinkerOptions: Perform an incremental link. The result can be passed directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer and is not supported by the driver linker backend. Intermediate - caching is disabled automatically when combined with link-time - optimization. + caching is disabled on nvJitLink versions older than 13.6 when combined + with link-time optimization, to work around an nvJitLink bug. Default: False. ptx : bool, optional Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``. @@ -318,8 +318,9 @@ class LinkerOptions: Default: 1. no_cache : bool, optional Do not cache the intermediate steps of nvJitLink. - This is also enabled automatically for incremental links that use - link-time optimization. + Caching is also disabled automatically for incremental links that use + link-time optimization on nvJitLink versions older than 13.6, to work + around an nvJitLink bug. Default: False. numba_debug : bool, optional Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option; @@ -435,10 +436,10 @@ class LinkerOptions: options.append(f"-split-compile={int(self.split_compile)}") if self.split_compile_extended is not None: options.append(f"-split-compile-extended={int(self.split_compile_extended)}") - # Repeated incremental LTO links can crash in nvJitLink's intermediate - # cache (reproduced with 13.4.92 on Windows). Avoid reusing cached - # state for each incremental stage. - if self.no_cache is True or (self.incremental and self.link_time_optimization): + # nvbugs 6659652 + if self.no_cache is True or ( + _nvjitlink_version < (13, 6) and self.incremental and self.link_time_optimization + ): options.append("-no-cache") if as_bytes: diff --git a/cuda_core/tests/test_linker.py b/cuda_core/tests/test_linker.py index 95e7270efcf..5b0bc0a9cbc 100644 --- a/cuda_core/tests/test_linker.py +++ b/cuda_core/tests/test_linker.py @@ -246,15 +246,22 @@ def test_linker_options_incremental_as_bytes(value, expected_count): @pytest.mark.agent_authored(model="gpt-6") @pytest.mark.skipif(is_culink_backend, reason="as_bytes() only supported for nvjitlink backend") -def test_linker_options_incremental_lto_disables_cache(): - options = LinkerOptions(arch="sm_80", incremental=True, link_time_optimization=True) - assert b"-no-cache" in options.as_bytes() - - options = LinkerOptions(arch="sm_80", incremental=True) - assert b"-no-cache" not in options.as_bytes() - - options = LinkerOptions(arch="sm_80", link_time_optimization=True) - assert b"-no-cache" not in options.as_bytes() +@pytest.mark.parametrize( + "version,needs_workaround", [((13, 4), True), ((13, 5), True), ((13, 6), False), ((14, 0), False)] +) +@pytest.mark.parametrize( + "incremental,link_time_optimization", [(False, False), (False, True), (True, False), (True, True)] +) +@pytest.mark.parametrize("no_cache", [None, False, True]) +def test_linker_options_incremental_lto_disables_cache( + monkeypatch, version, needs_workaround, incremental, link_time_optimization, no_cache +): + options = LinkerOptions( + arch="sm_80", incremental=incremental, link_time_optimization=link_time_optimization, no_cache=no_cache + ) + monkeypatch.setattr(_linker, "_nvjitlink_version", version) + expected = no_cache is True or (needs_workaround and incremental and link_time_optimization) + assert options.as_bytes().count(b"-no-cache") == int(expected) @pytest.mark.parametrize("backend", ("invalid", "driver")) From 16f89fac4202a5e7ba438877178c66742c438fd3 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Sun, 1 Nov 2026 13:37:41 -0500 Subject: [PATCH 3/3] remove redundant defensive line that is problematic in FT Python --- cuda_core/cuda/core/_linker.pyx | 2 -- 1 file changed, 2 deletions(-) diff --git a/cuda_core/cuda/core/_linker.pyx b/cuda_core/cuda/core/_linker.pyx index a45969eb24d..e278f0842f2 100644 --- a/cuda_core/cuda/core/_linker.pyx +++ b/cuda_core/cuda/core/_linker.pyx @@ -790,8 +790,6 @@ def _decide_nvjitlink_or_driver() -> bool: if _use_nvjitlink_backend is not None: return not _use_nvjitlink_backend - _nvjitlink_version = None - warn_txt_common = ( "the driver APIs will be used instead, which do not support" " minor version compatibility or linking LTO IRs."