From f2a3da38f3afd513bea146c17ef445d789595e72 Mon Sep 17 00:00:00 2001 From: s1arsky Date: Tue, 3 Nov 2026 16:51:31 +0100 Subject: [PATCH] Model: Skip standalone load for drafters that load with their target exllamav3 1.6.6 (commit 07015a36) gave Gemma4AssistantForCausalLM the capability `load_with_target`: its modules are loaded by the target's load() on the devices of the K/V cache layers it reads, and a standalone load now raises a RuntimeError. Gate the standalone draft load on that cap and let the existing `draft_model=` hand-off to the target's load_gen do the work, mirroring exllamav3's own model_init wiring. Drafters without the cap (MTP component, DFlash) keep their standalone load, and on exllamav3 < 1.6.6 no drafter carries the cap, so behavior there is unchanged. Tensor-parallel targets are unaffected either way: the drafter load path in the target's load_gen already raises a NotImplementedError for them before any allocation. Fixes theroyallab/tabbyAPI#514 --- backends/exllamav3/model.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backends/exllamav3/model.py b/backends/exllamav3/model.py index 4bbefd34..5f6a9ddd 100644 --- a/backends/exllamav3/model.py +++ b/backends/exllamav3/model.py @@ -1192,7 +1192,12 @@ def load_model_sync(self, progress_callback=None): if value: yield value - if self.use_draft_model: + if self.draft_model is not None and self.draft_model.caps.get("load_with_target"): + xlogger.info( + f"Draft model {self.draft_model_dir} loads with its target model, " + "skipping its standalone load" + ) + elif self.use_draft_model: for value in self.draft_model.load_gen( reserve_per_device=( None if (self.gpu_split or self.draft_gpu_split) else self.autosplit_reserve