diff --git a/README.md b/README.md index 58ba5178..41e55c27 100644 --- a/README.md +++ b/README.md @@ -737,6 +737,32 @@ You can also rename specific stems: } ``` +#### Separating audio arrays in memory + +`separate_numpy()` runs the loaded model on a NumPy array and returns the stems as arrays. No audio files are read or written during the separation; model files are still downloaded and loaded as usual. + +```python +import soundfile as sf +from audio_separator.separator import Separator + +separator = Separator() +separator.load_model(model_filename='model_bs_roformer_ep_317_sdr_12.9755.ckpt') + +# Rows are samples and columns are channels: (frames,), (frames, 1) or (frames, 2) +audio, sample_rate = sf.read('audio1.wav', dtype='float32') + +stems = separator.separate_numpy(audio, sample_rate=sample_rate) +vocals = stems["Vocals"] # (frames, 2) array at separator.sample_rate +instrumental = stems["Instrumental"] +``` + +- The input must be a floating-point array. It is copied, so the caller's array is never modified. Mono input is duplicated to two channels. Empty arrays and NaN or infinite samples raise `InvalidAudioDataError`. +- The result maps stem names to independent `(frames, 2)` arrays at `separator.sample_rate` (44100 by default). Input at another sample rate is resampled in memory, so the number of frames can change. +- `normalization_threshold` and `amplification_threshold` apply as they do for file output, as long as `amplification_threshold` is not above `normalization_threshold`. +- `output_single_stem` limits the result to that stem. One difference from file output: for MDXC models with more than two stems, file output writes every stem by default even when `output_single_stem` is set, but `separate_numpy()` returns only the selected stem. +- A single MDX, MDXC (including Roformer), VR or Demucs model is supported. Ensembles raise `NotImplementedError`, and `chunk_duration` does not apply. +- VR models rebuild audio from STFT frames, so their stems can be a few milliseconds shorter than the input, as with file output. Stem dtypes follow each architecture's processing (VR stems are float64). + ## Parameters for the Separator class - **`log_level`:** (Optional) Logging level, e.g., INFO, DEBUG, WARNING. `Default: logging.INFO` diff --git a/audio_separator/separator/architectures/demucs_separator.py b/audio_separator/separator/architectures/demucs_separator.py index 5b2fa116..ec0f449e 100644 --- a/audio_separator/separator/architectures/demucs_separator.py +++ b/audio_separator/separator/architectures/demucs_separator.py @@ -92,28 +92,33 @@ def __init__(self, common_config, arch_config): self.logger.info("Demucs Separator initialisation complete") - def separate(self, audio_file_path, custom_output_names=None): + def separate(self, audio_file_path, custom_output_names=None, *, return_numpy=False): """ Separates the audio file into its component stems using the Demucs model. Args: - audio_file_path (str): The path to the audio file to be processed. + audio_file_path (str or np.ndarray): Input file path, or audio frames in numpy mode. custom_output_names (dict, optional): Custom names for the output files. Defaults to None. + return_numpy (bool): Accept an audio array and return stems without writing files. Returns: - list: A list of paths to the output files generated by the separation process. + list or dict: Output file paths, or stem names mapped to audio arrays. """ self.logger.debug("Starting separation process...") source = None stem_source = None inst_source = {} - self.audio_file_path = audio_file_path - self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] + if return_numpy: + self.clear_file_specific_paths() + mix = self.prepare_mix(audio_file_path) + else: + self.audio_file_path = audio_file_path + self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] - # Prepare the mix for processing - self.logger.debug("Preparing mix...") - mix = self.prepare_mix(self.audio_file_path) + # Prepare the mix for processing + self.logger.debug("Preparing mix...") + mix = self.prepare_mix(self.audio_file_path) self.logger.debug(f"Mix prepared for demixing. Shape: {mix.shape}") @@ -171,6 +176,13 @@ def separate(self, audio_file_path, custom_output_names=None): self.logger.debug("Setting source map to 4-stem...") self.demucs_source_map = DEMUCS_4_SOURCE_MAPPER + if return_numpy: + return { + stem_name: self._prepare_numpy_stem(source[stem_value].T) + for stem_name, stem_value in self.demucs_source_map.items() + if self.output_single_stem is None or stem_name.lower() == self.output_single_stem.lower() + } + self.logger.debug("Processing for all stems...") for stem_name, stem_value in self.demucs_source_map.items(): if self.output_single_stem is not None: diff --git a/audio_separator/separator/architectures/mdx_separator.py b/audio_separator/separator/architectures/mdx_separator.py index af5e311e..6bf8208f 100644 --- a/audio_separator/separator/architectures/mdx_separator.py +++ b/audio_separator/separator/architectures/mdx_separator.py @@ -145,24 +145,29 @@ def load_model(self): self.model_run.to(self.torch_device).eval() self.logger.warning("Model converted from onnx to pytorch due to segment size not matching dim_t, processing may be slower.") - def separate(self, audio_file_path, custom_output_names=None): + def separate(self, audio_file_path, custom_output_names=None, *, return_numpy=False): """ Separates the audio file into primary and secondary sources based on the model's configuration. - It processes the mix, demixes it into sources, normalizes the sources, and saves the output files. + It processes the mix, demixes it into sources, and saves files or returns stem arrays. Args: - audio_file_path (str): The path to the audio file to be processed. + audio_file_path (str or np.ndarray): Input file path, or audio frames in numpy mode. custom_output_names (dict, optional): Custom names for the output files. Defaults to None. + return_numpy (bool): Accept an audio array and return stems without writing files. Returns: - list: A list of paths to the output files generated by the separation process. + list or dict: Output file paths, or stem names mapped to audio arrays. """ - self.audio_file_path = audio_file_path - self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] + if return_numpy: + self.clear_file_specific_paths() + mix = self.prepare_mix(audio_file_path) + else: + self.audio_file_path = audio_file_path + self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] - # Prepare the mix for processing - self.logger.debug(f"Preparing mix for input audio file {self.audio_file_path}...") - mix = self.prepare_mix(self.audio_file_path) + # Prepare the mix for processing + self.logger.debug(f"Preparing mix for input audio file {self.audio_file_path}...") + mix = self.prepare_mix(self.audio_file_path) self.logger.debug("Normalizing mix before demixing...") peak = np.abs(mix).max() @@ -194,6 +199,17 @@ def separate(self, audio_file_path, custom_output_names=None): self.logger.debug("Inverting secondary stem by subtracting of transposed demixed stem from transposed original mix") self.secondary_source = (-self.primary_source * self.compensate) + mix.T + if return_numpy: + stems = {} + for stem_name, source in ( + (self.secondary_stem_name, self.secondary_source), + (self.primary_stem_name, self.primary_source), + ): + if self.output_single_stem and self.output_single_stem.lower() != stem_name.lower(): + continue + stems[stem_name] = self._prepare_numpy_stem(source) + return stems + # Save and process the secondary stem if needed if not self.output_single_stem or self.output_single_stem.lower() == self.secondary_stem_name.lower(): self.secondary_stem_output_path = self.get_stem_output_path(self.secondary_stem_name, custom_output_names) diff --git a/audio_separator/separator/architectures/mdxc_separator.py b/audio_separator/separator/architectures/mdxc_separator.py index d03f2be2..9a09d571 100644 --- a/audio_separator/separator/architectures/mdxc_separator.py +++ b/audio_separator/separator/architectures/mdxc_separator.py @@ -305,26 +305,31 @@ def _run_roformer_model(self, part): ) return output - def separate(self, audio_file_path, custom_output_names=None): + def separate(self, audio_file_path, custom_output_names=None, *, return_numpy=False): """ Separates the audio file into primary and secondary sources based on the model's configuration. - It processes the mix, demixes it into sources, normalizes the sources, and saves the output files. + It processes the mix, demixes it into sources, and saves files or returns stem arrays. Args: - audio_file_path (str): The path to the audio file to be processed. + audio_file_path (str or np.ndarray): Input file path, or audio frames in numpy mode. custom_output_names (dict, optional): Custom names for the output files. Defaults to None. + return_numpy (bool): Accept an audio array and return stems without writing files. Returns: - list: A list of paths to the output files generated by the separation process. + list or dict: Output file paths, or stem names mapped to audio arrays. """ self.primary_source = None self.secondary_source = None - self.audio_file_path = audio_file_path - self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] + if return_numpy: + self.clear_file_specific_paths() + mix = self.prepare_mix(audio_file_path) + else: + self.audio_file_path = audio_file_path + self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] - self.logger.debug(f"Preparing mix for input audio file {self.audio_file_path}...") - mix = self.prepare_mix(self.audio_file_path) + self.logger.debug(f"Preparing mix for input audio file {self.audio_file_path}...") + mix = self.prepare_mix(self.audio_file_path) # Short inputs need the configured segment size, but this automatic # override must not persist when the separator instance is reused. @@ -337,6 +342,21 @@ def separate(self, audio_file_path, custom_output_names=None): source = self.demix(mix=mix, override_model_segment_size=override_model_segment_size) self.logger.debug("Demixing completed.") + if return_numpy: + if isinstance(source, dict): + training = self.model_data_cfgdict.training + stem_names = [training.target_instrument] if training.target_instrument else training.instruments + if not (self.process_all_stems and len(stem_names) > 2): + stem_names = [self.secondary_stem_name, self.primary_stem_name] + sources = {name: source[name] for name in stem_names} + else: + sources = {self.primary_stem_name: source} + return { + name: self._prepare_numpy_stem(stem.T) + for name, stem in sources.items() + if not self.output_single_stem or self.output_single_stem.lower() == name.lower() + } + output_files = [] self.logger.debug("Processing output files...") diff --git a/audio_separator/separator/architectures/vr_separator.py b/audio_separator/separator/architectures/vr_separator.py index 1de806e5..57fac488 100644 --- a/audio_separator/separator/architectures/vr_separator.py +++ b/audio_separator/separator/architectures/vr_separator.py @@ -138,51 +138,57 @@ def _ensure_model_loaded(self, nn_arch_size): self.is_vr_51_model = is_vr_51_model self.logger.debug("Model loaded and moved to device.") - def separate(self, audio_file_path, custom_output_names=None): + def separate(self, audio_file_path, custom_output_names=None, *, return_numpy=False): """ Separates the audio file into primary and secondary sources based on the model's configuration. - It processes the mix, demixes it into sources, normalizes the sources, and saves the output files. + It processes the mix, demixes it into sources, and saves files or returns stem arrays. Args: - audio_file_path (str): The path to the audio file to be processed. + audio_file_path (str or np.ndarray): Input file path, or audio frames in numpy mode. custom_output_names (dict, optional): Custom names for the output files. Defaults to None. + return_numpy (bool): Accept an audio array and return stems without writing files. Returns: - list: A list of paths to the output files generated by the separation process. + list or dict: Output file paths, or stem names mapped to audio arrays. """ self.primary_source = None self.secondary_source = None - self.audio_file_path = audio_file_path - self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[ 0] - - # Detect input audio bit depth for output preservation - try: - import soundfile as sf - info = sf.info(audio_file_path) - self.input_audio_subtype = info.subtype - self.logger.info(f"Input audio subtype: {self.input_audio_subtype}") - - # Map subtype to wav_subtype for soundfile and set input_bit_depth for pydub - if "24" in self.input_audio_subtype: - self.wav_subtype = "PCM_24" - self.input_bit_depth = 24 - self.logger.info("Detected 24-bit input audio") - elif "32" in self.input_audio_subtype: - self.wav_subtype = "PCM_32" - self.input_bit_depth = 32 - self.logger.info("Detected 32-bit input audio") - else: + mix = None + if return_numpy: + self.clear_file_specific_paths() + mix = self.prepare_mix(audio_file_path) + else: + self.audio_file_path = audio_file_path + self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[ 0] + + # Detect input audio bit depth for output preservation + try: + import soundfile as sf + info = sf.info(audio_file_path) + self.input_audio_subtype = info.subtype + self.logger.info(f"Input audio subtype: {self.input_audio_subtype}") + + # Map subtype to wav_subtype for soundfile and set input_bit_depth for pydub + if "24" in self.input_audio_subtype: + self.wav_subtype = "PCM_24" + self.input_bit_depth = 24 + self.logger.info("Detected 24-bit input audio") + elif "32" in self.input_audio_subtype: + self.wav_subtype = "PCM_32" + self.input_bit_depth = 32 + self.logger.info("Detected 32-bit input audio") + else: + self.wav_subtype = "PCM_16" + self.input_bit_depth = 16 + self.logger.info("Detected 16-bit input audio") + except Exception as e: + self.logger.warning(f"Could not detect input audio bit depth: {e}. Defaulting to PCM_16") self.wav_subtype = "PCM_16" + self.input_audio_subtype = None self.input_bit_depth = 16 - self.logger.info("Detected 16-bit input audio") - except Exception as e: - self.logger.warning(f"Could not detect input audio bit depth: {e}. Defaulting to PCM_16") - self.wav_subtype = "PCM_16" - self.input_audio_subtype = None - self.input_bit_depth = 16 - self.logger.debug(f"Starting separation for input audio file {self.audio_file_path}...") + self.logger.debug(f"Starting separation for input audio file {self.audio_file_path}...") nn_arch_sizes = [31191, 33966, 56817, 123821, 123812, 129605, 218409, 537238, 537227] # default model_size = math.ceil(os.stat(self.model_path).st_size / 1024) @@ -191,7 +197,7 @@ def separate(self, audio_file_path, custom_output_names=None): self._ensure_model_loaded(nn_arch_size) - y_spec, v_spec = self.inference_vr(self.loading_mix(), self.torch_device, self.aggressiveness) + y_spec, v_spec = self.inference_vr(self.loading_mix(mix), self.torch_device, self.aggressiveness) self.logger.debug("Inference completed.") # Sanitize y_spec and v_spec to replace NaN and infinite values @@ -227,6 +233,18 @@ def separate(self, audio_file_path, custom_output_names=None): self.output_single_stem = None self.logger.warning(f"The output_single_stem setting '{self.output_single_stem}' does not match any of the output files: '{self.primary_stem_name}' and '{self.secondary_stem_name}'. For this model '{self.model_name}', the output_single_stem setting will be ignored and all output files will be saved.") + if return_numpy: + # Arrays are returned at self.sample_rate; file output keeps its fixed 44100 Hz resampling below. + stems = {} + for stem_name, spec in ((self.primary_stem_name, y_spec), (self.secondary_stem_name, v_spec)): + if self.output_single_stem and self.output_single_stem.lower() != stem_name.lower(): + continue + source = self.spec_to_wav(spec) + if self.model_samplerate != self.sample_rate: + source = librosa.resample(source, orig_sr=self.model_samplerate, target_sr=self.sample_rate) + stems[stem_name] = self._prepare_numpy_stem(source.T) + return stems + # Save and process the primary stem if needed if not self.output_single_stem or self.output_single_stem.lower() == self.primary_stem_name.lower(): self.logger.debug(f"Processing primary stem: {self.primary_stem_name}") @@ -269,12 +287,13 @@ def separate(self, audio_file_path, custom_output_names=None): return output_files - def loading_mix(self): + def loading_mix(self, mix=None): X_wave, X_spec_s = {}, {} bands_n = len(self.model_params.param["band"]) - audio_file = spec_utils.write_array_to_mem(self.audio_file_path, subtype=self.wav_subtype) + # An in-memory mix skips the lossy WAV round trip through write_array_to_mem. + audio_file = None if mix is not None else spec_utils.write_array_to_mem(self.audio_file_path, subtype=self.wav_subtype) is_mp3 = audio_file.endswith(".mp3") if isinstance(audio_file, str) else False self.logger.debug(f"loading_mix iteraring through {bands_n} bands") @@ -287,7 +306,13 @@ def loading_mix(self): wav_resolution = "polyphase" if d == bands_n: # high-end band - X_wave[d], _ = librosa.load(audio_file, sr=bp["sr"], mono=False, dtype=np.float32, res_type=wav_resolution) + if mix is None: + X_wave[d], _ = librosa.load(audio_file, sr=bp["sr"], mono=False, dtype=np.float32, res_type=wav_resolution) + elif bp["sr"] == self.sample_rate: + X_wave[d] = mix + else: + # Resample in memory exactly as librosa.load would for a file at self.sample_rate. + X_wave[d] = librosa.resample(mix, orig_sr=self.sample_rate, target_sr=bp["sr"], res_type=wav_resolution) X_spec_s[d] = spec_utils.wave_to_spectrogram(X_wave[d], bp["hl"], bp["n_fft"], self.model_params, band=d, is_v51_model=self.is_vr_51_model) if not np.any(X_wave[d]) and is_mp3: diff --git a/audio_separator/separator/common_separator.py b/audio_separator/separator/common_separator.py index be204264..179f575d 100644 --- a/audio_separator/separator/common_separator.py +++ b/audio_separator/separator/common_separator.py @@ -202,6 +202,15 @@ def final_process(self, stem_path, source, stem_name): return {stem_name: source} + def _prepare_numpy_stem(self, source): + """Validate and normalize a sample-major stem, returning an owned array.""" + source = validate_audio_source(source) + if source.size == 0 or not np.isfinite(source).all(): + raise InvalidAudioDataError("Stem contains empty or non-finite audio") + return spec_utils.normalize( + wave=source, max_peak=self.normalization_threshold, min_peak=self.amplification_threshold + ).copy() + def cached_sources_clear(self): """ Clears the cache dictionaries for VR, MDX, and Demucs models. diff --git a/audio_separator/separator/separator.py b/audio_separator/separator/separator.py index a71f3f8e..ebe48b08 100644 --- a/audio_separator/separator/separator.py +++ b/audio_separator/separator/separator.py @@ -1092,6 +1092,42 @@ def load_model(self, model_filename="model_bs_roformer_ep_317_sdr_12.9755.ckpt", self.logger.debug("Loading model completed.") self.logger.info(f'Load model duration: {time.strftime("%H:%M:%S", time.gmtime(int(time.perf_counter() - load_model_start_time)))}') + def separate_numpy(self, audio, sample_rate): + """Separate floating-point audio without reading or writing audio files. + + Input is mono frames or a (frames, channels) array with one or two channels. + Returns a dict of stem names to independent (frames, 2) arrays at + ``self.sample_rate``. Input at another sample rate is resampled in memory. + Currently supports a single MDX, MDXC, VR or Demucs model; file chunking does not apply. + """ + if not isinstance(audio, np.ndarray) or not np.issubdtype(audio.dtype, np.floating): + raise TypeError("audio must be a floating-point NumPy array") + audio = validate_audio_source(audio) + if isinstance(sample_rate, (bool, np.bool_)) or not isinstance(sample_rate, (int, np.integer)) or sample_rate <= 0: + raise ValueError("sample_rate must be a positive integer") + audio = np.array(audio, dtype=np.float32, copy=True) + if audio.size == 0 or not np.isfinite(audio).all(): + raise InvalidAudioDataError("Audio array must contain nonempty, finite audio frames") + # load_model() records an ensemble without creating a model instance, so reject it first. + if len(self.model_filenames) > 1: + raise NotImplementedError("NumPy separation currently supports a single MDX, MDXC, VR or Demucs model") + if not (self.torch_device and self.model_instance): + raise ValueError("Initialization failed or model not loaded. Please load a model before attempting to separate.") + + from audio_separator.separator.architectures.mdx_separator import MDXSeparator + from audio_separator.separator.architectures.mdxc_separator import MDXCSeparator + from audio_separator.separator.architectures.vr_separator import VRSeparator + from audio_separator.separator.architectures.demucs_separator import DemucsSeparator + + separator_types = (MDXSeparator, MDXCSeparator, VRSeparator, DemucsSeparator) + if not isinstance(self.model_instance, separator_types): + raise NotImplementedError("NumPy separation currently supports a single MDX, MDXC, VR or Demucs model") + if audio.ndim == 2 and audio.shape[1] == 1: + audio = audio[:, 0] + if sample_rate != self.sample_rate: + audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=self.sample_rate, axis=0) + return self._separate_file(audio, return_numpy=True) + def separate(self, audio_file_path, custom_output_names=None): """ Separates the audio file(s) into different stems (e.g., vocals, instruments) using the loaded model. @@ -1185,19 +1221,20 @@ def separate(self, audio_file_path, custom_output_names=None): raise BatchSeparationError(output_files, failures) return output_files - def _separate_file(self, audio_file_path, custom_output_names=None): + def _separate_file(self, audio_file_path, custom_output_names=None, *, return_numpy=False): """ - Internal method to handle separation for a single audio file. - This method performs the actual separation process for a single audio file. It logs the start and end of the process, + Internal method to handle separation for a single audio file or array. + This method performs the actual separation process for a single input. It logs the start and end of the process, handles autocast if enabled, and ensures GPU cache is cleared after processing. Parameters: - - audio_file_path (str): The path to the audio file. + - audio_file_path (str or np.ndarray): The path to the audio file, or in-memory audio. - custom_output_names (dict, optional): Custom names for the output files. Defaults to None. + - return_numpy (bool): Return stem arrays instead of writing audio files. Returns: - - output_files (list of str): A list containing the paths to the separated audio stem files. + - output_files (list of str or dict): Output paths, or stem names mapped to arrays. """ # Check if chunking is enabled and file is large enough - if self.chunk_duration is not None: + if not return_numpy and self.chunk_duration is not None: import librosa duration = librosa.get_duration(path=audio_file_path) @@ -1209,7 +1246,7 @@ def _separate_file(self, audio_file_path, custom_output_names=None): return self._process_with_chunking(audio_file_path, custom_output_names) # Log the start of the separation process - self.logger.info(f"Starting separation process for audio_file_path: {audio_file_path}") + self.logger.info("Starting separation process for %s", "audio array" if return_numpy else f"audio_file_path: {audio_file_path}") separate_start_time = time.perf_counter() # Log normalization and amplification thresholds @@ -1222,17 +1259,18 @@ def _separate_file(self, audio_file_path, custom_output_names=None): effective_precision = self.effective_precision inference_device = getattr(self.model_instance, "torch_device", self.torch_device) inference_device_type = getattr(inference_device, "type", str(inference_device)) + separation_options = {"return_numpy": True} if return_numpy else {} try: if effective_precision == NATIVE_FP16: self.logger.debug("Using native float16 inference.") - output_files = self.model_instance.separate(audio_file_path, custom_output_names) + output_files = self.model_instance.separate(audio_file_path, custom_output_names, **separation_options) elif effective_precision == AUTOCAST and inference_device_type != "privateuseone": self.logger.debug("Using autocast inference on %s.", inference_device_type) with autocast_mode.autocast(inference_device_type): - output_files = self.model_instance.separate(audio_file_path, custom_output_names) + output_files = self.model_instance.separate(audio_file_path, custom_output_names, **separation_options) else: self.logger.debug("Using float32 inference.") - output_files = self.model_instance.separate(audio_file_path, custom_output_names) + output_files = self.model_instance.separate(audio_file_path, custom_output_names, **separation_options) finally: # Clear per-file state even when inference or audio export fails. active_error = sys.exc_info()[1] diff --git a/tests/integration/test_numpy_separation.py b/tests/integration/test_numpy_separation.py new file mode 100644 index 00000000..d99bed6a --- /dev/null +++ b/tests/integration/test_numpy_separation.py @@ -0,0 +1,63 @@ +from itertools import combinations +from pathlib import Path + +import numpy as np +import pytest +import soundfile as sf + +from audio_separator.separator import Separator +from audio_separator.separator.architectures.demucs_separator import DemucsSeparator +from audio_separator.separator.architectures.mdx_separator import MDXSeparator +from audio_separator.separator.architectures.mdxc_separator import MDXCSeparator +from audio_separator.separator.architectures.vr_separator import VRSeparator + +VOCALS_AND_INSTRUMENTAL = {"Vocals", "Instrumental"} +DEMUCS_6_STEMS = {"Vocals", "Drums", "Bass", "Other", "Guitar", "Piano"} + +# The MDXC models cover both MDXC demix paths: MDX23C (TFC_TDF_net) with two +# trained stems, and RoFormer with a single target plus a residual stem. +# VR rebuilds waveforms from STFT frames, so its stems may end a few milliseconds early. +NUMPY_MODELS = [ + pytest.param("kuielab_b_vocals.onnx", MDXSeparator, False, 0.0, VOCALS_AND_INSTRUMENTAL, id="mdx"), + pytest.param("MDX23C-8KFFT-InstVoc_HQ.ckpt", MDXCSeparator, False, 0.0, VOCALS_AND_INSTRUMENTAL, id="mdxc-mdx23c"), + pytest.param("model_bs_roformer_ep_317_sdr_12.9755.ckpt", MDXCSeparator, True, 0.0, VOCALS_AND_INSTRUMENTAL, id="mdxc-roformer"), + pytest.param("MGM_MAIN_v4.pth", VRSeparator, False, 0.05, VOCALS_AND_INSTRUMENTAL, id="vr"), + pytest.param("htdemucs_6s.yaml", DemucsSeparator, False, 0.0, DEMUCS_6_STEMS, id="demucs"), +] + + +@pytest.mark.parametrize("model_filename, architecture, is_roformer, max_length_diff_seconds, expected_stems", NUMPY_MODELS) +def test_numpy_separation_returns_arrays_without_output_files( + tmp_path, monkeypatch, model_filename, architecture, is_roformer, max_length_diff_seconds, expected_stems +): + repo = Path(__file__).resolve().parents[2] + audio, sample_rate = sf.read(repo / "tests/inputs/mardy20s.flac", dtype="float32") + separator = Separator(output_dir=str(tmp_path)) + separator.load_model(model_filename=model_filename) + assert type(separator.model_instance) is architecture + assert getattr(separator.model_instance, "is_roformer", False) is is_roformer + original_audio = audio.copy() + + def unexpected_audio_io(*args, **kwargs): + raise AssertionError("NumPy separation must not read or write audio files") + + for method in ("load", "get_duration"): + monkeypatch.setattr(f"audio_separator.separator.common_separator.librosa.{method}", unexpected_audio_io) + monkeypatch.setattr("audio_separator.separator.common_separator.CommonSeparator.write_audio", unexpected_audio_io) + + stems = separator.separate_numpy(audio, sample_rate=sample_rate) + + np.testing.assert_array_equal(audio, original_audio) + assert isinstance(stems, dict) + assert set(stems) == expected_stems + max_length_diff = int(max_length_diff_seconds * sample_rate) + for stem in stems.values(): + assert isinstance(stem, np.ndarray) + assert stem.ndim == 2 and stem.shape[1] == audio.shape[1] + assert abs(stem.shape[0] - audio.shape[0]) <= max_length_diff + assert np.issubdtype(stem.dtype, np.floating) + assert np.isfinite(stem).all() + assert np.any(stem) + assert not np.shares_memory(stem, audio) + assert not any(np.shares_memory(first, second) for first, second in combinations(stems.values(), 2)) + assert not any(tmp_path.iterdir()) diff --git a/tests/unit/test_numpy_api.py b/tests/unit/test_numpy_api.py new file mode 100644 index 00000000..eb276026 --- /dev/null +++ b/tests/unit/test_numpy_api.py @@ -0,0 +1,143 @@ +import logging +from unittest.mock import Mock + +import numpy as np +import pytest +import torch + +from audio_separator.separator import Separator +from audio_separator.separator.architectures.mdx_separator import MDXSeparator +from audio_separator.separator.exceptions import InvalidAudioDataError +from audio_separator.separator.execution_policy import FP32 + + +@pytest.fixture +def numpy_separator(tmp_path, monkeypatch): + separator = Separator(model_file_dir=tmp_path / "models", output_dir=tmp_path / "output", info_only=True) + model = MDXSeparator.__new__(MDXSeparator) + model.logger = logging.getLogger(__name__) + model.torch_device = torch.device("cpu") + model.effective_precision = FP32 + model.sample_rate = 44100 + model.normalization_threshold = 0.9 + model.amplification_threshold = 0.0 + model.output_single_stem = None + model.input_bit_depth = None + model.input_subtype = None + model.primary_stem_name = "Vocals" + model.secondary_stem_name = "Instrumental" + model.compensate = 1.0 + model.invert_using_spec = False + model.demix = Mock(side_effect=lambda mix, is_match_mix=False: mix.copy()) + model.clear_gpu_cache = Mock() + model.write_audio = Mock(side_effect=AssertionError("Unexpected audio file write")) + for method in ("load", "get_duration"): + monkeypatch.setattr( + f"audio_separator.separator.common_separator.librosa.{method}", + Mock(side_effect=AssertionError("Unexpected audio file read")), + ) + separator.torch_device = model.torch_device + separator.model_instance = model + separator.model_filename = "test.onnx" + separator.model_filenames = ["test.onnx"] + return separator + + +def test_numpy_separation_preserves_read_only_input_and_avoids_audio_io(numpy_separator): + audio = np.full((32, 2), 2.0, dtype=np.float32) + audio.flags.writeable = False + numpy_separator.chunk_duration = 0.001 + + stems = numpy_separator.separate_numpy(audio, 44100) + + np.testing.assert_array_equal(audio, np.full((32, 2), 2.0, dtype=np.float32)) + assert set(stems) == {"Vocals", "Instrumental"} + for stem in stems.values(): + assert stem.shape == audio.shape + assert np.isfinite(stem).all() + assert not np.shares_memory(stem, audio) + assert numpy_separator.model_instance.primary_source is None + assert numpy_separator.model_instance.secondary_source is None + + +@pytest.mark.parametrize("column", [False, True]) +def test_numpy_separation_converts_mono_to_stereo(numpy_separator, column): + audio = np.linspace(-0.5, 0.5, 64, dtype=np.float32) + if column: + audio = audio[:, None] + + stems = numpy_separator.separate_numpy(audio, 44100) + + for stem in stems.values(): + assert stem.shape == (64, 2) + np.testing.assert_array_equal(stem[:, 0], stem[:, 1]) + + +def test_numpy_separation_resamples_on_the_frame_axis(numpy_separator): + audio = np.full((100, 2), 0.25, dtype=np.float32) + + stems = numpy_separator.separate_numpy(audio, 22050) + + assert set(stems) == {"Vocals", "Instrumental"} + assert all(stem.shape == (200, 2) for stem in stems.values()) + + +def test_numpy_separation_keeps_results_independent_between_calls(numpy_separator): + first = numpy_separator.separate_numpy(np.full((32, 2), 0.25, dtype=np.float32), 44100) + saved = {name: stem.copy() for name, stem in first.items()} + + second = numpy_separator.separate_numpy(np.full((32, 2), 0.5, dtype=np.float32), 44100) + + for name, stem in first.items(): + np.testing.assert_array_equal(stem, saved[name]) + assert not np.shares_memory(stem, second[name]) + assert not np.array_equal(stem, second[name]) + + +def test_numpy_separation_honors_single_stem_selection(numpy_separator): + numpy_separator.model_instance.output_single_stem = "vocals" + + stems = numpy_separator.separate_numpy(np.ones((32, 2), dtype=np.float32), 44100) + + assert set(stems) == {"Vocals"} + + +def test_numpy_separation_cleans_state_after_inference_failure(numpy_separator): + model = numpy_separator.model_instance + model.demix.side_effect = [np.ones((2, 32), dtype=np.float32), RuntimeError("inference failed")] + + with pytest.raises(RuntimeError, match="inference failed"): + numpy_separator.separate_numpy(np.ones((32, 2), dtype=np.float32), 44100) + + assert model.primary_source is None + assert model.secondary_source is None + assert model.audio_file_path is None + + +@pytest.mark.parametrize( + "audio,error", + [ + ("input.wav", TypeError), + (np.zeros((32, 2), dtype=np.int16), TypeError), + (np.empty((0, 2), dtype=np.float32), InvalidAudioDataError), + (np.full((32, 2), np.nan, dtype=np.float32), InvalidAudioDataError), + (np.zeros((32, 3), dtype=np.float32), InvalidAudioDataError), + ], +) +def test_numpy_separation_rejects_invalid_audio(numpy_separator, audio, error): + with pytest.raises(error): + numpy_separator.separate_numpy(audio, 44100) + + +@pytest.mark.parametrize("sample_rate", [0, -1, True, 44100.5]) +def test_numpy_separation_rejects_invalid_sample_rates(numpy_separator, sample_rate): + with pytest.raises(ValueError, match="positive integer"): + numpy_separator.separate_numpy(np.ones((32, 2), dtype=np.float32), sample_rate) + + +def test_numpy_separation_rejects_ensemble_before_any_model_is_loaded(tmp_path): + separator = Separator(model_file_dir=tmp_path / "models", output_dir=tmp_path / "output", info_only=True) + separator.load_model(model_filename=["first.ckpt", "second.ckpt"]) + + with pytest.raises(NotImplementedError, match="single MDX"): + separator.separate_numpy(np.ones((32, 2), dtype=np.float32), 44100) diff --git a/tests/unit/test_numpy_demucs_api.py b/tests/unit/test_numpy_demucs_api.py new file mode 100644 index 00000000..de999876 --- /dev/null +++ b/tests/unit/test_numpy_demucs_api.py @@ -0,0 +1,107 @@ +import logging +from unittest.mock import Mock + +import numpy as np +import pytest +import torch + +from audio_separator.separator import Separator +from audio_separator.separator.architectures import demucs_separator +from audio_separator.separator.architectures.demucs_separator import DEMUCS_4_SOURCE_MAPPER, DemucsSeparator +from audio_separator.separator.execution_policy import FP32 + +STEMS_BY_SOURCE_COUNT = { + 2: {"Instrumental", "Vocals"}, + 4: {"Bass", "Drums", "Other", "Vocals"}, + 6: {"Bass", "Drums", "Other", "Vocals", "Guitar", "Piano"}, +} + + +def fake_demix(num_sources): + """Return a demix stand-in whose source i is the mix scaled by (i + 1) / 10.""" + return lambda mix: np.stack([mix * ((index + 1) / 10) for index in range(num_sources)]) + + +@pytest.fixture +def demucs_numpy_separator(tmp_path, monkeypatch): + for method in ("load", "get_duration"): + monkeypatch.setattr( + f"audio_separator.separator.common_separator.librosa.{method}", + Mock(side_effect=AssertionError("Unexpected audio file read")), + ) + # Only model loading and inference are mocked; stem mapping and post-processing run for real. + monkeypatch.setattr(demucs_separator, "HDemucs", Mock()) + monkeypatch.setattr(demucs_separator, "get_demucs_model", Mock()) + monkeypatch.setattr(demucs_separator, "demucs_segments", lambda segment_size, model: model) + + separator = Separator(model_file_dir=tmp_path / "models", output_dir=tmp_path / "output", info_only=True) + model = DemucsSeparator.__new__(DemucsSeparator) + model.logger = logging.getLogger(__name__) + model.torch_device = torch.device("cpu") + model.effective_precision = FP32 + model.sample_rate = 44100 + model.normalization_threshold = 0.9 + model.amplification_threshold = 0.0 + model.output_single_stem = None + model.input_bit_depth = None + model.input_subtype = None + model.model_name = "test" + model.model_path = str(tmp_path / "test.yaml") + model.segment_size = "Default" + model.demucs_source_map = DEMUCS_4_SOURCE_MAPPER + model.demix_demucs = Mock(side_effect=fake_demix(4)) + model.clear_gpu_cache = Mock() + model.write_audio = Mock(side_effect=AssertionError("Unexpected audio file write")) + separator.torch_device = model.torch_device + separator.model_instance = model + separator.model_filename = "test.yaml" + separator.model_filenames = ["test.yaml"] + return separator + + +@pytest.mark.parametrize("num_sources", [2, 4, 6]) +def test_demucs_numpy_maps_every_source_to_its_stem(demucs_numpy_separator, num_sources): + model = demucs_numpy_separator.model_instance + model.demix_demucs.side_effect = fake_demix(num_sources) + audio = np.full((32, 2), 0.5, dtype=np.float32) + audio.flags.writeable = False + + stems = demucs_numpy_separator.separate_numpy(audio, 44100) + + np.testing.assert_array_equal(audio, np.full((32, 2), 0.5, dtype=np.float32)) + assert set(stems) == STEMS_BY_SOURCE_COUNT[num_sources] + for stem_name, stem in stems.items(): + index = model.demucs_source_map[stem_name] + np.testing.assert_allclose(stem, audio * ((index + 1) / 10), rtol=1e-6) + assert not np.shares_memory(stem, audio) + + +def test_demucs_numpy_honors_single_stem_selection(demucs_numpy_separator): + demucs_numpy_separator.model_instance.output_single_stem = "vocals" + + stems = demucs_numpy_separator.separate_numpy(np.full((32, 2), 0.5, dtype=np.float32), 44100) + + assert set(stems) == {"Vocals"} + + +def test_demucs_numpy_converts_mono_to_stereo(demucs_numpy_separator): + stems = demucs_numpy_separator.separate_numpy(np.linspace(-0.5, 0.5, 64, dtype=np.float32), 44100) + + for stem in stems.values(): + assert stem.shape == (64, 2) + np.testing.assert_array_equal(stem[:, 0], stem[:, 1]) + + +def test_demucs_file_mode_still_loads_and_writes_audio_files(demucs_numpy_separator, monkeypatch, tmp_path): + model = demucs_numpy_separator.model_instance + model.output_format = "FLAC" + model.write_audio = Mock() + load = Mock(return_value=(np.full((2, 32), 0.5, dtype=np.float32), 44100)) + monkeypatch.setattr("audio_separator.separator.common_separator.librosa.load", load) + input_path = str(tmp_path / "song.flac") + + output_files = model.separate(input_path) + + assert output_files == [f"song_({name})_test.flac" for name in ("Bass", "Drums", "Other", "Vocals")] + assert load.call_args.args[0] == input_path + assert model.write_audio.call_count == 4 diff --git a/tests/unit/test_numpy_mdxc_api.py b/tests/unit/test_numpy_mdxc_api.py new file mode 100644 index 00000000..31e50b9a --- /dev/null +++ b/tests/unit/test_numpy_mdxc_api.py @@ -0,0 +1,99 @@ +import logging +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest +import torch + +from audio_separator.separator import Separator +from audio_separator.separator.architectures.mdxc_separator import MDXCSeparator +from audio_separator.separator.execution_policy import FP32 + + +@pytest.fixture +def mdxc_numpy_separator(tmp_path, monkeypatch): + separator = Separator(model_file_dir=tmp_path / "models", output_dir=tmp_path / "output", info_only=True) + model = MDXCSeparator.__new__(MDXCSeparator) + model.logger = logging.getLogger(__name__) + model.torch_device = torch.device("cpu") + model.effective_precision = FP32 + model.sample_rate = 44100 + model.normalization_threshold = 0.9 + model.amplification_threshold = 0.0 + model.output_single_stem = None + model.input_bit_depth = None + model.input_subtype = None + model.primary_stem_name = "Vocals" + model.secondary_stem_name = "Instrumental" + model.process_all_stems = True + model.override_model_segment_size = False + model.model_data_cfgdict = SimpleNamespace( + training=SimpleNamespace(target_instrument="Vocals", instruments=["Vocals", "Instrumental"]) + ) + model.demix = Mock(side_effect=lambda mix, **kwargs: {"Vocals": mix * 0.25, "Instrumental": mix * 0.75}) + model.clear_gpu_cache = Mock() + model.write_audio = Mock(side_effect=AssertionError("Unexpected audio file write")) + for method in ("load", "get_duration"): + monkeypatch.setattr( + f"audio_separator.separator.common_separator.librosa.{method}", + Mock(side_effect=AssertionError("Unexpected audio file read")), + ) + separator.torch_device = model.torch_device + separator.model_instance = model + separator.model_filename = "test.ckpt" + separator.model_filenames = ["test.ckpt"] + return separator + + +def test_mdxc_numpy_returns_stereo_arrays_without_changing_input(mdxc_numpy_separator): + audio = np.full((32, 2), 0.5, dtype=np.float32) + audio.flags.writeable = False + + stems = mdxc_numpy_separator.separate_numpy(audio, 44100) + + np.testing.assert_array_equal(audio, np.full((32, 2), 0.5, dtype=np.float32)) + np.testing.assert_allclose(stems["Vocals"], audio * 0.25) + np.testing.assert_allclose(stems["Instrumental"], audio * 0.75) + assert all(stem.shape == audio.shape for stem in stems.values()) + + +@pytest.mark.parametrize("selected_stem", [None, "bass"]) +def test_mdxc_numpy_returns_selected_multistem_outputs(mdxc_numpy_separator, selected_stem): + model = mdxc_numpy_separator.model_instance + model.model_data_cfgdict.training = SimpleNamespace(target_instrument=None, instruments=["Vocals", "Drums", "Bass"]) + model.output_single_stem = selected_stem + model.demix.side_effect = lambda mix, **kwargs: { + "Vocals": mix * 0.25, + "Drums": mix * 0.5, + "Bass": mix * 0.75, + } + + stems = mdxc_numpy_separator.separate_numpy(np.ones((32, 2), dtype=np.float32), 44100) + + assert set(stems) == ({"Bass"} if selected_stem else {"Vocals", "Drums", "Bass"}) + for stem in stems.values(): + assert stem.shape == (32, 2) + assert np.isfinite(stem).all() + + +def test_mdxc_numpy_returns_single_array_source(mdxc_numpy_separator): + mdxc_numpy_separator.model_instance.demix.side_effect = lambda mix, **kwargs: mix * 0.25 + + stems = mdxc_numpy_separator.separate_numpy(np.ones((32, 2), dtype=np.float32), 44100) + + assert set(stems) == {"Vocals"} + assert stems["Vocals"].shape == (32, 2) + + +def test_mdxc_numpy_results_remain_independent_between_calls(mdxc_numpy_separator): + first = mdxc_numpy_separator.separate_numpy(np.full((32, 2), 0.25, dtype=np.float32), 44100) + saved = {name: stem.copy() for name, stem in first.items()} + + second = mdxc_numpy_separator.separate_numpy(np.full((32, 2), 0.5, dtype=np.float32), 44100) + + for name, stem in first.items(): + np.testing.assert_array_equal(stem, saved[name]) + assert not np.shares_memory(stem, second[name]) + assert not np.array_equal(stem, second[name]) + assert mdxc_numpy_separator.model_instance.override_model_segment_size is False diff --git a/tests/unit/test_numpy_vr_api.py b/tests/unit/test_numpy_vr_api.py new file mode 100644 index 00000000..f7149fa6 --- /dev/null +++ b/tests/unit/test_numpy_vr_api.py @@ -0,0 +1,134 @@ +import logging +from pathlib import Path +from unittest.mock import Mock + +import numpy as np +import pytest +import torch + +from audio_separator.separator import Separator +from audio_separator.separator.architectures import vr_separator +from audio_separator.separator.architectures.vr_separator import VRSeparator +from audio_separator.separator.execution_policy import FP32 +from audio_separator.separator.uvr_lib_v5.vr_network.model_param_init import ModelParameters + +MODEL_PARAMS_DIR = Path(vr_separator.__file__).resolve().parents[1] / "uvr_lib_v5" / "vr_network" / "modelparams" +# VR rebuilds waveforms from STFT frames, so stems may end a few milliseconds early. +MAX_LENGTH_DIFF = int(0.05 * 44100) + + +def stereo_tone(frames=44100): + t = np.arange(frames, dtype=np.float32) / 44100 + return np.stack([0.5 * np.sin(2 * np.pi * 440 * t), 0.25 * np.sin(2 * np.pi * 660 * t)], axis=1).astype(np.float32) + + +@pytest.fixture +def make_vr_numpy_separator(tmp_path, monkeypatch): + for method in ("load", "get_duration"): + monkeypatch.setattr( + f"audio_separator.separator.common_separator.librosa.{method}", + Mock(side_effect=AssertionError("Unexpected audio file read")), + ) + monkeypatch.setattr( + "audio_separator.separator.uvr_lib_v5.spec_utils.write_array_to_mem", + Mock(side_effect=AssertionError("Unexpected in-memory WAV round trip")), + ) + # Use scipy's polyphase resampler, as VR does on Apple Silicon, so these tests don't need libsamplerate. + monkeypatch.setattr("audio_separator.separator.uvr_lib_v5.spec_utils.wav_resolution", "polyphase") + weights = tmp_path / "test.pth" + weights.write_bytes(b"\0") + + def make(param_name="1band_sr44100_hl512"): + separator = Separator(model_file_dir=tmp_path / "models", output_dir=tmp_path / "output", info_only=True) + model = VRSeparator.__new__(VRSeparator) + model.logger = logging.getLogger(__name__) + model.torch_device = torch.device("cpu") + model.torch_device_mps = None + model.effective_precision = FP32 + model.sample_rate = 44100 + model.normalization_threshold = 0.9 + model.amplification_threshold = 0.0 + model.output_single_stem = None + model.input_bit_depth = None + model.input_subtype = None + model.primary_stem_name = "Instrumental" + model.secondary_stem_name = "Vocals" + model.model_name = "test" + model.model_path = str(weights) + model.model_params = ModelParameters(str(MODEL_PARAMS_DIR / f"{param_name}.json")) + for band in model.model_params.param["band"].values(): + band["res_type"] = "polyphase" + model.model_samplerate = model.model_params.param["sr"] + model.is_vr_51_model = False + model.high_end_process = False + model.input_high_end_h = None + model.input_high_end = None + model.aggressiveness = {} + model._ensure_model_loaded = Mock() + # Only the network is mocked; band splitting, STFT and waveform rebuilding run for real. + model.inference_vr = Mock(side_effect=lambda spec, device, aggressiveness: (spec * 0.25, spec * 0.75)) + model.clear_gpu_cache = Mock() + model.write_audio = Mock(side_effect=AssertionError("Unexpected audio file write")) + separator.torch_device = model.torch_device + separator.model_instance = model + separator.model_filename = "test.pth" + separator.model_filenames = ["test.pth"] + return separator + + return make + + +# 2band_32000 runs at 32 kHz, which covers the in-memory input and output resampling. +@pytest.mark.parametrize("param_name", ["1band_sr44100_hl512", "2band_32000"]) +def test_vr_numpy_returns_stereo_arrays_without_audio_io(make_vr_numpy_separator, param_name): + separator = make_vr_numpy_separator(param_name) + audio = stereo_tone() + audio.flags.writeable = False + + stems = separator.separate_numpy(audio, 44100) + + np.testing.assert_array_equal(audio, stereo_tone()) + assert set(stems) == {"Instrumental", "Vocals"} + for stem in stems.values(): + assert stem.ndim == 2 and stem.shape[1] == 2 + assert abs(stem.shape[0] - audio.shape[0]) <= MAX_LENGTH_DIFF + assert np.isfinite(stem).all() + assert np.any(stem) + assert not np.shares_memory(stem, audio) + assert not np.shares_memory(stems["Instrumental"], stems["Vocals"]) + separator.model_instance.inference_vr.assert_called_once() + + +def test_vr_numpy_converts_mono_to_stereo(make_vr_numpy_separator): + separator = make_vr_numpy_separator() + + stems = separator.separate_numpy(stereo_tone()[:, 0], 44100) + + for stem in stems.values(): + assert stem.shape[1] == 2 + np.testing.assert_allclose(stem[:, 0], stem[:, 1], atol=1e-6) + + +def test_vr_numpy_honors_single_stem_selection(make_vr_numpy_separator): + separator = make_vr_numpy_separator() + separator.model_instance.output_single_stem = "vocals" + + stems = separator.separate_numpy(stereo_tone(), 44100) + + assert set(stems) == {"Vocals"} + + +def test_vr_file_mode_still_loads_and_writes_audio_files(make_vr_numpy_separator, monkeypatch, tmp_path): + model = make_vr_numpy_separator().model_instance + model.output_format = "FLAC" + model.write_audio = Mock() + load = Mock(return_value=(stereo_tone().T.copy(), 44100)) + monkeypatch.setattr("audio_separator.separator.architectures.vr_separator.librosa.load", load) + monkeypatch.setattr("audio_separator.separator.uvr_lib_v5.spec_utils.write_array_to_mem", lambda path, subtype: path) + input_path = str(tmp_path / "song.flac") + + output_files = model.separate(input_path) + + assert output_files == ["song_(Instrumental)_test.flac", "song_(Vocals)_test.flac"] + assert load.call_args.args[0] == input_path + assert model.write_audio.call_count == 2