Repository navigation
feat: add Separator.separate_numpy() for in-memory separation #322
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
6caec95
3d7b04f
2be1ba1
74c27fd
b4f70b0
da0d1fa
54b1b03
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -305,26 +305,31 @@ def _run_roformer_model(self, part): | |
| ) | ||
| return output | ||
|
|
||
| def separate(self, audio_file_path, custom_output_names=None): | ||
| def separate(self, audio_file_path, custom_output_names=None, *, return_numpy=False): | ||
| """ | ||
| Separates the audio file into primary and secondary sources based on the model's configuration. | ||
| It processes the mix, demixes it into sources, normalizes the sources, and saves the output files. | ||
| It processes the mix, demixes it into sources, and saves files or returns stem arrays. | ||
|
|
||
| Args: | ||
| audio_file_path (str): The path to the audio file to be processed. | ||
| audio_file_path (str or np.ndarray): Input file path, or audio frames in numpy mode. | ||
| custom_output_names (dict, optional): Custom names for the output files. Defaults to None. | ||
| return_numpy (bool): Accept an audio array and return stems without writing files. | ||
|
|
||
| Returns: | ||
| list: A list of paths to the output files generated by the separation process. | ||
| list or dict: Output file paths, or stem names mapped to audio arrays. | ||
| """ | ||
| self.primary_source = None | ||
| self.secondary_source = None | ||
|
|
||
| self.audio_file_path = audio_file_path | ||
| self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] | ||
| if return_numpy: | ||
| self.clear_file_specific_paths() | ||
| mix = self.prepare_mix(audio_file_path) | ||
| else: | ||
| self.audio_file_path = audio_file_path | ||
| self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] | ||
|
|
||
| self.logger.debug(f"Preparing mix for input audio file {self.audio_file_path}...") | ||
| mix = self.prepare_mix(self.audio_file_path) | ||
| self.logger.debug(f"Preparing mix for input audio file {self.audio_file_path}...") | ||
| mix = self.prepare_mix(self.audio_file_path) | ||
|
|
||
| # Short inputs need the configured segment size, but this automatic | ||
| # override must not persist when the separator instance is reused. | ||
|
|
@@ -337,6 +342,21 @@ def separate(self, audio_file_path, custom_output_names=None): | |
| source = self.demix(mix=mix, override_model_segment_size=override_model_segment_size) | ||
| self.logger.debug("Demixing completed.") | ||
|
|
||
| if return_numpy: | ||
| if isinstance(source, dict): | ||
| training = self.model_data_cfgdict.training | ||
| stem_names = [training.target_instrument] if training.target_instrument else training.instruments | ||
| if not (self.process_all_stems and len(stem_names) > 2): | ||
| stem_names = [self.secondary_stem_name, self.primary_stem_name] | ||
| sources = {name: source[name] for name in stem_names} | ||
| else: | ||
| sources = {self.primary_stem_name: source} | ||
| return { | ||
| name: self._prepare_numpy_stem(stem.T) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Resolve conflicting peak thresholds for MDXC output. The constructor accepts
|
||
| for name, stem in sources.items() | ||
| if not self.output_single_stem or self.output_single_stem.lower() == name.lower() | ||
| } | ||
|
|
||
| output_files = [] | ||
| self.logger.debug("Processing output files...") | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -138,51 +138,57 @@ def _ensure_model_loaded(self, nn_arch_size): | |
| self.is_vr_51_model = is_vr_51_model | ||
| self.logger.debug("Model loaded and moved to device.") | ||
|
|
||
| def separate(self, audio_file_path, custom_output_names=None): | ||
| def separate(self, audio_file_path, custom_output_names=None, *, return_numpy=False): | ||
| """ | ||
| Separates the audio file into primary and secondary sources based on the model's configuration. | ||
| It processes the mix, demixes it into sources, normalizes the sources, and saves the output files. | ||
| It processes the mix, demixes it into sources, and saves files or returns stem arrays. | ||
|
|
||
| Args: | ||
| audio_file_path (str): The path to the audio file to be processed. | ||
| audio_file_path (str or np.ndarray): Input file path, or audio frames in numpy mode. | ||
| custom_output_names (dict, optional): Custom names for the output files. Defaults to None. | ||
| return_numpy (bool): Accept an audio array and return stems without writing files. | ||
|
|
||
| Returns: | ||
| list: A list of paths to the output files generated by the separation process. | ||
| list or dict: Output file paths, or stem names mapped to audio arrays. | ||
| """ | ||
| self.primary_source = None | ||
| self.secondary_source = None | ||
|
|
||
| self.audio_file_path = audio_file_path | ||
| self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[ 0] | ||
|
|
||
| # Detect input audio bit depth for output preservation | ||
| try: | ||
| import soundfile as sf | ||
| info = sf.info(audio_file_path) | ||
| self.input_audio_subtype = info.subtype | ||
| self.logger.info(f"Input audio subtype: {self.input_audio_subtype}") | ||
|
|
||
| # Map subtype to wav_subtype for soundfile and set input_bit_depth for pydub | ||
| if "24" in self.input_audio_subtype: | ||
| self.wav_subtype = "PCM_24" | ||
| self.input_bit_depth = 24 | ||
| self.logger.info("Detected 24-bit input audio") | ||
| elif "32" in self.input_audio_subtype: | ||
| self.wav_subtype = "PCM_32" | ||
| self.input_bit_depth = 32 | ||
| self.logger.info("Detected 32-bit input audio") | ||
| else: | ||
| mix = None | ||
| if return_numpy: | ||
| self.clear_file_specific_paths() | ||
| mix = self.prepare_mix(audio_file_path) | ||
| else: | ||
| self.audio_file_path = audio_file_path | ||
| self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[ 0] | ||
|
|
||
| # Detect input audio bit depth for output preservation | ||
| try: | ||
| import soundfile as sf | ||
| info = sf.info(audio_file_path) | ||
| self.input_audio_subtype = info.subtype | ||
| self.logger.info(f"Input audio subtype: {self.input_audio_subtype}") | ||
|
|
||
| # Map subtype to wav_subtype for soundfile and set input_bit_depth for pydub | ||
| if "24" in self.input_audio_subtype: | ||
| self.wav_subtype = "PCM_24" | ||
| self.input_bit_depth = 24 | ||
| self.logger.info("Detected 24-bit input audio") | ||
| elif "32" in self.input_audio_subtype: | ||
| self.wav_subtype = "PCM_32" | ||
| self.input_bit_depth = 32 | ||
| self.logger.info("Detected 32-bit input audio") | ||
| else: | ||
| self.wav_subtype = "PCM_16" | ||
| self.input_bit_depth = 16 | ||
| self.logger.info("Detected 16-bit input audio") | ||
| except Exception as e: | ||
| self.logger.warning(f"Could not detect input audio bit depth: {e}. Defaulting to PCM_16") | ||
| self.wav_subtype = "PCM_16" | ||
| self.input_audio_subtype = None | ||
| self.input_bit_depth = 16 | ||
| self.logger.info("Detected 16-bit input audio") | ||
| except Exception as e: | ||
| self.logger.warning(f"Could not detect input audio bit depth: {e}. Defaulting to PCM_16") | ||
| self.wav_subtype = "PCM_16" | ||
| self.input_audio_subtype = None | ||
| self.input_bit_depth = 16 | ||
|
|
||
| self.logger.debug(f"Starting separation for input audio file {self.audio_file_path}...") | ||
| self.logger.debug(f"Starting separation for input audio file {self.audio_file_path}...") | ||
|
|
||
| nn_arch_sizes = [31191, 33966, 56817, 123821, 123812, 129605, 218409, 537238, 537227] # default | ||
| model_size = math.ceil(os.stat(self.model_path).st_size / 1024) | ||
|
|
@@ -191,7 +197,7 @@ def separate(self, audio_file_path, custom_output_names=None): | |
|
|
||
| self._ensure_model_loaded(nn_arch_size) | ||
|
|
||
| y_spec, v_spec = self.inference_vr(self.loading_mix(), self.torch_device, self.aggressiveness) | ||
| y_spec, v_spec = self.inference_vr(self.loading_mix(mix), self.torch_device, self.aggressiveness) | ||
| self.logger.debug("Inference completed.") | ||
|
|
||
| # Sanitize y_spec and v_spec to replace NaN and infinite values | ||
|
|
@@ -227,6 +233,18 @@ def separate(self, audio_file_path, custom_output_names=None): | |
| self.output_single_stem = None | ||
| self.logger.warning(f"The output_single_stem setting '{self.output_single_stem}' does not match any of the output files: '{self.primary_stem_name}' and '{self.secondary_stem_name}'. For this model '{self.model_name}', the output_single_stem setting will be ignored and all output files will be saved.") | ||
|
|
||
| if return_numpy: | ||
| # Arrays are returned at self.sample_rate; file output keeps its fixed 44100 Hz resampling below. | ||
| stems = {} | ||
| for stem_name, spec in ((self.primary_stem_name, y_spec), (self.secondary_stem_name, v_spec)): | ||
| if self.output_single_stem and self.output_single_stem.lower() != stem_name.lower(): | ||
| continue | ||
| source = self.spec_to_wav(spec) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keep sub-hop VR inputs from producing empty stems. A finite 128-frame clip at 44.1 kHz passes In array mode, pad sub-hop input before the STFT and trim the reconstructed stem back to the original duration. Leave the existing file path unchanged.
|
||
| if self.model_samplerate != self.sample_rate: | ||
| source = librosa.resample(source, orig_sr=self.model_samplerate, target_sr=self.sample_rate) | ||
| stems[stem_name] = self._prepare_numpy_stem(source.T) | ||
| return stems | ||
|
|
||
| # Save and process the primary stem if needed | ||
| if not self.output_single_stem or self.output_single_stem.lower() == self.primary_stem_name.lower(): | ||
| self.logger.debug(f"Processing primary stem: {self.primary_stem_name}") | ||
|
|
@@ -269,12 +287,13 @@ def separate(self, audio_file_path, custom_output_names=None): | |
|
|
||
| return output_files | ||
|
|
||
| def loading_mix(self): | ||
| def loading_mix(self, mix=None): | ||
| X_wave, X_spec_s = {}, {} | ||
|
|
||
| bands_n = len(self.model_params.param["band"]) | ||
|
|
||
| audio_file = spec_utils.write_array_to_mem(self.audio_file_path, subtype=self.wav_subtype) | ||
| # An in-memory mix skips the lossy WAV round trip through write_array_to_mem. | ||
| audio_file = None if mix is not None else spec_utils.write_array_to_mem(self.audio_file_path, subtype=self.wav_subtype) | ||
| is_mp3 = audio_file.endswith(".mp3") if isinstance(audio_file, str) else False | ||
|
|
||
| self.logger.debug(f"loading_mix iteraring through {bands_n} bands") | ||
|
|
@@ -287,7 +306,13 @@ def loading_mix(self): | |
| wav_resolution = "polyphase" | ||
|
|
||
| if d == bands_n: # high-end band | ||
| X_wave[d], _ = librosa.load(audio_file, sr=bp["sr"], mono=False, dtype=np.float32, res_type=wav_resolution) | ||
| if mix is None: | ||
| X_wave[d], _ = librosa.load(audio_file, sr=bp["sr"], mono=False, dtype=np.float32, res_type=wav_resolution) | ||
| elif bp["sr"] == self.sample_rate: | ||
| X_wave[d] = mix | ||
| else: | ||
| # Resample in memory exactly as librosa.load would for a file at self.sample_rate. | ||
| X_wave[d] = librosa.resample(mix, orig_sr=self.sample_rate, target_sr=bp["sr"], res_type=wav_resolution) | ||
| X_spec_s[d] = spec_utils.wave_to_spectrogram(X_wave[d], bp["hl"], bp["n_fft"], self.model_params, band=d, is_v51_model=self.is_vr_51_model) | ||
|
|
||
| if not np.any(X_wave[d]) and is_mp3: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -202,6 +202,15 @@ def final_process(self, stem_path, source, stem_name): | |
|
|
||
| return {stem_name: source} | ||
|
|
||
| def _prepare_numpy_stem(self, source): | ||
| """Validate and normalize a sample-major stem, returning an owned array.""" | ||
| source = validate_audio_source(source) | ||
| if source.size == 0 or not np.isfinite(source).all(): | ||
| raise InvalidAudioDataError("Stem contains empty or non-finite audio") | ||
| return spec_utils.normalize( | ||
| wave=source, max_peak=self.normalization_threshold, min_peak=self.amplification_threshold | ||
| ).copy() | ||
|
Comment on lines
+210
to
+212
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Prevent amplification from returning non-finite samples. A finite
|
||
|
|
||
| def cached_sources_clear(self): | ||
| """ | ||
| Clears the cache dictionaries for VR, MDX, and Demucs models. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Resample array input to the model's sample rate before inference.
Separator.sample_ratesets the rate for both file and array input.separate_numpyresamples only to that value. When the value differs from the model rate, Demucs receives audio at the wrong rate. For example,Separator(sample_rate=22050)with an htdemucs model at 44,100 Hz givesapply_model22,050 frames per second, and it sizes its windows at 44,100 Hz. The model then processes the audio at the wrong time scale, which can degrade the returned stems.Convert the input to the model's
sampleratebefore inference. Return stems atself.sample_rate. The file path uses the same rate and has the same problem, so apply the fix to the shared processing path, not only toseparate_numpy.