From 379720f77c212c37d931e75ef8a50fcc9b27793c Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Sat, 16 May 2026 04:44:16 +0000 Subject: [PATCH] Add mu-law support for voicing/energy Introduce 'mulaw' voicing_domain and mu-law energy handling across configs and preprocessing. Configs updated to include voicing_domain='mulaw' (acoustic/variance templates and instances). Acoustic/variance binarizers now pass hparams['voicing_domain'] to get_voicing. Parameter adaptor clamps voicing max to 0 when domain is 'mulaw'. get_energy_librosa and get_voicing gain new domain/mu params and implement mu-law compression (default mu=255) to produce a dB-like range compatible with existing pipelines. These changes add support for mu-law representation of voicing/energy. --- configs/acoustic.yaml | 1 + configs/templates/config_acoustic.yaml | 1 + configs/templates/config_variance.yaml | 1 + configs/variance.yaml | 1 + docs/ConfigurationSchemas.md | 18 ++++++++++++++++++ modules/fastspeech/param_adaptor.py | 2 +- preprocessing/acoustic_binarizer.py | 2 +- preprocessing/variance_binarizer.py | 2 +- utils/binarizer_utils.py | 17 +++++++++++++---- 9 files changed, 38 insertions(+), 7 deletions(-) diff --git a/configs/acoustic.yaml b/configs/acoustic.yaml index e49eb1142..499cd3f26 100644 --- a/configs/acoustic.yaml +++ b/configs/acoustic.yaml @@ -47,6 +47,7 @@ mel_base: 'e' energy_smooth_width: 0.06 breathiness_smooth_width: 0.06 voicing_smooth_width: 0.06 +voicing_domain: 'mulaw' tension_smooth_width: 0.06 use_lang_id: false diff --git a/configs/templates/config_acoustic.yaml b/configs/templates/config_acoustic.yaml index 68dabb3a7..a3582fb0b 100644 --- a/configs/templates/config_acoustic.yaml +++ b/configs/templates/config_acoustic.yaml @@ -52,6 +52,7 @@ use_energy_embed: false use_breathiness_embed: false use_voicing_embed: false use_tension_embed: false +voicing_domain: 'mulaw' use_key_shift_embed: true use_speed_embed: true diff --git a/configs/templates/config_variance.yaml b/configs/templates/config_variance.yaml index 7c5af0ea2..2f9eee2f5 100644 --- a/configs/templates/config_variance.yaml +++ b/configs/templates/config_variance.yaml @@ -59,6 +59,7 @@ breathiness_db_max: -20.0 voicing_db_min: -96.0 voicing_db_max: -12.0 +voicing_domain: 'mulaw' tension_logit_min: -10.0 tension_logit_max: 10.0 diff --git a/configs/variance.yaml b/configs/variance.yaml index 2547e47a6..b18517bb8 100644 --- a/configs/variance.yaml +++ b/configs/variance.yaml @@ -89,6 +89,7 @@ breathiness_smooth_width: 0.06 voicing_db_min: -96.0 voicing_db_max: -12.0 voicing_smooth_width: 0.06 +voicing_domain: 'mulaw' tension_logit_min: -10.0 tension_logit_max: 10.0 diff --git a/docs/ConfigurationSchemas.md b/docs/ConfigurationSchemas.md index 449e9e3c0..3ec18fb39 100644 --- a/docs/ConfigurationSchemas.md +++ b/docs/ConfigurationSchemas.md @@ -2284,6 +2284,24 @@ Minimum voicing value in dB used for normalization to [-1, 1]. Note that with [d default-96.0 +### voicing_domain + +Domain of the extracted voicing curve and of the RMS energy used to compute it. The following values are currently available: + +- `'db'`: the raw RMS of the harmonic part converted to decibels. +- `'mulaw'`: mu-law compression of the raw RMS values with `mu = 255`, rescaled to a `[-96, 0]` range. This keeps the values compatible with the dB-like range expected by the rest of the pipeline, while allocating more of the available range to low-energy frames, which gives finer resolution on unvoiced and weakly voiced segments. + +The value is read when extracting voicing during preprocessing, and by the variance parameter adaptor that builds the normalization range of the voicing branch. It must therefore stay consistent between the binarized dataset and the training/inference configuration, and switching it requires re-running binarization and training from scratch. When it is `'mulaw'`, the upper normalization bound of the voicing branch is clamped to `0` regardless of [voicing_db_max](#voicing_db_max). + + + + + + + + +
visibilityacoustic, variance
scopepreprocessing, training, inference
customizabilitynormal
typestr
defaultdb
constraintsChoose from 'db' or 'mulaw'.
+ ### voicing_smooth_width Length of sinusoidal smoothing convolution kernel (in seconds) on the extracted voicing curve. diff --git a/modules/fastspeech/param_adaptor.py b/modules/fastspeech/param_adaptor.py index 77ebb8331..28735c5ce 100644 --- a/modules/fastspeech/param_adaptor.py +++ b/modules/fastspeech/param_adaptor.py @@ -49,7 +49,7 @@ def build_adaptor(self, cls=MultiVarianceDiffusion): if self.predict_voicing: ranges.append(( hparams['voicing_db_min'], - hparams['voicing_db_max'] + 0. if hparams.get('voicing_domain', 'db') == 'mulaw' else hparams['voicing_db_max'] )) clamps.append((hparams['voicing_db_min'], 0.)) diff --git a/preprocessing/acoustic_binarizer.py b/preprocessing/acoustic_binarizer.py index dff5e679b..ba8e04fc2 100644 --- a/preprocessing/acoustic_binarizer.py +++ b/preprocessing/acoustic_binarizer.py @@ -193,7 +193,7 @@ def process_item(self, item_name, meta_data, binarization_args): if self.need_voicing: # get ground truth voicing voicing = get_voicing( - dec_waveform, None, None, length=length + dec_waveform, None, None, length=length, domain=hparams.get('voicing_domain', 'db') ) global voicing_smooth diff --git a/preprocessing/variance_binarizer.py b/preprocessing/variance_binarizer.py index a20160ca6..59371904b 100644 --- a/preprocessing/variance_binarizer.py +++ b/preprocessing/variance_binarizer.py @@ -492,7 +492,7 @@ def process_item(self, item_name, meta_data, binarization_args): ) if voicing is None: voicing = get_voicing( - dec_waveform, None, None, length=length + dec_waveform, None, None, length=length, domain=hparams.get('voicing_domain', 'db') ) voicing_from_wav = True diff --git a/utils/binarizer_utils.py b/utils/binarizer_utils.py index de11dd03d..4d99ddf5e 100644 --- a/utils/binarizer_utils.py +++ b/utils/binarizer_utils.py @@ -79,14 +79,15 @@ def get_pitch_parselmouth( return f0, uv -def get_energy_librosa(waveform, length, *, hop_size, win_size, domain='db'): +def get_energy_librosa(waveform, length, *, hop_size, win_size, domain='db', mu=255.0): """ Definition of energy: RMS of the waveform, in dB representation :param waveform: [T] :param length: Expected number of frames :param hop_size: Frame width, in number of samples :param win_size: Window size, in number of samples - :param domain: db or amplitude + :param domain: 'db', 'amplitude', or 'mulaw' + :param mu: mu parameter for mu-law compression :return: energy """ energy = librosa.feature.rms(y=waveform, frame_length=win_size, hop_length=hop_size)[0] @@ -97,6 +98,10 @@ def get_energy_librosa(waveform, length, *, hop_size, win_size, domain='db'): energy = librosa.amplitude_to_db(energy, top_db=None) elif domain == 'amplitude': pass + elif domain == 'mulaw': + energy = np.log1p(mu * energy) / np.log1p(mu) + # Since modifications to the API have been frozen, this approach is adopted for compatibility. + energy = energy * 96 - 96 else: raise ValueError(f'Invalid domain: {domain}') return energy @@ -134,7 +139,8 @@ def get_breathiness( def get_voicing( waveform: Union[np.ndarray, DecomposedWaveform], samplerate, f0, length, - *, hop_size=None, fft_size=None, win_size=None + *, hop_size=None, fft_size=None, win_size=None, + domain='db', mu=255.0 ): """ Definition of voicing: RMS of the harmonic part, in dB representation @@ -145,6 +151,8 @@ def get_voicing( :param hop_size: Frame width, in number of samples :param fft_size: Number of fft bins :param win_size: Window size, in number of samples + :param domain: 'db', 'amplitude', or 'mulaw' + :param mu: mu parameter for mu-law compression :return: voicing """ if not isinstance(waveform, DecomposedWaveform): @@ -155,7 +163,8 @@ def get_voicing( waveform_sp = waveform.harmonic() voicing = get_energy_librosa( waveform_sp, length=length, - hop_size=waveform.hop_size, win_size=waveform.win_size + hop_size=waveform.hop_size, win_size=waveform.win_size, + domain=domain, mu=mu ) return voicing