音频数据处理基础#

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

from datasets import load_dataset, Audio

# 加载数据集并设置采样率
dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")
dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))

# 查看音频数据
example = dataset[0]["audio"]
print(f"采样率: {example['sampling_rate']} Hz")
print(f"音频数组形状: {example['array'].shape}")
print(f"时长: {len(example['array']) / example['sampling_rate']:.2f} 秒")

1
2
3
4
5

{
    'path': '/path/to/audio.wav',
    'array': array([0.001, -0.002, ...]),  # 波形数据
    'sampling_rate': 16000                  # 采样率
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

import numpy as np
import matplotlib.pyplot as plt

# 对音频信号进行 FFT
audio_array = example['array']
fft_result = np.fft.fft(audio_array)
frequencies = np.fft.fftfreq(len(audio_array), 1/16000)

# 只取正频率部分
positive_freq_idx = frequencies > 0
plt.plot(frequencies[positive_freq_idx],
         np.abs(fft_result)[positive_freq_idx])
plt.xlabel('Frequency (Hz)')
plt.ylabel('Magnitude')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

import librosa
import librosa.display

# 计算 STFT
stft = librosa.stft(audio_array,
                    n_fft=1024,      # FFT 窗口大小
                    hop_length=512)  # 帧移

# 转为分贝刻度
stft_db = librosa.amplitude_to_db(np.abs(stft))

# 可视化频谱图
librosa.display.specshow(stft_db,
                         sr=16000,
                         hop_length=512,
                         x_axis='time',
                         y_axis='hz')
plt.colorbar(format='%+2.0f dB')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 计算梅尔频谱
mel_spectrogram = librosa.feature.melspectrogram(
    y=audio_array,
    sr=16000,
    n_mels=80,        # 梅尔滤波器数量
    n_fft=1024,
    hop_length=512
)

# 转为对数刻度（log-mel spectrogram）
log_mel = librosa.power_to_db(mel_spectrogram)

librosa.display.specshow(log_mel,
                         sr=16000,
                         x_axis='time',
                         y_axis='mel')

1
2
3
4
5
6
7
8
9

# 计算 MFCC
mfcc = librosa.feature.mfcc(
    y=audio_array,
    sr=16000,
    n_mfcc=13,     # MFCC 系数数量
    n_mels=80
)

print(f"MFCC 形状: {mfcc.shape}")  # (n_mfcc, time_steps)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

from transformers import AutoFeatureExtractor

# 加载预训练模型的特征提取器
feature_extractor = AutoFeatureExtractor.from_pretrained(
    "facebook/wav2vec2-base"
)

# 查看预期的采样率
print(f"期望采样率: {feature_extractor.sampling_rate}")

# 处理音频
inputs = feature_extractor(
    example['array'],
    sampling_rate=16000,
    return_tensors="pt"
)

print(f"输入形状: {inputs.input_values.shape}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

from datasets import Audio

# 方法 1：使用 datasets 自动重采样
dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))

# 方法 2：使用 librosa 手动重采样
import librosa
audio_resampled = librosa.resample(
    audio_array,
    orig_sr=44100,
    target_sr=16000
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 批量处理时自动填充
inputs = feature_extractor(
    [audio1, audio2, audio3],
    sampling_rate=16000,
    padding=True,          # 填充到最长
    max_length=16000 * 30, # 最大长度（30秒）
    truncation=True,       # 超长截断
    return_tensors="pt"
)

# 返回 attention_mask 标记有效位置
print(f"attention_mask 形状: {inputs.attention_mask.shape}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

import numpy as np

def add_noise(audio, noise_level=0.005):
    """添加高斯噪声"""
    noise = np.random.randn(len(audio)) * noise_level
    return audio + noise

def time_shift(audio, shift_max=0.1):
    """时间平移"""
    shift = int(len(audio) * np.random.uniform(-shift_max, shift_max))
    return np.roll(audio, shift)

def speed_change(audio, sr, rate_range=(0.9, 1.1)):
    """变速（不改变音调）"""
    rate = np.random.uniform(*rate_range)
    return librosa.effects.time_stretch(audio, rate=rate)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

from datasets import load_dataset, Audio
from transformers import AutoFeatureExtractor

# 1. 加载数据集
dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")

# 2. 加载特征提取器
feature_extractor = AutoFeatureExtractor.from_pretrained(
    "openai/whisper-small"
)

# 3. 统一采样率
dataset = dataset.cast_column(
    "audio",
    Audio(sampling_rate=feature_extractor.sampling_rate)
)

# 4. 定义预处理函数
def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]

    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=feature_extractor.sampling_rate,
        padding=True,
        return_tensors="pt"
    )

    return inputs

# 5. 批量应用预处理
processed_dataset = dataset.map(
    preprocess_function,
    batched=True,
    batch_size=8,
    remove_columns=dataset.column_names
)