音频应用与 Transformers Pipelines#

1
2
3
4

from transformers import pipeline

# 一行代码创建音频分类器
classifier = pipeline("audio-classification")

1
2
3
4
5
6
7
8
9

from transformers import pipeline

# 创建分类器（自动下载默认模型）
classifier = pipeline("audio-classification")

# 对音频文件分类
result = classifier("audio_sample.wav")
print(result)
# [{'label': 'speech', 'score': 0.95}, {'label': 'music', 'score': 0.03}, ...]

1
2
3
4
5
6
7
8

# 使用特定模型进行情感识别
classifier = pipeline(
    "audio-classification",
    model="superb/hubert-base-superb-er"  # 情感识别模型
)

result = classifier("speech.wav")
# [{'label': 'hap', 'score': 0.8}, {'label': 'sad', 'score': 0.1}, ...]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

from datasets import load_dataset, Audio

# 加载数据集
dataset = load_dataset("PolyAI/minds14", name="en-US", split="train[:10]")

# 创建分类器
classifier = pipeline(
    "audio-classification",
    model="facebook/wav2vec2-base"
)

# 直接传入音频数组
example = dataset[0]
result = classifier(example["audio"]["array"])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 批量处理音频文件
audio_files = ["audio1.wav", "audio2.wav", "audio3.wav"]
results = classifier(audio_files)

# 或者处理数据集
def classify_batch(batch):
    audio_arrays = [x["array"] for x in batch["audio"]]
    predictions = classifier(audio_arrays)
    batch["predictions"] = predictions
    return batch

classified_dataset = dataset.map(classify_batch, batched=True, batch_size=4)

1
2
3
4
5
6
7
8
9

from transformers import pipeline

# 创建 ASR pipeline（默认使用 Whisper）
transcriber = pipeline("automatic-speech-recognition")

# 转录音频
result = transcriber("speech.wav")
print(result["text"])
# "Hello, how are you today?"

1
2
3
4
5
6
7

# 使用 Whisper 大模型
transcriber = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-large-v3"
)

result = transcriber("speech.wav")

1
2
3
4
5
6
7
8

# 中文语音识别
transcriber = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-large-v3",
    generate_kwargs={"language": "chinese", "task": "transcribe"}
)

result = transcriber("chinese_speech.wav")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 语音翻译（任意语言 → 英语）
translator = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-large-v3",
    generate_kwargs={"task": "translate"}
)

# 中文语音直接输出英文
result = translator("chinese_speech.wav")
print(result["text"])  # English translation

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 获取词级时间戳
transcriber = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-large-v3",
    return_timestamps="word"  # 或 "chunk" 获取句子级
)

result = transcriber("speech.wav")
print(result)
# {
#   'text': 'Hello world',
#   'chunks': [
#     {'text': 'Hello', 'timestamp': (0.0, 0.5)},
#     {'text': 'world', 'timestamp': (0.5, 1.0)}
#   ]
# }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 使用分块处理长音频
transcriber = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-large-v3",
    chunk_length_s=30,      # 每块 30 秒
    stride_length_s=5       # 重叠 5 秒
)

# 处理长音频文件
result = transcriber("long_audio.wav")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

from transformers import pipeline

# 创建零样本分类器
classifier = pipeline(
    "zero-shot-audio-classification",
    model="laion/clap-htsat-unfused"
)

# 自定义候选标签
candidate_labels = ["music", "speech", "noise", "silence"]

result = classifier(
    "audio.wav",
    candidate_labels=candidate_labels
)

print(result)
# [{'label': 'music', 'score': 0.85}, {'label': 'speech', 'score': 0.10}, ...]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 音乐流派分类
candidate_labels = [
    "classical music",
    "rock music",
    "jazz music",
    "electronic music",
    "hip hop music"
]

result = classifier("song.wav", candidate_labels=candidate_labels)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

from transformers import pipeline
import soundfile as sf

# 创建 TTS pipeline
synthesizer = pipeline("text-to-speech", model="microsoft/speecht5_tts")

# 需要说话人嵌入
from datasets import load_dataset
embeddings_dataset = load_dataset(
    "Matthijs/cmu-arctic-xvectors",
    split="validation"
)
speaker_embedding = embeddings_dataset[0]["xvector"]

# 生成语音
speech = synthesizer(
    "Hello, this is a test of text to speech.",
    forward_params={"speaker_embeddings": speaker_embedding}
)

# 保存音频
sf.write("output.wav", speech["audio"], samplerate=speech["sampling_rate"])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

from transformers import pipeline

# 创建 Bark pipeline
synthesizer = pipeline("text-to-speech", model="suno/bark-small")

# Bark 不需要说话人嵌入
speech = synthesizer("Hello! This is Bark speaking.")

# 保存音频
import soundfile as sf
sf.write("bark_output.wav", speech["audio"], samplerate=speech["sampling_rate"])

1
2
3
4
5
6
7
8

# Bark 支持特殊标记控制语音效果
text_with_effects = """
[laughs] That's so funny! [sighs]
Can you believe it? [clears throat]
Let me tell you something important.
"""

speech = synthesizer(text_with_effects)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 使用 GPU
classifier = pipeline(
    "audio-classification",
    model="facebook/wav2vec2-base",
    device=0  # 使用第一个 GPU
)

# 使用 MPS (Apple Silicon)
classifier = pipeline(
    "audio-classification",
    model="facebook/wav2vec2-base",
    device="mps"
)

# 自动选择设备
classifier = pipeline(
    "audio-classification",
    model="facebook/wav2vec2-base",
    device_map="auto"
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 设置批处理大小
transcriber = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-small",
    batch_size=8  # 批处理大小
)

# 批量处理
audio_files = ["file1.wav", "file2.wav", "file3.wav"]
results = transcriber(audio_files)

1
2
3
4
5
6

# 从本地路径加载模型
classifier = pipeline(
    "audio-classification",
    model="./my_local_model",
    tokenizer="./my_local_model"
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

from transformers import pipeline
from datasets import load_dataset, Audio
import soundfile as sf

# 1. 初始化 ASR pipeline
asr = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-small",
    return_timestamps=True
)

# 2. 准备音频数据
def load_audio(file_path, target_sr=16000):
    """加载并重采样音频"""
    import librosa
    audio, sr = librosa.load(file_path, sr=target_sr)
    return audio

# 3. 转录函数
def transcribe_audio(audio_input):
    """
    转录音频，支持文件路径或音频数组
    """
    result = asr(audio_input)
    return {
        "text": result["text"],
        "chunks": result.get("chunks", [])
    }

# 4. 批量处理示例
def batch_transcribe(audio_files):
    """批量转录多个音频文件"""
    results = []
    for file in audio_files:
        try:
            result = transcribe_audio(file)
            results.append({
                "file": file,
                "success": True,
                **result
            })
        except Exception as e:
            results.append({
                "file": file,
                "success": False,
                "error": str(e)
            })
    return results

# 5. 使用示例
if __name__ == "__main__":
    # 单文件转录
    result = transcribe_audio("speech.wav")
    print(f"转录结果: {result['text']}")

    # 带时间戳的输出
    for chunk in result.get("chunks", []):
        start, end = chunk["timestamp"]
        print(f"[{start:.2f}s - {end:.2f}s]: {chunk['text']}")