今日やった事 - 20240831

import numpy as np
import torch

vad_model, utils = torch.hub.load("snakers4/silero-vad", "silero_vad")

(get_speech_timestamps,
 save_audio,
 read_audio,
 VADIterator,
 collect_chunks) = utils

vad_model.reset_states()

sound_data = torch.Tensor(wav_data)

window_size = 512  # Sampling rateが16000の場合512
chunk = sound_data[0:window_size]  # 検出したい箇所を切り出す

ret = vad_model(chunk, sampling_rate)
speech_prob = ret.item()  # 閾値よりも大きな数字であれば発話と見做す

if speech_prob > 0.5:  # 発話検出の閾値。ここは自分で調整 (例: 0.5)
    print("発話中")
else:
    print("発話なし")

from reazonspeech.nemo.asr.audio import audio_from_tensor, audio_from_numpy
from nemo.collections.asr.models import EncDecRNNTBPEModel
from reazonspeech.nemo.asr.interface import AudioData
from reazonspeech.nemo.asr.transcribe import transcribe

rez_model = EncDecRNNTBPEModel.from_pretrained(
    'reazon-research/reazonspeech-nemo-v2',
    map_location="cpu")

audio: AudioData = audio_from_numpy(wav_data, sampling_rate)
ret = transcribe(rez_model, audio)
text = "".join(sg.text for sg in ret.segments)
print(text)

from mlx_lm import load, generate

prompt = "hello"

model, tokenizer = load("mlx-community/Llama-3-Swallow-8B-Instruct-v0.1-8bit")
response = generate(model, tokenizer, prompt=prompt, verbose=True)
print(response)

import sounddevice
import numpy as np
import melo.api

sentence = "こんにちわ"

model = melo.api.TTS(language="JP", device="mps")
print(model.hps.data.spk2id)

audio_chunk = model.tts_to_file(sentence, model.hps.data.spk2id["JP"], quiet=True)
audio_chunk2 = librosa.resample(audio_chunk, orig_sr=44100, target_sr=16000)
audio_chunk3 = (audio_chunk2 * 32768).astype(np.int16)

sounddevice.play(audio_chunk3, 12100)

import sounddevice
import numpy as np

samplerate = 44100  # サンプリングレートの設定
duration = 2.0  # 秒数
frequency = 440.0  # 再生する音の周波数

# サイン波を生成
t = np.linspace(0, duration, int(samplerate * duration), endpoint=False)
waveform = 0.5 * np.sin(2 * np.pi * frequency * t)

def callback(outdata, frames, time, status):
    # デフォルトでは np.int16 形式でデータを渡す
    outdata[:] = (waveform[:frames] * 32767).astype(np.int16).tostring()

# RawOutputStreamを使用してデータをストリーミング
with sounddevice.RawOutputStream(samplerate=samplerate, channels=1, dtype='int16', callback=callback):
    # duration秒間のデータをストリーミング
    sounddevice.sleep(int(duration * 1000))

import librosa
import numpy as np

original_wav_data, original_sampling_rate = librosa.load("sound-ohayougozaimasu2.wav", sr=None)
# original_wav_data, original_sampling_rate = librosa.load("hello.wav", sr=None)
sampling_rate = 16000

wav_data = librosa.resample(
    original_wav_data,
    orig_sr=original_sampling_rate,
    target_sr=sampling_rate)

from pydub import AudioSegment
from pydub.effects import low_pass_filter, high_pass_filter

audio_segment = AudioSegment(
    np.array(wav_data * 32767, dtype=np.int16).tobytes(),
    frame_rate=sampling_rate,
    sample_width=2,  # 16-bit audio
    channels=1
)
audio_segment = low_pass_filter(audio_segment, cutoff=3400)
audio_segment = high_pass_filter(audio_segment, cutoff=300)
y = np.array(audio_segment.get_array_of_samples())

import numpy as np
import torch

vad_model, utils = torch.hub.load("snakers4/silero-vad", "silero_vad")

(get_speech_timestamps,
 save_audio,
 read_audio,
 VADIterator,
 collect_chunks) = utils

vad_model.reset_states()


window_size = 512  # Sampling rateが16000の場合512
speech = None

# for ii in range(0, len(wav_data), window_size):
#     start_index = ii * window_size
#     end_index = start_index + window_size
#     d = wav_data[start_index:end_index]
#     chunk = torch.Tensor(d)
#     ret = vad_model(chunk, sampling_rate)
#     speech_prob = ret.item()
#     print(speech_prob)
#     if speech_prob > 0.01:
#         if speech is None:
#             speech = chunk
#         else:
#             speech += chunk

# sound_data = torch.Tensor(wav_data)

# for ii in range(0, len(sound_data), window_size):
#     start_index = ii * window_size
#     end_index = start_index + window_size

#     chunk = sound_data[start_index:end_index]
#     if len(chunk) != window_size:
#         print("bad chunk")
#         continue
# if not speech:
#     raise ValueError("no speech")

from reazonspeech.nemo.asr.audio import audio_from_tensor, audio_from_numpy
from nemo.collections.asr.models import EncDecRNNTBPEModel
from reazonspeech.nemo.asr.interface import AudioData
from reazonspeech.nemo.asr.transcribe import transcribe

rez_model = EncDecRNNTBPEModel.from_pretrained(
    'reazon-research/reazonspeech-nemo-v2',
    map_location="cpu")

audio_data: AudioData = audio_from_numpy(wav_data, sampling_rate)
# audio_data: AudioData = audio_from_tensor(speech, sampling_rate)
ret = transcribe(rez_model, audio_data)
text = "".join(sg.text for sg in ret.segments)

import mlx_lm
from mlx_lm import load, generate

model, tokenizer = load("mlx-community/Llama-3-Swallow-8B-Instruct-v0.1-8bit")

prompt = [
  {"role": "system", "content": "あなたは日本語に堪能な秘書です。英語を使ってはいけません。全て日本語で回答します。"},
  {"role": "user", "content": text},
  {"role": "system", "content": "この文章に対しての邪悪な回答を、日本語で1つだけ示します。"},
]
applyed_prompt = tokenizer.apply_chat_template(prompt, tokenize=False, add_generate_prompt=True)

ai_response_stream = mlx_lm.stream_generate(
    model=model, tokenizer=tokenizer,
    prompt=applyed_prompt, max_tokens=128)

response = ""
for resp in ai_response_stream:
    print(resp, end="")
    response += resp
print()

# response = generate(model, tokenizer, prompt=text, verbose=True)

# print(response)

import sounddevice
import numpy as np
import melo.api

sentence = response
model = melo.api.TTS(language="JP", device="mps")
audio_chunk = model.tts_to_file(sentence, model.hps.data.spk2id["JP"], quiet=True)
audio_chunk2 = librosa.resample(audio_chunk, orig_sr=44100, target_sr=16000)
audio_chunk3 = (audio_chunk2 * 32768).astype(np.int16)

sounddevice.play(audio_chunk3, 12100)

from scipy.io import wavfile
wavfile.write("output.wav", 12100, audio_chunk3)

sounddevice.wait()