Voice detection and recording with PyAudio

Question

I am using OpenSesame (an experiment builder using Python) to collect participants' voice response to specific stimuli using PyAudio.

What I want to achieve is the following :

1. Start recording sound for X seconds (timeout).
2. Compute "loudness" of sound and compare to a pre-defined threshold.
3. Get the time when the sound reaches the threshold if it does.
4. Save the sound in a .wav file when recording is done.

It's working quite well except that the sound recorded stops when the threshold is reached.

How could I continue recording the sound (and compute the loudness) when the threshold is reached (until timeout, for example)?

The code was adapted from [here][1]. Here is my code:

import pyaudio
import struct
import math 
import wave

timeout = 5000
sound_threshold = 0.001

CHUNK = 1024
SHORT_NORMALIZE = (1.0/32768.0)
FORMAT = pyaudio.paInt16
CHANNELS = 2
RATE = 44100
RECORD_SECONDS = 5
WAVE_OUTPUT_FILENAME = "output.wav"

p = pyaudio.PyAudio()

def get_rms(block):

    """Get root mean square as a measure of loudness"""

    count = len(block)/2
    format = "%dh" % (count)
    shorts = struct.unpack( format, block )
    sum_squares = 0.0
    for sample in shorts:
        n = sample * SHORT_NORMALIZE
        sum_squares += n*n
    return math.sqrt( sum_squares / count )

stream = p.open(format=FORMAT,
            channels=CHANNELS,
            rate=RATE,
            input=True,
            frames_per_buffer=CHUNK)

print("* recording")
frames = []

start_time = clock.time()
while True:
    if clock.time() - start_time >= timeout:
        var.response_time = timeout
        var.response = u'timeout'
        var.loudness = None
        var.in_clock_time = clock.time()
        var.start = start_time
        break   
    try:
        block = stream.read(CHUNK)
        frames.append(block)
    except IOError as e:
        print(e)
    loudness = get_rms(block)
    print(loudness)
    if loudness > sound_threshold:
        var.response_time = clock.time() - start_time
        var.response = u'detected'
        var.in_clock_time = clock.time()
        var.start = start_time
        var.loudness = loudness
        break
        
print(response)
print(response_time)
print("* done recording")

stream.stop_stream()
stream.close()
p.terminate()

wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
wf.setnchannels(CHANNELS)
wf.setsampwidth(p.get_sample_size(FORMAT))
wf.setframerate(RATE)
wf.writeframes(b''.join(frames))
wf.close()
´´´

  [1]: https://forum.cogsci.nl/discussion/1772/detecting-voice-onsets-voicekey

Answer 1

i understand you are just trying to start recording time in seconds and save then output in .wav

try:

recorder.py

import pyaudio
import wave

# Record in chunks of 1024 samples
chunk = 1024 

# 16 bits per sample
sample_format = pyaudio.paInt16 
chanels = 2

# Record at 44400 samples per second
smpl_rt = 44400 
seconds = 15 # seconds its gonna be recording
filename = "output.wav" # file its gonna be saved

# Create an interface to PortAudio
pa = pyaudio.PyAudio() 

stream = pa.open(format=sample_format, channels=chanels,
                rate=smpl_rt, input=True,
                frames_per_buffer=chunk)

print('recording started')

# Initialize array that be used for storing frames
frames = [] 

# Store data in chunks by seconds
for i in range(0, int(smpl_rt / chunk * seconds)):
    data = stream.read(chunk)
    frames.append(data)

# Stop and close the stream
stream.stop_stream()
stream.close()

# Terminate - PortAudio interface
pa.terminate()

print('recording ended')

# Save the recorded data in a .wav format
sf = wave.open(filename, 'wb')
sf.setnchannels(chanels)
sf.setsampwidth(pa.get_sample_size(sample_format))
sf.setframerate(smpl_rt)
sf.writeframes(b''.join(frames))
sf.close()

voice recognize.py (if you want to recognize voice)

import speech_recognition as sr

r = sr.Recognizer() # initialize recognizer
with sr.AudioFile('output.wav') as source:
    audio = r.record(source) # process audio
    try:
        text = r.recognize_google(audio, language='en-EN') # language is gonna recognize
        print('Transcription: {}'.format(text))
    except:
        print('Could not recognize voice, please try again.')

that works for me

Answer 2

Here is how I solved the issue (even though not sure it's the most optimal way to do it) :

1. I started a timer in the While loop until timeout is reached

2. I created a list gathering the loudness and clock.time at each iteration

3. I extracted the first time at which the loudness threshold is reached with :

   var.response_time = (df.loc[df['loudness'] > 
   sound_threshold,'clock_time'].iloc[0])- start_time
   

4. I used Try/Except to deal with situations when timeout is reached without response

Here is the full code:

import pyaudio
import struct
import math 
import wave
import pandas as pd

timeout = 5000
sound_threshold = 0.001

FORMAT = pyaudio.paInt16
SHORT_NORMALIZE = (1.0/32768.0)
CHANNELS = 2
RATE = 44100
INPUT_BLOCK_TIME = 0.01
INPUT_FRAMES_PER_BLOCK = int(RATE*INPUT_BLOCK_TIME)
filename = "output.wav"
chunk=1024

p = pyaudio.PyAudio()
def get_rms(block):
"""Get root mean square as a measure of loudness"""
   count = len(block)/2
   format = "%dh" % (count)
   shorts = struct.unpack( format, block )
   sum_squares = 0.0
   for sample in shorts:
       n = sample * SHORT_NORMALIZE
       sum_squares += n*n
   return math.sqrt( sum_squares / count )

# Open the mic
stream = p.open(format=FORMAT,channels=CHANNELS,
               rate=RATE, input=True, input_device_index=0, 
               frames_per_buffer=INPUT_FRAMES_PER_BLOCK)

# Listen for sounds until a sound is detected or a timeout occurs.
print("* recording")
frames = []
list_ldn = []
list_ct = []

# Start a timer until timeout and compute loudness/clocktime for each block - append to lists
start_time = clock.time()
while clock.time() - start_time <= timeout:
    try:
       block = stream.read(chunk)
       frames.append(block)
    except IOError as e:
       print(e)
   loudness = get_rms(block)
   list_ldn.append(loudness)
   list_ct.append(clock.time())

# Close the audio stream
stream.stop_stream()
stream.close()
p.terminate()

wf = wave.open(filename, 'wb')
wf.setnchannels(CHANNELS)
wf.setsampwidth(p.get_sample_size(FORMAT))
wf.setframerate(RATE)
wf.writeframes(b''.join(frames))
wf.close()

# merge the 2 lists to a df and compute response_time & response
df = pd.DataFrame({'clock_time':list_ct,'loudness':list_ldn})
   try:
     var.response_time = (df.loc[df['loudness'] > sound_threshold, 
     'clock_time'].iloc[0]) - start_time
     var.response = u'detected'
# use except to deal with errors when no response is detected
   except IndexError:
     var.response_time = timeout
     var.response = u'timeout'