Swift - read two audio files and calculate their cross-correlation

Question

(I'm using this as a reading file reference, this as an Objective-C clue that lacks implementation, this as the closest implementation I've seen so far in Objective-C.)

I'm trying to get a cross-correlation array, calculated from two audio arrays in the Swift Playgrounds.

The steps I'm trying to do (and I did it easily in Python, consuming scipy.signal.correlate) are, in this order:

read both audio files (as two Float arrays) and sample rates (Int-like). I select only a single channel from both if they are stereo.
append a zero padding at the end of the shorter array, so both arrays will have the same length. This is a necessary step.
use the Accelerate library and calculate the cross-correlation with a DSP function.
find the sample in the correlation array associated with the highest coefficient (or the "peak" value from the correlation array), and calculate:

sample_index = max_index - (array_length / 2)
delay_in_seconds = sample_index / sample_rate

Down below is what I've worked so far, but I'm stuck in the correlation function. I don't know how to call vDSP_conv or correlate.

Personally, I prefer the vDSP_conv, since it has compatibility with older Apple devices, but achieving a working script is already good. I don't even know how to call vDSP functions passing a mocked array, much less with an array taken from an audio file.

Also, there's a conceptual difference between convolution and correlation, regarding the orientation of the sliding signal. I'm not sure how to handle this option in the vDSP_conv function, I think I should pass the __IF argument as -1.

import AVFoundation
import Accelerate

enum ExampleError: Error {
    case mismatchingSampleRates
}

enum TestError: Error {
    case delayCalculationError
    case zeroPaddingError
}

func main() {
    let time_delay: Float = try! get_delay(filename_1: "metronome", extension_1: ".aif",
                               filename_2: "metronome_100ms_delay", extension_2: ".aiff")
    print(time_delay)
}

func get_delay(filename_1: String, extension_1: String, filename_2: String, extension_2: String) throws -> Float {
    var (audio_1, sr_1) = readAudio(filename: filename_1, file_extension: extension_1)
    var (audio_2, sr_2) = readAudio(filename: filename_2, file_extension: extension_2)
    
    if (sr_1 != sr_2) {
        throw ExampleError.mismatchingSampleRates
    }
    (audio_1, audio_2) = append_zeros_at_shortest(audio_1, audio_2)
    let correlation_array: [Float] = [0, 10, 20, 30, 40, 41, 20, 10, 0] //mock value
    return get_time_at_peak(correlation_array, sample_rate: 1) // mock sample rate
    }

func readAudio(filename: String, file_extension: String) -> ([Float], Int) {
    let audioUrl = Bundle.main.url(forResource: filename, withExtension: file_extension)!
    let audioFile = try! AVAudioFile(forReading: audioUrl)
    let audioFileFormat = audioFile.processingFormat
    let audioFileSize = UInt32(audioFile.length)
    let audioBuffer = AVAudioPCMBuffer(pcmFormat: audioFileFormat, frameCapacity: audioFileSize)!
    try! audioFile.read(into: audioBuffer) //not sure why to call this
    return (Array(UnsafeBufferPointer(start: audioBuffer.floatChannelData![0], count: Int(audioBuffer.frameLength))), Int(audioFile.fileFormat.sampleRate))
}

func append_zeros_at_shortest(_ audio_1: [Float],_ audio_2: [Float]) -> ([Float], [Float]) {
    if audio_1.count == audio_2.count {
        return (audio_1, audio_2)
    }
    if audio_1.count < audio_2.count {
        return (append_zeros(at: audio_1, new_len: audio_2.count), audio_2)
    }
    //if audio_2.count < audio_1.count ...
    return (audio_1, append_zeros(at: audio_2, new_len: audio_1.count))
}

func append_zeros(at old_array: [Float], new_len: Int) -> [Float] {
    var new_array = Array<Float>(repeating: 0, count: new_len)
    new_array[0..<old_array.count] = old_array[0..<old_array.count]
    return new_array
}

func get_time_at_peak(_ array: [Float],sample_rate sr: Int) -> Float {
    var center_index =  Float(array.count / 2)
    if array.count % 2 == 0 {
        center_index -= 0.5
    }
    let max_index = Float(array.indices.first{array[$0] == array.max()}!)
    return (max_index - center_index) / Float(sr)
}


func test_get_time_at_peak() throws -> Bool {
    if get_time_at_peak([0,10, 20, 30, 40, 50, 40, 30, 20, 10, 0], sample_rate: 1) != 0 {
        throw TestError.delayCalculationError
    }
    if get_time_at_peak([0,10, 20, 30, 40, 50, 51, 30, 20, 10, 0], sample_rate: 1) != 1 {
        throw TestError.delayCalculationError
    }
    if get_time_at_peak([0,10, 20, 30, 51, 50, 40, 30, 20, 10, 0], sample_rate: 1) != -1 {
        throw TestError.delayCalculationError
    }
    if get_time_at_peak([0,10, 20, 30, 40, 50, 40, 51, 20, 10, 0], sample_rate: 2) != 1 {
        throw TestError.delayCalculationError
    }
    if get_time_at_peak([0,10, 20, 30, 40, 50, 51, 40, 20, 10, 0], sample_rate: 2) != 0.5 {
        throw TestError.delayCalculationError
    }
    if get_time_at_peak([0, 10, 20, 30, 0, 41, 20, 10], sample_rate: 1) != 1.5 {
        throw TestError.delayCalculationError
    }
    return true
}

func test_append_zeros() throws -> Bool {
    let (v1, v2) = append_zeros_at_shortest([1,2,3], [10,20,30,40,50])
    if v1 != [1,2,3,0,0] {
        throw TestError.zeroPaddingError
    }
    if v2 != [10,20,30,40,50] {
        throw TestError.zeroPaddingError
    }
    return true
}

func call_tests() {
    try! test_get_time_at_peak()
    try! test_append_zeros()
}

call_tests()
main()

Instead of the mocked array, I've tried to achieve the correlation with:

var correlation_array: UnsafeMutablePointer<Float>
vDSP_conv(audio_1, 1, audio_2, 1, correlation_array, 1, vDSP_Length(2 * audio_1.count - 1), vDSP_Length(audio_1.count))

But I got the following error at compile time:

Cannot convert value of type 'UnsafeMutablePointer' to expected argument type '[Float]'

I have also tried to:

let correlation_array: [Float] = vDSP.correlate(audio_1, withKernel: audio_2)

But I got the following error at runtime:

error: Execution was interrupted, reason: EXC_BAD_INSTRUCTION (code=EXC_I386_INVOP, subcode=0x0).

With this, I get an empty array as result:

let correlation_array: [Float] = vDSP.correlate([0,10,20,30,20,10,0], withKernel: [0,0,10,20,30,20,10])

Miguel de Sousa · Answer 1 · 2021-01-13T15:26:24.847

Searching in the Github, I found the implementation below. I've tested it and gave me 98.97ms of delay between my two recordings. That's approximately the delay value according to the Audacity probe (99ms).

Now I can see that I should pass a pointer to the result Float array as an argument.

There's also a zero padding in the implementation below, so both x and y have the same length. After that, x is padded both in the end and the beginning, so it can shifts over the y array, following the theory behind the convolution operation.

// Cross-correlation of a signal [x], with another signal [y]. The signal [y]
// is padded so that it is the same length as [x].
public func xcorr(_ x: [Float], _ y: [Float]) -> [Float] {
    precondition(x.count >= y.count, "Input vector [x] must have at least as many elements as [y]")
    var yPadded = y
    if x.count > y.count {
        let padding = repeatElement(Float(0.0), count: x.count - y.count)
        yPadded = y + padding
    }
    
    let resultSize = x.count + yPadded.count - 1
    var result = [Float](repeating: 0, count: resultSize)
    let xPad = repeatElement(Float(0.0), count: yPadded.count-1)
    let xPadded = xPad + x + xPad
    vDSP_conv(xPadded, 1, yPadded, 1, &result, 1, vDSP_Length(resultSize), vDSP_Length(yPadded.count))
    
    return result
}

Swift - read two audio files and calculate their cross-correlation

1 Answers1