Twilio TwiML Connect bidirectional synchronous stream

Question

I am trying to make an outgoing call from my Twilio account/number to a PSTN mobile number. I am using this TwiML: <Response> <Connect><Stream url="{ngrok_url}"/></Connect><Response>
With this I am able to recieve and play incoming audio from PSTN mobile phone however I am unable to send outgoing audio to PSTN mobile phone.

Here is the main websocket function which handles incoming audio and tries to send outgoing packet in response via function "sendremotemediatotwilio":

    async def websocket_rpc_endpoint(ws: WebSocket):
        # A lot of messages will be sent rapidly. We'll stop showing after the first one.
        global output_queue
        inbuffer = bytearray(b'')
        inbound_chunks_started = False
        latest_inbound_timestamp = 0
        BUFFER_SIZE = frames_per_buffer
        await ws.accept()
        while True:
            data = await ws.receive_json()
               
            # Using the event type you can determine what type of message you are receiving
            if data['event'] == "connected":
                print("Connected Message received: {}".format(data))
            if data['event'] == "start":
                print("Start Message received: {}".format(data))
            if data['event'] == "media":
                # print("Media message: {}".format(data))
                media = data['media']
                chunk = base64.b64decode(media['payload'])
                # print("Payload is: {}".format(payload))
                if media['track'] == 'inbound':
                    # fills in silence if there have been dropped packets
                    if inbound_chunks_started:
                        if latest_inbound_timestamp + 20 < int(media['timestamp']):
                            bytes_to_fill = 8 * (int(media['timestamp']) - (latest_inbound_timestamp + 20))
                            # NOTE: 0xff is silence for mulaw audio
                            # and there are 8 bytes per ms of data for our format (8 bit, 8000 Hz)
                            inbuffer.extend(b'\xff' * bytes_to_fill)
                    else:
                        # make it known that inbound chunks have started arriving
                        inbound_chunks_started = True
                        latest_inbound_timestamp = int(media['timestamp'])
                    latest_inbound_timestamp = int(media['timestamp'])
                    # extend the inbound audio buffer with data
                    inbuffer.extend(chunk)
                while len(inbuffer) >= BUFFER_SIZE:
                    asinbound = AudioSegment(inbuffer[:BUFFER_SIZE], sample_width=1, frame_rate=8000, channels=1)
                # # print("That's {} bytes".format(len(chunk)))
                # print("That's {} bytes".format(len(chunk)))
                # print("Additional media messages from WebSocket are being suppressed....")
                    output_queue.put_nowait(base64.b64encode(asinbound.raw_data))
                    inbuffer = inbuffer[BUFFER_SIZE:]
                await sendremotemediatotwilio(ws,media,data['streamSid'])
                # print("Added data in queue")
            if data['event'] == "closed":
                print("Closed Message received: {}".format(data))
                break
    async def sendremotemediatotwilio(ws,media,streamSid):
        global input_queue
        # if input_queue.empty():
        #     await asyncio.sleep(1)
        if not input_queue.empty():
            base64_data = input_queue.get_nowait()
            media_data = {
                "event": "media",
                "streamSid": streamSid,
                "media": {
                    "payload": base64.b64encode(base64_data).decode('utf-8')
                }
            }
            # media = json.dumps(media_data)
            # print(f"media: {media}")
            print("sending json")
            await ws.send_json(media_data)
        else:
            # print("Queue is empty")
            pass

What do you mean that you are unable to send outgoing audio? Is there an error? — philnash, May 16 '22 at 22:52
Thank you for replying. I meant that at PSTN mobile side I could not hear packets sent by this program. Now I have resolved this issue — SAURABH PANDEY, May 18 '22 at 05:01

score 1 · Answer 1 · answered May 18 '22 at 05:06

I have resolved this issue. Now I can successfully send and recieve packets to and from PSTN mobile phones. The issue was in transcoding from pcmu to linear pcm. Once it was resolved the same code is working OK

@app.websocket("/twiliomedia")
async def websocket_rpc_endpoint(ws: WebSocket):
    # A lot of messages will be sent rapidly. We'll stop showing after the first one.
    global output_queue,pushdatainqueue
    inbuffer = bytearray(b'')
    inbound_chunks_started = False
    latest_inbound_timestamp = 0
    BUFFER_SIZE = frames_per_buffer
    await ws.accept()
    while True:
        data = await ws.receive_json()
        # print("Json data",data)
        # if message is None:
        #     print("No message received...")
        #     continue
        #
        # # Messages are a JSON encoded string
        # data = json.loads(message)

        # Using the event type you can determine what type of message you are receiving
        if data['event'] == "connected":
            print("Connected Message received: {}".format(data))
        if data['event'] == "start":
            print("Start Message received: {}".format(data))
        if data['event'] == "media":
            # print("Media message: {}".format(data))
            media = data['media']
            chunk = base64.b64decode(media['payload'])
            # print("Payload is: {}".format(payload))
            if media['track'] == 'inbound':
                # fills in silence if there have been dropped packets
                if inbound_chunks_started:
                    if latest_inbound_timestamp + 20 < int(media['timestamp']):
                        bytes_to_fill = 8 * (int(media['timestamp']) - (latest_inbound_timestamp + 20))
                        # NOTE: 0xff is silence for mulaw audio
                        # and there are 8 bytes per ms of data for our format (8 bit, 8000 Hz)
                        inbuffer.extend(b'\xff' * bytes_to_fill)
                else:
                    # make it known that inbound chunks have started arriving
                    inbound_chunks_started = True
                    pushdatainqueue = True
                    latest_inbound_timestamp = int(media['timestamp'])
                latest_inbound_timestamp = int(media['timestamp'])
                # extend the inbound audio buffer with data
                inbuffer.extend(chunk)
            while len(inbuffer) >= BUFFER_SIZE:
                asinbound = AudioSegment(inbuffer[:BUFFER_SIZE], sample_width=1, frame_rate=8000, channels=1)
                # # print("That's {} bytes".format(len(chunk)))
                # print("That's {} bytes".format(len(chunk)))
                # print("Additional media messages from WebSocket are being suppressed....")
                output_queue.put_nowait(base64.b64encode(asinbound.raw_data))
                inbuffer = inbuffer[BUFFER_SIZE:]
            await sendremotemediatotwilio(ws, media, data['streamSid'])
            # print("Added data in queue")
        if data['event'] == "closed":
            print("Closed Message received: {}".format(data))
            break


async def sendremotemediatotwilio(ws, media, streamSid):
    global input_queue
    # if input_queue.empty():
    #     await asyncio.sleep(1)
    if not input_queue.empty():
        base64_data = input_queue.get_nowait()
        media_data = {
            "event": "media",
            "streamSid": streamSid,
            "media": {
                "payload": base64.b64encode(base64_data).decode("utf-8")
                # "payload": base64_data.decode("utf-8")
            }
        }
        # media = json.dumps(media_data)
        # print(f"media: {media}")
        print("sending json")
        await ws.send_json(media_data)
    else:
        # print("Queue is empty")
        pass

Twilio TwiML Connect bidirectional synchronous stream

1 Answers1