I am currently working on a video calling application for Linux desktops. To establish peer-to-peer connections, I have used aiortc for RTC connections and utilized WebSockets for handling offer, answer, candidate, and message data transactions. Moreover, I have integrated OpenCV for video capture and streaming through aiortc.
However, I am facing an issue where the ICE connection state remains stuck in the "checking" phase and does not progress further.
Please have a look at my code and help me for the issue:
import asyncio
import json
import time
import cv2
import websocket
import threading
from aiortc import (RTCIceCandidate,
RTCPeerConnection,
RTCSessionDescription,
VideoStreamTrack,
RTCConfiguration,
RTCIceServer)
from aiortc.contrib.media import MediaBlackhole
from CameraVideoStreamTrack import CameraVideoStreamTrack
from OpenCVVideoStreamTrack import OpenCVVideoStreamTrack
from VideoReceiver import VideoReceiver
class WebRTCClient:
def __init__(self, signaling_url, main):
self.local_video_track = None
self.Main = main
self.signaling_url = signaling_url
self.ws = None
self.pc = RTCPeerConnection(
configuration=RTCConfiguration(iceServers=[RTCIceServer(urls=['stun:stun.l.google.com:19302'])]))
self.playing = False
self.pc.on('datachannel', self.on_datachannel)
self.pc.on('track', self.on_track)
#self.pc.on("track", lambda track: asyncio.ensure_future(self.on_track(track)))
self.pc.on('iceconnectionstatechange', self.on_iceconnectionstatechange)
self.pc.on("icecandidate",self.on_icecandidate)
self.pc.on("icegatheringstatechange",self.on_icegatheringstatechange)
def connect(self):
self.ws = websocket.WebSocketApp(
self.signaling_url,
on_open=self.ws_on_open,
on_message=self.ws_on_message,
on_error=self.ws_on_error,
on_close=self.ws_on_close)
ws_thread = threading.Thread(target=self.ws.run_forever)
ws_thread.daemon = True
ws_thread.start()
async def create_offer(self):
# Add an audio transceiver with sendrecv direction
self.pc.addTransceiver("audio", direction="sendrecv")
self.pc.addTransceiver("video", direction="sendrecv")
if self.local_video_track is None:
# Capture local video stream
self.local_video_track = OpenCVVideoStreamTrack()
# Create a local video receiver thread
local_video_receiver = VideoReceiver(self.local_video_track)
local_video_receiver.update_signal.connect(lambda img: self.Main.update_image(img, "local"))
local_video_receiver.start()
# Add the local video track to the connection
self.pc.addTrack(self.local_video_track)
offer = await self.pc.createOffer()
print(f'Offer sending: {offer}')
await self.pc.setLocalDescription(offer)
print('Offer sending 2')
self.ws.send(json.dumps(self.pc.localDescription, default=lambda o: o.__dict__, sort_keys=True, indent=4))
print('Offer sent 2')
async def on_track(self, track):
print(f"Track {track.kind} received")
if track.kind == "audio":
print(f"Track {track.kind} received")
#self.pc.addTrack(self.player.audio)
#self.recorder.addTrack(track)
elif track.kind == "video":
print(f"Track {track.kind} received")
while True:
_, frame = await track.recv()
return frame
if not frame:
print(f"Track {track.kind} not received done")
break
# Convert frame to numpy array and update QLabel
img = frame.to_ndarray(format="bgr24")
self.Main.update_image(img, "remote")
print(f"Track {track.kind} received done 2")
print(f"Track {track.kind} received done")
# remote_video = VideoTransformTrack(track)
#
# # Create a remote video receiver thread
# remote_video_receiver = VideoReceiver(remote_video)
# remote_video_receiver.update_signal.connect(lambda img: self.Main.update_image(img, "remote"))
# remote_video_receiver.start()
def on_datachannel(self, channel):
@channel.on("message")
def on_message(message):
print(f"datachannel message: {message}")
if isinstance(message, str) and message.startswith("ping"):
channel.send("pong" + message[4:])
async def on_iceconnectionstatechange(self):
print(f"ICE connection state is {self.pc.iceConnectionState}")
print(f"Connection state is {self.pc.connectionState}")
if self.pc.iceConnectionState == "failed":
await self.pc.close()
async def on_icecandidate(self, candidate):
if candidate:
print("Local ICE candidate:", candidate)
# Send the candidate to the remote peer using your signaling server
jsoncandidate = json.dumps(candidate, default=lambda o: o.__dict__, sort_keys=True, indent=4)
print(f"Local ICE candidate: {candidate}")
self.ws.send(jsoncandidate)
async def on_icegatheringstatechange(self):
print("ICE gathering state changed to:", self.pc.iceGatheringState)
if self.pc.iceGatheringState == "complete":
print("All local ICE candidates have been gathered")
async def add_ice_candidate(self, message):
print(f'Candidate message {message}')
#message = json.loads(candidate)
if message["candidate"]["sdp"] is not None:
parts = message["candidate"]["sdp"].split()
foundation = parts[0].split(':')[1]
component_id = int(parts[1])
protocol = parts[2]
priority = int(parts[3])
ip_address = parts[4]
port = int(parts[5])
candidate_type = parts[7]
candidate = RTCIceCandidate(
foundation=foundation,
component=component_id,
protocol=protocol,
priority=priority,
ip=ip_address,
port=port,
type=candidate_type
)
candidate.sdpMLineIndex = message["candidate"]["sdpMLineIndex"]
candidate.sdpMid = message["candidate"]["sdpMid"]
print("Start Adding Candidate")
await self.pc.addIceCandidate(candidate)
print("End Adding Candidate")
async def close(self):
await self.ws.close()
await self.pc.close()
def ws_disconnect(self):
self.ws.close()
def ws_on_open(self, ws):
print("WebSocket connection opened")
def ws_on_message(self, ws, message):
print(f"Received message: {message}")
msg = self.parse_json(message)
asyncio.run(self.read_message(msg))
def ws_on_error(self, ws, error):
print(f"WebSocket error: {error}")
def ws_on_close(self, ws):
self.pc.close()
print("WebSocket connection closed")
async def read_message(self, message):
if message["type"] and message["type"].lower() == "offer":
print('Handle Offer')
offer = RTCSessionDescription(message["sdp"], "offer")
await self.handle_offer(offer)
elif message["type"] and message["type"].lower() == "answer":
print('Handle Answer')
self.pc.addTransceiver("video", direction="sendrecv")
self.pc.addTransceiver("audio", direction="sendrecv")
# Make sure to convert the remote_answer string to a RTCSessionDescription object
answer = RTCSessionDescription(message["sdp"], message["type"].lower())
print('Handle Answer 1')
await self.pc.setRemoteDescription(answer)
print('Handle Answer Done')
elif message["candidate"]:
await self.add_ice_candidate(message)
def parse_json(self, json_str):
# Parse the JSON string into a dictionary
data = json.loads(json_str)
# Convert all property names to lowercase
data = {key.lower(): value for key, value in data.items()}
return data
async def handle_offer(self, offer):
# handle offer
# Add transceiver
self.pc.addTransceiver("video")
self.pc.addTransceiver("audio")
time.sleep(1)
print("Handling Offer")
await self.pc.setRemoteDescription(offer)
print("Set Remote Description Done")
print("Recoder Started")
# send answer
answer = await self.pc.createAnswer()
print(f"Answer Created: {answer}")
await self.pc.setLocalDescription(answer)
print("Set Local Description Done")
# Send the answer to the other peer
ans = json.dumps(answer, default=lambda o: o.__dict__, sort_keys=True, indent=4)
print(f"answer sent json {ans}")
self.ws.send(ans)
print(f"answer sent {answer}")
class VideoTransformTrack(VideoStreamTrack):
def __init__(self, track):
super().__init__() # don't forget this!
self.track = track
async def recv(self):
return await self.track.recv()
