I've got an issue with implementing WebRTC on RN and would appreciate some help. I have a signaling server set up using socketIO and sometimes manage to get audio to work but it's unreliable and I can't seem to make video work. I'm using supabase db and RT events to advertise new users. Note: this implementation is a bit different because I'm trying to make a matchmaking system. I often get the sdp: called in wrong state: stable error which I don't seem to be able to solve (there are only two test users in the lobby so I don't see why this is being triggered). Also, am getting a new warn today: PeerConnection doesn't exist.
I'll give you access to the various repos. For now, here's the code (I apologise for the length and I can develop further if you comment/dm) :
Client side code (minimal test):
import { useUser } from "@clerk/clerk-expo";
import { io } from "socket.io-client";
import {
RTCPeerConnection,
RTCSessionDescription,
RTCIceCandidate,
mediaDevices,
MediaStream,
} from "react-native-webrtc";
import supabase from "../hooks/initSupabase";
class VoIP {
//socket = io("ws://localhost:3000"); // local testing
socket = io("https://squid-app-mz65h.ondigitalocean.app"); // live / anything not simulator
id = useUser().user.id;
searching = true;
remoteID: string;
remoteCandidates = [];
localCandidates = [];
localDescription: RTCSessionDescription;
peerConnection: RTCPeerConnection;
connectionState: string;
sessionConstraints: RTCOfferOptions = {
offerToReceiveAudio: true,
offerToReceiveVideo: true,
};
mediaConstraints = {
audio: true,
video: {
frameRate: 30,
facingMode: "user",
},
};
localStream: MediaStream;
remoteMediaStream: MediaStream;
listenForDeletes = supabase.channel("matchmaking-deletes").on(
"postgres_changes",
{
event: "DELETE",
schema: "public",
table: "matchmaking",
filter: "id=eq." + this.id,
},
(payload) => {
this.searching = false;
}
);
listenForInserts = supabase.channel("matchmaking-inserts").on(
"postgres_changes",
{
event: "INSERT",
schema: "public",
table: "matchmaking",
},
(payload) => {
console.log(payload.new.id, "entered matchmaking");
if (payload.new.id === this.id) {
return;
} else {
if (
this.peerConnection.signalingState !== "have-remote-offer" ||
this.searching
) {
console.log("handling offer");
this.handleOffer(payload.new);
}
}
}
);
listenForUpdates = supabase.channel("matchmaking-updates").on(
"postgres_changes",
{
event: "UPDATE",
schema: "public",
table: "matchmaking",
},
(payload) => {
console.log(payload.new.id, "updated their entry");
if (payload.new.id === this.id) {
return;
} else {
if (
this.peerConnection.signalingState !== "have-remote-offer" ||
this.searching
) {
this.handleOffer(payload.new);
}
}
}
);
getLocalStream = async () => {
try {
const mediaStream = await mediaDevices.getUserMedia(
this.mediaConstraints
);
return mediaStream;
} catch (err) {
console.error(err);
}
};
createOffer = async () => {
try {
const offerDescription = await this.peerConnection.createOffer(
this.sessionConstraints
);
await this.peerConnection.setLocalDescription(offerDescription);
return this.peerConnection.localDescription;
} catch (err) {
// Handle Errors
console.error(err);
}
};
sendOffer = (offer, tag: string | undefined = undefined) => {
if (this.peerConnection.localDescription == null && offer == null) {
return;
} else {
if (tag === "renegotiate" && this.remoteID) {
this.socket.emit(
"renegotiating_offer",
{
id: this.id, // me
remoteID: this.remoteID, // them
offerDescription: this.peerConnection.localDescription ?? offer,
},
(response) => {
console.log(response);
}
);
} else {
this.socket.emit(
"enter_matchmaking",
{
id: this.id, // me
offerDescription: this.peerConnection.localDescription ?? offer,
},
(response) => {
console.log(response);
}
);
}
}
};
processCandidates = () => {
if (this.remoteCandidates.length < 1) {
return;
}
this.remoteCandidates.map((candidate) =>
this.peerConnection.addIceCandidate(candidate)
);
console.log("Added remote candidates");
this.remoteCandidates = [];
};
handleOffer = async (payload) => {
try {
const offerDescription = new RTCSessionDescription(
payload.offerDescription
);
await this.peerConnection.setRemoteDescription(offerDescription);
const answerDescription = await this.peerConnection.createAnswer();
await this.peerConnection.setLocalDescription(answerDescription);
this.processCandidates();
// Send the answerDescription back as a response to the offerDescription.
this.socket.emit("client_answer", {
id: payload.id, // The ID of the user who sent the offerDescription
remoteID: this.id, // me
answerDescription: answerDescription,
});
} catch (err) {
// Handle Errors
console.error(err);
}
};
handleAnswer = async (payload) => {
try {
const answerDescription = new RTCSessionDescription(
payload.answerDescription
);
await this.peerConnection.setRemoteDescription(answerDescription);
this.processCandidates();
} catch (err) {
// Handle Errors
console.error(err);
}
};
handleIceCandidate = (iceCandidate) => {
iceCandidate = new RTCIceCandidate(iceCandidate);
if (this.peerConnection.remoteDescription == null) {
return this.remoteCandidates.push(iceCandidate);
}
return this.peerConnection.addIceCandidate(iceCandidate);
};
constructor() {
this.peerConnection = new RTCPeerConnection({
iceServers: [
{
urls: "stun:stun.l.google.com:19302",
},
],
});
}
}
export default VoIP;
/*
You're Alice, you want to call someone so you enter matchmaking and send an offer to all users
You're Bob, you want to call someone so you enter matchmaking and send an offer to all users
Alice receives Bob's offer and sends an answer back to Bob
Bob receives Alice's answer and adds it to his peer connection
Alice receives Bob's ice candidate and adds it to her peer connection
Bob receives Alice's ice candidate and adds it to his peer connection
Alice and Bob are now connected
Alice and Bob can now send media streams to each other
*/
import { useState, useEffect } from "react";
import { View, Button } from "react-native";
import { RTCView, MediaStream } from "react-native-webrtc";
import VoIP from "../components/VoIP";
import supabase from "../hooks/initSupabase";
const Admin = () => {
const voip: VoIP = new VoIP();
voip.peerConnection.onicecandidate = (event) => {
if (!event.candidate) {
return; // All ICE candidates have been sent
}
if (!voip.remoteID || !voip.peerConnection.remoteDescription) {
voip.localCandidates.push(event.candidate);
return;
}
if (voip.id && voip.remoteID && event.candidate) {
console.log("Sending ice candidate", event.candidate);
voip.socket.emit("send_ice", {
sender: voip.id,
receiver: voip.remoteID,
ice_candidate: event.candidate,
});
}
};
voip.peerConnection.onconnectionstatechange = (event) => {
voip.connectionState = voip.peerConnection.connectionState;
console.log("Connection state changed to", voip.connectionState);
if (voip.connectionState === "connected") {
supabase.from("matchmaking").delete().match({ id: voip.id });
// send all local candidates to the remote peer
voip.localCandidates.forEach((candidate) => {
if (voip.id && voip.remoteID && candidate) {
voip.socket.emit("send_ice", {
id: voip.id,
remoteID: voip.remoteID,
ice_candidate: candidate,
});
}
});
// consume all stashed remote candidates
voip.remoteCandidates.forEach((candidate) => {
voip.peerConnection.addIceCandidate(candidate);
});
}
};
voip.peerConnection.addEventListener("track", (event) => {
console.log("Got remote track:", event);
if (!voip.remoteMediaStream) {
voip.remoteMediaStream = new MediaStream(undefined);
}
voip.remoteMediaStream.addTrack(event.track);
setUrl(voip.remoteMediaStream?.toURL());
});
voip.peerConnection.addEventListener("negotiationneeded", async () => {
console.log("Negotiation needed");
const offerDescription = await voip.createOffer();
if (offerDescription.type === "offer") {
voip.sendOffer(offerDescription, "renegotiate");
}
});
voip.peerConnection.addEventListener("iceconnectionstatechange", (event) => {
if (voip.peerConnection.iceConnectionState === "disconnected") {
voip.peerConnection.close();
}
if (voip.peerConnection.iceConnectionState === "connected") {
setUrl(voip.remoteMediaStream?.toURL());
supabase.from("matchmaking").delete().match({ id: voip.id });
console.log(url);
}
console.log("ICE connection state changed to", event);
});
voip.socket.on("server_answer", async (payload) => {
if (payload.id === voip.id && voip.searching) {
console.log(`${voip.id}: Got server answer from ${payload.remoteID}`);
voip.remoteID = payload.remoteID;
voip.searching = false;
voip.handleAnswer(payload);
}
});
voip.socket.on("renegotiation", async (payload) => {
if (payload.remoteID === voip.id && voip.searching) {
console.log(`${voip.id}: Got renegotiation from ${payload.id}`);
voip.remoteID = payload.id;
voip.handleOffer(payload.offerDescription);
}
});
voip.socket.on("ice_candidate", (payload) => {
if (payload.receiver === voip.id) {
console.log("Got ice candidate", payload);
voip.handleIceCandidate(payload.ice_candidate);
}
});
voip.getLocalStream().then((stream) => {
voip.localStream = stream;
stream.getTracks().forEach((track) => {
if (track.kind === "audio") {
return;
}
console.log("Adding track to call", track);
voip.peerConnection.addTrack(track, voip.localStream);
});
});
const [url, setUrl] = useState<string | undefined>(undefined);
useEffect(() => {
// always check that the user is receiving the correct data
// offer: {
// "id:" "Alice",
// "offerDescription": ...
// }
// answer: {
// id: "Alice", (set by Alice, read by Bob)
// remoteID: "Bob", (set by Bob, read by Alice)
// answerDescription: ... (intended for Alice)
// }
voip.listenForInserts.subscribe();
voip.listenForUpdates.subscribe();
if (voip.searching) {
console.log("Listening...");
} else {
console.log("Not listening...");
}
return () => {
voip.listenForInserts.unsubscribe();
voip.listenForUpdates.unsubscribe();
};
}, [voip, voip.searching, voip.remoteID, voip.id, url]);
return (
<View
style={{
flex: 1,
justifyContent: "center",
alignItems: "center",
}}
>
<Button
title="Start Call"
onPress={async () => {
const offerDescription = await voip.createOffer();
voip.sendOffer(offerDescription);
voip.searching = true;
}}
/>
{url && (
<>
<RTCView
mirror={true}
objectFit={"cover"}
streamURL={voip.remoteMediaStream?.toURL()}
zOrder={0}
style={{
width: "60%",
height: "60%",
position: "absolute",
backgroundColor: "black",
}}
/>
</>
)}
</View>
);
};
export default Admin;
Server side code
require("dotenv").config();
import { Server } from "socket.io";
import supabase from "./useSupabase";
const express = require("express");
const http = require("http");
const app = express();
const server = http.createServer(app);
const io = new Server(server, {
cors: {
origin: "*",
methods: ["GET", "POST"],
},
});
io.on("open", () => {
console.log("socket.io server open");
});
io.on("connection", (socket) => {
socket.on("enter_matchmaking", async (arg, callback) => {
// more user information here later (socials, username, profile_picture, etc.)
if (!arg.id || arg.id.slice(0, 5) !== "user_") {
callback("invalid id");
return;
}
const { data, error } = await supabase.from("matchmaking").insert([
{
id: arg.id,
offerDescription: arg.offerDescription,
},
]);
if (!error) callback("success");
if (error?.code === "23505") {
// duplicate key error
const { data, error } = await supabase
.from("matchmaking")
.update({ offerDescription: arg.offerDescription })
.eq("id", arg.id);
if (!error) callback("updated entry");
else callback(error);
}
});
/*
an interested peer that is open to handshake sent their answer
it is being sent privately to the initial offer holder
*/
socket.on("client_answer", (arg) => {
console.log("transmiting answer to initial peer", arg);
io.emit("server_answer", {
id: arg.id, // id of the initial sender
remoteID: arg.remoteID, // id of the responder
answerDescription: arg.answerDescription,
});
});
/*
a peer has found a match (they exchanged offers already) and is now sending ice candidates privately to the other peer
*/
socket.on("send_ice", (arg) => {
console.log(arg);
if (!arg.sender || !arg.receiver || !arg.ice_candidate) {
console.log("invalid ice candidate");
return;
} else {
io.emit("ice_candidate", {
sender: arg.sender,
receiver: arg.receiver,
ice_candidate: arg.ice_candidate,
});
console.log("transmitted ice candidate to another peer");
}
});
// sometimes we need to renegotiate the connection so one peer resends an offer and the other peer resends an answer
socket.on("renegotiating_offer", (arg) => {
io.emit("renegotiation", {
id: arg.id,
remoteID: arg.remoteID,
offerDescription: arg.offerDescription,
});
});
});
io.on("listening", () => {
console.log("listening on port 3000");
});
io.on("error", (err) => {
console.log(err);
});
server.listen(3000);
I found that sometimes this issue can be solved by processing remoteCandidates once the connection is done, this didn't work for me personally. I also tried switching from supabase to only sockets.
