[Shanda conference] private chat channel webrtc tools

preface

In the Shanda Conference , We not only need to implement multi person video conferencing , We also need to implement a similar QQ、 WeChat Such instant messaging services . In this one-to-one private chat service , We have added a one-to-one private video call function , One is to increase the diversity of software functions , Second, it also paves the way for the realization of multi person chat , To be familiar with the first WebRTC Operation in actual environment .

logic design

First , We need to design the code logic of private video calls . We were 【 Yamada conference 】WebRTC Underlying peer connections This article introduces the basic WebRTC The process of peer connection . Its establishment is essentially a process of two handshakes ：

• The initiator sends... To the receiver OFFER request , And bring your own SessionDescriptionProtocol （ Later abbreviated as sdp）;
• The receiver receives from the sender sdp , establish ANSWER , Generate sdp Return to the sender ;
• The sender receives... From the receiver sdp after , Set up WebRTC Peer connection .

In the private video chat module , We decided to adopt De centralization Of 、P2P Architecture design , The central server only does Signaling forwarding function , Thus, the insufficient bandwidth of the server itself can be bypassed , It is difficult to support high-resolution pictures .
secondly , Because we added session encryption , The initiator can choose whether this call needs to be encrypted , The receiver needs to know whether the other party has enabled encryption , To prompt the user whether to have an encrypted conversation .
Because our server does not SSL certificate , The data sent by the server and the client are in clear text , This means that we cannot pass keys over untrusted channels . therefore , We use a negotiation algorithm to generate a one-time key , The negotiation process also requires a handshake . Final , We simplify the process , The logical flow of the final connection establishment process is obtained ：

For the convenience of description , We will The initiative be called A , Passive party be called B .

1. A towards B Initiate a session request , Which carries Key agreement Some information needed ;
2. B Receive A Request , According to what you carry Negotiation information Judge A Whether to enable encryption , And reply , If you agree to a conversation and A Encryption enabled , Then continue according to Negotiation information To calculate the Public key And Private key , take Public key Send back to A;
3. A After receiving the reply message , Judge B Agree to session , If agreed, generate OFFER request , carry sdp Send to B , If encryption is enabled , Then get B Returned Public key , Through the algorithm Private key ;
4. B Receive OFFER Ask for something to do with sdp, take sdp Save as Remote descriptor , And create ANSWER Request to get Local descriptor , And send it back to A;
5. A Receive ANSWER , Will be one of the sdp Save as Remote descriptor , Both parties add Ice candidates , Establish peer connection .

private WebRTC Tool class code

// ChatRTC.tsx
import {
    
	AlertOutlined,
	CheckOutlined,
	CloseOutlined,
	ExclamationCircleOutlined,
} from '@ant-design/icons';
import Modal from 'antd/lib/modal';
import {
     globalMessage } from 'Components/GlobalMessage/GlobalMessage';
import {
     EventEmitter } from 'events';
import React from 'react';
import {
     ChatSocket } from 'Utils/ChatSocket/ChatSocket';
import {
    
	CALL_STATUS_ANSWERING,
	CALL_STATUS_CALLING,
	CALL_STATUS_FREE,
	CALL_STATUS_OFFERING,
	ChatWebSocketType,
	DEVICE_TYPE,
	PRIVATE_WEBRTC_ANSWER_TYPE,
	receiverCodecs,
	senderCodecs,
} from 'Utils/Constraints';
import eventBus from 'Utils/EventBus/EventBus';
import {
     getDeviceStream, getMainContent } from 'Utils/Global';
import {
     AUDIO_TYPE, buildPropmt } from 'Utils/Prompt/Prompt';
import {
     setCallStatus, setNowChattingId, setNowWebrtcFriendId } from 'Utils/Store/actions';
import store from 'Utils/Store/store';
import {
     eWindow } from 'Utils/Types';
import {
     setupReceiverTransform, setupSenderTransform } from 'Utils/WebRTC/RtcEncrypt';

interface ChatRtcProps {
    
	socket: ChatSocket;
	myId: number;
}

export class ChatRTC extends EventEmitter {
    
	callAudioPrompt: (() => void)[];
	answerAudioPrompt: (() => void)[];
	socket: ChatSocket;
	myId: number;
	localStream: null | MediaStream;
	remoteStream: null | MediaStream;
	sender?: number;
	receiver?: number;
	peer!: RTCPeerConnection;
	answerModal!: null | {
    
		destroy: () => void;
	};
	offerModal!: null | {
    
		destroy: () => void;
	};
	candidateQueue: Array<any>;
	useSecurity: boolean;
	security: string;

	constructor(props: ChatRtcProps) {
    
		super();
		this.callAudioPrompt = buildPropmt(AUDIO_TYPE.WEBRTC_CALLING, true);
		this.answerAudioPrompt = buildPropmt(AUDIO_TYPE.WEBRTC_ANSWERING, true);

		this.socket = props.socket;
		this.myId = props.myId;
		this.localStream = null;
		this.remoteStream = null;
		this.useSecurity = false;
		this.security = '[]';
		this.candidateQueue = new Array();

		this.socket.on('ON_PRIVATE_WEBRTC_REQUEST', (msg) => {
    
			this.responseCall(msg);
		});

		this.socket.on('ON_PRIVATE_WEBRTC_RESPONSE', ({
     accept, sender, receiver, security }) => {
    
			if (sender === this.sender && receiver === this.receiver) {
    
				this.callAudioPrompt[1]();
				if (this.offerModal) {
    
					this.offerModal.destroy();
					this.offerModal = null;
				}
				if (accept === PRIVATE_WEBRTC_ANSWER_TYPE.ACCEPT) {
    
					this.createOffer(security);
				} else {
    
					switch (accept) {
    
						case PRIVATE_WEBRTC_ANSWER_TYPE.BUSY:
							globalMessage.error({
    
								content: ' The other party is on the phone ',
								duration: 1.5,
							});
							break;
						case PRIVATE_WEBRTC_ANSWER_TYPE.OFFLINE:
							globalMessage.error({
    
								content: ' The caller is not online ',
								duration: 1.5,
							});
							break;
						case PRIVATE_WEBRTC_ANSWER_TYPE.REJECT:
							globalMessage.error({
    
								content: ' The other party rejected your call invitation ',
								duration: 1.5,
							});
							break;
					}
					this.onEnded();
				}
			}
		});

		this.socket.on('ON_PRIVATE_WEBRTC_OFFER', (msg) => {
    
			if (msg.sender === this.sender && msg.receiver === this.receiver)
				this.createAnswer(msg.sdp);
		});

		this.socket.on('ON_PRIVATE_WEBRTC_ANSWER', (msg) => {
    
			this.receiveAnswer(msg.sdp);
		});

		this.socket.on('ON_PRIVATE_WEBRTC_CANDIDATE', (msg) => {
    
			if (msg.sender === this.sender && msg.receiver === this.receiver) {
    
				this.handleCandidate(msg);
			}
		});

		this.socket.on('ON_PRIVATE_WEBRTC_DISCONNECT', (msg) => {
    
			globalMessage.info(' The other party has hung up ');
			this.onHangUp(msg);
		});
	}

	callRemote(targetId: number, myName: string, offerModal: any) {
    
		this.useSecurity = localStorage.getItem('securityPrivateWebrtc') === 'true';
		this.callAudioPrompt[0]();
		store.dispatch(setCallStatus(CALL_STATUS_OFFERING));
		store.dispatch(setNowWebrtcFriendId(targetId));
		this.sender = this.myId;
		this.receiver = targetId;
		let pgArr: Array<string> = [];
		(async () => {
    
			if (this.useSecurity) {
    
				pgArr = await eWindow.ipc.invoke('DIFFIE_HELLMAN');
			}
			this.socket.send({
    
				type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_REQUEST,
				sender: this.myId,
				senderName: myName,
				security: JSON.stringify(pgArr),
				receiver: targetId,
			});
			this.offerModal = offerModal;
		})();
	}

	responseCall(msg: any) {
    
		this.sender = msg.sender;
		this.receiver = this.myId;

		const rejectOffer = (reason: number) => {
    
			this.socket.send({
    
				type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_RESPONSE,
				security: '',
				accept: reason,
				sender: msg.sender,
				receiver: msg.receiver,
			});
		};

		if (store.getState().callStatus === CALL_STATUS_FREE) {
    
			eventBus.emit('GET_PRIVATE_CALLED');
			this.answerAudioPrompt[0]();
			store.dispatch(setNowChattingId(msg.sender));
			const pgArr = JSON.parse(msg.security);
			const useSecurity = pgArr.length === 3;
			this.answerModal = Modal.confirm({
    
				icon: useSecurity ? <AlertOutlined /> : <ExclamationCircleOutlined />,
				title: ' Video call invitation ',
				content: (
					<span>
						 user  {
    msg.senderName}(id: {
    msg.sender}) Send you a video call request , Accept or not ？
						{
    useSecurity ? (
							<span>
								<br />
								 Be careful ： The other party has enabled the private chat video session encryption function , Accepting this session may result in your CPU Occupancy has been greatly increased , Please confirm with the other party and choose whether to accept this session 
							</span>
						) : (
							''
						)}
					</span>
				),
				cancelText: (
					<>
						<CloseOutlined />
						 Refuse to accept 
					</>
				),
				okText: (
					<>
						<CheckOutlined />
						 Agree to the request 
					</>
				),
				onOk: () => {
    
					this.useSecurity = useSecurity;
					if (useSecurity) {
    
						eWindow.ipc
							.invoke('DIFFIE_HELLMAN', pgArr[0], pgArr[1], pgArr[2])
							.then((serverArr) => {
    
								const [privateKey, publicKey] = serverArr;
								this.security = privateKey;
								this.socket.send({
    
									type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_RESPONSE,
									accept: PRIVATE_WEBRTC_ANSWER_TYPE.ACCEPT,
									sender: this.sender,
									receiver: this.receiver,
									security: publicKey,
								});
							});
					} else {
    
						this.socket.send({
    
							type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_RESPONSE,
							accept: PRIVATE_WEBRTC_ANSWER_TYPE.ACCEPT,
							sender: this.sender,
							receiver: this.receiver,
							security: '',
						});
					}

					store.dispatch(setCallStatus(CALL_STATUS_ANSWERING));
					store.dispatch(setNowWebrtcFriendId(msg.sender));
				},
				onCancel: () => {
    
					rejectOffer(PRIVATE_WEBRTC_ANSWER_TYPE.REJECT);
					this.answerModal = null;
					this.sender = undefined;
					this.receiver = undefined;
				},
				afterClose: this.answerAudioPrompt[1],
				centered: true,
				getContainer: getMainContent,
			});
		} else rejectOffer(PRIVATE_WEBRTC_ANSWER_TYPE.BUSY);
	}

	async createOffer(publicKey: string) {
    
		this.peer = this.buildPeer();
		this.localStream = new MediaStream();
		this.localStream.addTrack(
			(await getDeviceStream(DEVICE_TYPE.VIDEO_DEVICE)).getVideoTracks()[0]
		);
		this.localStream.addTrack(
			(await getDeviceStream(DEVICE_TYPE.AUDIO_DEVICE)).getAudioTracks()[0]
		);
		for (const track of this.localStream.getTracks()) {
    
			this.peer.addTrack(track, this.localStream);
		}

		// NOTE:  encryption 
		if (publicKey) {
    
			const privateKey = await eWindow.ipc.invoke('DIFFIE_HELLMAN', publicKey);
			this.security = privateKey;
			this.peer.getSenders().forEach((sender) => {
    
				setupSenderTransform(sender, privateKey);
			});
		} else {
    
			this.peer
				.getTransceivers()
				.find((t) => t.sender.track?.kind === 'video')
				?.setCodecPreferences(senderCodecs);
		}

		this.peer
			.createOffer({
    
				offerToReceiveAudio: true,
				offerToReceiveVideo: true,
			})
			.then((sdp) => {
    
				this.peer.setLocalDescription(sdp);
				this.socket.send({
    
					type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_OFFER,
					sdp: sdp.sdp,
					sender: this.sender,
					receiver: this.receiver,
				});
			});
	}

	async createAnswer(remoteSdp: any) {
    
		this.peer = this.buildPeer();

		this.peer.setRemoteDescription(
			new RTCSessionDescription({
    
				sdp: remoteSdp,
				type: 'offer',
			})
		);

		while (this.candidateQueue.length > 0) {
    
			this.peer.addIceCandidate(this.candidateQueue.shift());
		}

		this.localStream = new MediaStream();
		this.localStream.addTrack(
			(await getDeviceStream(DEVICE_TYPE.VIDEO_DEVICE)).getVideoTracks()[0]
		);
		this.localStream.addTrack(
			(await getDeviceStream(DEVICE_TYPE.AUDIO_DEVICE)).getAudioTracks()[0]
		);
		this.emit('LOCAL_STREAM_READY', this.localStream);
		for (const track of this.localStream.getTracks()) {
    
			this.peer.addTrack(track, this.localStream);
		}

		// NOTE:  encryption 
		if (this.useSecurity) {
    
			this.peer.getSenders().forEach((sender) => {
    
				setupSenderTransform(sender, this.security);
			});
		} else {
    
			this.peer
				.getTransceivers()
				.find((t) => t.sender.track?.kind === 'video')
				?.setCodecPreferences(senderCodecs);
		}

		this.peer
			.createAnswer({
    
				mandatory: {
    
					OfferToReceiveAudio: true,
					OfferToReceiveVideo: true,
				},
			})
			.then((sdp) => {
    
				this.peer.setLocalDescription(sdp);
				this.socket.send({
    
					type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_ANSWER,
					sdp: sdp.sdp,
					sender: this.sender,
					receiver: this.receiver,
				});
				store.dispatch(setCallStatus(CALL_STATUS_CALLING));
			});
	}

	async receiveAnswer(remoteSdp: any) {
    
		this.peer.setRemoteDescription(
			new RTCSessionDescription({
    
				sdp: remoteSdp,
				type: 'answer',
			})
		);
		store.dispatch(setCallStatus(CALL_STATUS_CALLING));
		this.emit('LOCAL_STREAM_READY', this.localStream);
	}

	handleCandidate(data: RTCIceCandidateInit) {
    
		this.candidateQueue = this.candidateQueue || new Array();
		if (data.candidate) {
    
			// NOTE:  Need to wait  signalingState  Turn into  stable  To add a candidate 
			if (this.peer && this.peer.signalingState === 'stable') {
    
				this.peer.addIceCandidate(data);
			} else {
    
				this.candidateQueue.push(data);
			}
		}
	}

	hangUp() {
    
		this.callAudioPrompt[1]();
		this.socket.send({
    
			type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_DISCONNECT,
			sender: this.sender,
			receiver: this.receiver,
			target: store.getState().nowWebrtcFriendId,
		});
		this.offerModal = null;
		this.onEnded();
	}

	onHangUp(data: {
     sender: number; receiver: number }) {
    
		const {
     sender, receiver } = data;
		if (sender === this.sender && receiver === this.receiver) {
    
			if (this.answerModal) {
    
				this.answerAudioPrompt[1]();
				this.answerModal.destroy();
			}
			this.answerModal = null;
			this.onEnded();
		}
	}

	/** *  establish  RTCPeer  Connect  * @returns  After creation  RTCPeer  Connect  */
	private buildPeer(): RTCPeerConnection {
    
		const peer = new (RTCPeerConnection as any)({
    
			iceServers: [
				{
    
					urls: 'stun:stun.stunprotocol.org:3478',
				},
			],
			encodedInsertableStreams: this.useSecurity,
		}) as RTCPeerConnection;
		peer.onicecandidate = (evt) => {
    
			if (evt.candidate) {
    
				const message = {
    
					type: ChatWebSocketType.CHAT_PRIVATE_WEBRTC_CANDIDATE,
					candidate: evt.candidate.candidate,
					sdpMid: evt.candidate.sdpMid,
					sdpMLineIndex: evt.candidate.sdpMLineIndex,
					sender: this.sender,
					receiver: this.receiver,
					target: store.getState().nowWebrtcFriendId,
				};
				this.socket.send(message);
			}
		};
		peer.ontrack = (evt) => {
    
			// NOTE:  Decrypt 
			if (this.useSecurity) setupReceiverTransform(evt.receiver, this.security);
			else
				peer.getTransceivers()
					.find((t) => t.receiver.track.kind === 'video')
					?.setCodecPreferences(receiverCodecs);

			this.remoteStream = this.remoteStream || new MediaStream();
			this.remoteStream.addTrack(evt.track);
			if (this.remoteStream.getTracks().length === 2)
				this.emit('REMOTE_STREAM_READY', this.remoteStream);
		};

		// NOTE:  Disconnection detection 
		peer.oniceconnectionstatechange = () => {
    
			if (peer.iceConnectionState === 'disconnected') {
    
				this.emit('ICE_DISCONNECT');
			}
		};
		peer.onconnectionstatechange = () => {
    
			if (peer.connectionState === 'failed') {
    
				this.emit('RTC_CONNECTION_FAILED');
			}
		};
		return peer;
	}

	changeVideoTrack(newTrack: MediaStreamTrack) {
    
		if (this.localStream && this.peer) {
    
			const oldTrack = this.localStream.getVideoTracks()[0];
			this.localStream.removeTrack(oldTrack);
			this.localStream.addTrack(newTrack);
			this.peer
				.getSenders()
				.find((s) => s.track === oldTrack)
				?.replaceTrack(newTrack);
		}
	}

	/** *  Clear the data after ending the call  */
	onEnded() {
    
		this.sender = undefined;
		this.receiver = undefined;
		this.useSecurity = false;
		this.security = '[]';
		store.dispatch(setNowWebrtcFriendId(null));
		this.localStream = null;
		this.remoteStream = null;
		if (this.peer) this.peer.close();
		this.candidateQueue = new Array();
		store.dispatch(setCallStatus(CALL_STATUS_FREE));
	}
}