From 37b90354337efb0257bd27da22adf789e8fd71d7 Mon Sep 17 00:00:00 2001 From: Taichi Kato Date: Mon, 1 Jun 2020 12:30:23 +0800 Subject: [PATCH 01/10] Updated CSS to adopt better to mobile screens --- public/css/chat.css | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/public/css/chat.css b/public/css/chat.css index aef5584..0a746cc 100644 --- a/public/css/chat.css +++ b/public/css/chat.css @@ -111,7 +111,9 @@ a { #wrapper { display: flex; - flex-direction: column; + /* flex: 1 1 20em; */ + /* flex-basis: 30%; */ + flex-direction: row; align-items: center; flex-wrap: wrap; justify-content: center; @@ -122,8 +124,8 @@ a { left: 50%; -ms-transform: translate(-50%, -50%); transform: translate(-50%, -50%); - width: 65%; - max-height: 100%; + width: 100%; + max-height: 90vh; max-width: 100%; } @@ -146,7 +148,7 @@ a { } #remote-video:first-child:nth-last-child(1) { /* -or- li:only-child { */ - max-height: 80vh; +width: 80vw; } /* two items */ @@ -415,7 +417,22 @@ button:hover { margin: 0; line-height: 2rem; } - + #wrapper { + display: flex; + flex-direction: row; + align-items: center; + flex-wrap: wrap; + justify-content: center; + padding: 0; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, calc(-50% - 3rem)); + max-height: 90%; + max-width: 100%; + } #remote-video { /* width: 75vw; height: calc((16/9) * 75vw); */ From 003fca03990111c08e8f0c4bf8d76a812cf1dc4e Mon Sep 17 00:00:00 2001 From: Khush Jammu Date: Mon, 1 Jun 2020 12:45:10 +0800 Subject: [PATCH 02/10] add stub for bitrate downscaling --- public/js/chat.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/js/chat.js b/public/js/chat.js index 8634978..49838d9 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -499,6 +499,12 @@ function sendToAllDataChannels(message) { // } // End Fullscreen + +// Downscale own stream +function downscaleStreams() { + // To be implemented +} + // Mute microphone function muteMicrophone() { var audioTrack = null; From f72d77494e2050ffbffc5f007a43018fbb2e3b22 Mon Sep 17 00:00:00 2001 From: Arya Vohra Date: Tue, 2 Jun 2020 10:52:54 +0800 Subject: [PATCH 03/10] Testing bitrate/res downscaling on n>4 --- public/js/chat.js | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/public/js/chat.js b/public/js/chat.js index 49838d9..eb52b4e 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -225,6 +225,15 @@ var VideoChat = { break; } }; + + + // Downscale send resolution and bitrate if num in room > 4 + if (VideoChat.peerConnections.size > 3) { + for (var pc in VideoChat.peerConnections) { + downscaleStream(pc); + } + } + callback(uuid); }; }, @@ -500,9 +509,25 @@ function sendToAllDataChannels(message) { // End Fullscreen -// Downscale own stream -function downscaleStreams() { - // To be implemented +// Downscale single stream +async function downscaleStream(pc) { + height = 480; + rate = 800000; + try { + do { + h = height; + r = rate; + const [sender] = pc.getSenders(); + const ratio = sender.track.getSettings().height / height; + const params = sender.getParameters(); + if (!params.encodings) params.encodings = [{}]; // Firefox workaround! + params.encodings[0].scaleResolutionDownBy = Math.max(ratio, 1); + params.encodings[0].maxBitrate = rate; + await sender.setParameters(params); + } while (h != height); + } catch (e) { + logIt(e); + } } // Mute microphone From 38ead5c785bcdd87cb43338287f3eb31d0a50628 Mon Sep 17 00:00:00 2001 From: Arya Vohra Date: Tue, 2 Jun 2020 11:33:15 +0800 Subject: [PATCH 04/10] Downscaling to 360p for larger calls working --- public/css/chat.css | 2 +- public/js/chat.js | 79 +++++++++++++++++++++++++++------------------ public/landing.html | 15 ++++----- 3 files changed, 55 insertions(+), 41 deletions(-) diff --git a/public/css/chat.css b/public/css/chat.css index cc49ab6..3c80d13 100644 --- a/public/css/chat.css +++ b/public/css/chat.css @@ -145,7 +145,7 @@ a { padding: 10px; } #remote-video:first-child:nth-last-child(1) { -/* -or- li:only-child { */ + /* -or- li:only-child { */ max-height: 80vh; } diff --git a/public/js/chat.js b/public/js/chat.js index eb52b4e..3404d46 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -204,7 +204,9 @@ var VideoChat = { break; case "disconnected": logIt("disconnected - UUID " + uuid); - VideoChat.remoteVideoWrapper.removeChild(document.querySelectorAll(`[uuid="${uuid}"]`)[0]); + VideoChat.remoteVideoWrapper.removeChild( + document.querySelectorAll(`[uuid="${uuid}"]`)[0] + ); VideoChat.connected.delete(uuid); VideoChat.peerConnections.delete(uuid); dataChannel.delete(uuid); @@ -225,15 +227,6 @@ var VideoChat = { break; } }; - - - // Downscale send resolution and bitrate if num in room > 4 - if (VideoChat.peerConnections.size > 3) { - for (var pc in VideoChat.peerConnections) { - downscaleStream(pc); - } - } - callback(uuid); }; }, @@ -370,6 +363,12 @@ var VideoChat = { VideoChat.connected.set(uuid, true); // Hide caption status text captionText.fadeOut(); + // Downscale send resolution and bitrate if num in room > 4 + if (VideoChat.peerConnections.size > 3) { + VideoChat.peerConnections.forEach(function (value, key, map) { + downscaleStream(value); + }); + } // Reposition local video after a second, as there is often a delay // between adding a stream and the height of the video div changing setTimeout(() => rePositionLocalVideo(), 500); @@ -508,16 +507,18 @@ function sendToAllDataChannels(message) { // } // End Fullscreen - // Downscale single stream -async function downscaleStream(pc) { - height = 480; +async function downscaleStream(pc, applying = false) { + height = 240; rate = 800000; + if (applying) return; try { + applying = true; do { h = height; - r = rate; - const [sender] = pc.getSenders(); + const sender = pc.getSenders().find(function (s) { + return s.track.kind === "video"; + }); const ratio = sender.track.getSettings().height / height; const params = sender.getParameters(); if (!params.encodings) params.encodings = [{}]; // Firefox workaround! @@ -527,6 +528,8 @@ async function downscaleStream(pc) { } while (h != height); } catch (e) { logIt(e); + } finally { + applying = false; } } @@ -542,7 +545,7 @@ function muteMicrophone() { }); audioTrack.enabled = VideoChat.audioEnabled; }); - + // select mic button and mic button text const micButtonIcon = document.getElementById("mic-icon"); const micButtonText = document.getElementById("mic-text"); @@ -894,15 +897,16 @@ function uuidToColor(uuid) { // Using uuid to generate random. unique pastel color var hash = 0; for (var i = 0; i < uuid.length; i++) { - hash = uuid.charCodeAt(i) + ((hash << 5) - hash); - hash = hash & hash; + hash = uuid.charCodeAt(i) + ((hash << 5) - hash); + hash = hash & hash; } var hue = Math.abs(hash % 360); - console.log(hue); // Ensure color is not similar to other colors - var availColors = Array.from({length: 18}, (x,i) => i*20); - VideoChat.peerColors.forEach(function(value, key, map) {availColors[Math.floor(value/20)] = null}); - if (availColors[Math.floor(hue/20)] == null) { + var availColors = Array.from({ length: 18 }, (x, i) => i * 20); + VideoChat.peerColors.forEach(function (value, key, map) { + availColors[Math.floor(value / 20)] = null; + }); + if (availColors[Math.floor(hue / 20)] == null) { for (var i = 0; i < availColors.length; i++) { if (availColors[i] != null) { hue = (hue % 20) + availColors[i]; @@ -917,7 +921,9 @@ function uuidToColor(uuid) { // Sets the border color of uuid's stream function setStreamColor(uuid) { const color = uuidToColor(uuid); - document.querySelectorAll(`[uuid="${uuid}"]`)[0].style.border = `3px solid ${color}`; + document.querySelectorAll( + `[uuid="${uuid}"]` + )[0].style.border = `3px solid ${color}`; VideoChat.peerColors[uuid] = color; } @@ -952,18 +958,27 @@ function togglePictureInPicture() { logIt("Error exiting pip."); logIt(error); }); - } else if (VideoChat.remoteVideoWrapper.lastChild.webkitPresentationMode === "inline") { - VideoChat.remoteVideoWrapper.lastChild.webkitSetPresentationMode("picture-in-picture"); } else if ( - VideoChat.remoteVideoWrapper.lastChild.webkitPresentationMode === "picture-in-picture" + VideoChat.remoteVideoWrapper.lastChild.webkitPresentationMode === "inline" ) { - VideoChat.remoteVideoWrapper.lastChild.webkitSetPresentationMode("inline"); + VideoChat.remoteVideoWrapper.lastChild.webkitSetPresentationMode( + "picture-in-picture" + ); + } else if ( + VideoChat.remoteVideoWrapper.lastChild.webkitPresentationMode === + "picture-in-picture" + ) { + VideoChat.remoteVideoWrapper.lastChild.webkitSetPresentationMode( + "inline" + ); } else { - VideoChat.remoteVideoWrapper.lastChild.requestPictureInPicture().catch((error) => { - alert( - "You must be connected to another person to enter picture in picture." - ); - }); + VideoChat.remoteVideoWrapper.lastChild + .requestPictureInPicture() + .catch((error) => { + alert( + "You must be connected to another person to enter picture in picture." + ); + }); } } else { alert( diff --git a/public/landing.html b/public/landing.html index 7e1b62d..693a094 100755 --- a/public/landing.html +++ b/public/landing.html @@ -67,9 +67,9 @@ class="mt-0 mb-32 reveal-from-bottom" data-reveal-delay="300" > - Simple, Secure, and Fast. Peer to peer group video - calling provides quality and latency simply not - available with traditional technology. + Simple, Secure, and Fast. Peer to peer group video calling + provides quality and latency simply not available with + traditional technology.

Decentralized group calls

- Zipcall lets you talk to up to four friends by - directly connecting to them, completely - decentralized. + Zipcall lets you talk to up to four friends by directly + connecting to them, completely decentralized.

@@ -248,8 +247,8 @@

Total Privacy and Security

Zipcall is built privacy first. Each chat is single use, - and end to end state of the art encryption means your calls - are exactly that. Your calls. + and end to end state of the art encryption means your + calls are exactly that. Your calls.

From fb3c28b6aa3b081739f102a4e725cc21c595f4ef Mon Sep 17 00:00:00 2001 From: Khush Jammu Date: Wed, 3 Jun 2020 17:44:05 +0800 Subject: [PATCH 05/10] add reloading for erronous disconnects --- public/js/chat.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/js/chat.js b/public/js/chat.js index 3404d46..4c41278 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -203,6 +203,12 @@ var VideoChat = { logIt("connected"); break; case "disconnected": + // First possibility: we disconnected from the peer + if (VideoChat.socket.connect().connected === false) { + location.reload(); + } + + // Second possibility: the peer disconnected from us logIt("disconnected - UUID " + uuid); VideoChat.remoteVideoWrapper.removeChild( document.querySelectorAll(`[uuid="${uuid}"]`)[0] From 604099e097d71af53415ff716fd984ed1b517a8e Mon Sep 17 00:00:00 2001 From: Khush Jammu Date: Wed, 3 Jun 2020 17:50:45 +0800 Subject: [PATCH 06/10] fix property in disconnect --- public/js/chat.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/chat.js b/public/js/chat.js index 4c41278..87c882e 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -204,7 +204,7 @@ var VideoChat = { break; case "disconnected": // First possibility: we disconnected from the peer - if (VideoChat.socket.connect().connected === false) { + if (VideoChat.socket.connected === false) { location.reload(); } From 04c9f14d084e7d7e9d8b65b6e42668df854afa15 Mon Sep 17 00:00:00 2001 From: Arya Vohra Date: Wed, 3 Jun 2020 17:53:55 +0800 Subject: [PATCH 07/10] Color fix and removed redundant definition of borderColor --- public/js/chat.js | 69 +++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/public/js/chat.js b/public/js/chat.js index 3404d46..7feaebd 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -32,7 +32,6 @@ var dataChannel = new Map(); var VideoChat = { videoEnabled: true, audioEnabled: true, - borderColor: `hsl(${Math.floor(Math.random() * 360)}, 70%, 80%)`, connected: new Map(), localICECandidates: {}, socket: io(), @@ -364,11 +363,11 @@ var VideoChat = { // Hide caption status text captionText.fadeOut(); // Downscale send resolution and bitrate if num in room > 4 - if (VideoChat.peerConnections.size > 3) { - VideoChat.peerConnections.forEach(function (value, key, map) { - downscaleStream(value); - }); - } + // if (VideoChat.peerConnections.size > 3) { + // VideoChat.peerConnections.forEach(function (value, key, map) { + // downscaleStream(value); + // }); + // } // Reposition local video after a second, as there is often a delay // between adding a stream and the height of the video div changing setTimeout(() => rePositionLocalVideo(), 500); @@ -508,30 +507,30 @@ function sendToAllDataChannels(message) { // End Fullscreen // Downscale single stream -async function downscaleStream(pc, applying = false) { - height = 240; - rate = 800000; - if (applying) return; - try { - applying = true; - do { - h = height; - const sender = pc.getSenders().find(function (s) { - return s.track.kind === "video"; - }); - const ratio = sender.track.getSettings().height / height; - const params = sender.getParameters(); - if (!params.encodings) params.encodings = [{}]; // Firefox workaround! - params.encodings[0].scaleResolutionDownBy = Math.max(ratio, 1); - params.encodings[0].maxBitrate = rate; - await sender.setParameters(params); - } while (h != height); - } catch (e) { - logIt(e); - } finally { - applying = false; - } -} +// async function downscaleStream(pc, applying = false) { +// height = 240; +// rate = 800000; +// if (applying) return; +// try { +// applying = true; +// do { +// h = height; +// const sender = pc.getSenders().find(function (s) { +// return s.track.kind === "video"; +// }); +// const ratio = sender.track.getSettings().height / height; +// const params = sender.getParameters(); +// if (!params.encodings) params.encodings = [{}]; // Firefox workaround! +// params.encodings[0].scaleResolutionDownBy = Math.max(ratio, 1); +// params.encodings[0].maxBitrate = rate; +// await sender.setParameters(params); +// } while (h != height); +// } catch (e) { +// logIt(e); +// } finally { +// applying = false; +// } +// } // Mute microphone function muteMicrophone() { @@ -902,20 +901,20 @@ function uuidToColor(uuid) { } var hue = Math.abs(hash % 360); // Ensure color is not similar to other colors - var availColors = Array.from({ length: 18 }, (x, i) => i * 20); + var availColors = Array.from({ length: 9 }, (x, i) => i * 40); VideoChat.peerColors.forEach(function (value, key, map) { - availColors[Math.floor(value / 20)] = null; + availColors[Math.floor(value / 40)] = null; }); - if (availColors[Math.floor(hue / 20)] == null) { + if (availColors[Math.floor(hue / 40)] == null) { for (var i = 0; i < availColors.length; i++) { if (availColors[i] != null) { - hue = (hue % 20) + availColors[i]; + hue = (hue % 40) + availColors[i]; availColors[i] = null; break; } } } - return `hsl(${hue},100%,70%)`; + return `hsl(${hue},100%,60%)`; } // Sets the border color of uuid's stream From 5bddd4463dbdd31ca7e093166b9dcd5537dd601d Mon Sep 17 00:00:00 2001 From: Arya Vohra Date: Thu, 4 Jun 2020 12:49:34 +0800 Subject: [PATCH 08/10] Generating colors at the right rtime --- public/js/chat.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/js/chat.js b/public/js/chat.js index df4812d..0c528af 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -37,10 +37,10 @@ var VideoChat = { socket: io(), remoteVideoWrapper: document.getElementById("wrapper"), localVideo: document.getElementById("local-video"), + peerColors: new Map(), peerConnections: new Map(), recognition: undefined, borderColor: undefined, - peerColors: new Map(), // Call to getUserMedia (provided by adapter.js for cross browser compatibility) // asking for access to both the video and audio streams. If the request is @@ -98,17 +98,17 @@ var VideoChat = { }, }); - VideoChat.borderColor = uuidToColor(VideoChat.socket.id); VideoChat.localVideo.srcObject = stream; - VideoChat.localVideo.style.border = `3px solid ${VideoChat.borderColor}`; // Now we're ready to join the chat room. VideoChat.socket.emit("join", roomHash); + VideoChat.borderColor = uuidToColor(VideoChat.socket.id); + VideoChat.localVideo.style.border = `3px solid ${VideoChat.borderColor}`; // Add listeners to the websocket VideoChat.socket.on("full", chatRoomFull); VideoChat.socket.on("offer", VideoChat.onOffer); - VideoChat.socket.on("willInitiateCall", VideoChat.call); + VideoChat.socket.on("initiateCall", VideoChat.call); // Set up listeners on the socket VideoChat.socket.on("candidate", VideoChat.onCandidate); From 84716b21616f0ec6532b5f4d44c2f93c0b6bf828 Mon Sep 17 00:00:00 2001 From: Arya Vohra Date: Thu, 4 Jun 2020 13:00:28 +0800 Subject: [PATCH 09/10] Change listener name --- server.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server.js b/server.js index de98ade..a82b726 100644 --- a/server.js +++ b/server.js @@ -8,6 +8,7 @@ var twillioAccountSID = var twilio = require("twilio")(twillioAccountSID, twillioAuthToken); var express = require("express"); var app = express(); +const fs = require('fs'); var http = require("http").createServer(app); var io = require("socket.io")(http); var path = require("path"); @@ -80,9 +81,9 @@ io.on("connection", function (socket) { socket.join(room); } else if (numClients < 5) { socket.join(room); - // When the client is not the first to join the room, all clients are ready. - logIt("Broadcasting ready message", room); - socket.broadcast.to(room).emit("willInitiateCall", socket.id, room); + logIt("Broadcasting request to connect with new peer...", room); + // Emits message to ask peers in room to connect with joining peer + socket.broadcast.to(room).emit("initiateCall", socket.id, room); } else { logIt( "room already full with " + numClients + " people in the room.", @@ -132,7 +133,7 @@ io.on("connection", function (socket) { }); // Listen for Heroku port, otherwise just use 3000 -var port = process.env.PORT || 3000; +var port = process.env.PORT || 443; http.listen(port, function () { console.log("http://localhost:" + port); }); From 9ccb1b138ea28784c6193656f48889576ac675fa Mon Sep 17 00:00:00 2001 From: Khush Jammu Date: Thu, 4 Jun 2020 15:41:28 +0800 Subject: [PATCH 10/10] overhaul disconnect handling --- public/js/chat.js | 46 +++++++++++++++++++++++++++++----------------- server.js | 5 +++++ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/public/js/chat.js b/public/js/chat.js index 0c528af..b3b8224 100644 --- a/public/js/chat.js +++ b/public/js/chat.js @@ -106,11 +106,10 @@ var VideoChat = { VideoChat.localVideo.style.border = `3px solid ${VideoChat.borderColor}`; // Add listeners to the websocket + VideoChat.socket.on("leave", VideoChat.onLeave); VideoChat.socket.on("full", chatRoomFull); VideoChat.socket.on("offer", VideoChat.onOffer); VideoChat.socket.on("initiateCall", VideoChat.call); - - // Set up listeners on the socket VideoChat.socket.on("candidate", VideoChat.onCandidate); VideoChat.socket.on("answer", VideoChat.onAnswer); VideoChat.socket.on("requestToggleCaptions", () => toggleSendCaptions()); @@ -119,6 +118,7 @@ var VideoChat = { ); }, + // Initiate a call with newly joined peer call: function (uuid, room) { logIt(`call >>> Initiating call with ${uuid}...`); VideoChat.socket.on( @@ -130,6 +130,24 @@ var VideoChat = { VideoChat.socket.emit("token", roomHash, uuid); }, + // Handle a peer leaving the room + onLeave: function(uuid) { + logIt("disconnected - UUID " + uuid); + // Remove video element + VideoChat.remoteVideoWrapper.removeChild( + document.querySelectorAll(`[uuid="${uuid}"]`)[0] + ); + + // Delete connection & metadata + VideoChat.connected.delete(uuid); + VideoChat.peerConnections.delete(uuid); + dataChannel.delete(uuid); + + if (VideoChat.peerConnections.size === 0) { + displayWaitingCaption(); + } + }, + establishConnection: function (correctUuid, callback) { return function (token, uuid) { if (correctUuid != uuid) { @@ -202,23 +220,12 @@ var VideoChat = { logIt("connected"); break; case "disconnected": - // First possibility: we disconnected from the peer - if (VideoChat.socket.connected === false) { + // Remove UUID if connection to server is intact + if (VideoChat.socket.connected) { + VideoChat.onLeave(uuid); + } else { location.reload(); } - - // Second possibility: the peer disconnected from us - logIt("disconnected - UUID " + uuid); - VideoChat.remoteVideoWrapper.removeChild( - document.querySelectorAll(`[uuid="${uuid}"]`)[0] - ); - VideoChat.connected.delete(uuid); - VideoChat.peerConnections.delete(uuid); - dataChannel.delete(uuid); - - if (VideoChat.peerConnections.size === 0) { - displayWaitingCaption(); - } break; case "failed": logIt("failed"); @@ -1001,6 +1008,11 @@ function displayWaitingCaption() { rePositionCaptions(); } +window.onbeforeunload = function () { + VideoChat.socket.emit("leave", roomHash); + return null; +}; + function startUp() { // Try and detect in-app browsers and redirect var ua = navigator.userAgent || navigator.vendor || window.opera; diff --git a/server.js b/server.js index a82b726..84c4d00 100644 --- a/server.js +++ b/server.js @@ -93,6 +93,11 @@ io.on("connection", function (socket) { } }); + socket.on("leave", function (room) { + logIt("A client has left the room", room); + socket.broadcast.to(room).emit("leave", socket.id); + }); + // When receiving the token message, use the Twilio REST API to request an // token to get ephemeral credentials to use the TURN server. socket.on("token", function (room, uuid) {