import * as matrixcs from 'matrix-js-sdk'; import cytoscape from 'cytoscape'; function captureRumIssue(message, severity = 'ERROR', extra = {}) { const rum = globalThis.OtelRUM?.last; if (!rum || typeof rum.captureMessage !== 'function') return; try { rum.captureMessage(message, severity, extra); } catch (err) { console.warn('RUM capture failed', err); } } window.addEventListener('error', (event) => { captureRumIssue(event.message || 'Unhandled browser error', 'ERROR', { source: event.filename, line: event.lineno, column: event.colno, }); }); window.addEventListener('unhandledrejection', (event) => { const reason = event.reason; captureRumIssue(`Unhandled promise rejection: ${reason?.message || reason || 'unknown error'}`, 'ERROR', { name: reason?.name, stack: reason?.stack, }); }); // --------------------------------------------------------------------------- // state // --------------------------------------------------------------------------- const ROOM_UNITS = 100; // spatial plane is 0..100 x 0..100 const SPATIAL_SCALE = 0.12; // room units -> audio world units const FALLBACK_ICE = { iceServers: [{ urls: ['stun:turn.k6n.net:3478', 'stun:stun.l.google.com:19302'] }] }; const LS_NAME = 'voicepoc.nick'; const LS_ICE = 'voicepoc.iceOverride'; const LS_AUDIO_INPUT = 'voicepoc.audioInput'; const S = { ws: null, myId: null, room: null, name: localStorage.getItem(LS_NAME) || '', peers: new Map(), // id -> peer record micStream: null, screenStreams: new Map(), // streamId -> local MediaStream screenSenders: new Map(), // peerId -> Map(streamId -> RTCRtpSender[]) audioCtx: null, me: { x: 50, y: 50 }, muted: false, deafened: false, iceServers: null, // resolved on join sharing: false, connecting: false, pingTimer: null, reconnect: { room: null, attempt: 0, timer: null, countdownTimer: null }, channelRosters: {}, // roomId -> [snapshot] }; window.__dbg = S; // POC debug handle const $ = (sel) => document.querySelector(sel); const els = { voiceView: $('#voice-view'), stage: $('#stage'), screens: $('#screens'), spatial: $('#spatial'), status: $('#voice-status'), statusDot: $('#status-dot'), statusRow: $('#status-row'), nick: $('#nick'), meAvatar: $('#me-avatar'), mic: $('#btn-mic'), deafen: $('#btn-deafen'), reconnectBanner: $('#reconnect-banner'), reconnectText: $('#reconnect-text'), mxPanel: $('#mx-panel'), mxMessages: $('#mx-messages'), mxInput: $('#mx-input'), welcomeBackdrop: $('#welcome-backdrop'), welcomeNick: $('#welcome-nick'), welcomeNickError: $('#welcome-nick-error'), sidebar: $('#sidebar'), sidebarBackdrop: $('#sidebar-backdrop'), btnMenu: $('#btn-menu'), }; // buttons that appear twice (sidebar + mobile bottom bar) — keep both in sync const micButtons = [...document.querySelectorAll('#btn-mic, .mirror-mic')]; const deafenButtons = [...document.querySelectorAll('#btn-deafen, .mirror-deafen')]; const shareButtons = [...document.querySelectorAll('#btn-share, .mirror-share')]; const statusTextEls = [...document.querySelectorAll('#voice-status, #mobile-status-text')]; const statusDotEls = [...document.querySelectorAll('#status-dot, #mobile-status-dot')]; function mIcon(name, extraClass = '') { const span = document.createElement('span'); span.className = `material-symbols-outlined${extraClass ? ` ${extraClass}` : ''}`; span.textContent = name; return span; } // --------------------------------------------------------------------------- // helpers // --------------------------------------------------------------------------- function toast(msg, kind = '') { const t = document.createElement('div'); t.className = `toast ${kind}`; t.textContent = msg; $('#toasts').appendChild(t); setTimeout(() => t.remove(), 5000); } const ICE_NOTICE_INTERVAL = 15000; function notifyIceIssue(peer, message, extra = {}) { const now = Date.now(); const key = `${message}:${extra.url || ''}`; if (!peer || peer.lastIceNotice?.key !== key || now - peer.lastIceNotice.at > ICE_NOTICE_INTERVAL) { toast(`${peer?.name || 'Voice'}: ${message}`, 'warn'); if (peer) peer.lastIceNotice = { key, at: now }; } captureRumIssue(message, 'WARNING', { peerId: peer?.id, peerName: peer?.name, ...extra, }); } async function selectedIcePath(pc) { if (!pc?.getStats) return null; const stats = await pc.getStats(); let pair; stats.forEach((report) => { if (report.type === 'candidate-pair' && (report.selected || report.nominated) && report.state === 'succeeded') { pair = report; } }); if (!pair) return null; const local = stats.get(pair.localCandidateId); const remote = stats.get(pair.remoteCandidateId); return { localType: local?.candidateType, remoteType: remote?.candidateType, protocol: pair.protocol, path: local?.candidateType === 'relay' || remote?.candidateType === 'relay' ? 'TURN relay' : 'direct/STUN', }; } async function tuneAudioSender(sender) { if (!sender?.getParameters || !sender?.setParameters) return; const params = sender.getParameters(); if (!params.encodings?.length) params.encodings = [{}]; params.encodings[0].maxBitrate = 64000; params.encodings[0].dtx = false; params.encodings[0].priority = 'high'; await sender.setParameters(params); } function preferOpus(pc) { const capabilities = globalThis.RTCRtpReceiver?.getCapabilities?.('audio'); const codecs = capabilities?.codecs || []; const opus = codecs.filter((codec) => codec.mimeType?.toLowerCase() === 'audio/opus'); if (!opus.length) return; const ordered = [...opus, ...codecs.filter((codec) => !opus.includes(codec))]; for (const transceiver of pc.getTransceivers()) { if (transceiver.receiver?.track?.kind === 'audio' || transceiver.sender?.track?.kind === 'audio') { try { transceiver.setCodecPreferences(ordered); } catch (err) { captureRumIssue('Could not prefer Opus audio codec', 'WARNING', { error: err?.message || String(err) }); } } } } function monitorPeerQuality(peer) { if (peer.qualityTimer) clearInterval(peer.qualityTimer); peer.qualityTimer = setInterval(async () => { const pc = peer.pc; if (!pc || pc.connectionState !== 'connected' || !pc.getStats) return; try { const stats = await pc.getStats(); let inbound; let outbound; let remoteInbound; stats.forEach((report) => { if (report.kind !== 'audio' && report.mediaType !== 'audio') return; if (report.type === 'inbound-rtp') inbound = report; if (report.type === 'outbound-rtp') outbound = report; if (report.type === 'remote-inbound-rtp') remoteInbound = report; }); const packets = (inbound?.packetsReceived || 0) + (inbound?.packetsLost || 0); const lossRatio = packets > 20 ? (inbound.packetsLost || 0) / packets : 0; const jitterMs = (inbound?.jitter || 0) * 1000; const rttMs = (remoteInbound?.roundTripTime || 0) * 1000; const issue = lossRatio >= 0.05 || jitterMs >= 80 || rttMs >= 500; if (issue && (!peer.lastQualityNotice || Date.now() - peer.lastQualityNotice > ICE_NOTICE_INTERVAL)) { peer.lastQualityNotice = Date.now(); const details = `quality degraded (loss ${Math.round(lossRatio * 100)}%, jitter ${Math.round(jitterMs)}ms${rttMs ? `, RTT ${Math.round(rttMs)}ms` : ''})`; toast(`${peer.name}: ${details}`, 'warn'); captureRumIssue('WebRTC audio quality degraded', 'WARNING', { peerId: peer.id, peerName: peer.name, lossRatio, jitterMs, rttMs, packetsReceived: inbound?.packetsReceived, packetsLost: inbound?.packetsLost, bytesReceived: inbound?.bytesReceived, bytesSent: outbound?.bytesSent, path: peer.icePath?.path, }); } } catch (err) { captureRumIssue('Could not inspect WebRTC audio quality', 'WARNING', { peerId: peer.id, error: err?.message || String(err), }); } }, 5000); } function playRetroCue(name) { const ctx = S.audioCtx; if (!ctx) return; ctx.resume().catch(() => {}); const cues = { join: [ { frequency: 659, offset: 0, duration: 0.09, type: 'square' }, { frequency: 988, offset: 0.08, duration: 0.16, type: 'square' }, ], leave: [ { frequency: 523, offset: 0, duration: 0.11, type: 'triangle' }, { frequency: 392, offset: 0.09, duration: 0.18, type: 'triangle' }, ], share: [ { frequency: 523, offset: 0, duration: 0.08, type: 'square' }, { frequency: 659, offset: 0.07, duration: 0.08, type: 'square' }, { frequency: 784, offset: 0.14, duration: 0.08, type: 'square' }, { frequency: 1047, offset: 0.21, duration: 0.22, type: 'triangle' }, ], }; const cue = cues[name]; if (!cue) return; const start = ctx.currentTime; for (const note of cue) { const oscillator = ctx.createOscillator(); const gain = ctx.createGain(); const noteStart = start + note.offset; oscillator.type = note.type; oscillator.frequency.setValueAtTime(note.frequency, noteStart); gain.gain.setValueAtTime(0.0001, noteStart); gain.gain.exponentialRampToValueAtTime(0.055, noteStart + 0.008); gain.gain.exponentialRampToValueAtTime(0.0001, noteStart + note.duration); oscillator.connect(gain).connect(ctx.destination); oscillator.start(noteStart); oscillator.stop(noteStart + note.duration + 0.02); } } const PEER_COLORS = ['#5865f2', '#23a55a', '#f0b232', '#eb459e', '#3ba55c', '#faa61a', '#e67e22', '#9b59b6']; function colorFor(id) { let h = 0; for (const c of id) h = (h * 31 + c.charCodeAt(0)) >>> 0; return PEER_COLORS[h % PEER_COLORS.length]; } function setAvatar(el, name, id) { el.style.background = colorFor(id); el.textContent = (name || '?').slice(0, 1).toUpperCase(); } function effectiveIce() { const raw = localStorage.getItem(LS_ICE); if (raw) { try { return JSON.parse(raw); } catch { /* fall through */ } } return S.iceServers || FALLBACK_ICE; } function microphoneConstraints(deviceId = localStorage.getItem(LS_AUDIO_INPUT)) { const audio = { echoCancellation: true, noiseSuppression: true, autoGainControl: true, channelCount: 1, sampleRate: 48000, sampleSize: 16, }; if (deviceId) audio.deviceId = { exact: deviceId }; return { audio }; } async function populateAudioInputs() { const select = $('#audio-input'); if (!select || !navigator.mediaDevices?.enumerateDevices) return; try { const devices = await navigator.mediaDevices.enumerateDevices(); const inputs = devices.filter((device) => device.kind === 'audioinput'); const selected = localStorage.getItem(LS_AUDIO_INPUT) || ''; select.replaceChildren(new Option('System default', '')); inputs.forEach((device, index) => { select.add(new Option(device.label || `Microphone ${index + 1}`, device.deviceId)); }); select.value = inputs.some((device) => device.deviceId === selected) ? selected : ''; if (selected && !select.value) localStorage.removeItem(LS_AUDIO_INPUT); } catch (err) { console.warn('enumerating audio inputs failed', err); captureRumIssue('Could not enumerate audio inputs', 'WARNING', { error: err?.message || String(err) }); } } async function switchAudioInput(deviceId) { const previousDeviceId = localStorage.getItem(LS_AUDIO_INPUT) || ''; if (deviceId) localStorage.setItem(LS_AUDIO_INPUT, deviceId); else localStorage.removeItem(LS_AUDIO_INPUT); if (!S.room || !S.micStream) return; let nextStream; try { nextStream = await navigator.mediaDevices.getUserMedia(microphoneConstraints(deviceId)); } catch (err) { if (previousDeviceId) localStorage.setItem(LS_AUDIO_INPUT, previousDeviceId); else localStorage.removeItem(LS_AUDIO_INPUT); $('#audio-input').value = previousDeviceId; toast(`Could not use that microphone: ${err.message || err}`, 'error'); return; } const nextTrack = nextStream.getAudioTracks()[0]; nextTrack.enabled = !S.muted; await Promise.all([...S.peers.values()].map(async (peer) => { if (!peer.pc) return; const sender = peer.pc.getSenders().find((item) => item.track && item.track.kind === 'audio'); if (sender) await sender.replaceTrack(nextTrack); })); S.micStream.getTracks().forEach((track) => track.stop()); S.micStream = nextStream; updateMicButton(); toast('Microphone changed', 'ok'); } // --------------------------------------------------------------------------- // join / leave // --------------------------------------------------------------------------- async function joinVoice(room) { if (S.room === room || S.connecting) return; if (S.room) leaveVoice('switching channel'); S.connecting = true; setStatus(`connecting to ${room}…`, 'connecting'); // Mic permission & AudioContext: request immediately on the user-click gesture // so browsers don't block the microphone permission prompt or audio autoplay policy. if (!S.audioCtx) S.audioCtx = new AudioContext(); await S.audioCtx.resume().catch(() => {}); orientListener(); // Attach a one-time gesture listener in case browser autoplay policy keeps AudioContext suspended const resumeCtx = () => { if (S.audioCtx && S.audioCtx.state === 'suspended') { S.audioCtx.resume().catch(() => {}); } }; window.addEventListener('click', resumeCtx, { passive: true }); window.addEventListener('keydown', resumeCtx, { passive: true }); try { S.micStream = await navigator.mediaDevices.getUserMedia(microphoneConstraints()); populateAudioInputs(); } catch { S.micStream = null; toast('No microphone available — joining in listen-only mode', 'warn'); } S.muted = false; updateMicButton(); for (const btn of shareButtons) btn.disabled = false; try { S.iceServers = await fetchIce(); } catch { toast('Could not load ICE config from server — using fallback STUN', 'warn'); } await openSignaling(room); S.room = room; S.connecting = false; playRetroCue('join'); renderAll(); } async function fetchIce() { const headers = window.rum && typeof window.rum.traceHeaders === 'function' ? window.rum.traceHeaders() : {}; const res = await fetch('/api/ice', { headers }); if (!res.ok) throw new Error(`ice ${res.status}`); return res.json(); } async function fetchRooms() { try { const headers = window.rum && typeof window.rum.traceHeaders === 'function' ? window.rum.traceHeaders() : {}; const res = await fetch('/api/rooms', { headers }); if (res.ok) { S.channelRosters = await res.json(); renderTiles(); renderChannelCounts(); } } catch (err) { console.warn('fetchRooms failed', err); captureRumIssue('Could not refresh voice channel rosters', 'WARNING', { error: err?.message || String(err) }); } } function leaveVoice(reason) { if (!S.room && !S.ws) return; const wasConnected = Boolean(S.room); if (wasConnected) playRetroCue('leave'); if (S.sharing) stopScreenShare(undefined, true); for (const peer of S.peers.values()) destroyPeer(peer); S.peers.clear(); if (S.pingTimer) { clearInterval(S.pingTimer); S.pingTimer = null; } if (S.ws) { try { S.ws.close(); } catch {} S.ws = null; } if (S.micStream) { S.micStream.getTracks().forEach((t) => t.stop()); S.micStream = null; } // Keep share buttons enabled so clicking one prompts the user to join a channel updateMicButton(); document.querySelectorAll('.channel.voice').forEach((c) => c.classList.remove('active')); S.room = null; S.myId = null; S.muted = false; S.deafened = false; updateDeafenButton(); renderAll(); setStatus(reason || 'not connected'); fetchRooms(); } function setStatus(text, state = '') { // Concise display text let shortText = text; if (text.startsWith('voice connected')) { shortText = S.room ? `#${S.room}` : 'Connected'; } else if (text.startsWith('connecting to')) { shortText = 'Connecting…'; } for (const el of statusTextEls) el.textContent = shortText; if (els.statusRow) els.statusRow.title = text; for (const el of statusDotEls) el.className = 'status-indicator' + (state ? ` ${state}` : ''); } // --------------------------------------------------------------------------- // reconnection (server restarts / redeploys, dropped signaling, etc.) // --------------------------------------------------------------------------- const RECONNECT_DELAYS = [1000, 2000, 4000, 8000, 15000]; // caps at 15s function showReconnectBanner(text) { els.reconnectText.textContent = text; els.reconnectBanner.classList.remove('hidden'); } function hideReconnectBanner() { els.reconnectBanner.classList.add('hidden'); } function beginReconnect(room) { cancelReconnect(); S.reconnect.room = room; attemptReconnect(); } function cancelReconnect() { const r = S.reconnect; clearTimeout(r.timer); clearInterval(r.countdownTimer); r.timer = r.countdownTimer = null; r.room = null; r.attempt = 0; hideReconnectBanner(); } function attemptReconnect() { const r = S.reconnect; if (!r.room) return; const delay = RECONNECT_DELAYS[Math.min(r.attempt, RECONNECT_DELAYS.length - 1)]; r.attempt++; let remaining = delay; const label = () => `Connection lost — reconnecting in ${Math.ceil(remaining / 1000)}s (attempt ${r.attempt})`; showReconnectBanner(label()); clearInterval(r.countdownTimer); r.countdownTimer = setInterval(() => { remaining = Math.max(0, remaining - 1000); if (remaining > 0) showReconnectBanner(label()); }, 1000); clearTimeout(r.timer); r.timer = setTimeout(() => doReconnectAttempt(), delay); } async function doReconnectAttempt() { const r = S.reconnect; if (!r.room) return; clearInterval(r.countdownTimer); const room = r.room; showReconnectBanner('Reconnecting…'); try { await joinVoice(room); r.room = null; r.attempt = 0; hideReconnectBanner(); toast('Reconnected', 'ok'); } catch { attemptReconnect(); } } $('#reconnect-retry').onclick = () => { clearTimeout(S.reconnect.timer); clearInterval(S.reconnect.countdownTimer); doReconnectAttempt(); }; $('#reconnect-dismiss').onclick = () => cancelReconnect(); // --------------------------------------------------------------------------- // signaling // --------------------------------------------------------------------------- function openSignaling(room) { return new Promise((resolve, reject) => { const proto = location.protocol === 'https:' ? 'wss' : 'ws'; const url = `${proto}://${location.host}/ws?room=${encodeURIComponent(room)}&name=${encodeURIComponent(S.name)}`; const ws = new WebSocket(url); S.ws = ws; ws.onerror = () => { S.connecting = false; setStatus('connection failed', 'error'); captureRumIssue('Voice signaling connection failed', 'ERROR', { room }); reject(new Error('websocket failed')); }; ws.onmessage = (ev) => { let msg; try { msg = JSON.parse(ev.data); } catch { return; } try { handleSignal(msg); } catch (err) { console.error('signal handler error', msg.type, err); captureRumIssue('Voice signaling message handler failed', 'ERROR', { type: msg.type, error: err?.message || String(err) }); } }; ws.onclose = () => { if (S.pingTimer) { clearInterval(S.pingTimer); S.pingTimer = null; } if (S.ws === ws && S.room) { const room = S.room; leaveVoice('disconnected'); beginReconnect(room); } }; ws.onopen = () => { if (S.pingTimer) clearInterval(S.pingTimer); S.pingTimer = setInterval(() => { send({ type: 'ping' }); }, 10000); resolve(); }; }); } function send(msg) { if (S.ws && S.ws.readyState === 1) S.ws.send(JSON.stringify(msg)); } function handleSignal(msg) { switch (msg.type) { case 'rosters': { S.channelRosters = msg.rooms || {}; renderTiles(); renderChannelCounts(); break; } case 'hello': { S.myId = msg.id; S.room = msg.room; for (const p of msg.peers) { S.peers.set(p.id, makePeerRecord(p)); // deterministic initiator: lower id offers (no glare storm on join) ensurePC(S.peers.get(p.id), S.myId < p.id); } sendMyPos(true); renderAll(); setStatus(`voice connected — ${msg.peers.length + 1} in ${msg.room}`, 'online'); toast(`Joined ${msg.room}`, 'ok'); break; } case 'joined': S.peers.set(msg.peer.id, makePeerRecord(msg.peer)); playRetroCue('join'); renderAll(); toast(`${msg.peer.name} joined`, 'ok'); break; case 'left': { const peer = S.peers.get(msg.id); if (peer) { playRetroCue('leave'); destroyPeer(peer); S.peers.delete(msg.id); renderAll(); toast(`${peer.name} left`); } break; } case 'pos': { const peer = S.peers.get(msg.from); if (peer) { peer.x = msg.x; peer.y = msg.y; updatePanner(peer); } break; } case 'state': { const peer = S.peers.get(msg.from); if (peer) { peer.state = msg.state; renderTiles(); } break; } case 'offer': case 'answer': { const peer = ensurePC(S.peers.get(msg.from) || createOrphanPeer(msg.from), false); onDescription(peer, msg.sdp).catch((e) => { console.error('description failed', e); captureRumIssue('WebRTC session description failed', 'ERROR', { peerId: peer.id, type: msg.type, error: e?.message || String(e) }); }); break; } case 'candidate': { const peer = S.peers.get(msg.from); if (!peer || !peer.pc) return; // candidates only after description path created pc onCandidate(peer, msg.candidate).catch((e) => { console.error('candidate failed', e); captureRumIssue('WebRTC ICE candidate failed', 'WARNING', { peerId: peer.id, error: e?.message || String(e) }); }); break; } case 'reset': { // The other side's connection to us failed and it's rebuilding from // scratch — tear down our half so a fresh offer lands on a clean // RTCPeerConnection instead of an already-established (and now // mismatched) DTLS transport. See recreatePeerConnection(). const peer = S.peers.get(msg.from); if (peer) recreatePeerConnection(peer, { announce: false }); break; } default: break; } } // --------------------------------------------------------------------------- // peer records + WebRTC (perfect negotiation) // --------------------------------------------------------------------------- function makePeerRecord(p) { return { id: p.id, name: p.name, x: p.x, y: p.y, state: p.state || { muted: false, deafened: false, sharing: false }, pc: null, polite: false, makingOffer: false, ignoreOffer: false, pendingCandidates: [], micStream: null, screenStreams: new Map(), gain: null, panner: null, analyser: null, vol: 1, speaking: false, resetAttempts: 0, lastIceNotice: null, icePath: null, qualityTimer: null, lastQualityNotice: 0, els: {}, }; } function createOrphanPeer(id) { const p = makePeerRecord({ id, name: 'peer-' + id.slice(0, 4), x: 50, y: 50 }); S.peers.set(id, p); return p; } function ensurePC(peer, initiator) { if (peer.pc) return peer; const config = { ...effectiveIce(), iceCandidatePoolSize: 4 }; // After a failed retry, force everything through TURN relay — sidesteps // AP/router client isolation and NAT-hairpin issues that are common when // both peers are on the same LAN (direct/STUN candidates get silently // dropped by the router in that case, so give up trying them). if (peer.resetAttempts > 0) config.iceTransportPolicy = 'relay'; const pc = new RTCPeerConnection(config); peer.pc = pc; peer.polite = S.myId > peer.id; pc.onnegotiationneeded = async () => { try { peer.makingOffer = true; await pc.setLocalDescription(); send({ type: 'offer', to: peer.id, sdp: pc.localDescription }); } catch (err) { console.error('negotiation failed', err); captureRumIssue('WebRTC negotiation failed', 'ERROR', { peerId: peer.id, error: err?.message || String(err) }); } finally { peer.makingOffer = false; } }; pc.onicecandidate = ({ candidate }) => { if (candidate) send({ type: 'candidate', to: peer.id, candidate }); }; pc.onicecandidateerror = (event) => { notifyIceIssue(peer, `ICE server unavailable (${event.errorCode || 'unknown'})`, { peerId: peer.id, iceConnectionState: pc.iceConnectionState, iceGatheringState: pc.iceGatheringState, errorCode: event.errorCode, errorText: event.errorText, url: event.url, }); }; pc.oniceconnectionstatechange = () => { const state = pc.iceConnectionState; if (state === 'disconnected') { notifyIceIssue(peer, 'ICE disconnected; audio may stop', { iceConnectionState: state, iceGatheringState: pc.iceGatheringState, }); } else if (state === 'failed') { notifyIceIssue(peer, 'ICE failed; trying TURN relay', { iceConnectionState: state, iceGatheringState: pc.iceGatheringState, }); } }; pc.onconnectionstatechange = () => { renderTiles(); if (pc.connectionState === 'failed') { console.warn('peer connection failed, rebuilding it from scratch', peer.id); notifyIceIssue(peer, 'connection failed; rebuilding voice path', { connectionState: pc.connectionState, iceConnectionState: pc.iceConnectionState, iceGatheringState: pc.iceGatheringState, signalingState: pc.signalingState, resetAttempts: peer.resetAttempts, }); captureRumIssue('WebRTC peer connection failed', 'ERROR', { peerId: peer.id, state: pc.connectionState, resetAttempts: peer.resetAttempts, }); recreatePeerConnection(peer, { announce: true }); } else if (pc.connectionState === 'connected') { peer.resetAttempts = 0; // a successful connection clears the failure count selectedIcePath(pc).then((path) => { if (!path) return; peer.icePath = path; toast(`${peer.name}: ${path.path} connection`, 'ok'); captureRumIssue('WebRTC connection established', 'INFO', { peerId: peer.id, peerName: peer.name, connectionState: pc.connectionState, iceConnectionState: pc.iceConnectionState, ...path, }); }).catch((err) => { captureRumIssue('Could not inspect selected ICE path', 'WARNING', { peerId: peer.id, error: err?.message || String(err), }); }); monitorPeerQuality(peer); } }; pc.ontrack = (ev) => onRemoteTrack(peer, ev); // send my mic; if I have none, ask to receive theirs if (S.micStream) { for (const track of S.micStream.getAudioTracks()) { const sender = pc.addTrack(track, S.micStream); tuneAudioSender(sender).catch((err) => captureRumIssue('Could not tune Opus audio sender', 'WARNING', { peerId: peer.id, error: err?.message || String(err), })); } } else { pc.addTransceiver('audio', { direction: 'recvonly' }); } preferOpus(pc); // send my screen share if I am currently sharing when this peer connects if (S.sharing) { const peerSenders = new Map(); for (const [streamId, stream] of S.screenStreams) { const senders = []; for (const track of stream.getTracks()) { senders.push(pc.addTrack(track, stream)); } peerSenders.set(streamId, senders); } S.screenSenders.set(peer.id, peerSenders); } if (initiator) pc.onnegotiationneeded(); return peer; } const RESET_MAX_ATTEMPTS = 3; // Once a connection reaches 'failed', its DTLS transport is done — restartIce() // only restarts ICE gathering and reuses the same transport/certificate, which // throws "Failed to set SSL role for the transport" if you try to renegotiate // on it. The only real fix is a brand-new RTCPeerConnection (fresh certificate, // no established role to conflict with). Since only one side may notice the // failure first, whichever side detects it also tells the other to rebuild via // a 'reset' signal — otherwise the other side keeps its old, now-mismatched // transport and the fresh offer we send will fail against it too. function recreatePeerConnection(peer, { announce }) { if (announce) { peer.resetAttempts++; if (peer.resetAttempts > RESET_MAX_ATTEMPTS) { toast(`Connection to ${peer.name} keeps failing — try leaving and rejoining the channel`, 'error'); captureRumIssue('WebRTC peer connection could not be rebuilt', 'ERROR', { peerId: peer.id, resetAttempts: peer.resetAttempts, }); return; } send({ type: 'reset', to: peer.id }); } if (peer.pc) { try { peer.pc.close(); } catch {} } if (peer.qualityTimer) { clearInterval(peer.qualityTimer); peer.qualityTimer = null; } peer.pc = null; peer.makingOffer = false; peer.ignoreOffer = false; peer.pendingCandidates = []; detachSpatial(peer); if (peer.audioEl) { try { peer.audioEl.pause(); peer.audioEl.srcObject = null; } catch {} peer.audioEl = null; } peer.micStream = null; peer.screenStreams.clear(); S.screenSenders.delete(peer.id); renderScreens(); renderTiles(); ensurePC(peer, S.myId < peer.id); // same deterministic-initiator rule as the first connection } async function onDescription(peer, desc) { const pc = peer.pc; const collision = desc.type === 'offer' && (peer.makingOffer || pc.signalingState !== 'stable'); peer.ignoreOffer = !peer.polite && collision; if (peer.ignoreOffer) return; // impolite side ignores the rival offer await pc.setRemoteDescription(desc); // modern browsers roll back implicitly when polite for (const c of peer.pendingCandidates.splice(0)) { await pc.addIceCandidate(c).catch((e) => { console.warn('buffered candidate failed', e); captureRumIssue('WebRTC buffered ICE candidate failed', 'WARNING', { peerId: peer.id, error: e?.message || String(e) }); }); } if (desc.type === 'offer') { await pc.setLocalDescription(); send({ type: 'answer', to: peer.id, sdp: pc.localDescription }); } } async function onCandidate(peer, candidate) { if (peer.ignoreOffer) return; if (!peer.pc.remoteDescription) { peer.pendingCandidates.push(candidate); return; } await peer.pc.addIceCandidate(candidate); } function destroyPeer(peer) { if (peer.pc) { try { peer.pc.close(); } catch {} peer.pc = null; } if (peer.qualityTimer) { clearInterval(peer.qualityTimer); peer.qualityTimer = null; } detachSpatial(peer); if (peer.audioEl) { try { peer.audioEl.pause(); peer.audioEl.srcObject = null; } catch {} peer.audioEl = null; } peer.micStream = null; peer.screenStreams.clear(); peer.els = {}; renderScreens(); } // --------------------------------------------------------------------------- // remote media: mic -> spatial chain, screen -> big video + direct audio // --------------------------------------------------------------------------- function onRemoteTrack(peer, ev) { const track = ev.track; const stream = (ev.streams && ev.streams[0]) ? ev.streams[0] : new MediaStream([track]); if (track.kind === 'video') { const isNewScreenShare = !peer.screenStreams.has(stream.id); peer.screenStreams.set(stream.id, stream); track.addEventListener('ended', () => { if (peer.screenStreams.get(stream.id) === stream) { peer.screenStreams.delete(stream.id); if (!peer.screenStreams.size) { peer.state = { ...peer.state, sharing: false }; captureRumIssue('Remote screen share ended', 'INFO', { peerId: peer.id, peerName: peer.name, streamId: stream.id, }); } renderScreens(); renderTiles(); } }); stream.addEventListener('inactive', () => { if (peer.screenStreams.get(stream.id) === stream) { peer.screenStreams.delete(stream.id); peer.state = { ...peer.state, sharing: false }; renderScreens(); renderTiles(); } }, { once: true }); renderScreens(); if (isNewScreenShare) playRetroCue('share'); console.log('remote screen share from', peer.name); } else if (track.kind === 'audio') { peer.micStream = stream; // Ensure AudioContext is active (resumes on Firefox/Safari if suspended) if (S.audioCtx && S.audioCtx.state === 'suspended') { S.audioCtx.resume().catch(() => {}); } // Chrome/Safari benefit from having an Audio element attached to pump the stream if (!peer.audioEl) { peer.audioEl = new Audio(); peer.audioEl.srcObject = stream; peer.audioEl.muted = true; peer.audioEl.play().catch((e) => console.warn('peer audioEl autoplay', e)); } attachSpatial(peer, stream); console.log('remote mic from', peer.name); } renderTiles(); } function attachSpatial(peer, stream) { const ctx = S.audioCtx; if (!ctx || peer.panner) return; const src = ctx.createMediaStreamSource(stream); const panner = ctx.createPanner(); panner.panningModel = 'HRTF'; panner.distanceModel = 'inverse'; panner.refDistance = 4; panner.rolloffFactor = 1.4; panner.maxDistance = 200; const gain = ctx.createGain(); gain.gain.value = S.deafened ? 0 : peer.vol; const analyser = ctx.createAnalyser(); analyser.fftSize = 256; src.connect(panner).connect(gain).connect(analyser).connect(ctx.destination); Object.assign(peer, { panner, gain, analyser }); updatePanner(peer); } function detachSpatial(peer) { if (peer.gain) { try { peer.gain.disconnect(); } catch {} } if (peer.panner) { try { peer.panner.disconnect(); } catch {} } peer.panner = peer.gain = peer.analyser = null; } function updatePanner(peer) { if (!peer.panner) return; const x = (peer.x - S.me.x) * SPATIAL_SCALE; const z = (peer.y - S.me.y) * SPATIAL_SCALE; // screen-down = behind listener const p = peer.panner; if (p.positionX) { p.positionX.value = x; p.positionY.value = 0; p.positionZ.value = z; } else { // old Safari p.setPosition(x, 0, z); } } function orientListener() { const l = S.audioCtx && S.audioCtx.listener; if (!l) return; if (l.forwardX) { l.forwardX.value = 0; l.forwardY.value = 0; l.forwardZ.value = -1; l.upX.value = 0; l.upY.value = 1; l.upZ.value = 0; } else { l.setOrientation(0, 0, -1, 0, 1, 0); } } // --------------------------------------------------------------------------- // screen share (local) // --------------------------------------------------------------------------- async function startScreenShare() { if (!S.room) { toast('Join a voice channel first to share your screen', 'warn'); return; } if (!navigator.mediaDevices || !navigator.mediaDevices.getDisplayMedia) { toast('Screen sharing is not supported in this browser or context', 'error'); return; } let stream; try { // Try requesting display media with audio first stream = await navigator.mediaDevices.getDisplayMedia({ video: { frameRate: 30 }, audio: true, }); } catch (err) { // If user cancelled / denied, don't toast as error unless it's not NotAllowedError/AbortError if (err.name === 'NotAllowedError' || err.name === 'AbortError') { return; } // Try fallback without audio constraint if audio failed try { stream = await navigator.mediaDevices.getDisplayMedia({ video: { frameRate: 30 }, }); } catch (fallbackErr) { if (fallbackErr.name === 'NotAllowedError' || fallbackErr.name === 'AbortError') { return; } console.error('Failed to get display media:', fallbackErr); captureRumIssue('WebRTC display media capture failed', 'ERROR', { error: fallbackErr?.message || String(fallbackErr) }); toast(`Could not share screen: ${fallbackErr.message || fallbackErr}`, 'error'); return; } } S.screenStreams.set(stream.id, stream); S.sharing = true; playRetroCue('share'); send({ type: 'state', state: { muted: S.muted, deafened: S.deafened, sharing: true } }); for (const peer of S.peers.values()) addScreenToPeer(peer); const videoTrack = stream.getVideoTracks()[0]; if (videoTrack) { videoTrack.addEventListener('ended', () => stopScreenShare(stream)); } renderScreens(); // Open a desktop preview automatically; mobile keeps the in-page preview. if (window.matchMedia('(min-width: 700px)').matches) { popOutShare({ key: `me:${stream.id}`, name: `${S.name} (you)`, stream, local: true }); } renderTiles(); for (const btn of shareButtons) btn.classList.add('active'); toast('You are sharing your screen', 'ok'); } function addScreenToPeer(peer) { if (!peer.pc || !S.sharing) return; const peerSenders = new Map(); for (const [streamId, stream] of S.screenStreams) { const senders = []; for (const track of stream.getTracks()) { senders.push(peer.pc.addTrack(track, stream)); } peerSenders.set(streamId, senders); } S.screenSenders.set(peer.id, peerSenders); // renegotiation fires automatically } function stopScreenShare(stream, silent) { const streams = stream ? [stream] : [...S.screenStreams.values()]; if (!streams.length) return; for (const [peerId, peerSenders] of S.screenSenders) { const peer = S.peers.get(peerId); if (!peer || !peer.pc) continue; for (const localStream of streams) { for (const sender of peerSenders.get(localStream.id) || []) { try { peer.pc.removeTrack(sender); } catch {} } peerSenders.delete(localStream.id); } } for (const localStream of streams) { if (S.screenStreams.get(localStream.id) !== localStream) continue; localStream.getTracks().forEach((t) => t.stop()); S.screenStreams.delete(localStream.id); } S.sharing = S.screenStreams.size > 0; if (!S.sharing) S.screenSenders.clear(); send({ type: 'state', state: { muted: S.muted, deafened: S.deafened, sharing: S.sharing } }); renderScreens(); renderTiles(); for (const btn of shareButtons) btn.classList.toggle('active', S.sharing); if (!silent) toast(S.sharing ? 'Screen share stopped' : 'Screen sharing stopped'); } // --------------------------------------------------------------------------- // position sync // --------------------------------------------------------------------------- let lastPosSend = 0; function sendMyPos(force) { const now = performance.now(); if (!force && now - lastPosSend < 60) return; lastPosSend = now; send({ type: 'pos', x: S.me.x, y: S.me.y }); for (const peer of S.peers.values()) updatePanner(peer); } // --------------------------------------------------------------------------- // speaking detection (local meter + per-peer analyser RMS) // --------------------------------------------------------------------------- const meterBuf = new Uint8Array(256); function rms(analyser) { analyser.getByteTimeDomainData(meterBuf); let sum = 0; for (let i = 0; i < meterBuf.length; i++) { const v = (meterBuf[i] - 128) / 128; sum += v * v; } return Math.sqrt(sum / meterBuf.length); } let localAnalyser = null; setInterval(() => { if (!S.audioCtx) return; if (S.micStream && !localAnalyser) { const src = S.audioCtx.createMediaStreamSource(S.micStream); localAnalyser = S.audioCtx.createAnalyser(); localAnalyser.fftSize = 256; src.connect(localAnalyser); // tap only — not connected to destination (no echo) } const meSpeaking = localAnalyser && !S.muted && rms(localAnalyser) > 0.04; els.meAvatar.classList.toggle('speaking', !!meSpeaking); let changed = false; for (const peer of S.peers.values()) { const speaking = !!(peer.analyser && !S.deafened && rms(peer.analyser) > 0.03); if (speaking !== peer.speaking) { peer.speaking = speaking; changed = true; } } if (changed) renderTiles(); updateScreenTileIndicators(); }, 150); // --------------------------------------------------------------------------- // rendering // --------------------------------------------------------------------------- function renderAll() { renderTiles(); renderScreens(); renderStageEmpty(); renderChannelCounts(); } function renderStageEmpty() { const hasScreens = activeShares().length > 0; els.stage.classList.toggle('empty', !hasScreens); } function renderChannelCounts() { const counts = {}; // Count users across all rooms known from snapshot for (const [rId, users] of Object.entries(S.channelRosters || {})) { counts[rId] = users ? users.length : 0; } // If active room has peers, ensure active count includes me + peers if (S.room) { counts[S.room] = S.peers.size + 1; } document.querySelectorAll('[data-room-count]').forEach((el) => { const n = counts[el.dataset.roomCount] || 0; el.textContent = n ? `${n}` : ''; }); } function renderTiles() { // Clear any existing user rosters in voice channels document.querySelectorAll('.channel-users').forEach((el) => { el.innerHTML = ''; }); // For each voice channel element in the sidebar document.querySelectorAll('.channel.voice').forEach((channelEl) => { const roomId = channelEl.dataset.room; if (!roomId) return; let roster = channelEl.nextElementSibling; if (!roster || !roster.classList.contains('channel-users')) { roster = document.createElement('div'); roster.className = 'channel-users'; channelEl.after(roster); } if (S.room && S.room === roomId) { // Current active room: render Me + live peers (with interactive volume sliders) const meItem = document.createElement('div'); meItem.className = 'channel-user-item' + (els.meAvatar.classList.contains('speaking') ? ' speaking' : ''); const meAv = document.createElement('div'); meAv.className = 'avatar user-line-avatar'; setAvatar(meAv, S.name, S.myId || 'me'); const meName = document.createElement('span'); meName.className = 'user-line-name'; meName.textContent = `${S.name} (you)`; const meBadges = document.createElement('span'); meBadges.className = 'user-line-badges'; if (S.muted) { const b = document.createElement('span'); b.className = 'badge-mute'; b.title = 'Muted'; b.appendChild(mIcon('mic_off')); meBadges.appendChild(b); } if (S.deafened) { const b = document.createElement('span'); b.className = 'badge-deafen'; b.title = 'Deafened'; b.appendChild(mIcon('headset_off')); meBadges.appendChild(b); } if (S.sharing) { const b = document.createElement('span'); b.className = 'badge-share'; b.title = 'Sharing screen'; b.appendChild(mIcon('present_to_all')); meBadges.appendChild(b); } meItem.append(meAv, meName, meBadges); roster.appendChild(meItem); for (const peer of S.peers.values()) { const item = document.createElement('div'); item.className = 'channel-user-item' + (peer.speaking ? ' speaking' : ''); const av = document.createElement('div'); av.className = 'avatar user-line-avatar'; setAvatar(av, peer.name, peer.id); const name = document.createElement('span'); name.className = 'user-line-name'; name.textContent = peer.name; const badges = document.createElement('span'); badges.className = 'user-line-badges'; if (!peer.pc || peer.pc.connectionState !== 'connected') { badges.innerHTML += '…'; } if (peer.state.muted) { const b = document.createElement('span'); b.className = 'badge-mute'; b.title = 'Muted'; b.appendChild(mIcon('mic_off')); badges.appendChild(b); } if (peer.state.sharing) { const b = document.createElement('span'); b.className = 'badge-share'; b.title = 'Sharing screen'; b.appendChild(mIcon('present_to_all')); badges.appendChild(b); } // Volume hover popup const volPop = document.createElement('div'); volPop.className = 'volume-popup'; const volLabel = document.createElement('div'); volLabel.className = 'vol-title'; volLabel.textContent = `User Volume: ${Math.round(peer.vol * 100)}%`; const volInput = document.createElement('input'); volInput.type = 'range'; volInput.min = 0; volInput.max = 2; volInput.step = 0.05; volInput.value = peer.vol; volInput.oninput = (e) => { e.stopPropagation(); peer.vol = Number(volInput.value); volLabel.textContent = `User Volume: ${Math.round(peer.vol * 100)}%`; if (peer.gain && !S.deafened) peer.gain.gain.value = peer.vol; }; volPop.append(volLabel, volInput); // Touch has no :hover — tap the row to reveal the volume popup instead. item.addEventListener('click', (e) => { if (e.target === volInput) return; item.classList.toggle('show-vol'); }); item.append(av, name, badges, volPop); roster.appendChild(item); } } else { // Remote / unjoined channel: render users from server roster snapshot const occupants = S.channelRosters && S.channelRosters[roomId]; if (Array.isArray(occupants) && occupants.length > 0) { for (const user of occupants) { const item = document.createElement('div'); item.className = 'channel-user-item'; const av = document.createElement('div'); av.className = 'avatar user-line-avatar'; setAvatar(av, user.name, user.id); const name = document.createElement('span'); name.className = 'user-line-name'; name.textContent = user.name; const badges = document.createElement('span'); badges.className = 'user-line-badges'; if (user.state?.muted) { const b = document.createElement('span'); b.className = 'badge-mute'; b.title = 'Muted'; b.appendChild(mIcon('mic_off')); badges.appendChild(b); } if (user.state?.sharing) { const b = document.createElement('span'); b.className = 'badge-share'; b.title = 'Sharing screen'; b.appendChild(mIcon('present_to_all')); badges.appendChild(b); } item.append(av, name, badges); roster.appendChild(item); } } } }); renderStageEmpty(); } // share keys: 'me' or peer id; popped-out shares render in their own window const poppedWindows = new Map(); // key -> {win, name} function activeShares() { const shares = []; for (const [streamId, stream] of S.screenStreams) { shares.push({ key: `me:${streamId}`, name: `${S.name} (you)`, stream, local: true }); } for (const peer of S.peers.values()) { if (peer.state.sharing) { for (const stream of peer.screenStreams.values()) { shares.push({ key: `${peer.id}:${stream.id}`, name: peer.name, stream }); } } } return shares; } function renderScreens() { // prune popouts whose share ended const live = new Set(activeShares().map((s) => s.key)); for (const [key, p] of [...poppedWindows]) { if (!live.has(key)) { try { p.win.close(); } catch {} poppedWindows.delete(key); } } if (!els.screens) return; els.screens.innerHTML = ''; els.screens.classList.toggle('multi', activeShares().length > 1); for (const share of activeShares()) { if (poppedWindows.has(share.key)) { // rendered in its own window — show a small placeholder card to re-embed const ph = document.createElement('div'); ph.className = 'screen-card popped'; const btn = document.createElement('button'); btn.className = 'pop-btn'; btn.appendChild(mIcon('open_in_new')); btn.appendChild(document.createTextNode(' re-embed')); btn.onclick = () => closePopout(share.key); const label = document.createElement('div'); label.className = 'label'; label.textContent = `${share.name} — popped out`; ph.append(btn, label); if (share.local) { const stop = document.createElement('button'); stop.className = 'pop-btn stop-share-btn'; stop.title = 'Stop this screen share'; stop.appendChild(mIcon('stop_circle')); stop.appendChild(document.createTextNode(' stop')); stop.onclick = () => stopScreenShare(share.stream); ph.appendChild(stop); } els.screens.appendChild(ph); continue; } const card = document.createElement('div'); card.className = 'screen-card'; card.dataset.shareKey = share.key; const video = document.createElement('video'); video.autoplay = true; video.playsInline = true; video.muted = share.local; video.srcObject = share.stream; video.play().catch(() => {}); const label = document.createElement('div'); label.className = 'label'; label.innerHTML = '●'; label.appendChild(document.createTextNode(share.name)); const micBadge = document.createElement('span'); micBadge.className = 'tile-mic-badge'; micBadge.appendChild(mIcon('mic_off')); label.appendChild(micBadge); const pop = document.createElement('button'); pop.className = 'pop-btn'; pop.title = 'Pop out in new window'; pop.appendChild(mIcon('open_in_new')); pop.onclick = () => popOutShare(share); card.append(video, label, pop); if (share.local) { const stop = document.createElement('button'); stop.className = 'pop-btn stop-share-btn'; stop.title = 'Stop this screen share'; stop.appendChild(mIcon('stop_circle')); stop.appendChild(document.createTextNode(' stop')); stop.onclick = () => stopScreenShare(share.stream); card.appendChild(stop); } els.screens.appendChild(card); } updateScreenTileIndicators(); } // Cheap per-tick refresh of speaking/muted state on screen-share tiles — // avoids renderScreens()'s full teardown, which would restart