/* global AFRAME, THREE, NAF, CurrentWorld */ // Shared screens. Someone joins a room as "My screen" from a computer (see // share.js), and what they share shows up here as a big screen everyone sees, // with its sound coming from it. Anyone can move one: in VR point a hand at it // and squeeze grip, then squeeze the other grip too and move your hands apart // or together to resize it; on desktop look at it and press G, scroll to // resize, and press G again to put it down. A screen stays upright and turns to // face whoever is moving it. Where screens are is synced like wield.js's tools. // // The sharer's connection carries nothing else (no voice), so whatever video // and sound arrive on it are the screen's. They're read straight off the // peer connection rather than through EasyRTC's stream names, which can arrive // after the media does. (function () { const REACH = 25; // how far away you can grab a screen (m) const EDGE = 0.15; // grabbing just outside the picture still counts const DEFAULT_WIDTH = 3; const MIN_WIDTH = 0.6; const MAX_WIDTH = 12; const FRAME = 0.04; // dark border around the picture const CHIN = 0.16; // the border's bottom edge, with the name on it const POSE_MS = 66; // how often a moving screen's pose is sent const CHECK_MS = 1000; // how often to look for the sharer's video and sound const SMOOTH = 0.06; // seconds to ease toward someone else's move const MAX_SCREENS = 8; const v1 = new THREE.Vector3(); const v2 = new THREE.Vector3(); const v3 = new THREE.Vector3(); const q1 = new THREE.Quaternion(); const online = () => typeof NAF !== 'undefined' && NAF.connection && NAF.connection.isConnected(); const finite = (a, n) => Array.isArray(a) && a.length === n && a.every((x) => Number.isFinite(x) && Math.abs(x) < 1000); const round = (x) => Math.round(x * 1e4) / 1e4; const clampWidth = (w) => THREE.MathUtils.clamp(w, MIN_WIDTH, MAX_WIDTH); function cleanName(value) { return String(value || '').replace(/[\u0000-\u001f\u007f-\u009f]/g, '').trim().slice(0, 20) || 'Someone'; } // The video and sound coming in from a sharer, if any yet. function incoming(clientId) { const adapter = NAF.connection.adapter; const pc = adapter && adapter.easyrtc && adapter.easyrtc.getPeerConnectionByUserId(clientId); const found = { video: null, audio: null }; if (!pc || !pc.getTransceivers) return found; for (const t of pc.getTransceivers()) { const dir = t.currentDirection; if (dir !== 'recvonly' && dir !== 'sendrecv') continue; const track = t.receiver && t.receiver.track; if (track && track.readyState === 'live') found[track.kind] = track; } return found; } AFRAME.registerComponent('screens', { schema: { head: { type: 'selector', default: '#player' }, leftHand: { type: 'selector', default: '#left-hand' }, rightHand: { type: 'selector', default: '#right-hand' }, }, init() { this.screens = new Map(); // sharer's client id → screen this.pending = new Map(); // poses heard before the screen itself this.lastPose = 0; this.lastCheck = 0; this.hand = null; // what's moving a screen of yours: { screen, el?, dist, ox, oy } this.stretch = null; // your other hand while resizing: { el, from, width } this.listeners = []; const on = (target, type, fn, opts) => { target.addEventListener(type, fn, opts); this.listeners.push([target, type, fn, opts]); }; for (const el of [this.data.leftHand, this.data.rightHand]) { // After everyone else's listeners, so a pen grabbed by wield.js wins. on(el, 'gripdown', () => Promise.resolve().then(() => this.gripDown(el))); on(el, 'gripup', () => this.gripUp(el)); on(el, 'controllerdisconnected', () => this.gripUp(el)); } on(this.el.sceneEl, 'enter-vr', () => this.letGo()); on(this.el.sceneEl, 'exit-vr', () => this.letGo()); const typing = (evt) => evt.target.closest && evt.target.closest('input, textarea, select'); on(window, 'keydown', (evt) => { if (evt.code !== 'KeyG' || evt.repeat || typing(evt) || this.el.sceneEl.is('vr-mode')) return; if (this.hand) this.letGo(); else this.grabLooked(); }); on(window, 'wheel', (evt) => { if (!this.hand || this.hand.el) return; evt.preventDefault(); this.resize(this.hand.screen, this.hand.screen.width * Math.exp(-evt.deltaY * 0.0015)); }, { passive: false }); if (typeof NAF !== 'undefined' && NAF.connection) { NAF.connection.subscribeToDataChannel('screen', (from, type, msg) => this.onMessage(from, msg)); on(document.body, 'clientConnected', (evt) => this.catchUp(evt.detail.clientId)); on(document.body, 'clientDisconnected', (evt) => this.drop(evt.detail.clientId)); } }, remove() { for (const [target, type, fn, opts] of this.listeners) target.removeEventListener(type, fn, opts); for (const id of [...this.screens.keys()]) this.drop(id); }, // "Ann's screen", for the list of who's here. labels() { return [...this.screens.values()].map((s) => `${s.name}'s screen`); }, // ---- screens coming and going ---------------------------------------------- onMessage(from, msg) { if (!msg || typeof from !== 'string') return; if (msg.op === 'on') this.show(from, cleanName(msg.name)); else if (msg.op === 'off') this.drop(from); else if (msg.op === 'pose') this.onPose(msg); }, show(id, name) { let screen = this.screens.get(id); if (screen) { if (screen.name !== name) { screen.name = name; screen.label.setAttribute('text', 'value', `${name}'s screen`); } return; } if (this.screens.size >= MAX_SCREENS) return; screen = this.build(id, name); this.screens.set(id, screen); this.placeNew(screen); const early = this.pending.get(id); this.pending.delete(id); if (early) this.onPose(early); this.el.emit('screens-changed', null, false); }, build(id, name) { const el = document.createElement('a-entity'); el.classList.add('shared-screen'); this.el.sceneEl.appendChild(el); const group = el.object3D; const frame = new THREE.Mesh(new THREE.PlaneGeometry(1, 1), new THREE.MeshBasicMaterial({ color: '#0b0b0d', side: THREE.DoubleSide })); frame.position.z = -0.005; const picture = new THREE.Mesh(new THREE.PlaneGeometry(1, 1), new THREE.MeshBasicMaterial({ color: '#000000', toneMapped: false })); group.add(frame, picture); const label = document.createElement('a-entity'); label.setAttribute('text', { value: `${name}'s screen`, align: 'center', color: '#c8c8c8', width: 2, wrapCount: 30 }); const waiting = document.createElement('a-entity'); waiting.setAttribute('text', { value: 'Waiting for the picture…', align: 'center', color: '#9a9a9a', width: 2, wrapCount: 30 }); waiting.object3D.position.z = 0.01; el.append(label, waiting); const video = document.createElement('video'); video.muted = true; // the sound plays from the screen's spot instead (below) video.playsInline = true; video.autoplay = true; const texture = new THREE.VideoTexture(video); texture.colorSpace = THREE.SRGBColorSpace; const screen = { id, name, el, group, frame, picture, label, waiting, video, texture, sound: null, tracks: { video: null, audio: null }, width: DEFAULT_WIDTH, aspect: 16 / 9, yaw: 0, v: 0, by: null, mine: false, target: null, }; video.addEventListener('resize', () => this.fit(screen)); video.addEventListener('playing', () => { picture.material.map = texture; picture.material.color.set('#ffffff'); picture.material.needsUpdate = true; waiting.object3D.visible = false; this.fit(screen); }); this.fit(screen); return screen; }, drop(id) { this.pending.delete(id); const screen = this.screens.get(id); if (!screen) return; if (this.hand && this.hand.screen === screen) { this.hand = null; this.stretch = null; } if (screen.sound) { try { screen.sound.disconnect(); } catch (e) { /* never connected */ } } screen.video.srcObject = null; screen.texture.dispose(); for (const mesh of [screen.frame, screen.picture]) { mesh.geometry.dispose(); mesh.material.dispose(); } screen.el.remove(); this.screens.delete(id); this.el.emit('screens-changed', null, false); }, // Size the picture to the shared video's shape, with the name on the // border below it, where it reads in any light. fit(screen) { const { videoWidth: w, videoHeight: h } = screen.video; if (w && h) screen.aspect = THREE.MathUtils.clamp(w / h, 0.3, 4); const height = screen.width / screen.aspect; screen.picture.scale.set(screen.width, height, 1); screen.frame.scale.set(screen.width + 2 * FRAME, height + FRAME + CHIN, 1); screen.frame.position.y = -(CHIN - FRAME) / 2; screen.label.object3D.position.set(0, -height / 2 - CHIN / 2, 0.003); screen.label.setAttribute('text', 'width', THREE.MathUtils.clamp(screen.width * 0.45, 0.9, 2.4)); screen.waiting.setAttribute('text', 'width', THREE.MathUtils.clamp(screen.width * 0.6, 1, 3)); }, height(screen) { return screen.width / screen.aspect; }, // Where a screen first appears: the world's `screen` spot (see world.js), // or a few metres ahead of where people arrive. More screens line up beside // it, in the same order for everyone. placeNew(screen) { const world = typeof CurrentWorld === 'function' ? CurrentWorld(this.el.sceneEl) : null; let spot = world && world.screen; if (!spot) { const s = (world && world.spawn) || { x: 0, z: 0, yaw: 0 }; spot = { x: s.x - Math.sin(s.yaw) * 4, y: 1.8, z: s.z - Math.cos(s.yaw) * 4, yaw: s.yaw }; } screen.width = clampWidth(spot.width || DEFAULT_WIDTH); const rank = [...this.screens.keys()].sort().indexOf(screen.id); const step = rank === 0 ? 0 : (rank % 2 ? 1 : -1) * Math.ceil(rank / 2) * (screen.width + 0.4); screen.yaw = spot.yaw; screen.group.position.set(spot.x + Math.cos(spot.yaw) * step, spot.y, spot.z - Math.sin(spot.yaw) * step); screen.group.rotation.set(0, spot.yaw, 0); this.fit(screen); }, // ---- the sharer's video and sound ----------------------------------------- bind(screen) { const found = incoming(screen.id); if (found.video === screen.tracks.video && found.audio === screen.tracks.audio) return; screen.tracks = found; const tracks = [found.video, found.audio].filter(Boolean); // Chrome only lets remote sound into Web Audio while a media element is // playing the stream too, so the (muted) video element gets both. screen.video.srcObject = tracks.length ? new MediaStream(tracks) : null; if (tracks.length) screen.video.play().catch(() => {}); if (screen.sound) { try { screen.sound.disconnect(); } catch (e) { /* never connected */ } } if (found.audio) { if (!screen.sound) { screen.sound = new THREE.PositionalAudio(this.listener()); screen.sound.setDistanceModel('inverse'); screen.sound.setRefDistance(3); screen.sound.setRolloffFactor(0.6); screen.group.add(screen.sound); } screen.sound.setMediaStreamSource(new MediaStream([found.audio])); } }, listener() { const sceneEl = this.el.sceneEl; if (!sceneEl.audioListener) { sceneEl.audioListener = new THREE.AudioListener(); if (sceneEl.camera) sceneEl.camera.add(sceneEl.audioListener); sceneEl.addEventListener('camera-set-active', (e) => e.detail.cameraEl.getObject3D('camera').add(sceneEl.audioListener)); } return sceneEl.audioListener; }, // ---- moving and resizing --------------------------------------------------- myId() { return online() ? NAF.clientId : 'me'; }, // The nearest screen along a ray, and where it hits in the screen's own // coordinates (x right, y up from the middle). aim(origin, dir) { let best = null; for (const screen of this.screens.values()) { const g = screen.group; const normal = v1.set(Math.sin(screen.yaw), 0, Math.cos(screen.yaw)); const denom = dir.dot(normal); if (Math.abs(denom) < 1e-4) continue; const t = v2.subVectors(g.position, origin).dot(normal) / denom; if (t < 0 || t > REACH || (best && t > best.dist)) continue; const hit = v3.copy(origin).addScaledVector(dir, t).sub(g.position); const ox = hit.x * Math.cos(screen.yaw) - hit.z * Math.sin(screen.yaw); const oy = hit.y; if (Math.abs(ox) > screen.width / 2 + EDGE || Math.abs(oy) > this.height(screen) / 2 + EDGE) continue; best = { screen, dist: t, ox, oy }; } return best; }, ray(el, origin, dir) { el.object3D.updateWorldMatrix(true, false); el.object3D.getWorldPosition(origin); dir.set(0, 0, -1).applyQuaternion(el.object3D.getWorldQuaternion(q1)); return [origin, dir]; }, take(grab, el) { const screen = grab.screen; if (screen.by && screen.by !== this.myId()) { this.el.emit('screen-busy', { name: screen.name }, false); return; } this.hand = { ...grab, el }; screen.mine = true; screen.by = null; screen.target = null; screen.v++; this.sendPose(screen, true); }, grabLooked() { const [origin, dir] = this.ray(this.data.head, new THREE.Vector3(), new THREE.Vector3()); const grab = this.aim(origin, dir); if (grab) this.take(grab, null); }, gripDown(el) { if (!this.el.sceneEl.is('vr-mode') || !this.screens.size) return; const wield = this.el.components.wield; const held = wield && wield.hands.find((h) => h.el === el); if (held && held.tool) return; // it picked up a pen instead const board = this.el.sceneEl.querySelector('[chess]'); if (board && board.components.chess && board.components.chess.holds(el)) return; // or a chess piece if (this.hand && this.hand.el && this.hand.el !== el) { // The other hand joins in: resize by how far apart your hands are. const other = this.hand.el.object3D.getWorldPosition(v1); const from = el.object3D.getWorldPosition(v2).distanceTo(other); this.stretch = { el, from: Math.max(from, 0.05), width: this.hand.screen.width }; return; } if (this.hand) return; const [origin, dir] = this.ray(el, new THREE.Vector3(), new THREE.Vector3()); const grab = this.aim(origin, dir); if (grab) this.take(grab, el); if (this.hand) this.buzz(el); }, gripUp(el) { if (this.stretch && this.stretch.el === el) this.stretch = null; else if (this.hand && this.hand.el === el) this.letGo(); }, letGo() { const hand = this.hand; if (!hand) return; this.hand = null; this.stretch = null; hand.screen.mine = false; hand.screen.v++; this.sendPose(hand.screen, true); }, resize(screen, width) { screen.width = clampWidth(width); this.fit(screen); }, // Follows your hand (or gaze) at the distance you grabbed it, upright and // turned to face you, and never into the floor. move() { const hand = this.hand; const screen = hand.screen; const [origin, dir] = this.ray(hand.el || this.data.head, v1, v2); if (this.stretch) { const apart = this.stretch.el.object3D.getWorldPosition(v3).distanceTo(origin); this.resize(screen, this.stretch.width * (apart / this.stretch.from)); } const point = v3.copy(origin).addScaledVector(dir, hand.dist); const g = screen.group; const sx = Math.cos(screen.yaw); const sz = -Math.sin(screen.yaw); g.position.set(point.x - sx * hand.ox, point.y - hand.oy, point.z - sz * hand.ox); const head = this.data.head.object3D.getWorldPosition(v1); screen.yaw = Math.atan2(head.x - g.position.x, head.z - g.position.z); g.rotation.set(0, screen.yaw, 0); const world = typeof CurrentWorld === 'function' ? CurrentWorld(this.el.sceneEl) : null; const floor = world && world.floorAt ? world.floorAt(g.position.x, g.position.z) : 0; g.position.y = Math.max(g.position.y, floor + this.height(screen) / 2 + CHIN + 0.2); }, buzz(el) { try { const tracked = el.components['tracked-controls']; const pad = tracked && tracked.controller && tracked.controller.gamepad; const motor = pad && pad.hapticActuators && pad.hapticActuators[0]; if (motor && motor.pulse) Promise.resolve(motor.pulse(0.4, 40)).catch(() => {}); } catch (e) { // No haptics on this controller. } }, // ---- syncing where screens are --------------------------------------------- sendPose(screen, guaranteed, to) { if (!online()) return; const p = screen.group.position; const msg = { op: 'pose', s: screen.id, v: screen.v, by: screen.mine ? this.myId() : null, p: [round(p.x), round(p.y), round(p.z)], yaw: round(screen.yaw), w: round(screen.width), }; if (to) NAF.connection.sendDataGuaranteed(to, 'screen', msg); else if (guaranteed) NAF.connection.broadcastDataGuaranteed('screen', msg); else NAF.connection.broadcastData('screen', msg); }, onPose(msg) { if (typeof msg.s !== 'string' || msg.s.length > 40 || !Number.isInteger(msg.v) || !finite(msg.p, 3)) return; if (!Number.isFinite(msg.yaw) || !Number.isFinite(msg.w)) return; const screen = this.screens.get(msg.s); if (!screen) { // Heard where it is before hearing it's there; keep that for a moment. if (this.pending.size < MAX_SCREENS * 2) this.pending.set(msg.s, msg); return; } const by = typeof msg.by === 'string' ? msg.by : null; const current = screen.mine ? this.myId() : screen.by; if (msg.v < screen.v) return; // Two people grabbed it at once: the lower id keeps it, the same for everyone. if (msg.v === screen.v && by !== current && (by || '￿') > (current || '￿')) return; screen.v = msg.v; if (screen.mine && by !== this.myId()) { this.hand = null; this.stretch = null; screen.mine = false; } if (screen.mine) return; screen.by = by; screen.target = { p: new THREE.Vector3().fromArray(msg.p), yaw: msg.yaw, w: clampWidth(msg.w) }; }, // Tells someone who just arrived where the screens have been moved to. catchUp(clientId) { for (const screen of this.screens.values()) if (screen.v > 0) this.sendPose(screen, true, clientId); }, // ---- every frame ----------------------------------------------------------- tick(time, delta) { if (!this.screens.size) return; if (time - this.lastCheck >= CHECK_MS) { this.lastCheck = time; for (const screen of this.screens.values()) this.bind(screen); } const ease = 1 - Math.exp(-(Math.min(delta, 100) / 1000) / SMOOTH); for (const screen of this.screens.values()) { const target = screen.target; if (!target || screen.mine) continue; const g = screen.group; g.position.lerp(target.p, ease); let turn = target.yaw - screen.yaw; turn = Math.atan2(Math.sin(turn), Math.cos(turn)); screen.yaw += turn * ease; g.rotation.set(0, screen.yaw, 0); if (Math.abs(target.w - screen.width) > 1e-3) this.resize(screen, screen.width + (target.w - screen.width) * ease); if (!screen.by && g.position.distanceToSquared(target.p) < 1e-8 && Math.abs(turn) < 1e-4) screen.target = null; } if (this.hand) { this.move(); if (time - this.lastPose >= POSE_MS) { this.lastPose = time; this.sendPose(this.hand.screen, false); } } }, }); })();