From e69ad01e700e80fa959dfee08cd556ca0d777ffd Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 1 Nov 2026 05:43:24 -0800 Subject: Yesod: self-hosted WebXR rooms Start a room in a world and hang out in VR from the headset's browser: A-Frame + Networked-A-Frame over EasyRTC, with a coturn relay that can hide everyone's IP address. Ships with the Campfire, Temple, Cabin, Balcony and Test room worlds. Co-Authored-By: Claude Opus 5.5 --- public/js/components/screens.js | 483 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 483 insertions(+) create mode 100644 public/js/components/screens.js (limited to 'public/js/components/screens.js') diff --git a/public/js/components/screens.js b/public/js/components/screens.js new file mode 100644 index 0000000..89586af --- /dev/null +++ b/public/js/components/screens.js @@ -0,0 +1,483 @@ +/* global AFRAME, THREE, NAF, CurrentWorld */ +// Shared screens. Someone joins a room as "My screen" from a computer (see +// share.js), and what they share shows up here as a big screen everyone sees, +// with its sound coming from it. Anyone can move one: in VR point a hand at it +// and squeeze grip, then squeeze the other grip too and move your hands apart +// or together to resize it; on desktop look at it and press G, scroll to +// resize, and press G again to put it down. A screen stays upright and turns to +// face whoever is moving it. Where screens are is synced like wield.js's tools. +// +// The sharer's connection carries nothing else (no voice), so whatever video +// and sound arrive on it are the screen's. They're read straight off the +// peer connection rather than through EasyRTC's stream names, which can arrive +// after the media does. +(function () { + const REACH = 25; // how far away you can grab a screen (m) + const EDGE = 0.15; // grabbing just outside the picture still counts + const DEFAULT_WIDTH = 3; + const MIN_WIDTH = 0.6; + const MAX_WIDTH = 12; + const FRAME = 0.04; // dark border around the picture + const CHIN = 0.16; // the border's bottom edge, with the name on it + const POSE_MS = 66; // how often a moving screen's pose is sent + const CHECK_MS = 1000; // how often to look for the sharer's video and sound + const SMOOTH = 0.06; // seconds to ease toward someone else's move + const MAX_SCREENS = 8; + + const v1 = new THREE.Vector3(); + const v2 = new THREE.Vector3(); + const v3 = new THREE.Vector3(); + const q1 = new THREE.Quaternion(); + + const online = () => typeof NAF !== 'undefined' && NAF.connection && NAF.connection.isConnected(); + const finite = (a, n) => Array.isArray(a) && a.length === n && a.every((x) => Number.isFinite(x) && Math.abs(x) < 1000); + const round = (x) => Math.round(x * 1e4) / 1e4; + const clampWidth = (w) => THREE.MathUtils.clamp(w, MIN_WIDTH, MAX_WIDTH); + + function cleanName(value) { + return String(value || '').replace(/[\u0000-\u001f\u007f-\u009f]/g, '').trim().slice(0, 20) || 'Someone'; + } + + // The video and sound coming in from a sharer, if any yet. + function incoming(clientId) { + const adapter = NAF.connection.adapter; + const pc = adapter && adapter.easyrtc && adapter.easyrtc.getPeerConnectionByUserId(clientId); + const found = { video: null, audio: null }; + if (!pc || !pc.getTransceivers) return found; + for (const t of pc.getTransceivers()) { + const dir = t.currentDirection; + if (dir !== 'recvonly' && dir !== 'sendrecv') continue; + const track = t.receiver && t.receiver.track; + if (track && track.readyState === 'live') found[track.kind] = track; + } + return found; + } + + AFRAME.registerComponent('screens', { + schema: { + head: { type: 'selector', default: '#player' }, + leftHand: { type: 'selector', default: '#left-hand' }, + rightHand: { type: 'selector', default: '#right-hand' }, + }, + + init() { + this.screens = new Map(); // sharer's client id → screen + this.pending = new Map(); // poses heard before the screen itself + this.lastPose = 0; + this.lastCheck = 0; + this.hand = null; // what's moving a screen of yours: { screen, el?, dist, ox, oy } + this.stretch = null; // your other hand while resizing: { el, from, width } + + this.listeners = []; + const on = (target, type, fn, opts) => { + target.addEventListener(type, fn, opts); + this.listeners.push([target, type, fn, opts]); + }; + + for (const el of [this.data.leftHand, this.data.rightHand]) { + // After everyone else's listeners, so a pen grabbed by wield.js wins. + on(el, 'gripdown', () => Promise.resolve().then(() => this.gripDown(el))); + on(el, 'gripup', () => this.gripUp(el)); + on(el, 'controllerdisconnected', () => this.gripUp(el)); + } + on(this.el.sceneEl, 'enter-vr', () => this.letGo()); + on(this.el.sceneEl, 'exit-vr', () => this.letGo()); + + const typing = (evt) => evt.target.closest && evt.target.closest('input, textarea, select'); + on(window, 'keydown', (evt) => { + if (evt.code !== 'KeyG' || evt.repeat || typing(evt) || this.el.sceneEl.is('vr-mode')) return; + if (this.hand) this.letGo(); + else this.grabLooked(); + }); + on(window, 'wheel', (evt) => { + if (!this.hand || this.hand.el) return; + evt.preventDefault(); + this.resize(this.hand.screen, this.hand.screen.width * Math.exp(-evt.deltaY * 0.0015)); + }, { passive: false }); + + if (typeof NAF !== 'undefined' && NAF.connection) { + NAF.connection.subscribeToDataChannel('screen', (from, type, msg) => this.onMessage(from, msg)); + on(document.body, 'clientConnected', (evt) => this.catchUp(evt.detail.clientId)); + on(document.body, 'clientDisconnected', (evt) => this.drop(evt.detail.clientId)); + } + }, + + remove() { + for (const [target, type, fn, opts] of this.listeners) target.removeEventListener(type, fn, opts); + for (const id of [...this.screens.keys()]) this.drop(id); + }, + + // "Ann's screen", for the list of who's here. + labels() { + return [...this.screens.values()].map((s) => `${s.name}'s screen`); + }, + + // ---- screens coming and going ---------------------------------------------- + + onMessage(from, msg) { + if (!msg || typeof from !== 'string') return; + if (msg.op === 'on') this.show(from, cleanName(msg.name)); + else if (msg.op === 'off') this.drop(from); + else if (msg.op === 'pose') this.onPose(msg); + }, + + show(id, name) { + let screen = this.screens.get(id); + if (screen) { + if (screen.name !== name) { + screen.name = name; + screen.label.setAttribute('text', 'value', `${name}'s screen`); + } + return; + } + if (this.screens.size >= MAX_SCREENS) return; + screen = this.build(id, name); + this.screens.set(id, screen); + this.placeNew(screen); + const early = this.pending.get(id); + this.pending.delete(id); + if (early) this.onPose(early); + this.el.emit('screens-changed', null, false); + }, + + build(id, name) { + const el = document.createElement('a-entity'); + el.classList.add('shared-screen'); + this.el.sceneEl.appendChild(el); + const group = el.object3D; + + const frame = new THREE.Mesh(new THREE.PlaneGeometry(1, 1), new THREE.MeshBasicMaterial({ color: '#0b0b0d', side: THREE.DoubleSide })); + frame.position.z = -0.005; + const picture = new THREE.Mesh(new THREE.PlaneGeometry(1, 1), new THREE.MeshBasicMaterial({ color: '#000000', toneMapped: false })); + group.add(frame, picture); + + const label = document.createElement('a-entity'); + label.setAttribute('text', { value: `${name}'s screen`, align: 'center', color: '#c8c8c8', width: 2, wrapCount: 30 }); + const waiting = document.createElement('a-entity'); + waiting.setAttribute('text', { value: 'Waiting for the picture…', align: 'center', color: '#9a9a9a', width: 2, wrapCount: 30 }); + waiting.object3D.position.z = 0.01; + el.append(label, waiting); + + const video = document.createElement('video'); + video.muted = true; // the sound plays from the screen's spot instead (below) + video.playsInline = true; + video.autoplay = true; + const texture = new THREE.VideoTexture(video); + texture.colorSpace = THREE.SRGBColorSpace; + + const screen = { + id, name, el, group, frame, picture, label, waiting, video, texture, + sound: null, tracks: { video: null, audio: null }, + width: DEFAULT_WIDTH, aspect: 16 / 9, yaw: 0, + v: 0, by: null, mine: false, target: null, + }; + video.addEventListener('resize', () => this.fit(screen)); + video.addEventListener('playing', () => { + picture.material.map = texture; + picture.material.color.set('#ffffff'); + picture.material.needsUpdate = true; + waiting.object3D.visible = false; + this.fit(screen); + }); + this.fit(screen); + return screen; + }, + + drop(id) { + this.pending.delete(id); + const screen = this.screens.get(id); + if (!screen) return; + if (this.hand && this.hand.screen === screen) { + this.hand = null; + this.stretch = null; + } + if (screen.sound) { + try { screen.sound.disconnect(); } catch (e) { /* never connected */ } + } + screen.video.srcObject = null; + screen.texture.dispose(); + for (const mesh of [screen.frame, screen.picture]) { + mesh.geometry.dispose(); + mesh.material.dispose(); + } + screen.el.remove(); + this.screens.delete(id); + this.el.emit('screens-changed', null, false); + }, + + // Size the picture to the shared video's shape, with the name on the + // border below it, where it reads in any light. + fit(screen) { + const { videoWidth: w, videoHeight: h } = screen.video; + if (w && h) screen.aspect = THREE.MathUtils.clamp(w / h, 0.3, 4); + const height = screen.width / screen.aspect; + screen.picture.scale.set(screen.width, height, 1); + screen.frame.scale.set(screen.width + 2 * FRAME, height + FRAME + CHIN, 1); + screen.frame.position.y = -(CHIN - FRAME) / 2; + screen.label.object3D.position.set(0, -height / 2 - CHIN / 2, 0.003); + screen.label.setAttribute('text', 'width', THREE.MathUtils.clamp(screen.width * 0.45, 0.9, 2.4)); + screen.waiting.setAttribute('text', 'width', THREE.MathUtils.clamp(screen.width * 0.6, 1, 3)); + }, + + height(screen) { + return screen.width / screen.aspect; + }, + + // Where a screen first appears: the world's `screen` spot (see world.js), + // or a few metres ahead of where people arrive. More screens line up beside + // it, in the same order for everyone. + placeNew(screen) { + const world = typeof CurrentWorld === 'function' ? CurrentWorld(this.el.sceneEl) : null; + let spot = world && world.screen; + if (!spot) { + const s = (world && world.spawn) || { x: 0, z: 0, yaw: 0 }; + spot = { x: s.x - Math.sin(s.yaw) * 4, y: 1.8, z: s.z - Math.cos(s.yaw) * 4, yaw: s.yaw }; + } + screen.width = clampWidth(spot.width || DEFAULT_WIDTH); + const rank = [...this.screens.keys()].sort().indexOf(screen.id); + const step = rank === 0 ? 0 : (rank % 2 ? 1 : -1) * Math.ceil(rank / 2) * (screen.width + 0.4); + screen.yaw = spot.yaw; + screen.group.position.set(spot.x + Math.cos(spot.yaw) * step, spot.y, spot.z - Math.sin(spot.yaw) * step); + screen.group.rotation.set(0, spot.yaw, 0); + this.fit(screen); + }, + + // ---- the sharer's video and sound ----------------------------------------- + + bind(screen) { + const found = incoming(screen.id); + if (found.video === screen.tracks.video && found.audio === screen.tracks.audio) return; + screen.tracks = found; + const tracks = [found.video, found.audio].filter(Boolean); + // Chrome only lets remote sound into Web Audio while a media element is + // playing the stream too, so the (muted) video element gets both. + screen.video.srcObject = tracks.length ? new MediaStream(tracks) : null; + if (tracks.length) screen.video.play().catch(() => {}); + if (screen.sound) { + try { screen.sound.disconnect(); } catch (e) { /* never connected */ } + } + if (found.audio) { + if (!screen.sound) { + screen.sound = new THREE.PositionalAudio(this.listener()); + screen.sound.setDistanceModel('inverse'); + screen.sound.setRefDistance(3); + screen.sound.setRolloffFactor(0.6); + screen.group.add(screen.sound); + } + screen.sound.setMediaStreamSource(new MediaStream([found.audio])); + } + }, + + listener() { + const sceneEl = this.el.sceneEl; + if (!sceneEl.audioListener) { + sceneEl.audioListener = new THREE.AudioListener(); + if (sceneEl.camera) sceneEl.camera.add(sceneEl.audioListener); + sceneEl.addEventListener('camera-set-active', (e) => e.detail.cameraEl.getObject3D('camera').add(sceneEl.audioListener)); + } + return sceneEl.audioListener; + }, + + // ---- moving and resizing --------------------------------------------------- + + myId() { + return online() ? NAF.clientId : 'me'; + }, + + // The nearest screen along a ray, and where it hits in the screen's own + // coordinates (x right, y up from the middle). + aim(origin, dir) { + let best = null; + for (const screen of this.screens.values()) { + const g = screen.group; + const normal = v1.set(Math.sin(screen.yaw), 0, Math.cos(screen.yaw)); + const denom = dir.dot(normal); + if (Math.abs(denom) < 1e-4) continue; + const t = v2.subVectors(g.position, origin).dot(normal) / denom; + if (t < 0 || t > REACH || (best && t > best.dist)) continue; + const hit = v3.copy(origin).addScaledVector(dir, t).sub(g.position); + const ox = hit.x * Math.cos(screen.yaw) - hit.z * Math.sin(screen.yaw); + const oy = hit.y; + if (Math.abs(ox) > screen.width / 2 + EDGE || Math.abs(oy) > this.height(screen) / 2 + EDGE) continue; + best = { screen, dist: t, ox, oy }; + } + return best; + }, + + ray(el, origin, dir) { + el.object3D.updateWorldMatrix(true, false); + el.object3D.getWorldPosition(origin); + dir.set(0, 0, -1).applyQuaternion(el.object3D.getWorldQuaternion(q1)); + return [origin, dir]; + }, + + take(grab, el) { + const screen = grab.screen; + if (screen.by && screen.by !== this.myId()) { + this.el.emit('screen-busy', { name: screen.name }, false); + return; + } + this.hand = { ...grab, el }; + screen.mine = true; + screen.by = null; + screen.target = null; + screen.v++; + this.sendPose(screen, true); + }, + + grabLooked() { + const [origin, dir] = this.ray(this.data.head, new THREE.Vector3(), new THREE.Vector3()); + const grab = this.aim(origin, dir); + if (grab) this.take(grab, null); + }, + + gripDown(el) { + if (!this.el.sceneEl.is('vr-mode') || !this.screens.size) return; + const wield = this.el.components.wield; + const held = wield && wield.hands.find((h) => h.el === el); + if (held && held.tool) return; // it picked up a pen instead + const board = this.el.sceneEl.querySelector('[chess]'); + if (board && board.components.chess && board.components.chess.holds(el)) return; // or a chess piece + if (this.hand && this.hand.el && this.hand.el !== el) { + // The other hand joins in: resize by how far apart your hands are. + const other = this.hand.el.object3D.getWorldPosition(v1); + const from = el.object3D.getWorldPosition(v2).distanceTo(other); + this.stretch = { el, from: Math.max(from, 0.05), width: this.hand.screen.width }; + return; + } + if (this.hand) return; + const [origin, dir] = this.ray(el, new THREE.Vector3(), new THREE.Vector3()); + const grab = this.aim(origin, dir); + if (grab) this.take(grab, el); + if (this.hand) this.buzz(el); + }, + + gripUp(el) { + if (this.stretch && this.stretch.el === el) this.stretch = null; + else if (this.hand && this.hand.el === el) this.letGo(); + }, + + letGo() { + const hand = this.hand; + if (!hand) return; + this.hand = null; + this.stretch = null; + hand.screen.mine = false; + hand.screen.v++; + this.sendPose(hand.screen, true); + }, + + resize(screen, width) { + screen.width = clampWidth(width); + this.fit(screen); + }, + + // Follows your hand (or gaze) at the distance you grabbed it, upright and + // turned to face you, and never into the floor. + move() { + const hand = this.hand; + const screen = hand.screen; + const [origin, dir] = this.ray(hand.el || this.data.head, v1, v2); + if (this.stretch) { + const apart = this.stretch.el.object3D.getWorldPosition(v3).distanceTo(origin); + this.resize(screen, this.stretch.width * (apart / this.stretch.from)); + } + const point = v3.copy(origin).addScaledVector(dir, hand.dist); + const g = screen.group; + const sx = Math.cos(screen.yaw); + const sz = -Math.sin(screen.yaw); + g.position.set(point.x - sx * hand.ox, point.y - hand.oy, point.z - sz * hand.ox); + const head = this.data.head.object3D.getWorldPosition(v1); + screen.yaw = Math.atan2(head.x - g.position.x, head.z - g.position.z); + g.rotation.set(0, screen.yaw, 0); + const world = typeof CurrentWorld === 'function' ? CurrentWorld(this.el.sceneEl) : null; + const floor = world && world.floorAt ? world.floorAt(g.position.x, g.position.z) : 0; + g.position.y = Math.max(g.position.y, floor + this.height(screen) / 2 + CHIN + 0.2); + }, + + buzz(el) { + try { + const tracked = el.components['tracked-controls']; + const pad = tracked && tracked.controller && tracked.controller.gamepad; + const motor = pad && pad.hapticActuators && pad.hapticActuators[0]; + if (motor && motor.pulse) Promise.resolve(motor.pulse(0.4, 40)).catch(() => {}); + } catch (e) { + // No haptics on this controller. + } + }, + + // ---- syncing where screens are --------------------------------------------- + + sendPose(screen, guaranteed, to) { + if (!online()) return; + const p = screen.group.position; + const msg = { + op: 'pose', s: screen.id, v: screen.v, by: screen.mine ? this.myId() : null, + p: [round(p.x), round(p.y), round(p.z)], yaw: round(screen.yaw), w: round(screen.width), + }; + if (to) NAF.connection.sendDataGuaranteed(to, 'screen', msg); + else if (guaranteed) NAF.connection.broadcastDataGuaranteed('screen', msg); + else NAF.connection.broadcastData('screen', msg); + }, + + onPose(msg) { + if (typeof msg.s !== 'string' || msg.s.length > 40 || !Number.isInteger(msg.v) || !finite(msg.p, 3)) return; + if (!Number.isFinite(msg.yaw) || !Number.isFinite(msg.w)) return; + const screen = this.screens.get(msg.s); + if (!screen) { + // Heard where it is before hearing it's there; keep that for a moment. + if (this.pending.size < MAX_SCREENS * 2) this.pending.set(msg.s, msg); + return; + } + const by = typeof msg.by === 'string' ? msg.by : null; + const current = screen.mine ? this.myId() : screen.by; + if (msg.v < screen.v) return; + // Two people grabbed it at once: the lower id keeps it, the same for everyone. + if (msg.v === screen.v && by !== current && (by || '￿') > (current || '￿')) return; + screen.v = msg.v; + if (screen.mine && by !== this.myId()) { + this.hand = null; + this.stretch = null; + screen.mine = false; + } + if (screen.mine) return; + screen.by = by; + screen.target = { p: new THREE.Vector3().fromArray(msg.p), yaw: msg.yaw, w: clampWidth(msg.w) }; + }, + + // Tells someone who just arrived where the screens have been moved to. + catchUp(clientId) { + for (const screen of this.screens.values()) if (screen.v > 0) this.sendPose(screen, true, clientId); + }, + + // ---- every frame ----------------------------------------------------------- + + tick(time, delta) { + if (!this.screens.size) return; + if (time - this.lastCheck >= CHECK_MS) { + this.lastCheck = time; + for (const screen of this.screens.values()) this.bind(screen); + } + const ease = 1 - Math.exp(-(Math.min(delta, 100) / 1000) / SMOOTH); + for (const screen of this.screens.values()) { + const target = screen.target; + if (!target || screen.mine) continue; + const g = screen.group; + g.position.lerp(target.p, ease); + let turn = target.yaw - screen.yaw; + turn = Math.atan2(Math.sin(turn), Math.cos(turn)); + screen.yaw += turn * ease; + g.rotation.set(0, screen.yaw, 0); + if (Math.abs(target.w - screen.width) > 1e-3) this.resize(screen, screen.width + (target.w - screen.width) * ease); + if (!screen.by && g.position.distanceToSquared(target.p) < 1e-8 && Math.abs(turn) < 1e-4) screen.target = null; + } + if (this.hand) { + this.move(); + if (time - this.lastPose >= POSE_MS) { + this.lastPose = time; + this.sendPose(this.hand.screen, false); + } + } + }, + }); +})(); -- cgit v1.2.3