path: root/public/js/components/screens.js

/* global AFRAME, THREE, NAF, CurrentWorld */
// Shared screens. Someone joins a room as "My screen" from a computer (see
// share.js), and what they share shows up here as a big screen everyone sees,
// with its sound coming from it. Anyone can move one: in VR point a hand at it
// and squeeze grip, then squeeze the other grip too and move your hands apart
// or together to resize it; on desktop look at it and press G, scroll to
// resize, and press G again to put it down. A screen stays upright and turns to
// face whoever is moving it. Where screens are is synced like wield.js's tools.
//
// The sharer's connection carries nothing else (no voice), so whatever video
// and sound arrive on it are the screen's. They're read straight off the
// peer connection rather than through EasyRTC's stream names, which can arrive
// after the media does.
(function () {
  const REACH = 25;          // how far away you can grab a screen (m)
  const EDGE = 0.15;         // grabbing just outside the picture still counts
  const DEFAULT_WIDTH = 3;
  const MIN_WIDTH = 0.6;
  const MAX_WIDTH = 12;
  const FRAME = 0.04;        // dark border around the picture
  const CHIN = 0.16;         // the border's bottom edge, with the name on it
  const POSE_MS = 66;        // how often a moving screen's pose is sent
  const CHECK_MS = 1000;     // how often to look for the sharer's video and sound
  const SMOOTH = 0.06;       // seconds to ease toward someone else's move
  const MAX_SCREENS = 8;

  const v1 = new THREE.Vector3();
  const v2 = new THREE.Vector3();
  const v3 = new THREE.Vector3();
  const q1 = new THREE.Quaternion();

  const online = () => typeof NAF !== 'undefined' && NAF.connection && NAF.connection.isConnected();
  const finite = (a, n) => Array.isArray(a) && a.length === n && a.every((x) => Number.isFinite(x) && Math.abs(x) < 1000);
  const round = (x) => Math.round(x * 1e4) / 1e4;
  const clampWidth = (w) => THREE.MathUtils.clamp(w, MIN_WIDTH, MAX_WIDTH);

  function cleanName(value) {
    return String(value || '').replace(/[\u0000-\u001f\u007f-\u009f]/g, '').trim().slice(0, 20) || 'Someone';
  }

  // The video and sound coming in from a sharer, if any yet.
  function incoming(clientId) {
    const adapter = NAF.connection.adapter;
    const pc = adapter && adapter.easyrtc && adapter.easyrtc.getPeerConnectionByUserId(clientId);
    const found = { video: null, audio: null };
    if (!pc || !pc.getTransceivers) return found;
    for (const t of pc.getTransceivers()) {
      const dir = t.currentDirection;
      if (dir !== 'recvonly' && dir !== 'sendrecv') continue;
      const track = t.receiver && t.receiver.track;
      if (track && track.readyState === 'live') found[track.kind] = track;
    }
    return found;
  }

  AFRAME.registerComponent('screens', {
    schema: {
      head: { type: 'selector', default: '#player' },
      leftHand: { type: 'selector', default: '#left-hand' },
      rightHand: { type: 'selector', default: '#right-hand' },
    },

    init() {
      this.screens = new Map();  // sharer's client id → screen
      this.pending = new Map();  // poses heard before the screen itself
      this.lastPose = 0;
      this.lastCheck = 0;
      this.hand = null;          // what's moving a screen of yours: { screen, el?, dist, ox, oy }
      this.stretch = null;       // your other hand while resizing: { el, from, width }

      this.listeners = [];
      const on = (target, type, fn, opts) => {
        target.addEventListener(type, fn, opts);
        this.listeners.push([target, type, fn, opts]);
      };

      for (const el of [this.data.leftHand, this.data.rightHand]) {
        // After everyone else's listeners, so a pen grabbed by wield.js wins.
        on(el, 'gripdown', () => Promise.resolve().then(() => this.gripDown(el)));
        on(el, 'gripup', () => this.gripUp(el));
        on(el, 'controllerdisconnected', () => this.gripUp(el));
      }
      on(this.el.sceneEl, 'enter-vr', () => this.letGo());
      on(this.el.sceneEl, 'exit-vr', () => this.letGo());

      const typing = (evt) => evt.target.closest && evt.target.closest('input, textarea, select');
      on(window, 'keydown', (evt) => {
        if (evt.code !== 'KeyG' || evt.repeat || typing(evt) || this.el.sceneEl.is('vr-mode')) return;
        if (this.hand) this.letGo();
        else this.grabLooked();
      });
      on(window, 'wheel', (evt) => {
        if (!this.hand || this.hand.el) return;
        evt.preventDefault();
        this.resize(this.hand.screen, this.hand.screen.width * Math.exp(-evt.deltaY * 0.0015));
      }, { passive: false });

      if (typeof NAF !== 'undefined' && NAF.connection) {
        NAF.connection.subscribeToDataChannel('screen', (from, type, msg) => this.onMessage(from, msg));
        on(document.body, 'clientConnected', (evt) => this.catchUp(evt.detail.clientId));
        on(document.body, 'clientDisconnected', (evt) => this.drop(evt.detail.clientId));
      }
    },

    remove() {
      for (const [target, type, fn, opts] of this.listeners) target.removeEventListener(type, fn, opts);
      for (const id of [...this.screens.keys()]) this.drop(id);
    },

    // "Ann's screen", for the list of who's here.
    labels() {
      return [...this.screens.values()].map((s) => `${s.name}'s screen`);
    },

    // ---- screens coming and going ----------------------------------------------

    onMessage(from, msg) {
      if (!msg || typeof from !== 'string') return;
      if (msg.op === 'on') this.show(from, cleanName(msg.name));
      else if (msg.op === 'off') this.drop(from);
      else if (msg.op === 'pose') this.onPose(msg);
    },

    show(id, name) {
      let screen = this.screens.get(id);
      if (screen) {
        if (screen.name !== name) {
          screen.name = name;
          screen.label.setAttribute('text', 'value', `${name}'s screen`);
        }
        return;
      }
      if (this.screens.size >= MAX_SCREENS) return;
      screen = this.build(id, name);
      this.screens.set(id, screen);
      this.placeNew(screen);
      const early = this.pending.get(id);
      this.pending.delete(id);
      if (early) this.onPose(early);
      this.el.emit('screens-changed', null, false);
    },

    build(id, name) {
      const el = document.createElement('a-entity');
      el.classList.add('shared-screen');
      this.el.sceneEl.appendChild(el);
      const group = el.object3D;

      const frame = new THREE.Mesh(new THREE.PlaneGeometry(1, 1), new THREE.MeshBasicMaterial({ color: '#0b0b0d', side: THREE.DoubleSide }));
      frame.position.z = -0.005;
      const picture = new THREE.Mesh(new THREE.PlaneGeometry(1, 1), new THREE.MeshBasicMaterial({ color: '#000000', toneMapped: false }));
      group.add(frame, picture);

      const label = document.createElement('a-entity');
      label.setAttribute('text', { value: `${name}'s screen`, align: 'center', color: '#c8c8c8', width: 2, wrapCount: 30 });
      const waiting = document.createElement('a-entity');
      waiting.setAttribute('text', { value: 'Waiting for the picture…', align: 'center', color: '#9a9a9a', width: 2, wrapCount: 30 });
      waiting.object3D.position.z = 0.01;
      el.append(label, waiting);

      const video = document.createElement('video');
      video.muted = true;  // the sound plays from the screen's spot instead (below)
      video.playsInline = true;
      video.autoplay = true;
      const texture = new THREE.VideoTexture(video);
      texture.colorSpace = THREE.SRGBColorSpace;

      const screen = {
        id, name, el, group, frame, picture, label, waiting, video, texture,
        sound: null, tracks: { video: null, audio: null },
        width: DEFAULT_WIDTH, aspect: 16 / 9, yaw: 0,
        v: 0, by: null, mine: false, target: null,
      };
      video.addEventListener('resize', () => this.fit(screen));
      video.addEventListener('playing', () => {
        picture.material.map = texture;
        picture.material.color.set('#ffffff');
        picture.material.needsUpdate = true;
        waiting.object3D.visible = false;
        this.fit(screen);
      });
      this.fit(screen);
      return screen;
    },

    drop(id) {
      this.pending.delete(id);
      const screen = this.screens.get(id);
      if (!screen) return;
      if (this.hand && this.hand.screen === screen) {
        this.hand = null;
        this.stretch = null;
      }
      if (screen.sound) {
        try { screen.sound.disconnect(); } catch (e) { /* never connected */ }
      }
      screen.video.srcObject = null;
      screen.texture.dispose();
      for (const mesh of [screen.frame, screen.picture]) {
        mesh.geometry.dispose();
        mesh.material.dispose();
      }
      screen.el.remove();
      this.screens.delete(id);
      this.el.emit('screens-changed', null, false);
    },

    // Size the picture to the shared video's shape, with the name on the
    // border below it, where it reads in any light.
    fit(screen) {
      const { videoWidth: w, videoHeight: h } = screen.video;
      if (w && h) screen.aspect = THREE.MathUtils.clamp(w / h, 0.3, 4);
      const height = screen.width / screen.aspect;
      screen.picture.scale.set(screen.width, height, 1);
      screen.frame.scale.set(screen.width + 2 * FRAME, height + FRAME + CHIN, 1);
      screen.frame.position.y = -(CHIN - FRAME) / 2;
      screen.label.object3D.position.set(0, -height / 2 - CHIN / 2, 0.003);
      screen.label.setAttribute('text', 'width', THREE.MathUtils.clamp(screen.width * 0.45, 0.9, 2.4));
      screen.waiting.setAttribute('text', 'width', THREE.MathUtils.clamp(screen.width * 0.6, 1, 3));
    },

    height(screen) {
      return screen.width / screen.aspect;
    },

    // Where a screen first appears: the world's `screen` spot (see world.js),
    // or a few metres ahead of where people arrive. More screens line up beside
    // it, in the same order for everyone.
    placeNew(screen) {
      const world = typeof CurrentWorld === 'function' ? CurrentWorld(this.el.sceneEl) : null;
      let spot = world && world.screen;
      if (!spot) {
        const s = (world && world.spawn) || { x: 0, z: 0, yaw: 0 };
        spot = { x: s.x - Math.sin(s.yaw) * 4, y: 1.8, z: s.z - Math.cos(s.yaw) * 4, yaw: s.yaw };
      }
      screen.width = clampWidth(spot.width || DEFAULT_WIDTH);
      const rank = [...this.screens.keys()].sort().indexOf(screen.id);
      const step = rank === 0 ? 0 : (rank % 2 ? 1 : -1) * Math.ceil(rank / 2) * (screen.width + 0.4);
      screen.yaw = spot.yaw;
      screen.group.position.set(spot.x + Math.cos(spot.yaw) * step, spot.y, spot.z - Math.sin(spot.yaw) * step);
      screen.group.rotation.set(0, spot.yaw, 0);
      this.fit(screen);
    },

    // ---- the sharer's video and sound -----------------------------------------

    bind(screen) {
      const found = incoming(screen.id);
      if (found.video === screen.tracks.video && found.audio === screen.tracks.audio) return;
      screen.tracks = found;
      const tracks = [found.video, found.audio].filter(Boolean);
      // Chrome only lets remote sound into Web Audio while a media element is
      // playing the stream too, so the (muted) video element gets both.
      screen.video.srcObject = tracks.length ? new MediaStream(tracks) : null;
      if (tracks.length) screen.video.play().catch(() => {});
      if (screen.sound) {
        try { screen.sound.disconnect(); } catch (e) { /* never connected */ }
      }
      if (found.audio) {
        if (!screen.sound) {
          screen.sound = new THREE.PositionalAudio(this.listener());
          screen.sound.setDistanceModel('inverse');
          screen.sound.setRefDistance(3);
          screen.sound.setRolloffFactor(0.6);
          screen.group.add(screen.sound);
        }
        screen.sound.setMediaStreamSource(new MediaStream([found.audio]));
      }
    },

    listener() {
      const sceneEl = this.el.sceneEl;
      if (!sceneEl.audioListener) {
        sceneEl.audioListener = new THREE.AudioListener();
        if (sceneEl.camera) sceneEl.camera.add(sceneEl.audioListener);
        sceneEl.addEventListener('camera-set-active', (e) => e.detail.cameraEl.getObject3D('camera').add(sceneEl.audioListener));
      }
      return sceneEl.audioListener;
    },

    // ---- moving and resizing ---------------------------------------------------

    myId() {
      return online() ? NAF.clientId : 'me';
    },

    // The nearest screen along a ray, and where it hits in the screen's own
    // coordinates (x right, y up from the middle).
    aim(origin, dir) {
      let best = null;
      for (const screen of this.screens.values()) {
        const g = screen.group;
        const normal = v1.set(Math.sin(screen.yaw), 0, Math.cos(screen.yaw));
        const denom = dir.dot(normal);
        if (Math.abs(denom) < 1e-4) continue;
        const t = v2.subVectors(g.position, origin).dot(normal) / denom;
        if (t < 0 || t > REACH || (best && t > best.dist)) continue;
        const hit = v3.copy(origin).addScaledVector(dir, t).sub(g.position);
        const ox = hit.x * Math.cos(screen.yaw) - hit.z * Math.sin(screen.yaw);
        const oy = hit.y;
        if (Math.abs(ox) > screen.width / 2 + EDGE || Math.abs(oy) > this.height(screen) / 2 + EDGE) continue;
        best = { screen, dist: t, ox, oy };
      }
      return best;
    },

    ray(el, origin, dir) {
      el.object3D.updateWorldMatrix(true, false);
      el.object3D.getWorldPosition(origin);
      dir.set(0, 0, -1).applyQuaternion(el.object3D.getWorldQuaternion(q1));
      return [origin, dir];
    },

    take(grab, el) {
      const screen = grab.screen;
      if (screen.by && screen.by !== this.myId()) {
        this.el.emit('screen-busy', { name: screen.name }, false);
        return;
      }
      this.hand = { ...grab, el };
      screen.mine = true;
      screen.by = null;
      screen.target = null;
      screen.v++;
      this.sendPose(screen, true);
    },

    grabLooked() {
      const [origin, dir] = this.ray(this.data.head, new THREE.Vector3(), new THREE.Vector3());
      const grab = this.aim(origin, dir);
      if (grab) this.take(grab, null);
    },

    gripDown(el) {
      if (!this.el.sceneEl.is('vr-mode') || !this.screens.size) return;
      const wield = this.el.components.wield;
      const held = wield && wield.hands.find((h) => h.el === el);
      if (held && held.tool) return;  // it picked up a pen instead
      const board = this.el.sceneEl.querySelector('[chess]');
      if (board && board.components.chess && board.components.chess.holds(el)) return;  // or a chess piece
      if (this.hand && this.hand.el && this.hand.el !== el) {
        // The other hand joins in: resize by how far apart your hands are.
        const other = this.hand.el.object3D.getWorldPosition(v1);
        const from = el.object3D.getWorldPosition(v2).distanceTo(other);
        this.stretch = { el, from: Math.max(from, 0.05), width: this.hand.screen.width };
        return;
      }
      if (this.hand) return;
      const [origin, dir] = this.ray(el, new THREE.Vector3(), new THREE.Vector3());
      const grab = this.aim(origin, dir);
      if (grab) this.take(grab, el);
      if (this.hand) this.buzz(el);
    },

    gripUp(el) {
      if (this.stretch && this.stretch.el === el) this.stretch = null;
      else if (this.hand && this.hand.el === el) this.letGo();
    },

    letGo() {
      const hand = this.hand;
      if (!hand) return;
      this.hand = null;
      this.stretch = null;
      hand.screen.mine = false;
      hand.screen.v++;
      this.sendPose(hand.screen, true);
    },

    resize(screen, width) {
      screen.width = clampWidth(width);
      this.fit(screen);
    },

    // Follows your hand (or gaze) at the distance you grabbed it, upright and
    // turned to face you, and never into the floor.
    move() {
      const hand = this.hand;
      const screen = hand.screen;
      const [origin, dir] = this.ray(hand.el || this.data.head, v1, v2);
      if (this.stretch) {
        const apart = this.stretch.el.object3D.getWorldPosition(v3).distanceTo(origin);
        this.resize(screen, this.stretch.width * (apart / this.stretch.from));
      }
      const point = v3.copy(origin).addScaledVector(dir, hand.dist);
      const g = screen.group;
      const sx = Math.cos(screen.yaw);
      const sz = -Math.sin(screen.yaw);
      g.position.set(point.x - sx * hand.ox, point.y - hand.oy, point.z - sz * hand.ox);
      const head = this.data.head.object3D.getWorldPosition(v1);
      screen.yaw = Math.atan2(head.x - g.position.x, head.z - g.position.z);
      g.rotation.set(0, screen.yaw, 0);
      const world = typeof CurrentWorld === 'function' ? CurrentWorld(this.el.sceneEl) : null;
      const floor = world && world.floorAt ? world.floorAt(g.position.x, g.position.z) : 0;
      g.position.y = Math.max(g.position.y, floor + this.height(screen) / 2 + CHIN + 0.2);
    },

    buzz(el) {
      try {
        const tracked = el.components['tracked-controls'];
        const pad = tracked && tracked.controller && tracked.controller.gamepad;
        const motor = pad && pad.hapticActuators && pad.hapticActuators[0];
        if (motor && motor.pulse) Promise.resolve(motor.pulse(0.4, 40)).catch(() => {});
      } catch (e) {
        // No haptics on this controller.
      }
    },

    // ---- syncing where screens are ---------------------------------------------

    sendPose(screen, guaranteed, to) {
      if (!online()) return;
      const p = screen.group.position;
      const msg = {
        op: 'pose', s: screen.id, v: screen.v, by: screen.mine ? this.myId() : null,
        p: [round(p.x), round(p.y), round(p.z)], yaw: round(screen.yaw), w: round(screen.width),
      };
      if (to) NAF.connection.sendDataGuaranteed(to, 'screen', msg);
      else if (guaranteed) NAF.connection.broadcastDataGuaranteed('screen', msg);
      else NAF.connection.broadcastData('screen', msg);
    },

    onPose(msg) {
      if (typeof msg.s !== 'string' || msg.s.length > 40 || !Number.isInteger(msg.v) || !finite(msg.p, 3)) return;
      if (!Number.isFinite(msg.yaw) || !Number.isFinite(msg.w)) return;
      const screen = this.screens.get(msg.s);
      if (!screen) {
        // Heard where it is before hearing it's there; keep that for a moment.
        if (this.pending.size < MAX_SCREENS * 2) this.pending.set(msg.s, msg);
        return;
      }
      const by = typeof msg.by === 'string' ? msg.by : null;
      const current = screen.mine ? this.myId() : screen.by;
      if (msg.v < screen.v) return;
      // Two people grabbed it at once: the lower id keeps it, the same for everyone.
      if (msg.v === screen.v && by !== current && (by || '￿') > (current || '￿')) return;
      screen.v = msg.v;
      if (screen.mine && by !== this.myId()) {
        this.hand = null;
        this.stretch = null;
        screen.mine = false;
      }
      if (screen.mine) return;
      screen.by = by;
      screen.target = { p: new THREE.Vector3().fromArray(msg.p), yaw: msg.yaw, w: clampWidth(msg.w) };
    },

    // Tells someone who just arrived where the screens have been moved to.
    catchUp(clientId) {
      for (const screen of this.screens.values()) if (screen.v > 0) this.sendPose(screen, true, clientId);
    },

    // ---- every frame -----------------------------------------------------------

    tick(time, delta) {
      if (!this.screens.size) return;
      if (time - this.lastCheck >= CHECK_MS) {
        this.lastCheck = time;
        for (const screen of this.screens.values()) this.bind(screen);
      }
      const ease = 1 - Math.exp(-(Math.min(delta, 100) / 1000) / SMOOTH);
      for (const screen of this.screens.values()) {
        const target = screen.target;
        if (!target || screen.mine) continue;
        const g = screen.group;
        g.position.lerp(target.p, ease);
        let turn = target.yaw - screen.yaw;
        turn = Math.atan2(Math.sin(turn), Math.cos(turn));
        screen.yaw += turn * ease;
        g.rotation.set(0, screen.yaw, 0);
        if (Math.abs(target.w - screen.width) > 1e-3) this.resize(screen, screen.width + (target.w - screen.width) * ease);
        if (!screen.by && g.position.distanceToSquared(target.p) < 1e-8 && Math.abs(turn) < 1e-4) screen.target = null;
      }
      if (this.hand) {
        this.move();
        if (time - this.lastPose >= POSE_MS) {
          this.lastPose = time;
          this.sendPose(this.hand.screen, false);
        }
      }
    },
  });
})();