Zoom-lite with per-speaker transcription

import { Agent, callable, type Connection, type WSMessage } from "agents";
import type { GeneratedEnv } from "@ayjnt/env";

type Participant = {
  id: string;             // client-minted, shared with the user's Transcriber
  displayName: string;
  joinedAt: number;
  muted: boolean;
  cameraOn: boolean;
  screenSharing: boolean;
};

type TranscriptEntry = {
  id: string;
  participantId: string;
  displayName: string;
  text: string;
  at: number;
};

type State = { participants: Participant[]; transcript: TranscriptEntry[] };

export default class ConferenceRoom extends Agent<GeneratedEnv, State> {
  override initialState: State = { participants: [], transcript: [] };

  override async onMessage(conn: Connection, message: WSMessage) {
    // Audio doesn't flow through this agent at all — it lives in the
    // Transcriber. Drop any stray binary frames as a safety net.
    if (typeof message !== "string") return;
    const frame = JSON.parse(message);
    // ... handle hello / media-state / webrtc relay ...
  }

  /** Inter-agent RPC entry point — called by Transcriber DOs.
   *  Returns void; errors propagate through the await on the other side. */
  async recordUtterance(participantId: string, text: string): Promise<void> {
    const participant = this.state.participants.find((p) => p.id === participantId);
    if (!participant) return; // unknown speaker — drop
    this.setState({
      ...this.state,
      transcript: [
        ...this.state.transcript,
        { id: crypto.randomUUID(), participantId, displayName: participant.displayName, text, at: Date.now() },
      ].slice(-200),
    });
  }

  @callable({ description: "Clear the conversation transcript." })
  async clearTranscript() {
    this.setState({ ...this.state, transcript: [] });
  }
}

import { Agent, type Connection, type WSMessage } from "agents";
import { getAgent } from "ayjnt/rpc";
import { WorkersAIFluxSTT, type TranscriberSession } from "@cloudflare/voice";
import type { GeneratedEnv } from "@ayjnt/env";
import type ConferenceRoom from "../room/agent.ts";

type ConnState = {
  roomId: string | null;
  participantId: string | null;
  displayName: string | null;
};

export default class Transcriber extends Agent<GeneratedEnv> {
  private sessions = new Map<string, TranscriberSession>();

  override async onConnect(conn: Connection) {
    conn.setState({ roomId: null, participantId: null, displayName: null });
  }

  override async onMessage(conn: Connection, message: WSMessage) {
    if (message instanceof ArrayBuffer) {
      this.sessions.get(conn.id)?.feed(message);
      return;
    }
    if (typeof message !== "string") return;
    const { kind, roomId, participantId, displayName } = JSON.parse(message);
    if (kind !== "bind") return;

    conn.setState({ roomId, participantId, displayName });

    // One Whisper session per WebSocket connection. onUtterance fires
    // once the model finalizes a turn → RPC to the room.
    const session = new WorkersAIFluxSTT(this.env.AI).createSession({
      language: "en",
      onUtterance: async (text: string) => {
        const room = await getAgent<ConferenceRoom>(
          this.env.CONFERENCE_ROOM,
          roomId,
        );
        await room.recordUtterance(participantId, text);
      },
    });
    this.sessions.set(conn.id, session);
  }

  override async onClose(conn: Connection) {
    this.sessions.get(conn.id)?.close();
    this.sessions.delete(conn.id);
  }
}

import { useAgent } from "@ayjnt/room";
import { startAudioCapture } from "./audio-capture.ts";

export default function ConferenceUI() {
  const agent = useAgent();                         // WS #1 → room
  const participantId = useMemo(() => crypto.randomUUID(), []);

  const onJoin = async (name: string) => {
    const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: true });

    // Tell the room who we are — it'll see this participantId everywhere.
    agent.send(JSON.stringify({ kind: "hello", participantId, displayName: name }));

    // Open WS #2 → this user's own Transcriber DO.
    const ws = new WebSocket(`wss://${location.host}/transcriber/${participantId}`);
    ws.addEventListener("open", () => {
      ws.send(JSON.stringify({ kind: "bind", roomId: agent.name, participantId, displayName: name }));
    });

    // Mic frames → Transcriber (NOT room). The Transcriber feeds them
    // to Whisper, then RPCs the room with the finalized utterance.
    const micTrack = stream.getAudioTracks()[0];
    await startAudioCapture(micTrack, (pcm) => ws.send(pcm));
  };
  // ... peer-mesh setup, video tiles, transcript pane ...
}