Streaming audio

This room is in dual mode: each participant joins from their own device,
so a join token must request exactly 1 language.

import { RoomEvent, Track, type RemoteTrack } from "livekit-client"

const container = document.querySelector("#glot-audio")!

room
  .on(RoomEvent.TrackSubscribed, (track: RemoteTrack) => {
    if (track.kind !== Track.Kind.Audio) return
    container.appendChild(track.attach())
  })
  .on(RoomEvent.TrackUnsubscribed, (track: RemoteTrack) => {
    track.detach().forEach((el) => el.remove())
  })

import { Room } from "livekit-client"

const TRANSCRIPT_TOPIC = "glot.transcript"

const room = new Room()
let entries: TranscriptEntry[] = []

// Reads are chained, not run concurrently. A text-stream handler returns void, so the
// SDK does not await it — two overlapping readAll() promises can settle out of order and
// interleave one utterance's chunks into another's. Chaining costs nothing here: each
// frame is one small, self-contained stream.
let reads = Promise.resolve()

room.registerTextStreamHandler(TRANSCRIPT_TOPIC, (reader) => {
  reads = reads
    .then(async () => {
      const frame = parseTranscriptFrame(await reader.readAll())
      if (frame === null) return
      entries = applyTranscriptFrame(entries, frame)
      render(entries)
    })
    .catch((err) => {
      // An unreadable frame costs one caption; a rejection left on the chain would
      // poison every read after it.
      console.error("Failed to read a transcript frame", err)
    })
})

await room.connect(url, token)

const TRANSCRIPT_VERSION = 1

export function parseTranscriptFrame(payload: string): TranscriptFrame | null {
  let raw: unknown
  try {
    raw = JSON.parse(payload)
  } catch {
    return null
  }
  if (typeof raw !== "object" || raw === null) return null

  const { v, kind, segment, language, text } = raw as Record<string, unknown>
  if (v !== TRANSCRIPT_VERSION) return null
  if (typeof segment !== "string" || typeof language !== "string" || typeof text !== "string") {
    return null
  }

  if (kind === "partial") {
    return { kind, segment, language, text, last: (raw as { last?: unknown }).last === true }
  }
  if (kind === "segment") {
    const { source_language: sourceLanguage, source_text: sourceText } = raw as Record<string, unknown>
    // Both halves of the pair or nothing: half a pair renders as a caption that silently
    // drops the direction it claims to show.
    if (typeof sourceLanguage !== "string" || typeof sourceText !== "string") return null
    return { kind, segment, language, text, sourceLanguage, sourceText }
  }
  return null
}

type TranscriptStatus = "streaming" | "stopped" | "final"

type TranscriptEntry = {
  segment: string
  /** Language translated into. */
  language: string
  /** Accumulated while streaming, replaced wholesale when confirmed. */
  text: string
  /** Known only once the utterance is confirmed. */
  sourceLanguage: string | null
  sourceText: string | null
  status: TranscriptStatus
}

export function applyTranscriptFrame(
  entries: TranscriptEntry[],
  frame: TranscriptFrame,
): TranscriptEntry[] {
  const index = entries.findIndex((entry) => entry.segment === frame.segment)

  if (frame.kind === "segment") {
    const confirmed: TranscriptEntry = {
      segment: frame.segment,
      language: frame.language,
      text: frame.text,
      sourceLanguage: frame.sourceLanguage,
      sourceText: frame.sourceText,
      status: "final",
    }
    // Appended when unseen rather than dropped: partials can be lost across a reconnect,
    // and the confirmed text is the half worth showing.
    if (index === -1) return [...entries, confirmed]
    return entries.map((entry, i) => (i === index ? confirmed : entry))
  }

  if (index === -1) {
    return [
      ...entries,
      {
        segment: frame.segment,
        language: frame.language,
        text: frame.text,
        sourceLanguage: null,
        sourceText: null,
        status: frame.last ? "stopped" : "streaming",
      },
    ]
  }

  const current = entries[index]
  // A partial that arrives after its utterance was confirmed is stale — the frames raced,
  // or the translator re-sent one. Appending it would tack a fragment onto a finished caption.
  if (current.status === "final") return entries

  return entries.map((entry, i) =>
    i === index
      ? { ...current, text: current.text + frame.text, status: frame.last ? "stopped" : "streaming" }
      : entry,
  )
}