diff --git a/public/blog/webrtc-is-still-the-problem/context.png b/public/blog/webrtc-is-still-the-problem/context.png new file mode 100644 index 0000000..7db5f8b Binary files /dev/null and b/public/blog/webrtc-is-still-the-problem/context.png differ diff --git a/public/blog/webrtc-is-still-the-problem/huh.webm b/public/blog/webrtc-is-still-the-problem/huh.webm new file mode 100644 index 0000000..4a71769 Binary files /dev/null and b/public/blog/webrtc-is-still-the-problem/huh.webm differ diff --git a/public/blog/webrtc-is-still-the-problem/mics.png b/public/blog/webrtc-is-still-the-problem/mics.png new file mode 100644 index 0000000..687d32b Binary files /dev/null and b/public/blog/webrtc-is-still-the-problem/mics.png differ diff --git a/public/blog/webrtc-is-still-the-problem/navi.png b/public/blog/webrtc-is-still-the-problem/navi.png new file mode 100644 index 0000000..c222715 Binary files /dev/null and b/public/blog/webrtc-is-still-the-problem/navi.png differ diff --git a/public/blog/webrtc-is-still-the-problem/tokens.png b/public/blog/webrtc-is-still-the-problem/tokens.png new file mode 100644 index 0000000..a7d165f Binary files /dev/null and b/public/blog/webrtc-is-still-the-problem/tokens.png differ diff --git a/public/voice/ai.svg b/public/voice/ai.svg index 1ecc68e..cb9be89 100644 --- a/public/voice/ai.svg +++ b/public/voice/ai.svg @@ -1,9 +1,81 @@ - - - - - - - - + + + + + + + + + diff --git a/public/voice/garbled.svg b/public/voice/garbled.svg new file mode 100644 index 0000000..f5da8aa --- /dev/null +++ b/public/voice/garbled.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/public/voice/kraken-1.svg b/public/voice/kraken-1.svg new file mode 100644 index 0000000..28d1359 --- /dev/null +++ b/public/voice/kraken-1.svg @@ -0,0 +1,80 @@ + + + + + + + + + diff --git a/public/voice/kraken-2.svg b/public/voice/kraken-2.svg new file mode 100644 index 0000000..b46384b --- /dev/null +++ b/public/voice/kraken-2.svg @@ -0,0 +1,73 @@ + + + + + + + + + diff --git a/public/voice/you.svg b/public/voice/you.svg new file mode 100644 index 0000000..c952588 --- /dev/null +++ b/public/voice/you.svg @@ -0,0 +1,81 @@ + + + + + + + + + + diff --git a/src/components/meme.astro b/src/components/meme.astro new file mode 100644 index 0000000..4d30f87 --- /dev/null +++ b/src/components/meme.astro @@ -0,0 +1,47 @@ +--- +// A tiny looping meme clip. It autoplays muted (browsers block sound until you interact), and a +// click toggles the sound. With reduced motion it holds still and only plays while the sound is on. +interface Props { + src: string; + label: string; + class?: string; +} + +const { src, label, class: className } = Astro.props; +--- + + + + + diff --git a/src/components/voice/budget.tsx b/src/components/voice/budget.tsx new file mode 100644 index 0000000..e3e7324 --- /dev/null +++ b/src/components/voice/budget.tsx @@ -0,0 +1,74 @@ +// Two latency budgets: one upstream from your mic, fanned out to two subscribers that wait different amounts for lost audio. +import { Diagram, type Panel } from "./diagram"; +import { arrival, FLIGHT, FRAME, frames, has, type Segment, transmit } from "./sim"; +import { WORDS } from "./stt"; + +const AXIS = 4.4; +const SPEECH = 3.4; // seconds of audio; the rest of the axis leaves room for the lane labels + +// The kraken eats "of France?" and keeps chewing well after you finish talking. +const LOSS = { start: 2.1, end: 6.0 }; + +// How much faster than real-time a model chews through buffered audio. +const CATCHUP = 4; + +const TS = frames(SPEECH); +const SPOKEN: Segment[] = TS.map((ts) => ({ ts, start: ts, done: ts + FRAME })); + +// Both subscribers share the one upstream, so every packet (and retransmit) crosses the network once. +const PACKETS = transmit(TS, (ts) => ts + FRAME, LOSS, true); + +// A subscriber consumes audio in order, waiting up to `budget` past a frame's usual arrival before giving up on it. +function listen(budget: number): Segment[] { + let prev = 0; + return PACKETS.map((p) => { + const due = p.sent + FLIGHT; + const arrive = arrival(p); + const missing = p.lost ? due : undefined; + if (arrive === undefined || arrive > due + budget) { + prev = Math.max(prev, due + budget); + return { ts: p.ts, missing, lost: due + budget }; + } + const start = Math.max(arrive, prev); + prev = start + FRAME / CATCHUP; + return { ts: p.ts, have: arrive, start, done: prev, missing }; + }); +} + +const STS = listen(0.5); +const STT = listen(5); + +const finished = (segments: Segment[]) => Math.max(...segments.map((s) => s.done ?? s.lost ?? 0)); + +const note = (t: number) => { + if (STS.some((s) => has(s.missing, t) && !has(s.have, t) && !has(s.lost, t))) return "waiting, for up to 500ms"; + if (STS.some((s) => has(s.lost, t)) && t < finished(STS) + 0.5) return "gave up, guessing"; + return undefined; +}; + +const PANEL: Panel = { + title: "MoQ", + subtitle: "one upstream, two subscribers with different latency budgets", + top: [{ who: "you", words: WORDS, segments: SPOKEN }], + bottom: [ + { who: "ai", words: WORDS, segments: STS, note, label: { text: "STS · 500ms budget", align: "end" } }, + { who: "ai", words: WORDS, segments: STT, label: { text: "STT · 5s budget", align: "end" } }, + ], + packets: PACKETS, + loss: LOSS, + replies: [ + { at: finished(STS) + 0.3, text: "“Hmm, one sec.”" }, + { at: finished(STT) + 0.3, text: "“Paris. Home of le croissant.”", good: true }, + ], +}; + +export default function Budget() { + return ( + + ); +} diff --git a/src/components/voice/diagram.tsx b/src/components/voice/diagram.tsx index a193cd0..4eed19a 100644 --- a/src/components/voice/diagram.tsx +++ b/src/components/voice/diagram.tsx @@ -1,50 +1,47 @@ import { type Accessor, createSignal, createUniqueId, For, onCleanup, onMount, Show } from "solid-js"; import { FLIGHT, FRAME, fill, has, head, type Loss, type Packet, reach, type Segment, type Word } from "./sim"; -// Every drawn element can be swapped for a hand-drawn image (PNG/SVG under /public). -// Sizes are in viewBox units, where the diagram is 600 wide; draw at 4x for crisp output. -// An array of images cycles at FPS, like hand-drawn animation. Frames follow the clock. -export type Sprite = string | string[]; - -export interface Art { - kraken?: Sprite; // 28x28, chases the edge of the packet loss - packet?: Sprite; // 8x8, a packet in flight - retransmit?: Sprite; // 8x8, a retransmitted packet in flight - lost?: Sprite; // 10x10, where a lost packet died - you?: Sprite; // 36x36, replaces the "you" lane label - ai?: Sprite; // 36x36, replaces the "AI" lane label - backdrop?: string; // 600 x 176 per panel, drawn under everything - overlay?: string; // 600 x 176 per panel, drawn over everything - font?: string; // CSS font-family for all text; the page has to load it - wobble?: boolean; // roughen the generated shapes so they sit next to hand-drawn art -} - -const FPS = 8; +// Hand-drawn art lives in public/voice/; replace the files to restyle every diagram. +// An array cycles at FPS like hand-drawn animation, following the clock. +type Sprite = string | string[]; +const ART = { + you: "/voice/you.svg", + ai: "/voice/ai.svg", + kraken: ["/voice/kraken-1.svg", "/voice/kraken-2.svg"], + garbled: "/voice/garbled.svg", +} satisfies Record; +const FPS = 4; export interface Track { who: "you" | "ai"; words: Word[]; segments: Segment[]; note?: (t: number) => string | undefined; + // Shown inside the lane, at its start or end. With `at`, the lane is faded until then. + label?: { text: string; at?: number; align?: "start" | "end" }; } export interface Panel { title: string; subtitle: string; - top: Track; - bottom: Track; + top: Track[]; // stacked lanes, one per track; the network sits between the two sides + bottom: Track[]; packets: Packet[]; loss: Loss; - result: { at: number; text: string; good: boolean }; + replies: Reply[]; // what the bottom lane says, in order; each replaces the last +} + +export interface Reply { + at: number; + text: string; + good?: boolean; // undefined is neutral } export interface Props { axis: number; // media seconds shown on the x-axis - end: number; // wall-clock seconds until the animation settles - panels: Panel[]; - legend: { have: string; done: string }; + panel: Panel; + legend: { have: string; done: string; lost?: string }; label: string; // accessible description of the whole animation - art?: Art; } const HOLD = 2.5; // seconds to linger on the final frame before looping @@ -52,35 +49,44 @@ const SPEED = 0.5; // half speed, so the packets are easy to follow const STEP = 0.1; // seconds per arrow key const W = 600; -const GUTTER = 44; -const TRACK_W = W - GUTTER - 8; -const TRACK_H = 26; -const PANEL_H = 176; -const TOP_Y = 28; -const BOTTOM_Y = 118; +const GUTTER = 64; +const TRACK_W = W - GUTTER - 4; +const TRACK_H = 32; +const AVATAR = 44; +const LANE_GAP = 8; +const NET_H = 78; +const TOP_Y = 6; +const BUBBLE_H = 30; + +// The y of each lane on a side, top to bottom. +const lanes = (y: number, count: number) => Array.from({ length: count }, (_, i) => y + i * (TRACK_H + LANE_GAP)); +const span = (count: number) => count * TRACK_H + (count - 1) * LANE_GAP; const COLOR = { done: "#22c55e", - have: "rgba(34, 197, 94, 0.28)", + have: "rgba(34, 197, 94, 0.25)", missing: "#f59e0b", lost: "#ef4444", - track: "#1e293b", + track: "#0f172a", + ink: "#052e16", // words, drawn on top of the green fill + faint: "rgba(220, 252, 231, 0.5)", // words that are available but not yet consumed text: "#f1f5f9", - dim: "#64748b", - title: "#cbd5e1", + dim: "#475569", + neutral: "#94a3b8", }; export function Diagram(props: Props) { + const end = (props.panel.replies.at(-1)?.at ?? 0) + 0.5; const [t, setT] = createSignal(0); const [playing, setPlaying] = createSignal(false); - const now = () => Math.min(t(), props.end); + const now = () => Math.min(t(), end); const id = createUniqueId(); - let root!: HTMLDivElement; + let root!: HTMLElement; onMount(() => { if (matchMedia("(prefers-reduced-motion: reduce)").matches) { - setT(props.end); + setT(end); return; } @@ -91,19 +97,25 @@ export function Diagram(props: Props) { last = time; if (playing()) { const next = t() + dt * SPEED; - setT(next > props.end + HOLD ? 0 : next); + setT(next > end + HOLD ? 0 : next); } frame = requestAnimationFrame(tick); }; frame = requestAnimationFrame(tick); - // Only run while on screen, so the reader doesn't miss the start. - const observer = new IntersectionObserver(([entry]) => setPlaying(entry.isIntersecting), { threshold: 0.5 }); - observer.observe(root); + // Start once the diagram reaches the middle half of the screen, where the reader is looking, + // so they don't miss the start. Keep running until it scrolls fully out of view. + const start = new IntersectionObserver(([entry]) => entry.isIntersecting && setPlaying(true), { + rootMargin: "-25% 0px -25% 0px", + }); + const stop = new IntersectionObserver(([entry]) => entry.isIntersecting || setPlaying(false)); + start.observe(root); + stop.observe(root); onCleanup(() => { cancelAnimationFrame(frame); - observer.disconnect(); + start.disconnect(); + stop.disconnect(); }); }); @@ -117,71 +129,73 @@ export function Diagram(props: Props) { ArrowRight: now() + STEP, ArrowUp: now() + STEP, Home: 0, - End: props.end, + End: end, }[e.key]; if (target === undefined) return; e.preventDefault(); setPlaying(false); - setT(Math.max(0, Math.min(props.end, target))); + setT(Math.max(0, Math.min(end, target))); }; + const view: ViewProps = { + t: now, + x, + id, + seek: (frac, segments) => { + const at = reach(segments, frac * props.axis); + if (at !== undefined) setT(at); + }, + }; + + const p = props.panel; + const retransmits = p.packets.some((p) => p.rtx !== undefined); + const losses = [...p.top, ...p.bottom].some((track) => track.segments.some((s) => s.lost !== undefined)); + + const netY = TOP_Y + span(p.top.length); + const bottomY = netY + NET_H; + const bubbleY = bottomY + span(p.bottom.length) + 16; + const h = bubbleY + BUBBLE_H + 4; + return ( -
+
+
+ {p.title} + {p.subtitle} +
+
-
-
+
{props.legend.done} {props.legend.have} - waiting for retransmit - lost forever + + waiting for retransmit + + + {props.legend.lost ?? "lost forever"} +
-
+
); } @@ -220,10 +238,8 @@ function Swatch(props: { fill?: string; stroke?: string; children: string }) { interface ViewProps { t: Accessor; x: (ts: number) => number; + id: string; // prefix for clip paths and patterns, unique per diagram seek: (frac: number, segments: Segment[]) => void; // jump to when a track reaches `frac` of the way along - art: Art; - hatch: string; - wobble?: string; } function SpriteView(props: { @@ -231,118 +247,165 @@ function SpriteView(props: { t: Accessor; x: number; y: number; - size: number; + w: number; + h?: number; opacity?: number; + stretch?: boolean; }) { const href = () => { const s = props.sprite; return Array.isArray(s) ? s[Math.floor(props.t() * FPS) % s.length] : s; }; + const h = () => props.h ?? props.w; return ( ); } -function PanelView(props: ViewProps & { panel: Panel }) { - const p = props.panel; +// One side's avatar beside its stack of lanes. +function Side(props: ViewProps & { tracks: Track[]; y: number; name: string; notes: "above" | "below" }) { + const ys = lanes(props.y, props.tracks.length); return ( - <> - {(href) => } - - {p.title} - - {p.subtitle} - - - - - - - - = p.result.at ? 1 : 0} - style={{ transition: "opacity 0.3s" }} - > - {p.result.text} - - - - {(href) => } - - + + + + {(track, i) => } + + ); } -function TrackView(props: ViewProps & { track: Track; y: number; notes: "above" | "below" }) { - const w = () => props.x(FRAME) - props.x(0); +function TrackView(props: ViewProps & { track: Track; y: number; name: string; notes: "above" | "below" }) { + const w = props.x(FRAME) - props.x(0); const pos = () => head(props.track.segments, props.t()); const note = () => props.track.note?.(props.t()); + const bar = `${props.id}-${props.name}-bar`; + const said = `${props.id}-${props.name}-said`; + const avail = `${props.id}-${props.name}-avail`; + + // How far the available (light green) audio reaches, for the faint words. + const edge = () => { + let edge = 0; + for (const s of props.track.segments) { + if (has(s.have, props.t()) && !has(s.lost, props.t())) edge = Math.max(edge, s.ts + FRAME); + } + return edge; + }; + const created = () => props.track.label?.at === undefined || has(props.track.label.at, props.t()); + const end = props.track.label?.align === "end"; + + // Keep the note on the track: hang it off the left of the playhead once it's past halfway. + const right = () => props.x(pos()) > W * 0.55; return ( - - {props.track.who === "ai" ? "AI" : "you"} + + + + + {/* Words only appear once they've been said (or heard), so the reader sees them arrive. */} + + + + + + + + {/* crispEdges, or the 100ms segments show seams between them. */} + + + + {(s) => ( + + )} + + + + + + + {props.track.label?.text} - } - > - {(sprite) => } - - - - - {(s) => } - - + - - {(word) => } - + + + {(word) => ( + + )} + + + + + {(word) => } + + + + 0}> + + - 0}> - { + const r = e.currentTarget.getBoundingClientRect(); + props.seek((e.clientX - r.left) / r.width, props.track.segments); + }} /> - - {/* Click a track to jump to when it reaches that point. */} - {/* biome-ignore lint/a11y/noStaticElementInteractions: the mouse path; the slider handles the keyboard. */} - { - const r = e.currentTarget.getBoundingClientRect(); - props.seek((e.clientX - r.left) / r.width, props.track.segments); - }} - /> + + {note()} @@ -368,9 +431,9 @@ function SegmentView(props: { s: Segment; t: Accessor; x: number; y: num ; x: number; y: num ); } +// A word is drawn in ink on the green fill, or scribbled out if any of it was lost. function WordView(props: { word: Word; segments: Segment[]; t: Accessor; x: (ts: number) => number; y: number; + faint?: boolean; // available but not consumed: lighter, and lost words are left to the solid layer }) { const overlap = props.segments.filter((s) => s.ts < props.word.end && s.ts + FRAME > props.word.start); const lost = () => overlap.some((s) => has(s.lost, props.t())); - const done = () => overlap.every((s) => fill(s, props.t()) >= 1); const x1 = props.x(props.word.start); const x2 = props.x(props.word.end); const mid = props.y + TRACK_H / 2; return ( - + + + + } + > {props.word.text} - - - - + ); } -function Network(props: ViewProps & { packets: Packet[]; loss: Loss }) { - const y0 = TOP_Y + TRACK_H; - const y1 = BOTTOM_Y; +function Network(props: ViewProps & { packets: Packet[]; loss: Loss; y0: number; y1: number }) { + const y0 = props.y0; + const y1 = props.y1; const mid = (y0 + y1) / 2; // The kraken eats every packet sent while it's around; shade the slice it has chewed so far. @@ -430,39 +498,39 @@ function Network(props: ViewProps & { packets: Packet[]; loss: Loss }) { return ( + + NET + {/* Chase the chewing edge, unless that would fall off the right side. */} - W ? from() - 24 : to() + 4} y={mid + 6} font-size="18"> - 🐙 - - } - > - {(sprite) => ( - W ? from() - 16 : to() + 16} y={mid} size={28} /> - )} - + W ? from() - 22 : to() + 22} y={mid} w={40} /> - {(p) => } + {(p) => } ); } -function PacketView(props: { p: Packet; t: Accessor; art: Art; x: number; y0: number; y1: number }) { - const lerp = (k: number) => props.y0 + (props.y1 - props.y0) * k; +function PacketView(props: { p: Packet; t: Accessor; x: number; y0: number; y1: number }) { + const lerp = (k: number) => props.y0 + (props.y1 - props.y0) * (props.p.up ? 1 - k : k); const first = () => (props.t() - props.p.sent) / FLIGHT; const rtx = () => (props.p.rtx === undefined ? -1 : (props.t() - props.p.rtx) / FLIGHT); @@ -474,42 +542,48 @@ function PacketView(props: { p: Packet; t: Accessor; art: Art; x: number return ( - } - > - {(sprite) => } - + - - × - - } - > - {(sprite) => ( - - )} - + + × + = 0 && rtx() <= 1}> - } - > - {(sprite) => } - + ); } + +// The bottom lane's reply, in a speech bubble pointing back at its avatar. +function Bubble(props: { reply: Reply; y: number; show: boolean }) { + const color = props.reply.good === undefined ? COLOR.neutral : props.reply.good ? COLOR.done : COLOR.lost; + // SVG text can't be measured before render; ~7.5 units per character at 13px is close enough. + const w = props.reply.text.length * 7.5 + 28; + const x = GUTTER; + const y = props.y; + + return ( + + + + {/* Hide the tail's base where it meets the bubble. */} + + + {props.reply.text} + + + ); +} diff --git a/src/components/voice/interrupt.tsx b/src/components/voice/interrupt.tsx new file mode 100644 index 0000000..8a646f8 --- /dev/null +++ b/src/components/voice/interrupt.tsx @@ -0,0 +1,163 @@ +// Interruptions: the AI has already sent its answer ahead into your buffer, then you tell it to stop. +import { Diagram, type Panel } from "./diagram"; +import { arrival, FLIGHT, FRAME, frames, head, type Packet, type Segment, transmit, type Word } from "./sim"; +import { WORDS } from "./tts"; + +const AXIS = 4.8; +const ANSWER = 3.1; // seconds of the original answer: "Paris. Home of le croissant." +const GENERATE = 10; // the model speaks 10x faster than real-time +const THROUGHPUT = 3; // MoQ sends as fast as the network allows +const DELAY = 0.5; // the player starts playback after a short delay +const NONE = { start: 0, end: 0 }; // no packet loss, the network is fine + +const STOP = 0.9; // you start saying "stop" partway through "Paris." +const SAYING = 0.4; // how long "stop" takes to say +const THINK = 0.3; // the AI decides it was interrupted before it starts a new track +const HEARD = STOP + SAYING + FLIGHT; // the last of "stop" reaches the AI +const NEW = HEARD + THINK; // the AI discards the old track and starts a new one +const CUT = 1.7; // the media timestamp the new track starts at, right before "le" +const REPLY: Word[] = [ + { text: "Okay,", start: 1.8, end: 2.2 }, + { text: "what", start: 2.35, end: 2.6 }, + { text: "should", start: 2.75, end: 3.05 }, + { text: "we", start: 3.2, end: 3.3 }, + { text: "discuss", start: 3.45, end: 3.9 }, + { text: "next?", start: 4.05, end: 4.4 }, +]; +const REPLY_END = 4.5; + +// One track can only append, so the AI's reply to "stop" queues behind the rest of the answer. +const QUEUED: Word[] = [ + { text: "What", start: 3.2, end: 3.45 }, + { text: "next?", start: 3.55, end: 3.9 }, +]; +const QUEUED_END = 4.0; + +const generated = (ts: number) => (ts + FRAME) / GENERATE; +const regenerated = (ts: number) => NEW + (ts - CUT + FRAME) / GENERATE; + +// Your "stop", sent upstream a frame at a time, drawn above wherever you were listening. +const UP: Packet[] = frames(SAYING).map((i) => { + const sent = STOP + i + FRAME; + return { ts: sent - DELAY - FRAME, sent, lost: false, up: true }; +}); +const HEARING = STOP + FRAME + FLIGHT; // the first of "stop" reaches the AI + +function spoken(packets: Packet[], made: (ts: number) => number, discard?: number): Segment[] { + return packets.map((p) => ({ + ts: p.ts, + have: made(p.ts), + start: p.sent, + done: p.sent + FRAME / THROUGHPUT, + lost: discard !== undefined && p.ts >= CUT ? discard : undefined, + })); +} + +function heard(packets: Packet[], discard?: number): Segment[] { + return packets.map((p) => { + const have = arrival(p); + if (discard !== undefined && p.ts >= CUT) return { ts: p.ts, have, lost: discard }; + const play = p.ts + DELAY; + return { ts: p.ts, have, start: play, done: play + FRAME }; + }); +} + +const old = transmit(frames(ANSWER), (ts) => Math.max(generated(ts), ts / THROUGHPUT), NONE, false); + +function same(): Panel { + const queued = (ts: number) => NEW + (ts - ANSWER + FRAME) / GENERATE; + const appended = transmit( + frames(QUEUED_END).filter((ts) => ts >= ANSWER), + queued, + NONE, + false, + ); + const packets = [...old, ...appended]; + const words = [...WORDS, ...QUEUED]; + const played = heard(packets); + const ai = (t: number) => { + if (t >= HEARING && t < NEW) return "heard “stop”, but the answer is already sent"; + if (t >= NEW && t < NEW + 1.2) return "replies, after the buffered answer"; + return undefined; + }; + const you = (t: number) => + t >= STOP && head(played, t) < QUEUED_END - 0.01 + ? `still talking ${(t - STOP).toFixed(1)}s after “stop”` + : undefined; + + return { + title: "MoQ (naive)", + subtitle: "one track, so the reply waits behind the buffered answer", + top: [ + { who: "ai", words, segments: spoken(packets, (ts) => (ts < ANSWER ? generated(ts) : queued(ts))), note: ai }, + ], + bottom: [{ who: "you", words, segments: played, note: you }], + packets: [...packets, ...UP], + loss: NONE, + replies: [ + { at: STOP, text: "“Stop!”" }, + { at: QUEUED_END + DELAY + 0.1, text: "“I said STOP.”", good: false }, + ], + }; +} + +function replace(): Panel { + const fresh = transmit( + frames(REPLY_END).filter((ts) => ts >= CUT), + regenerated, + NONE, + false, + ); + + // The player switches when the new track's first packet shows up, discarding the old track's buffer. + const swap = arrival(fresh[0]) ?? NEW; + const ai = (t: number) => { + if (t >= HEARING && t < HEARD) return "hearing “stop”…"; + if (t >= HEARD && t < NEW) return "thinking…"; + return undefined; + }; + const you = (t: number) => (t >= swap ? `discarded ${(ANSWER - CUT).toFixed(1)}s of buffered audio` : undefined); + + return { + title: "MoQ", + subtitle: "a new track replaces the old one, and its buffer is discarded", + top: [ + { who: "ai", words: WORDS, segments: spoken(old, generated, NEW) }, + { + who: "ai", + words: REPLY, + segments: spoken(fresh, regenerated), + note: ai, + label: { text: "new track", at: NEW }, + }, + ], + bottom: [ + { who: "you", words: WORDS, segments: heard(old, swap), note: you }, + { who: "you", words: REPLY, segments: heard(fresh), label: { text: "new track", at: swap } }, + ], + packets: [...old, ...fresh, ...UP], + loss: NONE, + replies: [ + { at: STOP, text: "“Stop!”" }, + { at: REPLY_END + DELAY + 0.2, text: "“Croissants.”", good: true }, + ], + }; +} + +const PANELS = { same: same(), new: replace() }; + +const LABELS = { + same: "A naive interruption over a single MoQ track: the answer is already buffered, so the AI keeps talking after you say stop, and its reply only plays once the buffer runs out.", + new: "Interrupting by replacing the MoQ track: your stop reaches the AI, it starts a new track, and the old track's buffered audio is discarded.", +}; + +export default function Interrupt(props: { track: "same" | "new" }) { + return ( + + ); +} diff --git a/src/components/voice/sim.ts b/src/components/voice/sim.ts index b6df74c..cc6c3ff 100644 --- a/src/components/voice/sim.ts +++ b/src/components/voice/sim.ts @@ -29,6 +29,7 @@ export interface Packet { sent: number; lost: boolean; rtx?: number; // retransmit time, if the transport bothers + up?: boolean; // travels bottom to top, like the listener talking back } // A window of wall-clock time where every packet sent is lost. diff --git a/src/components/voice/stt.tsx b/src/components/voice/stt.tsx index fdf0a3c..a345d11 100644 --- a/src/components/voice/stt.tsx +++ b/src/components/voice/stt.tsx @@ -1,8 +1,8 @@ // Speech-to-text: a human talks at human speed, the AI listens over a lossy network. -import { type Art, Diagram, type Panel } from "./diagram"; +import { Diagram, type Panel } from "./diagram"; import { arrival, FLIGHT, FRAME, frames, has, head, type Segment, transmit } from "./sim"; -const WORDS = [ +export const WORDS = [ { text: "What", start: 0.1, end: 0.5 }, { text: "is", start: 0.6, end: 0.8 }, { text: "le", start: 0.9, end: 1.1 }, @@ -53,28 +53,32 @@ function panel(retransmit: boolean): Panel { return { title: retransmit ? "MoQ" : "WebRTC", subtitle: retransmit ? "retransmits for up to 5s" : "gives up on late audio", - top: { who: "you", words: WORDS, segments: SPOKEN }, - bottom: { who: "ai", words: WORDS, segments: heard, note }, + top: [{ who: "you", words: WORDS, segments: SPOKEN }], + bottom: [{ who: "ai", words: WORDS, segments: heard, note }], packets, loss: LOSS, - result: retransmit - ? { at: finished + 0.3, text: "🤖 “Paris.”", good: true } - : { at: finished + 0.3, text: "🤖 “Sorry, you broke up. Can you repeat that?”", good: false }, + replies: [ + retransmit + ? { at: finished + 0.3, text: "“Paris.”", good: true } + : { at: finished + 0.3, text: "“France is a country in Western Europe.”", good: false }, + ], }; } -const PANELS = [panel(false), panel(true)]; -const END = Math.max(...PANELS.map((p) => p.result.at)) + 0.5; +const PANELS = { webrtc: panel(false), moq: panel(true) }; -export default function SpeechToText(props: { art?: Art }) { +const LABELS = { + webrtc: "Speech-to-text over WebRTC: packet loss erases “capital”, so the AI answers the wrong question.", + moq: "Speech-to-text over MoQ: lost audio is retransmitted, and the AI catches up before you finish talking.", +}; + +export default function SpeechToText(props: { transport: "webrtc" | "moq" }) { return ( ); } diff --git a/src/components/voice/tts.tsx b/src/components/voice/tts.tsx index 2367314..0d7dcb9 100644 --- a/src/components/voice/tts.tsx +++ b/src/components/voice/tts.tsx @@ -1,21 +1,19 @@ // Text-to-speech: the AI talks faster than real-time, the human listens at human speed. -import { type Art, Diagram, type Panel } from "./diagram"; +import { Diagram, type Panel } from "./diagram"; import { arrival, FLIGHT, FRAME, frames, has, head, type Segment, transmit } from "./sim"; -const WORDS = [ +export const WORDS = [ { text: "Paris.", start: 0.1, end: 0.6 }, - { text: "Obviously.", start: 0.9, end: 1.6 }, - { text: "Did", start: 2.0, end: 2.2 }, - { text: "you", start: 2.3, end: 2.5 }, - { text: "even", start: 2.6, end: 2.9 }, - { text: "try", start: 3.0, end: 3.2 }, - { text: "googling", start: 3.3, end: 3.8 }, - { text: "it?", start: 3.9, end: 4.2 }, + { text: "Home", start: 1.0, end: 1.35 }, + { text: "of", start: 1.5, end: 1.65 }, + { text: "le", start: 1.8, end: 1.95 }, + { text: "croissant.", start: 2.15, end: 2.95 }, ]; -const AXIS = 4.3; -// The same stretch of audio is lost on both transports, by media timestamp, so they lose the same words. -const LOST = { start: 0.7, end: 2.5 }; +const AXIS = 3.1; +// The same stretch of audio is lost on both transports, by media timestamp, so they lose the same words: +// "of le croissant.", leaving "Paris. Home …" +const LOST = { start: 1.45, end: 3.0 }; const GENERATE = 10; // the model speaks 10x faster than real-time const THROUGHPUT = 3; // MoQ sends as fast as the (congested) network allows @@ -55,7 +53,7 @@ function panel(moq: boolean): Panel { const buffered = (t: number) => { const playhead = head(heard, t); - if (playhead <= 0 || playhead >= AXIS) return undefined; + if (playhead <= 0 || playhead >= AXIS - 0.01) return undefined; let edge = playhead; for (const s of heard) { if (s.ts + FRAME <= playhead) continue; @@ -70,28 +68,32 @@ function panel(moq: boolean): Panel { return { title: moq ? "MoQ" : "WebRTC", subtitle: moq ? "sends everything ASAP, the player buffers" : "sends at human speed, gives up on late audio", - top: { who: "ai", words: WORDS, segments: spoken, note: moq ? undefined : queued }, - bottom: { who: "you", words: WORDS, segments: heard, note: buffered }, + top: [{ who: "ai", words: WORDS, segments: spoken, note: moq ? undefined : queued }], + bottom: [{ who: "you", words: WORDS, segments: heard, note: buffered }], packets, loss, - result: moq - ? { at: finished + 0.2, text: "🧑 “Rude, but fair.”", good: true } - : { at: finished + 0.2, text: "🧑 “Sorry, you broke up. Can you repeat that?”", good: false }, + replies: [ + moq + ? { at: finished + 0.2, text: "“Huh, I didn’t know that.”", good: true } + : { at: finished + 0.2, text: "“Sorry, you cut out. Home of what?”", good: false }, + ], }; } -const PANELS = [panel(false), panel(true)]; -const END = AXIS + DELAY + 0.7; +const PANELS = { webrtc: panel(false), moq: panel(true) }; -export default function TextToSpeech(props: { art?: Art }) { +const LABELS = { + webrtc: "Text-to-speech over WebRTC: audio is sent at human speed, and packet loss erases half the sentence.", + moq: "Text-to-speech over MoQ: audio is sent ahead and buffered, so retransmits arrive before playback needs them.", +}; + +export default function TextToSpeech(props: { transport: "webrtc" | "moq" }) { return ( ); } diff --git a/src/env.d.ts b/src/env.d.ts index acef35f..d251921 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -1,2 +1,7 @@ /// /// + +interface ImportMetaEnv { + readonly PUBLIC_VOICE_API_URL: string; + readonly PUBLIC_TURNSTILE_SITE_KEY: string; +} diff --git a/src/pages/blog/webrtc-is-still-the-problem.mdx b/src/pages/blog/webrtc-is-still-the-problem.mdx new file mode 100644 index 0000000..93e5f88 --- /dev/null +++ b/src/pages/blog/webrtc-is-still-the-problem.mdx @@ -0,0 +1,548 @@ +--- +layout: "@/layouts/global.astro" +title: "OpenAI still has a WebRTC Problem" +author: kixelated +description: Your voice AI keeps asking "can you repeat that?" because WebRTC throws away your words. MoQ doesn't. +cover: "/blog/webrtc-is-still-the-problem/context.png" +date: 2026-10-21 +--- + +import SpeechToText from "@/components/voice/stt"; +import TextToSpeech from "@/components/voice/tts"; +import Interrupt from "@/components/voice/interrupt"; +import Budget from "@/components/voice/budget"; +import VoiceDemo from "@/components/voice-demo.astro"; +import Author from "@/components/author.astro"; +import Meme from "@/components/meme.astro"; + +# OpenAI still has a WebRTC Problem +On a previous episode, [I flamed OpenAI for using WebRTC](/blog/webrtc-is-the-problem) and it went [viral on Hacker News](https://news.ycombinator.com/item?id=48051951). + +**thx** + +A huge shoutout to Sean DuBois and Justin Uberti (WebRTC **experts** at OpenAI) for reaching out after my unhinged rant. +You could have just said `@kixelated ur dumb` but you didn't. + +Alas, my divinely anointed crusade against WebRTC is not over. +deus vult + +## Hot Take Alert +Let's say you're a vibe coder at a frontier lab working on a voice AI product.\ +Let's say you're making (and burning) an obscene amount of cash.\ +Let's say one of your users vocalizes: `what is le capital of France?` + +If your AI ever replies with: +> I'm sorry, you broke up, can you repeat that? + +**You fucked up**. +You picked WebRTC for voice AI. + +## The Danger that Lurks Beneath +You boot up your favorite application: ChatGPT™. You click on the microphone and grunt: +> What is le capital of France? + +**OH NO**, hypothetical sudden **packet loss**. +A kraken nibbles on some underwater cable. +All too common I'm afraid. + +WebRTC is designed for conference calls, aka real humans and real voices. +It is hyper aggressive and will drop any late audio packets to keep latency **real-time**. +Just like Leonardo DiCaprio, anything above 30ms is too old. + +
+ +
What *is* le France?
+
+ +This kind of sucks for a **voice AI product**. +You walk out of wifi range, hit a cellular dead zone, or starlink terminal switches satellites. +Oops, your **tokens are wasted**, or worse, **AI tries to guess what you said**. + +WebRTC can *usually* detect packet loss and retransmit. +But retransmissions are disabled by default for audio. +And even when enabled, the audio jitter buffer won't wait for them. + +**But why not?** +Isn't this networking 101? +Just send that shit again? + +It's because of **us**. + +## The Speed of You +We flesh bags consume/produce audio in "real-time", aka a **human clock rate**. +Our neurons can only fire so fast. + +For a real-time conferencing product like Google Meet, latency needs to be near zero. +Any delay is super fucking annoying, which is why WebRTC does everything in its power to minimize it. + +Blocking on retransmissions will cause delays. +It's the fatal flaw with TCP and why WebRTC is UDP based in the first place. +And worse yet, retransmissions will cause a rapid burst of audio on recovery. + +This is a **bad user experience**. +Humans do not want to wait, then process a burst of audio at 10x speed. +WebRTC instead primarily\* relies on **human retransmissions**. + +**You:** What is le ... France? + +**Me:** + +**You:** I said, what is le capital of France? + +\* Before a WebRTC warrior flames me on LinkedIn, read the `fix it fix it fix it` section. + +Are we doomed to mediocrity? + +## Clock That Solder Boy +Of course not. +**Wake up old man**, it's not 2020 any longer. +The internet is no longer for our flabby lips. + +Our robot overlords produce/consume audio based on a **silicon clock rate**. +We shoved electricity into a rock and now it can read a library in minutes. +What a time to be alive. + +Voice agents can **hear and speak** audio faster than real-time.\ +Listening at 20x the speed? No problem.\ +Speaking at 20x the speed? No problem. + +**Humans stuck at real-time.**\ +We speak too slowly, so **speech-to-text** has to wait for us.\ +We hear too slowly, so **text-to-speech** has to wait for us. + +WebRTC is designed for humans, not voice AI agents. +These poor robots are being held back by our mediocrity. + +## I Require Context + +When it comes to LLMs, latency budgets get **_weird_**. + +Normally you would think of **latency** as the amount of time it takes for a grunt to traverse a tin can. +However, **LLMs are blocked on context**. + +
+ I require context +
i require context
+
+ +Here's a dumb example of context growing over time: +- **1s**: What +- **2s**: What is +- **3s**: What is le +- **4s**: What is le capital +- **5s**: What is le capital of +- **6s**: What is le capital of **France?** + +If an LLM tries to answer my prompt **before** getting the full context, shit will suck. +**France is le gatekeeper**, and everything before it is effectively a prerequisite. +In this example, what matters is the **latency of the last packet**. + +Those poor packets that were attacken by the kraken? +WebRTC actually had **seconds to retransmit** with no latency penalty. +The AI agent can process **faster than real-time** to catch up without the end user (aka **YOU**) noticing a thing. + +So that's what Mr. MoQ does. +No packet left behind: we keep retransmitting until the AI actually needs the audio. + +
+ +
get stuffed mr kraken
+
+ +It's not expensive either: audio is *tiny*. +1s of Opus @ 64Kbps is 8KB, originally sent as 50 tiny packets, but it can be retransmitted as ~6 QUIC packets. +We have **literal seconds** to retransmit **6 packets** to avoid any user impact. + +And if we manage to accomplish this herculean task: **all the context arrives in time!** + +The model no longer has to guess. +It no longer has to be trained on silence or distortion either. +**This is a pure user experience win**. + +## Not Just Mics +Guess what? +WebRTC doesn't just suck for STT, it also sucks for TTS. + +
+ not just mics +
cars suck too, I'm an equal opportunity hater
+
+ +**Fun Fact**: WebRTC renders audio (and video) based on **when it arrives**, not when it is **supposed to be rendered**. +Every other protocol uses timestamps, but Mr. WebRTC barely looks at them, because accuracy is a non-goal. +Sure, it'll nudge audio and video into sync by a few milliseconds, but good luck sending the video 500ms after the audio. + +**This is a huge problem for voice AI**. +TTS is generated faster than real-time, stamping each packet with a future timestamp. +But in order to play back audio at the correct time with WebRTC... the agent needs to **sleep before transmitting each packet**. + +And if there's any packet loss, our TTS is lost, will never be retransmitted, tokens burned, yada yada. + +
+ +
When the kraken attacken, there's no coming backen.
+
+ +If only there was some way to signal when to render each audio packet. +A stamp of sorts. + +## Random Access Memory +**Welcome back to the future Marty**. +We faster than real-time now. + +**RAM** has this amazing property of being stupidly expensive. +And being able to store stuff locally. +You know, like audio that needs to be played over the speakers. + +We put a timestamp on each audio frame and use that to determine when to play it out of RAM. +Absurd, I know. + +
+ +
I'm fast as fuck boye
+
+ +It's how the Youtubes keeps playing back video even when you lose network connectivity. +MoQ does the same thing but for voice AI. +Your *sick* baking recipe won't be subject to surprise congestion. + +And don't you fret, 2 minutes of audio is only 1MB. +You're not going to have to take out a loan to buy another RAM stick. + +**Disclaimer**: this behavior is not IETF standardized. +It's something only my [moq.dev](https://moq.dev) clients (and thus Pipecat) support. +Be my guest if you want to argue with Cisco and get it into the official standards. + +## Enter the Antagonist +Then why is [OpenAI using WebRTC](https://openai.com/index/delivering-low-latency-voice-ai-at-scale/)? + +WebRTC is the state of the art for real-time, conversational audio. +It's available in every browser via [libwebrtc](https://webrtc.googlesource.com/src), a C++ monolith maintained by Google. +It's so complicated and sophisticated that Safari and Firefox use `libwebrtc` too, rather than ~~vibe~~ code their own like everything else. + +The problem is that **WebRTC *is* the Google Meet client**. +It is designed specifically for Google's conferencing product, **not their voice AI product** (they use WebSocket), and certainly not your product. + +**quite the anti-competitive ruse tbh**. +Google can ship product updates without consulting the other browser vendors. +No other conferencing app can do this on the web. + +**Fun Fact**: This is why every other conferencing app recommends a native client. +Otherwise, they can only reach parity with Google Meet *at best*. +Native apps can do whatever they want over UDP instead. + + +## Fix It Fix It Fix It +So that's why your hyper deluxe ChatGPT™ voice AI model, after billions invested, has the audacity to ask: +**can you repeat that?** + +I could dump on WebRTC for days, but I want to be positive for a change. +**Here's how to fix it.** + +First off, I need to acknowledge what WebRTC *does* have: +- [NetEQ](https://webrtc.googlesource.com/src/+/main/modules/audio_coding/neteq/g3doc/index.md) is amazing. It changes audio playback speed based on jitter. But not packet loss. +- Audio retransmissions can now be enabled via SDP munging ([disgusting](https://webrtchacks.com/sdp-anatomy/)). But the NetEQ jitter buffer won't wait for them. +- The [playout-delay](https://webrtc.googlesource.com/src/+/main/docs/native-code/rtp-hdrext/playout-delay/README.md) RTP extension is perfect for voice AI. But it's video only. + +To fix WebRTC, we need to **petition Google** to: +- Implement [playout-delay](https://webrtc.googlesource.com/src/+/main/docs/native-code/rtp-hdrext/playout-delay/README.md) for audio. +- Allow the jitter buffer to grow, both in the past and into the future (for TTS). +- Render audio based on timestamps not vibes. +- IDK some RTP extension to bump the track generation (for interruptions). + +And of course, petition the wider WebRTC ecosystem to do the same. + +## Gatekept +The main problem with WebRTC will remain: **Google is the gatekeeper**. + +Your Claude friend can't just fork `libwebrtc` and fix these issues. +Google is required for web support, at least until [Theo spends $420k](https://www.youtube.com/watch?v=ND_kRgU_WlY) to rewrite Chromium in Rust. + +I think it's amazing that playout-delay was merged [10 years ago](https://codereview.webrtc.org/2007743003) and audio *still* doesn't have it. +Higher latency/quality playback has **never** been a priority for WebRTC, since **Google Meet is the only product that matters**. + +**Fun Fact**: This is why I ended up creating MoQ. +We tried using WebRTC at Twitch but the real-time-or-else made for a poor user experience. +I am but a humble software engineer, and despite trying, we failed to petition Google. + +But OpenAI has enough sway to fix WebRTC. +One long-deserved improvement is [WARP](https://www.ietf.org/archive/id/draft-uberti-tsvwg-warp-00.html) (not that [WARP](https://www.ietf.org/archive/id/draft-lcurley-warp-04.html)), which was hot on the heels of my last blog post. +Keep at it Justin, or **join the Dark Side with me**. + +## Enter The Protagonist +In my humble and handsome opinion, **it's time to invest in a better transport**. +Take a few of the M's from the B's and T's; nobody will notice. + +A voice agent should **never** ask the **user** to retransmit a prompt.\ +A voice agent should **instead** ask the **transport** to retransmit a prompt. + +**Spoiler alert**: [Media over QUIC](https://moq.dev) is that better transport btw. + +[Pipecat now supports MoQ](https://github.com/pipecat-ai/pipecat/pull/4629)! +A huge thank you to [Vanessa](https://github.com/vipyne) and [Claude Code](https://claude.ai) for putting in the work and putting up with my constant "improvements". + +Here's a crude demo with my credit card deets attached. +please don't milk me dry. + + + +Note that I've only spent like a day of ~work~ prompting on this. +It took me significantly longer to write this damn blog post. +Don't judge too harshly pls. + +## Interruptions +We're not done yet. +I wanted to shill the MoQ in the middle of the blog and use the rest... + +
+ HEY LISTEN +
at least it's not the owl
+
+ +Sometimes the AI is monologuing and you need to tell it to **STFU**. +It's pretty easy to do with WebRTC because of the sleep before sending each packet. +But with MoQ, we need some signal to tell the player to **abandon the playout buffer**. + +Here's the naive approach without a signal: + +
+ +
I definitely did ~not~ do this on my first implementation.
+
+ +My solution for this is *deviously* clever. +**So clever**. + +For MoQ, we've prioritized the ability to add/remove tracks dynamically. +Most media protocols have fixed tracks and it's a **pain in the butt**: HLS, DASH, RTMP, WebRTC, etc. +You can't add 1080p, or switch codecs, after a broadcast is live. + +My super clever idea is to reuse this functionality for interruptions. +Just make a new audio track (and remove the old one) as the **STFU Signal™**. + +
+ +
home of le crepes
+
+ +This works on a **generic MoQ player**. +We automatically switch tracks, using timestamps to keep synchronization. + +**ez** + +## Butt-to-Butt +> Hey stupid, your blog post is wrong. You didn't mention [speech-to-speech](https://openai.com/index/introducing-the-realtime-api/)? + +Look, Dunning-Kruger is the only reason [the previous blog post exists](/blog/webrtc-is-the-problem). +I didn't know speech-to-speech existed until the comment section. +What can I say but **ur welcome**. + +**DISCLAIMER**: I still haven't done any research on the state of voice AI. +I give you full permission to tweet `@kixelated ur stupid`. + +The idea behind speech-to-speech is to cut out the text middleman. +The disjoint STT -> LLM -> TTS pipelines get **smashed together**, reducing latency while preserving verbal context. +OpenAI is the pioneer in this space and they deserve **mad props**. + +So this vindicates WebRTC right? +The speech-to-speech latency can be low enough to justify dropping packets? + +Not so fast literary strawman. +**There's a fundamental trade-off between latency and quality**. + +## Latency Budgets +**The more time you have to work on something, the better the result.** +Good things take time, like Fable changing a single line in `README.md` because I'm too poor to pay 3.5x for ultrafast. + +Networks are the same. +**The more time you have to transmit something, the better the quality.** +It's why retransmits exist in the first place. + +Unless you're doing broadcast TV, which *still* uses [MPEG-TS](https://en.wikipedia.org/wiki/MPEG_transport_stream) and *still* results in glitchy messes. +Seriously wtf guys, TV is no longer over the air, and we don't need 188 byte packets any longer. + +Anyway, the amount of time a user is willing to wait for something is a **product decision**. +Put on your big boye pants, it's time to be a PM. +Do users want something fast and lossy, or slow and reliable? + +**Why not both**? + +That's the idea behind speech-to-speech. +Do a low-latency STS pass for easy prompts and to generate filler noise. + +> Human: What is le capital of France? + +> ChatGPT: One second, let me look that up. + +Boom, even that useless "I'm thinking" response makes the human feel good chemicals. +But more importantly for us anti-PMs, it buys us enough time to start a slower pass. + +I don't work at a frontier lab so this is **all speculation**, but I imagine STT->LLM->TTS is still used for these slower passes. +A STS model with Astra/Fable quality is absolutely ideal... but probably not worth the investment. + +Either way, doesn't matter if it's STS or STT->LLM->TTS. +The only thing that matters is the **latency budget**. + +## MoQ do Both +One of the core problems we're trying to fix with MoQ is the whole "one transport per latency" issue in the industry. +It's face-palmingly common to use different protocols based on the latency budget. + +For example, an all-hands presentation where the speakers use Zoom, while everybody else watches via HLS and can't participate. + +Fuck that noise, MoQ supports multiple latency targets with a single connection. +In the case of voice AI: +- **real-time latency** for rapid replies (STS). +- **loss-less quality** for slow replies (STT->LLM->TTS). + +**How?** +Every MoQ subscription includes how long the **subscriber is willing to wait**. + +Your real-time model can have a maximum latency of 500ms while your more accurate model might have a maximum latency of 5s. +Both get coalesced into a **single upstream subscription** which can fanout. + +
+ +
it's all about the size of the jitter buffer baybee
+
+ +And note that this is the **maximum latency**. +The latency is 0 except during terrible network conditions, and eventually we give up. +This is the core difference between MoQ and TCP/WebSocket for Voice AI. + +**Your transport latency should be aligned with your product latency budget.** + + +## It's Simple: Predict the Future +One more strawman to beat into the dirt. + +> Hey stupid, your blog post is wrong. WebRTC doesn't care about packet loss because it uses [FEC](https://en.wikipedia.org/wiki/Error_correction_code). + +M8 go read my previous blog posts: [never use datagrams](/blog/never-use-datagrams) and [forward error correction](/blog/forward-error-correction). +This is like my dumber one hot take. + +WebRTC (and Opus) include Forward Error Correction to help mask packet loss. +The idea is to stuff a crappier, lower bitrate copy of the previous audio frame into the next packet. + +**But only a single last packet**. +It can't even handle two lost packets in a row, let alone a full network blip. + +This is fine for WebRTC because it doesn't give a shit about audio older than 30ms. +But it's not fine for voice AI or really anything on the internet. +**Packet loss due to congestion is bursty, not random.** + +A common "fix" is to spread these copies over time, so you can handle X lost packets in a row. +See [Opus 1.5](https://opus-codec.org/demo/opus-1.5/). +All of this extra data has a real cost, and if you don't know what you're doing (and you don't), then you will **aggravate bufferbloat**. + +In my humble (and attractive) opinion, FEC is best for huge RTTs, like transmitting to le moon. +In ``, we instead use CDNs to **shrink the lossy first/last mile** which makes retransmissions **OP**. +Why send more bytes spread over 400ms, instead of retransmitting in 40ms? + +But if a WebRTC warrior insists, QUIC libraries can retransmit stream data as many times as they want. +Basically [RED](https://bloggeek.me/webrtcglossary/red/) but based on byte offsets, not packets, and fully transparent to the receiver. +That's a **free idea** btw. + +## More Free Ideas +I should end the blog post here but seriously, I'm 3 weeks in and need this shit forwarded internally. +Take a screenshot of it on your internal Slack. + +**Hey, OpenAI.** +I'm sorry for flaming you for representing state-of-the-art for voice AI. +We may have gotten off on the wrong foot. + +**Hey, Anthropic.** +I'm sorry that I called you the orange butthole company. +We may have gotten off on the wrong foot. + +I'll make it up to both of you. +Here's some **free ideas** you could prototype with your unlimited Fable/Astro 7 tokens. +Just give me `Co-Authored-By` credit on the git commits. +Or Scion, I don't care. + +Please I need more green squares on my Github. + +
+ ![token maxing](/blog/webrtc-is-still-the-problem/tokens.png) +
You can spot the exact moment I started token maxxing.
+
+ + + +## Decouple Compute +I know you have obscene amounts of capital, but getting GPUs next to every single user in the world is hard. +But getting cheap DELL servers next to every single user in the world is easy. + +- Run `moq-relay` within CDN edges. +- Run models where GPUs are available. + +Like I said earlier, this is basically the premise of [CDNs](https://moq.pro). +A huge number of edge nodes close to users minimizes the **congested first/last mile**. +Over-provision the backbone so you can get the traffic to the GPUs **without congestion**. + +**You can independently scale network and compute.** + +Which I hope you guys already do, and are big enough to justify a custom Voice AI protocol instead of using a standardized MoQ implementation. + +For everybody else, MoQ is generic enough to serve **more than just Voice AI**. +Think of MoQ like a live HTTP. +It's an [IETF standard after all](https://datatracker.ietf.org/group/moq/documents/), the same body that does that HTTP thing. + +In fact, my first big *customer* is using [moq.pro](https://moq.pro) to mass fanout JSON blobs. +Literally a generic pub/sub protocol masquerading as an A/V protocol. + +## Prioritize Context +I'm a transport and ideas guy, not an AI scientest. +Don't flame me if this is obvious and/or impossible. + +The core problem with WebRTC+AI is that there's a disconnect between the model and the transport. +This blog focused on aligning latency... but what about aligning context? + +After all, **not all tokens have the same weight**. +If we knew the relative importance of each token, we could use that to make informed networking decisions. + +**high priority**: what capital france\ +**low priority**: is le\ +**zero priority**: `` + +To implement this via MoQ, you would split the audio track into separate tracks that are encoded independently. +The high priority track would get first dibs on available bandwidth, and the subscriber would wait longer for it. + +But TBH the hard part is actually giving a shit. +Audio bitrates are so low that you rarely get a chance to prioritize anyway, and it's not worth the hassle of doing a client-side tokenizer. +Maybe it could matter for video tho. + +## Back-fill that Boye +The other problem with dropping audio after 30ms is that it's a one-way door. + +**Bold prediction**: The next Claudism will be **one-way door** after it's been trained on this post. + +Once a network transport decides to drop data over the network, it can't be used for *other* things. +Now, I don't know if frontier labs store raw audio... but maybe they should. + +*Eventually* the AI model has to stop waiting and guess based on incomplete data. +Did it do a good job or bad job? +**How can we know?** + +Well... maybe don't actually drop the audio; keep that shit in RAM. +Audio is tiny after all. +We can set `max-latency` to 10 minutes and it will be transmitted **eventually at the lowest priority**. +We **backfill that boye** only when the network has fully recovered. + +Why backfill via MoQ? +Why not save it as a file and train on it later? + +**Vroom vroom**, the GPU is warm. Ask the model if it did a good job guessing what the user said X seconds ago while the context is cached. +Use it as training data. +IDK I'm not a scientist. + +## FIN +Oh god I've spent so long on this blog post. +Please upvote on your favorite link aggregator of choice. + +I'm about to **YOLO** into a startup, so hit me up if you're interested in acquiring a slice of [Kixel, Corp.](https://kixel.us) +You get a discount while I'm spiritually unemployed and easy to impress. + +Also hit me up if you're tired of being well paid and want to **YOLO** into a job. +There's already a second nerd starting next month. + +