1import {2 audioMinusVideoSkewMs,3 audioOutputTimelineFromStats,4 exactMediaTimestampMicros,5 isCurrentAvSyncEpoch,6 nextAvSyncEpoch,7 projectAudioMediaPtsMicros,8} from './av-sync.mjs';9import {10 isCurrentAudioDelivery,11 isCurrentAudioGeneration,12 parseAudioEnvelope,13 stageAudioTerminalState,14 takeAudioTerminalState,15} from './audio-envelope.mjs';16import { newAudioSession } from './audio-ui.mjs';1718export const AUDIO_SAMPLE_RATE = 48000;1920const AUDIO_DECODER_CONFIG = Object.freeze({21 codec: 'opus',22 sampleRate: AUDIO_SAMPLE_RATE,23 numberOfChannels: 2,24});25const MAX_AUDIO_DECODE_QUEUE_SIZE = 3;2627export function createAudioPipelineSession({28 invokeCommand,29 onUpdate = () => {},30 resetAvSync = () => {},31 isDisconnecting = () => false,32 hasAudioDecoder = () => typeof globalThis.AudioDecoder === 'function',33 hasAudioWorklet = () => typeof globalThis.AudioWorkletNode === 'function',34 getAudioContextConstructor = () => globalThis.AudioContext || globalThis.webkitAudioContext,35 probeDecoderSupport = (config) => globalThis.AudioDecoder.isConfigSupported(config),36 createDecoder = (callbacks) => new globalThis.AudioDecoder(callbacks),37 createWorkletNode = (context, name, options) => new globalThis.AudioWorkletNode(38 context,39 name,40 options,41 ),42 createEncodedChunk = (init) => new globalThis.EncodedAudioChunk(init),43} = {}) {44 if (typeof invokeCommand !== 'function') throw new TypeError('invokeCommand must be a function');4546 let activeSession = newAudioSession();4748 function update() {49 onUpdate(activeSession);50 }5152 function setState(state, detail = state) {53 activeSession.state = state;54 activeSession.stateDetail = detail;55 update();56 }5758 function resetSyncTelemetry(session = activeSession, { recoverRing = false } = {}) {59 if (!session) return;60 session.audioTimeline = null;61 session.avSyncEpoch = nextAvSyncEpoch(session.avSyncEpoch);62 try {63 session.workletNode?.port.postMessage({64 type: recoverRing ? 'recover' : 'av-sync-epoch',65 avSyncEpoch: session.avSyncEpoch,66 });67 } catch (_) {}68 }6970 function sampleSkew(videoPtsMicros, presentationTimeMs) {71 const session = activeSession;72 if (73 exactMediaTimestampMicros(videoPtsMicros) === null74 || !session75 || session.failed76 || !session.audioTimeline77 || !session.available78 || session.context?.state !== 'running'79 || typeof session.context.getOutputTimestamp !== 'function'80 ) return null;8182 let outputTimestamp;83 try {84 outputTimestamp = session.context.getOutputTimestamp();85 } catch (_) {86 return null;87 }88 const audioPtsMicros = projectAudioMediaPtsMicros(89 session.audioTimeline,90 outputTimestamp,91 presentationTimeMs,92 );93 return audioMinusVideoSkewMs(audioPtsMicros, videoPtsMicros);94 }9596 function resetTelemetry(session = activeSession) {97 session.packetsReceived = 0;98 session.decoderDropped = 0;99 session.bufferedFrames = 0;100 session.underflows = 0;101 session.underflowDurationMicros = 0;102 session.silentDurationMicros = 0;103 session.transportDropped = 0;104 session.frontendDropped = 0;105 resetAvSync(session);106 update();107 }108109 function acknowledge(generation, deliveryId) {110 void invokeCommand('iroh_client_ack_audio', { generation, deliveryId }).catch((error) => {111 console.warn('audio acknowledgment failed:', error);112 });113 }114115 function requestNativeStop(session) {116 const generation = session?.expectedGeneration;117 if (session !== activeSession118 || !Number.isSafeInteger(generation)119 || generation < 0120 || session.stopRequested) return;121 session.stopRequested = true;122 void invokeCommand('iroh_client_stop_audio', { expectedGeneration: generation }).catch((error) => {123 console.warn('audio stop failed:', error);124 });125 }126127 function disable(detail, session = activeSession) {128 if (!session || session !== activeSession) return;129 resetAvSync(session);130 session.failed = true;131 session.failureDetail = detail;132 session.available = false;133 if (session.decoder) {134 try { session.decoder.close(); } catch (_) {}135 session.decoder = null;136 }137 session.messageTracker.clear();138 try { session.workletNode?.port.postMessage({ type: 'clear' }); } catch (_) {}139 requestNativeStop(session);140 setState('error', detail);141 }142143 function configureDecoder(session = activeSession) {144 if (session.decoder) {145 try { session.decoder.close(); } catch (_) {}146 }147 session.decoder = createDecoder({148 output: (audioData) => {149 try {150 if (session !== activeSession || session.failed) return;151 const workletNode = session?.workletNode;152 if (!workletNode || audioData.numberOfChannels !== 2) {153 throw new Error(`unexpected decoded audio channel count: ${audioData.numberOfChannels}`);154 }155 const timestampMicros = exactMediaTimestampMicros(audioData.timestamp);156 if (timestampMicros === null) throw new Error('decoded audio has an invalid media timestamp');157 // Reserve ownership before allocating/copying PCM. The MessagePort has158 // no useful queue-depth API, so this makes overload latest-frame-wins159 // at a hard ceiling of three decoded messages.160 const messageId = session.messageTracker.reserve();161 if (messageId === null) {162 update();163 return;164 }165 const channels = [];166 const transfers = [];167 try {168 for (let planeIndex = 0; planeIndex < 2; planeIndex++) {169 const channel = new Float32Array(audioData.numberOfFrames);170 audioData.copyTo(channel, { planeIndex, format: 'f32-planar' });171 channels.push(channel);172 transfers.push(channel.buffer);173 }174 workletNode.port.postMessage({175 type: 'samples',176 id: messageId,177 channels,178 timestampMicros,179 }, transfers);180 } catch (error) {181 session.messageTracker.accept(messageId);182 throw error;183 }184 } catch (error) {185 disable(`Decoded audio output failed: ${error}`, session);186 } finally {187 audioData.close();188 }189 },190 error: (error) => {191 disable(`Opus decoder failed: ${error}`, session);192 },193 });194 session.decoder.configure(AUDIO_DECODER_CONFIG);195 }196197 function primeOutputForActivation() {198 const session = activeSession;199 const AudioContextConstructor = getAudioContextConstructor();200 if (typeof AudioContextConstructor !== 'function') return;201 if (!session.context || session.context.state === 'closed') {202 try {203 session.context = new AudioContextConstructor({204 latencyHint: 'interactive',205 sampleRate: AUDIO_SAMPLE_RATE,206 });207 } catch (_) {208 return;209 }210 }211 // This function is deliberately synchronous and is called before the first212 // await in a click/keyboard connect handler, preserving WebKit user activation.213 void session.context.resume().catch(() => {});214 }215216 async function initialize(session) {217 if (session.decoder) {218 try { session.decoder.close(); } catch (_) {}219 session.decoder = null;220 }221 if (session.workletNode) {222 try { session.workletNode.disconnect(); } catch (_) {}223 session.workletNode = null;224 }225 session.messageTracker.clear();226 if (!hasAudioDecoder()) {227 return { supported: false, error: 'WebCodecs AudioDecoder is unavailable' };228 }229 if (!hasAudioWorklet()) {230 return { supported: false, error: 'AudioWorklet is unavailable' };231 }232 let support;233 try {234 support = await probeDecoderSupport(AUDIO_DECODER_CONFIG);235 } catch (error) {236 return { supported: false, error: `Opus capability probe failed: ${error}` };237 }238 if (!support.supported) {239 return { supported: false, error: 'WebCodecs Opus decoding is unsupported' };240 }241 if (!session.context) {242 return { supported: false, error: 'Web Audio output is unavailable' };243 }244 try {245 if (session.context.sampleRate !== AUDIO_SAMPLE_RATE) {246 throw new Error(`audio output opened at ${session.context.sampleRate} Hz instead of 48000 Hz`);247 }248 await session.context.audioWorklet.addModule(new URL('./audio-worklet.js', import.meta.url));249 session.workletNode = createWorkletNode(session.context, 'sigil-audio-processor', {250 numberOfInputs: 0,251 numberOfOutputs: 1,252 outputChannelCount: [2],253 });254 session.workletNode.port.onmessage = (event) => {255 if (event.data?.type === 'accepted') {256 session.messageTracker.accept(event.data.id);257 if (session === activeSession) update();258 } else if (event.data?.type === 'stats') {259 if (!isCurrentAvSyncEpoch(event.data.avSyncEpoch, session.avSyncEpoch)) return;260 session.workletDroppedFrames = Number.isSafeInteger(event.data.droppedFrames)261 && event.data.droppedFrames >= 0262 ? event.data.droppedFrames : session.workletDroppedFrames;263 session.workletRecoveryDiscardedFrames = Number.isSafeInteger(264 event.data.recoveryDiscardedFrames,265 ) && event.data.recoveryDiscardedFrames >= 0266 ? event.data.recoveryDiscardedFrames : session.workletRecoveryDiscardedFrames;267 session.audioTimeline = audioOutputTimelineFromStats(event.data);268 if (session !== activeSession) return;269 session.bufferedFrames = Number.isSafeInteger(event.data.bufferedFrames)270 ? event.data.bufferedFrames : session.bufferedFrames;271 session.underflows = Number.isSafeInteger(event.data.underflows)272 ? event.data.underflows : session.underflows;273 session.underflowDurationMicros = Number.isFinite(event.data.underflowDurationMicros)274 && event.data.underflowDurationMicros >= 0275 ? event.data.underflowDurationMicros : session.underflowDurationMicros;276 session.silentDurationMicros = Number.isFinite(event.data.silentDurationMicros)277 && event.data.silentDurationMicros >= 0278 ? event.data.silentDurationMicros : session.silentDurationMicros;279 if (session.available && !session.muted) {280 setState(event.data.started ? 'playing' : 'priming', 'bounded Opus playback');281 } else {282 update();283 }284 } else if (event.data?.type === 'error') {285 disable(`AudioWorklet failed: ${event.data.error}`, session);286 }287 };288 session.workletNode.port.onmessageerror = () => {289 disable('AudioWorklet returned an unreadable message', session);290 };291 session.workletNode.onprocessorerror = () => {292 disable('AudioWorklet processor stopped unexpectedly', session);293 };294 session.workletNode.connect(session.context.destination);295 session.workletNode.port.postMessage({ type: 'mute', muted: session.muted });296 configureDecoder(session);297 await session.context.resume();298 setState(299 session.context.state === 'running' ? 'negotiating' : 'blocked',300 session.context.state === 'running'301 ? 'Opus output ready'302 : 'WebKit suspended audio output; activate the audio button to retry',303 );304 return { supported: true, error: null };305 } catch (error) {306 await teardown(false);307 return { supported: false, error: `Audio initialization failed: ${error}` };308 }309 }310311 async function teardown(resetStatus = true) {312 const session = activeSession;313 resetAvSync(session);314 const decoder = session.decoder;315 const workletNode = session.workletNode;316 const context = session.context;317 // Replace the active owner before closing resources so delayed callbacks318 // cannot mutate the disconnected UI or a successor session. Seed the next319 // dormant session with the user's mute preference across reconnects.320 activeSession = newAudioSession({ muted: session.muted });321 session.expectedGeneration = null;322 session.channel = null;323 session.available = false;324 session.messageTracker.clear();325 session.decoder = null;326 session.workletNode = null;327 session.context = null;328 if (decoder) {329 try { decoder.close(); } catch (_) {}330 }331 if (workletNode) {332 try { workletNode.port.postMessage({ type: 'clear' }); } catch (_) {}333 try { workletNode.disconnect(); } catch (_) {}334 }335 if (context && context.state !== 'closed') {336 try { await context.close(); } catch (_) {}337 }338 if (resetStatus) update();339 }340341 async function toggleMute() {342 const session = activeSession;343 if (!session.available) return;344 session.muted = !session.muted;345 try {346 if (!session.muted && session.context?.state !== 'running') await session.context?.resume();347 session.workletNode?.port.postMessage({ type: 'mute', muted: session.muted });348 if (!session.muted && session.context?.state !== 'running') {349 setState('blocked', 'WebKit suspended audio output; click the audio button to retry');350 } else {351 update();352 }353 } catch (error) {354 setState('error', `Audio output activation failed: ${error}`);355 }356 }357358 function prepareForConnection() {359 activeSession = newAudioSession({ muted: activeSession.muted });360 activeSession.acceptsNativeEvents = true;361 // Keep this synchronous and before createConnectionAttempt's first await so362 // WebKit observes the originating click/keyboard user activation.363 primeOutputForActivation();364 }365366 function beginAttempt() {367 const session = activeSession;368 resetTelemetry(session);369 return session;370 }371372 async function createAttemptChannel(Channel, session = activeSession) {373 const support = await initialize(session);374 if (!support.supported) setState('unavailable', support.error);375 session.channel = new Channel((message) => handleBinaryMessage(message, session));376 return { session, support, channel: session.channel };377 }378379 function releaseAttempt(session) {380 if (session && session !== activeSession) session.channel = null;381 }382383 async function acceptConnectedResult(result, { session, support }) {384 session.expectedGeneration = Number.isSafeInteger(result.audio_generation)385 && result.audio_generation > 0386 ? result.audio_generation : null;387 const pendingTerminal = takeAudioTerminalState(388 session.pendingTerminalStates,389 session.expectedGeneration,390 );391 if (pendingTerminal !== null) {392 session.failed = true;393 session.failureDetail = pendingTerminal.error || 'Audio connection ended';394 }395 session.available = result.audio_available === true;396 if (session.available && session.failed) {397 disable(session.failureDetail || 'Audio output failed', session);398 } else if (session.available) {399 setState(400 session.context?.state === 'running' ? 'priming' : 'blocked',401 session.context?.state === 'running'402 ? 'Waiting for bounded Opus prebuffer'403 : 'WebKit suspended audio output; activate the audio button to retry',404 );405 } else {406 await teardown(false);407 setState('unavailable', result.audio_error || support.error || 'host audio unavailable');408 }409 }410411 function handleBinaryMessage(message, session) {412 let packet;413 try {414 packet = parseAudioEnvelope(message);415 } catch (error) {416 console.error('invalid audio packet:', error);417 disable(`Audio packet failed: ${error}`, session);418 return;419 }420421 // Every valid delivery is released using the token embedded in that exact422 // message. A delayed delivery from an old connection is acknowledged but423 // cannot enter the current decoder.424 acknowledge(packet.generation, packet.deliveryId);425 if (session !== activeSession426 || session.failed427 || !isCurrentAudioDelivery(packet, session.expectedGeneration)) return;428429 try {430 session.packetsReceived++;431 if (!session.decoder || session.decoder.state === 'closed') {432 session.decoderDropped++;433 update();434 return;435 }436 const queuedAudioPackets = session.decoder.decodeQueueSize;437 if (packet.discontinuity || queuedAudioPackets >= MAX_AUDIO_DECODE_QUEUE_SIZE) {438 resetAvSync(session, { recoverRing: true });439 session.decoderDropped += queuedAudioPackets;440 configureDecoder(session);441 }442 session.decoder.decode(createEncodedChunk({443 type: 'key',444 timestamp: packet.ptsMicros,445 duration: (packet.frameSamples * 1_000_000) / packet.sampleRate,446 data: packet.data,447 }));448 update();449 } catch (error) {450 console.error('undecodable audio packet:', error);451 disable(`Audio packet failed: ${error}`, session);452 }453 }454455 function handleNativeState(payload) {456 const session = activeSession;457 if (isDisconnecting()458 || !session.acceptsNativeEvents459 || payload?.available !== false) return;460 if (session.expectedGeneration === null) {461 try {462 stageAudioTerminalState(session.pendingTerminalStates, payload);463 } catch (error) {464 console.error('invalid pre-connect audio terminal state:', error);465 }466 return;467 }468 if (!isCurrentAudioGeneration(payload?.generation, session.expectedGeneration)) return;469 disable(payload.error || 'Audio connection ended', session);470 }471472 function handleNativeStats(payload) {473 const session = activeSession;474 if (!isCurrentAudioGeneration(payload?.generation, session.expectedGeneration)) return;475 session.transportDropped = Number.isSafeInteger(payload?.sequence_dropped_total)476 ? payload.sequence_dropped_total : session.transportDropped;477 session.frontendDropped = Number.isSafeInteger(payload?.frontend_dropped_total)478 ? payload.frontend_dropped_total : session.frontendDropped;479 update();480 }481482 return Object.freeze({483 acceptConnectedResult,484 beginAttempt,485 createAttemptChannel,486 handleNativeState,487 handleNativeStats,488 prepareForConnection,489 releaseAttempt,490 resetSyncTelemetry,491 sampleSkew,492 teardown,493 toggleMute,494 get session() {495 return activeSession;496 },497 });498}