diff --git a/README.md b/README.md index d5e7ec12..b4705451 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,14 @@ it is unavailable or declined; the signed integrity trail and the report record that neutral condition and why. A recorded interview still requires its camera before it can start. +Where the browser supports device selection, preflight offers microphone and +output selectors using System default until another device is chosen. Changing +the microphone releases the previous track and requires a fresh sound check; +the selected track is published to the interview. The output selection applies +to test sounds and Jim's voice, is shared with the Meet sidebar, and is +remembered across reloads. A saved output that is no longer listed falls back +to System default. + Candidates can present the interview in Google Meet by sharing the CodeTrial tab with tab audio enabled. Meet owns the shared tab after that, and face-presence analysis is disabled for the session. See the diff --git a/tests/browser/audio-input.test.js b/tests/browser/audio-input.test.js new file mode 100644 index 00000000..7e711aaf --- /dev/null +++ b/tests/browser/audio-input.test.js @@ -0,0 +1,479 @@ +import { after, before, test } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_RUNTIME_CONFIG, + FAKE_SPEECH_CAPTURE, + launchChromium, + startStaticServer, +} from "./source.js"; + +let browser; +let server; +let base; +before(async () => { + browser = await launchChromium({ + args: [ + "--use-fake-device-for-media-stream", + "--use-fake-ui-for-media-stream", + FAKE_SPEECH_CAPTURE, + ], + }); + if (browser) + ({ server, base } = await startStaticServer({ + runtimeConfig: DEFAULT_RUNTIME_CONFIG, + })); +}); +after(async () => { + await browser?.close(); + await new Promise((resolve) => (server ? server.close(resolve) : resolve())); +}); + +async function preflight(t, { enumeration = true, deviceIds = true } = {}) { + const page = await browser.newPage(); + t.after(() => page.close()); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + t.after(() => assert.deepEqual(errors, [])); + await page.addInitScript( + ({ enumeration, deviceIds }) => { + if (!deviceIds) + navigator.mediaDevices.getSupportedConstraints = () => ({}); + window.inputs = [ + { + kind: "audioinput", + deviceId: "default", + label: "Default microphone", + }, + { kind: "audioinput", deviceId: "usb", label: "USB microphone" }, + { kind: "audioinput", deviceId: "headset", label: "Headset" }, + { kind: "audiooutput", deviceId: "speaker", label: "Speaker" }, + ]; + navigator.mediaDevices.enumerateDevices = enumeration + ? async () => window.inputs + : undefined; + const capture = navigator.mediaDevices.getUserMedia.bind( + navigator.mediaDevices, + ); + window.microphones = []; + window.audioRequests = []; + navigator.mediaDevices.getUserMedia = async (constraints) => { + if (!constraints.audio) { + const stream = await capture(constraints); + window.cameraTrack = stream.getVideoTracks()[0]; + return stream; + } + const deviceId = constraints.audio.deviceId?.exact || ""; + window.audioRequests.push(deviceId); + if (window.refusedInput === deviceId) + throw new DOMException( + "Selected microphone refused", + window.refusedInputError || "NotAllowedError", + ); + // Hardware ids are synthetic; capture real fake-media tracks so the + // production meter and publication code still receive MediaStreamTracks. + const audio = { ...constraints.audio }; + delete audio.deviceId; + const stream = await capture({ audio }); + const track = stream.getAudioTracks()[0]; + track.testDeviceId = deviceId; + const stop = track.stop.bind(track); + track.stop = () => { + const stopped = JSON.parse( + sessionStorage.getItem("microphone-stops") || "[]", + ); + stopped.push(track.id); + sessionStorage.setItem("microphone-stops", JSON.stringify(stopped)); + stop(); + }; + window.microphones.push(track); + if (window.heldInput === deviceId) + await new Promise((resolve) => { + window.releaseInput = resolve; + }); + return stream; + }; + }, + { enumeration, deviceIds }, + ); + await page.goto(`${base}/interview.html?problem=chargeback-pair-match`); + await page.waitForFunction( + (count) => + window.microphones.length && + window.cameraTrack && + document.querySelector("#preflight-input").options.length === count, + enumeration && deviceIds ? 3 : 1, + ); + return page; +} + +test("preflight offers one default and microphone devices without changing initial capture", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + assert.deepEqual( + await page.locator("#preflight-input option").allTextContents(), + ["System default", "USB microphone", "Headset"], + ); + assert.equal(await page.locator("#preflight-input").inputValue(), ""); + assert.deepEqual(await page.evaluate(() => window.audioRequests), [""]); + const original = await page.evaluate(() => window.microphones[0].id); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => + window.microphones.some((track) => track.testDeviceId === "usb"), + ); + assert.deepEqual( + await page.evaluate(() => ({ + old: window.microphones[0].readyState, + current: window.microphones.at(-1).readyState, + camera: window.cameraTrack.readyState, + })), + { old: "ended", current: "live", camera: "live" }, + ); + assert.notEqual( + await page.evaluate(() => window.microphones.at(-1).id), + original, + ); +}); + +test("switching microphones closes the gate until the replacement proves sound", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.locator("#camera-skip").click(); + await page.locator("#audio-heard").click(); + await page.waitForFunction( + () => !document.querySelector("#audio-check-join").disabled, + ); + await page.evaluate(() => { + window.heldInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => Boolean(window.releaseInput)); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.equal( + await page + .locator("#audio-step-mic") + .evaluate((node) => node.classList.contains("done")), + false, + ); + assert.equal( + await page + .locator("#audio-meter-fill") + .evaluate((node) => node.style.width), + "0%", + ); + await page.evaluate(() => { + window.heldInput = null; + window.releaseInput(); + }); + await page.waitForFunction( + () => !document.querySelector("#audio-check-join").disabled, + ); + assert.equal(await page.locator("#audio-heard").textContent(), "Confirmed"); +}); + +test("rapid microphone selections discard an earlier pending grant", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + window.heldInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => Boolean(window.releaseInput)); + await page.locator("#preflight-input").selectOption("headset"); + assert.deepEqual(await page.evaluate(() => window.audioRequests), [ + "", + "usb", + ]); + await page.evaluate(() => { + window.heldInput = null; + window.releaseInput(); + }); + await page.waitForFunction( + () => window.microphones.at(-1).testDeviceId === "headset", + ); + assert.equal( + await page.evaluate( + () => + window.microphones.find((track) => track.testDeviceId === "usb") + .readyState, + ), + "ended", + ); + assert.equal(await page.locator("#preflight-input").inputValue(), "headset"); + assert.equal( + await page.evaluate(() => window.microphones.at(-1).readyState), + "live", + ); +}); + +test("a browser without device enumeration still captures the default microphone", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, { enumeration: false }); + assert.equal(await page.locator("#preflight-input").isDisabled(), true); + assert.match( + await page.locator("#preflight-input-note").textContent(), + /system default/, + ); + assert.equal( + await page.evaluate(() => window.microphones[0].readyState), + "live", + ); + await page.waitForFunction(() => + document.querySelector("#audio-step-mic").classList.contains("done"), + ); +}); + +test("unsupported device-id constraints keep microphone selection on default", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, { deviceIds: false }); + assert.equal(await page.locator("#preflight-input").isDisabled(), true); + assert.equal(await page.locator("#preflight-input").inputValue(), ""); + assert.deepEqual(await page.evaluate(() => window.audioRequests), [""]); + await page.waitForFunction(() => + document.querySelector("#audio-step-mic").classList.contains("done"), + ); +}); + +test("a refused microphone blocks preflight until a different device succeeds", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + window.refusedInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => + document + .querySelector("#audio-check-status") + .textContent.includes("Selected microphone refused"), + ); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.equal( + await page.evaluate(() => window.microphones[0].readyState), + "ended", + ); + await page.locator("#preflight-input").selectOption("headset"); + await page.waitForFunction(() => + document.querySelector("#audio-step-mic").classList.contains("done"), + ); + assert.equal( + await page.evaluate(() => window.microphones.at(-1).testDeviceId), + "headset", + ); +}); + +test("failed capture retries preserve device menus until a grant or devicechange", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + const enumerate = navigator.mediaDevices.enumerateDevices; + window.deviceListings = 0; + navigator.mediaDevices.enumerateDevices = async () => { + window.deviceListings += 1; + return enumerate(); + }; + window.inputOption = document.querySelector("#preflight-input").options[1]; + window.outputOption = + document.querySelector("#preflight-output").options[0]; + window.refusedInput = "usb"; + window.refusedInputError = "OverconstrainedError"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction( + () => window.audioRequests.filter((id) => id === "usb").length >= 2, + ); + assert.deepEqual( + await page.evaluate(() => ({ + listings: window.deviceListings, + inputRetained: window.inputOption.isConnected, + outputRetained: window.outputOption.isConnected, + })), + { listings: 0, inputRetained: true, outputRetained: true }, + ); + await page.evaluate(() => { + window.inputs.find((device) => device.deviceId === "headset").label = + "Granted headset"; + }); + await page.locator("#preflight-input").selectOption("headset"); + await page.waitForFunction( + () => + document.querySelector('#preflight-input option[value="headset"]') + .textContent === "Granted headset", + ); + await page.evaluate(() => { + window.inputs.find((device) => device.deviceId === "usb").label = + "Reconnected USB"; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => + document.querySelector('#preflight-input option[value="usb"]') + .textContent === "Reconnected USB", + ); +}); + +test("unplugging the selected microphone requests default and reconnecting refreshes choices", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction( + () => window.microphones.at(-1).testDeviceId === "usb", + ); + await page.evaluate(() => { + window.removedInput = window.inputs.find( + (device) => device.deviceId === "usb", + ); + window.inputs = window.inputs.filter((device) => device.deviceId !== "usb"); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => + document.querySelector("#preflight-input").value === "" && + window.microphones.at(-1).testDeviceId === "", + ); + assert.equal( + await page.evaluate( + () => + window.microphones.find((track) => track.testDeviceId === "usb") + .readyState, + ), + "ended", + ); + await page.evaluate(() => { + window.inputs.push(window.removedInput); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-input").options.length === 3, + ); + assert.equal(await page.locator("#preflight-input").inputValue(), ""); +}); + +test("microphone enumeration errors recover and ignore an older reply", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => { + throw new Error("Listing refused"); + }; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-input-note") + .textContent.includes("could not be listed"), + ); + assert.equal( + await page.locator("#preflight-input-note").textContent(), + "Microphones could not be listed. Check your browser permissions and reload the page.", + ); + await page.evaluate(() => { + window.pendingLists = []; + window.holdLists = true; + navigator.mediaDevices.enumerateDevices = () => + window.holdLists + ? new Promise((resolve) => { + window.pendingLists.push(resolve); + }) + : Promise.resolve(window.inputs); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + window.staleLists = [...window.pendingLists]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + for (const resolve of window.pendingLists.slice(window.staleLists.length)) + resolve(window.inputs); + window.holdLists = false; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-input-note").hidden, + ); + await page.evaluate(async () => { + for (const resolve of window.staleLists) resolve([]); + await Promise.resolve(); + }); + assert.equal(await page.locator("#preflight-input option").count(), 3); +}); + +test("the chosen microphone track is the one handed to LiveKit", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.locator("#preflight-input").selectOption("usb"); + await page.locator("#camera-skip").click(); + await page.locator("#audio-heard").click(); + await page.waitForFunction( + () => !document.querySelector("#audio-check-join").disabled, + ); + const selected = await page.evaluate(() => window.microphones.at(-1).id); + await page.route("**/api/token", (route) => + route.fulfill({ + json: { serverUrl: "wss://test.invalid", token: "test", durationMin: 45 }, + }), + ); + await page.evaluate(() => { + const events = window.LivekitClient.RoomEvent; + window.published = []; + window.LivekitClient = { + RoomEvent: events, + Track: { Source: { Microphone: "microphone", Camera: "camera" } }, + Room: class { + localParticipant = { + async publishTrack(track, options) { + window.published.push({ + id: track.id, + kind: track.kind, + deviceId: track.testDeviceId, + source: options.source, + }); + // Stop at publication: the test has no LiveKit service or agent. + throw new Error("Test publication captured"); + }, + }; + on() { + return this; + } + async connect() {} + async disconnect() {} + }, + }; + }); + await page.locator("#audio-check-join").click(); + await page.waitForFunction(() => window.published.length > 0); + assert.deepEqual(await page.evaluate(() => window.published), [ + { id: selected, kind: "audio", deviceId: "usb", source: "microphone" }, + ]); +}); + +test("leaving preflight stops a microphone grant that arrives late", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + window.heldInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => Boolean(window.releaseInput)); + let releaseLobby; + const lobby = new Promise((resolve) => { + releaseLobby = resolve; + }); + await page.route(`${base}/`, async (route) => { + await lobby; + await route.continue(); + }); + try { + const held = await page.evaluate(() => { + const ids = window.microphones.map((track) => track.id); + document.querySelector("#audio-check-leave").click(); + window.releaseInput(); + return ids; + }); + releaseLobby(); + await page.waitForURL(`${base}/`); + const stopped = await page.evaluate(() => + JSON.parse(sessionStorage.getItem("microphone-stops") || "[]"), + ); + assert.ok(held.length >= 2); + assert.deepEqual( + held.filter((id) => !stopped.includes(id)), + [], + ); + } finally { + releaseLobby(); + } +}); diff --git a/tests/browser/audio-playback-browser.test.js b/tests/browser/audio-playback-browser.test.js index 62c0b33a..ccb1e385 100644 --- a/tests/browser/audio-playback-browser.test.js +++ b/tests/browser/audio-playback-browser.test.js @@ -86,9 +86,12 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon for (const error of errors) t.diagnostic(`page error: ${error}`); }); await page.addInitScript(() => { + delete HTMLMediaElement.prototype.setSinkId; const nativePlay = HTMLMediaElement.prototype.play; window.blockJim = true; HTMLMediaElement.prototype.play = function () { + if (this.id === "jim-audio" && window.jimMutedAtPlay === undefined) + window.jimMutedAtPlay = this.muted; return this.id === "jim-audio" && window.blockJim ? Promise.reject( new DOMException("Test autoplay denial", "NotAllowedError"), @@ -137,6 +140,8 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon await page.waitForFunction( () => !document.querySelector("#audio-check-join").disabled, ); + const outputNote = await page.locator("#preflight-output-note").textContent(); + assert.match(outputNote, /cannot choose an output device/); await page.locator("#audio-check-join").click(); await page.waitForFunction( () => @@ -144,6 +149,14 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon document.querySelector("#mic").textContent === "Mic on", ); await page.evaluate(() => { + window.fakeLivekit = window.LivekitClient; + }); + await page.addScriptTag({ + url: `${base}/vendor/livekit-client.js?real-attach`, + }); + await page.evaluate(async () => { + const RemoteAudioTrack = window.LivekitClient.RemoteAudioTrack; + window.LivekitClient = window.fakeLivekit; const room = window.testRoom; const participant = { identity: "interviewer-jim", @@ -152,19 +165,34 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon room.remoteParticipants.set(participant.identity, participant); room.emit("Reconnecting"); room.emit("Reconnected"); - window.testTrack = { - kind: "audio", - attach: (audio) => { - // A live track never ends; loop the short fixture so it does not - // report paused again before the wait after resume sees it. - audio.loop = true; - audio.src = "/audio/test-voice.wav"; - }, - detach: () => [document.querySelector("#jim-audio")], + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + window.testTrack = new RemoteAudioTrack( + stream.getAudioTracks()[0], + "test-jim", + ); + const attach = window.testTrack.attach.bind(window.testTrack); + window.testTrack.attach = (audio) => { + const result = attach(audio); + window.jimMutedAtAttach = audio.muted; + return result; }; room.emit("TrackSubscribed", window.testTrack, {}, participant); }); await page.waitForSelector("#audio-playback-notice:not([hidden])"); + assert.deepEqual( + await page.evaluate(() => [window.jimMutedAtAttach, window.jimMutedAtPlay]), + [false, true], + ); + assert.equal( + await page.locator("#preflight-output-note").textContent(), + outputNote, + ); + assert.equal( + await page + .locator("#preflight-output-note") + .evaluate((note) => note.hidden), + false, + ); assert.equal(await page.locator("#jim-stage #resume-audio").count(), 1); await page.locator("#resume-audio").click(); await page.waitForFunction( @@ -172,6 +200,10 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon document.querySelector("#audio-playback-notice").hidden && !document.querySelector("#jim-audio").paused, ); + assert.equal( + await page.locator("#jim-audio").evaluate((audio) => audio.muted), + false, + ); assert.equal(await page.evaluate(() => window.testRoom.starts), 1); assert.equal( await page.evaluate(() => document.activeElement.id), diff --git a/tests/browser/audio-playback.test.js b/tests/browser/audio-playback.test.js index 1d2c1c65..62d49598 100644 --- a/tests/browser/audio-playback.test.js +++ b/tests/browser/audio-playback.test.js @@ -6,6 +6,7 @@ import { playbackBlocked, } from "../../web/audio-playback.js"; import { loadInterview } from "./source.js"; +import { createJimVolume } from "../../web/audio-output.js"; const blocked = () => new DOMException("Gesture required", "NotAllowedError"); const tick = () => new Promise((resolve) => setImmediate(resolve)); @@ -328,12 +329,158 @@ function audioTrack() { }; } +test("releasing a routing hold detects blocked audible play and offers recovery", async () => { + const route = deferred(); + let allowed = false; + const element = { + volume: 1, + muted: false, + isConnected: false, + play() { + return this.muted || allowed + ? Promise.resolve() + : Promise.reject(blocked()); + }, + }; + let volume; + const f = fixture({ reapply: () => volume.apply() }); + const own = { + jimAudio: null, + jimTrack: null, + audioPlayback: f.playback, + isCurrentAgent: () => true, + applyAudioOutput: async () => {}, + audioOutputSettled: () => route.promise, + setTimeout: () => 1, + clearTimeout() {}, + setBanner() {}, + document: { + createElement: () => element, + body: { appendChild: (audio) => (audio.isConnected = true) }, + }, + }; + volume = own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: null, + }); + interviewFunction("playRemoteAudio", own)({ kind: "audio", attach() {} }, {}); + await tick(); + assert.equal(element.muted, true); + assert.equal(f.notice.hidden, true); + route.resolve(); + await tick(); + assert.equal(element.muted, false); + assert.equal(f.notice.hidden, false); + allowed = true; + f.click(); + await tick(); + assert.equal(f.notice.hidden, true); +}); + +test("a native pause after successful play rechecks permission and drops old listeners", async () => { + const f = fixture(); + let allowed = true; + let attempts = 0; + const audio = Object.assign(new EventTarget(), { + paused: false, + ended: false, + play() { + attempts += 1; + return allowed ? Promise.resolve() : Promise.reject(blocked()); + }, + }); + await f.playback.play(audio); + allowed = false; + audio.paused = true; + audio.dispatchEvent(new Event("pause")); + await tick(); + assert.equal(attempts, 2); + assert.equal(f.notice.hidden, false); + f.playback.forget(audio); + audio.dispatchEvent(new Event("pause")); + await tick(); + assert.equal(attempts, 2); + assert.equal(f.notice.hidden, true); +}); + +test("fresh Jim audio stays muted until the selected output route settles", async () => { + const route = deferred(); + const observed = []; + const element = { muted: false, volume: 1, isConnected: false }; + let expire; + const messages = []; + const own = { + jimAudio: null, + jimTrack: null, + isCurrentAgent: () => true, + audioOutputId: () => "headphones", + setTimeout(callback) { + expire = callback; + return 1; + }, + clearTimeout() {}, + setBanner(source, message) { + messages.push([source, message]); + }, + applyAudioOutput(id) { + assert.equal(id, "headphones"); + return Promise.resolve(); + }, + audioOutputSettled() { + return route.promise; + }, + audioPlayback: { + play(audio) { + observed.push(["play", audio.muted]); + return Promise.resolve(); + }, + }, + document: { + createElement: () => element, + body: { appendChild: (audio) => (audio.isConnected = true) }, + }, + }; + own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: null, + }); + const track = { + kind: "audio", + attach(audio) { + audio.muted = false; + observed.push(["attach", audio.muted]); + }, + }; + const play = interviewFunction("playRemoteAudio", own); + play(track, {}); + assert.deepEqual(observed, [ + ["attach", false], + ["play", true], + ]); + await tick(); + assert.equal(element.muted, true); + expire(); + assert.match(messages.at(-1)[1], /reload/); + route.resolve(); + await tick(); + assert.equal(element.muted, false); + assert.deepEqual(messages.at(-1), ["output", ""]); + play(track, {}); + assert.equal(own.jimAudio, element); + assert.deepEqual(observed.slice(-2), [ + ["attach", false], + ["play", false], + ]); +}); + test("a late unsubscribe for Jim's old track leaves the new track playing and recoverable", async () => { const f = fixture(); const own = { jimAudio: null, jimTrack: null, audioPlayback: f.playback, + applyAudioOutput: async () => {}, + audioOutputSettled: async () => {}, isCurrentAgent: () => true, isAvatarAnalyserTrack: () => false, document: { @@ -354,6 +501,10 @@ test("a late unsubscribe for Jim's old track leaves the new track playing and re body: { appendChild: (element) => (element.isConnected = true) }, }, }; + own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: null, + }); const play = interviewFunction("playRemoteAudio", own); const drop = interviewFunction("dropRemoteAudio", own); const old = audioTrack(); @@ -361,6 +512,7 @@ test("a late unsubscribe for Jim's old track leaves the new track playing and re play(old, {}); await tick(); const element = own.jimAudio; + assert.equal(element.muted, false); // Reconnect order LiveKit is allowed to use: the new subscribe first. play(replacement, {}); await tick(); @@ -368,6 +520,7 @@ test("a late unsubscribe for Jim's old track leaves the new track playing and re drop(old); assert.equal(own.jimAudio, element); assert.equal(element.isConnected, true); + assert.equal(element.muted, false); assert.equal(element.paused, false); assert.equal(f.notice.hidden, false); drop(replacement); diff --git a/tests/browser/devices.test.js b/tests/browser/devices.test.js index 0c8312c4..4da35e69 100644 --- a/tests/browser/devices.test.js +++ b/tests/browser/devices.test.js @@ -44,6 +44,166 @@ test("a granted track of each kind lands in one stream", async () => { assert.equal(pool.errorOf("audio"), null); }); +test("selecting a microphone replaces only audio and preserves processing constraints", async () => { + const asked = []; + const { pool } = poolWith(async (constraints) => { + asked.push(constraints); + return new FakeStream([ + new FakeTrack(constraints.audio ? "audio" : "video"), + ]); + }); + let lost = 0; + pool.configure("audio", { + onLost: () => { + lost += 1; + }, + }); + pool.start(); + await settle(); + const original = pool.trackOf("audio"); + const camera = pool.trackOf("video"); + const stream = pool.stream; + pool.selectMicrophone("usb"); + assert.equal(original.stopped, true); + assert.equal(pool.trackOf("audio"), null); + assert.equal(lost, 1); + await settle(); + assert.deepEqual(asked.at(-1), { + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + deviceId: { exact: "usb" }, + }, + }); + assert.equal(pool.stream, stream); + assert.equal(pool.trackOf("video"), camera); + assert.equal(camera.stopped, false); + assert.notEqual(pool.trackOf("audio"), original); + const count = asked.length; + pool.selectMicrophone("usb"); + assert.equal(asked.length, count); + pool.selectMicrophone(""); + await settle(); + assert.equal(asked.at(-1).audio.deviceId, undefined); + assert.equal(pool.stream.getTracks().length, 2); +}); + +test("rapid microphone choices stop the old grant and request only the latest device", async () => { + const pending = []; + const asked = []; + const { pool } = poolWith((constraints) => { + if (constraints.video) + return Promise.resolve(new FakeStream([new FakeTrack("video")])); + asked.push(constraints.audio.deviceId?.exact || ""); + return new Promise((resolve) => pending.push(resolve)); + }); + pool.start(); + pool.selectMicrophone("usb"); + pool.selectMicrophone("headset"); + assert.deepEqual(asked, [""]); + const abandoned = new FakeTrack("audio"); + pending.shift()(new FakeStream([abandoned])); + await settle(); + assert.equal(abandoned.stopped, true); + assert.equal(pool.trackOf("audio"), null); + assert.deepEqual(asked, ["", "headset"]); + const latest = new FakeTrack("audio"); + pending.shift()(new FakeStream([latest])); + await settle(); + assert.equal(pool.trackOf("audio"), latest); + assert.equal(latest.stopped, false); +}); + +test("an obsolete microphone rejection does not poison the latest choice", async () => { + let rejectOld; + let grantNew; + const { pool } = poolWith((constraints) => { + if (constraints.video) + return Promise.resolve(new FakeStream([new FakeTrack("video")])); + if (!constraints.audio.deviceId) + return new Promise((resolve, reject) => { + rejectOld = reject; + }); + return new Promise((resolve) => { + grantNew = resolve; + }); + }); + pool.start(); + pool.selectMicrophone("usb"); + rejectOld(new Error("Old device refused")); + await settle(); + assert.equal(pool.errorOf("audio"), null); + const selected = new FakeTrack("audio"); + grantNew(new FakeStream([selected])); + await settle(); + assert.equal(pool.trackOf("audio"), selected); +}); + +test("a failed selected microphone retries that device and can return to default", async () => { + const asked = []; + const retries = []; + const { pool } = poolWith( + async (constraints) => { + if (constraints.audio?.deviceId) { + asked.push(constraints.audio.deviceId.exact); + throw new Error("Selected device unavailable"); + } + return new FakeStream([ + new FakeTrack(constraints.audio ? "audio" : "video"), + ]); + }, + { + schedule: (callback) => { + retries.push(callback); + return retries.length; + }, + unschedule: () => {}, + }, + ); + pool.start(); + await settle(); + pool.selectMicrophone("missing"); + await settle(); + assert.equal(pool.trackOf("audio"), null); + assert.match(pool.errorOf("audio"), /Selected device unavailable/); + retries.shift()(); + await settle(); + assert.deepEqual(asked, ["missing", "missing"]); + pool.selectMicrophone(""); + await settle(); + assert.ok(pool.trackOf("audio")); + assert.equal(pool.errorOf("audio"), null); +}); + +test("leaving during a microphone change stops a late grant without requesting again", async () => { + let finished = false; + let grant; + let asked = 0; + const { pool } = poolWith( + (constraints) => { + if (constraints.video) + return Promise.resolve(new FakeStream([new FakeTrack("video")])); + asked += 1; + return new Promise((resolve) => { + grant = resolve; + }); + }, + { isFinished: () => finished }, + ); + pool.start(); + pool.selectMicrophone("usb"); + finished = true; + const late = new FakeTrack("audio"); + grant(new FakeStream([late])); + await settle(); + assert.equal(late.stopped, true); + assert.equal(pool.trackOf("audio"), null); + assert.equal(asked, 1); + pool.selectMicrophone("headset"); + assert.equal(asked, 1); +}); + // The bug this replaces: each device scheduled its own retry and both called // back into the same start, so denying both prompts doubled the number of // in-flight requests every two seconds until the tab died. diff --git a/tests/browser/jim-volume.test.js b/tests/browser/jim-volume.test.js index 5fbfcc84..a31f518c 100644 --- a/tests/browser/jim-volume.test.js +++ b/tests/browser/jim-volume.test.js @@ -19,6 +19,26 @@ function fixture(stored = null, audio = { volume: 1, muted: false }) { }; } +test("routing hold and candidate mute remain independent through volumechange", () => { + const audio = Object.assign(new EventTarget(), { volume: 1, muted: false }); + const f = fixture("37", audio); + f.volume.setRoutingHeld(true); + assert.equal(audio.muted, true); + assert.equal(f.volume.muted, false); + f.volume.setVolume(45); + audio.muted = false; + audio.dispatchEvent(new Event("volumechange")); + assert.equal(audio.muted, true); + f.volume.toggleMute(); + f.volume.setRoutingHeld(false); + assert.equal(audio.muted, true); + assert.equal(f.volume.muted, true); + f.volume.toggleMute(); + assert.equal(audio.muted, false); + assert.equal(audio.volume, 0.45); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "45"); +}); + for (const stored of [ null, "", diff --git a/tests/browser/meet-tab-audio.test.js b/tests/browser/meet-tab-audio.test.js index d5933e73..8a9f648d 100644 --- a/tests/browser/meet-tab-audio.test.js +++ b/tests/browser/meet-tab-audio.test.js @@ -12,12 +12,11 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { - assertIncludesCompact, firstPartyScripts as webScripts, interviewSource, root, } from "./source.js"; -import { outputAfterRouting, outputOptions } from "../../web/meet-audio.js"; +import { outputOptions } from "../../web/meet-audio.js"; const web = join(root, "web"); const read = (name) => readFileSync(join(web, name), "utf8"); @@ -121,102 +120,49 @@ test("meet-mode getusermedia allowlist", () => { // These execute the decisions instead of reading them. The text assertions // below still guard the wiring, but a wiring assertion cannot tell whether the // logic is right, and both bugs this feature shipped with were logic. -test("sink routing offers only devices the browser has named", () => { +test("output choices include one default and only named device ids", () => { const before = [{ kind: "audiooutput", deviceId: "", label: "" }]; const after = [ + { kind: "audiooutput", deviceId: "default", label: "System speaker" }, { kind: "audiooutput", deviceId: "speakers", label: "Built-in" }, { kind: "audioinput", deviceId: "mic", label: "Mic" }, { kind: "audiooutput", deviceId: "usb", label: "" }, ]; - // Pre-grant placeholder: an empty deviceId renders as a real choice and then - // routes to nothing, so it must not reach the list at all. - assert.deepEqual(outputOptions(before, null), []); + // The explicit default replaces blank placeholders and the browser's default + // alias, so neither can produce a duplicate choice. + assert.deepEqual(outputOptions(before, null), [ + { deviceId: "", label: "System default", selected: true }, + ]); assert.deepEqual(outputOptions(after, "usb"), [ + { deviceId: "", label: "System default", selected: false }, { deviceId: "speakers", label: "Built-in", selected: false }, { deviceId: "usb", label: "Output 2", selected: true }, ]); // An unnamed device still gets a usable label, and inputs never appear. assert.deepEqual( outputOptions(after, null).map((option) => option.deviceId), - ["speakers", "usb"], - ); -}); - -test("sink routing keeps the preference and the real sink in agreement", () => { - // Success: the pick becomes the preference. - assert.equal(outputAfterRouting("old", "new", true), "new"); - // Refusal rolls back. Keeping the refused device would leave the select - // showing a speaker Jim never played through, until the next reload. - assert.equal(outputAfterRouting("old", "new", false), "old"); - // Refused with nothing to fall back to: no preference at all, so the caller - // clears the key rather than persisting the device that just failed. - assert.equal(outputAfterRouting(null, "new", false), null); - // Reapplying a stored preference passes the same id as both, so there is no - // earlier device to fall back to. Returning it would write the refused id - // back and retry it on every reload. - assert.equal( - outputAfterRouting("same", "same", false), - "", - "a refused reapply must clear the preference, not re-store it", - ); - assert.equal( - outputAfterRouting("same", "same", true), - "same", - "and a reapply that works keeps it", + ["", "speakers", "usb"], ); }); // What the executable tests above cannot see: that interview.js actually wires // those decisions to the right nodes and the right track. test("sink routing", () => { - // setSinkId targets Jim's element and nothing else, so the candidate cannot - // accidentally reroute their own microphone monitoring. + // Playback routing must never target the candidate's microphone monitoring. const sinkCalls = [...script.matchAll(/(\w+)\s*\n?\s*\.setSinkId\(/g)].map( (match) => match[1], ); assert.deepEqual( sinkCalls, - ["jimAudio"], - "setSinkId may only target Jim's audio element", + ["target", "jimAudio", "jimAudio"], + "setSinkId targets only the test sounds and Jim's audio", ); assert.match( script, /element\.id = "jim-audio"/, "the remote audio element needs a stable id", ); - assert.match( - script, - /outputOptions\(devices, readStored\(AUDIO_OUTPUT_KEY\)\)/, - "the selector is built from the shared decision", - ); - // Persist before routing. A choice made during the preflight, before Jim's - // track exists, is dropped if it only survives a successful setSinkId. - // `applyAudioOutput` is the queue in front of this; the routing itself is - // what has to get the order right. - const apply = script.slice(script.indexOf("async function routeAudioOutput")); - assert.ok( - apply.indexOf("writeStored(AUDIO_OUTPUT_KEY") < - apply.indexOf(".setSinkId("), - "routing must persist before it routes", - ); - // Serialized and stamped. Two picks in flight can settle in either order, and - // the older one must not write its device over the newer one's. - assertIncludesCompact( - script, - "routingChain = routingChain.then(() => routeAudioOutput(deviceId, ++routingRequest))", - "routing requests are serialized", - ); - assert.match( - apply, - /if \(request !== routingRequest\) return;/, - "and a superseded request does not get the last word on the preference", - ); - assert.match( - apply, - /outputAfterRouting\(previous, deviceId, routed\)/, - "rollback comes from the tested decision", - ); // One element for the session, and it is released on unsubscribe. assert.match( script, @@ -228,11 +174,6 @@ test("sink routing", () => { /TrackUnsubscribed, dropRemoteAudio/, "unsubscribed audio must be detached", ); - // Unsupported browsers get told, not silently downgraded, and the whole row - // hides so the label does not dangle over a control that is gone. - assert.match(script, /"setSinkId" in HTMLMediaElement\.prototype/); - assert.match(script, /meetOutputRow\.hidden = true/); - assert.match(script, /showOutputNote\(OUTPUT_NOTES\.unsupported\)/); }); test("no synthetic microphone", () => { diff --git a/tests/browser/mic-meter.test.js b/tests/browser/mic-meter.test.js index ef1cca86..d0dfc958 100644 --- a/tests/browser/mic-meter.test.js +++ b/tests/browser/mic-meter.test.js @@ -122,6 +122,28 @@ test("a replacement microphone is proven on its own frames", () => { ); }); +test("callbacks from a replaced microphone cannot prove or fail the current one", () => { + const started = fakeMeter(); + const pool = fakePool(); + const meter = createMicMeter({ + pool, + isFinished: () => false, + onLevel: () => {}, + onFailure: () => assert.fail("a stale meter failed the current microphone"), + startMeter: started.start, + }); + meter.start(); + const previous = started.latest(); + meter.forget(); + meter.start(); + for (let index = 0; index < MIC_CONFIRM_FRAMES; index++) previous.onPeak(0.8); + previous.onError("Old meter stopped"); + assert.equal(meter.peak(), 0); + assert.equal(pool.errorOf("audio"), null); + loudEnough(started); + assert.ok(meter.peak() > 0); +}); + /// Timers a test can run by hand. /// /// The retry is the whole point of these two, and a real `setTimeout` would diff --git a/tests/browser/preflight-speech.test.js b/tests/browser/preflight-speech.test.js index d50db53d..507fa7d0 100644 --- a/tests/browser/preflight-speech.test.js +++ b/tests/browser/preflight-speech.test.js @@ -46,9 +46,10 @@ async function playLoopedSpeech(page) { ); } -async function preflight(t) { +async function preflight(t, initialize) { const page = await browser.newPage(); t.after(() => page.close()); + if (initialize) await page.addInitScript(initialize); await page.addInitScript(() => { const capture = navigator.mediaDevices.getUserMedia.bind( navigator.mediaDevices, @@ -66,6 +67,777 @@ async function preflight(t) { return page; } +for (const reset of [false, true]) { + for (const listing of ["empty", "placeholder", "failure"]) { + test(`saved output survives a pre-grant ${listing} listing and ${reset ? "can return to default" : "routes both test sounds"}`, async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await browser.newPage(); + t.after(() => page.close()); + await page.addInitScript((listing) => { + localStorage.setItem("codetrial:audioOutputId", "headphones"); + const capture = navigator.mediaDevices.getUserMedia.bind( + navigator.mediaDevices, + ); + navigator.mediaDevices.getUserMedia = async (constraints) => { + if (constraints.audio) + await new Promise((resolve) => { + window.grantMicrophone = resolve; + }); + const stream = await capture(constraints); + if (constraints.audio) window.microphoneGranted = true; + return stream; + }; + navigator.mediaDevices.enumerateDevices = async () => { + if (listing === "failure") throw new Error("Listing unavailable"); + if (window.microphoneGranted) + return [ + { + kind: "audiooutput", + deviceId: "headphones", + label: "Headphones", + }, + ]; + return listing === "empty" + ? [] + : [{ kind: "audiooutput", deviceId: "", label: "" }]; + }; + window.probeRoutes = []; + HTMLMediaElement.prototype.setSinkId = async function (id) { + window.probeRoutes.push(["speech", id]); + Object.defineProperty(this, "sinkId", { + configurable: true, + value: id, + }); + }; + AudioContext.prototype.setSinkId = async function (id) { + window.probeRoutes.push(["tone", id]); + Object.defineProperty(this, "sinkId", { + configurable: true, + value: id, + }); + }; + }, listing); + await page.goto(`${base}/interview.html?problem=chargeback-pair-match`); + await page.waitForFunction( + () => + window.grantMicrophone && + !document.querySelector("#preflight-output-note").hidden, + ); + for (const selector of ["#preflight-output", "#meet-output-select"]) { + assert.equal(await page.locator(selector).inputValue(), "headphones"); + assert.equal( + await page.locator(`${selector} option:checked`).textContent(), + "Saved output device", + ); + } + assert.equal( + await page.evaluate(() => + localStorage.getItem("codetrial:audioOutputId"), + ), + "headphones", + ); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction( + () => + document.querySelector("#audio-test-tone").textContent === + "Play it again", + ); + assert.deepEqual( + await page.evaluate(() => + window.probeRoutes.filter(([sound]) => sound === "tone"), + ), + [["tone", "headphones"]], + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction( + () => + !document.querySelector("#audio-test-voice").paused || + document.querySelector("#audio-test-voice").ended, + ); + assert.deepEqual( + await page.evaluate(() => + window.probeRoutes.filter(([sound]) => sound === "speech"), + ), + [["speech", "headphones"]], + ); + if (reset) { + await page.locator("#preflight-output").selectOption(""); + for (const selector of ["#preflight-output", "#meet-output-select"]) + assert.equal(await page.locator(selector).inputValue(), ""); + assert.equal( + await page.evaluate(() => + localStorage.getItem("codetrial:audioOutputId"), + ), + null, + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction(() => + window.probeRoutes.some( + ([sound, id]) => sound === "speech" && id === "", + ), + ); + } + await page.evaluate(() => window.grantMicrophone()); + await page.waitForFunction(() => window.microphoneGranted); + if (listing !== "failure") { + await page.waitForFunction( + () => + document.querySelector("#preflight-output option[value=headphones]") + ?.textContent === "Headphones", + ); + } + for (const selector of ["#preflight-output", "#meet-output-select"]) + assert.equal( + await page.locator(selector).inputValue(), + reset ? "" : "headphones", + ); + assert.equal( + await page.evaluate(() => + localStorage.getItem("codetrial:audioOutputId"), + ), + reset ? null : "headphones", + ); + }); + } +} + +test("preflight output defaults to the system and routes test sounds to the chosen device", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "default", label: "Default speaker" }, + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + { kind: "audioinput", deviceId: "microphone", label: "Microphone" }, + ]; + window.outputRoutes = []; + HTMLMediaElement.prototype.setSinkId = async function (id) { + window.outputRoutes.push({ target: this.id, id }); + Object.defineProperty(this, "sinkId", { configurable: true, value: id }); + }; + AudioContext.prototype.setSinkId = async function (id) { + window.outputRoutes.push({ target: "tone", id }); + Object.defineProperty(this, "sinkId", { configurable: true, value: id }); + }; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.deepEqual( + await page.locator("#preflight-output option").allTextContents(), + ["System default", "Headphones"], + ); + await page.locator("#audio-heard").click(); + await page.waitForFunction(() => + document.querySelector("#audio-step-output").classList.contains("done"), + ); + await page.locator("#preflight-output").selectOption("headphones"); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => route.target === "tone" && route.id === "headphones", + ), + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => + route.target === "audio-test-voice" && route.id === "headphones", + ), + ); + await page.locator("#preflight-output").selectOption(""); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => route.target === "tone" && route.id === "", + ), + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => route.target === "audio-test-voice" && route.id === "", + ), + ); +}); + +test("system-default test sounds leave already-default sinks unchanged", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + window.routeAttempts = 0; + const route = async () => { + window.routeAttempts += 1; + }; + HTMLMediaElement.prototype.setSinkId = route; + AudioContext.prototype.setSinkId = route; + }); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction( + () => + document.querySelector("#audio-test-tone").textContent === + "Play it again", + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction( + () => document.querySelector("#audio-test-voice").ended, + ); + assert.equal(await page.evaluate(() => window.routeAttempts), 0); +}); + +async function outputRoutingPage(t) { + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "default", label: "Default" }, + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + { kind: "audiooutput", deviceId: "speakers", label: "Speakers" }, + ]; + }); + await page.evaluate(async () => { + window.outputApi = await import("./audio-output.js"); + window.jimRoutes = []; + window.mockJim = { + sinkId: "", + async setSinkId(id) { + window.jimRoutes.push(id); + if (window.holdRoute) + await new Promise((resolve) => { + window.releaseRoute = resolve; + }); + if (id === window.refusedOutput) throw new Error("Device refused"); + this.sinkId = id; + }, + }; + window.outputApi.initAudioOutput({ + nodes: { + meetOutputSelect: document.querySelector("#meet-output-select"), + meetOutputRow: document.querySelector("#meet-output-row"), + meetOutputNote: document.querySelector("#meet-output-note"), + preflightOutput: document.querySelector("#preflight-output"), + preflightOutputNote: document.querySelector("#preflight-output-note"), + }, + jimAudioElement: () => window.mockJim, + }); + await window.outputApi.refreshAudioOutputs(); + }); + return page; +} + +test("both output selectors share choices and reset preflight confirmation", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + assert.deepEqual( + await page.locator("#meet-output-select option").allTextContents(), + await page.locator("#preflight-output option").allTextContents(), + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + assert.equal( + await page.locator("#meet-output-select").inputValue(), + "headphones", + ); + await page.locator("#audio-heard").click(); + await page.evaluate(() => { + const select = document.querySelector("#meet-output-select"); + select.value = "speakers"; + select.dispatchEvent(new Event("change")); + }); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + assert.equal( + await page.locator("#preflight-output").inputValue(), + "speakers", + ); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); +}); + +test("a refused Jim route restores both selectors to the actual previous sink", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.locator("#preflight-output").selectOption("speakers"); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + await page.evaluate(() => { + window.refusedOutput = "headphones"; + }); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("refused"), + ); + for (const id of ["preflight-output", "meet-output-select"]) { + assert.equal(await page.locator(`#${id}`).inputValue(), "speakers"); + } + assert.equal( + await page.evaluate(() => window.outputApi.audioOutputId()), + "speakers", + ); + assert.equal(await page.evaluate(() => window.mockJim.sinkId), "speakers"); + assert.equal( + await page.locator("#preflight-output-note").textContent(), + "That output device was refused. The previous output or system default is selected.", + ); + const refusal = await page.locator("#preflight-output-note").textContent(); + await page.evaluate(async () => { + await window.outputApi.refreshAudioOutputs(); + window.previousOutputOption = + document.querySelector("#preflight-output").options[0]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction(() => !window.previousOutputOption.isConnected); + assert.equal( + await page.locator("#preflight-output-note").textContent(), + refusal, + ); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + "speakers", + ); + await page.evaluate(() => { + window.holdRoute = true; + }); + await page.locator("#preflight-output").selectOption(""); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + assert.match( + await page.locator("#preflight-output-note").textContent(), + /refused/, + ); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => window.mockJim.sinkId === ""); + assert.equal(await page.locator("#preflight-output-note").isHidden(), true); +}); + +test("a pending Jim route finishes before the newest choice and cannot roll it back", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate(() => { + window.holdRoute = true; + }); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + await page.locator("#preflight-output").selectOption("speakers"); + assert.deepEqual(await page.evaluate(() => window.jimRoutes), ["headphones"]); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + assert.deepEqual(await page.evaluate(() => window.jimRoutes), [ + "headphones", + "speakers", + ]); + for (const id of ["preflight-output", "meet-output-select"]) { + assert.equal(await page.locator(`#${id}`).inputValue(), "speakers"); + } +}); + +test("a refused Jim route returns to default when its previous sink was unplugged", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + await page.evaluate(() => { + window.refusedOutput = "speakers"; + window.holdRoute = true; + }); + await page.locator("#preflight-output").selectOption("speakers"); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "speakers", label: "Speakers" }, + ]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("refused"), + ); + for (const id of ["preflight-output", "meet-output-select"]) { + assert.equal(await page.locator(`#${id}`).inputValue(), ""); + } + assert.equal(await page.evaluate(() => window.mockJim.sinkId), ""); + assert.equal(await page.evaluate(() => window.outputApi.audioOutputId()), ""); +}); + +test("output survives a replaced Jim element with storage blocked", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate(() => { + Object.defineProperty(window, "localStorage", { + configurable: true, + get() { + throw new Error("Storage blocked"); + }, + }); + }); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + await page.evaluate(async () => { + window.mockJim = { + sinkId: "", + async setSinkId(id) { + this.sinkId = id; + }, + }; + await window.outputApi.applyAudioOutput(window.outputApi.audioOutputId()); + }); + assert.equal(await page.evaluate(() => window.mockJim.sinkId), "headphones"); + await page.addInitScript(() => { + Object.defineProperty(window, "localStorage", { + get() { + throw new Error("Storage blocked"); + }, + }); + }); + await page.reload(); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 3, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); +}); + +test("output choices survive reload and System default clears the saved preference", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate(() => localStorage.setItem("reload-check", "available")); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + await page.evaluate(() => { + const select = document.querySelector("#meet-output-select"); + select.value = "speakers"; + select.dispatchEvent(new Event("change")); + }); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + "speakers", + ); + await page.reload(); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 3, + ); + assert.equal( + await page.locator("#preflight-output").inputValue(), + "speakers", + ); + assert.equal( + await page.locator("#meet-output-select").inputValue(), + "speakers", + ); + assert.equal( + await page.evaluate(() => localStorage.getItem("reload-check")), + "available", + ); + await page.locator("#preflight-output").selectOption(""); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + null, + ); + await page.reload(); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 3, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.equal(await page.locator("#meet-output-select").inputValue(), ""); +}); + +test("a saved output that is no longer listed falls back to System default", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + localStorage.setItem("codetrial:audioOutputId", "removed"); + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.waitForFunction( + () => localStorage.getItem("codetrial:audioOutputId") === null, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.equal(await page.locator("#meet-output-select").inputValue(), ""); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + null, + ); +}); + +for (const superseded of ["before routing starts", "during routing"]) { + test(`output settlement follows newer choices ${superseded}`, async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate((when) => { + window.holdRoute = true; + window.outputSettled = false; + void window.outputApi.applyAudioOutput("headphones"); + window.outputApi.audioOutputSettled().then(() => { + window.outputSettled = true; + }); + if (when === "before routing starts") + void window.outputApi.applyAudioOutput("speakers"); + }, superseded); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + if (superseded === "during routing") { + await page.evaluate(() => { + void window.outputApi.applyAudioOutput("speakers"); + window.releaseRoute(); + }); + await page.waitForFunction(() => window.jimRoutes.length === 2); + } + assert.equal(await page.evaluate(() => window.outputSettled), false); + await page.evaluate(() => { + void window.outputApi.applyAudioOutput(""); + window.releaseRoute(); + }); + await page.waitForFunction(() => window.jimRoutes.at(-1) === ""); + assert.equal(await page.evaluate(() => window.outputSettled), false); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => window.outputSettled); + assert.equal(await page.evaluate(() => window.mockJim.sinkId), ""); + }); +} + +test("output permission hint appears without specific devices and clears when they appear", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "", label: "" }, + ]; + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("microphone access"), + ); + assert.deepEqual( + await page.locator("#preflight-output option").allTextContents(), + ["System default"], + ); + for (const id of ["preflight-output-note", "meet-output-note"]) { + assert.equal( + await page.locator(`#${id}`).evaluate((note) => note.hidden), + false, + ); + } + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + for (const id of ["preflight-output-note", "meet-output-note"]) { + assert.equal( + await page.locator(`#${id}`).evaluate((note) => note.hidden), + true, + ); + } +}); + +test("unsupported output selection keeps system-default playback available", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + localStorage.setItem("codetrial:audioOutputId", "headphones"); + delete HTMLMediaElement.prototype.setSinkId; + }); + assert.equal(await page.locator("#preflight-output").isDisabled(), true); + assert.match( + await page.locator("#preflight-output-note").textContent(), + /system default/, + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction( + () => document.querySelector("#audio-test-voice").ended, + ); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); +}); + +test("output enumeration recovers and ignores an older failed refresh", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => { + throw new Error("Device list unavailable"); + }; + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("could not be listed"), + ); + await page.evaluate(() => { + window.lists = []; + window.holdLists = true; + const devices = [ + { kind: "audiooutput", deviceId: "speaker", label: "Speaker" }, + ]; + navigator.mediaDevices.enumerateDevices = () => + window.holdLists + ? new Promise((resolve, reject) => { + window.lists.push({ resolve, reject }); + }) + : Promise.resolve(devices); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + window.staleLists = [...window.lists]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + for (const request of window.lists.slice(window.staleLists.length)) + request.resolve(devices); + window.holdLists = false; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.evaluate(async () => { + for (const request of window.staleLists) + request.reject(new Error("Stale failure")); + await Promise.resolve(); + }); + assert.equal(await page.locator("#preflight-output-note").isHidden(), true); + assert.deepEqual( + await page.locator("#preflight-output option").allTextContents(), + ["System default", "Speaker"], + ); +}); + +test("removing the selected output restores default and requires confirmation again", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator("#audio-heard").click(); + await page.waitForFunction(() => + document.querySelector("#audio-step-output").classList.contains("done"), + ); + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => []; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 1, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.match( + await page.locator("#preflight-output-note").textContent(), + /microphone access/, + ); + assert.equal( + await page + .locator("#preflight-output-note") + .evaluate((note) => note.hidden), + false, + ); +}); + +test("a pending tone route cannot play after selecting away and back", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + window.routes = []; + window.tones = []; + AudioContext.prototype.setSinkId = async function (id) { + window.routes.push(id); + if (window.routes.length === 1) + await new Promise((resolve) => { + window.releaseRoute = resolve; + }); + this.testSink = id; + Object.defineProperty(this, "sinkId", { configurable: true, value: id }); + }; + const createOscillator = AudioContext.prototype.createOscillator; + AudioContext.prototype.createOscillator = function () { + const oscillator = createOscillator.call(this); + const start = oscillator.start.bind(oscillator); + oscillator.start = () => { + window.tones.push(this.testSink); + start(); + }; + return oscillator; + }; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + await page.locator("#preflight-output").selectOption(""); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator("#audio-test-tone").click(); + assert.deepEqual(await page.evaluate(() => window.tones), []); + await page.evaluate(() => window.releaseRoute()); + await page.waitForFunction(() => window.tones.length > 0); + assert.deepEqual(await page.evaluate(() => window.tones), ["headphones"]); + assert.deepEqual(await page.evaluate(() => window.routes), ["headphones"]); +}); + +for (const sound of ["tone", "speech"]) { + test(`a refused ${sound} output can be retried without confirming it`, async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + window.refuseOutput = true; + window.routedOutput = false; + const route = async () => { + if (window.refuseOutput) + throw new DOMException("Device refused", "NotAllowedError"); + window.routedOutput = true; + }; + HTMLMediaElement.prototype.setSinkId = route; + AudioContext.prototype.setSinkId = route; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator(`#audio-test-${sound}`).click(); + await page.waitForFunction(() => + document + .querySelector("#audio-check-status") + .textContent.includes("Could not"), + ); + assert.equal( + await page.locator("#audio-heard").textContent(), + "I heard it", + ); + await page.evaluate(() => { + window.refuseOutput = false; + }); + await page.locator(`#audio-test-${sound}`).click(); + await page.waitForFunction(() => window.routedOutput); + assert.equal( + await page.locator("#audio-heard").textContent(), + "I heard it", + ); + }); +} + test("preflight speech replays during capture and still requires the candidate's confirmation", async (t) => { if (!browser) return t.skip("playwright chromium unavailable"); const page = await preflight(t); diff --git a/web/audio-input.js b/web/audio-input.js new file mode 100644 index 00000000..3bb8acce --- /dev/null +++ b/web/audio-input.js @@ -0,0 +1,70 @@ +// Microphone choices are limited to preflight, before its stream is published. +export function createAudioInput({ + mediaDevices, + pool, + select, + note, + isFinished, +}) { + let listing = 0; + const showNote = (text) => { + note.textContent = text; + note.hidden = !text; + }; + const supported = + pool.supported && + Boolean(mediaDevices?.enumerateDevices) && + Boolean(mediaDevices?.getSupportedConstraints?.().deviceId); + select.disabled = !supported; + if (!supported) { + showNote("Microphone selection is unavailable. Use the system default."); + } + const refresh = async () => { + if (!supported || isFinished()) return; + const request = ++listing; + try { + const devices = await mediaDevices.enumerateDevices(); + if (isFinished() || request !== listing) return; + const selected = select.value; + const options = [new Option("System default", "")]; + for (const device of devices) { + if ( + device.kind !== "audioinput" || + !device.deviceId || + device.deviceId === "default" + ) + continue; + options.push( + new Option( + device.label || `Microphone ${options.length}`, + device.deviceId, + ), + ); + } + select.replaceChildren(...options); + select.value = options.some((option) => option.value === selected) + ? selected + : ""; + showNote(""); + if (selected && !select.value) pool.selectMicrophone(""); + } catch { + if (!isFinished() && request === listing) + showNote( + "Microphones could not be listed. Check your browser permissions and reload the page.", + ); + } + }; + const change = () => { + if (!isFinished()) pool.selectMicrophone(select.value); + }; + select.addEventListener("change", change); + mediaDevices?.addEventListener?.("devicechange", refresh); + return { + refresh, + close() { + listing += 1; + select.removeEventListener("change", change); + mediaDevices?.removeEventListener?.("devicechange", refresh); + }, + }; +} diff --git a/web/audio-output.js b/web/audio-output.js index d8e792ea..6bbf644c 100644 --- a/web/audio-output.js +++ b/web/audio-output.js @@ -1,114 +1,162 @@ -/// Which speaker plays the tab's audio, and the two preferences that outlive a -/// reload. -/// -/// No mixer and no synthetic microphone: Meet captures the tab, and CodeTrial -/// only decides which output device the tab's own audio lands on. Split out of -/// `interview.js` because routing, the stored preference and the note the -/// candidate reads about it are one decision made in three places, and the -/// order they happen in is the part that matters. - -import { - OUTPUT_NOTES, - outputAfterRouting, - outputOptions, -} from "./meet-audio.js"; +// One output selection for preflight playback and the interviewer's voice. + +import { OUTPUT_NOTES, outputOptions } from "./meet-audio.js"; let nodes = null; -/// Read through a function rather than held: `playRemoteAudio` creates and -/// drops Jim's element as LiveKit re-subscribes him, and a copy taken at init -/// would route the one from the previous subscription. let jimAudioElement = () => null; +let selectedOutputId = ""; +let outputNote = ""; +let listRequest = 0; +// A pre-grant listing can hide a saved device; only a granted listing can +// prove that it disappeared. +let outputAccessGranted = false; +// Serialize routes so an older setSinkId cannot finish after a newer one. +// Stamp requests to skip superseded choices and ignore their late failures. +let routingRequest = 0; +let routingChain = Promise.resolve(); + +export const AUDIO_OUTPUT_KEY = "codetrial:audioOutputId"; +export const MEET_PRESENTATION_KEY = "codetrial:meetPresentation"; export function initAudioOutput(deps) { ({ nodes, jimAudioElement } = deps); + selectedOutputId = readStored(AUDIO_OUTPUT_KEY) || ""; + renderOutputs(outputOptions([], selectedOutputId)); } -export const AUDIO_OUTPUT_KEY = "codetrial:audioOutputId"; -export const MEET_PRESENTATION_KEY = "codetrial:meetPresentation"; +export function audioOutputId() { + return selectedOutputId; +} -/// Lists output devices so Jim can be routed away from the shared tab's -/// default. No mixer and no synthetic microphone: Meet captures the tab, and -/// CodeTrial only decides which speaker plays the tab's own audio. -export async function refreshAudioOutputs() { +function renderOutputs(options) { + if ( + selectedOutputId && + !outputAccessGranted && + !options.some((option) => option.deviceId === selectedOutputId) + ) { + options = [ + ...options, + { + deviceId: selectedOutputId, + label: "Saved output device", + selected: true, + }, + ]; + } + for (const select of [nodes.meetOutputSelect, nodes.preflightOutput]) { + select.replaceChildren( + ...options.map((option) => { + const element = document.createElement("option"); + element.value = option.deviceId; + element.textContent = option.label; + element.selected = option.selected; + return element; + }), + ); + } +} + +function selectOutput(deviceId) { + const changed = selectedOutputId !== deviceId; + selectedOutputId = deviceId; + writeStored(AUDIO_OUTPUT_KEY, deviceId); + nodes.meetOutputSelect.value = deviceId; + nodes.preflightOutput.value = deviceId; + if (changed) nodes.preflightOutput.dispatchEvent(new Event("outputchange")); +} + +export async function refreshAudioOutputs({ granted = false } = {}) { + if (granted) outputAccessGranted = true; + const request = ++listRequest; const mediaDevices = navigator.mediaDevices; if ( !mediaDevices?.enumerateDevices || !("setSinkId" in HTMLMediaElement.prototype) ) { - // Hide the row, not just the control: a label pointing at a hidden select - // renders as a heading with nothing under it. nodes.meetOutputRow.hidden = true; + nodes.preflightOutput.disabled = true; showOutputNote(OUTPUT_NOTES.unsupported); + if (selectedOutputId) void applyAudioOutput(""); return; } - let devices = []; + nodes.meetOutputRow.hidden = false; + nodes.preflightOutput.disabled = false; try { - devices = await mediaDevices.enumerateDevices(); + const devices = await mediaDevices.enumerateDevices(); + if (request !== listRequest) return; + const options = outputOptions(devices, selectedOutputId); + const missing = + outputAccessGranted && + !options.some((option) => option.deviceId === selectedOutputId); + if (missing) { + options[0].selected = true; + } + renderOutputs(options); + if (outputNote !== OUTPUT_NOTES.refused) + showOutputNote(options.length === 1 ? OUTPUT_NOTES.empty : ""); + if (missing) void applyAudioOutput(""); } catch { - showOutputNote(OUTPUT_NOTES.listFailed); - return; - } - const options = outputOptions(devices, readStored(AUDIO_OUTPUT_KEY)); - nodes.meetOutputSelect.replaceChildren(); - for (const option of options) { - const element = document.createElement("option"); - element.value = option.deviceId; - element.textContent = option.label; - element.selected = option.selected; - nodes.meetOutputSelect.append(element); + if (request === listRequest) showOutputNote(OUTPUT_NOTES.listFailed); } - showOutputNote(options.length ? "" : OUTPUT_NOTES.empty); } -/// The routing request this one has to finish behind, and the number that says -/// which request is the newest. -/// -/// `setSinkId` is asynchronous and the candidate can pick again before it -/// settles. Two in flight can resolve in either order, and the older one would -/// then write its device into the preference and the select, undoing a choice -/// the candidate made after it. Serialized so they resolve in the order they -/// were asked for, and stamped so a request that is no longer the newest -/// routes but does not get the last word on what is stored or shown. -let routingChain = Promise.resolve(); -let routingRequest = 0; - export function applyAudioOutput(deviceId) { - routingChain = routingChain.then(() => - routeAudioOutput(deviceId, ++routingRequest), - ); + deviceId = deviceId || ""; + const request = ++routingRequest; + selectOutput(deviceId); + routingChain = routingChain + .catch(() => {}) + .then(() => routeAudioOutput(deviceId, request)); return routingChain; } +export async function audioOutputSettled() { + let pending; + do { + pending = routingChain; + await pending; + } while (pending !== routingChain); +} + async function routeAudioOutput(deviceId, request) { - if (!deviceId) return; - const previous = readStored(AUDIO_OUTPUT_KEY); - writeStored(AUDIO_OUTPUT_KEY, deviceId); - const jimAudio = jimAudioElement(); - if (!jimAudio || typeof jimAudio.setSinkId !== "function") return; - const routed = await jimAudio - .setSinkId(deviceId) - .then(() => true) - .catch(() => false); - // Cleared, not left alone, when there is no previous device to restore. The - // refused id was written before routing was attempted, so leaving it would - // persist a device Jim never played through: the next reload would pre-select - // it and retry the same refusal. - // A newer pick arrived while this one was routing. It has already routed, so - // Jim plays through whatever the last `setSinkId` chose, and the newest - // request is the one entitled to say so in the preference and the select. if (request !== routingRequest) return; - const stored = outputAfterRouting(previous, deviceId, routed); - writeStored(AUDIO_OUTPUT_KEY, stored); - nodes.meetOutputSelect.value = stored || ""; - // An empty list keeps its explanation: routing succeeding says nothing about - // whether there was anything to choose from. - const note = routed ? "" : OUTPUT_NOTES.refused; - if (note || nodes.meetOutputSelect.length) showOutputNote(note); + // The element changes across subscriptions; read it when routing starts. + const jimAudio = jimAudioElement(); + if (!jimAudio) return; + const previous = jimAudio.sinkId || ""; + try { + if (typeof jimAudio.setSinkId === "function") { + if (jimAudio.sinkId !== deviceId) await jimAudio.setSinkId(deviceId); + } else if (deviceId) throw new Error("Output selection is unavailable"); + if (request === routingRequest && outputNote === OUTPUT_NOTES.refused) + showOutputNote(""); + } catch { + if (request !== routingRequest) return; + // The previous sink may have disappeared while this request was pending. + const fallback = Array.from(nodes.meetOutputSelect.options).some( + (option) => option.value === previous, + ) + ? previous + : ""; + if (fallback !== previous) { + try { + await jimAudio.setSinkId(fallback); + } catch { + // Keep the refusal visible if even the system output is unavailable. + } + } + if (request !== routingRequest) return; + selectOutput(fallback); + showOutputNote(OUTPUT_NOTES.refused); + } } -export function showOutputNote(text) { - nodes.meetOutputNote.textContent = text; - nodes.meetOutputNote.hidden = !text; +function showOutputNote(text) { + outputNote = text; + for (const note of [nodes.meetOutputNote, nodes.preflightOutputNote]) { + note.textContent = text; + note.hidden = !text; + } } /// `localStorage` or `sessionStorage`, or `null` where site data is blocked: @@ -169,13 +217,14 @@ export function createJimVolume({ const stored = readStored(JIM_VOLUME_KEY, storage); let level = /^(?:[1-9][0-9]?|100)$/.test(stored ?? "") ? Number(stored) : 100; let muted = false; + let routingHeld = false; let supported = true; let tested = null; const restore = () => { const audio = element(); if ( audio && - (audio.muted !== muted || + (audio.muted !== (muted || routingHeld) || (supported && !sameVolume(audio.volume, level / 100))) ) apply(); @@ -192,7 +241,7 @@ export function createJimVolume({ // another level synchronously, then restore the requested attenuation. if (fresh) supported = setsVolume(audio, 0.5); if (supported) supported = setsVolume(audio, level / 100); - audio.muted = muted; + audio.muted = muted || routingHeld; if (fresh) audio.addEventListener?.("volumechange", restore); } onChange(); @@ -225,6 +274,10 @@ export function createJimVolume({ return supported; }, apply, + setRoutingHeld(value) { + routingHeld = Boolean(value); + apply(); + }, setVolume, toggleMute() { muted = !muted; diff --git a/web/audio-playback.js b/web/audio-playback.js index 249690cc..f9e4cb3b 100644 --- a/web/audio-playback.js +++ b/web/audio-playback.js @@ -31,6 +31,12 @@ export function createAudioPlayback({ let pending = false; let blocked = false; + function paused() { + // Unmuting can pause native playback after its play promise succeeded. + if (element?.paused && !element.ended && !pending && !blocked) + void play(element); + } + function render() { if (!blocked && button.ownerDocument.activeElement === button) focusTarget.focus({ preventScroll: true }); @@ -60,6 +66,7 @@ export function createAudioPlayback({ /// restored, so nothing is announced. function clear() { revision += 1; + element?.removeEventListener?.("pause", paused); element = null; pending = false; blocked = false; @@ -84,6 +91,10 @@ export function createAudioPlayback({ function play(nextElement) { if (!room) return Promise.resolve(); revision += 1; + if (element !== nextElement) { + element?.removeEventListener?.("pause", paused); + nextElement.addEventListener?.("pause", paused); + } element = nextElement; pending = false; reapply(); diff --git a/web/devices.js b/web/devices.js index b931fa35..ee010469 100644 --- a/web/devices.js +++ b/web/devices.js @@ -48,6 +48,7 @@ export function createDevicePool({ kind, constraints: { [kind]: CONSTRAINTS[kind] }, pending: false, + generation: 0, disabled: false, error: null, accept: () => true, @@ -87,6 +88,7 @@ export function createDevicePool({ // `pending` matters as much as the shared timer: a retry must not open a // second prompt while the first is still on screen. const requestDevice = (device) => { + if (isFinished()) return; // An ended track never revives, and while it sits in the stream it looks // like a device this pool already holds, so the retry asks for nothing and // the candidate is stranded with no way back. @@ -103,10 +105,15 @@ export function createDevicePool({ if (device.disabled || device.pending || trackOf(device.kind)) return; device.pending = true; + const generation = device.generation; void mediaDevices .getUserMedia(device.constraints) .then((granted) => { - if (device.disabled) { + if ( + device.disabled || + isFinished() || + generation !== device.generation + ) { granted.getTracks().forEach((track) => track.stop()); return; } @@ -122,6 +129,7 @@ export function createDevicePool({ device.onTrack(); }) .catch((error) => { + if (isFinished() || generation !== device.generation) return; // Some browsers leave a DOMException's message empty and name the // failure instead. Keep both: the preflight words are the candidate's // only explanation, and the optional-camera path distinguishes denied @@ -133,6 +141,9 @@ export function createDevicePool({ }) .finally(() => { device.pending = false; + // A selection changed while the browser was answering the old request. + // Reuse the pending guard so repeated changes never open more prompts. + if (generation !== device.generation) requestDevice(device); onChange(); }); }; @@ -162,6 +173,30 @@ export function createDevicePool({ }, start, retry, + selectMicrophone(deviceId) { + const audio = devices.audio; + if ( + isFinished() || + (audio.constraints.audio.deviceId?.exact || "") === deviceId + ) + return; + audio.generation += 1; + audio.constraints = { + audio: { + ...CONSTRAINTS.audio, + ...(deviceId ? { deviceId: { exact: deviceId } } : {}), + }, + }; + audio.error = null; + const held = trackOf("audio"); + if (held) { + stream.removeTrack(held); + held.stop(); + } + audio.onLost(); + start(); + onChange(); + }, disable(kind) { const device = devices[kind]; device.disabled = true; diff --git a/web/interview.html b/web/interview.html index ce46ce5d..58c3d279 100644 --- a/web/interview.html +++ b/web/interview.html @@ -561,6 +561,21 @@

Devices

+
+ + +
+

Sound — @@ -607,6 +622,21 @@

+
+ + +
+

Microphone — say something and watch the diff --git a/web/interview.js b/web/interview.js index 7080a53b..b018ba85 100644 --- a/web/interview.js +++ b/web/interview.js @@ -45,6 +45,7 @@ import { toggleLineComment, } from "./editor.js"; import { createDevicePool } from "./devices.js"; +import { createAudioInput } from "./audio-input.js"; import { createFaceCheck } from "./face-check.js"; import { attachSelfView } from "./self-view.js"; import { createMicMeter, startMediaMeter } from "./mic-meter.js"; @@ -119,7 +120,8 @@ import { responseWindowIndex, } from "./replay-feed.js"; import { - AUDIO_OUTPUT_KEY, + audioOutputId, + audioOutputSettled, MEET_PRESENTATION_KEY, applyAudioOutput, createJimVolume, @@ -617,6 +619,10 @@ const nodes = { meetOutputRow: document.querySelector("#meet-output-row"), meetOutputSelect: document.querySelector("#meet-output-select"), meetOutputNote: document.querySelector("#meet-output-note"), + preflightOutput: document.querySelector("#preflight-output"), + preflightOutputNote: document.querySelector("#preflight-output-note"), + preflightInput: document.querySelector("#preflight-input"), + preflightInputNote: document.querySelector("#preflight-input-note"), sidebar: document.querySelector("#interview-sidebar"), sidebarResizer: document.querySelector("#sidebar-resizer"), editorPanel: document.querySelector(".editor-panel"), @@ -1039,9 +1045,11 @@ function bindEvents() { removeCandidateCase(Number(button.dataset.removeCase)); }); void initializeCandidateCases(); - nodes.meetOutputSelect.addEventListener("change", () => { - void applyAudioOutput(nodes.meetOutputSelect.value); - }); + for (const select of [nodes.meetOutputSelect, nodes.preflightOutput]) { + select.addEventListener("change", () => { + void applyAudioOutput(select.value); + }); + } // The checkbox is the only copy of this answer. A mirrored state field had to // be written in two places and read in four, and the one reader that forgot // it published a wrong camera status for the whole interview. @@ -1421,6 +1429,38 @@ function runAudioCheck() { let hint = null; let hintUntil = 0; let finished = false; + const microphoneSelector = createAudioInput({ + mediaDevices, + pool, + select: nodes.preflightInput, + note: nodes.preflightInputNote, + isFinished: () => finished, + }); + let playbackRequest = 0; + let playbackRouting = Promise.resolve(); + const outputChanged = () => { + if (finished) return; + playbackRequest += 1; + outputConfirmed = false; + nodes.audioTestVoice.pause(); + refresh(); + }; + nodes.preflightOutput.addEventListener("outputchange", outputChanged); + // A stale setSinkId can change the sink even when its caller never plays. + // Keep routing ordered, including when the selection returns to the same id. + const routeTestOutput = (target, outputId, request) => { + const routing = playbackRouting.then(async () => { + if (finished || request !== playbackRequest) return false; + if (typeof target.setSinkId === "function") { + if (target.sinkId !== outputId) await target.setSinkId(outputId); + } else if (outputId) { + throw new Error("Output selection is unavailable"); + } + return !finished && request === playbackRequest; + }); + playbackRouting = routing.catch(() => {}); + return routing; + }; // Set once, because refresh() runs on every animation frame: the // candidate is the only reliable output sensor, so confirming is // accepted whenever they click rather than gated on tone timing. @@ -1481,6 +1521,9 @@ function runAudioCheck() { // so a request still pending when this runs never lights the camera. const stopChecks = () => { finished = true; + microphoneSelector.close(); + playbackRequest += 1; + nodes.preflightOutput.removeEventListener("outputchange", outputChanged); nodes.audioTestVoice.pause(); nodes.audioTestVoice.currentTime = 0; meter.stop(); @@ -1525,11 +1568,29 @@ function runAudioCheck() { }; nodes.audioTestTone.addEventListener("click", async () => { + const outputId = audioOutputId(); + const request = ++playbackRequest; nodes.audioTestVoice.pause(); if (!(await unblockOutput())) { showHint("The browser is still blocking audio. Click the tone again."); return; } + try { + if (typeof context.setSinkId !== "function" && outputId) { + showHint( + "This browser cannot route the test tone. Try the test voice instead.", + ); + return; + } + if (!(await routeTestOutput(context, outputId, request))) return; + } catch { + if (finished || request !== playbackRequest) return; + showHint( + "Could not use that output device. Choose another device and try again.", + ); + return; + } + if (finished || request !== playbackRequest) return; const oscillator = context.createOscillator(); const gain = context.createGain(); oscillator.frequency.value = 440; @@ -1546,17 +1607,25 @@ function runAudioCheck() { // speech. A local sample needs no interview session or synthesis service, // and playing it says nothing about whether the candidate actually heard it. nodes.audioTestSpeech.addEventListener("click", async () => { + const outputId = audioOutputId(); + const request = ++playbackRequest; nodes.audioTestSpeech.disabled = true; try { nodes.audioTestVoice.pause(); nodes.audioTestVoice.currentTime = 0; if (nodes.audioTestVoice.error) nodes.audioTestVoice.load(); + if (!(await routeTestOutput(nodes.audioTestVoice, outputId, request))) + return; await nodes.audioTestVoice.play(); - if (finished) return; + if (finished || request !== playbackRequest) return; nodes.audioTestSpeech.textContent = "Play speech again"; showHint('If you hear the test voice, click "I heard it".'); } catch (error) { - if (!finished && error?.name !== "AbortError") { + if ( + !finished && + request === playbackRequest && + error?.name !== "AbortError" + ) { showHint( "Could not play the test voice. Try again and check your audio settings.", ); @@ -1620,7 +1689,13 @@ function runAudioCheck() { // actually carries sound, and it runs for the rest of the preflight. pool.configure("audio", { accept: (track) => Boolean(track), - onTrack: () => meter.start(), + onTrack: () => { + meter.start(); + // A grant reveals labels; failed retries must leave open menus alone + // while the candidate chooses a different device. + void refreshAudioOutputs({ granted: true }); + void microphoneSelector.refresh(); + }, onLost: () => { meter.forget(); nodes.audioMeterFill.style.width = "0%"; @@ -1647,6 +1722,8 @@ function runAudioCheck() { }, }); pool.start(); + void microphoneSelector.refresh(); + void refreshAudioOutputs(); refresh(); }); } @@ -2293,6 +2370,7 @@ function playRemoteAudio(track, participant) { !isCurrentAgent(participant) ) return; + const fresh = !jimAudio; const element = jimAudio ?? document.createElement("audio"); // LiveKit can deliver the old track's unsubscribe after the new track's // subscribe. Its detach pauses every element the track still holds, so the @@ -2303,11 +2381,30 @@ function playRemoteAudio(track, participant) { element.id = "jim-audio"; jimAudio = element; jimTrack = track; - jimVolume.apply(); + // LiveKit unmutes during attach; every volume reapply must retain this hold. + if (fresh) jimVolume.setRoutingHeld(true); + else jimVolume.apply(); if (!element.isConnected) document.body.appendChild(element); void audioPlayback.play(element); // The candidate may have chosen an output before Jim's track arrived. - void applyAudioOutput(readStored(AUDIO_OUTPUT_KEY)); + void applyAudioOutput(audioOutputId()); + if (fresh) { + const pending = setTimeout(() => { + if (jimAudio === element) + setBanner( + "output", + "Audio output selection is taking longer than expected. Check your device, or reload the page to retry.", + ); + }, 5000); + void audioOutputSettled().finally(() => { + clearTimeout(pending); + if (jimAudio !== element) return; + setBanner("output", ""); + jimVolume.setRoutingHeld(false); + // Muted play can succeed without permission to play audibly. + void audioPlayback.play(element); + }); + } attachAvatarAnalyser(track, participant); } @@ -2322,7 +2419,8 @@ function dropRemoteAudio(track) { element.remove(); if (element === jimAudio) { jimAudio = jimTrack = null; - jimVolume.apply(); + jimVolume.setRoutingHeld(false); + setBanner("output", ""); } } // Only if it was the analyser's own track. The guard above already refuses to @@ -2509,8 +2607,9 @@ function setLanguage(language) { /// deferred-text field that only one of the two owners got, and a lock-breaking /// special case for session-ending events. const BANNER_RANK = { - session: 5, - connection: 4, + session: 6, + connection: 5, + output: 4, interviewer: 3, reconnect: 2, face: 1, @@ -2518,6 +2617,7 @@ const BANNER_RANK = { const banners = { session: "", connection: "", + output: "", interviewer: "", reconnect: "", face: "", diff --git a/web/meet-audio.js b/web/meet-audio.js index a0dec281..cbc7a74a 100644 --- a/web/meet-audio.js +++ b/web/meet-audio.js @@ -1,9 +1,4 @@ -// Output routing for Google Meet presentation mode. -// -// Meet re-shares the CodeTrial tab, so Jim reaches the interviewer through -// whatever speaker plays this tab. Choosing that speaker is the whole feature, -// and it is all decisions: which devices are worth offering, what to call them, -// and what to keep when the browser refuses one. +// Output choices shared by the preflight and Google Meet controls. // // Free of DOM and Web Audio types on purpose, like audio-check.js, so these // decisions run under `node --test` instead of being asserted as source text; @@ -13,44 +8,33 @@ export const OUTPUT_NOTES = { unsupported: "This browser cannot choose an output device. Use the system default, or switch to Chrome.", listFailed: - "Output devices could not be listed. The system default is in use.", + "Output devices could not be listed. Check the selected device or use the system default.", empty: - "No output devices yet. Finish the media preflight; this list fills in once the browser grants access.", - refused: "That output device was refused. Jim stays on the previous device.", + "Specific output devices may appear after you allow microphone access. You can still use the system default.", + refused: + "That output device was refused. The previous output or system default is selected.", }; /// Devices worth offering. Outputs only, and only those the browser has named: /// before a `getUserMedia` grant it reports a placeholder whose `deviceId` is /// empty, which renders as a real choice and then selects to "", so a candidate /// picking it silently gets nothing. -export function outputOptions(devices, savedId) { - return devices - .filter((device) => device.kind === "audiooutput" && device.deviceId) - .map((device, index) => ({ - deviceId: device.deviceId, - // Labels stay blank until a permission grant; number them so the list is - // still usable rather than a column of empty rows. - label: device.label || `Output ${index + 1}`, - selected: device.deviceId === savedId, - })); -} - -/// Which device the preference should name after an attempt to route. -/// `routed` is whether `setSinkId` resolved. Null means no preference at all. -/// -/// The preference is written before routing is attempted, because a candidate -/// can pick a device during the preflight, long before Jim's track exists. A -/// preference that only survived a successful `setSinkId` would be dropped -/// exactly then, and the select would keep showing a device Jim never used. -/// A refusal therefore has to roll the stored value back, or the select and the -/// real sink disagree until the next reload. -export function outputAfterRouting(previous, requested, routed) { - if (routed) return requested; - // A refusal falls back to the device that was working before, except when - // that is the same device: reapplying a stored preference on reload passes - // the same id as both, so returning `previous` would write the refused id - // straight back and retry it on every reload for the life of the browser - // profile. There is no earlier device to fall back to, so the preference - // clears and the next load uses the system default. - return previous === requested ? "" : previous; +export function outputOptions(devices, selectedId) { + return [ + { deviceId: "", label: "System default", selected: !selectedId }, + ...devices + .filter( + (device) => + device.kind === "audiooutput" && + device.deviceId && + device.deviceId !== "default", + ) + .map((device, index) => ({ + deviceId: device.deviceId, + // Labels stay blank until a permission grant; number them so the list is + // still usable rather than a column of empty rows. + label: device.label || `Output ${index + 1}`, + selected: device.deviceId === selectedId, + })), + ]; } diff --git a/web/mic-meter.js b/web/mic-meter.js index 33129928..41db0c15 100644 --- a/web/mic-meter.js +++ b/web/mic-meter.js @@ -66,6 +66,7 @@ export function createMicMeter({ return startMeter( pool.stream, (level) => { + if (isFinished() || mine !== generation) return; pool.setError("audio", null); recent.push(level); if (recent.length > MIC_CONFIRM_FRAMES) recent.shift(); @@ -75,6 +76,7 @@ export function createMicMeter({ onLevel(level); }, (error) => { + if (isFinished() || mine !== generation) return; pool.setError("audio", error); forgetPeak(); onFailure(error);