1// tunebench measures how long channel changes take on a running airwavesd,2// as the app sees them: the tune call, then playback in headless Chrome3// with hls.js set up as the app sets it up, to the first frame and through4// the first seconds, counting stalls. Tunes run one at a time, as their own5// client, and each is stopped before the next.6//7// node scripts/tunebench.mjs --server http://nas:8089 --plan 9.1x3,1.2x3,201.0x28//9// Options:10// --server URL the airwavesd to measure (required)11// --plan LIST channel numbers, each with "xN" for N tunes; tunes go12// round by round through the list, so a channel's tunes13// are spread out14// --watch S seconds to watch each tune after it plays (30)15// --gap S seconds between tunes (5)16// --client ID the client ID to tune as ("bench")17// --token T the API token, when the server has one18// --out FILE append each tune's result to FILE as JSON lines19// --label NAME names the run in the results ("run")20// --chrome PATH a Chrome or Chromium that plays H.264 (default: the21// installed Google Chrome)22//23// Playwright is loaded from PLAYWRIGHT (a path to its index.mjs) or the24// "playwright" package.25import fs from 'node:fs';26import path from 'node:path';27import { fileURLToPath } from 'node:url';2829const args = parseArgs(process.argv.slice(2));30if (!args.server || !args.plan) {31 console.error('usage: node scripts/tunebench.mjs --server URL --plan 9.1x3,1.2x3 [--watch 30] [--gap 5] [--out FILE]');32 process.exit(2);33}34const server = args.server.replace(/\/+$/, '');35const watch = Number(args.watch ?? 30);36const gap = Number(args.gap ?? 5);37const client = args.client || 'bench';38const label = args.label || 'run';39const headers = { 'Content-Type': 'application/json', ...(args.token ? { Authorization: `Bearer ${args.token}` } : {}) };4041const here = path.dirname(fileURLToPath(import.meta.url));42const hlsjs = path.join(here, '..', 'frontend', 'dist', 'vendor', 'hls.min.js');43const { chromium } = await import(process.env.PLAYWRIGHT || 'playwright');4445function parseArgs(argv) {46 const out = {};47 for (let i = 0; i < argv.length; i++) {48 const a = argv[i];49 if (!a.startsWith('--')) continue;50 const [k, v] = a.slice(2).split('=', 2);51 out[k] = v ?? argv[++i];52 }53 return out;54}5556// plan expands "9.1x3,1.2x2" into tunes, round by round.57function plan(spec) {58 const want = spec.split(',').map((s) => s.trim()).filter(Boolean).map((s) => {59 const [number, n] = s.split('x');60 return { number, n: Number(n || 1) };61 });62 const out = [];63 for (let round = 0; want.some((w) => w.n > round); round++) {64 for (const w of want) if (w.n > round) out.push(w.number);65 }66 return out;67}6869async function api(method, p, body) {70 const r = await fetch(server + p, { method, headers, body: body ? JSON.stringify(body) : undefined });71 const text = await r.text();72 let json;73 try { json = JSON.parse(text); } catch { json = { error: text }; }74 if (!r.ok) throw new Error(`${p}: HTTP ${r.status}: ${json.error || text}`.slice(0, 300));75 return json;76}7778async function kinds() {79 const snap = await api('GET', '/api/snapshot');80 const m = new Map();81 for (const c of snap.custom || []) m.set(c.number, c.kind || 'folder');82 return m;83}8485const sleep = (ms) => new Promise((r) => setTimeout(r, ms));86const quantile = (xs, q) => {87 const s = xs.filter((x) => x != null && Number.isFinite(x)).sort((a, b) => a - b);88 if (!s.length) return null;89 return s[Math.min(s.length - 1, Math.max(0, Math.ceil(q * s.length) - 1))];90};91const secs = (ms) => (ms == null ? '-' : (ms / 1000).toFixed(2));9293// The page plays a stream as the app does (frontend/dist/js/player.js attempt()),94// and keeps a record of what happened when.95const pageScript = () => {96 window.bench = (src, watchMs) => new Promise((resolve) => {97 const video = document.querySelector('video');98 const t0 = Date.now();99 const rec = { start: t0, events: {}, stalls: [], errors: [], buffers: [] };100 const mark = (name) => { if (rec.events[name] == null) rec.events[name] = Date.now() - t0; };101 const h = new window.Hls({ liveSyncDurationCount: 2, maxBufferLength: 12, subtitleDisplay: false });102 const E = window.Hls.Events;103 h.on(E.MANIFEST_PARSED, (_, d) => { mark('manifest'); rec.audioTracks = (d.audioTracks || []).length; rec.subtitleTracks = (d.subtitleTracks || []).length; });104 h.on(E.LEVEL_LOADED, (_, d) => { mark('level'); if (rec.firstLevel == null) rec.firstLevel = { segments: d.details.fragments.length, targetDuration: d.details.targetduration, first: d.details.fragments.map((f) => +f.duration.toFixed(2)).slice(0, 4) }; });105 h.on(E.FRAG_LOADED, (_, d) => { if (d.frag.type === 'main') mark('fragLoaded'); });106 h.on(E.FRAG_BUFFERED, (_, d) => { if (d.frag.type === 'main') mark('fragBuffered'); });107 h.on(E.BUFFER_CREATED, (_, d) => { rec.buffers = Object.keys(d.tracks || {}); });108 h.on(E.ERROR, (_, d) => { rec.errors.push(`${d.fatal ? 'fatal ' : ''}${d.details}`); });109 let playing = false;110 let stallAt = 0;111 video.addEventListener('playing', () => {112 if (!playing) { playing = true; mark('playing'); rec.startPos = video.currentTime; rec.liveEdge = h.liveSyncPosition; }113 if (stallAt) { rec.stalls[rec.stalls.length - 1].ms = Date.now() - stallAt; stallAt = 0; }114 });115 video.addEventListener('waiting', () => {116 if (!playing || stallAt) return;117 stallAt = Date.now();118 rec.stalls.push({ at: stallAt - t0, ms: null });119 });120 // A picture that stops without a waiting event counts too.121 let lastPos = -1;122 let still = 0;123 const tick = setInterval(() => {124 if (!playing || video.paused) return;125 if (video.currentTime === lastPos && !stallAt) {126 if (++still === 2) rec.stalls.push({ at: Date.now() - t0 - 1000, ms: null, frozen: true });127 } else still = 0;128 lastPos = video.currentTime;129 }, 500);130 video.muted = true;131 h.loadSource(src);132 h.attachMedia(video);133 video.play().catch(() => {});134 const giveUp = setTimeout(() => finish(), watchMs + 20000);135 const finish = () => {136 clearInterval(tick);137 clearTimeout(giveUp);138 if (stallAt) rec.stalls[rec.stalls.length - 1].ms = Date.now() - stallAt;139 const q = video.getVideoPlaybackQuality ? video.getVideoPlaybackQuality() : {};140 rec.dropped = q.droppedVideoFrames;141 rec.frames = q.totalVideoFrames;142 rec.width = video.videoWidth;143 rec.latency = h.latency;144 rec.played = video.currentTime - (rec.startPos || 0);145 h.destroy();146 resolve(rec);147 };148 // Watch for watchMs after playing starts, or give up 20 s after that.149 const wait = setInterval(() => {150 if (rec.events.playing != null && Date.now() - t0 - rec.events.playing >= watchMs) { clearInterval(wait); finish(); }151 }, 200);152 });153};154155const browser = await chromium.launch({156 ...(args.chrome ? { executablePath: args.chrome } : { channel: 'chrome' }),157 args: ['--autoplay-policy=no-user-gesture-required', '--mute-audio'],158});159const page = await browser.newPage();160await page.setContent('<!doctype html><video width="640" height="360"></video>');161await page.addScriptTag({ path: hlsjs });162await page.evaluate(pageScript);163164const kindOf = await kinds();165const tunes = plan(args.plan);166const results = [];167console.log(`${tunes.length} tunes on ${server}, ${watch} s each, ${gap} s apart`);168for (const [i, number] of tunes.entries()) {169 if (i > 0) await sleep(gap * 1000);170 const kind = kindOf.get(number) || 'antenna';171 const r = { label, number, kind, at: new Date().toISOString() };172 const t0 = Date.now();173 try {174 const pb = await api('POST', '/api/tune', { client, number });175 r.api = Date.now() - t0;176 r.timing = pb.timing;177 const rec = await page.evaluate(([src, ms]) => window.bench(src, ms), [server + pb.path, watch * 1000]);178 const off = rec.start - t0; // tune call to the page's start179 r.page = rec;180 r.ttff = rec.events.playing != null ? off + rec.events.playing : null;181 if (r.ttff == null) r.error = `never played${rec.errors.length ? ': ' + rec.errors.join(' ') : ''}`;182 r.stalls = rec.stalls.length;183 r.stallMs = rec.stalls.reduce((s, x) => s + (x.ms || 0), 0);184 } catch (e) {185 r.error = String(e.message || e);186 }187 try { await api('POST', '/api/stop', { client }); } catch (e) { r.stopError = String(e.message || e); }188 results.push(r);189 if (args.out) fs.appendFileSync(args.out, JSON.stringify(r) + '\n');190 const ev = r.page ? r.page.events : {};191 const t = r.timing ? ' | server: ' + r.timing.map((m) => `${m.name} ${secs(m.ms)}`).join(', ') : '';192 console.log(`${String(i + 1).padStart(2)} ${number.padEnd(6)} ${kind.padEnd(8)} api ${secs(r.api)} s, first frame ${secs(r.ttff)} s` +193 ` (manifest +${secs(ev.manifest)}, fragment +${secs(ev.fragLoaded)}, playing +${secs(ev.playing)}),` +194 ` stalls ${r.stalls ?? '-'}${r.stallMs ? ` (${secs(r.stallMs)} s)` : ''}${r.page && r.page.errors.length ? ' errors ' + r.page.errors.join(' ') : ''}${r.error ? ' ERROR ' + r.error : ''}${t}`);195}196await browser.close();197198// The summary: per kind, the median and worst of each measure.199const byKind = new Map();200for (const r of results) {201 if (!byKind.has(r.kind)) byKind.set(r.kind, []);202 byKind.get(r.kind).push(r);203}204console.log('\nkind tunes api p50 api max frame p50 frame max stalls failed');205for (const [kind, rs] of byKind) {206 const ok = rs.filter((r) => !r.error);207 console.log(`${kind.padEnd(9)} ${String(rs.length).padStart(5)} ${secs(quantile(ok.map((r) => r.api), 0.5)).padStart(7)} ${secs(quantile(ok.map((r) => r.api), 1)).padStart(7)}` +208 ` ${secs(quantile(ok.map((r) => r.ttff), 0.5)).padStart(9)} ${secs(quantile(ok.map((r) => r.ttff), 1)).padStart(9)}` +209 ` ${String(ok.reduce((s, r) => s + (r.stalls || 0), 0)).padStart(6)} ${String(rs.length - ok.length).padStart(6)}`);210}211212// The server's steps, when it reports them: per kind, each step's median.213const steps = [...byKind].map(([kind, rs]) => {214 const at = new Map();215 for (const r of rs) for (const m of r.timing || []) {216 if (!at.has(m.name)) at.set(m.name, []);217 at.get(m.name).push(m.ms);218 }219 return [kind, [...at].map(([name, ms]) => `${name} ${secs(quantile(ms, 0.5))}`).join(', ')];220}).filter(([, s]) => s);221if (steps.length) {222 console.log('\nserver steps, median seconds from the tune call');223 for (const [kind, s] of steps) console.log(`${kind.padEnd(9)} ${s}`);224}