An episode's English captions (the uploader's, else YouTube's own) are fetched with where it streams from, ahead for the newest, and go to the app as a WebVTT rendition timed from where the stream starts: its first sound, since a copied picture starts at the key frame before. WebKit calls a player stalled when it has all it needs and stops fetching, as an on-demand stream does; the waiting mark then stayed up over a picture that played on. It now shows only while the picture has stopped, and goes once it moves.
- commit
- b7fa79df4c0b287deb4035649bf617a915722eb4
- parent
- fec7b36daab98507804a811045f0da73552c3040
- author
- FelineStateMachine <226808055+FelineStateMachine@users.noreply.github.com>
- kind
- feat risk score 1.9 of 3, Jev's estimate
| M | frontend/ | 16 | ++++++++-- |
| M | internal/ | 51 | +++++++++++++++++++++++++++++- |
| M | internal/ | 14 | ++++++++ |
| M | internal/ | 7 | ++++- |
| M | internal/ | 11 | +++++-- |
| M | internal/ | 51 | ++++++++++++++++++++++++++---- |
| M | internal/ | 36 | +++++++++++++++++++++ |
@@ -187,7 +187,8 @@ if (video.audioTracks) video.audioTracks.addEventListener('addtrack', applyAudio
187187 // A small badge at the picture's bottom right says what playback is doing,188188 // on the TV view. It stays only while there's something to do about it:189189 // paused (with how far behind live, or where in a recording), and190-// buffering (once a stream has played, after 300 ms of waiting). The rest190+// buffering (once a stream has played, after 300 ms of waiting with the191+// picture stopped). The rest191192 // flashes and fades: a skip (the amount, adding up over quick presses) and192193 // then where it left the picture; how far behind live on resuming, or on193194 // the info key; "Live" on going back to live. Playing behind live, it's@@ -205,7 +206,7 @@ const PS_PLACE_MS = 2500; // then where it left the picture, this long
205206 const PS_WAIT_MS = 300; // shorter waits for data go unmarked206207 const ps = {207208 el: $('#pstate'), skip: 0, dir: 0, skipUntil: 0, skipTimer: 0, placeUntil: 0, placeLive: false, placeTimer: 0,208- played: false, paused: false, waiting: false, waitTimer: 0, tick: 0, shown: '',209+ played: false, paused: false, waiting: false, waitAt: 0, waitTimer: 0, tick: 0, shown: '',209210 };210211 211212 // noteSkip shows a skip of delta seconds (what playback moved, which is@@ -302,9 +303,14 @@ function placePlayState() {
302303 303304 function startWaiting() {304305 if (!ps.played || ps.waitTimer || ps.waiting) return;306+ const at = video.currentTime;305307 ps.waitTimer = setTimeout(() => {306308 ps.waitTimer = 0;309+ // A stall the picture plays through isn't a wait: a player with all it310+ // needs stops fetching for a while, which WebKit calls stalled.311+ if (video.paused || video.currentTime > at + 0.1) return;307312 ps.waiting = true;313+ ps.waitAt = video.currentTime;308314 renderPlayState();309315 }, PS_WAIT_MS);310316 }@@ -335,7 +341,11 @@ video.addEventListener('emptied', () => {
335341 ps.placeUntil = 0;336342 stopWaiting();337343 });338-video.addEventListener('timeupdate', renderPlayState);344+// Playing again ends a wait, whatever the events said.345+video.addEventListener('timeupdate', () => {346+ if (ps.waiting && video.currentTime > ps.waitAt + 0.25) stopWaiting();347+ else renderPlayState();348+});339349 video.addEventListener('resize', placePlayState);340350 if (window.ResizeObserver) new ResizeObserver(placePlayState).observe(stage);341351 new MutationObserver(() => { placePlayState(); renderPlayState(); }).observe($('#banner'), { attributes: true, attributeFilter: ['class'] });@@ -21,6 +21,7 @@ import (
2121 "sync"2222 "time"2323 24+ "airwaves/internal/cc"2425 "airwaves/internal/store"2526 "airwaves/internal/tuner"2627 "airwaves/internal/youtube"@@ -58,6 +59,7 @@ type Library struct {
5859 loaded bool5960 st file6061 resolved map[string]resolved // where episodes stream from, by ID62+ captions map[string][]cc.Cue // episodes' English captions, nil for none6163 updating bool // listing a show6264 kick chan struct{}6365 now func() time.Time@@ -414,10 +416,57 @@ func (l *Library) Input(ctx context.Context, id string, from float64) (tuner.Inp
414416 if from <= 5 || length > 0 && from >= length-5 {415417 from = 0416418 }417- args, audio := youtube.InputArgs(in, time.Duration(from*float64(time.Second)), youtube.HTTPChunks(l.FFmpeg))419+ start := time.Duration(from * float64(time.Second))420+ args, audio := youtube.InputArgs(in, start, youtube.HTTPChunks(l.FFmpeg))418421 out := tuner.Input{Args: args, Video: "0:v:0", VOD: true, Offset: from, Copy: strings.HasPrefix(in[0].VCodec, "avc1")}419422 if audio != "" {420423 out.Audio = []tuner.AudioTrack{{Map: audio}}421424 }425+ // English captions, as text for the app to lay out, timed from where426+ // the stream starts.427+ cctx, cancel := context.WithTimeout(ctx, playCaptionsTime)428+ defer cancel()429+ if cues := l.captionsFor(cctx, id, info); len(cues) > 0 {430+ out.Subtitles = &cc.Script{}431+ out.Subtitles.Set(0, 1000*time.Hour, cc.Shift(cues, -start))432+ }422433 return out, nil423434 }435+436+const (437+ // playCaptionsTime bounds waiting for an episode's captions at Play438+ // (they're usually fetched ahead); captionsTime fetching them ahead.439+ playCaptionsTime = 8 * time.Second440+ captionsTime = 30 * time.Second441+ // keptCaptions bounds how many episodes' captions are kept.442+ keptCaptions = 64443+)444+445+// captionsFor are an episode's English captions, the uploader's or446+// YouTube's automatic ones, fetched once; nil when it has none. A failed447+// fetch is tried again next time.448+func (l *Library) captionsFor(ctx context.Context, id string, info youtube.Info) []cc.Cue {449+ l.mu.Lock()450+ cues, ok := l.captions[id]451+ l.mu.Unlock()452+ if ok {453+ return cues454+ }455+ if sub, ok := info.Captions(); ok {456+ raw, err := youtube.FetchCaptions(ctx, l.tool().Client(), sub)457+ if err == nil {458+ cues, err = cc.Parse(raw)459+ }460+ if err != nil {461+ log.Printf("podcasts: captions for %s: %v", id, err)462+ return nil463+ }464+ }465+ l.mu.Lock()466+ if l.captions == nil || len(l.captions) >= keptCaptions {467+ l.captions = map[string][]cc.Cue{}468+ }469+ l.captions[id] = cues470+ l.mu.Unlock()471+ return cues472+}@@ -85,6 +85,9 @@ func fakeYtDlp(args []string) int {
8585 if id == "feedclip" {8686 info["duration"] = 2008787 }88+ if id == "ep2" {89+ info["automatic_captions"] = map[string]any{"en": []map[string]any{{"ext": "vtt", "url": "https://example.com/ep2.en.vtt"}}}90+ }8891 if slices.Contains(args, "-f") {8992 exp := now + 6*36009093 info["requested_formats"] = []map[string]any{@@ -119,6 +122,8 @@ func (youtubeStub) RoundTrip(r *http.Request) (*http.Response, error) {
119122 </feed>`, http.StatusOK120123 case u == "https://i.ytimg.com/vi/ep3/maxresdefault.jpg":121124 code = http.StatusOK125+ case u == "https://example.com/ep2.en.vtt":126+ body, code = "WEBVTT\n\n00:09:58.000 --> 00:10:03.000\nbefore and after\n\n00:10:30.000 --> 00:10:32.000\nlater on\n", http.StatusOK122127 }123128 return &http.Response{StatusCode: code, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil124129 }@@ -293,6 +298,15 @@ func TestInput(t *testing.T) {
293298 if !in.VOD || !in.Copy || in.Offset != 600 || in.Video != "0:v:0" || len(in.Audio) != 1 || in.Audio[0].Map != "1:a:0" {294299 t.Errorf("input: %+v", in)295300 }301+ // Its captions, timed from where it starts: the one showing then is302+ // clipped there.303+ if in.Subtitles == nil {304+ t.Fatal("no captions")305+ }306+ cues := in.Subtitles.Between(0, time.Hour)307+ if len(cues) != 2 || cues[0].Start != 0 || cues[0].End != 3*time.Second || cues[1].Start != 30*time.Second || cues[1].Text != "later on" {308+ t.Errorf("captions: %+v", cues)309+ }296310 if in, err := l.Input(context.Background(), "ep2", 3); err != nil || in.Offset != 0 || strings.Contains(strings.Join(in.Args, " "), "-ss") {297311 t.Errorf("from the start: %+v %v", in, err)298312 }@@ -493,11 +493,16 @@ func (l *Library) ahead(ctx context.Context) {
493493 return494494 }495495 tool.Ahead.Lock()496- _, err := l.fetch(ctx, id)496+ info, err := l.fetch(ctx, id)497497 tool.Ahead.Unlock()498498 if err != nil && ctx.Err() == nil {499499 log.Printf("podcasts: finding %s ahead: %v", id, err)500500 }501+ if err == nil {502+ cctx, cancel := context.WithTimeout(ctx, captionsTime)503+ l.captionsFor(cctx, id, info)504+ cancel()505+ }501506 }502507 }503508 @@ -193,7 +193,11 @@ func (s *Server) Start(ctx context.Context, client string, in tuner.Input) (*Pla
193193 }()194194 }195195 if subtitled(in) {196- go newSubtitler(dir, in.Subtitles).run(runCtx)196+ sub := newSubtitler(dir, in.Subtitles)197+ if in.Copy {198+ sub.byAudio()199+ }200+ go sub.run(runCtx)197201 }198202 go func() {199203 _ = cmd.Wait()@@ -334,8 +338,9 @@ func playlists(in tuner.Input) []string {
334338 }335339 336340 // subtitled reports whether a session has a WebVTT subtitle rendition:337-// live streams with their captions as text.338-func subtitled(in tuner.Input) bool { return in.Subtitles != nil && !in.VOD }341+// live streams with their captions as text, and podcasts' episodes (a342+// recording's are in its picture).343+func subtitled(in tuner.Input) bool { return in.Subtitles != nil && (!in.VOD || in.Copy) }339344 340345 // ffmpegArgs transcodes to H.264 and AAC in HLS, or copies a picture341346 // that's H.264 already (Copy). Several audio tracks become separate@@ -41,6 +41,8 @@ type subtitler struct {
4141 script *cc.Script4242 first int64 // PTS of the stream's first picture; -1 until known4343 segs map[int]subSegment44+ audio bool // the script counts from the first sound (byAudio)45+ lead time.Duration // and that's this long after the first picture4446 }4547 4648 // subSegment is a video segment's place in time.@@ -55,6 +57,15 @@ func newSubtitler(dir string, script *cc.Script) *subtitler {
5557 return &subtitler{dir: dir, script: script, first: -1, segs: map[int]subSegment{}}5658 }5759 60+// byAudio has the script count from the stream's first sound rather than61+// its first picture: for a picture copied as it is (a podcast's), which62+// starts at the key frame before where the stream was asked to, while the63+// sound starts there.64+func (s *subtitler) byAudio() *subtitler {65+ s.audio = true66+ return s67+}68+5869 // run keeps the subtitles in step with the video until ctx ends.5970 func (s *subtitler) run(ctx context.Context) {6071 t := time.NewTicker(subsPoll)@@ -74,6 +85,8 @@ type mediaPlaylist struct {
7485 target string // EXT-X-TARGETDURATION7586 sequence int // of the first segment7687 segs []playlistSegment88+ kind string // EXT-X-PLAYLIST-TYPE, "" for none89+ ended bool // EXT-X-ENDLIST7790 }7891 7992 type playlistSegment struct {@@ -90,6 +103,10 @@ func parseMediaPlaylist(b []byte) (mediaPlaylist, error) {
90103 switch {91104 case strings.HasPrefix(line, "#EXT-X-TARGETDURATION:"):92105 p.target = strings.TrimPrefix(line, "#EXT-X-TARGETDURATION:")106+ case strings.HasPrefix(line, "#EXT-X-PLAYLIST-TYPE:"):107+ p.kind = strings.TrimPrefix(line, "#EXT-X-PLAYLIST-TYPE:")108+ case line == "#EXT-X-ENDLIST":109+ p.ended = true93110 case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):94111 p.sequence, _ = strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"))95112 case strings.HasPrefix(line, "#EXTINF:"):@@ -119,15 +136,23 @@ func (s *subtitler) update() error {
119136 }120137 var list bytes.Buffer121138 fmt.Fprintf(&list, "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:%s\n#EXT-X-MEDIA-SEQUENCE:%d\n", p.target, p.sequence)139+ if p.kind != "" {140+ fmt.Fprintf(&list, "#EXT-X-PLAYLIST-TYPE:%s\n", p.kind)141+ }142+ ended := p.ended // the video's ended, and so have these once all are written122143 for i, seg := range p.segs {123144 n := p.sequence + i124145 if _, done := s.segs[n]; !done {125146 if err := s.write(n, seg); err != nil {147+ ended = false126148 break // list only what's written; try again shortly127149 }128150 }129151 fmt.Fprintf(&list, "#EXTINF:%.6f,\n%s\n", seg.dur, subName(n))130152 }153+ if ended {154+ list.WriteString("#EXT-X-ENDLIST\n")155+ }131156 if err := writeAtomic(filepath.Join(s.dir, subsPlaylist), list.Bytes()); err != nil {132157 return err133158 }@@ -149,7 +174,7 @@ func subName(n int) string { return fmt.Sprintf("sub_%05d.vtt", n) }
149174 150175 // write makes the WebVTT segment for video segment n.151176 func (s *subtitler) write(n int, seg playlistSegment) error {152- pts, err := firstPTS(filepath.Join(s.dir, seg.uri))177+ pts, err := firstPTS(filepath.Join(s.dir, seg.uri), false)153178 if err != nil {154179 prev, ok := s.segs[n-1]155180 if !ok {@@ -163,6 +188,15 @@ func (s *subtitler) write(n int, seg playlistSegment) error {
163188 cur.pts = prev.pts + (pts-prev.raw)&ptsMask164189 } else if s.first < 0 {165190 s.first, cur.pts = pts, pts191+ if s.audio {192+ // How long the sound starts after the picture, which the193+ // script's times count from.194+ if a, err := firstPTS(filepath.Join(s.dir, seg.uri), true); err == nil {195+ if d := (a - pts) & ptsMask; d < 10*90000 {196+ s.lead = time.Duration(d) * time.Second / 90000197+ }198+ }199+ }166200 } else {167201 // A gap in what's been seen: the nearest way round the clock.168202 cur.pts = s.first + (pts-s.first)&ptsMask@@ -173,7 +207,7 @@ func (s *subtitler) write(n int, seg playlistSegment) error {
173207 b := fmt.Appendf(nil, "WEBVTT\nX-TIMESTAMP-MAP=MPEGTS:%d,LOCAL:%s\n\n", pts, cc.VTTTime(cur.start))174208 // Cues starting a little later come too, so players that lay captions175209 // out ahead know what's coming; they're repeated in their own segments.176- for _, c := range s.script.Between(cur.start, end+subsAhead) {210+ for _, c := range cc.Shift(s.script.Between(cur.start-s.lead, end+subsAhead-s.lead), s.lead) {177211 b = cc.AppendVTT(b, c)178212 }179213 if err := writeAtomic(filepath.Join(s.dir, subName(n)), b); err != nil {@@ -185,9 +219,9 @@ func (s *subtitler) write(n int, seg playlistSegment) error {
185219 186220 const ptsMask = 1<<33 - 1187221 188-// firstPTS reads the presentation time of a segment's first picture, in189-// 90 kHz ticks.190-func firstPTS(path string) (int64, error) {222+// firstPTS reads the presentation time of a segment's first picture, or223+// with audio its first sound, in 90 kHz ticks.224+func firstPTS(path string, audio bool) (int64, error) {191225 f, err := os.Open(path)192226 if err != nil {193227 return 0, err@@ -209,8 +243,11 @@ func firstPTS(path string) (int64, error) {
209243 }210244 p = pkt[5+int(pkt[4]):]211245 }212- // A video PES with a PTS.213- if len(p) >= 14 && p[0] == 0 && p[1] == 0 && p[2] == 1 && p[3]&0xf0 == 0xe0 && p[7]&0x80 != 0 {246+ // A video (or audio) PES with a PTS.247+ if len(p) < 14 || p[0] != 0 || p[1] != 0 || p[2] != 1 || p[7]&0x80 == 0 {248+ continue249+ }250+ if !audio && p[3]&0xf0 == 0xe0 || audio && p[3]&0xe0 == 0xc0 {214251 return int64(p[9]>>1&7)<<30 | int64(p[10])<<22 | int64(p[11]>>1)<<15 | int64(p[12])<<7 | int64(p[13]>>1), nil215252 }216253 }@@ -176,3 +176,39 @@ func TestMasterPlaylistForSubtitles(t *testing.T) {
176176 t.Error("a recording has no subtitle rendition")177177 }178178 }179+180+// TestSubtitlesByAudio: a copied picture starts at the key frame before181+// where the stream was asked to, the sound there; the captions, timed182+// from where it was asked to, come that much after the first picture. The183+// subtitles' playlist is of the video's kind, and ends with it.184+func TestSubtitlesByAudio(t *testing.T) {185+ dir := t.TempDir()186+ pes := func(id byte, pts int64) []byte {187+ p := []byte{0x47, 0x41, 0x00, 0x10, 0, 0, 1, id, 0, 0, 0x80, 0x80, 5,188+ byte(pts>>29&0x0e | 0x21), byte(pts >> 22), byte(pts>>14 | 1), byte(pts >> 7), byte(pts<<1 | 1)}189+ return append(p, make([]byte, 188-len(p))...)190+ }191+ picture := int64(900000)192+ seg := append(pes(0xe0, picture), pes(0xc0, picture+180000)...) // the sound two seconds on193+ if err := os.WriteFile(filepath.Join(dir, "seg_0_00000.ts"), seg, 0o644); err != nil {194+ t.Fatal(err)195+ }196+ list := "#EXTM3U\n#EXT-X-VERSION:6\n#EXT-X-TARGETDURATION:4\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4.000000,\nseg_0_00000.ts\n#EXT-X-ENDLIST\n"197+ if err := os.WriteFile(filepath.Join(dir, "stream_0.m3u8"), []byte(list), 0o644); err != nil {198+ t.Fatal(err)199+ }200+ var script cc.Script201+ script.Set(0, time.Hour, []cc.Cue{{Start: time.Second, End: 1500 * time.Millisecond, Text: "one second in"}})202+ s := newSubtitler(dir, &script).byAudio()203+ if err := s.update(); err != nil {204+ t.Fatal(err)205+ }206+ vtt, _ := os.ReadFile(filepath.Join(dir, subName(0)))207+ if !strings.Contains(string(vtt), "00:00:03.000 --> 00:00:03.500\none second in") {208+ t.Errorf("segment:\n%s", vtt)209+ }210+ subs, _ := os.ReadFile(filepath.Join(dir, subsPlaylist))211+ if !strings.Contains(string(subs), "#EXT-X-PLAYLIST-TYPE:EVENT\n") || !strings.HasSuffix(string(subs), "#EXT-X-ENDLIST\n") {212+ t.Errorf("playlist:\n%s", subs)213+ }214+}