A video's decoder no longer waits up to 1 s for its captions to load; only its picture waits for them on the way to the encoder, so no caption is lost and a Jellyfin tune is about half a second faster.
- commit
- 57ef46f4cb891c4fa74f033d3d096beb3813c69d
- parent
- 22dba1662cc8d3c13b3386f4160f060ddea9317f
- author
- FelineStateMachine <226808055+FelineStateMachine@users.noreply.github.com>
- kind
- fix risk score 1.8 of 3, Jev's estimate
| M | internal/ | 57 | ++++++++++++++----------- |
| M | internal/ | 70 | +++++++++++++++++++++++++++++- |
| M | internal/ | 27 | ++++++++++-- |
@@ -197,11 +197,14 @@ const (
197197 )198198 199199 // subtitle sets up a video's captions as it starts at span s, skip frames200-// in, and returns a subtitle file to burn into the picture, or "". The201-// file is the caller's to remove.202-func subtitle(ctx context.Context, ffmpeg string, it item, skip int64, s *span, out *output) string {200+// in. It returns a subtitle file to burn into the picture, or "" (the201+// file is the caller's to remove), and for closed captions, settled,202+// closed once they're in place or will come late: the video's picture203+// waits for it on its way to the encoder, so the video opens meanwhile.204+// settled is nil when there's nothing to wait for.205+func subtitle(ctx context.Context, ffmpeg string, it item, skip int64, s *span, out *output) (burn string, settled <-chan struct{}) {203206 if it.Captions == nil {204- return ""207+ return "", nil205208 }206209 offset := frameDuration(skip)207210 if !out.app && cc.Foreign(it.AudioLang) && canBurn(ffmpeg) {@@ -210,37 +213,41 @@ func subtitle(ctx context.Context, ffmpeg string, it item, skip int64, s *span,
210213 cancel()211214 if cues = cc.Shift(cues, -offset); err == nil && len(cues) > 0 {212215 if path, err := writeSubtitles(cc.Screen(cues)); err == nil {213- return path216+ return path, nil214217 }215218 }216219 }217220 // Captions not fetched yet get a moment before the video starts, then218221 // come in late.219- wait, cancel := context.WithTimeout(ctx, captionsWait)220- _, _ = it.Captions.get(wait)221- cancel()222- phase.Step(ctx, "captions")223- if cues, ok := it.Captions.ready(); ok {224- out.captions.set(s, cc.Shift(cues, -offset))225- return ""226- }222+ done := make(chan struct{})227223 go func() {228- cues, err := it.Captions.get(ctx)229- if err != nil {224+ defer close(done)225+ wait, cancel := context.WithTimeout(ctx, captionsWait)226+ _, _ = it.Captions.get(wait)227+ cancel()228+ phase.Step(ctx, "captions")229+ if cues, ok := it.Captions.ready(); ok {230+ out.captions.set(s, cc.Shift(cues, -offset))230231 return231232 }232- // They're late: start from a little after now, so every caption233- // that shows has been loaded.234- in := frameDuration(out.video.fed()-s.start) + lateCaptions235- var keep []cc.Cue236- for _, c := range cc.Shift(cues, -offset) {237- if c.Start >= in {238- keep = append(keep, c)233+ go func() {234+ cues, err := it.Captions.get(ctx)235+ if err != nil {236+ return239237 }240- }241- out.captions.set(s, keep)238+ // They're late: start from a little after now, so every caption239+ // that shows has been loaded.240+ in := frameDuration(out.video.fed()-s.start) + lateCaptions241+ var keep []cc.Cue242+ for _, c := range cc.Shift(cues, -offset) {243+ if c.Start >= in {244+ keep = append(keep, c)245+ }246+ }247+ out.captions.set(s, keep)248+ }()242249 }()243- return ""250+ return "", done244251 }245252 246253 // writeSubtitles writes cues to a temporary SRT file for ffmpeg's@@ -21,6 +21,7 @@ import (
2121 "airwaves/internal/cc"2222 "airwaves/internal/jellyfin"2323 "airwaves/internal/jellyfin/jellyfintest"24+ "airwaves/internal/phase"2425 "airwaves/internal/stream"2526 "airwaves/internal/tuner"2627 )@@ -87,6 +88,68 @@ func TestStreamCarriesCaptions(t *testing.T) {
8788 }8889 }8990 91+// TestCaptionsLoadWhileTheVideoOpens: captions that take a while to92+// fetch don't hold the video's opening up, only its picture going on to93+// the encoder, so they're all there from the start.94+func TestCaptionsLoadWhileTheVideoOpens(t *testing.T) {95+ ffmpeg, err := exec.LookPath("ffmpeg")96+ if err != nil {97+ t.Skip("no ffmpeg")98+ }99+ dir := t.TempDir()100+ video := filepath.Join(dir, "v.mp4")101+ cmd := exec.Command(ffmpeg, "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=640x360:r=25:d=4",102+ "-f", "lavfi", "-i", "sine=f=440:d=4", "-c:v", "libx264", "-c:a", "aac", video)103+ if out, err := cmd.CombinedOutput(); err != nil {104+ t.Fatalf("make video: %v: %s", err, out)105+ }106+ slow := newCaptions("slow", func(context.Context) ([]byte, error) {107+ time.Sleep(700 * time.Millisecond)108+ return []byte("1\n00:00:00,500 --> 00:00:02,000\nFrom the start\n"), nil109+ })110+ it := item{Path: video, Frames: 4 * loopFPS, Audio: true, Title: "Captioned", Captions: slow}111+ n := 0112+ cue := func(time.Time, bool) (item, int64, error) {113+ if n++; n == 1 {114+ return it, 0, nil115+ }116+ return item{Slate: true, Frames: 10 * loopFPS}, 0, nil117+ }118+ timer := phase.New("test", "folder")119+ ctx, cancel := context.WithTimeout(phase.NewContext(context.Background(), timer), 3*time.Second)120+ defer cancel()121+ var out bytes.Buffer122+ if err := playCues(ctx, &out, ffmpeg, "test", cue); err != nil {123+ t.Fatal(err)124+ }125+ var decoding, captioned time.Duration126+ for _, m := range timer.Marks() {127+ switch m.Name {128+ case "decoding":129+ decoding = m.At130+ case "captions":131+ captioned = m.At132+ }133+ }134+ if decoding == 0 || captioned < 600*time.Millisecond || decoding > captioned {135+ t.Errorf("the video waited for its captions to open: %s", timer)136+ }137+ if err := os.WriteFile(filepath.Join(dir, "out.ts"), out.Bytes(), 0o644); err != nil {138+ t.Fatal(err)139+ }140+ extract := exec.Command(ffmpeg, "-hide_banner", "-loglevel", "error", "-nostdin",141+ "-f", "lavfi", "-i", "movie=out.ts[out0+subcc]", "-map", "0:s", "-f", "srt", "-")142+ extract.Dir = dir143+ srt, err := extract.Output()144+ if err != nil {145+ t.Fatalf("extracting captions: %v", err)146+ }147+ got, err := cc.Parse(srt)148+ if err != nil || len(got) != 1 || got[0].Text != "From the start" || got[0].Start > time.Second {149+ t.Errorf("captions %q (%v), want the one from the start:\n%s", got, err, srt)150+ }151+}152+90153 func TestFeedCountsWholeFrames(t *testing.T) {91154 r, w, err := os.Pipe()92155 if err != nil {@@ -99,7 +162,7 @@ func TestFeedCountsWholeFrames(t *testing.T) {
99162 got <- b100163 }()101164 for _, data := range []string{"0123456789abcdefghij", "klmno", ""} {102- src, done, err := f.source(nil)165+ src, done, err := f.source(nil, nil)103166 if err != nil {104167 t.Fatal(err)105168 }@@ -187,7 +250,10 @@ func TestSubtitleChoosesBurnIn(t *testing.T) {
187250 out := &output{captions: &timeline{}, app: c.app, video: &feed{unit: frameBytes}}188251 it := item{AudioLang: c.lang, Captions: srtCaptions(srt)}189252 s := out.captions.begin(0)190- path := subtitle(context.Background(), ffmpeg, it, 2*loopFPS, s, out)253+ path, settled := subtitle(context.Background(), ffmpeg, it, 2*loopFPS, s, out)254+ if settled != nil {255+ <-settled256+ }191257 burnt := path != ""192258 if burnt {193259 b, _ := os.ReadFile(path)@@ -545,7 +545,8 @@ func (p *pacer) wait(n int64) bool {
545545 // decoder cut off mid-frame has its frame made up, so the next decoder's546546 // frames start whole. Close the pipe once the decoder has it. process, when547547 // set, sees what passes, whole units at a time, and may change it in place.548-func (f *feed) source(process func([]byte)) (w *os.File, done func() int64, err error) {548+// hold, when set, holds what the decoder writes back until it's closed.549+func (f *feed) source(process func([]byte), hold <-chan struct{}) (w *os.File, done func() int64, err error) {549550 r, w, err := os.Pipe()550551 if err != nil {551552 return nil, nil, err@@ -561,6 +562,9 @@ func (f *feed) source(process func([]byte)) (w *os.File, done func() int64, err
561562 if n == 0 && f.first != nil {562563 f.first()563564 }565+ if n == 0 && hold != nil {566+ <-hold567+ }564568 if process != nil {565569 process(b[:k-k%f.unit])566570 }@@ -726,7 +730,12 @@ func decode(ctx context.Context, ffmpeg, label string, it item, skip int64, out
726730 default:727731 input, sound = it.inputs(skip)728732 }729- video, videoDone, err := out.video.source(nil)733+ // The picture waits for the video's captions to be in place before it734+ // goes to the encoder (see subtitle), so the video opens meanwhile.735+ hold := make(chan struct{})736+ release := sync.OnceFunc(func() { close(hold) })737+ defer release()738+ video, videoDone, err := out.video.source(nil, hold)730739 if err != nil {731740 log.Printf("%s: %v", label, err)732741 return false@@ -739,8 +748,9 @@ func decode(ctx context.Context, ffmpeg, label string, it item, skip int64, out
739748 lev = newLeveler(it.Level)740749 level = lev.process741750 }742- audio, audioDone, err := out.audio.source(level)751+ audio, audioDone, err := out.audio.source(level, nil)743752 if err != nil {753+ release()744754 video.Close()745755 videoDone()746756 log.Printf("%s: %v", label, err)@@ -749,12 +759,21 @@ func decode(ctx context.Context, ffmpeg, label string, it item, skip int64, out
749759 span := out.captions.begin(out.video.units)750760 var args []string751761 if it.Slate {762+ release()752763 args = slateArgs(ffmpeg, it.Title, it.Frames-skip)753764 } else {754- subs := subtitle(ctx, ffmpeg, it, skip, span, out)765+ subs, settled := subtitle(ctx, ffmpeg, it, skip, span, out)755766 if subs != "" {756767 defer os.Remove(subs)757768 }769+ if settled == nil {770+ release()771+ } else {772+ go func() {773+ <-settled774+ release()775+ }()776+ }758777 args = decodeInputs(input, sound, it.Frames-skip, subs)759778 }760779 dec := exec.CommandContext(ctx, ffmpeg, args...)