1package podcast23import (4 "cmp"5 "context"6 "errors"7 "log"8 "math"9 "net/http"10 "regexp"11 "slices"12 "strconv"13 "strings"14 "time"1516 "golang.org/x/sync/errgroup"1718 "airwaves/internal/web"19 "airwaves/internal/youtube"20)2122// Keeping up: each show is listed every listEvery (its newest listLimit23// videos), and a channel's feed, YouTube's cheap word of new uploads but24// down at times, is read every feedEvery in between. A video new to a25// show is looked up (its length, exact date, description and whether it's26// an episode at all), a few an update with a gap between, so yt-dlp isn't27// run hard. The newest unwatched episode of each show has where it28// streams from found ahead, so Play starts at once.29const (30 listEvery = 6 * time.Hour31 feedEvery = 30 * time.Minute32 retryList = time.Hour // after a listing failed33 maxLookups = 12 // videos looked up per update34 tickEvery = time.Minute35 // minLeft is how long addresses must still work to be used; they're36 // found again sooner.37 minLeft = 20 * time.Minute38 // resolveTime bounds finding where an episode streams from.39 resolveTime = 90 * time.Second40 keepSkipped = 30041)4243// lookupGap spaces out yt-dlp runs; tests shorten it.44var lookupGap = 3 * time.Second4546// resolved is where an episode streams from, until when.47type resolved struct {48 info youtube.Info49 until time.Time50}5152// Run keeps the shows up to date until ctx ends.53func (l *Library) Run(ctx context.Context) {54 l.mu.Lock()55 l.load()56 kick := l.kick57 l.mu.Unlock()58 t := time.NewTicker(tickEvery)59 defer t.Stop()60 for {61 l.update(ctx)62 select {63 case <-ctx.Done():64 return65 case <-t.C:66 case <-kick:67 }68 }69}7071// poke has Run update now. l.mu is held.72func (l *Library) poke() {73 select {74 case l.kick <- struct{}{}:75 default:76 }77}7879// update lists the shows due, reads the feeds due, looks up new videos80// and finds the next episodes' streams ahead. One show failing doesn't81// stop the others.82func (l *Library) update(ctx context.Context) {83 now := l.clock()84 l.mu.Lock()85 l.load()86 var lists, feeds []*show87 for _, s := range l.st.Shows {88 switch {89 case l.listDue(s, now):90 lists = append(lists, s)91 case !s.source().playlist && s.ChannelID != "" && now.Sub(s.Checked) >= feedEvery:92 feeds = append(feeds, s)93 }94 }95 l.updating = len(lists) > 096 l.mu.Unlock()97 tool := l.tool()98 for _, s := range lists {99 if ctx.Err() != nil {100 return101 }102 tool.Jobs.Lock() // one listing job at a time across yt-dlp's users103 l.relist(ctx, s)104 tool.Jobs.Unlock()105 }106 l.mu.Lock()107 l.updating = false108 l.mu.Unlock()109 for _, s := range feeds {110 if ctx.Err() != nil {111 return112 }113 l.readFeed(ctx, s)114 }115 l.lookups(ctx)116 l.ahead(ctx)117}118119// listDue reports whether a show is due a listing. l.mu is held.120func (l *Library) listDue(s *show, now time.Time) bool {121 if s.Error != "" {122 return now.Sub(s.Listed) >= retryList123 }124 return now.Sub(s.Listed) >= listEvery125}126127// list lists a show's newest videos, a few seconds' work. It doesn't take128// its turn with yt-dlp's other listing jobs, which can be a channel's whole129// catalog and take minutes: the background updates take turns (see130// update), while someone adding a show or refreshing doesn't wait behind131// them.132func (l *Library) list(ctx context.Context, s src) (youtube.Listing, error) {133 ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)134 defer cancel()135 args := append(slices.Clone(youtube.ListingArgs), "-I", "1:"+strconv.Itoa(listLimit), s.url)136 raw, err := l.tool().Run(ctx, args...)137 if err != nil {138 return youtube.Listing{}, err139 }140 if s.playlist {141 return youtube.ParsePlaylist(raw)142 }143 return youtube.ParseListing(raw)144}145146// refreshAtOnce is how many shows Refresh lists at the same time.147const refreshAtOnce = 4148149// Refresh lists every show again now, rather than at its turn: for someone150// at the screen, so a few at once, and not behind yt-dlp's other jobs (see151// list). The new episodes are looked up after, in the background. It152// returns the library as refreshed.153func (l *Library) Refresh(ctx context.Context) State {154 l.mu.Lock()155 l.load()156 shows := slices.Clone(l.st.Shows)157 l.mu.Unlock()158 var g errgroup.Group159 g.SetLimit(refreshAtOnce)160 for _, s := range shows {161 g.Go(func() error {162 l.relist(ctx, s)163 return nil164 })165 }166 g.Wait()167 l.mu.Lock()168 l.poke()169 l.mu.Unlock()170 return l.State()171}172173// relist lists a show again and takes in what's new.174func (l *Library) relist(ctx context.Context, s *show) {175 lst, err := l.list(ctx, s.source())176 if err != nil && ctx.Err() != nil {177 return // stopped, not failed: it's listed at its turn178 }179 now := l.clock()180 l.mu.Lock()181 defer l.mu.Unlock()182 if l.find(s.ID) != s { // removed meanwhile183 return184 }185 s.Listed = now186 if err != nil {187 s.Error = err.Error()188 log.Printf("podcasts: %s: %v", s.Title, err)189 l.save()190 return191 }192 s.Error = ""193 s.merge(lst, s.source(), now)194 l.save()195}196197// merge takes in a listing: its newest episodes, keeping what's known of198// each, plus older ones part watched. Videos too short to be episodes,199// and streams to come or under way, are left out.200func (s *show) merge(lst youtube.Listing, sr src, now time.Time) {201 if t := showTitle(lst, sr); t != "" {202 s.Title = t203 }204 if lst.Image != "" {205 s.Thumbnail = lst.Image206 }207 if !sr.playlist {208 s.ChannelID = lst.ID209 } else {210 s.ChannelID = lst.OwnerID211 }212 s.Listed = now213 known := map[string]*episode{}214 for _, e := range s.Episodes {215 known[e.ID] = e216 }217 var out []*episode218 seen := map[string]bool{}219 for _, v := range lst.Entries {220 // A video without a title yet (a premiere to come, say) is221 // listed once it has one.222 if v.Title == "" {223 continue224 }225 if seen[v.ID] || !s.fits(v.Seconds, v.LiveStatus, sr) {226 if v.Seconds > 0 && float64(v.Seconds) < s.MinMinutes*60 {227 s.skip(v.ID)228 }229 continue230 }231 seen[v.ID] = true232 e := known[v.ID]233 if e == nil {234 e = &episode{ID: v.ID}235 }236 e.Title = v.Title237 if v.Seconds > 0 {238 e.Seconds = float64(v.Seconds)239 }240 if !v.Published.IsZero() && (e.Published.IsZero() || e.Approx) {241 e.Published, e.Approx = v.Published, true242 }243 out = append(out, e)244 }245 if sr.playlist && !slices.ContainsFunc(out, func(e *episode) bool { return e.Published.IsZero() }) {246 slices.SortStableFunc(out, func(a, b *episode) int { return b.Published.Compare(a.Published) })247 }248 // New uploads the feed found that the listing doesn't show yet stay.249 var fresh []*episode250 for _, e := range s.Episodes {251 if !seen[e.ID] && !e.Published.IsZero() && now.Sub(e.Published) < 3*24*time.Hour && !e.Approx {252 fresh = append(fresh, e)253 seen[e.ID] = true254 }255 }256 out = append(fresh, out...)257 if len(out) > keepEpisodes {258 out = out[:keepEpisodes]259 }260 for _, e := range s.Episodes { // part watched, kept till finished261 if !seen[e.ID] && e.Position > 0 && !e.Watched && !slices.Contains(out, e) {262 out = append(out, e)263 }264 }265 s.Episodes = out266}267268// fits reports whether a video can be one of the show's episodes, by269// what's known: long enough (when its length is), and not a stream to270// come, on now or not yet processed. A show of a channel's videos takes271// no past streams, which have their own tab.272func (s *show) fits(seconds int, live string, sr src) bool {273 switch live {274 case "is_upcoming", "is_live", "post_live":275 return false276 case "was_live":277 if sr.tab == "videos" {278 return false279 }280 }281 return seconds <= 0 || float64(seconds) >= s.MinMinutes*60282}283284// showTitle is what a listing calls the show: a playlist's title, unless285// it names no show ("Every Episode", "Full Episodes", "Podcast"), then its286// channel's name; a channel's name. A channel named as a handle is287// ("TheStandupPod") reads as words ("The Standup Pod").288func showTitle(lst youtube.Listing, sr src) string {289 if sr.playlist && lst.Owner != "" && strings.TrimSpace(generic.ReplaceAllString(lst.Name, "")) == "" {290 return spaced(lst.Owner)291 }292 if !sr.playlist {293 return spaced(lst.Name)294 }295 return cmp.Or(lst.Name, spaced(lst.Owner))296}297298// spaced puts spaces between the words of a name written as one299// ("TheStandupPod"), leaving any other name as it is.300func spaced(name string) string {301 if strings.ContainsAny(name, " _-.") || !camel.MatchString(name) {302 return name303 }304 return camel.ReplaceAllString(name, "$1 $2")305}306307var camel = regexp.MustCompile(`(\p{Ll})(\p{Lu})`)308309// generic are the words of a playlist's title that say what it holds, not310// which show.311var generic = regexp.MustCompile(`(?i)\b(the|every|all|full|latest|new|episodes?|podcasts?|shows?|uploads?|videos?)\b|[^\pL\pN]+`)312313// skip remembers a video that isn't an episode.314func (s *show) skip(id string) {315 if slices.Contains(s.Skipped, id) {316 return317 }318 s.Skipped = append(s.Skipped, id)319 if len(s.Skipped) > keepSkipped {320 s.Skipped = s.Skipped[len(s.Skipped)-keepSkipped:]321 }322}323324// readFeed reads a channel's feed: uploads the show hasn't seen join it,325// to be looked up, and ones it has get their exact dates and326// descriptions. Shorts are left out by their links.327func (l *Library) readFeed(ctx context.Context, s *show) {328 ctx, cancel := context.WithTimeout(ctx, 30*time.Second)329 defer cancel()330 l.mu.Lock()331 id := s.ChannelID332 l.mu.Unlock()333 raw, err := web.Get(ctx, l.tool().Client(), youtube.FeedURL(id))334 var entries []youtube.FeedEntry335 if err == nil {336 entries, err = youtube.ParseFeed(raw)337 }338 now := l.clock()339 l.mu.Lock()340 defer l.mu.Unlock()341 if l.find(s.ID) != s {342 return343 }344 s.Checked = now345 if err != nil {346 return // the listings will do347 }348 known := map[string]*episode{}349 for _, e := range s.Episodes {350 known[e.ID] = e351 }352 var fresh []*episode353 for _, f := range entries {354 if f.ID == "" || slices.Contains(s.Skipped, f.ID) {355 continue356 }357 if e := known[f.ID]; e != nil {358 if !f.Published.IsZero() && (e.Published.IsZero() || e.Approx) {359 e.Published, e.Approx = f.Published, false360 }361 if d := youtube.Description(f.Description); e.Description == "" && d != "" {362 e.Description = d363 }364 continue365 }366 if f.Short() {367 s.skip(f.ID)368 continue369 }370 // Older than the episodes kept: not new, just not kept.371 if n := len(s.Episodes); n >= keepEpisodes && !f.Published.After(s.Episodes[n-1].Published) {372 continue373 }374 fresh = append(fresh, &episode{ID: f.ID, Title: f.Title, Published: f.Published, Description: youtube.Description(f.Description)})375 }376 s.Episodes = append(fresh, s.Episodes...) // the next listing trims them377 l.save()378}379380// lookups looks up the details of videos new to their shows, newest381// first, a few an update: their lengths, dates and descriptions, and382// whether they're episodes at all.383func (l *Library) lookups(ctx context.Context) {384 type due struct {385 show *show386 id string387 published time.Time388 }389 l.mu.Lock()390 var todo []due391 for _, s := range l.st.Shows {392 for _, e := range s.Episodes {393 if e.Looked.IsZero() {394 todo = append(todo, due{s, e.ID, e.Published})395 }396 }397 }398 l.mu.Unlock()399 slices.SortStableFunc(todo, func(a, b due) int { return b.published.Compare(a.published) })400 for i, d := range todo {401 if i == maxLookups || ctx.Err() != nil {402 return403 }404 if i > 0 {405 select {406 case <-ctx.Done():407 return408 case <-time.After(lookupGap):409 }410 }411 l.lookup(ctx, d.show, d.id)412 }413}414415// lookup looks up a video's details, and drops it from its show when it416// isn't an episode after all.417func (l *Library) lookup(ctx context.Context, s *show, id string) {418 tool := l.tool()419 tool.Jobs.Lock() // its turn first: its time starts when it runs420 ctx, cancel := context.WithTimeout(ctx, resolveTime)421 defer cancel()422 raw, err := tool.Run(ctx, "-J", "--skip-download", "--no-playlist", "--ignore-no-formats-error", youtube.WatchURL(id))423 tool.Jobs.Unlock()424 var info youtube.Info425 if err == nil {426 info, err = youtube.ParseInfo(raw)427 }428 thumb := ""429 if err == nil {430 thumb = l.thumbnail(ctx, id)431 }432 now := l.clock()433 l.mu.Lock()434 defer l.mu.Unlock()435 if l.find(s.ID) != s {436 return437 }438 i := slices.IndexFunc(s.Episodes, func(e *episode) bool { return e.ID == id })439 if i < 0 {440 return441 }442 e := s.Episodes[i]443 if err != nil {444 if youtube.Gone(err) {445 s.Episodes = slices.Delete(s.Episodes, i, i+1)446 s.skip(id)447 l.save()448 } else if ctx.Err() == nil {449 e.Looked = now // tried; the next listing tries again450 log.Printf("podcasts: %s: looking up %s: %v", s.Title, id, err)451 }452 return453 }454 e.Looked = now455 if info.Title != "" {456 e.Title = info.Title457 }458 if info.Duration > 0 {459 e.Seconds = math.Round(info.Duration)460 }461 if info.Timestamp > 0 {462 e.Published, e.Approx = time.Unix(int64(info.Timestamp), 0).UTC(), false463 }464 if d := youtube.Description(info.Description); d != "" {465 e.Description = d466 }467 if thumb != "" {468 e.Thumbnail = thumb469 }470 live := info.LiveStatus471 if info.MediaType == "short" || !s.fits(int(e.Seconds), live, s.source()) {472 s.Episodes = slices.Delete(s.Episodes, i, i+1)473 if live != "is_upcoming" && live != "is_live" && live != "post_live" {474 s.skip(id) // for good; a stream to come may be one later475 }476 }477 l.save()478}479480// thumbnail is a video's largest picture: maxresdefault when YouTube has481// one, which not every video does, else "" for the one every video has.482func (l *Library) thumbnail(ctx context.Context, id string) string {483 u := "https://i.ytimg.com/vi/" + id + "/maxresdefault.jpg"484 req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)485 if err != nil {486 return ""487 }488 resp, err := l.tool().Client().Do(req)489 if err != nil {490 return ""491 }492 resp.Body.Close()493 if resp.StatusCode != http.StatusOK {494 return ""495 }496 return u497}498499// ahead finds where the newest unwatched episode of each show streams500// from, so Play starts at once, and forgets addresses that expired.501func (l *Library) ahead(ctx context.Context) {502 now := l.clock()503 l.mu.Lock()504 for id, r := range l.resolved {505 if now.After(r.until) {506 delete(l.resolved, id)507 }508 }509 var ids []string510 for _, s := range l.st.Shows {511 for _, e := range s.Episodes {512 if e.Watched || e.Looked.IsZero() {513 continue514 }515 if r, ok := l.resolved[e.ID]; !ok || r.until.Sub(now) < minLeft {516 ids = append(ids, e.ID)517 }518 break519 }520 }521 l.mu.Unlock()522 tool := l.tool()523 for _, id := range ids {524 if ctx.Err() != nil {525 return526 }527 tool.Ahead.Lock()528 info, err := l.fetch(ctx, id)529 tool.Ahead.Unlock()530 if err != nil && ctx.Err() == nil {531 log.Printf("podcasts: finding %s ahead: %v", id, err)532 }533 if err == nil {534 cctx, cancel := context.WithTimeout(ctx, captionsTime)535 l.captionsFor(cctx, id, info)536 cancel()537 }538 }539}540541// resolve finds where an episode streams from: as found ahead, else now.542func (l *Library) resolve(ctx context.Context, id string) (youtube.Info, error) {543 l.mu.Lock()544 r, ok := l.resolved[id]545 l.mu.Unlock()546 if ok && time.Until(r.until) >= minLeft {547 return r.info, nil548 }549 return l.fetch(ctx, id)550}551552// fetch asks yt-dlp where an episode streams from, and keeps it until it553// expires.554func (l *Library) fetch(ctx context.Context, id string) (youtube.Info, error) {555 ctx, cancel := context.WithTimeout(ctx, resolveTime)556 defer cancel()557 raw, err := l.tool().Run(ctx, "-J", "--no-playlist", "-f", youtube.FormatSpec(maxHeight), youtube.WatchURL(id))558 if err != nil {559 return youtube.Info{}, err560 }561 info, err := youtube.ParseInfo(raw)562 if err == nil && len(info.Inputs()) == 0 {563 err = errors.New("yt-dlp found no stream")564 }565 if err != nil {566 return youtube.Info{}, err567 }568 l.mu.Lock()569 l.resolved[id] = resolved{info: info, until: youtube.AddressesUntil(info, time.Now())}570 l.mu.Unlock()571 return info, nil572}