1package admin23// YouTube channels: a youtube.json in the channel's folder names the4// YouTube channels whose uploads play, or a playlist to play in order (see5// vchan.YouTube). The page finds them with yt-dlp, by searching YouTube's6// channels or looking up a link.78import (9 "context"10 "encoding/json"11 "errors"12 "fmt"13 "net/http"14 "net/url"15 "os"16 "regexp"17 "slices"18 "strings"19 "time"20 "unicode/utf8"2122 "airwaves/internal/vchan"23 "airwaves/internal/youtube"24)2526const (27 youtubeFile = "youtube.json"28 maxSources = 1029 // How long a search's results and a lookup are trusted, and a failed30 // lookup remembered.31 searchFor = 5 * time.Minute32 lookupFor = 6 * time.Hour33 lookupErrFor = 10 * time.Minute34 searchTime = 45 * time.Second35 lookupTime = 60 * time.Second // two runs of yt-dlp36 searchCount = 1537 // ytRuns is how many yt-dlps the page runs at once.38 ytRuns = 339)4041var (42 handleRe = regexp.MustCompile(`^@[\p{L}\p{M}\p{N}._-]{1,100}$`)43 channelIDRe = regexp.MustCompile(`^UC[0-9A-Za-z_-]{22}$`)44 customRe = regexp.MustCompile(`^[\p{L}\p{M}\p{N}._-]{1,100}$`) // /c/ and /user/ names45 heights = []int{360, 480, 720, 1080}46 // rerunMixes are how strongly reruns lean toward recent uploads.47 rerunMixes = []string{"recent", "balanced", "any"}48 // ytErrPrefix is how yt-dlp starts its errors: "[youtube:tab] @name: ".49 ytErrPrefix = regexp.MustCompile(`^yt-dlp: \[[^\]]+\] (\S+: )?`)50 // pathYtDlp runs yt-dlp from PATH when the library has none of its own.51 pathYtDlp = &youtube.YtDlp{}52)5354var (55 errNotChannel = errors.New("that isn't a YouTube channel: use its link (youtube.com/@handle), @handle or channel ID")56 errVideoLink = errors.New("that's a link to a video: use the channel's link instead, like youtube.com/@handle")57 errPlaylistLink = errors.New("that's a playlist: play it as the channel's playlist, or use a channel's link, like youtube.com/@handle")58 errBoth = errors.New("a channel plays YouTube channels' uploads or a playlist, not both")59)6061// ytConfigJSON is a YouTube channel's settings, as the page shows them.62type ytConfigJSON struct {63 Channels []string `json:"channels"`64 Playlist string `json:"playlist,omitempty"` // its page; played in order, in place of channels65 RepeatDays float64 `json:"repeatDays"`66 MaxHeight int `json:"maxHeight"`67 MinMinutes float64 `json:"minMinutes"`68 MaxMinutes float64 `json:"maxMinutes"` // 0: no limit69 RerunMix string `json:"rerunMix"`70 MaxAgeDays float64 `json:"maxAgeDays"` // 0: any age71 DeadAir bool `json:"deadAir"` // stand by rather than repeat within RepeatDays72}7374// ytReq is a YouTube channel's settings as the page sends them. Settings75// left out stay as they are (for a new channel, the defaults).76type ytReq struct {77 Channels []string `json:"channels"`78 Playlist *string `json:"playlist"` // a link or ID; "" for none79 RepeatDays *float64 `json:"repeatDays"`80 MaxHeight *int `json:"maxHeight"`81 MinMinutes *float64 `json:"minMinutes"`82 MaxMinutes *float64 `json:"maxMinutes"`83 RerunMix *string `json:"rerunMix"`84 MaxAgeDays *float64 `json:"maxAgeDays"`85 DeadAir *bool `json:"deadAir"`86}8788// ytSourceJSON is one of the YouTube channels a channel plays, or its89// playlist, and how its listing is going.90type ytSourceJSON struct {91 Channel string `json:"channel"` // as in youtube.json92 URL string `json:"url"` // its page93 ID string `json:"id,omitempty"`94 Name string `json:"name,omitempty"` // a playlist's title95 Handle string `json:"handle,omitempty"`96 Image string `json:"image,omitempty"` // a playlist's first video's thumbnail97 Owner string `json:"owner,omitempty"` // a playlist's channel98 OwnerURL string `json:"ownerUrl,omitempty"`99 Videos int `json:"videos"` // in the catalog100 Playable int `json:"playable"` // of those, the ones that fit the settings, age included101 Listed *time.Time `json:"listed,omitempty"`102 Listing bool `json:"listing"`103 Since *time.Time `json:"listingSince,omitempty"`104 Failed *time.Time `json:"failed,omitempty"`105 Error string `json:"error,omitempty"` // why it can't be listed, when known106}107108// ytChannelJSON is a channel a search or lookup found.109type ytChannelJSON struct {110 ID string `json:"id"`111 URL string `json:"url"`112 Handle string `json:"handle"`113 Name string `json:"name"`114 Subscribers int64 `json:"subscribers"`115 Description string `json:"description"`116 Image string `json:"image"`117 Videos int `json:"videos,omitempty"` // roughly, from a lookup118}119120func channelOut(c youtube.Channel) ytChannelJSON {121 return ytChannelJSON{ID: c.ID, URL: c.URL, Handle: c.Handle, Name: c.Name, Subscribers: c.Subscribers,122 Description: c.Description, Image: c.Image, Videos: c.Videos}123}124125type searched struct {126 found []ytChannelJSON127 at time.Time128}129130// ytPlaylistJSON is a playlist a lookup found.131type ytPlaylistJSON struct {132 ID string `json:"id"`133 URL string `json:"url"`134 Title string `json:"title"`135 Owner string `json:"owner"`136 OwnerURL string `json:"ownerUrl,omitempty"`137 Videos int `json:"videos"` // as YouTube counts them, ones that can't play included138 Image string `json:"image,omitempty"`139}140141func playlistOut(p youtube.PlaylistInfo) ytPlaylistJSON {142 return ytPlaylistJSON{ID: p.ID, URL: p.URL, Title: p.Title, Owner: p.Owner, OwnerURL: ownerURL(p.OwnerID), Videos: p.Videos, Image: p.Image}143}144145func ownerURL(id string) string {146 if id == "" {147 return ""148 }149 return "https://www.youtube.com/channel/" + id150}151152// lookedUp is a lookup of a channel, or of a playlist.153type lookedUp struct {154 ch ytChannelJSON155 pl ytPlaylistJSON156 err error157 at time.Time158}159160func (l lookedUp) fresh() bool {161 if l.err != nil {162 return time.Since(l.at) < lookupErrFor163 }164 return time.Since(l.at) < lookupFor165}166167// ytConfigOf is the settings the page edits, from a youtube.json read168// with the defaults filled in.169func ytConfigOf(cfg vchan.YouTubeConfig) ytConfigJSON {170 c := ytConfigJSON{Channels: cfg.Channels, Playlist: cfg.Playlist, RepeatDays: cfg.RepeatDays, MaxHeight: cfg.MaxHeight,171 MinMinutes: cfg.MinMinutes, MaxMinutes: cfg.MaxMinutes, RerunMix: cfg.RerunMix, MaxAgeDays: cfg.MaxAgeDays, DeadAir: cfg.DeadAir}172 if c.Channels == nil {173 c.Channels = []string{}174 }175 return c176}177178// describeYouTube fills in a YouTube channel's settings and how its179// sources are doing: Videos counts the catalogs, Items the videos that180// fit the settings, or a playlist's that can play.181func (s *Server) describeYouTube(c *channelJSON, y *vchan.YouTube) {182 st := y.Status()183 cfg := ytConfigOf(st.Config)184 c.YouTube = &cfg185 if st.Err != nil {186 c.Error = st.Err.Error()187 }188 videos, playable := 0, 0189 c.Sources = []ytSourceJSON{}190 for _, src := range st.Sources {191 j := ytSourceJSON{Channel: src.Channel, URL: src.URL, ID: src.ID, Name: src.Name, Videos: src.Videos,192 Playable: src.Playable, Listing: src.Listing, Listed: timeOrNil(src.Listed),193 Since: timeOrNil(src.Since), Failed: timeOrNil(src.Failed)}194 if cfg.Playlist != "" {195 s.describePlaylist(&j, src)196 } else {197 s.describeSource(&j, src)198 }199 c.Listing = c.Listing || j.Listing200 videos += src.Videos201 playable += src.Playable202 c.Sources = append(c.Sources, j)203 }204 c.Videos, c.Items = &videos, &playable205 c.Now = nowOn(y)206}207208// describeSource fills in one of the YouTube channels a channel plays:209// its page, and what a lookup found of it.210func (s *Server) describeSource(j *ytSourceJSON, src vchan.YouTubeSource) {211 page, err := youtubeChannel(src.Channel)212 if err != nil {213 page = strings.TrimSuffix(src.URL, "/videos")214 }215 j.URL = page216 if i := strings.LastIndex(page, "/@"); i >= 0 {217 j.Handle = page[i+1:]218 }219 if l, ok := s.lookedUp(page, src.ID); ok {220 if l.err == nil {221 j.Image, j.Handle = l.ch.Image, cmp(j.Handle, l.ch.Handle)222 j.Name = cmp(j.Name, l.ch.Name)223 } else if j.Failed != nil {224 j.Error = ytMessage(l.err)225 }226 }227}228229// describePlaylist fills in the playlist a channel plays: whose it is, its230// picture, and why it can't be listed, when a lookup found out.231func (s *Server) describePlaylist(j *ytSourceJSON, src vchan.YouTubeSource) {232 j.Owner, j.OwnerURL, j.Image = src.Owner, ownerURL(src.OwnerID), src.Image233 if l, ok := s.lookedUpPlaylist(src.URL); ok {234 if l.err == nil {235 j.Name, j.Owner, j.OwnerURL = cmp(j.Name, l.pl.Title), cmp(j.Owner, l.pl.Owner), cmp(j.OwnerURL, l.pl.OwnerURL)236 j.Image = cmp(j.Image, l.pl.Image)237 } else if j.Failed != nil {238 j.Error = ytMessage(l.err)239 }240 }241}242243func timeOrNil(t time.Time) *time.Time {244 if t.IsZero() {245 return nil246 }247 return &t248}249250// lookedUp finds a lookup of a channel, by its page or ID.251func (s *Server) lookedUp(page, id string) (lookedUp, bool) {252 s.mu.Lock()253 defer s.mu.Unlock()254 if l, ok := s.lookups[strings.ToLower(page)]; ok && l.fresh() {255 return l, true256 }257 if l, ok := s.lookups["id:"+id]; ok && id != "" && l.fresh() {258 return l, true259 }260 return lookedUp{}, false261}262263// lookedUpPlaylist finds a lookup of a playlist, by its page.264func (s *Server) lookedUpPlaylist(page string) (lookedUp, bool) {265 s.mu.Lock()266 defer s.mu.Unlock()267 l, ok := s.lookups["pl:"+page] // IDs differ by case268 return l, ok && l.fresh()269}270271// youtubeChannel checks a YouTube channel as the page gives it, a link,272// "@handle" or "UC…" ID, and returns its page: https://www.youtube.com/273// then @handle, channel/UC…, c/name or user/name.274func youtubeChannel(s string) (string, error) {275 s = strings.TrimSpace(s)276 switch {277 case handleRe.MatchString(s):278 return "https://www.youtube.com/" + s, nil279 case channelIDRe.MatchString(s):280 return "https://www.youtube.com/channel/" + s, nil281 }282 raw := s283 if !strings.Contains(s, "://") {284 raw = "https://" + s285 }286 u, err := url.Parse(raw)287 if err != nil || (u.Scheme != "http" && u.Scheme != "https") {288 return "", errNotChannel289 }290 host := strings.ToLower(u.Hostname())291 for _, p := range []string{"www.", "m."} {292 host = strings.TrimPrefix(host, p)293 }294 if host == "youtu.be" {295 return "", errVideoLink296 }297 if host != "youtube.com" {298 return "", errNotChannel299 }300 parts := strings.Split(strings.Trim(u.Path, "/"), "/")301 switch {302 case handleRe.MatchString(parts[0]):303 return "https://www.youtube.com/" + parts[0], nil304 case len(parts) > 1 && parts[0] == "channel" && channelIDRe.MatchString(parts[1]),305 len(parts) > 1 && (parts[0] == "c" || parts[0] == "user") && customRe.MatchString(parts[1]):306 return "https://www.youtube.com/" + parts[0] + "/" + parts[1], nil307 case parts[0] == "playlist":308 return "", errPlaylistLink309 case slices.Contains([]string{"watch", "shorts", "live", "embed"}, parts[0]):310 return "", errVideoLink311 }312 return "", errNotChannel313}314315// ytSettings checks a YouTube channel's settings from the page, over cur:316// the channel's settings now, or the defaults. A channel plays YouTube317// channels or a playlist: given alone, either takes the other's place.318func ytSettings(r *ytReq, cur ytConfigJSON) (ytConfigJSON, error) {319 if r == nil {320 r = &ytReq{}321 }322 c := cur323 c.Channels, c.Playlist = nil, ""324 if r.RepeatDays != nil {325 c.RepeatDays = *r.RepeatDays326 }327 if r.MaxHeight != nil {328 c.MaxHeight = *r.MaxHeight329 }330 if r.MinMinutes != nil {331 c.MinMinutes = *r.MinMinutes332 }333 if r.MaxMinutes != nil {334 c.MaxMinutes = *r.MaxMinutes335 }336 if r.RerunMix != nil {337 c.RerunMix = *r.RerunMix338 }339 if r.MaxAgeDays != nil {340 c.MaxAgeDays = *r.MaxAgeDays341 }342 if r.DeadAir != nil {343 c.DeadAir = *r.DeadAir344 }345 list, playlist := cur.Channels, cur.Playlist346 switch {347 case r.Channels != nil && r.Playlist != nil:348 list, playlist = r.Channels, *r.Playlist349 case r.Playlist != nil:350 if playlist = *r.Playlist; strings.TrimSpace(playlist) != "" {351 list = nil352 }353 case r.Channels != nil:354 if list = r.Channels; len(list) > 0 {355 playlist = ""356 }357 }358 if strings.TrimSpace(playlist) != "" {359 if len(list) > 0 {360 return c, errBoth361 }362 page, err := youtube.Playlist(playlist)363 if err != nil {364 return c, fmt.Errorf("%q: %w", strings.TrimSpace(playlist), err)365 }366 c.Playlist = page367 }368 seen := map[string]bool{}369 for _, ch := range list {370 page, err := youtubeChannel(ch)371 if err != nil {372 return c, fmt.Errorf("%q: %w", strings.TrimSpace(ch), err)373 }374 if k := strings.ToLower(page); !seen[k] {375 seen[k] = true376 c.Channels = append(c.Channels, page)377 }378 }379 switch {380 case len(c.Channels) == 0 && c.Playlist == "":381 return c, errors.New("add at least one YouTube channel, or a playlist")382 case len(c.Channels) > maxSources:383 return c, fmt.Errorf("a channel plays up to %d YouTube channels", maxSources)384 case c.RepeatDays < 0 || c.RepeatDays > 365:385 return c, errors.New("the repeat window is 0 to 365 days")386 case !slices.Contains(heights, c.MaxHeight):387 return c, errors.New("the quality is 360, 480, 720 or 1080")388 case c.MinMinutes < 0 || c.MinMinutes > 600:389 return c, errors.New("the shortest videos are 0 to 600 minutes long")390 case c.MaxMinutes < 0 || c.MaxMinutes > 1440 || c.MaxMinutes > 0 && c.MaxMinutes <= c.MinMinutes:391 return c, errors.New("the longest videos are longer than the shortest, up to 1440 minutes (0 for no limit)")392 case !slices.Contains(rerunMixes, c.RerunMix):393 return c, errors.New("reruns are recent, balanced or any")394 case c.MaxAgeDays < 0 || c.MaxAgeDays > 3650:395 return c, errors.New("the newest videos are from the last 1 to 3650 days (0 for any age)")396 }397 return c, nil398}399400// writeYtConfig updates a channel's youtube.json, keeping any settings the401// page doesn't edit.402func writeYtConfig(path string, c ytConfigJSON) error {403 m := map[string]any{}404 if raw, err := os.ReadFile(path); err == nil {405 if err := json.Unmarshal(raw, &m); err != nil {406 return fmt.Errorf("%s: %w", youtubeFile, err)407 }408 }409 if len(c.Channels) > 0 {410 m["channels"] = c.Channels411 } else {412 delete(m, "channels")413 }414 if c.Playlist != "" {415 m["playlist"] = c.Playlist416 } else {417 delete(m, "playlist")418 }419 m["repeatDays"] = c.RepeatDays420 m["maxHeight"] = c.MaxHeight421 m["minMinutes"] = c.MinMinutes422 m["maxMinutes"] = c.MaxMinutes423 m["rerunMix"] = c.RerunMix424 m["maxAgeDays"] = c.MaxAgeDays425 if c.DeadAir {426 m["deadAir"] = true427 } else {428 delete(m, "deadAir")429 }430 raw, err := json.MarshalIndent(m, "", " ")431 if err != nil {432 return err433 }434 return writeFile(path, append(raw, '\n'), 0o644)435}436437// ytdlp is the yt-dlp the library's YouTube channels use.438func (s *Server) ytdlp() *youtube.YtDlp {439 if s.Library.YtDlp != nil {440 return s.Library.YtDlp441 }442 return pathYtDlp443}444445// ytRun runs f with yt-dlp once it's ready and a run is free, and answers446// for it when it can't.447func (s *Server) ytRun(w http.ResponseWriter, r *http.Request, limit time.Duration, f func(context.Context, *youtube.YtDlp) error) bool {448 tool := s.ytdlp()449 if err := tool.Ready(); err != nil {450 writeErr(w, http.StatusServiceUnavailable, err)451 return false452 }453 ctx, cancel := context.WithTimeout(r.Context(), limit)454 defer cancel()455 select {456 case s.ytSlots <- struct{}{}:457 defer func() { <-s.ytSlots }()458 case <-ctx.Done():459 writeErr(w, http.StatusServiceUnavailable, errors.New("yt-dlp is busy; try again"))460 return false461 }462 if err := f(ctx, tool); err != nil {463 code := http.StatusBadGateway464 if ctx.Err() != nil {465 code, err = http.StatusGatewayTimeout, errors.New("YouTube took too long to answer; try again")466 }467 writeErr(w, code, errors.New(ytMessage(err)))468 return false469 }470 return true471}472473// ytMessage makes a yt-dlp error fit to show.474func ytMessage(err error) string {475 msg := err.Error()476 if strings.Contains(msg, "HTTP Error 404") {477 return "YouTube has no such channel"478 }479 return ytErrPrefix.ReplaceAllString(msg, "")480}481482// searchYouTube finds YouTube channels by name or topic.483func (s *Server) searchYouTube(w http.ResponseWriter, r *http.Request) {484 q := strings.Join(strings.Fields(r.URL.Query().Get("q")), " ")485 if q == "" || utf8.RuneCountInString(q) > 100 {486 writeErr(w, http.StatusBadRequest, errors.New("search for a channel or a topic, in up to 100 characters"))487 return488 }489 key := strings.ToLower(q)490 s.mu.Lock()491 if c, ok := s.searches[key]; ok && time.Since(c.at) < searchFor {492 s.mu.Unlock()493 writeJSON(w, map[string]any{"channels": c.found})494 return495 }496 s.mu.Unlock()497 var found []ytChannelJSON498 if !s.ytRun(w, r, searchTime, func(ctx context.Context, tool *youtube.YtDlp) error {499 list, err := tool.SearchChannels(ctx, q, searchCount)500 for _, c := range list {501 found = append(found, channelOut(c))502 }503 return err504 }) {505 return506 }507 if found == nil {508 found = []ytChannelJSON{}509 }510 s.mu.Lock()511 if s.searches == nil {512 s.searches = map[string]searched{}513 }514 for k, c := range s.searches {515 if time.Since(c.at) >= searchFor {516 delete(s.searches, k)517 }518 }519 s.searches[key] = searched{found: found, at: time.Now()}520 s.mu.Unlock()521 writeJSON(w, map[string]any{"channels": found})522}523524// lookupYouTube finds one YouTube channel from a link, @handle or ID,525// with roughly how many videos it has.526func (s *Server) lookupYouTube(w http.ResponseWriter, r *http.Request) {527 page, err := youtubeChannel(r.URL.Query().Get("u"))528 if err != nil {529 writeErr(w, http.StatusBadRequest, err)530 return531 }532 if l, ok := s.lookedUp(page, ""); ok {533 if l.err != nil {534 writeErr(w, http.StatusBadGateway, errors.New(ytMessage(l.err)))535 } else {536 writeJSON(w, l.ch)537 }538 return539 }540 var found youtube.Channel541 var failed error542 ok := s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *youtube.YtDlp) error {543 found, failed = tool.LookupChannel(ctx, page)544 return failed545 })546 if !ok && (failed == nil || errors.Is(failed, context.DeadlineExceeded) || errors.Is(failed, context.Canceled)) {547 return // yt-dlp didn't run, or ran out of time: nothing to remember548 }549 l := lookedUp{ch: channelOut(found), err: failed, at: time.Now()}550 s.remember(page, l)551 if ok {552 writeJSON(w, l.ch)553 }554}555556// remember keeps a lookup of the channel at page, by its page, and when557// it was found, by its own page and ID too.558func (s *Server) remember(page string, l lookedUp) {559 keys := []string{strings.ToLower(page)}560 if l.err == nil {561 keys = append(keys, strings.ToLower(l.ch.URL), "id:"+l.ch.ID)562 }563 s.keep(l, keys...)564}565566// keep holds a lookup by keys, letting go of those gone stale.567func (s *Server) keep(l lookedUp, keys ...string) {568 s.mu.Lock()569 defer s.mu.Unlock()570 if s.lookups == nil {571 s.lookups = map[string]lookedUp{}572 }573 for k, c := range s.lookups {574 if !c.fresh() {575 delete(s.lookups, k)576 }577 }578 for _, k := range keys {579 s.lookups[k] = l580 }581}582583// lookupPlaylist finds a YouTube playlist from its link or ID: its title,584// whose it is, and how many videos it has.585func (s *Server) lookupPlaylist(w http.ResponseWriter, r *http.Request) {586 page, err := youtube.Playlist(r.URL.Query().Get("u"))587 if err != nil {588 writeErr(w, http.StatusBadRequest, err)589 return590 }591 if l, ok := s.lookedUpPlaylist(page); ok {592 if l.err != nil {593 writeErr(w, http.StatusBadGateway, errors.New(ytMessage(l.err)))594 } else {595 writeJSON(w, l.pl)596 }597 return598 }599 var found youtube.PlaylistInfo600 var failed error601 ok := s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *youtube.YtDlp) error {602 found, failed = tool.LookupPlaylist(ctx, page)603 return failed604 })605 if !ok && (failed == nil || errors.Is(failed, context.DeadlineExceeded) || errors.Is(failed, context.Canceled)) {606 return // yt-dlp didn't run, or ran out of time: nothing to remember607 }608 l := lookedUp{pl: playlistOut(found), err: failed, at: time.Now()}609 s.keep(l, "pl:"+page)610 if ok {611 writeJSON(w, l.pl)612 }613}