internal/youtube has yt-dlp (kept up to date), channel and playlist listings, feeds, search and lookups, and a video's details with where to stream it from. The YouTube channels use it as before; the admin page and airwavesd name it directly. Nothing changes in what they do.
- commit
- 6af73667de0900ad072a29d7039e5074b2dd8bc2
- parent
- d0c9c3523d980295d0f4d7e4091cf3740997dd8f
- author
- FelineStateMachine <226808055+FelineStateMachine@users.noreply.github.com>
- kind
- refactor risk score 1.9 of 3, Jev's estimate
| M | README.md | 1 | + |
| M | cmd/ | 4 | +- |
| M | internal/ | 7 | +- |
| M | internal/ | 3 | +- |
| M | internal/ | 6 | +- |
| M | internal/ | 3 | +- |
| M | internal/ | 25 | +- |
| M | internal/ | 3 | +- |
| M | internal/ | 31 | +- |
| M | internal/ | 3 | +- |
| M | internal/ | 176 | ++----------- |
| M | internal/ | 32 | +- |
| M | internal/ | 59 | +--- |
| M | internal/ | 17 | +- |
| M | internal/ | 257 | ++---------------- |
| D | internal/ | 201 | -------------- |
| M | internal/ | 6 | +- |
| M | internal/ | 157 | +---------- |
| M | internal/ | 26 | +- |
| M | internal/ | 9 | +- |
| M | internal/ | 8 | +- |
| A | internal/ | 421 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 325 | +++++++++++++++++++++++ |
| R | internal/ | 95 | ++++++- |
@@ -671,6 +671,7 @@ internal/stream ffmpeg to HLS, one session per client
671671 internal/hdhr the HDHomeRun's HTTP API and discovery (hdhrtest fakes it); the emulated one672672 internal/vchan custom channels: Airwaves Weather, folder, Jellyfin and YouTube channels673673 internal/jellyfin Jellyfin client: sign-in, Quick Connect, browsing, stream URLs674+internal/youtube YouTube as a source: yt-dlp kept up to date, listings, feeds, search, where to stream from674675 deploy/, scripts/ Docker files for the server and the deploy script675676 deploy/standalone Docker Compose for custom channels only, built from source (docs/self-hosting.md)676677 frontend/dist the UI: plain HTML, CSS and JS, no build step; also served at /tv/@@ -28,8 +28,8 @@ import (
2828 "airwaves/internal/api"2929 "airwaves/internal/hdhr"3030 "airwaves/internal/service"31- "airwaves/internal/vchan"3231 "airwaves/internal/web"32+ "airwaves/internal/youtube"3333 )3434 3535 func main() {@@ -109,7 +109,7 @@ func run() error {
109109 110110 // YouTube changes faster than images are rebuilt, so by default111111 // airwavesd keeps the latest yt-dlp release itself.112- yt := &vchan.YtDlp{Path: *ytdlp, HTTP: web.NewClient()}112+ yt := &youtube.YtDlp{Path: *ytdlp, HTTP: web.NewClient()}113113 if yt.Path == "" {114114 yt.Path, yt.Update = filepath.Join(*data, "tools", "yt-dlp"), true115115 }@@ -20,6 +20,7 @@ import (
2020 2121 "airwaves/internal/jellyfin/jellyfintest"2222 "airwaves/internal/vchan"23+ "airwaves/internal/youtube"2324 )2425 2526 // TestMain lets the test binary stand in for yt-dlp: run with@@ -159,7 +160,7 @@ func newHarness(t *testing.T, opts ...func(*Server)) *harness {
159160 ytlog := filepath.Join(t.TempDir(), "yt-dlp.log")160161 t.Setenv("AIRWAVES_FAKE_YTDLP", ytlog)161162 lib := &vchan.Library{Root: root, FFmpeg: "ffmpeg", FFprobe: "ffprobe", Reserved: func() []string { return []string{"1.1"} },162- YtDlp: &vchan.YtDlp{Path: exe, HTTP: &http.Client{Transport: noNetwork{}}}}163+ YtDlp: &youtube.YtDlp{Path: exe, HTTP: &http.Client{Transport: noNetwork{}}}}163164 a := &Server{Library: lib, HTTP: jf.Client()}164165 for _, o := range opts {165166 o(a)@@ -657,7 +658,7 @@ func TestAdminYouTubeSearchAndLookup(t *testing.T) {
657658 }658659 659660 func TestAdminYouTubeWithoutYtDlp(t *testing.T) {660- lib := &vchan.Library{Root: t.TempDir(), YtDlp: &vchan.YtDlp{Path: filepath.Join(t.TempDir(), "yt-dlp")}}661+ lib := &vchan.Library{Root: t.TempDir(), YtDlp: &youtube.YtDlp{Path: filepath.Join(t.TempDir(), "yt-dlp")}}661662 srv := httptest.NewServer((&Server{Library: lib}).Handler())662663 defer srv.Close()663664 resp, err := http.Get(srv.URL + "/admin/api/youtube/search?q=dj+sets")@@ -678,7 +679,7 @@ func TestAdminYouTubeLive(t *testing.T) {
678679 if tool == "" {679680 t.Skip("set AIRWAVES_YTDLP to a yt-dlp to search YouTube")680681 }681- lib := &vchan.Library{Root: t.TempDir(), YtDlp: &vchan.YtDlp{Path: tool, Update: os.Getenv("AIRWAVES_YTDLP_UPDATE") != ""}}682+ lib := &vchan.Library{Root: t.TempDir(), YtDlp: &youtube.YtDlp{Path: tool, Update: os.Getenv("AIRWAVES_YTDLP_UPDATE") != ""}}682683 srv := httptest.NewServer((&Server{Library: lib}).Handler())683684 defer srv.Close()684685 get := func(path string) map[string]any {@@ -23,6 +23,7 @@ import (
2323 "time"2424 2525 "airwaves/internal/vchan"26+ "airwaves/internal/youtube"2627 )2728 2829 const (@@ -327,7 +328,7 @@ func (s *Server) youtubeAvatar(w http.ResponseWriter, r *http.Request, dir strin
327328 }328329 l, ok := s.lookedUp(page, "")329330 if !ok {330- if !s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *vchan.YtDlp) error {331+ if !s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *youtube.YtDlp) error {331332 found, err := tool.LookupChannel(ctx, page)332333 l = lookedUp{ch: channelOut(found), err: err, at: time.Now()}333334 return err@@ -17,9 +17,9 @@ import (
1717 "sync"1818 "testing"1919 20- "github.com/modelcontextprotocol/go-sdk/mcp"20+ "airwaves/internal/youtube"2121 22- "airwaves/internal/vchan"22+ "github.com/modelcontextprotocol/go-sdk/mcp"2323 )2424 2525 // agent connects to the harness's agent endpoint as an MCP client would.@@ -405,7 +405,7 @@ func TestAgentDemoServer(t *testing.T) {
405405 }406406 h := newHarness(t)407407 if tool := os.Getenv("AIRWAVES_YTDLP"); tool != "" {408- h.lib.YtDlp = &vchan.YtDlp{Path: tool, HTTP: http.DefaultClient}408+ h.lib.YtDlp = &youtube.YtDlp{Path: tool, HTTP: http.DefaultClient}409409 }410410 h.withWeather()411411 h.admin.Antenna = func(context.Context) map[string]string {@@ -43,6 +43,7 @@ import (
4343 "airwaves/internal/guide"4444 "airwaves/internal/jellyfin"4545 "airwaves/internal/vchan"46+ "airwaves/internal/youtube"4647 )4748 4849 // weatherKey is the weather channel among a backup's channels.@@ -181,7 +182,7 @@ func sourceOf(files map[string][]byte) string {
181182 return ""182183 }183184 if c.Playlist != "" {184- page, err := vchan.YouTubePlaylist(c.Playlist)185+ page, err := youtube.Playlist(c.Playlist)185186 if err != nil {186187 page = strings.TrimSpace(c.Playlist)187188 }@@ -20,6 +20,7 @@ import (
2020 "unicode/utf8"2121 2222 "airwaves/internal/vchan"23+ "airwaves/internal/youtube"2324 )2425 2526 const (@@ -47,7 +48,7 @@ var (
4748 // ytErrPrefix is how yt-dlp starts its errors: "[youtube:tab] @name: ".4849 ytErrPrefix = regexp.MustCompile(`^yt-dlp: \[[^\]]+\] (\S+: )?`)4950 // pathYtDlp runs yt-dlp from PATH when the library has none of its own.50- pathYtDlp = &vchan.YtDlp{}51+ pathYtDlp = &youtube.YtDlp{}5152 )5253 5354 var (@@ -116,7 +117,7 @@ type ytChannelJSON struct {
116117 Videos int `json:"videos,omitempty"` // roughly, from a lookup117118 }118119 119-func channelOut(c vchan.YouTubeChannel) ytChannelJSON {120+func channelOut(c youtube.Channel) ytChannelJSON {120121 return ytChannelJSON{ID: c.ID, URL: c.URL, Handle: c.Handle, Name: c.Name, Subscribers: c.Subscribers,121122 Description: c.Description, Image: c.Image, Videos: c.Videos}122123 }@@ -137,7 +138,7 @@ type ytPlaylistJSON struct {
137138 Image string `json:"image,omitempty"`138139 }139140 140-func playlistOut(p vchan.YouTubePlaylistInfo) ytPlaylistJSON {141+func playlistOut(p youtube.PlaylistInfo) ytPlaylistJSON {141142 return ytPlaylistJSON{ID: p.ID, URL: p.URL, Title: p.Title, Owner: p.Owner, OwnerURL: ownerURL(p.OwnerID), Videos: p.Videos, Image: p.Image}142143 }143144 @@ -358,7 +359,7 @@ func ytSettings(r *ytReq, cur ytConfigJSON) (ytConfigJSON, error) {
358359 if len(list) > 0 {359360 return c, errBoth360361 }361- page, err := vchan.YouTubePlaylist(playlist)362+ page, err := youtube.Playlist(playlist)362363 if err != nil {363364 return c, fmt.Errorf("%q: %w", strings.TrimSpace(playlist), err)364365 }@@ -434,7 +435,7 @@ func writeYtConfig(path string, c ytConfigJSON) error {
434435 }435436 436437 // ytdlp is the yt-dlp the library's YouTube channels use.437-func (s *Server) ytdlp() *vchan.YtDlp {438+func (s *Server) ytdlp() *youtube.YtDlp {438439 if s.Library.YtDlp != nil {439440 return s.Library.YtDlp440441 }@@ -443,7 +444,7 @@ func (s *Server) ytdlp() *vchan.YtDlp {
443444 444445 // ytRun runs f with yt-dlp once it's ready and a run is free, and answers445446 // for it when it can't.446-func (s *Server) ytRun(w http.ResponseWriter, r *http.Request, limit time.Duration, f func(context.Context, *vchan.YtDlp) error) bool {447+func (s *Server) ytRun(w http.ResponseWriter, r *http.Request, limit time.Duration, f func(context.Context, *youtube.YtDlp) error) bool {447448 tool := s.ytdlp()448449 if err := tool.Ready(); err != nil {449450 writeErr(w, http.StatusServiceUnavailable, err)@@ -494,7 +495,7 @@ func (s *Server) searchYouTube(w http.ResponseWriter, r *http.Request) {
494495 }495496 s.mu.Unlock()496497 var found []ytChannelJSON497- if !s.ytRun(w, r, searchTime, func(ctx context.Context, tool *vchan.YtDlp) error {498+ if !s.ytRun(w, r, searchTime, func(ctx context.Context, tool *youtube.YtDlp) error {498499 list, err := tool.SearchChannels(ctx, q, searchCount)499500 for _, c := range list {500501 found = append(found, channelOut(c))@@ -536,9 +537,9 @@ func (s *Server) lookupYouTube(w http.ResponseWriter, r *http.Request) {
536537 }537538 return538539 }539- var found vchan.YouTubeChannel540+ var found youtube.Channel540541 var failed error541- ok := s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *vchan.YtDlp) error {542+ ok := s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *youtube.YtDlp) error {542543 found, failed = tool.LookupChannel(ctx, page)543544 return failed544545 })@@ -582,7 +583,7 @@ func (s *Server) keep(l lookedUp, keys ...string) {
582583 // lookupPlaylist finds a YouTube playlist from its link or ID: its title,583584 // whose it is, and how many videos it has.584585 func (s *Server) lookupPlaylist(w http.ResponseWriter, r *http.Request) {585- page, err := vchan.YouTubePlaylist(r.URL.Query().Get("u"))586+ page, err := youtube.Playlist(r.URL.Query().Get("u"))586587 if err != nil {587588 writeErr(w, http.StatusBadRequest, err)588589 return@@ -595,9 +596,9 @@ func (s *Server) lookupPlaylist(w http.ResponseWriter, r *http.Request) {
595596 }596597 return597598 }598- var found vchan.YouTubePlaylistInfo599+ var found youtube.PlaylistInfo599600 var failed error600- ok := s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *vchan.YtDlp) error {601+ ok := s.ytRun(w, r, lookupTime, func(ctx context.Context, tool *youtube.YtDlp) error {601602 found, failed = tool.LookupPlaylist(ctx, page)602603 return failed603604 })@@ -41,6 +41,7 @@ import (
4141 "airwaves/internal/vchan"4242 "airwaves/internal/weather"4343 "airwaves/internal/web"44+ "airwaves/internal/youtube"4445 )4546 4647 // Version is reported to clients.@@ -178,7 +179,7 @@ type Options struct {
178179 // ChannelsDir holds folder channels, a subfolder of videos each.179180 ChannelsDir string180181 // YtDlp runs yt-dlp for YouTube channels; yt-dlp from PATH when nil.181- YtDlp *vchan.YtDlp182+ YtDlp *youtube.YtDlp182183 // Loudness is the level custom channels' sound is evened out to as183184 // it streams, in LUFS (vchan.DefaultLoudness is antenna TV's); 0184185 // leaves it as it is.@@ -24,6 +24,7 @@ import (
2424 "airwaves/internal/phase"2525 "airwaves/internal/stream"2626 "airwaves/internal/tuner"27+ "airwaves/internal/youtube"2728 )2829 2930 func srtCaptions(srt string) *captions {@@ -402,50 +403,50 @@ func TestYouTubeCaptions(t *testing.T) {
402403 }403404 }))404405 defer srv.Close()405- info := ytInfo{406- Subtitles: map[string][]ytSubtitle{406+ info := youtube.Info{407+ Subtitles: map[string][]youtube.Subtitle{407408 "de": {{Ext: "vtt", URL: srv.URL + "/de.vtt"}},408409 "en-GB": {{Ext: "srv3", URL: srv.URL + "/x"}, {Ext: "vtt", URL: srv.URL + "/manual.vtt"}, {Ext: "json3", URL: srv.URL + "/manual.json3"}},409410 },410- AutoCaptions: map[string][]ytSubtitle{"en": {{Ext: "vtt", URL: srv.URL + "/auto.m3u8", Protocol: "m3u8_native"}}},411+ AutoCaptions: map[string][]youtube.Subtitle{"en": {{Ext: "vtt", URL: srv.URL + "/auto.m3u8", Protocol: "m3u8_native"}}},411412 }412- sub, ok := info.captions()413+ sub, ok := info.Captions()413414 if !ok || sub.URL != srv.URL+"/manual.json3" {414415 t.Fatalf("picked %+v", sub)415416 }416- raw, err := ytCaptions(context.Background(), srv.Client(), sub)417+ raw, err := youtube.FetchCaptions(context.Background(), srv.Client(), sub)417418 if cues, _ := cc.Parse(raw); err != nil || len(cues) != 1 || cues[0].Text != "By hand" {418419 t.Errorf("manual captions %q, %v", raw, err)419420 }420421 info.Subtitles = nil421- sub, ok = info.captions()422+ sub, ok = info.Captions()422423 if !ok || sub.Protocol != "m3u8_native" {423424 t.Fatalf("picked %+v", sub)424425 }425- raw, err = ytCaptions(context.Background(), srv.Client(), sub)426+ raw, err = youtube.FetchCaptions(context.Background(), srv.Client(), sub)426427 cues, _ := cc.Parse(raw)427428 if err != nil || len(cues) != 2 || cues[0].Text != "Piece 1" || cues[1].Start != 20*time.Minute {428429 t.Errorf("automatic captions %q, %v", cues, err)429430 }430431 // Speech recognition's, as json3, time each word.431- info.AutoCaptions["en-orig"] = []ytSubtitle{{Ext: "vtt", URL: srv.URL + "/x"}, {Ext: "json3", URL: srv.URL + "/asr.json3"}}432- sub, _ = info.captions()433- raw, err = ytCaptions(context.Background(), srv.Client(), sub)432+ info.AutoCaptions["en-orig"] = []youtube.Subtitle{{Ext: "vtt", URL: srv.URL + "/x"}, {Ext: "json3", URL: srv.URL + "/asr.json3"}}433+ sub, _ = info.Captions()434+ raw, err = youtube.FetchCaptions(context.Background(), srv.Client(), sub)434435 cues, _ = cc.Parse(raw)435436 if err != nil || len(cues) != 1 || cues[0].Text != "all right" || len(cues[0].Words) != 2 || cues[0].Words[1].Start != 1400*time.Millisecond {436437 t.Errorf("speech recognition's captions %+v, %v", cues, err)437438 }438- if _, ok := (ytInfo{AutoCaptions: map[string][]ytSubtitle{"fr": {{Ext: "vtt", URL: "x"}}}}).captions(); ok {439+ if _, ok := (youtube.Info{AutoCaptions: map[string][]youtube.Subtitle{"fr": {{Ext: "vtt", URL: "x"}}}}).Captions(); ok {439440 t.Error("picked French captions")440441 }441- if _, err := ytCaptions(context.Background(), srv.Client(), ytSubtitle{URL: srv.URL + "/gone?signature=secret"}); err == nil || strings.Contains(err.Error(), "secret") {442+ if _, err := youtube.FetchCaptions(context.Background(), srv.Client(), youtube.Subtitle{URL: srv.URL + "/gone?signature=secret"}); err == nil || strings.Contains(err.Error(), "secret") {442443 t.Errorf("error %v", err)443444 }444- dubbed := ytInfo{Language: "ja", RequestedFormats: []ytFormat{{URL: "v", VCodec: "avc1", ACodec: "none"}, {URL: "a", VCodec: "none", ACodec: "mp4a", Language: "en-US"}}}445- if l := dubbed.audioLang(); l != "en-US" {445+ dubbed := youtube.Info{Language: "ja", RequestedFormats: []youtube.Format{{URL: "v", VCodec: "avc1", ACodec: "none"}, {URL: "a", VCodec: "none", ACodec: "mp4a", Language: "en-US"}}}446+ if l := dubbed.AudioLang(); l != "en-US" {446447 t.Errorf("dubbed sound %q", l)447448 }448- if l := (ytInfo{Language: "ja"}).audioLang(); l != "ja" {449+ if l := (youtube.Info{Language: "ja"}).AudioLang(); l != "ja" {449450 t.Errorf("sound %q", l)450451 }451452 }@@ -12,6 +12,7 @@ import (
1212 "time"1313 1414 "airwaves/internal/guide"15+ "airwaves/internal/youtube"1516 )1617 1718 // Library finds folder channels: each subfolder of Root is a channel named@@ -30,7 +31,7 @@ type Library struct {
3031 // when they change.3132 Reserved func() []string3233 // YtDlp runs yt-dlp for YouTube channels; yt-dlp from PATH when nil.33- YtDlp *YtDlp34+ YtDlp *youtube.YtDlp3435 // Loudness is the level the channels' sound is evened out to, in LUFS3536 // (see DefaultLoudness); 0 leaves it as it is.3637 Loudness float64@@ -8,19 +8,16 @@ import (
88 "hash/fnv"99 "io"1010 "log"11- "net/http"12- "net/url"1311 "os"14- "os/exec"1512 "path/filepath"1613 "slices"17- "sort"1814 "strconv"1915 "strings"2016 "sync"2117 "time"2218 2319 "airwaves/internal/web"20+ "airwaves/internal/youtube"2421 )2522 2623 // YouTube is a channel of uploads from YouTube channels, set up by a@@ -35,7 +32,7 @@ type YouTube struct {
3532 Title string3633 Dir string3734 FFmpeg string38- YtDlp *YtDlp // yt-dlp from PATH when nil35+ YtDlp *youtube.YtDlp // yt-dlp from PATH when nil3936 // Loudness is the level the videos' sound is evened out to, in LUFS;4037 // 0 leaves it as it is.4138 Loudness float64@@ -141,7 +138,7 @@ func (y *YouTube) Programs(from, to time.Time) []Program {
141138 break142139 }143140 p := Program{Start: a.Start.Local(), End: a.End.Local(), Title: a.Title, Subtitle: y.names[a.Channel],144- Image: ytThumbnail(a.ID), New: a.First}141+ Image: youtube.Thumbnail(a.ID), New: a.First}145142 if v := y.videos[a.ID]; v != nil {146143 p.Description = v.Description147144 p.Subtitle, p.Date = ytDated(p.Subtitle, v, y.clock())@@ -209,14 +206,14 @@ func (y *YouTube) clock() time.Time {
209206 return time.Now()210207 }211208 212-func (y *YouTube) tool() *YtDlp {209+func (y *YouTube) tool() *youtube.YtDlp {213210 if y.YtDlp != nil {214211 return y.YtDlp215212 }216213 return defaultYtDlp217214 }218215 219-var defaultYtDlp = &YtDlp{}216+var defaultYtDlp = &youtube.YtDlp{}220217 221218 // tend loads the channel's files on first use, keeps the schedule two days222219 // ahead, and starts a refresh of the catalog when one is due. y.mu is@@ -283,11 +280,11 @@ func (y *YouTube) readConfig(cat ytCatalog) {
283280 }284281 var sources []*ytSource285282 if y.cfg.Playlist != "" {286- url, _ := YouTubePlaylist(y.cfg.Playlist) // as readYtConfig checked283+ url, _ := youtube.Playlist(y.cfg.Playlist) // as readYtConfig checked287284 sources = append(sources, cat.source(url))288285 }289286 for _, c := range y.cfg.Channels {290- url := ytVideosURL(c)287+ url := youtube.VideosURL(c)291288 if !slices.ContainsFunc(sources, func(s *ytSource) bool { return s.URL == url }) {292289 sources = append(sources, cat.source(url))293290 }@@ -506,9 +503,9 @@ func (y *YouTube) refresh() {
506503 if j.list && playlist {507504 y.listPlaylist(ctx, j.url)508505 } else if j.list {509- tool.jobs.Lock()510- raw, err := tool.run(ctx, slices.Concat(ytListingArgs, []string{j.url})...)511- tool.jobs.Unlock()506+ tool.Jobs.Lock()507+ raw, err := tool.Run(ctx, slices.Concat(youtube.ListingArgs, []string{j.url})...)508+ tool.Jobs.Unlock()512509 var l ytListing513510 if err == nil {514511 l, err = parseYtListing(raw)@@ -542,12 +539,12 @@ func (y *YouTube) refresh() {
542539 case <-time.After(ytLookupGap):543540 }544541 }545- tool.jobs.Lock()546- raw, err := tool.run(ctx, "-J", "--skip-download", "--no-playlist", "--ignore-no-formats-error", ytWatchURL(id))547- tool.jobs.Unlock()548- var info ytInfo542+ tool.Jobs.Lock()543+ raw, err := tool.Run(ctx, "-J", "--skip-download", "--no-playlist", "--ignore-no-formats-error", youtube.WatchURL(id))544+ tool.Jobs.Unlock()545+ var info youtube.Info549546 if err == nil {550- info, err = parseYtInfo(raw)547+ info, err = youtube.ParseInfo(raw)551548 }552549 y.mu.Lock()553550 y.learn(id, info, err)@@ -567,17 +564,17 @@ func (y *YouTube) refresh() {
567564 // listed in full. Only changes between the two are logged.568565 func (y *YouTube) checkFeed(ctx context.Context, url, id string, listed bool) {569566 tool := y.tool()570- raw, err := web.Get(ctx, tool.client(), ytFeedURL(id))571- var entries []ytFeedEntry567+ raw, err := web.Get(ctx, tool.Client(), youtube.FeedURL(id))568+ var entries []youtube.FeedEntry572569 if err == nil {573- entries, err = parseYtFeed(raw)570+ entries, err = youtube.ParseFeed(raw)574571 }575572 var newest ytListing576573 ferr := err577574 if err != nil && !listed {578- tool.jobs.Lock()579- raw, ferr = tool.run(ctx, slices.Concat(ytListingArgs, []string{"--playlist-end", strconv.Itoa(ytNewAtTop), url})...)580- tool.jobs.Unlock()575+ tool.Jobs.Lock()576+ raw, ferr = tool.Run(ctx, slices.Concat(youtube.ListingArgs, []string{"--playlist-end", strconv.Itoa(ytNewAtTop), url})...)577+ tool.Jobs.Unlock()581578 if ferr == nil {582579 newest, ferr = parseYtListing(raw)583580 }@@ -664,7 +661,7 @@ func (y *YouTube) lookupsDue(now time.Time) []string {
664661 }665662 666663 // learn takes in a lookup of one video. y.mu is held.667-func (y *YouTube) learn(id string, info ytInfo, err error) {664+func (y *YouTube) learn(id string, info youtube.Info, err error) {668665 y.index()669666 v := y.videos[id]670667 if v == nil {@@ -674,7 +671,7 @@ func (y *YouTube) learn(id string, info ytInfo, err error) {
674671 switch {675672 case err == nil:676673 y.learnVideo(v, info)677- case gone(err):674+ case youtube.Gone(err):678675 v.Skip, v.New, v.Looked = "gone", false, now679676 log.Printf("%s: %s can't play: %v", y.label(), id, err)680677 default:@@ -778,7 +775,7 @@ func (y *YouTube) open(ctx context.Context, a ytAiring, offset time.Duration, fr
778775 info := r.info779776 y.mu.Lock()780777 if err != nil {781- if v := y.videos[a.ID]; v != nil && gone(err) {778+ if v := y.videos[a.ID]; v != nil && youtube.Gone(err) {782779 v.Skip, v.New, v.Looked = "gone", false, y.clock()783780 y.changed() // and so off the unlocked schedule784781 }@@ -794,129 +791,10 @@ func (y *YouTube) open(ctx context.Context, a ytAiring, offset time.Duration, fr
794791 if next.ID != "" {795792 go y.prefetch(ctx, next)796793 }797- in := info.inputs()798- o := opened{AudioLang: info.audioLang(), Captions: r.caps}799- o.Args, o.Audio = ytInputArgs(in, offset, httpChunks(y.FFmpeg))794+ in := info.Inputs()795+ o := opened{AudioLang: info.AudioLang(), Captions: r.caps}796+ o.Args, o.Audio = youtube.InputArgs(in, offset, youtube.HTTPChunks(y.FFmpeg))800797 log.Printf("%s: on air %s %s in (format %s, %dp, found in %s)", y.label(), a.ID, offset.Round(time.Second),801798 info.FormatID, in[0].Height, time.Since(began).Round(100*time.Millisecond))802799 return o, nil803800 }804-805-// ytCaptions fetches a caption track: a json3, WebVTT or SRT file, or a806-// playlist of WebVTT pieces, joined.807-func ytCaptions(ctx context.Context, client *http.Client, sub ytSubtitle) ([]byte, error) {808- raw, err := ytGet(ctx, client, sub.URL)809- if err != nil || !strings.HasPrefix(sub.Protocol, "m3u8") {810- return raw, err811- }812- base, err := url.Parse(sub.URL)813- if err != nil {814- return nil, errors.New("captions: bad address")815- }816- var all []byte817- pieces := 0818- for line := range strings.Lines(string(raw)) {819- line = strings.TrimSpace(line)820- if line == "" || strings.HasPrefix(line, "#") {821- continue822- }823- if pieces++; pieces > 500 {824- break825- }826- u, err := base.Parse(line)827- if err != nil {828- return nil, errors.New("captions: bad address")829- }830- piece, err := ytGet(ctx, client, u.String())831- if err != nil {832- return nil, err833- }834- all = append(append(all, piece...), "\n\n"...)835- }836- return all, nil837-}838-839-// ytGet fetches captions, leaving their address, which is signed, out of840-// errors.841-func ytGet(ctx context.Context, client *http.Client, address string) ([]byte, error) {842- req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil)843- if err != nil {844- return nil, errors.New("captions: bad address")845- }846- resp, err := client.Do(req)847- if uerr := (*url.Error)(nil); errors.As(err, &uerr) {848- return nil, fmt.Errorf("captions: %w", uerr.Err)849- }850- if err != nil {851- return nil, fmt.Errorf("captions: %w", err)852- }853- defer resp.Body.Close()854- if resp.StatusCode != http.StatusOK {855- return nil, fmt.Errorf("captions: HTTP %d", resp.StatusCode)856- }857- return io.ReadAll(io.LimitReader(resp.Body, 16<<20))858-}859-860-// ytFormatSpec picks H.264 picture up to height and AAC sound, English861-// sound of a video dubbed in several languages, else the best single file862-// up to height.863-func ytFormatSpec(height int) string {864- return fmt.Sprintf("bv*[height<=%[1]d][vcodec^=avc1]+ba[ext=m4a][language^=en]/bv*[height<=%[1]d][vcodec^=avc1]+ba[ext=m4a]/b[height<=%[1]d]", height)865-}866-867-// ytInputArgs are ffmpeg's inputs for a resolved video, played from868-// offset in: picture first, with yt-dlp's headers, reconnecting869-// when a connection drops. A stream that fails for good stops ffmpeg870-// (-xerror) rather than ending the video early, so it can be reopened.871-// chunked fetches in 10 MB ranges as yt-dlp does, which YouTube serves872-// without throttling.873-func ytInputArgs(fs []ytFormat, offset time.Duration, chunked bool) (args []string, audio string) {874- args = []string{"-xerror"}875- for i, f := range fs {876- args = append(args, "-reconnect", "1", "-reconnect_streamed", "1", "-reconnect_on_network_error", "1",877- "-reconnect_delay_max", "4", "-rw_timeout", "15000000")878- if chunked {879- args = append(args, "-request_size", "10485760", "-multiple_requests", "1")880- }881- if len(f.HTTPHeaders) > 0 {882- var h strings.Builder883- for _, k := range sortedKeys(f.HTTPHeaders) {884- h.WriteString(k + ": " + f.HTTPHeaders[k] + "\r\n")885- }886- args = append(args, "-headers", h.String())887- }888- if offset > 0 {889- args = append(args, "-ss", strconv.FormatFloat(offset.Seconds(), 'f', 3, 64))890- }891- args = append(args, "-i", f.URL)892- if audio == "" && f.ACodec != "none" && f.ACodec != "" {893- audio = fmt.Sprintf("%d:a:0", i)894- }895- }896- return args, audio897-}898-899-func sortedKeys(m map[string]string) []string {900- keys := make([]string, 0, len(m))901- for k := range m {902- keys = append(keys, k)903- }904- sort.Strings(keys)905- return keys906-}907-908-var chunks sync.Map // ffmpeg path: bool909-910-// httpChunks reports whether ffmpeg can fetch HTTP in ranges911-// (-request_size, FFmpeg 8).912-func httpChunks(ffmpeg string) bool {913- if ok, found := chunks.Load(ffmpeg); found {914- return ok.(bool)915- }916- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)917- defer cancel()918- out, _ := exec.CommandContext(ctx, ffmpeg, "-hide_banner", "-h", "protocol=http").Output()919- ok := strings.Contains(string(out), "-request_size")920- chunks.Store(ffmpeg, ok)921- return ok922-}@@ -15,6 +15,8 @@ import (
1515 "strings"1616 "testing"1717 "time"18+19+ "airwaves/internal/youtube"1820 )1921 2022 func readTestdata(t *testing.T, name string) []byte {@@ -55,7 +57,7 @@ func TestYtCatalogFromListingAndFeed(t *testing.T) {
5557 }5658 5759 now := time.Date(2026, 10, 3, 15, 0, 0, 0, time.UTC)58- s := &ytSource{URL: ytVideosURL("@TheLibraryofLetourneau")}60+ s := &ytSource{URL: youtube.VideosURL("@TheLibraryofLetourneau")}5961 if added := s.mergeListing(l, now); added != 0 {6062 t.Errorf("the first listing found %d new videos; it should take them all as the back catalog", added)6163 }@@ -65,7 +67,7 @@ func TestYtCatalogFromListingAndFeed(t *testing.T) {
6567 }6668 }6769 68- entries, err := parseYtFeed(readTestdata(t, "yt_feed.xml"))70+ entries, err := youtube.ParseFeed(readTestdata(t, "yt_feed.xml"))6971 if err != nil {7072 t.Fatal(err)7173 }@@ -96,7 +98,7 @@ func TestYtCatalogFromListingAndFeed(t *testing.T) {
9698 }9799 98100 // Its lookup gives the length; it's playable and still due a first run.99- info, err := parseYtInfo(readTestdata(t, "yt_video.json"))101+ info, err := youtube.ParseInfo(readTestdata(t, "yt_video.json"))100102 if err != nil {101103 t.Fatal(err)102104 }@@ -118,15 +120,15 @@ func TestYtCatalogFromListingAndFeed(t *testing.T) {
118120 }119121 120122 func TestYtInfoInputs(t *testing.T) {121- info, err := parseYtInfo(readTestdata(t, "yt_video.json"))123+ info, err := youtube.ParseInfo(readTestdata(t, "yt_video.json"))122124 if err != nil {123125 t.Fatal(err)124126 }125- in := info.inputs()127+ in := info.Inputs()126128 if len(in) != 2 || in[0].VCodec == "none" || in[1].ACodec == "none" || in[0].Height != 720 {127129 t.Fatalf("inputs: %+v", in)128130 }129- args, audio := ytInputArgs(in, 90*time.Second, true)131+ args, audio := youtube.InputArgs(in, 90*time.Second, true)130132 if audio != "1:a:0" || countInputs(args) != 2 || args[0] != "-xerror" {131133 t.Errorf("audio %q, %d inputs: %q", audio, countInputs(args), args)132134 }@@ -138,12 +140,12 @@ func TestYtInfoInputs(t *testing.T) {
138140 }139141 }140142 // One file with picture and sound.141- single := ytInfo{ID: "x", ytFormat: ytFormat{URL: "https://example.com/v.mp4", VCodec: "avc1", ACodec: "mp4a.40.2"}}142- if args, audio := ytInputArgs(single.inputs(), 0, false); audio != "0:a:0" || countInputs(args) != 1 ||143+ single := youtube.Info{ID: "x", Format: youtube.Format{URL: "https://example.com/v.mp4", VCodec: "avc1", ACodec: "mp4a.40.2"}}144+ if args, audio := youtube.InputArgs(single.Inputs(), 0, false); audio != "0:a:0" || countInputs(args) != 1 ||143145 slices.Contains(args, "-ss") || slices.Contains(args, "-request_size") {144146 t.Errorf("single file: %q %q", args, audio)145147 }146- if got := ytFormatSpec(480); got != "bv*[height<=480][vcodec^=avc1]+ba[ext=m4a][language^=en]/bv*[height<=480][vcodec^=avc1]+ba[ext=m4a]/b[height<=480]" {148+ if got := youtube.FormatSpec(480); got != "bv*[height<=480][vcodec^=avc1]+ba[ext=m4a][language^=en]/bv*[height<=480][vcodec^=avc1]+ba[ext=m4a]/b[height<=480]" {147149 t.Errorf("format %s", got)148150 }149151 }@@ -155,8 +157,8 @@ func TestYtDescription(t *testing.T) {
155157 "Lego Party on Steam: https://x.com\n\nApollo: http://twitch.tv/a\nJustin: http://twitch.tv/b\n\nFun with friends.": "Fun with friends.",156158 "https://only.links/here": "",157159 } {158- if got := ytDescription(in); got != want {159- t.Errorf("ytDescription(%q) = %q, want %q", in, got, want)160+ if got := youtube.Description(in); got != want {161+ t.Errorf("youtube.Description(%q) = %q, want %q", in, got, want)160162 }161163 }162164 }@@ -194,8 +196,8 @@ func TestYtVideosURL(t *testing.T) {
194196 "UC3tNpTOHsTnkmbwztCs30sA": "https://www.youtube.com/channel/UC3tNpTOHsTnkmbwztCs30sA/videos",195197 "https://www.youtube.com/@Northernlion/videos": "https://www.youtube.com/@Northernlion/videos",196198 } {197- if got := ytVideosURL(in); got != want {198- t.Errorf("ytVideosURL(%q) = %q, want %q", in, got, want)199+ if got := youtube.VideosURL(in); got != want {200+ t.Errorf("youtube.VideosURL(%q) = %q, want %q", in, got, want)199201 }200202 }201203 }@@ -432,7 +434,7 @@ func TestYouTubeGuide(t *testing.T) {
432434 write(youtubeConfig, `{"channels": ["@Northernlion", "https://www.youtube.com/@TheLibraryofLetourneau"], "minMinutes": 5}`)433435 cat := ytCatalog{}434436 for i, name := range []string{"Northernlion", "The Library of Letourneau"} {435- s := &ytSource{URL: ytVideosURL([]string{"@Northernlion", "@TheLibraryofLetourneau"}[i]),437+ s := &ytSource{URL: youtube.VideosURL([]string{"@Northernlion", "@TheLibraryofLetourneau"}[i]),436438 ID: fmt.Sprintf("UC%d", i), Name: name, Listed: now, Checked: now}437439 for j := range 40 {438440 s.Videos = append(s.Videos, &ytVideo{ID: fmt.Sprintf("v%d-%02d", i, j), Title: fmt.Sprintf("%s video %d", name, j),@@ -581,7 +583,7 @@ func TestYouTubeLive(t *testing.T) {
581583 if err := os.WriteFile(filepath.Join(dir, youtubeConfig), []byte(cfg), 0o644); err != nil {582584 t.Fatal(err)583585 }584- tool := &YtDlp{Path: os.Getenv("AIRWAVES_YTDLP")}586+ tool := &youtube.YtDlp{Path: os.Getenv("AIRWAVES_YTDLP")}585587 if tool.Path == "" {586588 tool.Path, tool.Update = filepath.Join(root, "tools", "yt-dlp"), true587589 }@@ -4,11 +4,10 @@ import (
44 "context"55 "errors"66 "log"7- "net/url"8- "strconv"97 "time"108 119 "airwaves/internal/phase"10+ "airwaves/internal/youtube"1211 )1312 1413 // Finding where to stream a video from takes yt-dlp seconds, too long to@@ -19,7 +18,7 @@ import (
1918 2019 // ytResolved is a video's details with its addresses, found ahead.2120 type ytResolved struct {22- info ytInfo21+ info youtube.Info2322 // caps are its English captions, fetched as soon as it's found, so2423 // tuning in doesn't wait for them either; nil when it has none.2524 caps *captions@@ -118,21 +117,21 @@ func (y *YouTube) lookAhead(id string) {
118117 height := y.cfg.MaxHeight119118 y.mu.Unlock()120119 tool := y.tool()121- tool.ahead.Lock()120+ tool.Ahead.Lock()122121 y.mu.Lock()123122 f.started = true124123 y.mu.Unlock()125124 ctx, cancel := context.WithTimeout(context.Background(), ytResolveTime)126125 info, err := y.fetch(ctx, id, height)127126 cancel()128- tool.ahead.Unlock()127+ tool.Ahead.Unlock()129128 y.mu.Lock()130129 delete(y.ahead, id)131130 switch {132131 case err == nil:133132 delete(y.missed, id)134133 y.keep(id, info, height)135- case gone(err):134+ case youtube.Gone(err):136135 if v := y.videos[id]; v != nil {137136 v.Skip, v.New, v.Looked = "gone", false, y.clock()138137 y.changed()@@ -148,47 +147,19 @@ func (y *YouTube) lookAhead(id string) {
148147 149148 // keep remembers a video's addresses until they expire, and starts150149 // fetching its captions. y.mu is held.151-func (y *YouTube) keep(id string, info ytInfo, height int) ytResolved {152- r := ytResolved{info: info, height: height, until: addressesUntil(info, time.Now())}150+func (y *YouTube) keep(id string, info youtube.Info, height int) ytResolved {151+ r := ytResolved{info: info, height: height, until: youtube.AddressesUntil(info, time.Now())}153152 if old, ok := y.resolved[id]; ok && old.caps != nil && !old.caps.failed() {154153 r.caps = old.caps // the same captions, fetched already or under way155- } else if sub, ok := info.captions(); ok {156- client := y.tool().client()157- r.caps = newCaptions(id, func(ctx context.Context) ([]byte, error) { return ytCaptions(ctx, client, sub) })154+ } else if sub, ok := info.Captions(); ok {155+ client := y.tool().Client()156+ r.caps = newCaptions(id, func(ctx context.Context) ([]byte, error) { return youtube.FetchCaptions(ctx, client, sub) })158157 r.caps.start()159158 }160159 y.resolved[id] = r161160 return r162161 }163162 164-// addressesUntil is when a video's addresses, found at, may stop working:165-// ten minutes before the earliest "expire" they carry (YouTube's last six166-// hours), or an hour after they were found when they carry none.167-func addressesUntil(info ytInfo, at time.Time) time.Time {168- until := at.Add(time.Hour)169- var earliest time.Time170- for _, f := range info.inputs() {171- u, err := url.Parse(f.URL)172- if err != nil {173- return until174- }175- sec, err := strconv.ParseInt(u.Query().Get("expire"), 10, 64)176- if err != nil || sec <= 0 {177- return until178- }179- if t := time.Unix(sec, 0); earliest.IsZero() || t.Before(earliest) {180- earliest = t181- }182- }183- if earliest.IsZero() {184- return until185- }186- if latest := at.Add(6 * time.Hour); earliest.After(latest) {187- return latest188- }189- return earliest.Add(-10 * time.Minute)190-}191-192163 // resolve finds where to stream a video from, and its captions: what was193164 // found ahead of time, waiting for it when that's under way, else yt-dlp194165 // now. fresh skips what was found before. What's found is kept for the@@ -233,15 +204,15 @@ func (y *YouTube) resolve(ctx context.Context, id string, fresh bool) (ytResolve
233204 }234205 235206 // fetch asks yt-dlp where to stream a video from.236-func (y *YouTube) fetch(ctx context.Context, id string, height int) (ytInfo, error) {207+func (y *YouTube) fetch(ctx context.Context, id string, height int) (youtube.Info, error) {237208 ctx, cancel := context.WithTimeout(ctx, ytResolveTime)238209 defer cancel()239- raw, err := y.tool().run(ctx, "-J", "--no-playlist", "-f", ytFormatSpec(height), ytWatchURL(id))210+ raw, err := y.tool().Run(ctx, "-J", "--no-playlist", "-f", youtube.FormatSpec(height), youtube.WatchURL(id))240211 if err != nil {241- return ytInfo{}, err212+ return youtube.Info{}, err242213 }243- info, err := parseYtInfo(raw)244- if err == nil && len(info.inputs()) == 0 {214+ info, err := youtube.ParseInfo(raw)215+ if err == nil && len(info.Inputs()) == 0 {245216 err = errors.New("yt-dlp found no stream")246217 }247218 return info, err@@ -11,6 +11,7 @@ import (
1111 "time"1212 1313 "airwaves/internal/phase"14+ "airwaves/internal/youtube"1415 )1516 1617 // TestYouTubeFindsAhead: the videos on now and next are found ahead,@@ -27,7 +28,7 @@ func TestYouTubeFindsAhead(t *testing.T) {
2728 t0 := time.Now().UTC().Truncate(time.Second)2829 stub := &feedStub{down: true}2930 y := &YouTube{Num: "1.8", Dir: t.TempDir(), loaded: true, now: func() time.Time { return t0 },30- YtDlp: &YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: stub}},31+ YtDlp: &youtube.YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: stub}},3132 cfg: ytConfig{MaxHeight: 720},3233 pool: &ytPool{Basis: "test", Lock: time.Hour}, plan: ytPlayout{Basis: "test", Airings: []ytAiring{3334 {ID: "aaaaaaaaaaa", Channel: "UCa", Title: "A", Start: t0.Add(-time.Minute), End: t0.Add(10 * time.Minute)},@@ -59,7 +60,7 @@ func TestYouTubeFindsAhead(t *testing.T) {
5960 // Tuning in uses what was found.6061 timer := phase.New("tune", "youtube")6162 r, err := y.resolve(phase.NewContext(context.Background(), timer), "aaaaaaaaaaa", false)62- if err != nil || len(r.info.inputs()) == 0 {63+ if err != nil || len(r.info.Inputs()) == 0 {6364 t.Fatalf("resolve: %v %+v", err, r)6465 }6566 if n := runs(); n != 2 || !timer.Has("found ahead") {@@ -116,16 +117,16 @@ func TestAddressesUntil(t *testing.T) {
116117 exp := func(d time.Duration) string {117118 return "https://rr1.example/videoplayback?expire=" + strconv.FormatInt(at.Add(d).Unix(), 10) + "&sig=x"118119 }119- two := ytInfo{RequestedFormats: []ytFormat{{URL: exp(6 * time.Hour), VCodec: "avc1"}, {URL: exp(5 * time.Hour), VCodec: "none", ACodec: "mp4a"}}}120- if got, want := addressesUntil(two, at), at.Add(5*time.Hour-10*time.Minute); !got.Equal(want) {120+ two := youtube.Info{RequestedFormats: []youtube.Format{{URL: exp(6 * time.Hour), VCodec: "avc1"}, {URL: exp(5 * time.Hour), VCodec: "none", ACodec: "mp4a"}}}121+ if got, want := youtube.AddressesUntil(two, at), at.Add(5*time.Hour-10*time.Minute); !got.Equal(want) {121122 t.Errorf("two addresses: until %v, want %v", got, want)122123 }123- none := ytInfo{ytFormat: ytFormat{URL: "https://example.com/v.mp4", VCodec: "avc1", ACodec: "mp4a"}}124- if got, want := addressesUntil(none, at), at.Add(time.Hour); !got.Equal(want) {124+ none := youtube.Info{Format: youtube.Format{URL: "https://example.com/v.mp4", VCodec: "avc1", ACodec: "mp4a"}}125+ if got, want := youtube.AddressesUntil(none, at), at.Add(time.Hour); !got.Equal(want) {125126 t.Errorf("no expiry: until %v, want %v", got, want)126127 }127- far := ytInfo{ytFormat: ytFormat{URL: exp(48 * time.Hour), VCodec: "avc1", ACodec: "mp4a"}}128- if got, want := addressesUntil(far, at), at.Add(6*time.Hour); !got.Equal(want) {128+ far := youtube.Info{Format: youtube.Format{URL: exp(48 * time.Hour), VCodec: "avc1", ACodec: "mp4a"}}129+ if got, want := youtube.AddressesUntil(far, at), at.Add(6*time.Hour); !got.Equal(want) {129130 t.Errorf("far expiry: until %v, want %v", got, want)130131 }131132 }@@ -2,16 +2,13 @@ package vchan
22 33 import (44 "encoding/json"5- "encoding/xml"6- "errors"75 "fmt"86 "math"97 "os"108 "path/filepath"11- "regexp"12- "slices"13- "strings"149 "time"10+11+ "airwaves/internal/youtube"1512 )1613 1714 // ytConfig is a YouTube channel's youtube.json.@@ -71,7 +68,7 @@ func readYtConfig(path string) (ytConfig, error) {
7168 case c.Playlist != "" && len(c.Channels) > 0:7269 return c, fmt.Errorf("%s names both channels and a playlist; it plays one or the other", filepath.Base(path))7370 case c.Playlist != "":74- if _, err := YouTubePlaylist(c.Playlist); err != nil {71+ if _, err := youtube.Playlist(c.Playlist); err != nil {7572 return c, fmt.Errorf("%s: playlist %q: %w", filepath.Base(path), c.Playlist, err)7673 }7774 case len(c.Channels) == 0:@@ -180,34 +177,8 @@ func (c *ytCatalog) source(url string) *ytSource {
180177 return s181178 }182179 183-// ytVideosURL turns a configured channel into its uploads tab.184-func ytVideosURL(s string) string {185- s = strings.TrimSpace(s)186- switch {187- case strings.HasPrefix(s, "@"):188- s = "https://www.youtube.com/" + s189- case strings.HasPrefix(s, "UC") && !strings.Contains(s, "/"):190- s = "https://www.youtube.com/channel/" + s191- case !strings.Contains(s, "://"):192- s = "https://" + s193- }194- s = strings.TrimRight(s, "/")195- for _, tab := range []string{"/videos", "/featured", "/streams", "/shorts", "/playlists", "/about"} {196- s = strings.TrimSuffix(s, tab)197- }198- return s + "/videos"199-}200-201-func ytFeedURL(channelID string) string {202- return "https://www.youtube.com/feeds/videos.xml?channel_id=" + channelID203-}204-205-func ytWatchURL(id string) string { return "https://www.youtube.com/watch?v=" + id }206-207-func ytThumbnail(id string) string { return "https://i.ytimg.com/vi/" + id + "/hqdefault.jpg" }208-209180 // ytListing is a channel's uploads tab, from yt-dlp --flat-playlist -J210-// with ytListingArgs.181+// with youtube.ListingArgs.211182 type ytListing struct {212183 ID, Name string213184 Videos []*ytVideo // newest first; a playlist's in its order@@ -216,47 +187,24 @@ type ytListing struct {
216187 }217188 218189 func parseYtListing(raw []byte) (ytListing, error) {219- var p struct {220- Channel string `json:"channel"`221- ChannelID string `json:"channel_id"`222- Entries []struct {223- ID string `json:"id"`224- Title string `json:"title"`225- Duration float64 `json:"duration"`226- Timestamp float64 `json:"timestamp"`227- LiveStatus string `json:"live_status"`228- } `json:"entries"`229- }230- if err := json.Unmarshal(raw, &p); err != nil {231- return ytListing{}, err232- }233- if p.ChannelID == "" {234- return ytListing{}, errors.New("not a YouTube channel")235- }236- l := ytListing{ID: p.ChannelID, Name: p.Channel}237- for _, e := range p.Entries {238- if e.ID == "" {239- continue240- }241- v := &ytVideo{ID: e.ID, Title: e.Title, Seconds: int(math.Round(e.Duration))}242- if e.Timestamp > 0 {243- v.Published, v.Approx = time.Unix(int64(e.Timestamp), 0).UTC(), true244- }245- switch e.LiveStatus {246- case "is_upcoming":247- v.Skip = "upcoming"248- case "is_live", "post_live", "was_live":249- v.Skip = "live"190+ l, err := youtube.ParseListing(raw)191+ return listingOf(l, true), err192+}193+194+// listingOf is a listing as the catalog keeps it: dated, its upload195+// dates, rough as they are.196+func listingOf(l youtube.Listing, dated bool) ytListing {197+ out := ytListing{ID: l.ID, Name: l.Name, Owner: l.Owner, OwnerID: l.OwnerID}198+ for _, e := range l.Entries {199+ v := &ytVideo{ID: e.ID, Title: e.Title, Seconds: e.Seconds, Skip: e.Skip, Channel: e.Channel}200+ if dated && !e.Published.IsZero() {201+ v.Published, v.Approx = e.Published, true250202 }251- l.Videos = append(l.Videos, v)203+ out.Videos = append(out.Videos, v)252204 }253- return l, nil205+ return out254206 }255207 256-// ytListingArgs list a channel's uploads tab with approximate upload dates257-// (from YouTube's "3 years ago"; no extra requests).258-var ytListingArgs = []string{"--flat-playlist", "-J", "--extractor-args", "youtubetab:approximate_date"}259-260208 // ytNewAtTop is how far down a listing a video it didn't have before261209 // counts as a new upload, due a first run, rather than an old one that262210 // came back.@@ -341,32 +289,12 @@ func (s *ytSource) mergeNewest(l ytListing, now time.Time) (added int, changed b
341289 return added, changed || len(fresh) > 0342290 }343291 344-// ytFeedEntry is an upload in a channel's feed: the newest 15, Shorts and345-// streams included, without lengths.346-type ytFeedEntry struct {347- ID string `xml:"videoId"`348- Title string `xml:"title"`349- Published time.Time `xml:"published"`350- Description string `xml:"group>description"`351- Link struct {352- Href string `xml:"href,attr"`353- } `xml:"link"`354-}355-356-func parseYtFeed(raw []byte) ([]ytFeedEntry, error) {357- var f struct {358- Entries []ytFeedEntry `xml:"entry"`359- }360- err := xml.Unmarshal(raw, &f)361- return f.Entries, err362-}363-364292 // mergeFeed adds the feed's uploads the source doesn't have yet, as new365293 // videos due a first run once their details are looked up, and fills in366294 // the dates and descriptions of ones it has. Shorts are recognized by367295 // their links, and kept only so they aren't looked at again. added counts368296 // the new uploads.369-func (s *ytSource) mergeFeed(entries []ytFeedEntry, now time.Time) (added int, changed bool) {297+func (s *ytSource) mergeFeed(entries []youtube.FeedEntry, now time.Time) (added int, changed bool) {370298 known := map[string]*ytVideo{}371299 for _, v := range s.Videos {372300 known[v.ID] = v@@ -380,14 +308,14 @@ func (s *ytSource) mergeFeed(entries []ytFeedEntry, now time.Time) (added int, c
380308 if (v.Published.IsZero() || v.Approx) && !e.Published.IsZero() {381309 v.Published, v.Approx, changed = e.Published, false, true382310 }383- if d := ytDescription(e.Description); v.Description == "" && d != "" {311+ if d := youtube.Description(e.Description); v.Description == "" && d != "" {384312 v.Description, changed = d, true385313 }386314 continue387315 }388316 v := &ytVideo{ID: e.ID, Title: e.Title, Published: e.Published,389- Description: ytDescription(e.Description), Found: now, New: true}390- if strings.Contains(e.Link.Href, "/shorts/") {317+ Description: youtube.Description(e.Description), Found: now, New: true}318+ if e.Short() {391319 v.Skip, v.New = "short", false392320 } else {393321 added++@@ -400,55 +328,8 @@ func (s *ytSource) mergeFeed(entries []ytFeedEntry, now time.Time) (added int, c
400328 return added, changed || len(fresh) > 0401329 }402330 403-// ytInfo is a video's details from yt-dlp -J, and with a format chosen,404-// where to stream it from.405-type ytInfo struct {406- ID string `json:"id"`407- Title string `json:"title"`408- Duration float64 `json:"duration"`409- Timestamp float64 `json:"timestamp"`410- Description string `json:"description"`411- Language string `json:"language"`412- LiveStatus string `json:"live_status"`413- MediaType string `json:"media_type"`414- FormatID string `json:"format_id"`415- ytFormat416- RequestedFormats []ytFormat `json:"requested_formats"`417- // Subtitles are the uploader's, AutoCaptions YouTube's speech418- // recognition, by language ("en", "en-US", "en-orig").419- Subtitles map[string][]ytSubtitle `json:"subtitles"`420- AutoCaptions map[string][]ytSubtitle `json:"automatic_captions"`421-}422-423-type ytFormat struct {424- URL string `json:"url"`425- VCodec string `json:"vcodec"`426- ACodec string `json:"acodec"`427- Height int `json:"height"`428- Language string `json:"language"`429- HTTPHeaders map[string]string `json:"http_headers"`430-}431-432-// ytSubtitle is a subtitle track in one format.433-type ytSubtitle struct {434- Ext string `json:"ext"`435- URL string `json:"url"`436- Protocol string `json:"protocol"` // "m3u8_native" for a playlist of pieces437-}438-439-func parseYtInfo(raw []byte) (ytInfo, error) {440- var i ytInfo441- if err := json.Unmarshal(raw, &i); err != nil {442- return i, err443- }444- if i.ID == "" {445- return i, errors.New("no video in yt-dlp's output")446- }447- return i, nil448-}449-450331 // learn takes in a video's looked-up details.451-func (v *ytVideo) learn(i ytInfo, now time.Time) {332+func (v *ytVideo) learn(i youtube.Info, now time.Time) {452333 v.Looked = now453334 if i.Title != "" {454335 v.Title = i.Title@@ -459,13 +340,13 @@ func (v *ytVideo) learn(i ytInfo, now time.Time) {
459340 if i.Timestamp > 0 {460341 v.Published, v.Approx = time.Unix(int64(i.Timestamp), 0).UTC(), false461342 }462- if d := ytDescription(i.Description); d != "" {343+ if d := youtube.Description(i.Description); d != "" {463344 v.Description = d464345 }465- if l := i.audioLang(); l != "" {346+ if l := i.AudioLang(); l != "" {466347 v.AudioLang = l467348 }468- _, v.Captions = i.captions()349+ _, v.Captions = i.Captions()469350 switch {470351 case i.MediaType == "short":471352 v.Skip = "short"@@ -480,89 +361,3 @@ func (v *ytVideo) learn(i ytInfo, now time.Time) {
480361 v.New = false481362 }482363 }483-484-var (485- separator = regexp.MustCompile(`(?m)^\s*[=\-_*~#]{5,}\s*$`)486- label = regexp.MustCompile(`^.{0,40}:$`)487-)488-489-// ytDescription makes a guide description of a YouTube description: its490-// first paragraph that says something, before the links section, without491-// addresses or the labels left of them ("Game on Steam:").492-func ytDescription(s string) string {493- if i := separator.FindStringIndex(s); i != nil {494- s = s[:i[0]]495- }496- for para := range strings.SplitSeq(strings.ReplaceAll(s, "\r", ""), "\n\n") {497- var keep []string498- for line := range strings.SplitSeq(para, "\n") {499- line = strings.TrimSpace(urls.ReplaceAllString(line, ""))500- if line != "" && !label.MatchString(line) {501- keep = append(keep, strings.TrimSuffix(line, ":"))502- }503- }504- if len(keep) > 0 {505- return firstParagraph(strings.Join(keep, "\n"), 300)506- }507- }508- return ""509-}510-511-// inputs are the ffmpeg inputs for a resolved video, picture first.512-func (i ytInfo) inputs() []ytFormat {513- fs := i.RequestedFormats514- if len(fs) == 0 && i.URL != "" {515- fs = []ytFormat{i.ytFormat}516- }517- out := make([]ytFormat, 0, len(fs))518- for _, f := range fs {519- if f.VCodec != "none" && f.VCodec != "" {520- out = append(out, f)521- }522- }523- for _, f := range fs {524- if f.VCodec == "none" || f.VCodec == "" {525- out = append(out, f)526- }527- }528- return out529-}530-531-// audioLang is the sound's language, "" when YouTube doesn't say.532-func (i ytInfo) audioLang() string {533- for _, f := range i.inputs() {534- if f.ACodec != "none" && f.ACodec != "" && f.Language != "" {535- return f.Language536- }537- }538- return i.Language539-}540-541-// captions picks the video's English captions: the uploader's, else542-// YouTube's automatic ones, as json3, which times each word of automatic543-// captions, else WebVTT or SRT. ok is false when there are none.544-func (i ytInfo) captions() (sub ytSubtitle, ok bool) {545- pick := func(tracks map[string][]ytSubtitle, langs ...string) (ytSubtitle, bool) {546- for _, l := range langs {547- for _, ext := range []string{"json3", "vtt", "srt"} {548- for _, s := range tracks[l] {549- if s.Ext == ext && s.URL != "" {550- return s, true551- }552- }553- }554- }555- return ytSubtitle{}, false556- }557- var others []string558- for l := range i.Subtitles {559- if strings.HasPrefix(l, "en-") {560- others = append(others, l)561- }562- }563- slices.Sort(others)564- if s, ok := pick(i.Subtitles, append([]string{"en"}, others...)...); ok {565- return s, true566- }567- return pick(i.AutoCaptions, "en-orig", "en")568-}@@ -1,201 +0,0 @@
1-package vchan2-3-import (4- "cmp"5- "context"6- "encoding/json"7- "errors"8- "fmt"9- "net/url"10- "os"11- "os/exec"12- "regexp"13- "strconv"14- "strings"15- "sync"16- "time"17-)18-19-// YouTubeChannel is a YouTube channel as a search or a lookup finds it.20-type YouTubeChannel struct {21- ID string // "UC…"22- URL string // its page: by handle, when it has one23- Handle string // "@name"; some channels have none24- Name string25- Subscribers int6426- Description string27- Image string // its avatar28- // Videos is roughly how many videos it has uploaded, Shorts and29- // streams included; 0 when unknown. Only lookups count them.30- Videos int31-}32-33-// ytChannelsOnly narrows YouTube's search results to channels.34-const ytChannelsOnly = "EgIQAg%3D%3D"35-36-// SearchChannels asks YouTube for the channels matching query, n at most.37-// YouTube's Topic channels, which it makes for artists' music, are left38-// out: they have no uploads to play.39-func (t *YtDlp) SearchChannels(ctx context.Context, query string, n int) ([]YouTubeChannel, error) {40- u := "https://www.youtube.com/results?search_query=" + url.QueryEscape(query) + "&sp=" + ytChannelsOnly41- raw, err := t.run(ctx, "--flat-playlist", "-J", "--playlist-end", strconv.Itoa(n), u)42- if err != nil {43- return nil, err44- }45- var p struct {46- Entries []ytChannelInfo `json:"entries"`47- }48- if err := json.Unmarshal(raw, &p); err != nil {49- return nil, err50- }51- out := []YouTubeChannel{}52- for _, e := range p.Entries {53- c := e.channel()54- if !strings.HasPrefix(c.ID, "UC") || c.Handle == "" && strings.HasSuffix(c.Name, " - Topic") {55- continue56- }57- out = append(out, c)58- }59- return out, nil60-}61-62-// LookupChannel finds the channel at page (its link, "@handle" or "UC…" ID,63-// as youtube.json names channels), and roughly how many videos it has.64-func (t *YtDlp) LookupChannel(ctx context.Context, page string) (YouTubeChannel, error) {65- raw, err := t.run(ctx, "--flat-playlist", "-J", "--playlist-end", "1", ytVideosURL(page))66- if err != nil {67- return YouTubeChannel{}, err68- }69- var e ytChannelInfo70- if err := json.Unmarshal(raw, &e); err != nil {71- return YouTubeChannel{}, err72- }73- c := e.channel()74- if !strings.HasPrefix(c.ID, "UC") {75- return YouTubeChannel{}, errors.New("not a YouTube channel")76- }77- // The channel's uploads playlist says how many there are; its tab78- // doesn't.79- if raw, err := t.run(ctx, "--flat-playlist", "-J", "--playlist-end", "1",80- "https://www.youtube.com/playlist?list=UU"+c.ID[2:]); err == nil {81- var p struct {82- Count int `json:"playlist_count"`83- }84- if json.Unmarshal(raw, &p) == nil {85- c.Videos = p.Count86- }87- }88- return c, nil89-}90-91-// ytChannelInfo is a channel in yt-dlp's output: a search result, or the92-// top of a channel's tab.93-type ytChannelInfo struct {94- ChannelID string `json:"channel_id"`95- Channel string `json:"channel"`96- Title string `json:"title"`97- UploaderID string `json:"uploader_id"`98- Followers float64 `json:"channel_follower_count"`99- Description string `json:"description"`100- Thumbnails []struct {101- URL string `json:"url"`102- Width int `json:"width"`103- Height int `json:"height"`104- ID string `json:"id"`105- } `json:"thumbnails"`106-}107-108-func (e ytChannelInfo) channel() YouTubeChannel {109- c := YouTubeChannel{ID: e.ChannelID, Name: cmp.Or(e.Channel, e.Title), Subscribers: int64(e.Followers),110- Description: ytBlurb(e.Description)}111- switch {112- case strings.HasPrefix(e.UploaderID, "@"):113- c.Handle = e.UploaderID114- c.URL = "https://www.youtube.com/" + c.Handle115- case c.ID != "":116- c.URL = "https://www.youtube.com/channel/" + c.ID117- }118- // The avatar is the largest square picture (the others are banners),119- // asked for at a size to show.120- size := 0121- for _, th := range e.Thumbnails {122- if th.Width > size && th.Width == th.Height {123- c.Image, size = th.URL, th.Width124- }125- }126- for _, th := range e.Thumbnails {127- if c.Image == "" && th.ID == "avatar_uncropped" {128- c.Image = th.URL129- }130- }131- if strings.HasPrefix(c.Image, "//") {132- c.Image = "https:" + c.Image133- }134- if i := strings.LastIndex(c.Image, "="); i > 0 {135- c.Image = c.Image[:i] + imageSize.ReplaceAllString(c.Image[i:], "=s176")136- }137- return c138-}139-140-// imageSize is the size option of a YouTube image address ("=s900-c-k").141-var imageSize = regexp.MustCompile(`^=s\d+`)142-143-// ytBlurb makes a channel's description one short paragraph. Search144-// results come cut short with "...".145-func ytBlurb(s string) string {146- s = strings.Join(strings.Fields(s), " ")147- if t, ok := strings.CutSuffix(s, " ..."); ok {148- s = t + "…"149- }150- if r := []rune(s); len(r) > 300 {151- s = strings.TrimSpace(string(r[:299])) + "…"152- }153- return s154-}155-156-// ErrYtDlpInstalling is Ready's answer while yt-dlp is being installed.157-var ErrYtDlpInstalling = errors.New("yt-dlp is being installed; try again in a minute")158-159-// ytdlpInstall is Ready's install of a managed yt-dlp.160-type ytdlpInstall struct {161- mu sync.Mutex162- running bool163- err error // the last try's164-}165-166-var ytdlpInstalls sync.Map // *YtDlp: *ytdlpInstall167-168-// Ready reports whether yt-dlp is there to run. When t keeps its own copy169-// and hasn't installed it yet, Ready starts installing it and reports170-// ErrYtDlpInstalling, or why the last try failed.171-func (t *YtDlp) Ready() error {172- if !t.Update {173- if _, err := exec.LookPath(cmp.Or(t.Path, "yt-dlp")); err != nil {174- return fmt.Errorf("yt-dlp isn't installed on the server: %w", err)175- }176- return nil177- }178- if _, err := os.Stat(t.Path); err == nil {179- return nil180- }181- v, _ := ytdlpInstalls.LoadOrStore(t, &ytdlpInstall{})182- in := v.(*ytdlpInstall)183- in.mu.Lock()184- defer in.mu.Unlock()185- failed := in.err186- if !in.running {187- in.running = true188- go func() {189- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)190- defer cancel()191- _, err := t.command(ctx)192- in.mu.Lock()193- in.running, in.err = false, err194- in.mu.Unlock()195- }()196- }197- if failed != nil {198- return fmt.Errorf("couldn't install yt-dlp (%v); trying again", failed)199- }200- return ErrYtDlpInstalling201-}@@ -14,6 +14,8 @@ import (
1414 "sync"1515 "testing"1616 "time"17+18+ "airwaves/internal/youtube"1719 )1820 1921 // TestMain lets the test binary stand in for yt-dlp: run with@@ -129,14 +131,14 @@ func TestYtFeedFallback(t *testing.T) {
129131 if err := os.WriteFile(filepath.Join(dir, youtubeConfig), []byte(`{"channels": ["@a"]}`), 0o644); err != nil {130132 t.Fatal(err)131133 }132- s := &ytSource{URL: ytVideosURL("@a"), ID: "UCa", Name: "A", Listed: clock, Dated: true}134+ s := &ytSource{URL: youtube.VideosURL("@a"), ID: "UCa", Name: "A", Listed: clock, Dated: true}133135 for i := range 100 {134136 s.Videos = append(s.Videos, &ytVideo{ID: fmt.Sprintf("v%03d", i), Title: fmt.Sprintf("Video %d", i),135137 Seconds: 1200, Published: clock.AddDate(0, 0, -i-1), Approx: true, Looked: clock})136138 }137139 feed := &feedStub{down: true, feed: []byte(strings.ReplaceAll(string(readTestdata(t, "yt_feed.xml")), "NewUpload01", "fromfeed001"))}138140 y := &YouTube{Num: "1.8", Title: "A", Dir: dir, now: func() time.Time { return clock },139- YtDlp: &YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: feed}}}141+ YtDlp: &youtube.YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: feed}}}140142 y.save(ytCatalogFile, ytCatalog{Sources: []*ytSource{s}})141143 refresh := func() {142144 y.mu.Lock()@@ -12,124 +12,20 @@ package vchan
1212 import (1313 "cmp"1414 "context"15- "encoding/json"16- "errors"1715 "log"18- "math"19- "net/url"20- "regexp"2116 "slices"2217 "strings"2318 "time"24-)25-26-var (27- errNotPlaylist = errors.New("that isn't a YouTube playlist: use its link (youtube.com/playlist?list=...) or its ID")28- errVideoNotPlaylist = errors.New("that's a link to a video: use the playlist's link, with list= in it")29- errMix = errors.New("that's one of YouTube's mixes, which change as they play: use a playlist's link")30- // ytListID is a playlist's ID in a link, and ytBareList one given on31- // its own: "PL…", a channel's uploads ("UU…") or an album ("OLAK5uy_…").32- ytListID = regexp.MustCompile(`^[A-Za-z0-9_-]{2,64}$`)33- ytBareList = regexp.MustCompile(`^(PL|UU|UL|FL|OLAK5uy_)[A-Za-z0-9_-]{10,62}$`)34-)35-36-// YouTubePlaylist checks a playlist as youtube.json or the admin page give37-// it, its link (or the link of a video in it) or its ID, and returns its38-// page: https://www.youtube.com/playlist?list= and the ID.39-func YouTubePlaylist(s string) (string, error) {40- s = strings.TrimSpace(s)41- id := s42- if !ytBareList.MatchString(s) {43- raw := s44- if !strings.Contains(s, "://") {45- raw = "https://" + s46- }47- u, err := url.Parse(raw)48- if err != nil || (u.Scheme != "http" && u.Scheme != "https") {49- return "", errNotPlaylist50- }51- host := strings.ToLower(u.Hostname())52- for _, p := range []string{"www.", "m.", "music."} {53- host = strings.TrimPrefix(host, p)54- }55- if host != "youtube.com" && host != "youtu.be" {56- return "", errNotPlaylist57- }58- id = u.Query().Get("list")59- switch {60- case id != "":61- case host == "youtu.be", u.Path == "/watch", strings.HasPrefix(u.Path, "/shorts/"), strings.HasPrefix(u.Path, "/live/"):62- return "", errVideoNotPlaylist63- default:64- return "", errNotPlaylist65- }66- }67- switch {68- case !ytListID.MatchString(id):69- return "", errNotPlaylist70- case strings.HasPrefix(id, "RD"):71- return "", errMix72- }73- return ytPlaylistURL(id), nil74-}7519 76-func ytPlaylistURL(id string) string { return "https://www.youtube.com/playlist?list=" + id }77-78-// ytPlaylistArgs list a playlist: its videos in order, with their lengths.79-var ytPlaylistArgs = []string{"--flat-playlist", "-J"}80-81-var (82- // ytHidden are the titles YouTube lists videos that can't play under.83- ytHidden = map[string]bool{"[Private video]": true, "[Deleted video]": true, "[Unavailable video]": true}84- // ytLocked are the availabilities of videos that only some can watch.85- ytLocked = map[string]bool{"private": true, "premium_only": true, "subscriber_only": true, "needs_auth": true}20+ "airwaves/internal/youtube"8621 )8722 8823 // parseYtPlaylist reads a playlist's listing, from yt-dlp --flat-playlist89-// -J with ytPlaylistArgs: its videos in its order, each once. Videos that90-// can't play here (private, deleted, members only) are left out; live and91-// upcoming ones are kept, marked, to join once they can play.24+// -J with youtube.PlaylistArgs (see youtube.ParsePlaylist), undated as a25+// playlist channel has it.9226 func parseYtPlaylist(raw []byte) (ytListing, error) {93- var p struct {94- Type string `json:"_type"`95- ID string `json:"id"`96- Title string `json:"title"`97- Channel string `json:"channel"`98- ChannelID string `json:"channel_id"`99- Uploader string `json:"uploader"`100- Entries []struct {101- ID string `json:"id"`102- Title string `json:"title"`103- Duration float64 `json:"duration"`104- Channel string `json:"channel"`105- Uploader string `json:"uploader"`106- LiveStatus string `json:"live_status"`107- Availability string `json:"availability"`108- } `json:"entries"`109- }110- if err := json.Unmarshal(raw, &p); err != nil {111- return ytListing{}, err112- }113- if p.Type != "playlist" || p.ID == "" || p.ID == p.ChannelID { // not a channel's tab114- return ytListing{}, errors.New("not a YouTube playlist")115- }116- l := ytListing{ID: p.ID, Name: p.Title, Owner: cmp.Or(p.Channel, p.Uploader), OwnerID: p.ChannelID}117- seen := map[string]bool{}118- for _, e := range p.Entries {119- if e.ID == "" || seen[e.ID] || ytHidden[e.Title] || ytLocked[e.Availability] {120- continue121- }122- seen[e.ID] = true123- v := &ytVideo{ID: e.ID, Title: e.Title, Seconds: int(math.Round(e.Duration)), Channel: cmp.Or(e.Channel, e.Uploader)}124- switch e.LiveStatus {125- case "is_upcoming":126- v.Skip = "upcoming"127- case "is_live", "post_live":128- v.Skip = "live"129- }130- l.Videos = append(l.Videos, v)131- }132- return l, nil27+ l, err := youtube.ParsePlaylist(raw)28+ return listingOf(l, false), err13329 }13430 13531 // mergePlaylist takes in a listing of the playlist: its videos in its@@ -244,7 +140,7 @@ func (y *YouTube) playlistPrograms(from, to time.Time) []Program {
244140 for k, sl := range slots {245141 if sl.I >= 0 {246142 v := seq[sl.I]247- p := Program{Start: sl.Start.Local(), End: sl.End.Local(), Title: v.Title, Image: ytThumbnail(v.ID),143+ p := Program{Start: sl.Start.Local(), End: sl.End.Local(), Title: v.Title, Image: youtube.Thumbnail(v.ID),248144 Description: v.Description, AudioLang: v.AudioLang, Captions: v.Captions}249145 p.Subtitle, p.Date = ytDated(y.channelOf(v), v, now)250146 out = append(out, p)@@ -314,9 +210,9 @@ func playlistAiring(v *ytVideo, sl slot) ytAiring {
314210 // listPlaylist lists the playlist at url again. y.mu isn't held.315211 func (y *YouTube) listPlaylist(ctx context.Context, url string) {316212 tool := y.tool()317- tool.jobs.Lock()318- raw, err := tool.run(ctx, slices.Concat(ytPlaylistArgs, []string{url})...)319- tool.jobs.Unlock()213+ tool.Jobs.Lock()214+ raw, err := tool.Run(ctx, slices.Concat(youtube.PlaylistArgs, []string{url})...)215+ tool.Jobs.Unlock()320216 var l ytListing321217 if err == nil {322218 l, err = parseYtPlaylist(raw)@@ -369,42 +265,9 @@ func (y *YouTube) playlistLookups(now time.Time) []string {
369265 370266 // learnVideo takes in a video's looked-up details. A playlist's Shorts371267 // and recorded streams play like any other video. y.mu is held.372-func (y *YouTube) learnVideo(v *ytVideo, i ytInfo) {268+func (y *YouTube) learnVideo(v *ytVideo, i youtube.Info) {373269 v.learn(i, y.clock())374270 if y.isPlaylist() && (v.Skip == "short" || v.Skip == "live" && i.LiveStatus == "was_live") {375271 v.Skip = ""376272 }377273 }378-379-// YouTubePlaylistInfo is a playlist as a lookup finds it.380-type YouTubePlaylistInfo struct {381- ID, URL string382- Title string383- Owner, OwnerID string // the channel that made it384- // Videos is how many it has, as YouTube counts them, ones that can't385- // play included.386- Videos int387- Image string // its first video's thumbnail388-}389-390-// LookupPlaylist finds the playlist at page (as YouTubePlaylist gives391-// it): its title, whose it is, and how many videos it has.392-func (t *YtDlp) LookupPlaylist(ctx context.Context, page string) (YouTubePlaylistInfo, error) {393- raw, err := t.run(ctx, "--flat-playlist", "-J", "--playlist-end", "1", page)394- if err != nil {395- return YouTubePlaylistInfo{}, err396- }397- l, err := parseYtPlaylist(raw)398- if err != nil {399- return YouTubePlaylistInfo{}, err400- }401- var p struct {402- Count int `json:"playlist_count"`403- }404- _ = json.Unmarshal(raw, &p)405- info := YouTubePlaylistInfo{ID: l.ID, URL: ytPlaylistURL(l.ID), Title: l.Name, Owner: l.Owner, OwnerID: l.OwnerID, Videos: p.Count}406- if len(l.Videos) > 0 {407- info.Image = ytThumbnail(l.Videos[0].ID)408- }409- return info, nil410-}@@ -10,6 +10,8 @@ import (
1010 "strings"1111 "testing"1212 "time"13+14+ "airwaves/internal/youtube"1315 )1416 1517 const csList = "PL8dPuuaLjXtNlUrzyH5r6jN9ulIgZBpdo"@@ -26,11 +28,11 @@ func TestYouTubePlaylistLinks(t *testing.T) {
2628 "http://www.youtube.com/embed/videoseries?list=" + csList,2729 csList,2830 } {29- if got, err := YouTubePlaylist(in); got != page || err != nil {30- t.Errorf("YouTubePlaylist(%q) = %q, %v", in, got, err)31+ if got, err := youtube.Playlist(in); got != page || err != nil {32+ t.Errorf("youtube.Playlist(%q) = %q, %v", in, got, err)3133 }3234 }33- if got, err := YouTubePlaylist("UU3tNpTOHsTnkmbwztCs30sA"); got != "https://www.youtube.com/playlist?list=UU3tNpTOHsTnkmbwztCs30sA" || err != nil {35+ if got, err := youtube.Playlist("UU3tNpTOHsTnkmbwztCs30sA"); got != "https://www.youtube.com/playlist?list=UU3tNpTOHsTnkmbwztCs30sA" || err != nil {3436 t.Errorf("uploads playlist: %q, %v", got, err)3537 }3638 for in, want := range map[string]string{@@ -43,8 +45,8 @@ func TestYouTubePlaylistLinks(t *testing.T) {
4345 "crash course": "isn't a YouTube playlist",4446 "https://www.youtube.com/playlist?list=PL<script>": "isn't a YouTube playlist",4547 } {46- if got, err := YouTubePlaylist(in); err == nil || !strings.Contains(err.Error(), want) {47- t.Errorf("YouTubePlaylist(%q) = %q, %v; want an error saying %q", in, got, err, want)48+ if got, err := youtube.Playlist(in); err == nil || !strings.Contains(err.Error(), want) {49+ t.Errorf("youtube.Playlist(%q) = %q, %v; want an error saying %q", in, got, err, want)4850 }4951 }5052 }@@ -100,7 +102,7 @@ func TestYtPlaylistListing(t *testing.T) {
100102 // Listed again: the playlist's new order, what's known kept, a video101103 // taken out gone, and one found unplayable still so.102104 now := time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)103- s := &ytSource{URL: ytPlaylistURL(csList)}105+ s := &ytSource{URL: youtube.PlaylistURL(csList)}104106 if added := s.mergePlaylist(l, now); added != 0 || s.Name != "Computer Science" || s.Owner != "CrashCourse" || len(s.Videos) != 9 {105107 t.Fatalf("first listing: %d added, %+v", added, s)106108 }@@ -150,7 +152,7 @@ func testPlaylist(t *testing.T, clock time.Time, schedule string) *YouTube {
150152 if schedule != "" {151153 write(DetailsFile, `{"schedule": `+schedule+`}`)152154 }153- s := &ytSource{URL: ytPlaylistURL("PLtestplaylist01"), ID: "PLtestplaylist01", Name: "A course", Owner: "Teacher",155+ s := &ytSource{URL: youtube.PlaylistURL("PLtestplaylist01"), ID: "PLtestplaylist01", Name: "A course", Owner: "Teacher",154156 Listed: clock, Checked: clock, Dated: true}155157 for i := range 5 {156158 s.Videos = append(s.Videos, &ytVideo{ID: fmt.Sprintf("part%07d", i+1), Title: fmt.Sprintf("Part %d", i+1),@@ -161,7 +163,7 @@ func testPlaylist(t *testing.T, clock time.Time, schedule string) *YouTube {
161163 }162164 s.Videos[1].Channel = "Guest"163165 y := &YouTube{Num: "1.9", Title: "Course", Dir: dir, now: func() time.Time { return clock },164- YtDlp: &YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: &feedStub{down: true}}}}166+ YtDlp: &youtube.YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: &feedStub{down: true}}}}165167 y.save(ytCatalogFile, ytCatalog{Sources: []*ytSource{s}})166168 return y167169 }@@ -187,7 +189,7 @@ func TestYtPlaylistAroundTheClock(t *testing.T) {
187189 t.Errorf("programs %v, want %v", titles, want)188190 }189191 if p := progs[0]; !p.Start.Equal(scheduleEpoch.Add(100*loop+600*time.Second)) || p.Subtitle != "Guest" ||190- p.Description != "About part 2" || p.Image != ytThumbnail("part0000002") {192+ p.Description != "About part 2" || p.Image != youtube.Thumbnail("part0000002") {191193 t.Errorf("part 2: %+v", p)192194 }193195 if p := progs[1]; p.Subtitle != "Teacher" {@@ -300,7 +302,7 @@ func TestYtPlaylistListedWithYtDlp(t *testing.T) {
300302 }301303 clock := time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)302304 y := &YouTube{Num: "1.9", Title: "CS", Dir: dir, now: func() time.Time { return clock },303- YtDlp: &YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: &feedStub{down: true}}}}305+ YtDlp: &youtube.YtDlp{Path: os.Args[0], HTTP: &http.Client{Transport: &feedStub{down: true}}}}304306 refresh := func() {305307 y.mu.Lock()306308 y.tend()@@ -351,8 +353,8 @@ func TestYtPlaylistListedWithYtDlp(t *testing.T) {
351353 if len(st.Sources) != 1 || st.Config.Playlist != given {352354 t.Fatalf("status: %+v", st)353355 }354- if s := st.Sources[0]; s.Channel != given || s.URL != ytPlaylistURL(csList) || s.Name != "Computer Science" || s.Owner != "CrashCourse" ||355- s.Videos != 9 || s.Playable != 7 || s.Listing || !s.Listed.Equal(clock) || s.Image != ytThumbnail("tpIctyqH29Q") {356+ if s := st.Sources[0]; s.Channel != given || s.URL != youtube.PlaylistURL(csList) || s.Name != "Computer Science" || s.Owner != "CrashCourse" ||357+ s.Videos != 9 || s.Playable != 7 || s.Listing || !s.Listed.Equal(clock) || s.Image != youtube.Thumbnail("tpIctyqH29Q") {356358 t.Errorf("status: %+v", s)357359 }358360 if y.Empty() {@@ -10,6 +10,7 @@ import (
1010 "time"1111 1212 "airwaves/internal/cc"13+ "airwaves/internal/youtube"1314 )1415 1516 func TestYtConfigDefaultsAndMix(t *testing.T) {@@ -279,7 +280,7 @@ func TestYouTubeTakesInEdits(t *testing.T) {
279280 if err := os.WriteFile(cfg, []byte(`{"channels": ["@a"]}`), 0o644); err != nil {280281 t.Fatal(err)281282 }282- s := &ytSource{URL: ytVideosURL("@a"), ID: "UCa", Name: "A", Listed: now, Checked: now, Dated: true}283+ s := &ytSource{URL: youtube.VideosURL("@a"), ID: "UCa", Name: "A", Listed: now, Checked: now, Dated: true}283284 for j := range 300 {284285 s.Videos = append(s.Videos, &ytVideo{ID: fmt.Sprintf("v%03d", j), Title: fmt.Sprintf("Video %d", j),285286 Seconds: 3600, Published: now.AddDate(0, 0, -3*j), Approx: true, Looked: now})@@ -318,7 +319,7 @@ func maxAgeChannel(t *testing.T, now time.Time, cfg string) *YouTube {
318319 if err := os.WriteFile(filepath.Join(dir, youtubeConfig), []byte(cfg), 0o644); err != nil {319320 t.Fatal(err)320321 }321- s := &ytSource{URL: ytVideosURL("@a"), ID: "UCa", Name: "A", Listed: now, Checked: now, Dated: true}322+ s := &ytSource{URL: youtube.VideosURL("@a"), ID: "UCa", Name: "A", Listed: now, Checked: now, Dated: true}322323 add := func(id string, published time.Time, approx bool) {323324 s.Videos = append(s.Videos, &ytVideo{ID: id, Title: "Video " + id, Seconds: 1800,324325 Published: published, Approx: approx, Looked: now})@@ -536,11 +537,11 @@ func TestYtAgingOutKeepsThePlan(t *testing.T) {
536537 // app turns captions on for a Spanish video.537538 func TestYtAudioLangAndCaptions(t *testing.T) {538539 now := time.Date(2026, 10, 3, 18, 0, 0, 0, time.UTC)539- es, err := parseYtInfo(readTestdata(t, "yt_video_es.json"))540+ es, err := youtube.ParseInfo(readTestdata(t, "yt_video_es.json"))540541 if err != nil {541542 t.Fatal(err)542543 }543- en, err := parseYtInfo(readTestdata(t, "yt_video.json"))544+ en, err := youtube.ParseInfo(readTestdata(t, "yt_video.json"))544545 if err != nil {545546 t.Fatal(err)546547 }@@ -4,6 +4,8 @@ import (
44 "path/filepath"55 "sync"66 "time"7+8+ "airwaves/internal/youtube"79 )810 911 // YouTubeConfig is a YouTube channel's youtube.json, with the settings it@@ -79,13 +81,13 @@ func (y *YouTube) Status() YouTubeStatus {
7981 st.Sources = append(st.Sources, src)8082 }8183 if y.isPlaylist() {82- url, _ := YouTubePlaylist(y.cfg.Playlist)84+ url, _ := youtube.Playlist(y.cfg.Playlist)8385 if s := y.source(url); s != nil {8486 seq, _ := y.sequence()8587 src := YouTubeSource{Channel: y.cfg.Playlist, URL: s.URL, ID: s.ID, Name: s.Name, Videos: len(s.Videos),8688 Playable: len(seq), Owner: s.Owner, OwnerID: s.OwnerID, Listed: s.Listed, Listing: y.listDue(s, now)}8789 if len(seq) > 0 {88- src.Image = ytThumbnail(seq[0].ID)90+ src.Image = youtube.Thumbnail(seq[0].ID)8991 }9092 add(src)9193 }@@ -93,7 +95,7 @@ func (y *YouTube) Status() YouTubeStatus {
9395 }9496 seen := map[string]bool{}9597 for _, c := range y.cfg.Channels {96- s := y.source(ytVideosURL(c))98+ s := y.source(youtube.VideosURL(c))9799 if s == nil || seen[s.URL] {98100 continue99101 }@@ -0,0 +1,421 @@
1+package youtube2+3+import (4+ "cmp"5+ "context"6+ "encoding/json"7+ "encoding/xml"8+ "errors"9+ "math"10+ "net/url"11+ "regexp"12+ "strconv"13+ "strings"14+ "time"15+)16+17+// VideosURL turns a channel, as its link, "@handle" or "UC…" ID, into its18+// uploads tab.19+func VideosURL(s string) string {20+ s = strings.TrimSpace(s)21+ switch {22+ case strings.HasPrefix(s, "@"):23+ s = "https://www.youtube.com/" + s24+ case strings.HasPrefix(s, "UC") && !strings.Contains(s, "/"):25+ s = "https://www.youtube.com/channel/" + s26+ case !strings.Contains(s, "://"):27+ s = "https://" + s28+ }29+ s = strings.TrimRight(s, "/")30+ for _, tab := range []string{"/videos", "/featured", "/streams", "/shorts", "/playlists", "/about"} {31+ s = strings.TrimSuffix(s, tab)32+ }33+ return s + "/videos"34+}35+36+// FeedURL is a channel's feed of its newest 15 uploads.37+func FeedURL(channelID string) string {38+ return "https://www.youtube.com/feeds/videos.xml?channel_id=" + channelID39+}40+41+// WatchURL is a video's page.42+func WatchURL(id string) string { return "https://www.youtube.com/watch?v=" + id }43+44+// Thumbnail is a video's thumbnail, 480x360, which every video has.45+func Thumbnail(id string) string { return "https://i.ytimg.com/vi/" + id + "/hqdefault.jpg" }46+47+// Listing is a channel's tab or a playlist, from yt-dlp --flat-playlist48+// -J with ListingArgs or PlaylistArgs.49+type Listing struct {50+ ID, Name string51+ Entries []Entry // a channel's newest first; a playlist's in its order52+ // Owner and OwnerID are a playlist's channel (see ParsePlaylist).53+ Owner, OwnerID string54+}55+56+// Entry is a video in a listing.57+type Entry struct {58+ ID, Title string59+ Seconds int // 0 when the listing doesn't say60+ // Published is the upload date, roughly, from YouTube's "3 years61+ // ago"; zero when the listing doesn't say.62+ Published time.Time63+ // Skip is "upcoming" for a premiere or a stream to come, "live" for64+ // one on now or not yet processed (and in a channel's tab, one that65+ // was live), else "".66+ Skip string67+ // Channel is a playlist's video's channel, which may not be the68+ // playlist's.69+ Channel string70+}71+72+// ListingArgs list a channel's tab with approximate upload dates (from73+// YouTube's "3 years ago"; no extra requests).74+var ListingArgs = []string{"--flat-playlist", "-J", "--extractor-args", "youtubetab:approximate_date"}75+76+// PlaylistArgs list a playlist: its videos in order, with their lengths.77+var PlaylistArgs = []string{"--flat-playlist", "-J"}78+79+// ParseListing reads a channel tab's listing.80+func ParseListing(raw []byte) (Listing, error) {81+ var p struct {82+ Channel string `json:"channel"`83+ ChannelID string `json:"channel_id"`84+ Entries []struct {85+ ID string `json:"id"`86+ Title string `json:"title"`87+ Duration float64 `json:"duration"`88+ Timestamp float64 `json:"timestamp"`89+ LiveStatus string `json:"live_status"`90+ } `json:"entries"`91+ }92+ if err := json.Unmarshal(raw, &p); err != nil {93+ return Listing{}, err94+ }95+ if p.ChannelID == "" {96+ return Listing{}, errors.New("not a YouTube channel")97+ }98+ l := Listing{ID: p.ChannelID, Name: p.Channel}99+ for _, e := range p.Entries {100+ if e.ID == "" {101+ continue102+ }103+ v := Entry{ID: e.ID, Title: e.Title, Seconds: int(math.Round(e.Duration))}104+ if e.Timestamp > 0 {105+ v.Published = time.Unix(int64(e.Timestamp), 0).UTC()106+ }107+ switch e.LiveStatus {108+ case "is_upcoming":109+ v.Skip = "upcoming"110+ case "is_live", "post_live", "was_live":111+ v.Skip = "live"112+ }113+ l.Entries = append(l.Entries, v)114+ }115+ return l, nil116+}117+118+var (119+ // hidden are the titles YouTube lists videos that can't play under.120+ hidden = map[string]bool{"[Private video]": true, "[Deleted video]": true, "[Unavailable video]": true}121+ // locked are the availabilities of videos that only some can watch.122+ locked = map[string]bool{"private": true, "premium_only": true, "subscriber_only": true, "needs_auth": true}123+)124+125+// ParsePlaylist reads a playlist's listing: its videos in its order, each126+// once. Videos that can't play here (private, deleted, members only) are127+// left out; live and upcoming ones are kept, marked, to join once they128+// can play.129+func ParsePlaylist(raw []byte) (Listing, error) {130+ var p struct {131+ Type string `json:"_type"`132+ ID string `json:"id"`133+ Title string `json:"title"`134+ Channel string `json:"channel"`135+ ChannelID string `json:"channel_id"`136+ Uploader string `json:"uploader"`137+ Entries []struct {138+ ID string `json:"id"`139+ Title string `json:"title"`140+ Duration float64 `json:"duration"`141+ Timestamp float64 `json:"timestamp"`142+ Channel string `json:"channel"`143+ Uploader string `json:"uploader"`144+ LiveStatus string `json:"live_status"`145+ Availability string `json:"availability"`146+ } `json:"entries"`147+ }148+ if err := json.Unmarshal(raw, &p); err != nil {149+ return Listing{}, err150+ }151+ if p.Type != "playlist" || p.ID == "" || p.ID == p.ChannelID { // not a channel's tab152+ return Listing{}, errors.New("not a YouTube playlist")153+ }154+ l := Listing{ID: p.ID, Name: p.Title, Owner: cmp.Or(p.Channel, p.Uploader), OwnerID: p.ChannelID}155+ seen := map[string]bool{}156+ for _, e := range p.Entries {157+ if e.ID == "" || seen[e.ID] || hidden[e.Title] || locked[e.Availability] {158+ continue159+ }160+ seen[e.ID] = true161+ v := Entry{ID: e.ID, Title: e.Title, Seconds: int(math.Round(e.Duration)), Channel: cmp.Or(e.Channel, e.Uploader)}162+ if e.Timestamp > 0 {163+ v.Published = time.Unix(int64(e.Timestamp), 0).UTC()164+ }165+ switch e.LiveStatus {166+ case "is_upcoming":167+ v.Skip = "upcoming"168+ case "is_live", "post_live":169+ v.Skip = "live"170+ }171+ l.Entries = append(l.Entries, v)172+ }173+ return l, nil174+}175+176+var (177+ errNotPlaylist = errors.New("that isn't a YouTube playlist: use its link (youtube.com/playlist?list=...) or its ID")178+ errVideoNotPlaylist = errors.New("that's a link to a video: use the playlist's link, with list= in it")179+ errMix = errors.New("that's one of YouTube's mixes, which change as they play: use a playlist's link")180+ // listID is a playlist's ID in a link, and bareList one given on its181+ // own: "PL…", a channel's uploads ("UU…") or an album ("OLAK5uy_…").182+ listID = regexp.MustCompile(`^[A-Za-z0-9_-]{2,64}$`)183+ bareList = regexp.MustCompile(`^(PL|UU|UL|FL|OLAK5uy_)[A-Za-z0-9_-]{10,62}$`)184+)185+186+// Playlist checks a playlist given as its link (or the link of a video in187+// it) or its ID, and returns its page: https://www.youtube.com/playlist?list=188+// and the ID.189+func Playlist(s string) (string, error) {190+ s = strings.TrimSpace(s)191+ id := s192+ if !bareList.MatchString(s) {193+ raw := s194+ if !strings.Contains(s, "://") {195+ raw = "https://" + s196+ }197+ u, err := url.Parse(raw)198+ if err != nil || (u.Scheme != "http" && u.Scheme != "https") {199+ return "", errNotPlaylist200+ }201+ host := strings.ToLower(u.Hostname())202+ for _, p := range []string{"www.", "m.", "music."} {203+ host = strings.TrimPrefix(host, p)204+ }205+ if host != "youtube.com" && host != "youtu.be" {206+ return "", errNotPlaylist207+ }208+ id = u.Query().Get("list")209+ switch {210+ case id != "":211+ case host == "youtu.be", u.Path == "/watch", strings.HasPrefix(u.Path, "/shorts/"), strings.HasPrefix(u.Path, "/live/"):212+ return "", errVideoNotPlaylist213+ default:214+ return "", errNotPlaylist215+ }216+ }217+ switch {218+ case !listID.MatchString(id):219+ return "", errNotPlaylist220+ case strings.HasPrefix(id, "RD"):221+ return "", errMix222+ }223+ return PlaylistURL(id), nil224+}225+226+// PlaylistURL is a playlist's page.227+func PlaylistURL(id string) string { return "https://www.youtube.com/playlist?list=" + id }228+229+// PlaylistInfo is a playlist as a lookup finds it.230+type PlaylistInfo struct {231+ ID, URL string232+ Title string233+ Owner, OwnerID string // the channel that made it234+ // Videos is how many it has, as YouTube counts them, ones that can't235+ // play included.236+ Videos int237+ Image string // its first video's thumbnail238+}239+240+// LookupPlaylist finds the playlist at page (as Playlist gives it): its241+// title, whose it is, and how many videos it has.242+func (t *YtDlp) LookupPlaylist(ctx context.Context, page string) (PlaylistInfo, error) {243+ raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", "1", page)244+ if err != nil {245+ return PlaylistInfo{}, err246+ }247+ l, err := ParsePlaylist(raw)248+ if err != nil {249+ return PlaylistInfo{}, err250+ }251+ var p struct {252+ Count int `json:"playlist_count"`253+ }254+ _ = json.Unmarshal(raw, &p)255+ info := PlaylistInfo{ID: l.ID, URL: PlaylistURL(l.ID), Title: l.Name, Owner: l.Owner, OwnerID: l.OwnerID, Videos: p.Count}256+ if len(l.Entries) > 0 {257+ info.Image = Thumbnail(l.Entries[0].ID)258+ }259+ return info, nil260+}261+262+// FeedEntry is an upload in a channel's feed: the newest 15, Shorts and263+// streams included, without lengths.264+type FeedEntry struct {265+ ID string `xml:"videoId"`266+ Title string `xml:"title"`267+ Published time.Time `xml:"published"`268+ Description string `xml:"group>description"`269+ Link struct {270+ Href string `xml:"href,attr"`271+ } `xml:"link"`272+}273+274+// Short reports whether the entry is a Short, by its link.275+func (e FeedEntry) Short() bool { return strings.Contains(e.Link.Href, "/shorts/") }276+277+// ParseFeed reads a channel's feed.278+func ParseFeed(raw []byte) ([]FeedEntry, error) {279+ var f struct {280+ Entries []FeedEntry `xml:"entry"`281+ }282+ err := xml.Unmarshal(raw, &f)283+ return f.Entries, err284+}285+286+// Channel is a YouTube channel as a search or a lookup finds it.287+type Channel struct {288+ ID string // "UC…"289+ URL string // its page: by handle, when it has one290+ Handle string // "@name"; some channels have none291+ Name string292+ Subscribers int64293+ Description string294+ Image string // its avatar295+ // Videos is roughly how many videos it has uploaded, Shorts and296+ // streams included; 0 when unknown. Only lookups count them.297+ Videos int298+}299+300+// channelsOnly narrows YouTube's search results to channels.301+const channelsOnly = "EgIQAg%3D%3D"302+303+// SearchChannels asks YouTube for the channels matching query, n at most.304+// YouTube's Topic channels, which it makes for artists' music, are left305+// out: they have no uploads to play.306+func (t *YtDlp) SearchChannels(ctx context.Context, query string, n int) ([]Channel, error) {307+ u := "https://www.youtube.com/results?search_query=" + url.QueryEscape(query) + "&sp=" + channelsOnly308+ raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", strconv.Itoa(n), u)309+ if err != nil {310+ return nil, err311+ }312+ var p struct {313+ Entries []channelInfo `json:"entries"`314+ }315+ if err := json.Unmarshal(raw, &p); err != nil {316+ return nil, err317+ }318+ out := []Channel{}319+ for _, e := range p.Entries {320+ c := e.channel()321+ if !strings.HasPrefix(c.ID, "UC") || c.Handle == "" && strings.HasSuffix(c.Name, " - Topic") {322+ continue323+ }324+ out = append(out, c)325+ }326+ return out, nil327+}328+329+// LookupChannel finds the channel at page (its link, "@handle" or "UC…"330+// ID), and roughly how many videos it has.331+func (t *YtDlp) LookupChannel(ctx context.Context, page string) (Channel, error) {332+ raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", "1", VideosURL(page))333+ if err != nil {334+ return Channel{}, err335+ }336+ var e channelInfo337+ if err := json.Unmarshal(raw, &e); err != nil {338+ return Channel{}, err339+ }340+ c := e.channel()341+ if !strings.HasPrefix(c.ID, "UC") {342+ return Channel{}, errors.New("not a YouTube channel")343+ }344+ // The channel's uploads playlist says how many there are; its tab345+ // doesn't.346+ if raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", "1",347+ "https://www.youtube.com/playlist?list=UU"+c.ID[2:]); err == nil {348+ var p struct {349+ Count int `json:"playlist_count"`350+ }351+ if json.Unmarshal(raw, &p) == nil {352+ c.Videos = p.Count353+ }354+ }355+ return c, nil356+}357+358+// channelInfo is a channel in yt-dlp's output: a search result, or the359+// top of a channel's tab.360+type channelInfo struct {361+ ChannelID string `json:"channel_id"`362+ Channel string `json:"channel"`363+ Title string `json:"title"`364+ UploaderID string `json:"uploader_id"`365+ Followers float64 `json:"channel_follower_count"`366+ Description string `json:"description"`367+ Thumbnails []struct {368+ URL string `json:"url"`369+ Width int `json:"width"`370+ Height int `json:"height"`371+ ID string `json:"id"`372+ } `json:"thumbnails"`373+}374+375+func (e channelInfo) channel() Channel {376+ c := Channel{ID: e.ChannelID, Name: cmp.Or(e.Channel, e.Title), Subscribers: int64(e.Followers),377+ Description: Blurb(e.Description)}378+ switch {379+ case strings.HasPrefix(e.UploaderID, "@"):380+ c.Handle = e.UploaderID381+ c.URL = "https://www.youtube.com/" + c.Handle382+ case c.ID != "":383+ c.URL = "https://www.youtube.com/channel/" + c.ID384+ }385+ // The avatar is the largest square picture (the others are banners),386+ // asked for at a size to show.387+ size := 0388+ for _, th := range e.Thumbnails {389+ if th.Width > size && th.Width == th.Height {390+ c.Image, size = th.URL, th.Width391+ }392+ }393+ for _, th := range e.Thumbnails {394+ if c.Image == "" && th.ID == "avatar_uncropped" {395+ c.Image = th.URL396+ }397+ }398+ if strings.HasPrefix(c.Image, "//") {399+ c.Image = "https:" + c.Image400+ }401+ if i := strings.LastIndex(c.Image, "="); i > 0 {402+ c.Image = c.Image[:i] + imageSize.ReplaceAllString(c.Image[i:], "=s176")403+ }404+ return c405+}406+407+// imageSize is the size option of a YouTube image address ("=s900-c-k").408+var imageSize = regexp.MustCompile(`^=s\d+`)409+410+// Blurb makes a channel's description one short paragraph. Search411+// results come cut short with "...".412+func Blurb(s string) string {413+ s = strings.Join(strings.Fields(s), " ")414+ if t, ok := strings.CutSuffix(s, " ..."); ok {415+ s = t + "…"416+ }417+ if r := []rune(s); len(r) > 300 {418+ s = strings.TrimSpace(string(r[:299])) + "…"419+ }420+ return s421+}@@ -0,0 +1,325 @@
1+package youtube2+3+import (4+ "context"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "net/http"10+ "net/url"11+ "os/exec"12+ "regexp"13+ "slices"14+ "sort"15+ "strconv"16+ "strings"17+ "sync"18+ "time"19+)20+21+// Info is a video's details from yt-dlp -J, and with a format chosen,22+// where to stream it from.23+type Info struct {24+ ID string `json:"id"`25+ Title string `json:"title"`26+ Duration float64 `json:"duration"`27+ Timestamp float64 `json:"timestamp"`28+ Description string `json:"description"`29+ Language string `json:"language"`30+ LiveStatus string `json:"live_status"`31+ MediaType string `json:"media_type"`32+ FormatID string `json:"format_id"`33+ Format34+ RequestedFormats []Format `json:"requested_formats"`35+ // Subtitles are the uploader's, AutoCaptions YouTube's speech36+ // recognition, by language ("en", "en-US", "en-orig").37+ Subtitles map[string][]Subtitle `json:"subtitles"`38+ AutoCaptions map[string][]Subtitle `json:"automatic_captions"`39+ // Thumbnails are its pictures, largest last as yt-dlp lists them.40+ Thumbnails []struct {41+ URL string `json:"url"`42+ Width int `json:"width"`43+ Height int `json:"height"`44+ } `json:"thumbnails"`45+}46+47+// Format is a stream of a video: its picture, its sound, or both.48+type Format struct {49+ URL string `json:"url"`50+ VCodec string `json:"vcodec"`51+ ACodec string `json:"acodec"`52+ Height int `json:"height"`53+ Language string `json:"language"`54+ HTTPHeaders map[string]string `json:"http_headers"`55+}56+57+// Subtitle is a subtitle track in one format.58+type Subtitle struct {59+ Ext string `json:"ext"`60+ URL string `json:"url"`61+ Protocol string `json:"protocol"` // "m3u8_native" for a playlist of pieces62+}63+64+// ParseInfo reads yt-dlp -J's details of a video.65+func ParseInfo(raw []byte) (Info, error) {66+ var i Info67+ if err := json.Unmarshal(raw, &i); err != nil {68+ return i, err69+ }70+ if i.ID == "" {71+ return i, errors.New("no video in yt-dlp's output")72+ }73+ return i, nil74+}75+76+// Inputs are the streams to play a resolved video from, picture first.77+func (i Info) Inputs() []Format {78+ fs := i.RequestedFormats79+ if len(fs) == 0 && i.URL != "" {80+ fs = []Format{i.Format}81+ }82+ out := make([]Format, 0, len(fs))83+ for _, f := range fs {84+ if f.VCodec != "none" && f.VCodec != "" {85+ out = append(out, f)86+ }87+ }88+ for _, f := range fs {89+ if f.VCodec == "none" || f.VCodec == "" {90+ out = append(out, f)91+ }92+ }93+ return out94+}95+96+// AudioLang is the sound's language, "" when YouTube doesn't say.97+func (i Info) AudioLang() string {98+ for _, f := range i.Inputs() {99+ if f.ACodec != "none" && f.ACodec != "" && f.Language != "" {100+ return f.Language101+ }102+ }103+ return i.Language104+}105+106+// Captions picks the video's English captions: the uploader's, else107+// YouTube's automatic ones, as json3, which times each word of automatic108+// captions, else WebVTT or SRT. ok is false when there are none.109+func (i Info) Captions() (sub Subtitle, ok bool) {110+ pick := func(tracks map[string][]Subtitle, langs ...string) (Subtitle, bool) {111+ for _, l := range langs {112+ for _, ext := range []string{"json3", "vtt", "srt"} {113+ for _, s := range tracks[l] {114+ if s.Ext == ext && s.URL != "" {115+ return s, true116+ }117+ }118+ }119+ }120+ return Subtitle{}, false121+ }122+ var others []string123+ for l := range i.Subtitles {124+ if strings.HasPrefix(l, "en-") {125+ others = append(others, l)126+ }127+ }128+ slices.Sort(others)129+ if s, ok := pick(i.Subtitles, append([]string{"en"}, others...)...); ok {130+ return s, true131+ }132+ return pick(i.AutoCaptions, "en-orig", "en")133+}134+135+// FormatSpec picks H.264 picture up to height and AAC sound, English136+// sound of a video dubbed in several languages, else the best single file137+// up to height.138+func FormatSpec(height int) string {139+ return fmt.Sprintf("bv*[height<=%[1]d][vcodec^=avc1]+ba[ext=m4a][language^=en]/bv*[height<=%[1]d][vcodec^=avc1]+ba[ext=m4a]/b[height<=%[1]d]", height)140+}141+142+// InputArgs are ffmpeg's inputs for a resolved video, played from offset143+// in: picture first, with yt-dlp's headers, reconnecting when a144+// connection drops. A stream that fails for good stops ffmpeg (-xerror)145+// rather than ending the video early, so it can be reopened. chunked146+// fetches in 10 MB ranges as yt-dlp does, which YouTube serves without147+// throttling (see HTTPChunks). audio is the sound's stream specifier.148+func InputArgs(fs []Format, offset time.Duration, chunked bool) (args []string, audio string) {149+ args = []string{"-xerror"}150+ for i, f := range fs {151+ args = append(args, "-reconnect", "1", "-reconnect_streamed", "1", "-reconnect_on_network_error", "1",152+ "-reconnect_delay_max", "4", "-rw_timeout", "15000000")153+ if chunked {154+ args = append(args, "-request_size", "10485760", "-multiple_requests", "1")155+ }156+ if len(f.HTTPHeaders) > 0 {157+ var h strings.Builder158+ for _, k := range sortedKeys(f.HTTPHeaders) {159+ h.WriteString(k + ": " + f.HTTPHeaders[k] + "\r\n")160+ }161+ args = append(args, "-headers", h.String())162+ }163+ if offset > 0 {164+ args = append(args, "-ss", strconv.FormatFloat(offset.Seconds(), 'f', 3, 64))165+ }166+ args = append(args, "-i", f.URL)167+ if audio == "" && f.ACodec != "none" && f.ACodec != "" {168+ audio = fmt.Sprintf("%d:a:0", i)169+ }170+ }171+ return args, audio172+}173+174+func sortedKeys(m map[string]string) []string {175+ keys := make([]string, 0, len(m))176+ for k := range m {177+ keys = append(keys, k)178+ }179+ sort.Strings(keys)180+ return keys181+}182+183+var chunks sync.Map // ffmpeg path: bool184+185+// HTTPChunks reports whether ffmpeg can fetch HTTP in ranges186+// (-request_size, FFmpeg 8).187+func HTTPChunks(ffmpeg string) bool {188+ if ok, found := chunks.Load(ffmpeg); found {189+ return ok.(bool)190+ }191+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)192+ defer cancel()193+ out, _ := exec.CommandContext(ctx, ffmpeg, "-hide_banner", "-h", "protocol=http").Output()194+ ok := strings.Contains(string(out), "-request_size")195+ chunks.Store(ffmpeg, ok)196+ return ok197+}198+199+// AddressesUntil is when a video's addresses, found at, may stop working:200+// ten minutes before the earliest "expire" they carry (YouTube's last six201+// hours), or an hour after they were found when they carry none.202+func AddressesUntil(info Info, at time.Time) time.Time {203+ until := at.Add(time.Hour)204+ var earliest time.Time205+ for _, f := range info.Inputs() {206+ u, err := url.Parse(f.URL)207+ if err != nil {208+ return until209+ }210+ sec, err := strconv.ParseInt(u.Query().Get("expire"), 10, 64)211+ if err != nil || sec <= 0 {212+ return until213+ }214+ if t := time.Unix(sec, 0); earliest.IsZero() || t.Before(earliest) {215+ earliest = t216+ }217+ }218+ if earliest.IsZero() {219+ return until220+ }221+ if latest := at.Add(6 * time.Hour); earliest.After(latest) {222+ return latest223+ }224+ return earliest.Add(-10 * time.Minute)225+}226+227+// FetchCaptions fetches a caption track: a json3, WebVTT or SRT file, or a228+// playlist of WebVTT pieces, joined.229+func FetchCaptions(ctx context.Context, client *http.Client, sub Subtitle) ([]byte, error) {230+ raw, err := get(ctx, client, sub.URL)231+ if err != nil || !strings.HasPrefix(sub.Protocol, "m3u8") {232+ return raw, err233+ }234+ base, err := url.Parse(sub.URL)235+ if err != nil {236+ return nil, errors.New("captions: bad address")237+ }238+ var all []byte239+ pieces := 0240+ for line := range strings.Lines(string(raw)) {241+ line = strings.TrimSpace(line)242+ if line == "" || strings.HasPrefix(line, "#") {243+ continue244+ }245+ if pieces++; pieces > 500 {246+ break247+ }248+ u, err := base.Parse(line)249+ if err != nil {250+ return nil, errors.New("captions: bad address")251+ }252+ piece, err := get(ctx, client, u.String())253+ if err != nil {254+ return nil, err255+ }256+ all = append(append(all, piece...), "\n\n"...)257+ }258+ return all, nil259+}260+261+// get fetches captions, leaving their address, which is signed, out of262+// errors.263+func get(ctx context.Context, client *http.Client, address string) ([]byte, error) {264+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil)265+ if err != nil {266+ return nil, errors.New("captions: bad address")267+ }268+ resp, err := client.Do(req)269+ if uerr := (*url.Error)(nil); errors.As(err, &uerr) {270+ return nil, fmt.Errorf("captions: %w", uerr.Err)271+ }272+ if err != nil {273+ return nil, fmt.Errorf("captions: %w", err)274+ }275+ defer resp.Body.Close()276+ if resp.StatusCode != http.StatusOK {277+ return nil, fmt.Errorf("captions: HTTP %d", resp.StatusCode)278+ }279+ return io.ReadAll(io.LimitReader(resp.Body, 16<<20))280+}281+282+var (283+ separator = regexp.MustCompile(`(?m)^\s*[=\-_*~#]{5,}\s*$`)284+ label = regexp.MustCompile(`^.{0,40}:$`)285+)286+287+// Description makes a short description of a YouTube description: its288+// first paragraph that says something, before the links section, without289+// addresses or the labels left of them ("Game on Steam:").290+func Description(s string) string {291+ if i := separator.FindStringIndex(s); i != nil {292+ s = s[:i[0]]293+ }294+ for para := range strings.SplitSeq(strings.ReplaceAll(s, "\r", ""), "\n\n") {295+ var keep []string296+ for line := range strings.SplitSeq(para, "\n") {297+ line = strings.TrimSpace(urls.ReplaceAllString(line, ""))298+ if line != "" && !label.MatchString(line) {299+ keep = append(keep, strings.TrimSuffix(line, ":"))300+ }301+ }302+ if len(keep) > 0 {303+ return firstParagraph(strings.Join(keep, "\n"), 300)304+ }305+ }306+ return ""307+}308+309+// firstParagraph is s's first paragraph on one line, cut at a word to n310+// characters at most.311+func firstParagraph(s string, n int) string {312+ s = strings.TrimSpace(s)313+ if i := strings.Index(s, "\n\n"); i >= 0 {314+ s = s[:i]315+ }316+ s = strings.Join(strings.Fields(s), " ")317+ if r := []rune(s); len(r) > n {318+ cut := string(r[:n])319+ if i := strings.LastIndex(cut, " "); i > n/2 {320+ cut = cut[:i]321+ }322+ s = cut + "…"323+ }324+ return s325+}@@ -1,4 +1,9 @@
1-package vchan1+// Package youtube is YouTube as a source: yt-dlp, kept up to date, to2+// list channels and playlists, find videos' details and where to stream3+// them from, and search for channels; and YouTube's feeds. The custom4+// channels (internal/vchan) and the podcasts (internal/podcast) both use5+// it.6+package youtube27 38 import (49 "bytes"@@ -15,12 +20,13 @@ import (
1520 "os/exec"1621 "path"1722 "path/filepath"23+ "regexp"1824 "strings"1925 "sync"2026 "time"2127 )2228 23-// YtDlp runs yt-dlp for YouTube channels. With Update, it keeps its own29+// YtDlp runs yt-dlp. With Update, it keeps its own2430 // copy at Path: the latest release is downloaded on first use and replaced2531 // about daily, so YouTube's changes don't wait for a new image. The release2632 // is a Python zipapp, so python3 must be installed, and YouTube needs a@@ -32,11 +38,14 @@ type YtDlp struct {
3238 // when nil.3339 HTTP *http.Client3440 41+ // Jobs is held for a listing job, one at a time across its users;42+ // Ahead while finding where to stream a video from ahead of time,43+ // likewise.44+ Jobs, Ahead sync.Mutex45+3546 mu sync.Mutex3647 checked time.Time // the last install or update check3748 updating bool38- jobs sync.Mutex // one catalog job at a time, across channels39- ahead sync.Mutex // one video found ahead of time at a time, likewise4049 }4150 4251 const (@@ -44,7 +53,8 @@ const (
4453 ytdlpUpdateEvery = 24 * time.Hour4554 )4655 47-func (t *YtDlp) client() *http.Client { return cmp.Or(t.HTTP, http.DefaultClient) }56+// Client is the HTTP client t reads with, for feeds and captions.57+func (t *YtDlp) Client() *http.Client { return cmp.Or(t.HTTP, http.DefaultClient) }4858 4959 // command returns the yt-dlp to run, installing it first when it's5060 // missing, and starts the daily update when one is due.@@ -138,7 +148,7 @@ func (t *YtDlp) latest(ctx context.Context) (string, error) {
138148 if err != nil {139149 return "", err140150 }141- c := *t.client()151+ c := *t.Client()142152 c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }143153 resp, err := c.Do(req)144154 if err != nil {@@ -157,7 +167,7 @@ func (t *YtDlp) get(ctx context.Context, url string) ([]byte, error) {
157167 if err != nil {158168 return nil, err159169 }160- resp, err := t.client().Do(req)170+ resp, err := t.Client().Do(req)161171 if err != nil {162172 return nil, err163173 }@@ -175,9 +185,9 @@ func ytdlpVersion(ctx context.Context, p string) (string, error) {
175185 return strings.TrimSpace(string(out)), err176186 }177187 178-// run runs yt-dlp with args and returns what it writes to stdout. Errors188+// Run runs yt-dlp with args and returns what it writes to stdout. Errors179189 // carry yt-dlp's own error message, without addresses.180-func (t *YtDlp) run(ctx context.Context, args ...string) ([]byte, error) {190+func (t *YtDlp) Run(ctx context.Context, args ...string) ([]byte, error) {181191 p, err := t.command(ctx)182192 if err != nil {183193 return nil, err@@ -195,17 +205,18 @@ func (t *YtDlp) run(ctx context.Context, args ...string) ([]byte, error) {
195205 if ctx.Err() != nil {196206 return nil, ctx.Err()197207 }198- return nil, &ytdlpError{msg: ytdlpMessage(stderr.String(), err)}208+ return nil, &Error{msg: message(stderr.String(), err)}199209 }200210 return out, nil201211 }202212 203-type ytdlpError struct{ msg string }213+// Error is yt-dlp's error message.214+type Error struct{ msg string }204215 205-func (e *ytdlpError) Error() string { return "yt-dlp: " + e.msg }216+func (e *Error) Error() string { return "yt-dlp: " + e.msg }206217 207-// ytdlpMessage picks yt-dlp's error line out of its output.208-func ytdlpMessage(stderr string, err error) string {218+// message picks yt-dlp's error line out of its output.219+func message(stderr string, err error) string {209220 msg := ""210221 for line := range strings.Lines(stderr) {211222 if s, ok := strings.CutPrefix(strings.TrimSpace(line), "ERROR: "); ok {@@ -227,10 +238,10 @@ func ytdlpMessage(stderr string, err error) string {
227238 return msg228239 }229240 230-// gone reports whether a yt-dlp error says the video can't be played241+// Gone reports whether a yt-dlp error says the video can't be played231242 // here at all, rather than for now.232-func gone(err error) bool {233- var e *ytdlpError243+func Gone(err error) bool {244+ var e *Error234245 if !errors.As(err, &e) {235246 return false236247 }@@ -246,3 +257,53 @@ func gone(err error) bool {
246257 }247258 return false248259 }260+261+// urls are addresses in text, left out of errors and descriptions.262+var urls = regexp.MustCompile(`https?://\S+`)263+264+// ErrInstalling is Ready's answer while yt-dlp is being installed.265+var ErrInstalling = errors.New("yt-dlp is being installed; try again in a minute")266+267+// installing is Ready's install of a managed yt-dlp.268+type installing struct {269+ mu sync.Mutex270+ running bool271+ err error // the last try's272+}273+274+var installs sync.Map // *YtDlp: *installing275+276+// Ready reports whether yt-dlp is there to run. When t keeps its own copy277+// and hasn't installed it yet, Ready starts installing it and reports278+// ErrInstalling, or why the last try failed.279+func (t *YtDlp) Ready() error {280+ if !t.Update {281+ if _, err := exec.LookPath(cmp.Or(t.Path, "yt-dlp")); err != nil {282+ return fmt.Errorf("yt-dlp isn't installed on the server: %w", err)283+ }284+ return nil285+ }286+ if _, err := os.Stat(t.Path); err == nil {287+ return nil288+ }289+ v, _ := installs.LoadOrStore(t, &installing{})290+ in := v.(*installing)291+ in.mu.Lock()292+ defer in.mu.Unlock()293+ failed := in.err294+ if !in.running {295+ in.running = true296+ go func() {297+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)298+ defer cancel()299+ _, err := t.command(ctx)300+ in.mu.Lock()301+ in.running, in.err = false, err302+ in.mu.Unlock()303+ }()304+ }305+ if failed != nil {306+ return fmt.Errorf("couldn't install yt-dlp (%v); trying again", failed)307+ }308+ return ErrInstalling309+}