1package youtube23import (4 "cmp"5 "context"6 "encoding/json"7 "encoding/xml"8 "errors"9 "math"10 "net/url"11 "regexp"12 "strconv"13 "strings"14 "time"15)1617// VideosURL turns a channel, as its link, "@handle" or "UC…" ID, into its18// uploads tab.19func VideosURL(s string) string {20 s = strings.TrimSpace(s)21 switch {22 case strings.HasPrefix(s, "@"):23 s = "https://www.youtube.com/" + s24 case strings.HasPrefix(s, "UC") && !strings.Contains(s, "/"):25 s = "https://www.youtube.com/channel/" + s26 case !strings.Contains(s, "://"):27 s = "https://" + s28 }29 s = strings.TrimRight(s, "/")30 for _, tab := range []string{"/videos", "/featured", "/streams", "/shorts", "/playlists", "/about"} {31 s = strings.TrimSuffix(s, tab)32 }33 return s + "/videos"34}3536// FeedURL is a channel's feed of its newest 15 uploads.37func FeedURL(channelID string) string {38 return "https://www.youtube.com/feeds/videos.xml?channel_id=" + channelID39}4041// WatchURL is a video's page.42func WatchURL(id string) string { return "https://www.youtube.com/watch?v=" + id }4344// Thumbnail is a video's thumbnail, 480x360, which every video has.45func Thumbnail(id string) string { return "https://i.ytimg.com/vi/" + id + "/hqdefault.jpg" }4647// Listing is a channel's tab or a playlist, from yt-dlp --flat-playlist48// -J with ListingArgs or PlaylistArgs.49type Listing struct {50 ID, Name string51 Entries []Entry // a channel's newest first; a playlist's in its order52 // Owner and OwnerID are a playlist's channel (see ParsePlaylist).53 Owner, OwnerID string54 // Image is a channel's avatar, or a playlist's first video's55 // thumbnail; "" when the listing has none.56 Image string57}5859// Entry is a video in a listing.60type Entry struct {61 ID, Title string62 Seconds int // 0 when the listing doesn't say63 // Published is the upload date, roughly, from YouTube's "3 years64 // ago"; zero when the listing doesn't say.65 Published time.Time66 // Skip is "upcoming" for a premiere or a stream to come, "live" for67 // one on now or not yet processed (and in a channel's tab, one that68 // was live), else "". LiveStatus is yt-dlp's own word for it69 // ("is_upcoming", "is_live", "post_live", "was_live", "not_live"), ""70 // when the listing doesn't say.71 Skip, LiveStatus string72 // Channel is a playlist's video's channel, which may not be the73 // playlist's.74 Channel string75}7677// ListingArgs list a channel's tab with approximate upload dates (from78// YouTube's "3 years ago"; no extra requests).79var ListingArgs = []string{"--flat-playlist", "-J", "--extractor-args", "youtubetab:approximate_date"}8081// PlaylistArgs list a playlist: its videos in order, with their lengths.82var PlaylistArgs = []string{"--flat-playlist", "-J"}8384// ParseListing reads a channel tab's listing.85func ParseListing(raw []byte) (Listing, error) {86 var p struct {87 channelInfo88 Entries []struct {89 ID string `json:"id"`90 Title string `json:"title"`91 Duration float64 `json:"duration"`92 Timestamp float64 `json:"timestamp"`93 LiveStatus string `json:"live_status"`94 } `json:"entries"`95 }96 if err := json.Unmarshal(raw, &p); err != nil {97 return Listing{}, err98 }99 if p.ChannelID == "" {100 return Listing{}, errors.New("not a YouTube channel")101 }102 l := Listing{ID: p.ChannelID, Name: strings.TrimSpace(p.Channel), Image: p.channel().Image}103 for _, e := range p.Entries {104 if e.ID == "" {105 continue106 }107 v := Entry{ID: e.ID, Title: e.Title, Seconds: int(math.Round(e.Duration)), LiveStatus: e.LiveStatus}108 if e.Timestamp > 0 {109 v.Published = time.Unix(int64(e.Timestamp), 0).UTC()110 }111 switch e.LiveStatus {112 case "is_upcoming":113 v.Skip = "upcoming"114 case "is_live", "post_live", "was_live":115 v.Skip = "live"116 }117 l.Entries = append(l.Entries, v)118 }119 return l, nil120}121122var (123 // hidden are the titles YouTube lists videos that can't play under.124 hidden = map[string]bool{"[Private video]": true, "[Deleted video]": true, "[Unavailable video]": true}125 // locked are the availabilities of videos that only some can watch.126 locked = map[string]bool{"private": true, "premium_only": true, "subscriber_only": true, "needs_auth": true}127)128129// ParsePlaylist reads a playlist's listing: its videos in its order, each130// once. Videos that can't play here (private, deleted, members only) are131// left out; live and upcoming ones are kept, marked, to join once they132// can play.133func ParsePlaylist(raw []byte) (Listing, error) {134 var p struct {135 Type string `json:"_type"`136 ID string `json:"id"`137 Title string `json:"title"`138 Channel string `json:"channel"`139 ChannelID string `json:"channel_id"`140 Uploader string `json:"uploader"`141 Entries []struct {142 ID string `json:"id"`143 Title string `json:"title"`144 Duration float64 `json:"duration"`145 Timestamp float64 `json:"timestamp"`146 Channel string `json:"channel"`147 Uploader string `json:"uploader"`148 LiveStatus string `json:"live_status"`149 Availability string `json:"availability"`150 } `json:"entries"`151 }152 if err := json.Unmarshal(raw, &p); err != nil {153 return Listing{}, err154 }155 if p.Type != "playlist" || p.ID == "" || p.ID == p.ChannelID { // not a channel's tab156 return Listing{}, errors.New("not a YouTube playlist")157 }158 l := Listing{ID: p.ID, Name: strings.TrimSpace(p.Title), Owner: strings.TrimSpace(cmp.Or(p.Channel, p.Uploader)), OwnerID: p.ChannelID}159 seen := map[string]bool{}160 for _, e := range p.Entries {161 if e.ID == "" || seen[e.ID] || hidden[e.Title] || locked[e.Availability] {162 continue163 }164 seen[e.ID] = true165 v := Entry{ID: e.ID, Title: e.Title, Seconds: int(math.Round(e.Duration)), Channel: cmp.Or(e.Channel, e.Uploader),166 LiveStatus: e.LiveStatus}167 if e.Timestamp > 0 {168 v.Published = time.Unix(int64(e.Timestamp), 0).UTC()169 }170 switch e.LiveStatus {171 case "is_upcoming":172 v.Skip = "upcoming"173 case "is_live", "post_live":174 v.Skip = "live"175 }176 l.Entries = append(l.Entries, v)177 }178 if len(l.Entries) > 0 {179 l.Image = Thumbnail(l.Entries[0].ID)180 }181 return l, nil182}183184var (185 errNotPlaylist = errors.New("that isn't a YouTube playlist: use its link (youtube.com/playlist?list=...) or its ID")186 errVideoNotPlaylist = errors.New("that's a link to a video: use the playlist's link, with list= in it")187 errMix = errors.New("that's one of YouTube's mixes, which change as they play: use a playlist's link")188 // listID is a playlist's ID in a link, and bareList one given on its189 // own: "PL…", a channel's uploads ("UU…") or an album ("OLAK5uy_…").190 listID = regexp.MustCompile(`^[A-Za-z0-9_-]{2,64}$`)191 bareList = regexp.MustCompile(`^(PL|UU|UL|FL|OLAK5uy_)[A-Za-z0-9_-]{10,62}$`)192)193194// Playlist checks a playlist given as its link (or the link of a video in195// it) or its ID, and returns its page: https://www.youtube.com/playlist?list=196// and the ID.197func Playlist(s string) (string, error) {198 s = strings.TrimSpace(s)199 id := s200 if !bareList.MatchString(s) {201 raw := s202 if !strings.Contains(s, "://") {203 raw = "https://" + s204 }205 u, err := url.Parse(raw)206 if err != nil || (u.Scheme != "http" && u.Scheme != "https") {207 return "", errNotPlaylist208 }209 host := strings.ToLower(u.Hostname())210 for _, p := range []string{"www.", "m.", "music."} {211 host = strings.TrimPrefix(host, p)212 }213 if host != "youtube.com" && host != "youtu.be" {214 return "", errNotPlaylist215 }216 id = u.Query().Get("list")217 switch {218 case id != "":219 case host == "youtu.be", u.Path == "/watch", strings.HasPrefix(u.Path, "/shorts/"), strings.HasPrefix(u.Path, "/live/"):220 return "", errVideoNotPlaylist221 default:222 return "", errNotPlaylist223 }224 }225 switch {226 case !listID.MatchString(id):227 return "", errNotPlaylist228 case strings.HasPrefix(id, "RD"):229 return "", errMix230 }231 return PlaylistURL(id), nil232}233234// PlaylistURL is a playlist's page.235func PlaylistURL(id string) string { return "https://www.youtube.com/playlist?list=" + id }236237// PlaylistInfo is a playlist as a lookup finds it.238type PlaylistInfo struct {239 ID, URL string240 Title string241 Owner, OwnerID string // the channel that made it242 // Videos is how many it has, as YouTube counts them, ones that can't243 // play included.244 Videos int245 Image string // its first video's thumbnail246}247248// LookupPlaylist finds the playlist at page (as Playlist gives it): its249// title, whose it is, and how many videos it has.250func (t *YtDlp) LookupPlaylist(ctx context.Context, page string) (PlaylistInfo, error) {251 raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", "1", page)252 if err != nil {253 return PlaylistInfo{}, err254 }255 l, err := ParsePlaylist(raw)256 if err != nil {257 return PlaylistInfo{}, err258 }259 var p struct {260 Count int `json:"playlist_count"`261 }262 _ = json.Unmarshal(raw, &p)263 info := PlaylistInfo{ID: l.ID, URL: PlaylistURL(l.ID), Title: l.Name, Owner: l.Owner, OwnerID: l.OwnerID, Videos: p.Count}264 if len(l.Entries) > 0 {265 info.Image = Thumbnail(l.Entries[0].ID)266 }267 return info, nil268}269270// FeedEntry is an upload in a channel's feed: the newest 15, Shorts and271// streams included, without lengths.272type FeedEntry struct {273 ID string `xml:"videoId"`274 Title string `xml:"title"`275 Published time.Time `xml:"published"`276 Description string `xml:"group>description"`277 Link struct {278 Href string `xml:"href,attr"`279 } `xml:"link"`280}281282// Short reports whether the entry is a Short, by its link.283func (e FeedEntry) Short() bool { return strings.Contains(e.Link.Href, "/shorts/") }284285// ParseFeed reads a channel's feed.286func ParseFeed(raw []byte) ([]FeedEntry, error) {287 var f struct {288 Entries []FeedEntry `xml:"entry"`289 }290 err := xml.Unmarshal(raw, &f)291 return f.Entries, err292}293294// Channel is a YouTube channel as a search or a lookup finds it.295type Channel struct {296 ID string // "UC…"297 URL string // its page: by handle, when it has one298 Handle string // "@name"; some channels have none299 Name string300 Subscribers int64301 Description string302 Image string // its avatar303 // Videos is roughly how many videos it has uploaded, Shorts and304 // streams included; 0 when unknown. Only lookups count them.305 Videos int306}307308// channelsOnly narrows YouTube's search results to channels.309const channelsOnly = "EgIQAg%3D%3D"310311// SearchChannels asks YouTube for the channels matching query, n at most.312// YouTube's Topic channels, which it makes for artists' music, are left313// out: they have no uploads to play.314func (t *YtDlp) SearchChannels(ctx context.Context, query string, n int) ([]Channel, error) {315 u := "https://www.youtube.com/results?search_query=" + url.QueryEscape(query) + "&sp=" + channelsOnly316 raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", strconv.Itoa(n), u)317 if err != nil {318 return nil, err319 }320 var p struct {321 Entries []channelInfo `json:"entries"`322 }323 if err := json.Unmarshal(raw, &p); err != nil {324 return nil, err325 }326 out := []Channel{}327 for _, e := range p.Entries {328 c := e.channel()329 if !strings.HasPrefix(c.ID, "UC") || c.Handle == "" && strings.HasSuffix(c.Name, " - Topic") {330 continue331 }332 out = append(out, c)333 }334 return out, nil335}336337// LookupChannel finds the channel at page (its link, "@handle" or "UC…"338// ID), and roughly how many videos it has.339func (t *YtDlp) LookupChannel(ctx context.Context, page string) (Channel, error) {340 raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", "1", VideosURL(page))341 if err != nil {342 return Channel{}, err343 }344 var e channelInfo345 if err := json.Unmarshal(raw, &e); err != nil {346 return Channel{}, err347 }348 c := e.channel()349 if !strings.HasPrefix(c.ID, "UC") {350 return Channel{}, errors.New("not a YouTube channel")351 }352 // The channel's uploads playlist says how many there are; its tab353 // doesn't.354 if raw, err := t.Run(ctx, "--flat-playlist", "-J", "--playlist-end", "1",355 "https://www.youtube.com/playlist?list=UU"+c.ID[2:]); err == nil {356 var p struct {357 Count int `json:"playlist_count"`358 }359 if json.Unmarshal(raw, &p) == nil {360 c.Videos = p.Count361 }362 }363 return c, nil364}365366// channelInfo is a channel in yt-dlp's output: a search result, or the367// top of a channel's tab.368type channelInfo struct {369 ChannelID string `json:"channel_id"`370 Channel string `json:"channel"`371 Title string `json:"title"`372 UploaderID string `json:"uploader_id"`373 Followers float64 `json:"channel_follower_count"`374 Description string `json:"description"`375 Thumbnails []struct {376 URL string `json:"url"`377 Width int `json:"width"`378 Height int `json:"height"`379 ID string `json:"id"`380 } `json:"thumbnails"`381}382383func (e channelInfo) channel() Channel {384 c := Channel{ID: e.ChannelID, Name: strings.TrimSpace(cmp.Or(e.Channel, e.Title)), Subscribers: int64(e.Followers),385 Description: Blurb(e.Description)}386 switch {387 case strings.HasPrefix(e.UploaderID, "@"):388 c.Handle = e.UploaderID389 c.URL = "https://www.youtube.com/" + c.Handle390 case c.ID != "":391 c.URL = "https://www.youtube.com/channel/" + c.ID392 }393 // The avatar is the largest square picture (the others are banners),394 // asked for at a size to show.395 size := 0396 for _, th := range e.Thumbnails {397 if th.Width > size && th.Width == th.Height {398 c.Image, size = th.URL, th.Width399 }400 }401 for _, th := range e.Thumbnails {402 if c.Image == "" && th.ID == "avatar_uncropped" {403 c.Image = th.URL404 }405 }406 if strings.HasPrefix(c.Image, "//") {407 c.Image = "https:" + c.Image408 }409 if i := strings.LastIndex(c.Image, "="); i > 0 {410 c.Image = c.Image[:i] + imageSize.ReplaceAllString(c.Image[i:], "=s176")411 }412 return c413}414415// imageSize is the size option of a YouTube image address ("=s900-c-k").416var imageSize = regexp.MustCompile(`^=s\d+`)417418// Blurb makes a channel's description one short paragraph. Search419// results come cut short with "...".420func Blurb(s string) string {421 s = strings.Join(strings.Fields(s), " ")422 if t, ok := strings.CutSuffix(s, " ..."); ok {423 s = t + "…"424 }425 if r := []rune(s); len(r) > 300 {426 s = strings.TrimSpace(string(r[:299])) + "…"427 }428 return s429}