1// Package youtube is YouTube as a source: yt-dlp, kept up to date, to2// list channels and playlists, find videos' details and where to stream3// them from, and search for channels; and YouTube's feeds. The custom4// channels (internal/vchan) and the podcasts (internal/podcast) both use5// it.6package youtube78import (9 "bytes"10 "cmp"11 "context"12 "crypto/sha256"13 "encoding/hex"14 "errors"15 "fmt"16 "io"17 "log"18 "net/http"19 "os"20 "os/exec"21 "path"22 "path/filepath"23 "regexp"24 "strings"25 "sync"26 "time"27)2829// YtDlp runs yt-dlp. With Update, it keeps its own30// copy at Path: the latest release is downloaded on first use and replaced31// about daily, so YouTube's changes don't wait for a new image. The release32// is a Python zipapp, so python3 must be installed, and YouTube needs a33// JavaScript runtime for yt-dlp (deno by default).34type YtDlp struct {35 Path string // "yt-dlp" from PATH when empty36 Update bool37 // HTTP downloads yt-dlp and reads channel feeds; http.DefaultClient38 // when nil.39 HTTP *http.Client4041 // Jobs is held for a listing job, one at a time across its users;42 // Ahead while finding where to stream a video from ahead of time,43 // likewise.44 Jobs, Ahead sync.Mutex4546 mu sync.Mutex47 checked time.Time // the last install or update check48 updating bool49}5051const (52 ytdlpReleases = "https://github.com/yt-dlp/yt-dlp/releases"53 ytdlpUpdateEvery = 24 * time.Hour54)5556// Client is the HTTP client t reads with, for feeds and captions.57func (t *YtDlp) Client() *http.Client { return cmp.Or(t.HTTP, http.DefaultClient) }5859// command returns the yt-dlp to run, installing it first when it's60// missing, and starts the daily update when one is due.61func (t *YtDlp) command(ctx context.Context) (string, error) {62 p := cmp.Or(t.Path, "yt-dlp")63 if !t.Update {64 return p, nil65 }66 t.mu.Lock()67 defer t.mu.Unlock()68 info, err := os.Stat(p)69 if err != nil {70 if err := t.install(ctx, p); err != nil {71 return "", fmt.Errorf("install yt-dlp: %w", err)72 }73 t.checked = time.Now()74 return p, nil75 }76 if t.checked.IsZero() {77 t.checked = info.ModTime() // installed or last checked then78 }79 if time.Since(t.checked) > ytdlpUpdateEvery && !t.updating {80 t.checked, t.updating = time.Now(), true81 go func() {82 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)83 defer cancel()84 if err := t.install(ctx, p); err != nil {85 log.Printf("yt-dlp: update: %v", err)86 }87 t.mu.Lock()88 t.updating = false89 t.mu.Unlock()90 }()91 }92 return p, nil93}9495// install puts the latest release at p unless it's already there. The new96// copy is checked against the release's checksums and run once before it97// replaces the old one, which running jobs keep using until they end.98func (t *YtDlp) install(ctx context.Context, p string) error {99 tag, err := t.latest(ctx)100 if err != nil {101 return err102 }103 if have, _ := ytdlpVersion(ctx, p); have == tag {104 now := time.Now()105 return os.Chtimes(p, now, now) // checked; next check in a day106 }107 bin, err := t.get(ctx, ytdlpReleases+"/download/"+tag+"/yt-dlp")108 if err != nil {109 return err110 }111 sums, err := t.get(ctx, ytdlpReleases+"/download/"+tag+"/SHA2-256SUMS")112 if err != nil {113 return err114 }115 sum := sha256.Sum256(bin)116 want := ""117 for line := range strings.Lines(string(sums)) {118 if f := strings.Fields(line); len(f) == 2 && f[1] == "yt-dlp" {119 want = f[0]120 }121 }122 if want != hex.EncodeToString(sum[:]) {123 return fmt.Errorf("yt-dlp %s: checksum mismatch", tag)124 }125 if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {126 return err127 }128 tmp := p + ".new"129 if err := os.WriteFile(tmp, bin, 0o755); err != nil {130 return err131 }132 if v, err := ytdlpVersion(ctx, tmp); err != nil || v != tag {133 os.Remove(tmp)134 return fmt.Errorf("yt-dlp %s doesn't run (is python3 installed?): %v", tag, err)135 }136 if err := os.Rename(tmp, p); err != nil {137 os.Remove(tmp)138 return err139 }140 log.Printf("yt-dlp %s installed at %s", tag, p)141 return nil142}143144// latest is the newest release's tag, read from where GitHub redirects145// the latest release page.146func (t *YtDlp) latest(ctx context.Context) (string, error) {147 req, err := http.NewRequestWithContext(ctx, http.MethodHead, ytdlpReleases+"/latest", nil)148 if err != nil {149 return "", err150 }151 c := *t.Client()152 c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }153 resp, err := c.Do(req)154 if err != nil {155 return "", err156 }157 resp.Body.Close()158 loc := resp.Header.Get("Location")159 if !strings.Contains(loc, "/releases/tag/") {160 return "", fmt.Errorf("latest yt-dlp release: HTTP %d", resp.StatusCode)161 }162 return path.Base(loc), nil163}164165func (t *YtDlp) get(ctx context.Context, url string) ([]byte, error) {166 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)167 if err != nil {168 return nil, err169 }170 resp, err := t.Client().Do(req)171 if err != nil {172 return nil, err173 }174 defer resp.Body.Close()175 if resp.StatusCode != http.StatusOK {176 return nil, fmt.Errorf("GET %s: HTTP %d", url, resp.StatusCode)177 }178 return io.ReadAll(io.LimitReader(resp.Body, 64<<20))179}180181func ytdlpVersion(ctx context.Context, p string) (string, error) {182 ctx, cancel := context.WithTimeout(ctx, 30*time.Second)183 defer cancel()184 out, err := exec.CommandContext(ctx, p, "--version").Output()185 return strings.TrimSpace(string(out)), err186}187188// Run runs yt-dlp with args and returns what it writes to stdout. Errors189// carry yt-dlp's own error message, without addresses.190func (t *YtDlp) Run(ctx context.Context, args ...string) ([]byte, error) {191 p, err := t.command(ctx)192 if err != nil {193 return nil, err194 }195 common := []string{"--ignore-config", "--no-warnings", "--no-progress"}196 if t.Update {197 // Beside the managed copy, so caches survive restarts.198 common = append(common, "--cache-dir", filepath.Join(filepath.Dir(p), "cache"))199 }200 cmd := exec.CommandContext(ctx, p, append(common, args...)...)201 var stderr bytes.Buffer202 cmd.Stderr = &stderr203 out, err := cmd.Output()204 if err != nil {205 if ctx.Err() != nil {206 return nil, ctx.Err()207 }208 return nil, &Error{msg: message(stderr.String(), err)}209 }210 return out, nil211}212213// Error is yt-dlp's error message.214type Error struct{ msg string }215216func (e *Error) Error() string { return "yt-dlp: " + e.msg }217218// message picks yt-dlp's error line out of its output.219func message(stderr string, err error) string {220 msg := ""221 for line := range strings.Lines(stderr) {222 if s, ok := strings.CutPrefix(strings.TrimSpace(line), "ERROR: "); ok {223 msg = s224 }225 }226 if msg == "" {227 var exit *exec.ExitError228 if errors.As(err, &exit) {229 if lines := strings.Split(strings.TrimSpace(stderr), "\n"); lines[len(lines)-1] != "" {230 msg = lines[len(lines)-1]231 }232 }233 }234 msg = urls.ReplaceAllString(cmp.Or(msg, err.Error()), "<url>")235 if len(msg) > 300 {236 msg = msg[:300] + "…"237 }238 return msg239}240241// Gone reports whether a yt-dlp error says the video can't be played242// here at all, rather than for now.243func Gone(err error) bool {244 var e *Error245 if !errors.As(err, &e) {246 return false247 }248 m := strings.ToLower(e.msg)249 if strings.Contains(m, "not a bot") {250 return false251 }252 for _, s := range []string{"video unavailable", "private video", "has been removed", "members-only",253 "join this channel", "confirm your age", "not available in your country", "terminated"} {254 if strings.Contains(m, s) {255 return true256 }257 }258 return false259}260261// urls are addresses in text, left out of errors and descriptions.262var urls = regexp.MustCompile(`https?://\S+`)263264// ErrInstalling is Ready's answer while yt-dlp is being installed.265var ErrInstalling = errors.New("yt-dlp is being installed; try again in a minute")266267// installing is Ready's install of a managed yt-dlp.268type installing struct {269 mu sync.Mutex270 running bool271 err error // the last try's272}273274var installs sync.Map // *YtDlp: *installing275276// Ready reports whether yt-dlp is there to run. When t keeps its own copy277// and hasn't installed it yet, Ready starts installing it and reports278// ErrInstalling, or why the last try failed.279func (t *YtDlp) Ready() error {280 if !t.Update {281 if _, err := exec.LookPath(cmp.Or(t.Path, "yt-dlp")); err != nil {282 return fmt.Errorf("yt-dlp isn't installed on the server: %w", err)283 }284 return nil285 }286 if _, err := os.Stat(t.Path); err == nil {287 return nil288 }289 v, _ := installs.LoadOrStore(t, &installing{})290 in := v.(*installing)291 in.mu.Lock()292 defer in.mu.Unlock()293 failed := in.err294 if !in.running {295 in.running = true296 go func() {297 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)298 defer cancel()299 _, err := t.command(ctx)300 in.mu.Lock()301 in.running, in.err = false, err302 in.mu.Unlock()303 }()304 }305 if failed != nil {306 return fmt.Errorf("couldn't install yt-dlp (%v); trying again", failed)307 }308 return ErrInstalling309}