1package sd23import (4 "bytes"5 "cmp"6 "context"7 "encoding/json"8 "errors"9 "fmt"10 "io"11 "net/http"12 "slices"13 "strconv"14 "strings"15 "time"16)1718// API is the Schedules Direct JSON API.19const API = "https://json.schedulesdirect.org/20141201/"2021// Request limits.22const (23 maxStations = 5000 // per /schedules and /schedules/md524 maxPrograms = 2000 // per /programs (5000 at most; smaller is kinder)25)2627// StopError is a refusal that ends updating until the account is28// changed: a wrong password, a locked, expired or disabled account, too29// many logins, or a request Schedules Direct calls a client bug. Trying30// again would only count against the account.31type StopError struct {32 Code int33 Message string34}3536func (e *StopError) Error() string {37 if e.Code == 0 {38 return e.Message39 }40 return fmt.Sprintf("%s (Schedules Direct code %d)", e.Message, e.Code)41}4243// apiError is any other error response.44type apiError struct {45 Code int46 Response string47 Message string48 Status int49}5051func (e *apiError) Error() string {52 return fmt.Sprintf("Schedules Direct: %s (code %d, HTTP %d)", cmp.Or(e.Message, e.Response), e.Code, e.Status)53}5455// stopCodes end updating until the account changes.56var stopCodes = map[int]string{57 4001: "The Schedules Direct account has expired",58 4002: "Schedules Direct didn't take the password's form",59 4003: "Schedules Direct doesn't know that user name and password",60 4004: "The Schedules Direct account is locked for a while after failed logins",61 4005: "Schedules Direct has turned off this account's access; open a ticket with them",62 4007: "Schedules Direct has turned off this application",63 4008: "The Schedules Direct account isn't active",64 4009: "Too many logins to Schedules Direct today",65 4010: "Schedules Direct saw this account used from too many places",66 1003: "Schedules Direct wants a User-Agent",67 1011: "Schedules Direct wants JSON",68 2055: "Schedules Direct refused a debug request",69 9999: "Schedules Direct reported an internal failure",70}7172// client is one update's conversation with Schedules Direct.73type client struct {74 base string // API, or a test server's75 http *http.Client76 ua string77 account Account78 st *state79 // relogged is set once a request has been retried with a new token.80 relogged bool81}8283// envelope is the code every response object may carry.84type envelope struct {85 Code *int `json:"code"`86 Response string `json:"response"`87 Message string `json:"message"`88}8990// do makes a request with JSON in and out. Errors in the response, by its91// code, become *StopError or *apiError; an expired token is renewed once.92func (c *client) do(ctx context.Context, method, path string, in, out any) error {93 var body io.Reader94 if in != nil {95 raw, err := json.Marshal(in)96 if err != nil {97 return err98 }99 body = bytes.NewReader(raw)100 }101 req, err := http.NewRequestWithContext(ctx, method, cmp.Or(c.base, API)+path, body)102 if err != nil {103 return err104 }105 req.Header.Set("User-Agent", c.ua)106 if in != nil || method == http.MethodPut {107 req.Header.Set("Content-Type", "application/json")108 }109 if path != "token" {110 req.Header.Set("Token", c.st.Token)111 }112 resp, err := c.http.Do(req)113 if err != nil {114 return fmt.Errorf("Schedules Direct: %w", err)115 }116 defer resp.Body.Close()117 raw, err := io.ReadAll(io.LimitReader(resp.Body, 512<<20))118 if err != nil {119 return fmt.Errorf("Schedules Direct: %w", err)120 }121 trimmed := bytes.TrimSpace(raw)122 if len(trimmed) == 0 || trimmed[0] != '{' && trimmed[0] != '[' {123 // Not JSON at all: a load balancer's refusal, or worse. Stop124 // rather than knock again.125 return &StopError{Message: fmt.Sprintf("Schedules Direct answered HTTP %d with something other than JSON; this server may be blocked", resp.StatusCode)}126 }127 if trimmed[0] == '{' {128 var env envelope129 if err := json.Unmarshal(trimmed, &env); err == nil && env.Code != nil && *env.Code != 0 {130 code := *env.Code131 // The token ran out (Schedules Direct may say so with 4003 as132 // well as 4006 outside a login): one new one, one retry.133 if (code == 4006 || code == 4003) && path != "token" && !c.relogged {134 c.relogged = true135 c.st.Token = ""136 if err := c.login(ctx, true); err != nil {137 return err138 }139 return c.do(ctx, method, path, in, out)140 }141 if msg, ok := stopCodes[code]; ok {142 return &StopError{Code: code, Message: msg}143 }144 return &apiError{Code: code, Response: env.Response, Message: env.Message, Status: resp.StatusCode}145 }146 }147 if resp.StatusCode >= 400 {148 return &apiError{Status: resp.StatusCode, Message: http.StatusText(resp.StatusCode)}149 }150 if out == nil {151 return nil152 }153 if err := json.Unmarshal(trimmed, out); err != nil {154 return fmt.Errorf("Schedules Direct %s: %w", path, err)155 }156 return nil157}158159// login gets a token, unless the saved one has more than an hour left. A160// token lasts a day from its first use, and asking again within that day161// gives the same one; fresh asks for a new one.162func (c *client) login(ctx context.Context, fresh bool) error {163 if !fresh && c.st.Token != "" && time.Until(c.st.TokenExpires) > time.Hour {164 return nil165 }166 in := map[string]any{"username": c.account.User, "password": c.account.PasswordSHA1}167 if fresh || c.st.Token != "" {168 in["newToken"] = true169 }170 var out struct {171 Token string `json:"token"`172 TokenExpires int64 `json:"tokenExpires"`173 ServerTime int64 `json:"serverTime"`174 }175 if err := c.do(ctx, http.MethodPost, "token", in, &out); err != nil {176 return err177 }178 if out.Token == "" || out.TokenExpires == 0 {179 return errors.New("Schedules Direct gave no token; it may be offline")180 }181 // The expiry by the server's clock, carried over to ours.182 life := time.Duration(out.TokenExpires-cmp.Or(out.ServerTime, time.Now().Unix())) * time.Second183 c.st.Token, c.st.TokenExpires = out.Token, time.Now().Add(life)184 return nil185}186187// update brings the state up to date: the account, the antenna lineup of188// zip, the schedules that changed and the programs they air.189func (c *client) update(ctx context.Context, zip, dir string, now time.Time) error {190 if err := c.login(ctx, false); err != nil {191 return err192 }193 id := "USA-OTA-" + zip194 modified, err := c.status(ctx, id)195 if err != nil {196 return err197 }198 if modified == "" {199 if err := c.addLineup(ctx, id); err != nil {200 return err201 }202 modified = "added"203 }204 if c.st.Lineup != id || c.st.LineupModified != modified || len(c.st.Stations) == 0 {205 if err := c.lineup(ctx, id); err != nil {206 return err207 }208 c.st.LineupModified = modified209 }210 if err := c.schedules(ctx, now); err != nil {211 return err212 }213 if err := c.programs(ctx); err != nil {214 return err215 }216 c.st.prune(now)217 c.artwork(ctx, dir, now)218 return nil219}220221// status reads the account: when it expires, whether the service is up,222// and when the lineup id last changed ("" when the account hasn't got it).223func (c *client) status(ctx context.Context, id string) (string, error) {224 var out struct {225 Account struct {226 Expires string `json:"expires"`227 MaxLineups int `json:"maxLineups"`228 } `json:"account"`229 Lineups []struct {230 Lineup string `json:"lineup"`231 ID string `json:"ID"`232 Modified string `json:"modified"`233 IsDeleted bool `json:"isDeleted"`234 } `json:"lineups"`235 SystemStatus []json.RawMessage `json:"systemStatus"`236 }237 if err := c.do(ctx, http.MethodGet, "status", nil, &out); err != nil {238 return "", err239 }240 if t, err := time.Parse(time.RFC3339, out.Account.Expires); err == nil {241 c.st.Expires = t242 }243 for _, raw := range out.SystemStatus {244 var sys struct {245 Status, Message string246 }247 if json.Unmarshal(raw, &sys) == nil && strings.EqualFold(sys.Status, "Offline") {248 return "", fmt.Errorf("Schedules Direct is offline: %s", sys.Message)249 }250 break // the first entry is the current one251 }252 for _, l := range out.Lineups {253 if cmp.Or(l.Lineup, l.ID) == id && !l.IsDeleted {254 return cmp.Or(l.Modified, "unknown"), nil255 }256 }257 return "", nil258}259260// addLineup adds the antenna lineup to the account, once: Schedules261// Direct allows a few changes a day.262func (c *client) addLineup(ctx context.Context, id string) error {263 err := c.do(ctx, http.MethodPut, "lineups/"+id, nil, nil)264 var ae *apiError265 switch {266 case errors.As(err, &ae) && ae.Code == 2100: // already there267 return nil268 case errors.As(err, &ae) && ae.Code == 2107:269 return errors.New("Schedules Direct is preparing this ZIP code's antenna lineup; trying again later")270 case errors.As(err, &ae) && (ae.Code == 4100 || ae.Code == 4101):271 return &StopError{Code: ae.Code, Message: "The Schedules Direct account can't take another lineup (" + ae.Message + "); remove one on schedulesdirect.org"}272 case errors.As(err, &ae) && ae.Code >= 2101 && ae.Code <= 2106:273 return &StopError{Code: ae.Code, Message: "Schedules Direct has no antenna lineup for ZIP code " + strings.TrimPrefix(id, "USA-OTA-")}274 }275 return err276}277278// lineup reads the stations of lineup id and their channel numbers.279func (c *client) lineup(ctx context.Context, id string) error {280 var out struct {281 Map []struct {282 StationID string `json:"stationID"`283 Major int `json:"atscMajor"`284 Minor int `json:"atscMinor"`285 RF int `json:"uhfVhf"`286 Channel json.RawMessage `json:"channel"`287 } `json:"map"`288 Stations []struct {289 StationID string `json:"stationID"`290 Name string `json:"name"`291 CallSign string `json:"callsign"`292 Affiliate string `json:"affiliate"`293 Logos []struct {294 URL string `json:"URL"`295 Category string `json:"category"`296 } `json:"stationLogo"`297 Logo *struct {298 URL string `json:"URL"`299 } `json:"logo"`300 } `json:"stations"`301 }302 if err := c.do(ctx, http.MethodGet, "lineups/"+id, nil, &out); err != nil {303 return err304 }305 number := map[string]string{}306 for _, m := range out.Map {307 switch {308 case m.Major > 0:309 number[m.StationID] = fmt.Sprintf("%d.%d", m.Major, m.Minor)310 default:311 var ch string312 if json.Unmarshal(m.Channel, &ch) == nil && ch != "" {313 number[m.StationID] = strings.TrimLeft(ch, "0")314 }315 }316 }317 var stations []station318 for _, s := range out.Stations {319 n := number[s.StationID]320 if n == "" {321 continue322 }323 sta := station{ID: s.StationID, Number: n, CallSign: s.CallSign, Name: s.Name, Affiliate: s.Affiliate, Logos: map[string]string{}}324 for _, l := range s.Logos {325 if l.URL != "" && l.Category != "" {326 sta.Logos[l.Category] = l.URL327 }328 }329 if s.Logo != nil {330 sta.Logo = s.Logo.URL331 }332 stations = append(stations, sta)333 }334 if len(stations) == 0 {335 return fmt.Errorf("Schedules Direct's lineup %s has no stations", id)336 }337 c.st.Lineup, c.st.Stations = id, stations338 if c.st.Days == nil {339 c.st.Days = map[string]map[string]day{}340 }341 keep := map[string]bool{}342 for _, s := range stations {343 keep[s.ID] = true344 }345 for id := range c.st.Days {346 if !keep[id] {347 delete(c.st.Days, id)348 }349 }350 return nil351}352353// dates are the UTC days to keep: yesterday (US evenings fall on the next354// UTC day) and Days from today.355func dates(now time.Time) []string {356 out := []string{}357 d := now.UTC().AddDate(0, 0, -1)358 for range Days + 1 {359 out = append(out, d.Format(time.DateOnly))360 d = d.AddDate(0, 0, 1)361 }362 return out363}364365type stationDates struct {366 StationID string `json:"stationID"`367 Date []string `json:"date"`368}369370// schedules fetches the station-days whose checksum changed since the371// last update, and only those.372func (c *client) schedules(ctx context.Context, now time.Time) error {373 ds := dates(now)374 var ask []stationDates375 for _, s := range c.st.Stations {376 ask = append(ask, stationDates{StationID: s.ID, Date: ds})377 }378 changed := map[string][]string{}379 for chunk := range slices.Chunk(ask, maxStations) {380 var out map[string]json.RawMessage381 if err := c.do(ctx, http.MethodPost, "schedules/md5", chunk, &out); err != nil {382 return err383 }384 for id, raw := range out {385 // A station with nothing comes as [] rather than {}.386 var days map[string]struct {387 Code int `json:"code"`388 MD5 string `json:"md5"`389 }390 if json.Unmarshal(raw, &days) != nil {391 continue392 }393 for date, d := range days {394 if d.Code == 0 && d.MD5 != "" && c.st.Days[id][date].MD5 != d.MD5 {395 changed[id] = append(changed[id], date)396 }397 }398 }399 }400 if len(changed) == 0 {401 return nil402 }403 var get []stationDates404 for id, ds := range changed {405 slices.Sort(ds)406 get = append(get, stationDates{StationID: id, Date: ds})407 }408 slices.SortFunc(get, func(a, b stationDates) int { return strings.Compare(a.StationID, b.StationID) })409 for chunk := range slices.Chunk(get, maxStations) {410 var out []json.RawMessage411 if err := c.do(ctx, http.MethodPost, "schedules", chunk, &out); err != nil {412 return err413 }414 for _, raw := range out {415 c.schedule(raw)416 }417 }418 return nil419}420421// schedule keeps one station-day of a /schedules response; errors for422// one station-day (deleted, queued, out of range) skip it.423func (c *client) schedule(raw json.RawMessage) {424 var s struct {425 StationID string `json:"stationID"`426 Code int `json:"code"`427 Programs []struct {428 ProgramID string `json:"programID"`429 AirDateTime string `json:"airDateTime"`430 Duration int `json:"duration"`431 MD5 string `json:"md5"`432 New bool `json:"new"`433 Premiere bool `json:"premiere"`434 LiveTapeDelay string `json:"liveTapeDelay"`435 IsPremiereOrFinale string `json:"isPremiereOrFinale"`436 AudioProperties []string `json:"audioProperties"`437 VideoProperties []string `json:"videoProperties"`438 Ratings []struct {439 Body string `json:"body"`440 Code string `json:"code"`441 } `json:"ratings"`442 } `json:"programs"`443 Metadata struct {444 MD5 string `json:"md5"`445 StartDate string `json:"startDate"`446 Code int `json:"code"`447 } `json:"metadata"`448 }449 if json.Unmarshal(raw, &s) != nil || s.StationID == "" {450 return451 }452 if s.Code == 2201 || s.Metadata.Code == 2201 { // the station is gone453 delete(c.st.Days, s.StationID)454 c.st.LineupModified = "" // read the lineup again next time455 return456 }457 if s.Code != 0 || s.Metadata.StartDate == "" {458 return459 }460 d := day{MD5: s.Metadata.MD5}461 for _, p := range s.Programs {462 start, err := time.Parse(time.RFC3339, p.AirDateTime)463 if err != nil || p.ProgramID == "" || p.Duration <= 0 {464 continue465 }466 a := airing{467 ProgramID: p.ProgramID, Start: start, Seconds: p.Duration, MD5: p.MD5,468 New: p.New, Live: p.LiveTapeDelay == "Live",469 Premiere: p.Premiere || strings.Contains(p.IsPremiereOrFinale, "Premiere"),470 Finale: strings.Contains(p.IsPremiereOrFinale, "Finale"),471 }472 for _, prop := range slices.Concat(p.AudioProperties, p.VideoProperties) {473 if tag := tags[strings.ToLower(prop)]; tag != "" && !slices.Contains(a.Tags, tag) {474 a.Tags = append(a.Tags, tag)475 }476 }477 for _, r := range p.Ratings {478 if a.Rating == "" || strings.Contains(r.Body, "USA") {479 a.Rating = ratingCode(r.Code)480 }481 }482 d.Airings = append(d.Airings, a)483 }484 if c.st.Days[s.StationID] == nil {485 c.st.Days[s.StationID] = map[string]day{}486 }487 c.st.Days[s.StationID][s.Metadata.StartDate] = d488}489490// tags are the guide's names for an airing's sound and picture.491var tags = map[string]string{492 "cc": "CC", "stereo": "Stereo", "dvs": "DVS", "sap": "SAP", "dd 5.1": "5.1", "surround": "5.1",493 "subtitled": "Subtitled", "hdtv": "HD",494}495496// ratingCode writes a rating as the guide does: "TV14" as "TV-14".497func ratingCode(code string) string {498 if rest, ok := strings.CutPrefix(code, "TV"); ok && rest != "" && rest[0] != '-' {499 return "TV-" + rest500 }501 return code502}503504// programs fetches the programs aired whose details changed (or were505// never fetched).506func (c *client) programs(ctx context.Context) error {507 want := map[string]bool{}508 for _, days := range c.st.Days {509 for _, d := range days {510 for _, a := range d.Airings {511 if p, ok := c.st.Programs[a.ProgramID]; !ok || p.MD5 != a.MD5 {512 want[a.ProgramID] = true513 }514 }515 }516 }517 ids := slices.Sorted(func(yield func(string) bool) {518 for id := range want {519 if !yield(id) {520 return521 }522 }523 })524 for chunk := range slices.Chunk(ids, maxPrograms) {525 var out []json.RawMessage526 if err := c.do(ctx, http.MethodPost, "programs", chunk, &out); err != nil {527 return err528 }529 for _, raw := range out {530 if id, p, ok := parseProgram(raw); ok {531 c.st.Programs[id] = p532 }533 }534 }535 return nil536}537538// sdProgram is the part of a /programs entry kept.539type sdProgram struct {540 ProgramID string `json:"programID"`541 Code int `json:"code"`542 MD5 string `json:"md5"`543 Titles []struct {544 Title120 string `json:"title120"`545 } `json:"titles"`546 EpisodeTitle150 string `json:"episodeTitle150"`547 Descriptions struct {548 Short []sdDescription `json:"description100"`549 Long []sdDescription `json:"description1000"`550 } `json:"descriptions"`551 OriginalAirDate string `json:"originalAirDate"`552 Genres []string `json:"genres"`553 EntityType string `json:"entityType"`554 Cast []struct {555 BillingOrder string `json:"billingOrder"`556 Role string `json:"role"`557 Name string `json:"name"`558 CharacterName string `json:"characterName"`559 } `json:"cast"`560 Metadata []map[string]struct {561 Season int `json:"season"`562 Episode int `json:"episode"`563 } `json:"metadata"`564 ContentRating []struct {565 Body string `json:"body"`566 Code string `json:"code"`567 } `json:"contentRating"`568 Movie *struct {569 Year flexInt `json:"year"`570 Duration int `json:"duration"`571 QualityRating []struct {572 RatingsBody string `json:"ratingsBody"`573 Rating string `json:"rating"`574 } `json:"qualityRating"`575 } `json:"movie"`576 EventDetails *struct {577 Teams []struct {578 Name string `json:"name"`579 IsHome bool `json:"isHome"`580 } `json:"teams"`581 } `json:"eventDetails"`582 HasImageArtwork bool `json:"hasImageArtwork"`583 HasSeriesArtwork bool `json:"hasSeriesArtwork"`584 HasMovieArtwork bool `json:"hasMovieArtwork"`585 HasSportsArtwork bool `json:"hasSportsArtwork"`586 HasEpisodeArtwork bool `json:"hasEpisodeArtwork"`587}588589type sdDescription struct {590 Language string `json:"descriptionLanguage"`591 Description string `json:"description"`592}593594// flexInt reads a number that may come as a string.595type flexInt int596597func (n *flexInt) UnmarshalJSON(raw []byte) error {598 s := strings.Trim(string(raw), `"`)599 if s == "" || s == "null" {600 *n = 0601 return nil602 }603 v, err := strconv.Atoi(s)604 *n = flexInt(v)605 return err606}607608// parseProgram turns a /programs entry into what's kept; ok is false for609// an entry that's an error (queued, unknown).610func parseProgram(raw json.RawMessage) (string, program, bool) {611 var sp sdProgram612 if json.Unmarshal(raw, &sp) != nil || sp.ProgramID == "" || sp.Code != 0 || sp.MD5 == "" {613 return "", program{}, false614 }615 p := program{616 MD5: sp.MD5, EpisodeTitle: sp.EpisodeTitle150, OriginalAirDate: sp.OriginalAirDate,617 Genres: sp.Genres, Kind: sp.EntityType,618 Art: sp.HasImageArtwork || sp.HasSeriesArtwork || sp.HasMovieArtwork || sp.HasSportsArtwork || sp.HasEpisodeArtwork,619 Description: cmp.Or(english(sp.Descriptions.Long), english(sp.Descriptions.Short)),620 }621 if len(sp.Titles) > 0 {622 p.Title = sp.Titles[0].Title120623 }624 for _, m := range sp.Metadata {625 if g, ok := m["Gracenote"]; ok {626 p.Season, p.Episode = g.Season, g.Episode627 }628 }629 for _, r := range sp.ContentRating {630 if p.Rating == "" || strings.Contains(r.Body, "USA") || strings.Contains(r.Body, "Motion Picture") {631 p.Rating = ratingCode(r.Code)632 }633 }634 cast := slices.Clone(sp.Cast)635 slices.SortStableFunc(cast, func(a, b struct {636 BillingOrder string `json:"billingOrder"`637 Role string `json:"role"`638 Name string `json:"name"`639 CharacterName string `json:"characterName"`640 }) int {641 return strings.Compare(a.BillingOrder, b.BillingOrder)642 })643 for _, m := range cast {644 if len(p.Cast) == 6 || m.Name == "" {645 continue646 }647 if m.CharacterName != "" {648 p.Cast = append(p.Cast, m.Name+" ("+m.CharacterName+")")649 } else {650 p.Cast = append(p.Cast, m.Name)651 }652 }653 if sp.Movie != nil {654 p.Year, p.Runtime = int(sp.Movie.Year), sp.Movie.Duration/60655 for _, q := range sp.Movie.QualityRating {656 if v, err := strconv.ParseFloat(q.Rating, 64); err == nil && strings.EqualFold(q.RatingsBody, "Gracenote") {657 p.Stars = v658 }659 }660 }661 if sp.EventDetails != nil {662 for _, t := range sp.EventDetails.Teams {663 if t.IsHome {664 p.HomeTeam = t.Name665 } else {666 p.AwayTeam = t.Name667 }668 }669 }670 return sp.ProgramID, p, true671}672673// english picks the English description, else the first.674func english(ds []sdDescription) string {675 for _, d := range ds {676 if strings.HasPrefix(d.Language, "en") {677 return d.Description678 }679 }680 if len(ds) > 0 {681 return ds[0].Description682 }683 return ""684}