1package sites23import (4 "bytes"5 "context"6 "crypto/sha256"7 "crypto/tls"8 "database/sql"9 "encoding/hex"10 "encoding/json"11 "errors"12 "fmt"13 "io"14 "net"15 "net/http"16 "net/url"17 "strconv"18 "strings"19 "time"2021 "github.com/FelineStateMachine/tinyrelay/internal/event"22 "github.com/FelineStateMachine/tinyrelay/internal/policy"23 "github.com/FelineStateMachine/tinyrelay/internal/storage"24)2526// Blob is a fetched site file and its declared content metadata.27type Blob struct {28 Body io.ReadCloser29 Type string30 Size int6431}3233// BlobGet opens a local blob by its SHA-256 hash. The caller closes Blob.Body.34type BlobGet func(ctx context.Context, blobSHA string) (Blob, error)3536// BlobPut stores bytes under blobSHA with the supplied content type. The37// callback consumes the reader before returning.38type BlobPut func(ctx context.Context, blobSHA string, contentType string, body io.Reader) error3940// Fetch performs an outbound site file request. The caller closes the41// returned response body.42type Fetch func(ctx context.Context, request *http.Request) (*http.Response, error)4344// ResolveIP resolves a host before an outbound site request is allowed.45type ResolveIP func(ctx context.Context, host string) ([]net.IP, error)4647// ResolveHost resolves a request host to a site label.48type ResolveHost func(ctx context.Context, host string) (label string, err error)4950// ReadAccess authorizes a request for site content.51type ReadAccess func(request *http.Request) bool5253// Config supplies storage, blob, network and request policy dependencies.54type Config struct {55 Store *storage.Store56 GetBlob BlobGet57 PutBlob BlobPut58 Fetch Fetch59 ResolveIP ResolveIP60 MaxFetchBytes int6461 Policy func() policy.Policy62 ReadAccess ReadAccess63 BaseDomain string64 ResolveHost ResolveHost65 RelayURL func(*http.Request) string66}6768type Service struct{ config Config }6970// Mirror copies every missing manifest file from its advertised servers into71// the local blob store. It is safe to retry after a partial failure.72func (s *Service) Mirror(ctx context.Context, manifest event.Event) error {73 if err := ValidateManifest(manifest); err != nil {74 return err75 }76 if !s.config.Policy().Features.Sites.Mirror {77 return nil78 }79 if s.config.PutBlob == nil {80 return errors.New("site mirror requires a blob writer")81 }82 for _, mapping := range SitePaths(manifest) {83 if s.config.GetBlob != nil {84 if local, err := s.config.GetBlob(ctx, mapping[2]); err == nil {85 if local.Body != nil {86 _ = local.Body.Close()87 }88 continue89 }90 }91 blob, err := s.blob(ctx, manifest, mapping)92 if err != nil {93 return fmt.Errorf("mirror %s: %w", mapping[1], err)94 }95 data, err := readVerified(blob.Body, mapping[2], s.config.MaxFetchBytes)96 _ = blob.Body.Close()97 if err != nil {98 return fmt.Errorf("verify %s: %w", mapping[1], err)99 }100 if err := s.config.PutBlob(ctx, mapping[2], chooseType(blob.Type, mapping[1]), bytes.NewReader(data)); err != nil {101 return fmt.Errorf("store %s: %w", mapping[1], err)102 }103 }104 return nil105}106107// RunMirrorIntent resolves the event captured by storage.AddIntents and runs108// the idempotent mirror operation. Unknown intent kinds are rejected so a109// worker cannot accidentally execute a different work queue entry here.110func (s *Service) RunMirrorIntent(ctx context.Context, intent storage.Intent) error {111 if intent.Kind != "site-mirror" {112 return fmt.Errorf("unsupported site intent %q", intent.Kind)113 }114 result, err := s.config.Store.Query(ctx, event.Filter{IDs: []string{intent.EventID}, Tags: map[string][]string{}}, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: 1})115 if err != nil {116 return err117 }118 if len(result.Events) == 0 {119 return nil120 }121 return s.Mirror(ctx, result.Events[0])122}123124func New(config Config) (*Service, error) {125 if config.Store == nil {126 return nil, errors.New("sites store is required")127 }128 if config.Policy == nil {129 config.Policy = func() policy.Policy { return policy.Defaults("") }130 }131 if config.BaseDomain == "" {132 config.BaseDomain = "localhost"133 }134 if config.ResolveIP == nil {135 config.ResolveIP = func(ctx context.Context, host string) ([]net.IP, error) {136 return net.DefaultResolver.LookupIP(ctx, "ip", host)137 }138 }139 if config.MaxFetchBytes < 0 {140 return nil, errors.New("sites maximum fetch bytes cannot be negative")141 }142 if err := ensureManifestSchema(context.Background(), config.Store.DB()); err != nil {143 return nil, fmt.Errorf("sites schema: %w", err)144 }145 return &Service{config: config}, nil146}147148func (s *Service) SaveOptions(e event.Event, now int64) storage.SaveOptions {149 if e.Kind != KindSite && e.Kind != KindNamedSite && e.Kind != KindSiteSnapshot {150 return storage.SaveOptions{Now: now}151 }152 if err := ValidateManifest(e); err != nil {153 return storage.SaveOptions{Now: now}154 }155 opts := storage.SaveOptions{Now: now, BeforeCommit: func(ctx context.Context, tx *sql.Tx) error { return s.applyManifest(ctx, tx, e) }}156 if s.config.Policy().Features.Sites.Mirror {157 opts.Intents = []storage.Intent{{Kind: "site-mirror", EventID: e.ID, Payload: SiteLabel(e)}}158 }159 return opts160}161162func (s *Service) ApplyTx(ctx context.Context, tx *sql.Tx, e event.Event) error {163 return s.applyManifest(ctx, tx, e)164}165166// DeleteTx removes a manifest index entry in the same transaction that167// removes its event. Callers should invoke this for deletion and replacement168// paths so routing never points at an event that is no longer visible.169func (s *Service) DeleteTx(ctx context.Context, tx *sql.Tx, eventID string) error {170 _, err := tx.ExecContext(ctx, "DELETE FROM site_manifests WHERE event_id=?", eventID)171 return err172}173174func (s *Service) applyManifest(ctx context.Context, tx *sql.Tx, e event.Event) error {175 if e.Kind != KindSite && e.Kind != KindNamedSite && e.Kind != KindSiteSnapshot {176 return nil177 }178 label := SiteLabel(e)179 if label == "" {180 return errors.New("invalid: site label")181 }182 _, err := tx.ExecContext(ctx, `INSERT INTO site_manifests(label,event_id,kind,pubkey,d,updated_at) VALUES(?,?,?,?,?,?) ON CONFLICT(label) DO UPDATE SET event_id=excluded.event_id,kind=excluded.kind,pubkey=excluded.pubkey,d=excluded.d,updated_at=excluded.updated_at`, label, e.ID, e.Kind, e.PubKey, event.Tag(e, "d"), time.Now().Unix())183 return err184}185186func (s *Service) Manifest(ctx context.Context, site Site) (event.Event, error) {187 f := event.Filter{Kinds: []int{site.Kind}, Tags: map[string][]string{}}188 if site.Kind == KindSiteSnapshot {189 f.IDs = []string{site.ID}190 } else {191 f.Authors = []string{site.PubKey}192 if site.Kind == KindNamedSite {193 f.Tags["d"] = []string{site.D}194 }195 }196 result, err := s.config.Store.Query(ctx, f, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: 1})197 if err != nil {198 return event.Event{}, err199 }200 if len(result.Events) == 0 {201 return event.Event{}, sql.ErrNoRows202 }203 if err := ValidateManifest(result.Events[0]); err != nil {204 return event.Event{}, err205 }206 return result.Events[0], nil207}208209func (s *Service) Handler() http.Handler { return http.HandlerFunc(s.serve) }210211// MatchesHost reports whether the daemon should dispatch this Host header to212// the site door before trying relay routes.213func (s *Service) MatchesHost(host string) bool {214 _, ok := s.labelForHost(context.Background(), host)215 return ok216}217218// HandlerForLabel returns a handler bound to a catalog-resolved site label.219// It is useful when the daemon has already selected a tenant from its host220// registry and must prevent a request from crossing into another tenant.221func (s *Service) HandlerForLabel(label string) http.Handler {222 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { s.serveLabel(w, r, label) })223}224225func (s *Service) serve(w http.ResponseWriter, r *http.Request) {226 label, ok := s.labelForHost(r.Context(), r.Host)227 if !ok {228 http.NotFound(w, r)229 return230 }231 s.serveLabel(w, r, label)232}233234func (s *Service) serveLabel(w http.ResponseWriter, r *http.Request, label string) {235 if !s.config.Policy().Features.Sites.Enabled {236 http.NotFound(w, r)237 return238 }239 if s.config.ReadAccess != nil && !s.config.ReadAccess(r) {240 http.Error(w, "authentication required", http.StatusUnauthorized)241 return242 }243 site, ok := ParseSite(label)244 if !ok {245 http.NotFound(w, r)246 return247 }248 manifest, err := s.Manifest(r.Context(), site)249 if err != nil {250 http.NotFound(w, r)251 return252 }253 if r.URL.Path == "/.well-known/nostr.json" {254 if _, present := r.URL.Query()["path"]; present {255 s.discovery(w, r, manifest)256 return257 }258 }259 if r.Method != http.MethodGet && r.Method != http.MethodHead {260 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)261 return262 }263 path, redirect, ok := sitePath(r.URL.EscapedPath(), SitePaths(manifest))264 if !ok {265 http.NotFound(w, r)266 return267 }268 if redirect {269 location := *r.URL270 location.Path = path271 location.RawPath = ""272 location.Host = r.Host273 location.Scheme = "http"274 if r.TLS != nil {275 location.Scheme = "https"276 }277 http.Redirect(w, r, location.String(), http.StatusPermanentRedirect)278 return279 }280 if err := s.servePath(w, r, manifest, path); err != nil {281 http.NotFound(w, r)282 }283}284285func (s *Service) discovery(w http.ResponseWriter, r *http.Request, manifest event.Event) {286 w.Header().Set("Access-Control-Allow-Origin", "*")287 w.Header().Set("Cache-Control", "no-store")288 w.Header().Set("Access-Control-Allow-Headers", "authorization, content-type, accept")289 w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS")290 if r.Method == http.MethodOptions {291 w.WriteHeader(http.StatusNoContent)292 return293 }294 if r.Method != http.MethodGet && r.Method != http.MethodHead {295 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)296 return297 }298 values, ok := r.URL.Query()["path"]299 if !ok || len(values) != 1 {300 writeDiscovery(w, http.StatusBadRequest, nil, r.Method == http.MethodHead)301 return302 }303 path, ok := RequestedPath(values[0])304 if !ok || !validDiscoveryPath(path) {305 writeDiscovery(w, http.StatusBadRequest, nil, r.Method == http.MethodHead)306 return307 }308 selected, redirect, valid := sitePath(path, SitePaths(manifest))309 if redirect {310 selected, _, valid = sitePath(path+"/", SitePaths(manifest))311 }312 if !valid || findPath(SitePaths(manifest), selected) == nil {313 writeDiscovery(w, http.StatusOK, nil, r.Method == http.MethodHead)314 return315 }316 filter := manifestFilter(manifest)317 relay := ""318 if s.config.RelayURL != nil {319 relay = s.config.RelayURL(r)320 }321 if relay == "" {322 relay = requestOrigin(r)323 }324 writeDiscovery(w, http.StatusOK, map[string]any{path: map[string]any{"filter": filter, "relays": []string{relay}}}, r.Method == http.MethodHead)325}326327func manifestFilter(e event.Event) map[string]any {328 if e.Kind == KindSiteSnapshot {329 return map[string]any{"ids": []string{e.ID}, "limit": 1}330 }331 result := map[string]any{"authors": []string{e.PubKey}, "kinds": []int{e.Kind}, "limit": 1}332 if e.Kind == KindNamedSite {333 result["#d"] = []string{event.Tag(e, "d")}334 }335 return result336}337338func validDiscoveryPath(path string) bool {339 if path == "" || !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "?#\\\x00\r\n") {340 return false341 }342 for _, part := range strings.Split(path, "/") {343 if part == ".." || part == "." {344 return false345 }346 }347 return true348}349350func writeDiscovery(w http.ResponseWriter, status int, value any, head bool) {351 w.Header().Set("Content-Type", "application/json")352 w.WriteHeader(status)353 if value != nil && !head {354 raw, _ := json.Marshal(value)355 _, _ = w.Write(raw)356 }357}358359func requestOrigin(r *http.Request) string {360 scheme := "http"361 if r.TLS != nil {362 scheme = "https"363 }364 return scheme + "://" + r.Host365}366367func (s *Service) labelForHost(ctx context.Context, host string) (string, bool) {368 host = strings.ToLower(strings.Split(host, ":")[0])369 if s.config.ResolveHost != nil {370 label, err := s.config.ResolveHost(ctx, host)371 if err == nil && label != "" {372 if _, ok := ParseSite(label); ok {373 return label, true374 }375 }376 }377 suffix := "." + strings.ToLower(strings.TrimSuffix(s.config.BaseDomain, "."))378 if strings.HasSuffix(host, suffix) {379 label := strings.TrimSuffix(host, suffix)380 if _, ok := ParseSite(label); ok {381 return label, true382 }383 }384 for _, custom := range s.config.Policy().CustomHosts {385 if strings.EqualFold(custom.Host, host) && custom.Site != "" {386 return custom.Site, true387 }388 }389 return "", false390}391392func sitePath(raw string, paths [][]string) (string, bool, bool) {393 decoded, err := url.PathUnescape(raw)394 if err != nil {395 return "", false, false396 }397 directory := !strings.Contains(pathBase(decoded), ".")398 redirect := directory && !strings.HasSuffix(decoded, "/")399 if redirect {400 decoded += "/"401 raw += "/"402 }403 lookup := decoded404 if directory {405 lookup += "index.html"406 }407 for _, tag := range paths {408 if len(tag) == 3 && (tag[1] == raw || tag[1] == lookup) {409 if redirect {410 return decoded, true, true411 }412 return tag[1], redirect, true413 }414 }415 return lookup, redirect, true416}417418func pathBase(path string) string {419 if i := strings.LastIndexByte(path, '/'); i >= 0 {420 return path[i+1:]421 }422 return path423}424425func (s *Service) servePath(w http.ResponseWriter, r *http.Request, manifest event.Event, path string) error {426 mapping := findPath(SitePaths(manifest), path)427 status := http.StatusOK428 if mapping == nil {429 mapping = findPath(SitePaths(manifest), "/404.html")430 status = http.StatusNotFound431 }432 if mapping == nil {433 return errors.New("site file not found")434 }435 blob, err := s.blob(r.Context(), manifest, mapping)436 if err != nil {437 return err438 }439 defer blob.Body.Close()440 data, err := readVerified(blob.Body, mapping[2], s.config.MaxFetchBytes)441 if err != nil {442 return err443 }444 w.Header().Set("ETag", `"`+mapping[2]+`"`)445 w.Header().Set("Content-Type", chooseType(blob.Type, path))446 w.Header().Set("Content-Length", strconv.Itoa(len(data)))447 cache := "public, no-cache, must-revalidate, no-transform"448 if s.config.Policy().Reads != "open" {449 cache = "private, no-store, no-transform"450 }451 w.Header().Set("Cache-Control", cache)452 w.Header().Set("X-Content-Type-Options", "nosniff")453 if r.Method == http.MethodHead {454 w.WriteHeader(status)455 return nil456 }457 if r.Header.Get("If-None-Match") == `"`+mapping[2]+`"` && status == http.StatusOK {458 w.WriteHeader(http.StatusNotModified)459 return nil460 }461 if status == http.StatusOK {462 http.ServeContent(w, r, path, time.Time{}, bytes.NewReader(data))463 return nil464 }465 w.WriteHeader(status)466 _, err = w.Write(data)467 return err468}469470func findPath(paths [][]string, wanted string) []string {471 for _, path := range paths {472 if len(path) == 3 && path[1] == wanted {473 return path474 }475 }476 return nil477}478func chooseType(blobType, path string) string {479 if blobType != "" {480 return blobType481 }482 return SiteType(path)483}484func readVerified(body io.Reader, want string, maxBytes int64) ([]byte, error) {485 limited := body486 if maxBytes > 0 {487 limited = io.LimitReader(body, maxBytes+1)488 }489 data, err := io.ReadAll(limited)490 if err != nil {491 return nil, err492 }493 if maxBytes > 0 && int64(len(data)) > maxBytes {494 return nil, errors.New("site file exceeds fetch limit")495 }496 sum := sha256.Sum256(data)497 if hex.EncodeToString(sum[:]) != want {498 return nil, errors.New("site file hash mismatch")499 }500 return data, nil501}502503func (s *Service) blob(ctx context.Context, manifest event.Event, mapping []string) (Blob, error) {504 if s.config.GetBlob != nil {505 if blob, err := s.config.GetBlob(ctx, mapping[2]); err == nil {506 return blob, nil507 }508 }509 if s.config.Fetch == nil {510 return Blob{}, errors.New("site file is not local")511 }512 for _, raw := range s.servers(ctx, manifest) {513 blob, err := s.fetchRemote(ctx, raw, mapping)514 if err == nil {515 return blob, nil516 }517 }518 return Blob{}, errors.New("site file unavailable")519}520521func (s *Service) fetchRemote(ctx context.Context, raw string, mapping []string) (Blob, error) {522 u, err := url.Parse(raw)523 if err != nil || !safeOrigin(u) || !s.publicOrigin(ctx, u) {524 return Blob{}, errors.New("unsafe site origin")525 }526 u.Path = strings.TrimSuffix(u.Path, "/") + "/" + mapping[2]527 u.RawQuery = ""528 var response *http.Response529 for redirects := 0; redirects <= 3; redirects++ {530 req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)531 if err != nil {532 return Blob{}, err533 }534 response, err = s.fetch(ctx, req)535 if err != nil {536 return Blob{}, err537 }538 if response == nil || response.Body == nil {539 return Blob{}, errors.New("empty origin response")540 }541 if response.StatusCode < 300 || response.StatusCode >= 400 {542 break543 }544 location := response.Header.Get("location")545 response.Body.Close()546 if location == "" {547 return Blob{}, errors.New("origin redirect has no location")548 }549 next, err := url.Parse(location)550 if err != nil || !safeOrigin(next) || !s.publicOrigin(ctx, next) {551 return Blob{}, errors.New("unsafe site redirect")552 }553 next.Path = strings.TrimSuffix(next.Path, "/") + "/" + mapping[2]554 next.RawQuery = ""555 u = next556 if redirects == 3 {557 return Blob{}, errors.New("too many origin redirects")558 }559 }560 if response.StatusCode < 200 || response.StatusCode >= 300 {561 response.Body.Close()562 return Blob{}, errors.New("origin response was not successful")563 }564 if s.config.MaxFetchBytes > 0 && response.ContentLength > s.config.MaxFetchBytes {565 response.Body.Close()566 return Blob{}, errors.New("site file exceeds operator fetch limit")567 }568 return Blob{Body: response.Body, Type: response.Header.Get("content-type"), Size: response.ContentLength}, nil569}570571func (s *Service) fetch(ctx context.Context, req *http.Request) (*http.Response, error) {572 if s.config.Fetch != nil {573 return s.config.Fetch(ctx, req)574 }575 return s.pinnedClient(ctx, req.URL.Hostname()).Do(req)576}577578func (s *Service) pinnedClient(ctx context.Context, hostname string) *http.Client {579 return &http.Client{Transport: &http.Transport{Proxy: nil, TLSClientConfig: &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12}, DialContext: func(dialCtx context.Context, network, address string) (net.Conn, error) {580 ips, err := s.config.ResolveIP(dialCtx, hostname)581 if err != nil || len(ips) == 0 {582 return nil, errors.New("origin DNS lookup failed")583 }584 for _, ip := range ips {585 if !isPublicIP(ip) {586 return nil, errors.New("origin resolved to a private address")587 }588 }589 port := "443"590 if _, p, err := net.SplitHostPort(address); err == nil {591 port = p592 }593 return (&net.Dialer{}).DialContext(dialCtx, network, net.JoinHostPort(ips[0].String(), port))594 }}, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }}595}596597func safeOrigin(u *url.URL) bool {598 if u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.Fragment != "" || net.ParseIP(u.Hostname()) != nil {599 return false600 }601 return !strings.HasSuffix(u.Hostname(), ".local") && !strings.HasSuffix(u.Hostname(), ".internal")602}603604func (s *Service) publicOrigin(ctx context.Context, u *url.URL) bool {605 ips, err := s.config.ResolveIP(ctx, u.Hostname())606 if err != nil || len(ips) == 0 {607 return false608 }609 for _, ip := range ips {610 if !isPublicIP(ip) {611 return false612 }613 }614 return true615}616617func isPublicIP(ip net.IP) bool {618 return !ip.IsLoopback() && !ip.IsPrivate() && !ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() && !ip.IsMulticast() && !ip.IsUnspecified()619}620621func (s *Service) servers(ctx context.Context, manifest event.Event) []string {622 servers := append([]string(nil), event.TagValues(manifest, "server")...)623 result, err := s.config.Store.Query(ctx, event.Filter{Authors: []string{manifest.PubKey}, Kinds: []int{10063}, Tags: map[string][]string{}}, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: 1})624 if err == nil && len(result.Events) > 0 {625 servers = append(servers, event.TagValues(result.Events[0], "server")...)626 }627 seen := make(map[string]bool, len(servers))628 out := make([]string, 0, len(servers))629 for _, value := range servers {630 if !seen[value] {631 seen[value] = true632 out = append(out, value)633 }634 }635 return out636}