- commit
- 8dd79a2655be8af901f089c924c060493daebb5b
- parent
- 2beb15ec7ff1e0ab7d5e55cf31c914fab06371f2
- author
- Dami <dami.etoile@gmail.com>
| A | internal/ | 606 | ++++++++++++++++++++++ |
| A | internal/ | 105 | ++++ |
| A | internal/ | 819 | ++++++++++++++++++++++++++++++ |
| M | internal/ | 4 | + |
| M | internal/ | 6 | +- |
| M | internal/ | 1 | + |
| M | internal/ | 18 | +- |
| M | internal/ | 6 | + |
| M | internal/ | 6 | + |
| M | internal/ | 4 | +- |
| A | internal/ | 155 | ++++++ |
| A | internal/ | 90 | +++ |
@@ -0,0 +1,606 @@
1+package daemon2+3+// Custom views hand fenced code blocks to an external transform and keep4+// what comes back as relay-signed artifacts attached to the source event.5+// This file holds the definition, its storage and the owner's management6+// methods; custom_views_transform.go holds the block extraction, the POST7+// and the artifacts, and custom_views_http.go serves them.8+9+import (10+ "context"11+ "database/sql"12+ "encoding/json"13+ "errors"14+ "fmt"15+ "sort"16+ "strconv"17+ "strings"18+ "time"19+20+ "github.com/FelineStateMachine/tinyrelay/internal/event"21+ "github.com/FelineStateMachine/tinyrelay/internal/storage"22+ "github.com/FelineStateMachine/tinyrelay/internal/views"23+)24+25+const (26+ viewTransform = "view-transform"27+ viewKindMax = 1628+ viewLanguageMax = 3229+ viewLanguageLength = 3230+ viewDefaultMaxBytes = 1 << 2031+ viewMaxBytesCap = 4 << 2032+ viewTimeout = 30 * time.Second33+ viewMaxAttempts = 334+ viewPauseFailures = 2035+ viewBackfillLimit = 50036+ viewHourlyPeriod = int64(3600)37+ viewResponseMaxBytes = int64(viewMaxBytesCap*4 + 1<<20)38+)39+40+// viewBackoff is the wait before the second and third attempt.41+var viewBackoff = []time.Duration{time.Minute, 5 * time.Minute}42+43+const customViewSchema = `CREATE TABLE IF NOT EXISTS custom_views(name TEXT PRIMARY KEY, kinds TEXT NOT NULL, transform TEXT NOT NULL, trigger TEXT NOT NULL, audience TEXT NOT NULL, languages TEXT NOT NULL, max_bytes INTEGER NOT NULL, secret TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, last_run_at INTEGER NOT NULL DEFAULT 0, last_status TEXT NOT NULL DEFAULT '', failures INTEGER NOT NULL DEFAULT 0);44+CREATE TABLE IF NOT EXISTS custom_view_artifacts(view TEXT NOT NULL, hash TEXT NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, engine TEXT NOT NULL DEFAULT '', audience TEXT NOT NULL, content TEXT NOT NULL, raw TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY(view,hash));45+CREATE TABLE IF NOT EXISTS custom_view_sources(view TEXT NOT NULL, hash TEXT NOT NULL, source TEXT NOT NULL, block INTEGER NOT NULL, expires INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(view,hash,source));46+CREATE INDEX IF NOT EXISTS custom_view_sources_source ON custom_view_sources(view,source);`47+48+// customView is one stored definition. The secret never leaves the tenant49+// except once, in the addcustomview answer.50+type customView struct {51+ Name string `json:"name"`52+ Kinds []int `json:"kinds"`53+ Transform string `json:"transform"`54+ Trigger string `json:"trigger"`55+ Audience string `json:"audience"`56+ Languages []string `json:"languages"`57+ MaxBytes int `json:"maxBytes"`58+ Enabled bool `json:"enabled"`59+ CreatedAt int64 `json:"created"`60+ LastRunAt int64 `json:"lastRun"`61+ LastStatus string `json:"lastStatus"`62+ Failures int `json:"failures"`63+ secret string64+}65+66+func (v customView) summary() map[string]any {67+ state := "active"68+ if !v.Enabled {69+ state = "paused"70+ }71+ return map[string]any{"name": v.Name, "kinds": v.Kinds, "transform": v.Transform, "host": callbackHost(v.Transform), "trigger": v.Trigger, "audience": v.Audience, "languages": v.Languages, "maxBytes": v.MaxBytes, "enabled": v.Enabled, "state": state, "created": v.CreatedAt, "lastRun": v.LastRunAt, "lastStatus": v.LastStatus, "failures": v.Failures}72+}73+74+func (v customView) watches(kind int) bool {75+ for _, k := range v.Kinds {76+ if k == kind {77+ return true78+ }79+ }80+ return false81+}82+83+func (t *Tenant) initCustomViews(ctx context.Context) error {84+ _, err := t.store.DB().ExecContext(ctx, customViewSchema)85+ return err86+}87+88+func customViewMethod(method string) bool {89+ return containsString([]string{"listcustomviews", "addcustomview", "removecustomview", "runcustomview", "pausecustomview", "resumecustomview"}, method)90+}91+92+// customViewExecute serves the owner's custom view methods under the view93+// operation. Every change is recorded in the audit log.94+func (t *Tenant) customViewExecute(ctx context.Context, actor, method string, params []json.RawMessage) (result any, err error) {95+ ctx, finish := t.app.telemetry.Start(ctx, "view")96+ outcome := "error"97+ defer func() {98+ if err != nil {99+ outcome = callbackOutcome(err)100+ }101+ finish(outcome)102+ }()103+ role, err := t.community.Role(ctx, actor)104+ if err != nil {105+ return nil, err106+ }107+ if role != "owner" {108+ return nil, fmt.Errorf("restricted: only the owner may %s", method)109+ }110+ switch method {111+ case "listcustomviews":112+ rows, err := t.customViewRows(ctx)113+ if err != nil {114+ return nil, err115+ }116+ out := make([]map[string]any, 0, len(rows))117+ for _, row := range rows {118+ out = append(out, row.summary())119+ }120+ outcome = "ok"121+ return out, nil122+ case "addcustomview":123+ result, err = t.addCustomView(ctx, actor, params)124+ default:125+ result, err = t.changeCustomView(ctx, actor, method, params)126+ }127+ if err == nil {128+ outcome = "ok"129+ }130+ return result, err131+}132+133+// customViewOptions is the addcustomview parameter. Kinds and languages may134+// arrive as lists or as comma-separated strings from a plain form.135+type customViewOptions struct {136+ Name string `json:"name"`137+ Kinds json.RawMessage `json:"kinds"`138+ Transform string `json:"transform"`139+ Trigger string `json:"trigger"`140+ Audience string `json:"audience"`141+ Languages json.RawMessage `json:"languages"`142+ MaxBytes json.RawMessage `json:"max_bytes"`143+ Secret string `json:"secret"`144+}145+146+func (t *Tenant) addCustomView(ctx context.Context, actor string, params []json.RawMessage) (any, error) {147+ if len(params) != 1 {148+ return nil, errors.New("invalid: addcustomview expects one object with name, kinds, transform, trigger, audience, languages and optional max_bytes and secret")149+ }150+ var options customViewOptions151+ if err := json.Unmarshal(params[0], &options); err != nil {152+ return nil, errors.New("invalid: addcustomview expects one object with name, kinds, transform, trigger, audience, languages and optional max_bytes and secret")153+ }154+ view, err := t.parseCustomView(options)155+ if err != nil {156+ return nil, err157+ }158+ generated := false159+ secret := strings.TrimSpace(options.Secret)160+ if secret == "" {161+ secret, err = randomHex(32)162+ if err != nil {163+ return nil, err164+ }165+ generated = true166+ } else if len(secret) < callbackSecretMin || len(secret) > callbackSecretMax || strings.Trim(secret, "!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~") != "" {167+ return nil, fmt.Errorf("invalid: secret must be %d to %d printable ASCII characters", callbackSecretMin, callbackSecretMax)168+ }169+ view.secret = secret170+ view.Enabled = true171+ view.CreatedAt = time.Now().Unix()172+ kinds, _ := json.Marshal(view.Kinds)173+ languages, _ := json.Marshal(view.Languages)174+ _, err = t.store.DB().ExecContext(ctx, `INSERT INTO custom_views(name,kinds,transform,trigger,audience,languages,max_bytes,secret,enabled,created_at) VALUES(?,?,?,?,?,?,?,?,1,?)`, view.Name, string(kinds), view.Transform, view.Trigger, view.Audience, string(languages), view.MaxBytes, secret, view.CreatedAt)175+ if err != nil {176+ if strings.Contains(err.Error(), "UNIQUE") {177+ return nil, errors.New("invalid: a view with that name exists")178+ }179+ return nil, err180+ }181+ t.invalidateCustomViews()182+ if err := t.community.Record(ctx, actor, "addcustomview", view.Name, ""); err != nil {183+ return nil, err184+ }185+ // Counts only: the transform host, path and secret stay out of the log.186+ t.app.telemetry.Logger().Info("custom view added", "tenant", t.meta.Name, "kinds", len(view.Kinds), "languages", len(view.Languages), "generated", generated)187+ out := view.summary()188+ out["secret"] = secret189+ return out, nil190+}191+192+// parseCustomView validates a definition: the name, the kinds it watches,193+// the https transform, the trigger, the audience, the languages and the194+// artifact size limit.195+func (t *Tenant) parseCustomView(options customViewOptions) (customView, error) {196+ view := customView{Name: strings.TrimSpace(options.Name), Transform: strings.TrimSpace(options.Transform), Trigger: strings.TrimSpace(options.Trigger), Audience: strings.TrimSpace(options.Audience), MaxBytes: viewDefaultMaxBytes}197+ if !views.NamePattern.MatchString(view.Name) {198+ return customView{}, errors.New("invalid: name must be 1 to 32 lowercase letters, digits or hyphens")199+ }200+ kinds, err := intList(options.Kinds)201+ if err != nil || len(kinds) == 0 || len(kinds) > viewKindMax {202+ return customView{}, fmt.Errorf("invalid: kinds must list 1 to %d event kinds", viewKindMax)203+ }204+ for _, kind := range kinds {205+ if kind < 0 || kind > 65535 {206+ return customView{}, errors.New("invalid: kinds must be between 0 and 65535")207+ }208+ }209+ sort.Ints(kinds)210+ view.Kinds = dedupeInts(kinds)211+ if err := t.checkCallbackURL(view.Transform); err != nil {212+ return customView{}, fmt.Errorf("invalid: transform %s", strings.TrimPrefix(err.Error(), "invalid: "))213+ }214+ if view.Trigger == "" {215+ view.Trigger = "write"216+ }217+ if view.Trigger != "write" && view.Trigger != "hourly" {218+ return customView{}, errors.New("invalid: trigger must be write or hourly")219+ }220+ if view.Audience == "" {221+ view.Audience = "public"222+ }223+ if view.Audience != "public" && view.Audience != "members" {224+ return customView{}, errors.New("invalid: audience must be public or members")225+ }226+ languages, err := stringList(options.Languages)227+ if err != nil || len(languages) == 0 || len(languages) > viewLanguageMax {228+ return customView{}, fmt.Errorf("invalid: languages must list 1 to %d fenced block languages", viewLanguageMax)229+ }230+ seen := map[string]bool{}231+ view.Languages = nil232+ for _, lang := range languages {233+ lang = strings.ToLower(strings.TrimSpace(lang))234+ if lang == "" || len(lang) > viewLanguageLength || strings.Trim(lang, "abcdefghijklmnopqrstuvwxyz0123456789+-_.#") != "" {235+ return customView{}, errors.New("invalid: languages must be lowercase words of letters, digits, +, -, _, . or #")236+ }237+ if !seen[lang] {238+ seen[lang] = true239+ view.Languages = append(view.Languages, lang)240+ }241+ }242+ if len(options.MaxBytes) > 0 && string(options.MaxBytes) != "null" && string(options.MaxBytes) != `""` {243+ var size int244+ var text string245+ if json.Unmarshal(options.MaxBytes, &size) != nil {246+ if json.Unmarshal(options.MaxBytes, &text) != nil {247+ return customView{}, errors.New("invalid: max_bytes must be a number")248+ }249+ if size, err = strconv.Atoi(strings.TrimSpace(text)); err != nil {250+ return customView{}, errors.New("invalid: max_bytes must be a number")251+ }252+ }253+ if size < 1 || size > viewMaxBytesCap {254+ return customView{}, fmt.Errorf("invalid: max_bytes must be between 1 and %d", viewMaxBytesCap)255+ }256+ view.MaxBytes = size257+ }258+ return view, nil259+}260+261+// intList reads a JSON list of integers, a list of numeric strings or a262+// comma-separated string.263+func intList(raw json.RawMessage) ([]int, error) {264+ if len(raw) == 0 {265+ return nil, nil266+ }267+ var out []int268+ if json.Unmarshal(raw, &out) == nil {269+ return out, nil270+ }271+ items, err := stringList(raw)272+ if err != nil {273+ return nil, err274+ }275+ for _, item := range items {276+ value, err := strconv.Atoi(strings.TrimSpace(item))277+ if err != nil {278+ return nil, err279+ }280+ out = append(out, value)281+ }282+ return out, nil283+}284+285+// stringList reads a JSON list of strings or a comma-separated string.286+func stringList(raw json.RawMessage) ([]string, error) {287+ if len(raw) == 0 {288+ return nil, nil289+ }290+ var out []string291+ if json.Unmarshal(raw, &out) == nil {292+ return out, nil293+ }294+ var text string295+ if err := json.Unmarshal(raw, &text); err != nil {296+ return nil, err297+ }298+ for _, item := range strings.FieldsFunc(text, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' }) {299+ if item = strings.TrimSpace(item); item != "" {300+ out = append(out, item)301+ }302+ }303+ return out, nil304+}305+306+func dedupeInts(values []int) []int {307+ out := values[:0]308+ for i, value := range values {309+ if i == 0 || value != values[i-1] {310+ out = append(out, value)311+ }312+ }313+ return out314+}315+316+func (t *Tenant) changeCustomView(ctx context.Context, actor, method string, params []json.RawMessage) (any, error) {317+ name := customViewNameParam(params)318+ if name == "" {319+ return nil, errors.New("invalid: view name required")320+ }321+ view, err := t.customViewByName(ctx, name)322+ if err != nil {323+ return nil, err324+ }325+ switch method {326+ case "removecustomview":327+ if err := t.removeCustomView(ctx, view); err != nil {328+ return nil, err329+ }330+ case "pausecustomview":331+ _, err = t.store.DB().ExecContext(ctx, `UPDATE custom_views SET enabled=0, last_status='paused' WHERE name=?`, name)332+ view.Enabled, view.LastStatus = false, "paused"333+ case "resumecustomview":334+ _, err = t.store.DB().ExecContext(ctx, `UPDATE custom_views SET enabled=1, failures=0, last_status='' WHERE name=?`, name)335+ view.Enabled, view.Failures, view.LastStatus = true, 0, ""336+ case "runcustomview":337+ if !view.Enabled {338+ return nil, errors.New("invalid: resume the view before running it")339+ }340+ queued, err := t.queueCustomViewBackfill(ctx, view, 0, strconv.FormatInt(time.Now().UnixNano(), 10))341+ if err != nil {342+ return nil, err343+ }344+ if err := t.community.Record(ctx, actor, method, name, strconv.Itoa(queued)); err != nil {345+ return nil, err346+ }347+ t.app.telemetry.Logger().Info("custom view run queued", "tenant", t.meta.Name, "queued", queued)348+ out := view.summary()349+ out["queued"] = queued350+ return out, nil351+ default:352+ return nil, fmt.Errorf("unsupported: management method %q", method)353+ }354+ if err != nil {355+ return nil, err356+ }357+ t.invalidateCustomViews()358+ if err := t.community.Record(ctx, actor, method, name, ""); err != nil {359+ return nil, err360+ }361+ if method == "removecustomview" {362+ return map[string]any{"name": name, "removed": true}, nil363+ }364+ return view.summary(), nil365+}366+367+// removeCustomView deletes the definition and every artifact it produced,368+// including the stored public records.369+func (t *Tenant) removeCustomView(ctx context.Context, view customView) error {370+ ids, err := t.customViewArtifactIDs(ctx, `SELECT event_id FROM custom_view_artifacts WHERE view=?`, view.Name)371+ if err != nil {372+ return err373+ }374+ if err := t.store.WithTx(ctx, func(tx *sql.Tx) error {375+ for _, table := range []string{"custom_view_sources", "custom_view_artifacts", "custom_views"} {376+ column := "view"377+ if table == "custom_views" {378+ column = "name"379+ }380+ if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+column+"=?", view.Name); err != nil {381+ return err382+ }383+ }384+ return nil385+ }); err != nil {386+ return err387+ }388+ return t.deleteStoredArtifacts(ctx, ids)389+}390+391+// customViewNameParam reads the name from a bare string or from {"name": ...}.392+func customViewNameParam(params []json.RawMessage) string {393+ if len(params) == 0 {394+ return ""395+ }396+ if name := stringParam(params, 0); name != "" {397+ return name398+ }399+ var options struct {400+ Name string `json:"name"`401+ }402+ _ = json.Unmarshal(params[0], &options)403+ return strings.TrimSpace(options.Name)404+}405+406+func (t *Tenant) customViewByName(ctx context.Context, name string) (customView, error) {407+ rows, err := t.customViewRows(ctx, name)408+ if err != nil {409+ return customView{}, err410+ }411+ if len(rows) == 0 {412+ return customView{}, errors.New("not found: view")413+ }414+ return rows[0], nil415+}416+417+// customViewRows lists definitions, narrowed to one name.418+func (t *Tenant) customViewRows(ctx context.Context, names ...string) ([]customView, error) {419+ query := `SELECT name,kinds,transform,trigger,audience,languages,max_bytes,secret,enabled,created_at,last_run_at,last_status,failures FROM custom_views`420+ args := []any{}421+ if len(names) > 0 {422+ query += ` WHERE name=?`423+ args = append(args, names[0])424+ }425+ query += ` ORDER BY created_at,name`426+ rows, err := t.store.DB().QueryContext(ctx, query, args...)427+ if err != nil {428+ return nil, fmt.Errorf("list custom views: %w", err)429+ }430+ defer rows.Close()431+ var out []customView432+ for rows.Next() {433+ var item customView434+ var kinds, languages string435+ var enabled int436+ if err := rows.Scan(&item.Name, &kinds, &item.Transform, &item.Trigger, &item.Audience, &languages, &item.MaxBytes, &item.secret, &enabled, &item.CreatedAt, &item.LastRunAt, &item.LastStatus, &item.Failures); err != nil {437+ return nil, err438+ }439+ item.Enabled = enabled != 0440+ if err := json.Unmarshal([]byte(kinds), &item.Kinds); err != nil {441+ return nil, fmt.Errorf("custom view kinds: %w", err)442+ }443+ if err := json.Unmarshal([]byte(languages), &item.Languages); err != nil {444+ return nil, fmt.Errorf("custom view languages: %w", err)445+ }446+ out = append(out, item)447+ }448+ return out, rows.Err()449+}450+451+// customViewIndex returns the enabled views indexed by kind, built on first452+// use and dropped whenever a view changes.453+func (t *Tenant) customViewIndex(ctx context.Context) map[int][]customView {454+ t.viewMu.Lock()455+ defer t.viewMu.Unlock()456+ if t.viewIndex != nil {457+ return t.viewIndex458+ }459+ index := map[int][]customView{}460+ rows, err := t.customViewRows(ctx)461+ if err != nil {462+ t.app.telemetry.Logger().Error("load custom views", "tenant", t.meta.Name, "error", err)463+ return index464+ }465+ for _, row := range rows {466+ if !row.Enabled {467+ continue468+ }469+ for _, kind := range row.Kinds {470+ index[kind] = append(index[kind], row)471+ }472+ }473+ t.viewIndex = index474+ return index475+}476+477+func (t *Tenant) invalidateCustomViews() {478+ t.viewMu.Lock()479+ t.viewIndex = nil480+ t.viewMu.Unlock()481+}482+483+// CustomViews is the summary the web UI renders with: the enabled views'484+// names and languages, nothing else.485+func (t *Tenant) CustomViews() []views.View {486+ seen := map[string]bool{}487+ var out []views.View488+ for _, list := range t.customViewIndex(context.Background()) {489+ for _, view := range list {490+ if !seen[view.Name] {491+ seen[view.Name] = true492+ out = append(out, views.View{Name: view.Name, Languages: append([]string(nil), view.Languages...)})493+ }494+ }495+ }496+ sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })497+ return out498+}499+500+// queueCustomViews runs after an event is stored or released. Each enabled501+// write-triggered view that watches the kind and finds a block it renders502+// gets one transform intent. It never fails the publish.503+func (t *Tenant) queueCustomViews(ctx context.Context, e event.Event) {504+ candidates := t.customViewIndex(ctx)[e.Kind]505+ if len(candidates) == 0 {506+ return507+ }508+ if e.Kind == event.KIND_REPO_STATE && (t.git == nil || t.git.IsPending(e.ID)) {509+ // The README is read from the objects, which have not arrived yet;510+ // releaseGit queues the state once they have.511+ return512+ }513+ var intents []storage.Intent514+ for _, view := range candidates {515+ if view.Trigger != "write" {516+ continue517+ }518+ if e.Kind != event.KIND_REPO_STATE && len(views.Matching(views.Blocks(e.Content), view.Languages)) == 0 {519+ continue520+ }521+ intents = append(intents, t.viewIntent(view, e, ""))522+ }523+ if len(intents) == 0 {524+ return525+ }526+ err := t.store.WithTx(ctx, func(tx *sql.Tx) error {527+ return storage.AddIntents(ctx, tx, intents, time.Now().Unix())528+ })529+ if err != nil {530+ t.app.telemetry.Logger().Error("view transforms not queued", "tenant", t.meta.Name, "error", err)531+ return532+ }533+ t.app.telemetry.Logger().Debug("view transforms queued", "tenant", t.meta.Name, "queued", len(intents))534+}535+536+// viewIntent is the transform intent for one source. The event travels in537+// the payload; run makes a backfill or an hourly pass distinct from the538+// write-time intent for the same event.539+func (t *Tenant) viewIntent(view customView, e event.Event, run string) storage.Intent {540+ payload, _ := json.Marshal(viewPayload{Event: e, Attempt: 1, Run: run})541+ id := e.ID542+ if run != "" {543+ id += "@" + run544+ }545+ return storage.Intent{Kind: viewTransform, EventID: id, Target: view.Name, Payload: string(payload)}546+}547+548+// queueCustomViewBackfill queues the newest sources of a view's kinds, at549+// most 500, written since the given time. Repository states are queued550+// whether or not their README has blocks; the handler reads it.551+func (t *Tenant) queueCustomViewBackfill(ctx context.Context, view customView, since int64, run string) (int, error) {552+ rows, err := t.store.Query(ctx, event.Filter{Kinds: view.Kinds, Tags: map[string][]string{}}, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: viewBackfillLimit})553+ if err != nil {554+ return 0, err555+ }556+ var intents []storage.Intent557+ for _, e := range rows.Events {558+ if e.CreatedAt < since {559+ continue560+ }561+ if e.Kind == event.KIND_REPO_STATE {562+ if t.git == nil || t.git.IsPending(e.ID) {563+ continue564+ }565+ } else if len(views.Matching(views.Blocks(e.Content), view.Languages)) == 0 {566+ continue567+ }568+ intents = append(intents, t.viewIntent(view, e, run))569+ }570+ if len(intents) == 0 {571+ return 0, nil572+ }573+ err = t.store.WithTx(ctx, func(tx *sql.Tx) error {574+ return storage.AddIntents(ctx, tx, intents, time.Now().Unix())575+ })576+ return len(intents), err577+}578+579+// tickCustomViews queues the hourly views that are due. It is called from580+// the tenant scheduler on its own timer.581+func (t *Tenant) tickCustomViews(ctx context.Context, now int64) error {582+ rows, err := t.customViewRows(ctx)583+ if err != nil {584+ return err585+ }586+ for _, view := range rows {587+ if !view.Enabled || view.Trigger != "hourly" || now-view.LastRunAt < viewHourlyPeriod {588+ continue589+ }590+ since := view.LastRunAt - 300591+ if view.LastRunAt == 0 {592+ since = 0593+ }594+ queued, err := t.queueCustomViewBackfill(ctx, view, since, strconv.FormatInt(now/viewHourlyPeriod, 10))595+ if err != nil {596+ return err597+ }598+ if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET last_run_at=? WHERE name=?`, now, view.Name); err != nil {599+ return err600+ }601+ if queued > 0 {602+ t.app.telemetry.Logger().Info("custom view hourly run queued", "tenant", t.meta.Name, "queued", queued)603+ }604+ }605+ return nil606+}@@ -0,0 +1,105 @@
1+package daemon2+3+// GET /views/<name>/<hash>.<svg|png> serves a custom view artifact under a4+// sandbox policy. A members-only view answers only members; a public one5+// follows the relay's read rule.6+7+import (8+ "net/http"9+ "strconv"10+ "strings"11+12+ "github.com/FelineStateMachine/tinyrelay/internal/views"13+)14+15+func isViewArtifactPath(path string) bool {16+ return strings.HasPrefix(path, "/views/")17+}18+19+func (t *Tenant) viewArtifactHTTP(w http.ResponseWriter, r *http.Request) {20+ ctx, finish := t.app.telemetry.Start(r.Context(), "view")21+ r = r.WithContext(ctx)22+ outcome := "error"23+ defer func() { finish(outcome) }()24+ if r.Method != http.MethodGet && r.Method != http.MethodHead {25+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)26+ outcome = "invalid"27+ return28+ }29+ parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/views/"), "/")30+ if len(parts) != 2 {31+ http.NotFound(w, r)32+ outcome = "invalid"33+ return34+ }35+ name := parts[0]36+ hash, extension, _ := strings.Cut(parts[1], ".")37+ if !views.NamePattern.MatchString(name) || !views.HashPattern.MatchString(hash) || (extension != "svg" && extension != "png") {38+ http.NotFound(w, r)39+ outcome = "invalid"40+ return41+ }42+ actor, err := t.resolveUIActor(r)43+ if err != nil {44+ http.Error(w, err.Error(), http.StatusUnauthorized)45+ outcome = "unauthorized"46+ return47+ }48+ record, err := t.customViewArtifact(ctx, name, hash)49+ if err != nil {50+ if strings.HasPrefix(err.Error(), "not found:") {51+ http.NotFound(w, r)52+ outcome = "invalid"53+ return54+ }55+ http.Error(w, err.Error(), http.StatusInternalServerError)56+ return57+ }58+ wantType := map[string]string{"svg": "image/svg+xml", "png": "image/png"}[extension]59+ if record.Type != wantType {60+ http.NotFound(w, r)61+ outcome = "invalid"62+ return63+ }64+ if record.Audience == "members" {65+ role, err := t.community.Role(ctx, actor)66+ if err != nil {67+ http.Error(w, err.Error(), http.StatusInternalServerError)68+ return69+ }70+ if actor == "" {71+ http.Error(w, "auth-required: this view is for members", http.StatusUnauthorized)72+ outcome = "unauthorized"73+ return74+ }75+ if role != "member" && role != "moderator" && role != "owner" {76+ http.Error(w, "restricted: this view is for members", http.StatusForbidden)77+ outcome = "unauthorized"78+ return79+ }80+ } else if err := t.browseRead(ctx, actor); err != nil {81+ browseHTTPError(w, err)82+ outcome = "unauthorized"83+ return84+ }85+ body, err := record.body()86+ if err != nil {87+ http.Error(w, "artifact is unreadable", http.StatusInternalServerError)88+ return89+ }90+ w.Header().Set("Content-Type", record.Type)91+ w.Header().Set("Content-Length", strconv.Itoa(len(body)))92+ w.Header().Set("Content-Security-Policy", "sandbox; default-src 'none'; style-src 'unsafe-inline'")93+ w.Header().Set("X-Content-Type-Options", "nosniff")94+ if record.Audience == "members" {95+ w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")96+ } else {97+ w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")98+ }99+ outcome = "ok"100+ if r.Method == http.MethodHead {101+ w.WriteHeader(http.StatusOK)102+ return103+ }104+ _, _ = w.Write(body)105+}@@ -0,0 +1,819 @@
1+package daemon2+3+// Custom view transforms. Each source event is one POST of its fenced4+// blocks to the view's transform, signed with the view's secret. What comes5+// back is checked, signed by the relay and attached to the source. A failed6+// POST is tried again after one minute and then after five; after 207+// failures in a row the view is paused until the owner resumes it.8+9+import (10+ "bytes"11+ "context"12+ "crypto/sha256"13+ "crypto/tls"14+ "database/sql"15+ "encoding/base64"16+ "encoding/hex"17+ "encoding/json"18+ "errors"19+ "fmt"20+ "io"21+ "net/http"22+ "regexp"23+ "sort"24+ "strconv"25+ "strings"26+ "sync"27+ "time"28+29+ "github.com/FelineStateMachine/tinyrelay/internal/event"30+ "github.com/FelineStateMachine/tinyrelay/internal/gitrelay"31+ "github.com/FelineStateMachine/tinyrelay/internal/replication"32+ "github.com/FelineStateMachine/tinyrelay/internal/views"33+ "github.com/FelineStateMachine/tinyrelay/internal/work"34+)35+36+// viewPayload is the work intent body: the source event, so the run does37+// not depend on the event still being stored, the attempt number and the38+// run that queued it.39+type viewPayload struct {40+ Event event.Event `json:"event"`41+ Attempt int `json:"attempt"`42+ Run string `json:"run,omitempty"`43+}44+45+// viewRequest is the body posted to a transform: only the blocks, with the46+// source named so the transform can correlate. No event content, tags,47+// author or time leaves the relay.48+type viewRequest struct {49+ Relay string `json:"relay"`50+ View string `json:"view"`51+ Source viewSource `json:"source"`52+ Blocks []views.Block `json:"blocks"`53+}54+55+type viewSource struct {56+ ID string `json:"id"`57+ Kind int `json:"kind"`58+}59+60+// viewResponse is what a transform answers with.61+type viewResponse struct {62+ Artifacts []viewArtifact `json:"artifacts"`63+ Errors []viewError `json:"errors"`64+}65+66+type viewArtifact struct {67+ Block int `json:"block"`68+ Type string `json:"type"`69+ Body string `json:"body"`70+ Engine string `json:"engine"`71+}72+73+type viewError struct {74+ Block int `json:"block"`75+ Error string `json:"error"`76+}77+78+// viewSourceKey names a source in the artifact tables: the event id, or the79+// coordinate of a repository state so a new state replaces its README.80+func viewSourceKey(e event.Event) string {81+ if e.Kind == event.KIND_REPO_STATE {82+ return strconv.Itoa(e.Kind) + ":" + e.PubKey + ":" + event.Tag(e, "d")83+ }84+ return e.ID85+}86+87+// viewSourceTag is the tag an artifact carries for a source.88+func viewSourceTag(source string) []string {89+ if strings.HasPrefix(source, strconv.Itoa(event.KIND_REPO_STATE)+":") {90+ return []string{"a", source}91+ }92+ return []string{"e", source}93+}94+95+// viewHTTPClient refuses private addresses and redirects unless the relay96+// itself runs on a loopback address.97+func (t *Tenant) viewHTTPClient() *http.Client {98+ if t.viewClient != nil {99+ return t.viewClient100+ }101+ if t.loopbackRelay() {102+ return &http.Client{Timeout: viewTimeout, Transport: &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}103+ }104+ return replication.NewPinnedClient(viewTimeout)105+}106+107+// viewLock bounds work to one transform run per view at a time.108+func (t *Tenant) viewLock(name string) *sync.Mutex {109+ t.viewMu.Lock()110+ defer t.viewMu.Unlock()111+ if t.viewLocks == nil {112+ t.viewLocks = map[string]*sync.Mutex{}113+ }114+ lock := t.viewLocks[name]115+ if lock == nil {116+ lock = &sync.Mutex{}117+ t.viewLocks[name] = lock118+ }119+ return lock120+}121+122+// sourceBlocks finds the blocks of a source: the fenced blocks of its123+// content, or of the README at the head of a repository state.124+func (t *Tenant) sourceBlocks(ctx context.Context, e event.Event) ([]views.Block, error) {125+ if e.Kind != event.KIND_REPO_STATE {126+ return views.Blocks(e.Content), nil127+ }128+ readme, err := t.stateReadme(ctx, e)129+ if err != nil {130+ return nil, err131+ }132+ return views.Blocks(readme), nil133+}134+135+// stateReadme reads README.md at the head named by a repository state. A136+// missing README yields no content.137+func (t *Tenant) stateReadme(ctx context.Context, e event.Event) (string, error) {138+ if t.git == nil {139+ return "", nil140+ }141+ head := strings.TrimPrefix(event.Tag(e, "HEAD"), "ref: ")142+ commit := ""143+ for _, tag := range e.Tags {144+ if len(tag) >= 2 && tag[0] == head {145+ commit = tag[1]146+ }147+ }148+ if head == "" || commit == "" {149+ return "", nil150+ }151+ for _, name := range []string{"README.md", "readme.md", "README"} {152+ page, err := t.git.Browse(ctx, gitrelay.BrowseRequest{Owner: e.PubKey, Repo: event.Tag(e, "d"), Ref: commit, Path: name, View: "file", Limit: 1})153+ if err != nil {154+ if strings.HasPrefix(err.Error(), "not found:") || strings.HasPrefix(err.Error(), "invalid:") {155+ continue156+ }157+ return "", err158+ }159+ if page.Binary || page.Content == "" {160+ continue161+ }162+ return page.Content, nil163+ }164+ return "", nil165+}166+167+// sourceCurrent reports whether the source is still the stored event: an168+// event that was deleted, or a state that a newer state replaced, renders169+// nothing.170+func (t *Tenant) sourceCurrent(ctx context.Context, e event.Event) (bool, error) {171+ var id string172+ var err error173+ if e.Kind == event.KIND_REPO_STATE {174+ err = t.store.DB().QueryRowContext(ctx, `SELECT id FROM events WHERE kind=? AND pubkey=? AND d=?`, e.Kind, e.PubKey, event.Tag(e, "d")).Scan(&id)175+ } else {176+ err = t.store.DB().QueryRowContext(ctx, `SELECT id FROM events WHERE id=?`, e.ID).Scan(&id)177+ }178+ if errors.Is(err, sql.ErrNoRows) {179+ return false, nil180+ }181+ return id == e.ID, err182+}183+184+// handleViewTransform serves one view-transform intent. It rechecks the185+// view and the source right before the POST, attaches artifacts that186+// already exist for a block's hash and posts only the blocks that need187+// rendering.188+func (t *Tenant) handleViewTransform(ctx context.Context, intent work.Intent) (err error) {189+ ctx, finish := t.app.telemetry.Start(ctx, "view")190+ outcome := "error"191+ defer func() { finish(outcome) }()192+ var payload viewPayload193+ if err := json.Unmarshal([]byte(intent.Payload), &payload); err != nil || payload.Event.ID == "" {194+ outcome = "invalid"195+ return errors.New("view-transform: invalid payload")196+ }197+ if payload.Attempt < 1 {198+ payload.Attempt = 1199+ }200+ lock := t.viewLock(intent.Target)201+ lock.Lock()202+ defer lock.Unlock()203+ view, err := t.customViewByName(ctx, intent.Target)204+ if err != nil {205+ if strings.HasPrefix(err.Error(), "not found:") {206+ outcome = "paused"207+ return nil208+ }209+ return err210+ }211+ if !view.Enabled {212+ outcome = "paused"213+ return nil214+ }215+ current, err := t.sourceCurrent(ctx, payload.Event)216+ if err != nil {217+ return err218+ }219+ if !current {220+ outcome = "invalid"221+ return nil222+ }223+ blocks, err := t.sourceBlocks(ctx, payload.Event)224+ if err != nil {225+ return err226+ }227+ blocks = views.Matching(blocks, view.Languages)228+ source := viewSourceKey(payload.Event)229+ expires := event.Expiration(payload.Event)230+ // Attach the blocks that already have an artifact and keep the rest.231+ var pending []views.Block232+ hashes := make([]string, 0, len(blocks))233+ for _, block := range blocks {234+ hash := views.Hash(block.Lang, block.Source)235+ hashes = append(hashes, hash)236+ exists, err := t.attachArtifact(ctx, view, hash, source, block.Index, expires)237+ if err != nil {238+ return err239+ }240+ if !exists {241+ pending = append(pending, block)242+ }243+ }244+ if err := t.detachStaleArtifacts(ctx, view.Name, source, hashes); err != nil {245+ return err246+ }247+ if len(pending) == 0 {248+ outcome = "ok"249+ return t.pruneViewArtifacts(ctx, view.Name, intent.ID)250+ }251+ status, response, postErr := t.postViewTransform(ctx, view, payload.Event, pending)252+ now := time.Now().Unix()253+ if postErr == nil {254+ stored, refused := 0, 0255+ sent := map[int]views.Block{}256+ for _, block := range pending {257+ sent[block.Index] = block258+ }259+ for _, artifact := range response.Artifacts {260+ block, ok := sent[artifact.Block]261+ if !ok {262+ refused++263+ continue264+ }265+ if err := t.storeArtifact(ctx, view, block, artifact, source, expires, now); err != nil {266+ if strings.HasPrefix(err.Error(), "invalid:") {267+ refused++268+ continue269+ }270+ return err271+ }272+ stored++273+ }274+ outcome = "ok"275+ if refused > 0 {276+ outcome = "invalid"277+ }278+ if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET last_run_at=?, last_status='ok', failures=0 WHERE name=?`, now, view.Name); err != nil {279+ return err280+ }281+ // Counts only: no source, block or body leaves the relay in a log.282+ t.app.telemetry.Logger().Info("view transform delivered", "tenant", t.meta.Name, "attempt", payload.Attempt, "status", status, "blocks", len(pending), "artifacts", stored, "refused", refused, "errors", len(response.Errors))283+ return t.pruneViewArtifacts(ctx, view.Name, intent.ID)284+ }285+ reason := postErr.Error()286+ failures := view.Failures + 1287+ paused := failures >= viewPauseFailures288+ if paused {289+ outcome = "paused"290+ if err := t.pauseCustomView(ctx, view.Name, failures, fmt.Sprintf("paused after %d failures: %s", failures, reason)); err != nil {291+ return err292+ }293+ } else if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET last_run_at=?, failures=?, last_status=? WHERE name=?`, now, failures, reason, view.Name); err != nil {294+ return err295+ }296+ t.app.telemetry.Logger().Info("view transform failed", "tenant", t.meta.Name, "attempt", payload.Attempt, "status", status, "failures", failures, "paused", paused)297+ if paused || payload.Attempt >= viewMaxAttempts {298+ return nil299+ }300+ return t.retryViewTransform(ctx, intent, payload)301+}302+303+// pauseCustomView stops the view and records the count and reason, then304+// drops the index so no new intents are queued for it.305+func (t *Tenant) pauseCustomView(ctx context.Context, name string, failures int, status string) error {306+ if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET enabled=0, failures=?, last_status=? WHERE name=?`, failures, status, name); err != nil {307+ return err308+ }309+ t.invalidateCustomViews()310+ return nil311+}312+313+// retryViewTransform queues the next attempt after its backoff as its own314+// intent so the durable queue's ordering and fencing still apply.315+func (t *Tenant) retryViewTransform(ctx context.Context, intent work.Intent, payload viewPayload) error {316+ delay := viewBackoff[len(viewBackoff)-1]317+ if payload.Attempt-1 < len(viewBackoff) {318+ delay = viewBackoff[payload.Attempt-1]319+ }320+ payload.Attempt++321+ encoded, err := json.Marshal(payload)322+ if err != nil {323+ return err324+ }325+ now := time.Now().Unix()326+ key := sha256.Sum256([]byte(viewTransform + "\x00" + intent.EventID + "\x00" + intent.Target + "\x00" + fmt.Sprint(payload.Attempt)))327+ return t.store.WithTx(ctx, func(tx *sql.Tx) error {328+ _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO work_intents(id,kind,event_id,target,payload,next_at,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`, hex.EncodeToString(key[:]), viewTransform, intent.EventID+"#"+fmt.Sprint(payload.Attempt), intent.Target, string(encoded), now+int64(delay/time.Second), now, now)329+ return err330+ })331+}332+333+// postViewTransform sends the blocks and reads the answer. It returns the334+// HTTP status with a bounded reason on failure; a response that is not335+// JSON counts as a failure.336+func (t *Tenant) postViewTransform(ctx context.Context, view customView, e event.Event, blocks []views.Block) (int, viewResponse, error) {337+ body, err := json.Marshal(viewRequest{Relay: t.publicURL, View: view.Name, Source: viewSource{ID: viewSourceKey(e), Kind: e.Kind}, Blocks: blocks})338+ if err != nil {339+ return 0, viewResponse{}, errors.New("encode request")340+ }341+ ctx, cancel := context.WithTimeout(ctx, viewTimeout)342+ defer cancel()343+ request, err := http.NewRequestWithContext(ctx, http.MethodPost, view.Transform, bytes.NewReader(body))344+ if err != nil {345+ return 0, viewResponse{}, errors.New("bad url")346+ }347+ request.Header.Set("Content-Type", "application/json")348+ request.Header.Set("Accept", "application/json")349+ request.Header.Set("X-Tiny-View", view.Name)350+ request.Header.Set("X-Tiny-Relay", t.publicURL)351+ request.Header.Set("X-Tiny-Signature", callbackSignature(view.secret, body))352+ request.Header.Set("User-Agent", "tinyrelay")353+ response, err := t.viewHTTPClient().Do(request)354+ if err != nil {355+ if errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {356+ return 0, viewResponse{}, errors.New("timeout")357+ }358+ return 0, viewResponse{}, errors.New("connection failed")359+ }360+ defer response.Body.Close()361+ raw, err := io.ReadAll(io.LimitReader(response.Body, viewResponseMaxBytes))362+ if err != nil {363+ return response.StatusCode, viewResponse{}, errors.New("read failed")364+ }365+ if response.StatusCode < 200 || response.StatusCode >= 300 {366+ return response.StatusCode, viewResponse{}, fmt.Errorf("HTTP %d", response.StatusCode)367+ }368+ var decoded viewResponse369+ if err := json.Unmarshal(raw, &decoded); err != nil {370+ return response.StatusCode, viewResponse{}, errors.New("invalid response")371+ }372+ return response.StatusCode, decoded, nil373+}374+375+var (376+ svgScript = regexp.MustCompile(`(?i)<script`)377+ svgHandler = regexp.MustCompile(`(?i)\bon[a-z]+\s*=`)378+ svgForeignObject = regexp.MustCompile(`(?i)<foreignObject`)379+ svgReference = regexp.MustCompile(`(?i)(?:xlink:)?href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))`)380+ pngSignature = []byte("\x89PNG\r\n\x1a\n")381+)382+383+// checkArtifact validates a transform's artifact: an accepted type, a body384+// within the view's size limit, a PNG that decodes from base64 and an SVG385+// without script, handlers, foreign objects or references outside itself.386+// It returns the body as stored: the SVG text or the PNG in base64.387+func checkArtifact(artifact viewArtifact, maxBytes int) (string, error) {388+ switch artifact.Type {389+ case "image/svg+xml":390+ if len(artifact.Body) > maxBytes {391+ return "", errors.New("invalid: artifact exceeds the size limit")392+ }393+ if err := checkSVG(artifact.Body); err != nil {394+ return "", err395+ }396+ return artifact.Body, nil397+ case "image/png":398+ body := strings.TrimSpace(artifact.Body)399+ decoded, err := base64.StdEncoding.DecodeString(body)400+ if err != nil {401+ if decoded, err = base64.RawStdEncoding.DecodeString(body); err != nil {402+ return "", errors.New("invalid: png body must be base64")403+ }404+ }405+ if len(decoded) > maxBytes {406+ return "", errors.New("invalid: artifact exceeds the size limit")407+ }408+ if !bytes.HasPrefix(decoded, pngSignature) {409+ return "", errors.New("invalid: png body is not a PNG")410+ }411+ return base64.StdEncoding.EncodeToString(decoded), nil412+ default:413+ return "", errors.New("invalid: artifact type must be image/svg+xml or image/png")414+ }415+}416+417+func checkSVG(body string) error {418+ if !strings.Contains(strings.ToLower(body), "<svg") {419+ return errors.New("invalid: svg body has no svg element")420+ }421+ if svgScript.MatchString(body) {422+ return errors.New("invalid: svg contains script")423+ }424+ if svgHandler.MatchString(body) {425+ return errors.New("invalid: svg contains an event handler")426+ }427+ if svgForeignObject.MatchString(body) {428+ return errors.New("invalid: svg contains a foreign object")429+ }430+ for _, match := range svgReference.FindAllStringSubmatch(body, -1) {431+ target := strings.TrimSpace(match[1] + match[2] + match[3])432+ if target == "" || strings.HasPrefix(target, "#") {433+ continue434+ }435+ if strings.HasPrefix(strings.ToLower(target), "javascript:") {436+ return errors.New("invalid: svg contains a javascript reference")437+ }438+ return errors.New("invalid: svg references an external target")439+ }440+ return nil441+}442+443+// storeArtifact checks one artifact and keeps it as a signed record444+// attached to its source.445+func (t *Tenant) storeArtifact(ctx context.Context, view customView, block views.Block, artifact viewArtifact, source string, expires int64, now int64) error {446+ content, err := checkArtifact(artifact, view.MaxBytes)447+ if err != nil {448+ return err449+ }450+ engine := strings.TrimSpace(artifact.Engine)451+ if len(engine) > 64 {452+ engine = engine[:64]453+ }454+ hash := views.Hash(block.Lang, block.Source)455+ err = t.store.WithTx(ctx, func(tx *sql.Tx) error {456+ if _, err := tx.ExecContext(ctx, `INSERT INTO custom_view_artifacts(view,hash,event_id,type,engine,audience,content,raw,created_at) VALUES(?,?,'',?,?,?,?,'',?) ON CONFLICT(view,hash) DO UPDATE SET type=excluded.type, engine=excluded.engine, audience=excluded.audience, content=excluded.content, created_at=excluded.created_at`, view.Name, hash, artifact.Type, engine, view.Audience, content, now); err != nil {457+ return err458+ }459+ _, err := tx.ExecContext(ctx, `INSERT INTO custom_view_sources(view,hash,source,block,expires) VALUES(?,?,?,?,?) ON CONFLICT(view,hash,source) DO UPDATE SET block=excluded.block, expires=excluded.expires`, view.Name, hash, source, block.Index, expires)460+ return err461+ })462+ if err != nil {463+ return err464+ }465+ return t.signArtifact(ctx, view, hash, now)466+}467+468+// attachArtifact reports whether an artifact exists for the hash and, when469+// it does, attaches the source to it through a fresh signed record.470+func (t *Tenant) attachArtifact(ctx context.Context, view customView, hash, source string, block int, expires int64) (bool, error) {471+ var exists int472+ err := t.store.DB().QueryRowContext(ctx, `SELECT 1 FROM custom_view_artifacts WHERE view=? AND hash=?`, view.Name, hash).Scan(&exists)473+ if errors.Is(err, sql.ErrNoRows) {474+ return false, nil475+ }476+ if err != nil {477+ return false, err478+ }479+ var current int480+ var currentBlock int481+ var currentExpires int64482+ err = t.store.DB().QueryRowContext(ctx, `SELECT 1,block,expires FROM custom_view_sources WHERE view=? AND hash=? AND source=?`, view.Name, hash, source).Scan(¤t, ¤tBlock, ¤tExpires)483+ if err == nil && currentBlock == block && currentExpires == expires {484+ return true, nil485+ }486+ if err != nil && !errors.Is(err, sql.ErrNoRows) {487+ return false, err488+ }489+ if _, err := t.store.DB().ExecContext(ctx, `INSERT INTO custom_view_sources(view,hash,source,block,expires) VALUES(?,?,?,?,?) ON CONFLICT(view,hash,source) DO UPDATE SET block=excluded.block, expires=excluded.expires`, view.Name, hash, source, block, expires); err != nil {490+ return false, err491+ }492+ return true, t.signArtifact(ctx, view, hash, time.Now().Unix())493+}494+495+// signArtifact signs the artifact with its current sources as a kind 30078496+// record and stores the record when the view is public.497+func (t *Tenant) signArtifact(ctx context.Context, view customView, hash string, now int64) error {498+ var typ, engine, content string499+ if err := t.store.DB().QueryRowContext(ctx, `SELECT type,engine,content FROM custom_view_artifacts WHERE view=? AND hash=?`, view.Name, hash).Scan(&typ, &engine, &content); err != nil {500+ return err501+ }502+ rows, err := t.store.DB().QueryContext(ctx, `SELECT source,block,expires FROM custom_view_sources WHERE view=? AND hash=? ORDER BY rowid`, view.Name, hash)503+ if err != nil {504+ return err505+ }506+ tags := [][]string{{"-"}, {"d", "bind.ws/view/" + view.Name + "/" + hash}, {"view", view.Name}}507+ var blocks [][]string508+ expires := int64(0)509+ unlimited := false510+ for rows.Next() {511+ var source string512+ var block int513+ var sourceExpires int64514+ if err := rows.Scan(&source, &block, &sourceExpires); err != nil {515+ rows.Close()516+ return err517+ }518+ tags = append(tags, viewSourceTag(source))519+ blocks = append(blocks, []string{"block", strconv.Itoa(block)})520+ if sourceExpires == 0 {521+ unlimited = true522+ } else if sourceExpires > expires {523+ expires = sourceExpires524+ }525+ }526+ if err := errors.Join(rows.Err(), rows.Close()); err != nil {527+ return err528+ }529+ tags = append(tags, blocks...)530+ tags = append(tags, []string{"type", typ})531+ if engine != "" {532+ tags = append(tags, []string{"engine", engine})533+ }534+ if expires > 0 && !unlimited {535+ tags = append(tags, []string{"expiration", strconv.FormatInt(expires, 10)})536+ }537+ var signed event.Event538+ if view.Audience == "public" {539+ signed, err = t.records.Generate(ctx, event.KIND_VIEW, tags, content, now)540+ } else {541+ signed, err = t.records.Sign(ctx, event.KIND_VIEW, tags, content, now)542+ }543+ if err != nil {544+ return err545+ }546+ raw, err := json.Marshal(signed)547+ if err != nil {548+ return err549+ }550+ _, err = t.store.DB().ExecContext(ctx, `UPDATE custom_view_artifacts SET event_id=?, raw=? WHERE view=? AND hash=?`, signed.ID, string(raw), view.Name, hash)551+ return err552+}553+554+// detachStaleArtifacts drops the source from artifacts whose block it no555+// longer carries, then removes artifacts left without a source.556+func (t *Tenant) detachStaleArtifacts(ctx context.Context, name, source string, hashes []string) error {557+ rows, err := t.store.DB().QueryContext(ctx, `SELECT hash FROM custom_view_sources WHERE view=? AND source=?`, name, source)558+ if err != nil {559+ return err560+ }561+ keep := map[string]bool{}562+ for _, hash := range hashes {563+ keep[hash] = true564+ }565+ var stale []string566+ for rows.Next() {567+ var hash string568+ if err := rows.Scan(&hash); err != nil {569+ rows.Close()570+ return err571+ }572+ if !keep[hash] {573+ stale = append(stale, hash)574+ }575+ }576+ if err := errors.Join(rows.Err(), rows.Close()); err != nil {577+ return err578+ }579+ for _, hash := range stale {580+ if _, err := t.store.DB().ExecContext(ctx, `DELETE FROM custom_view_sources WHERE view=? AND hash=? AND source=?`, name, hash, source); err != nil {581+ return err582+ }583+ }584+ return t.dropOrphanArtifacts(ctx, name)585+}586+587+// pruneViewArtifacts detaches sources that no longer exist and removes the588+// artifacts that lose their last source. A view with transform work still589+// queued is left alone so a replaced source can reattach its blocks first.590+func (t *Tenant) pruneViewArtifacts(ctx context.Context, name, except string) error {591+ var pending int592+ if err := t.store.DB().QueryRowContext(ctx, `SELECT count(*) FROM work_intents WHERE kind=? AND target=? AND state IN ('pending','running') AND id<>?`, viewTransform, name, except).Scan(&pending); err != nil {593+ return err594+ }595+ if pending > 0 {596+ return nil597+ }598+ rows, err := t.store.DB().QueryContext(ctx, `SELECT DISTINCT source FROM custom_view_sources WHERE view=?`, name)599+ if err != nil {600+ return err601+ }602+ var sources []string603+ for rows.Next() {604+ var source string605+ if err := rows.Scan(&source); err != nil {606+ rows.Close()607+ return err608+ }609+ sources = append(sources, source)610+ }611+ if err := errors.Join(rows.Err(), rows.Close()); err != nil {612+ return err613+ }614+ removed := 0615+ for _, source := range sources {616+ present, err := t.sourcePresent(ctx, source)617+ if err != nil {618+ return err619+ }620+ if present {621+ continue622+ }623+ if _, err := t.store.DB().ExecContext(ctx, `DELETE FROM custom_view_sources WHERE view=? AND source=?`, name, source); err != nil {624+ return err625+ }626+ removed++627+ }628+ if removed == 0 {629+ return nil630+ }631+ return t.dropOrphanArtifacts(ctx, name)632+}633+634+// pruneAllViewArtifacts runs the prune for every view; the maintenance635+// sweep calls it after expired events are removed.636+func (t *Tenant) pruneAllViewArtifacts(ctx context.Context) error {637+ rows, err := t.customViewRows(ctx)638+ if err != nil {639+ return err640+ }641+ for _, view := range rows {642+ if err := t.pruneViewArtifacts(ctx, view.Name, ""); err != nil {643+ return err644+ }645+ }646+ return nil647+}648+649+func (t *Tenant) sourcePresent(ctx context.Context, source string) (bool, error) {650+ var one int651+ var err error652+ if tag := viewSourceTag(source); tag[0] == "a" {653+ parts := strings.SplitN(source, ":", 3)654+ if len(parts) != 3 {655+ return false, nil656+ }657+ err = t.store.DB().QueryRowContext(ctx, `SELECT 1 FROM events WHERE kind=? AND pubkey=? AND d=? LIMIT 1`, event.KIND_REPO_STATE, parts[1], parts[2]).Scan(&one)658+ } else {659+ err = t.store.DB().QueryRowContext(ctx, `SELECT 1 FROM events WHERE id=? AND (expires=0 OR expires>?) LIMIT 1`, source, time.Now().Unix()).Scan(&one)660+ }661+ if errors.Is(err, sql.ErrNoRows) {662+ return false, nil663+ }664+ return err == nil, err665+}666+667+// dropOrphanArtifacts removes artifacts without a source, and their stored668+// records, then re-signs the artifacts whose source list shrank.669+func (t *Tenant) dropOrphanArtifacts(ctx context.Context, name string) error {670+ ids, err := t.customViewArtifactIDs(ctx, `SELECT event_id FROM custom_view_artifacts WHERE view=? AND NOT EXISTS(SELECT 1 FROM custom_view_sources WHERE custom_view_sources.view=custom_view_artifacts.view AND custom_view_sources.hash=custom_view_artifacts.hash)`, name)671+ if err != nil {672+ return err673+ }674+ if _, err := t.store.DB().ExecContext(ctx, `DELETE FROM custom_view_artifacts WHERE view=? AND NOT EXISTS(SELECT 1 FROM custom_view_sources WHERE custom_view_sources.view=custom_view_artifacts.view AND custom_view_sources.hash=custom_view_artifacts.hash)`, name); err != nil {675+ return err676+ }677+ if err := t.deleteStoredArtifacts(ctx, ids); err != nil {678+ return err679+ }680+ if len(ids) > 0 {681+ t.app.telemetry.Logger().Info("view artifacts removed", "tenant", t.meta.Name, "removed", len(ids))682+ }683+ // Artifacts that kept a source but lost one carry a tag for it; a fresh684+ // record drops it.685+ view, err := t.customViewByName(ctx, name)686+ if err != nil {687+ return nil688+ }689+ rows, err := t.store.DB().QueryContext(ctx, `SELECT a.hash, a.raw FROM custom_view_artifacts a WHERE a.view=?`, name)690+ if err != nil {691+ return err692+ }693+ type stale struct{ hash string }694+ var refresh []stale695+ for rows.Next() {696+ var hash, raw string697+ if err := rows.Scan(&hash, &raw); err != nil {698+ rows.Close()699+ return err700+ }701+ var signed event.Event702+ if json.Unmarshal([]byte(raw), &signed) != nil {703+ continue704+ }705+ var tagged []string706+ for _, tag := range signed.Tags {707+ if len(tag) == 2 && (tag[0] == "e" || tag[0] == "a") {708+ tagged = append(tagged, tag[1])709+ }710+ }711+ current, err := t.artifactSources(ctx, name, hash)712+ if err != nil {713+ rows.Close()714+ return err715+ }716+ sort.Strings(tagged)717+ sort.Strings(current)718+ if strings.Join(tagged, "\n") != strings.Join(current, "\n") {719+ refresh = append(refresh, stale{hash})720+ }721+ }722+ if err := errors.Join(rows.Err(), rows.Close()); err != nil {723+ return err724+ }725+ for _, item := range refresh {726+ if err := t.signArtifact(ctx, view, item.hash, time.Now().Unix()); err != nil {727+ return err728+ }729+ }730+ return nil731+}732+733+func (t *Tenant) artifactSources(ctx context.Context, name, hash string) ([]string, error) {734+ rows, err := t.store.DB().QueryContext(ctx, `SELECT source FROM custom_view_sources WHERE view=? AND hash=?`, name, hash)735+ if err != nil {736+ return nil, err737+ }738+ defer rows.Close()739+ var out []string740+ for rows.Next() {741+ var source string742+ if err := rows.Scan(&source); err != nil {743+ return nil, err744+ }745+ out = append(out, source)746+ }747+ return out, rows.Err()748+}749+750+func (t *Tenant) customViewArtifactIDs(ctx context.Context, query string, args ...any) ([]string, error) {751+ rows, err := t.store.DB().QueryContext(ctx, query, args...)752+ if err != nil {753+ return nil, err754+ }755+ defer rows.Close()756+ var ids []string757+ for rows.Next() {758+ var id string759+ if err := rows.Scan(&id); err != nil {760+ return nil, err761+ }762+ if id != "" {763+ ids = append(ids, id)764+ }765+ }766+ return ids, rows.Err()767+}768+769+// deleteStoredArtifacts removes the public records of artifacts by id.770+func (t *Tenant) deleteStoredArtifacts(ctx context.Context, ids []string) error {771+ for _, id := range ids {772+ if _, err := t.store.DeleteEvent(ctx, id); err != nil {773+ return err774+ }775+ }776+ return nil777+}778+779+// cascadeViewDeletion runs after a kind 5 deletion is stored: artifacts780+// attached to the deleted events or addresses lose that source at once.781+func (t *Tenant) cascadeViewDeletion(ctx context.Context, e event.Event) {782+ if e.Kind != event.KIND_DELETION {783+ return784+ }785+ rows, err := t.customViewRows(ctx)786+ if err != nil || len(rows) == 0 {787+ return788+ }789+ for _, view := range rows {790+ if err := t.pruneViewArtifacts(ctx, view.Name, ""); err != nil {791+ t.app.telemetry.Logger().Error("view artifacts not pruned", "tenant", t.meta.Name, "error", err)792+ }793+ }794+}795+796+// artifactRecord is one stored artifact as the route serves it.797+type artifactRecord struct {798+ Type string799+ Audience string800+ Content string801+ EventID string802+}803+804+func (t *Tenant) customViewArtifact(ctx context.Context, name, hash string) (artifactRecord, error) {805+ var record artifactRecord806+ err := t.store.DB().QueryRowContext(ctx, `SELECT type,audience,content,event_id FROM custom_view_artifacts WHERE view=? AND hash=?`, name, hash).Scan(&record.Type, &record.Audience, &record.Content, &record.EventID)807+ if errors.Is(err, sql.ErrNoRows) {808+ return artifactRecord{}, errors.New("not found: artifact")809+ }810+ return record, err811+}812+813+// body returns the bytes served for the artifact.814+func (r artifactRecord) body() ([]byte, error) {815+ if r.Type == "image/png" {816+ return base64.StdEncoding.DecodeString(r.Content)817+ }818+ return []byte(r.Content), nil819+}@@ -83,6 +83,10 @@ func (t *Tenant) ServeHTTP(w http.ResponseWriter, r *http.Request) {
8383 if t.tryBrowseHTTP(w, r) {8484 return8585 }86+ if isViewArtifactPath(r.URL.Path) {87+ t.viewArtifactHTTP(w, r)88+ return89+ }8690 if t.sites != nil && t.sites.MatchesHost(r.Host) {8791 t.sites.Handler().ServeHTTP(w, r)8892 return@@ -50,6 +50,10 @@ func (t *Tenant) executeManagement(ctx context.Context, actor, method string, pa
5050 result, err = t.callbackExecute(ctx, actor, method, params)5151 return result, true, err5252 }53+ if customViewMethod(method) {54+ result, err = t.customViewExecute(ctx, actor, method, params)55+ return result, true, err56+ }5357 if !managementAdapterMethod(method) {5458 return nil, false, nil5559 }@@ -138,7 +142,7 @@ func publicManagementMethod(method string) bool {
138142 // ManagementMethods is the source-compatible NIP-86 registry. The hosted139143 // lease, trial, fuel, and entitlement controls are intentionally absent.140144 func ManagementMethods() []string {141- return []string{"supportedmethods", "listaudit", "stats", "getpolicy", "setpolicy", "listviews", "banpubkey", "setblockedwords", "allowpubkey", "setmember", "unrulepubkey", "removemember", "listbannedpubkeys", "listallowedpubkeys", "listmembers", "listpeople", "listjoinrequests", "approvejoin", "denyjoin", "createinvite", "listinvites", "revokeinvite", "listclaims", "listlisthistory", "restorelist", "createclaim", "deleteclaim", "removesubtree", "banevent", "allowevent", "listeventsneedingmoderation", "blockip", "unblockip", "listblockedips", "listreports", "resolvereport", "exportconfig", "importconfig", "deleterelay", "listblobs", "listsites", "deleteblob", "deleteevent", "listrecentevents", "searchevents", "pinevent", "unpinevent", "listpins", "allowkind", "disallowkind", "unrulekind", "storagestats", "gitstorage", "setretention", "listretention", "purgekind", "listallowedkinds", "listblockedkinds", "notifytest", "resetrules", "listpresets", "listconnectiontemplates", "listconnections", "setconnections", "applypreset", "forkrelay", "pullfrom", "pullstatus", "listjobs", "deliverystatus", "addjob", "removejob", "runjob", "backfill", "transferowner", "listdumps", "deletedump", "dumpnow", "backupnow", "listbackups", "deletebackup", "setsuccession", "clearsuccession", "successionstatus", "changerelayname", "changerelaydescription", "changerelayicon", "adddomain", "setdomainsite", "checkdomain", "removedomain", "listdomains", "listagents", "pauseagent", "resumeagent", "revokeagent", "pauseallagents", "resumeallagents", "listcallbacks", "addcallback", "removecallback", "pausecallback", "resumecallback"}145+ return []string{"supportedmethods", "listaudit", "stats", "getpolicy", "setpolicy", "listviews", "banpubkey", "setblockedwords", "allowpubkey", "setmember", "unrulepubkey", "removemember", "listbannedpubkeys", "listallowedpubkeys", "listmembers", "listpeople", "listjoinrequests", "approvejoin", "denyjoin", "createinvite", "listinvites", "revokeinvite", "listclaims", "listlisthistory", "restorelist", "createclaim", "deleteclaim", "removesubtree", "banevent", "allowevent", "listeventsneedingmoderation", "blockip", "unblockip", "listblockedips", "listreports", "resolvereport", "exportconfig", "importconfig", "deleterelay", "listblobs", "listsites", "deleteblob", "deleteevent", "listrecentevents", "searchevents", "pinevent", "unpinevent", "listpins", "allowkind", "disallowkind", "unrulekind", "storagestats", "gitstorage", "setretention", "listretention", "purgekind", "listallowedkinds", "listblockedkinds", "notifytest", "resetrules", "listpresets", "listconnectiontemplates", "listconnections", "setconnections", "applypreset", "forkrelay", "pullfrom", "pullstatus", "listjobs", "deliverystatus", "addjob", "removejob", "runjob", "backfill", "transferowner", "listdumps", "deletedump", "dumpnow", "backupnow", "listbackups", "deletebackup", "setsuccession", "clearsuccession", "successionstatus", "changerelayname", "changerelaydescription", "changerelayicon", "adddomain", "setdomainsite", "checkdomain", "removedomain", "listdomains", "listagents", "pauseagent", "resumeagent", "revokeagent", "pauseallagents", "resumeallagents", "listcallbacks", "addcallback", "removecallback", "pausecallback", "resumecallback", "listcustomviews", "addcustomview", "removecustomview", "runcustomview", "pausecustomview", "resumecustomview"}142146 }143147 144148 func managementRoleAllows(method, role string) bool {@@ -44,6 +44,7 @@ func (t *Tenant) runScheduler() {
4444 }4545 if !wake {4646 report("records", t.records.Tick(ctx, now.Unix()))47+ report("custom views", t.tickCustomViews(ctx, now.Unix()))4748 if !now.Before(nextMaintenance) {4849 if t.blobs != nil {4950 report("partial uploads", t.blobs.CleanupMultipart(ctx))@@ -28,6 +28,7 @@ import (
2828 "github.com/FelineStateMachine/tinyrelay/internal/replication"2929 "github.com/FelineStateMachine/tinyrelay/internal/sites"3030 "github.com/FelineStateMachine/tinyrelay/internal/storage"31+ "github.com/FelineStateMachine/tinyrelay/internal/views"3132 "github.com/FelineStateMachine/tinyrelay/internal/webui"3233 "github.com/FelineStateMachine/tinyrelay/internal/work"3334 )@@ -46,6 +47,9 @@ func (t *Tenant) initServices(ctx context.Context) error {
4647 if err := t.initCallbacks(ctx); err != nil {4748 return err4849 }50+ if err := t.initCustomViews(ctx); err != nil {51+ return err52+ }4953 t.blobs, err = blob.New(ctx, blob.Config{5054 Root: t.meta.Paths.Root, PublicURL: t.publicURL, Store: t.store, Authorize: t.authorizeBlob, UploadTerms: t.blobUploadTerms,5155 Limits: func() blob.Limits {@@ -192,6 +196,10 @@ func (b backend) URL() string { return b.tenant.publicURL }
192196 func (b backend) Slug() string { return b.tenant.meta.Name }193197 func (b backend) Identity() string { return b.tenant.records.PublicKey() }194198 199+// CustomViews gives the page renderers the enabled custom views' names and200+// languages, never a transform or a secret.201+func (b backend) CustomViews() []views.View { return b.tenant.CustomViews() }202+195203 // ReadAllowed is consumed by the web UI private boundary. It deliberately196204 // delegates to the same uncached membership check used by private Git and197205 // NIP-42, so a revoked browser session cannot retain page access.@@ -581,6 +589,7 @@ func (t *Tenant) workHandlers() map[string]work.Handler {
581589 }582590 handlers[notificationPush] = t.handleNotificationPush583591 handlers[callbackDelivery] = t.handleCallbackDelivery592+ handlers[viewTransform] = t.handleViewTransform584593 return handlers585594 }586595 @@ -616,6 +625,7 @@ func (t *Tenant) releaseGit(ctx context.Context, id string) error {
616625 // A repository state is hidden until its objects arrive, so agents617626 // that wait for pushes are woken here rather than at publish time.618627 t.notifyCallbacks(ctx, released)628+ t.queueCustomViews(ctx, released)619629 }620630 return err621631 }@@ -657,8 +667,10 @@ func (t *Tenant) sweep(ctx context.Context, now int64) error {
657667 if err != nil {658668 return err659669 }660- _, err = t.store.SweepRetention(ctx, opts)661- return err670+ if _, err := t.store.SweepRetention(ctx, opts); err != nil {671+ return err672+ }673+ return t.pruneAllViewArtifacts(ctx)662674 }663675 func (t *Tenant) replacePolicy(p policy.Policy) {664676 t.mu.Lock()@@ -755,6 +767,8 @@ func (t *Tenant) commitImported(ctx context.Context, e event.Event, origin repli
755767 if err == nil {756768 t.notifyDevices(ctx, e)757769 t.notifyCallbacks(ctx, e)770+ t.queueCustomViews(ctx, e)771+ t.cascadeViewDeletion(ctx, e)758772 t.notifyWikiMerge(ctx, e)759773 t.notifyWikiProposal(ctx, e)760774 }@@ -76,6 +76,10 @@ type Tenant struct {
7676 callbackIndex map[int][]callbackRecord7777 callbackLocks map[string]*sync.Mutex7878 callbackClient *http.Client79+ viewMu sync.Mutex80+ viewIndex map[int][]customView81+ viewLocks map[string]*sync.Mutex82+ viewClient *http.Client7983 }8084 8185 func newTenant(ctx context.Context, cfg tenantConfig) (*Tenant, error) {@@ -396,6 +400,8 @@ func (t *Tenant) Publish(ctx context.Context, e event.Event, s relay.Session) (s
396400 }397401 t.notifyDevices(ctx, e)398402 t.notifyCallbacks(ctx, e)403+ t.queueCustomViews(ctx, e)404+ t.cascadeViewDeletion(ctx, e)399405 t.notifyWikiMerge(ctx, e)400406 t.notifyWikiProposal(ctx, e)401407 // Stage Git metadata before acknowledging the event. This closes the@@ -197,6 +197,12 @@ func (s *Service) Generate(ctx context.Context, kind int, tags [][]string, conte
197197 return s.signed(ctx, kind, tags, content, now)198198 }199199 200+// Sign signs a record without storing it, for records whose audience is201+// decided when they are served, such as members-only view artifacts.202+func (s *Service) Sign(ctx context.Context, kind int, tags [][]string, content string, now int64) (event.Event, error) {203+ return s.signedOnly(ctx, kind, tags, content, now)204+}205+200206 func (s *Service) profile(ctx context.Context, now int64) (event.Event, error) {201207 p := s.policy()202208 b, _ := json.Marshal(map[string]string{"name": p.Name, "about": p.Description, "picture": p.Icon})@@ -236,10 +236,10 @@ func latencyBuckets() []float64 {
236236 return []float64{.0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}237237 }238238 239-var operationNames = map[string]struct{}{"http": {}, "websocket": {}, "publish": {}, "subscribe": {}, "query": {}, "count": {}, "fanout": {}, "delivery": {}, "commit": {}, "save": {}, "parse": {}, "authorize": {}, "upgrade": {}, "callback": {}, "delivery-discovery": {}, "notification-broadcast": {}, "browserepos": {}, "browserepo": {}, "browsefiles": {}, "browsefile": {}, "browsestatus": {}, "browseissues": {}, "browsepulls": {}, "browseissue": {}, "browsepull": {}, "browse-download": {}, "push": {}, "mcp": {}, "rooms": {}, "blob-expiry": {}}239+var operationNames = map[string]struct{}{"http": {}, "websocket": {}, "publish": {}, "subscribe": {}, "query": {}, "count": {}, "fanout": {}, "delivery": {}, "commit": {}, "save": {}, "parse": {}, "authorize": {}, "upgrade": {}, "callback": {}, "delivery-discovery": {}, "notification-broadcast": {}, "browserepos": {}, "browserepo": {}, "browsefiles": {}, "browsefile": {}, "browsestatus": {}, "browseissues": {}, "browsepulls": {}, "browseissue": {}, "browsepull": {}, "browse-download": {}, "push": {}, "mcp": {}, "rooms": {}, "blob-expiry": {}, "view": {}}240240 var outcomeNames = map[string]struct{}{"success": {}, "ok": {}, "invalid": {}, "unauthorized": {}, "timeout": {}, "busy": {}, "closed": {}, "error": {}, "paused": {}}241241 var queueNames = map[string]struct{}{"inbox": {}, "outbox": {}, "write": {}, "delivery": {}}242-var workNames = map[string]struct{}{"delivery": {}, "push": {}, "replication": {}, "webhook": {}, "job": {}, "records-projection": {}, "site-mirror": {}, "git-metadata": {}, "catalog-owner": {}, "callback": {}, "delivery-discovery": {}, "notification-delivery": {}, "notification-broadcast": {}, "notification-push": {}, "notification": {}, "callback-delivery": {}}242+var workNames = map[string]struct{}{"delivery": {}, "push": {}, "replication": {}, "webhook": {}, "job": {}, "records-projection": {}, "site-mirror": {}, "git-metadata": {}, "catalog-owner": {}, "callback": {}, "delivery-discovery": {}, "notification-delivery": {}, "notification-broadcast": {}, "notification-push": {}, "notification": {}, "callback-delivery": {}, "view-transform": {}}243243 244244 func bounded(value string, allowed map[string]struct{}) string {245245 if _, ok := allowed[value]; ok {@@ -0,0 +1,155 @@
1+// Package views holds the parts of custom views that the renderers and the2+// relay share: finding fenced code blocks in text, naming a block by its3+// hash and the markup that shows an artifact in place of a block. It knows4+// nothing about any particular transform.5+package views6+7+import (8+ "crypto/sha256"9+ "encoding/hex"10+ "html"11+ "regexp"12+ "strings"13+)14+15+// Block is one fenced code block. Index counts every fenced block in the16+// text from zero, whether or not it carries a language.17+type Block struct {18+ Index int `json:"index"`19+ Lang string `json:"lang"`20+ Source string `json:"source"`21+}22+23+// NamePattern is the shape of a custom view name.24+var NamePattern = regexp.MustCompile(`^[a-z0-9-]{1,32}$`)25+26+// OpenFence reports whether a line opens a fenced code block: three or more27+// backticks or tildes, with the language as the first word after them. The28+// marker returned is the fence run, so the closing line must use the same29+// character.30+func OpenFence(line string) (marker, lang string, ok bool) {31+ trimmed := strings.TrimSpace(line)32+ if len(trimmed) < 3 || (trimmed[0] != '`' && trimmed[0] != '~') {33+ return "", "", false34+ }35+ run := len(trimmed) - len(strings.TrimLeft(trimmed, trimmed[:1]))36+ if run < 3 {37+ return "", "", false38+ }39+ info := strings.Fields(trimmed[run:])40+ if len(info) > 0 {41+ lang = strings.ToLower(info[0])42+ }43+ return trimmed[:run], lang, true44+}45+46+// ClosesFence reports whether a line closes the fence opened with marker: a47+// run of at least as many of the same character and nothing else.48+func ClosesFence(line, marker string) bool {49+ trimmed := strings.TrimSpace(line)50+ return len(trimmed) >= len(marker) && strings.Trim(trimmed, marker[:1]) == ""51+}52+53+// Blocks lists the fenced code blocks of a text in order. The source is the54+// lines between the fences joined by newlines, without a trailing newline.55+func Blocks(content string) []Block {56+ lines := strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n")57+ var out []Block58+ for i := 0; i < len(lines); i++ {59+ marker, lang, ok := OpenFence(lines[i])60+ if !ok {61+ continue62+ }63+ var body []string64+ for i++; i < len(lines) && !ClosesFence(lines[i], marker); i++ {65+ body = append(body, lines[i])66+ }67+ out = append(out, Block{Index: len(out), Lang: lang, Source: strings.Join(body, "\n")})68+ }69+ return out70+}71+72+// Matching keeps the blocks whose language is one of the given languages.73+func Matching(blocks []Block, languages []string) []Block {74+ var out []Block75+ for _, block := range blocks {76+ if block.Lang != "" && contains(languages, block.Lang) {77+ out = append(out, block)78+ }79+ }80+ return out81+}82+83+// Hash names a block: the SHA-256 of the language, a newline and the84+// source, in hex. The same block in two texts has the same hash.85+func Hash(lang, source string) string {86+ sum := sha256.Sum256([]byte(lang + "\n" + source))87+ return hex.EncodeToString(sum[:])88+}89+90+// HashPattern is the shape of a block hash in a path.91+var HashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)92+93+// Path is the address an artifact is served at.94+func Path(view, hash, extension string) string {95+ return "/views/" + view + "/" + hash + "." + extension96+}97+98+// Figure is the markup for a block a custom view renders: the artifact as99+// an object with the escaped code block inside it, so a missing or failed100+// artifact shows the code without any script.101+func Figure(view, lang, source string) string {102+ var b strings.Builder103+ b.WriteString(`<figure data-view="` + html.EscapeString(view) + `"><object data="` + Path(view, Hash(lang, source), "svg") + `" type="image/svg+xml"><pre><code`)104+ if lang != "" {105+ b.WriteString(` data-lang="` + html.EscapeString(lang) + `"`)106+ }107+ b.WriteString(">" + html.EscapeString(source) + "</code></pre></object></figure>\n")108+ return b.String()109+}110+111+// Renderer decides how a fenced block is shown: the view that renders the112+// language, if any. Languages maps a lowercase language to a view name.113+type Renderer struct {114+ Languages map[string]string115+}116+117+// NewRenderer indexes views by the languages they render. The first view118+// that names a language renders it.119+func NewRenderer(views []View) Renderer {120+ index := map[string]string{}121+ for _, view := range views {122+ for _, lang := range view.Languages {123+ lang = strings.ToLower(lang)124+ if _, taken := index[lang]; !taken {125+ index[lang] = view.Name126+ }127+ }128+ }129+ return Renderer{Languages: index}130+}131+132+// View is the summary a renderer needs: the name and the languages.133+type View struct {134+ Name string `json:"name"`135+ Languages []string `json:"languages"`136+}137+138+// Block returns the figure markup for a block when a view renders its139+// language.140+func (r Renderer) Block(lang, source string) (string, bool) {141+ view, ok := r.Languages[strings.ToLower(lang)]142+ if !ok || view == "" {143+ return "", false144+ }145+ return Figure(view, strings.ToLower(lang), source), true146+}147+148+func contains(values []string, value string) bool {149+ for _, item := range values {150+ if item == value {151+ return true152+ }153+ }154+ return false155+}@@ -0,0 +1,90 @@
1+package views2+3+import (4+ "crypto/sha256"5+ "encoding/hex"6+ "strings"7+ "testing"8+)9+10+func TestBlocksFindFencesInOrderWithLanguages(t *testing.T) {11+ content := "Intro\n\n```mermaid\ngraph TD;\n A-->B;\n```\n\nText\n\n~~~\nplain\n~~~\n\n ```` Python extra\nprint(1)\n```\nstill python\n````\n\n```dot\n"12+ blocks := Blocks(content)13+ if len(blocks) != 4 {14+ t.Fatalf("blocks = %+v", blocks)15+ }16+ for i, want := range []Block{17+ {0, "mermaid", "graph TD;\n A-->B;"},18+ {1, "", "plain"},19+ {2, "python", "print(1)\n```\nstill python"},20+ {3, "dot", ""},21+ } {22+ if blocks[i] != want {23+ t.Errorf("block %d = %+v, want %+v", i, blocks[i], want)24+ }25+ }26+ matched := Matching(blocks, []string{"mermaid", "dot"})27+ if len(matched) != 2 || matched[0].Index != 0 || matched[1].Index != 3 {28+ t.Fatalf("matching = %+v", matched)29+ }30+ if got := Matching(blocks, []string{"go"}); got != nil {31+ t.Fatalf("unexpected match %+v", got)32+ }33+}34+35+func TestFenceHelpersRejectShortAndMixedFences(t *testing.T) {36+ if _, _, ok := OpenFence("``"); ok {37+ t.Fatal("two backticks opened a fence")38+ }39+ if _, _, ok := OpenFence("text ```"); ok {40+ t.Fatal("fence in the middle of a line opened a fence")41+ }42+ marker, lang, ok := OpenFence("~~~~ Mermaid title")43+ if !ok || marker != "~~~~" || lang != "mermaid" {44+ t.Fatalf("open = %q %q %v", marker, lang, ok)45+ }46+ if ClosesFence("~~~", marker) || ClosesFence("````", marker) || !ClosesFence(" ~~~~~ ", marker) {47+ t.Fatal("closing fence rules")48+ }49+}50+51+func TestHashCoversLanguageAndSource(t *testing.T) {52+ sum := sha256.Sum256([]byte("mermaid\ngraph TD"))53+ if Hash("mermaid", "graph TD") != hex.EncodeToString(sum[:]) {54+ t.Fatal("hash differs from sha256(lang + newline + source)")55+ }56+ if Hash("mermaid", "graph TD") == Hash("dot", "graph TD") {57+ t.Fatal("language is not part of the hash")58+ }59+ if Blocks("```mermaid\na\n```")[0].Source != Blocks("```mermaid\r\na\r\n```")[0].Source {60+ t.Fatal("line endings change the source")61+ }62+}63+64+func TestFigureEmbedsTheArtifactWithTheCodeAsFallback(t *testing.T) {65+ got := Figure("diagrams", "mermaid", "graph TD;\n A-->B & <C>;")66+ hash := Hash("mermaid", "graph TD;\n A-->B & <C>;")67+ want := `<figure data-view="diagrams"><object data="/views/diagrams/` + hash + `.svg" type="image/svg+xml"><pre><code data-lang="mermaid">graph TD;` + "\n" + ` A-->B & <C>;</code></pre></object></figure>` + "\n"68+ if got != want {69+ t.Fatalf("figure =\n%s\nwant\n%s", got, want)70+ }71+ if strings.Contains(got, "class=") {72+ t.Fatal("figure carries a class attribute")73+ }74+}75+76+func TestRendererPicksTheFirstViewForALanguage(t *testing.T) {77+ r := NewRenderer([]View{{Name: "first", Languages: []string{"mermaid"}}, {Name: "second", Languages: []string{"Mermaid", "dot"}}})78+ if html, ok := r.Block("MERMAID", "x"); !ok || !strings.Contains(html, `data-view="first"`) {79+ t.Fatalf("mermaid = %q %v", html, ok)80+ }81+ if html, ok := r.Block("dot", "x"); !ok || !strings.Contains(html, `data-view="second"`) {82+ t.Fatalf("dot = %q %v", html, ok)83+ }84+ if _, ok := r.Block("go", "x"); ok {85+ t.Fatal("go rendered")86+ }87+ if _, ok := (Renderer{}).Block("mermaid", "x"); ok {88+ t.Fatal("empty renderer rendered")89+ }90+}