1package main23import (4 "context"5 "fmt"6 "io"7 "net/http"8 "os/exec"9 "path/filepath"10 "sync"11)1213// The workbench runs the pipeline's own tools rather than reimplementing14// them: a fetch is tools/crawl, pyramids are tools/tiles, composition is15// tools/generate, each invoked exactly as a person at a terminal would and16// each streamed back to the page as it speaks. Nothing runs on a schedule or17// at startup; an operation exists only between a submitted form and the18// subprocess's exit.1920// workshop holds where operations run: the repository whose tools are21// invoked, the archive captures land in, and the registry composed bundles22// install into. The lock keeps operations serial -- the tools are safe to23// interleave in principle, but two crawls sharing one progress pane help24// nobody.25type workshop struct {26 repo string27 archive string28 bundles string29 busy sync.Mutex30}3132// fetchArgv is the tools/crawl invocation for one source and target,33// verbatim what a person would type in the repository.34func fetchArgv(src Source, target, archive string) ([]string, error) {35 selected, err := src.FetchArgs(target)36 if err != nil {37 return nil, err38 }39 argv := []string{"go", "run", "./tools/crawl", "-archive", archive}40 return append(argv, selected...), nil41}4243// tilesArgv derives every pyramid the archive holds into build/tiles, which44// sits under the repository so the per-pyramid stamps persist and rebuilds45// stay incremental.46func tilesArgv(archive string) []string {47 return []string{"go", "run", "./tools/tiles", "-source", archive, "-output", "build/tiles"}48}4950// generateArgv composes bundles from the archive and the derived tiles into51// the given registry directory. tools/generate is addressed by the directory52// containing fmg-archive rather than the archive itself, so the archive path53// must actually be one.54func generateArgv(archive, bundles string) ([]string, error) {55 if filepath.Base(archive) != "fmg-archive" {56 return nil, fmt.Errorf(57 "the archive %s is not an fmg-archive directory, and tools/generate finds captures under that name", archive)58 }59 return []string{60 "go", "run", "./tools/generate",61 "-source", filepath.Dir(archive),62 "-tiles", "build/tiles/index.json",63 "-bundles", bundles,64 }, nil65}6667// streamCommand runs argv in dir and writes everything it says, both68// streams interleaved, to w as it arrives. The context is the request's: a69// page abandoned mid-operation stops its subprocess, so nothing crawls on70// with nobody watching.71func streamCommand(ctx context.Context, w io.Writer, dir string, argv []string) error {72 cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)73 cmd.Dir = dir74 out := flushWriter{w}75 cmd.Stdout = out76 cmd.Stderr = out77 return cmd.Run()78}7980// flushWriter pushes every write through to the client immediately, so a81// long crawl reads as progress rather than silence. Writers that cannot82// flush -- a buffer in a test -- just write.83type flushWriter struct {84 w io.Writer85}8687func (f flushWriter) Write(p []byte) (int, error) {88 n, err := f.w.Write(p)89 if flusher, ok := f.w.(http.Flusher); ok {90 flusher.Flush()91 }92 return n, err93}