- commit
- 66e4007b240540b0f73ef363b24aa64151cdc7fe
- parent
- 97e3b36916a1e2efb00ef5f36021390286c4505d
- author
- Dami <dami.etoile@gmail.com>
| A | internal/ | 398 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 143 | +++++++++++ |
@@ -0,0 +1,398 @@
1+// Package wiki implements the NIP-54 rules that do not depend on storage:2+// article name normalization, link discovery and a conservative Djot3+// renderer for article content.4+package wiki5+6+import (7+ "html/template"8+ "net/url"9+ "strings"10+ "unicode"11+)12+13+// Normalize turns a title into the article's d tag: lowercase, whitespace as14+// hyphens, punctuation and symbols removed, hyphens collapsed and trimmed.15+// Letters of every script, combining marks and digits are preserved.16+func Normalize(title string) string {17+ var out []rune18+ for _, r := range strings.ToLower(title) {19+ switch {20+ case unicode.IsSpace(r) || r == '-' || r == '_':21+ if len(out) > 0 && out[len(out)-1] != '-' {22+ out = append(out, '-')23+ }24+ case unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r):25+ out = append(out, r)26+ }27+ }28+ return strings.Trim(string(out), "-")29+}30+31+// Links lists the targets referenced from Djot content in order of first32+// appearance: normalized article names for wikilinks and reference-style33+// links with no definition, and full nostr: URIs for NIP-21 links.34+func Links(content string) []string {35+ refs := definitions(content)36+ seen := map[string]bool{}37+ var links []string38+ add := func(target string) {39+ if target != "" && !seen[target] {40+ seen[target] = true41+ links = append(links, target)42+ }43+ }44+ for _, def := range refs {45+ if strings.HasPrefix(strings.ToLower(def), "nostr:") {46+ add(def)47+ }48+ }49+ var visit func(text string)50+ visit = func(text string) {51+ for _, span := range inlineSpans(text) {52+ switch span.kind {53+ case "em", "strong":54+ visit(span.text)55+ case "wikilink":56+ add(Normalize(span.target))57+ case "link":58+ if strings.HasPrefix(strings.ToLower(span.target), "nostr:") {59+ add(span.target)60+ }61+ case "reference":62+ if def, ok := refs[strings.ToLower(span.target)]; ok {63+ if strings.HasPrefix(strings.ToLower(def), "nostr:") {64+ add(def)65+ }66+ } else {67+ add(Normalize(span.target))68+ }69+ }70+ }71+ }72+ for _, block := range blocks(content) {73+ switch block.kind {74+ case "pre", "def":75+ case "ul", "ol":76+ for _, item := range block.items {77+ visit(item)78+ }79+ default:80+ visit(block.text)81+ }82+ }83+ return links84+}85+86+// RenderHTML renders a subset of Djot: ATX headings, paragraphs, fenced87+// code, bullet and numbered lists, emphasis, strong, code spans, links,88+// [[wikilinks]] and reference-style links. Everything else is shown as89+// escaped text. Link targets are limited to http, https, mailto, nostr and90+// site-relative paths.91+func RenderHTML(content string) template.HTML {92+ refs := definitions(content)93+ var b strings.Builder94+ for _, block := range blocks(content) {95+ switch block.kind {96+ case "def":97+ continue98+ case "pre":99+ b.WriteString("<pre><code")100+ if block.lang != "" {101+ b.WriteString(` class="language-` + template.HTMLEscapeString(block.lang) + `"`)102+ }103+ b.WriteString(">" + template.HTMLEscapeString(block.text) + "</code></pre>\n")104+ case "heading":105+ level := string(rune('0' + block.level))106+ b.WriteString("<h" + level + ">" + renderInline(block.text, refs) + "</h" + level + ">\n")107+ case "ul", "ol":108+ b.WriteString("<" + block.kind + ">\n")109+ for _, item := range block.items {110+ b.WriteString("<li>" + renderInline(item, refs) + "</li>\n")111+ }112+ b.WriteString("</" + block.kind + ">\n")113+ default:114+ b.WriteString("<p>" + renderInline(block.text, refs) + "</p>\n")115+ }116+ }117+ return template.HTML(b.String())118+}119+120+type block struct {121+ kind string122+ level int123+ lang string124+ text string125+ items []string126+}127+128+// definitions collects reference link definitions, "[label]: target", keyed129+// by their lowercase label.130+func definitions(content string) map[string]string {131+ refs := map[string]string{}132+ for _, block := range blocks(content) {133+ if block.kind == "def" {134+ refs[strings.ToLower(block.lang)] = block.text135+ }136+ }137+ return refs138+}139+140+func blocks(content string) []block {141+ lines := strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n")142+ var out []block143+ var para []string144+ flush := func() {145+ if len(para) > 0 {146+ out = append(out, block{kind: "p", text: strings.Join(para, "\n")})147+ para = nil148+ }149+ }150+ for i := 0; i < len(lines); i++ {151+ trimmed := strings.TrimSpace(lines[i])152+ if trimmed == "" {153+ flush()154+ continue155+ }156+ if fence := trimmed[:min(3, len(trimmed))]; fence == "```" || fence == "~~~" {157+ flush()158+ lang := append(strings.Fields(trimmed[3:]), "")[0]159+ var body []string160+ for i++; i < len(lines); i++ {161+ if t := strings.TrimSpace(lines[i]); len(t) >= 3 && strings.Trim(t, fence[:1]) == "" {162+ break163+ }164+ body = append(body, lines[i])165+ }166+ out = append(out, block{kind: "pre", lang: lang, text: strings.Join(body, "\n") + "\n"})167+ continue168+ }169+ if level := headingLevel(trimmed); level > 0 {170+ flush()171+ out = append(out, block{kind: "heading", level: level, text: strings.TrimSpace(trimmed[level:])})172+ continue173+ }174+ if label, target, ok := definitionLine(trimmed); ok && len(para) == 0 {175+ out = append(out, block{kind: "def", lang: label, text: target})176+ continue177+ }178+ if kind, _ := listMarker(trimmed); kind != "" && len(para) == 0 {179+ var items []string180+ for ; i < len(lines); i++ {181+ current := strings.TrimSpace(lines[i])182+ itemKind, rest := listMarker(current)183+ if itemKind != kind {184+ break185+ }186+ items = append(items, rest)187+ }188+ i--189+ out = append(out, block{kind: kind, items: items})190+ continue191+ }192+ para = append(para, trimmed)193+ }194+ flush()195+ return out196+}197+198+func headingLevel(line string) int {199+ level := len(line) - len(strings.TrimLeft(line, "#"))200+ if level == 0 || level > 6 || level >= len(line) || line[level] != ' ' {201+ return 0202+ }203+ return level204+}205+206+func definitionLine(line string) (label, target string, ok bool) {207+ end := strings.Index(line, "]:")208+ if !strings.HasPrefix(line, "[") || end < 1 {209+ return "", "", false210+ }211+ target = strings.TrimSpace(line[end+2:])212+ return strings.TrimSpace(line[1:end]), target, target != "" && !strings.ContainsAny(target, " \t")213+}214+215+func listMarker(line string) (kind, rest string) {216+ if len(line) > 1 && (line[0] == '-' || line[0] == '*' || line[0] == '+') && line[1] == ' ' {217+ return "ul", strings.TrimSpace(line[2:])218+ }219+ digits := len(line) - len(strings.TrimLeft(line, "0123456789"))220+ if digits > 0 && digits+1 < len(line) && (line[digits] == '.' || line[digits] == ')') && line[digits+1] == ' ' {221+ return "ol", strings.TrimSpace(line[digits+2:])222+ }223+ return "", ""224+}225+226+type span struct {227+ kind string // text, code, em, strong, link, wikilink, reference228+ text, target string229+}230+231+// inlineSpans splits paragraph text into literal runs and recognized inline232+// elements. It never nests: the text inside a link or emphasis is rendered233+// as further spans by the caller.234+func inlineSpans(text string) []span {235+ var spans []span236+ var literal strings.Builder237+ flush := func() {238+ if literal.Len() > 0 {239+ spans = append(spans, span{kind: "text", text: literal.String()})240+ literal.Reset()241+ }242+ }243+ for i := 0; i < len(text); {244+ c := text[i]245+ switch {246+ case c == '\\' && i+1 < len(text):247+ literal.WriteByte(text[i+1])248+ i += 2249+ continue250+ case c == '`':251+ run := strings.TrimLeft(text[i:], "`")252+ marker := text[i : len(text)-len(run)]253+ if end := strings.Index(run, marker); end >= 0 {254+ flush()255+ spans = append(spans, span{kind: "code", text: strings.TrimSpace(run[:end])})256+ i += 2*len(marker) + end257+ continue258+ }259+ case c == '*' || c == '_':260+ if end := emphasisEnd(text, i); end > 0 {261+ flush()262+ kind := "em"263+ if c == '*' {264+ kind = "strong"265+ }266+ spans = append(spans, span{kind: kind, text: text[i+1 : end]})267+ i = end + 1268+ continue269+ }270+ case c == '[' && strings.HasPrefix(text[i:], "[["):271+ if end := strings.Index(text[i:], "]]"); end > 2 {272+ flush()273+ target, label, _ := strings.Cut(text[i+2:i+end], "|")274+ spans = append(spans, span{kind: "wikilink", target: strings.TrimSpace(target), text: strings.TrimSpace(label)})275+ i += end + 2276+ continue277+ }278+ case c == '[':279+ if s, n := linkAt(text[i:]); n > 0 {280+ flush()281+ spans = append(spans, s)282+ i += n283+ continue284+ }285+ }286+ literal.WriteByte(c)287+ i++288+ }289+ flush()290+ return spans291+}292+293+// emphasisEnd finds the closing marker for Djot emphasis: the opener is not294+// followed by whitespace and the closer is not preceded by it.295+func emphasisEnd(text string, start int) int {296+ marker := text[start]297+ if start+1 >= len(text) || text[start+1] == ' ' || text[start+1] == '\n' {298+ return 0299+ }300+ for i := start + 2; i < len(text); i++ {301+ if text[i] == marker && text[i-1] != ' ' && text[i-1] != '\n' {302+ return i303+ }304+ }305+ return 0306+}307+308+// linkAt recognizes [text](target), [text][label] and [text][] at the start309+// of the string and reports how many bytes it consumed.310+func linkAt(text string) (span, int) {311+ depth, end := 0, -1312+ for i := 0; i < len(text); i++ {313+ if text[i] == '[' {314+ depth++315+ } else if text[i] == ']' {316+ if depth--; depth == 0 {317+ end = i318+ break319+ }320+ }321+ }322+ if end <= 0 {323+ return span{}, 0324+ }325+ label := text[1:end]326+ rest := text[end+1:]327+ if strings.HasPrefix(rest, "(") {328+ if close := strings.IndexByte(rest, ')'); close > 0 {329+ return span{kind: "link", text: label, target: strings.TrimSpace(rest[1:close])}, end + 2 + close330+ }331+ }332+ if strings.HasPrefix(rest, "[") {333+ if close := strings.IndexByte(rest, ']'); close > 0 {334+ target := strings.TrimSpace(rest[1:close])335+ if target == "" {336+ target = label337+ }338+ return span{kind: "reference", text: label, target: target}, end + 2 + close339+ }340+ }341+ return span{}, 0342+}343+344+func renderInline(text string, refs map[string]string) string {345+ var b strings.Builder346+ for _, s := range inlineSpans(text) {347+ switch s.kind {348+ case "text":349+ b.WriteString(template.HTMLEscapeString(s.text))350+ case "code":351+ b.WriteString("<code>" + template.HTMLEscapeString(s.text) + "</code>")352+ case "em", "strong":353+ b.WriteString("<" + s.kind + ">" + renderInline(s.text, refs) + "</" + s.kind + ">")354+ case "wikilink":355+ label := s.text356+ if label == "" {357+ label = s.target358+ }359+ b.WriteString(anchor("/wiki/"+url.PathEscape(Normalize(s.target)), renderInline(label, refs)))360+ case "link":361+ b.WriteString(anchor(safeHref(s.target), renderInline(s.text, refs)))362+ case "reference":363+ href := "/wiki/" + url.PathEscape(Normalize(s.target))364+ if def, ok := refs[strings.ToLower(s.target)]; ok {365+ href = safeHref(def)366+ }367+ b.WriteString(anchor(href, renderInline(s.text, refs)))368+ }369+ }370+ return b.String()371+}372+373+func anchor(href, inner string) string {374+ if href == "" {375+ return inner376+ }377+ return `<a href="` + template.HTMLEscapeString(href) + `">` + inner + `</a>`378+}379+380+// safeHref keeps http, https, mailto and site-relative targets, routes381+// nostr: links through the relay's open page and drops everything else.382+func safeHref(target string) string {383+ target = strings.TrimSpace(target)384+ lower := strings.ToLower(target)385+ switch {386+ case strings.HasPrefix(lower, "nostr:"):387+ return "/open?target=" + url.QueryEscape(target)388+ case strings.HasPrefix(lower, "mailto:"):389+ return target390+ case strings.HasPrefix(lower, "http://"), strings.HasPrefix(lower, "https://"):391+ if u, err := url.Parse(target); err == nil && u.Host != "" {392+ return target393+ }394+ case strings.HasPrefix(target, "/") && !strings.HasPrefix(target, "//") && !strings.HasPrefix(target, "/\\"):395+ return target396+ }397+ return ""398+}@@ -0,0 +1,143 @@
1+package wiki2+3+import (4+ "fmt"5+ "strings"6+ "testing"7+)8+9+func TestNormalizeFollowsNIP54Examples(t *testing.T) {10+ cases := map[string]string{11+ "Wiki Article": "wiki-article",12+ "What's Up?": "whats-up",13+ " Hello World ": "hello-world",14+ "Article 1": "article-1",15+ "ウィキペディア": "ウィキペディア",16+ "Ñoño": "ñoño",17+ "Москва": "москва",18+ "日本語 Article": "日本語-article",19+ "proof-of-work": "proof-of-work",20+ "--Lightning--": "lightning",21+ "a -- b\tc": "a-b-c",22+ "C++ & C#": "c-c",23+ "e\u0301lan": "e\u0301lan",24+ "": "",25+ "!!!": "",26+ "snake_case_name": "snake-case-name",27+ "Tabs\tand\nbreaks": "tabs-and-breaks",28+ }29+ for input, want := range cases {30+ if got := Normalize(input); got != want {31+ t.Errorf("Normalize(%q) = %q, want %q", input, got, want)32+ }33+ }34+}35+36+func TestLinksFindWikilinksReferencesAndNostrTargets(t *testing.T) {37+ content := "Bitcoin is a [cryptocurrency][] invented by [Satoshi Nakamoto][].\n\n" +38+ "See also: [proof of work][] and [lightning network][Lightning Network], [[Mining|the miners]] and _[Биткойн][]_.\n\n" +39+ "- [Bob](nostr:npub1bob) wrote `[[not a link]]`\n" +40+ "- [web](https://example.com/[[x]])\n\n" +41+ "```\n[[ignored in code]]\n```\n\n" +42+ "[Satoshi Nakamoto]: nostr:npub1satoshi\n"43+ got := Links(content)44+ want := []string{"nostr:npub1satoshi", "cryptocurrency", "proof-of-work", "lightning-network", "mining", "биткойн", "nostr:npub1bob"}45+ if fmt.Sprint(got) != fmt.Sprint(want) {46+ t.Fatalf("Links = %v, want %v", got, want)47+ }48+ if links := Links("plain text"); len(links) != 0 {49+ t.Fatalf("plain text links = %v", links)50+ }51+}52+53+func TestRenderHTMLSupportedSyntax(t *testing.T) {54+ content := "# Title with _emphasis_\n\n" +55+ "A paragraph with *strong*, `code <b>` and a [link](https://example.com/a?b=1&c=2).\n" +56+ "Second line of the paragraph.\n\n" +57+ "## Lists\n\n" +58+ "- one [[Wiki Page]]\n- two [[Другая|shown]]\n\n" +59+ "1. first\n2) second\n\n" +60+ "``` go\nfunc main() {}\n<script>alert(1)</script>\n```\n\n" +61+ "[Bob](nostr:npub1bob) and [reference][] and [defined][site].\n\n" +62+ "[site]: https://example.org/\n"63+ got := string(RenderHTML(content))64+ for _, want := range []string{65+ "<h1>Title with <em>emphasis</em></h1>",66+ "<p>A paragraph with <strong>strong</strong>, <code>code <b></code> and a <a href=\"https://example.com/a?b=1&c=2\">link</a>.\nSecond line of the paragraph.</p>",67+ "<h2>Lists</h2>",68+ "<ul>\n<li>one <a href=\"/wiki/wiki-page\">Wiki Page</a></li>\n<li>two <a href=\"/wiki/%D0%B4%D1%80%D1%83%D0%B3%D0%B0%D1%8F\">shown</a></li>\n</ul>",69+ "<ol>\n<li>first</li>\n<li>second</li>\n</ol>",70+ "<pre><code class=\"language-go\">func main() {}\n<script>alert(1)</script>\n</code></pre>",71+ "<a href=\"/open?target=nostr%3Anpub1bob\">Bob</a>",72+ "<a href=\"/wiki/reference\">reference</a>",73+ "<a href=\"https://example.org/\">defined</a>",74+ } {75+ if !strings.Contains(got, want) {76+ t.Errorf("missing %q in\n%s", want, got)77+ }78+ }79+ if strings.Contains(got, "[site]") {80+ t.Errorf("definition line rendered:\n%s", got)81+ }82+}83+84+func TestRenderHTMLEscapesEverythingElse(t *testing.T) {85+ cases := []struct{ content, forbidden, required string }{86+ {"<script>alert(1)</script>", "<script", "<script>"},87+ {"[click](javascript:alert(1))", "javascript:", "click"},88+ {"[click](JavaScript:alert(1))", "avaScript:", "click"},89+ {"[click](data:text/html;base64,AAAA)", "data:", "click"},90+ {"[click](vbscript:msgbox)", "vbscript:", "click"},91+ {"[click](//evil.example/x)", "evil.example", "click"},92+ {"[click](/\\evil.example)", "href", "click"},93+ {"[[Page]] [x](\"><img src=x onerror=alert(1)>)", "<img", "x>"},94+ {"[ref][]\n\n[ref]: javascript:alert(1)", "javascript:", "<p>ref</p>"},95+ {"<a href=\"https://x\">raw html</a>", "<a href", "<a href"},96+ {"<img src=x onerror=alert(1)>", "<img", "<img"},97+ {"_<b>bold</b>_", "<b>", "<em><b>bold</b></em>"},98+ {"[[<script>]]", "<script", "/wiki/script"},99+ {"[[x|<script>]]", "<script>", "<script>"},100+ {"`<script>`", "<script>", "<code><script></code>"},101+ {"[text](https://example.com/\"onmouseover=\"alert(1))", "onmouseover=\"", ""onmouseover=""},102+ }103+ for _, tc := range cases {104+ got := string(RenderHTML(tc.content))105+ if strings.Contains(got, tc.forbidden) {106+ t.Errorf("%q rendered %q:\n%s", tc.content, tc.forbidden, got)107+ }108+ if !strings.Contains(got, tc.required) {109+ t.Errorf("%q lacks %q:\n%s", tc.content, tc.required, got)110+ }111+ }112+}113+114+func TestRenderHTMLHandlesEdgeCases(t *testing.T) {115+ cases := map[string]string{116+ "": "",117+ "\n\n\n": "",118+ "```": "<pre><code>\n</code></pre>\n",119+ "```\nunterminated": "<pre><code>unterminated\n</code></pre>\n",120+ "#NoSpace": "<p>#NoSpace</p>\n",121+ "####### seven": "<p>####### seven</p>\n",122+ "* not strong": "<ul>\n<li>not strong</li>\n</ul>\n",123+ "a * b * c": "<p>a * b * c</p>\n",124+ "a_b_c": "<p>a<em>b</em>c</p>\n",125+ "\\*escaped\\*": "<p>*escaped*</p>\n",126+ "``code with ` inside``": "<p><code>code with ` inside</code></p>\n",127+ "[[]]": "<p>[[]]</p>\n",128+ "[unclosed": "<p>[unclosed</p>\n",129+ "[text](": "<p>[text](</p>\n",130+ "[mailto](mailto:a@b.example)": "<p><a href=\"mailto:a@b.example\">mailto</a></p>\n",131+ "[rel](/files)": "<p><a href=\"/files\">rel</a></p>\n",132+ "[bad](http://)": "<p>bad</p>\n",133+ "- item\nplain": "<ul>\n<li>item</li>\n</ul>\n<p>plain</p>\n",134+ "para\n- not a list": "<p>para\n- not a list</p>\n",135+ "Ünïcödé & <ampersand>": "<p>Ünïcödé & <ampersand></p>\n",136+ "line\r\nbreak": "<p>line\nbreak</p>\n",137+ }138+ for content, want := range cases {139+ if got := string(RenderHTML(content)); got != want {140+ t.Errorf("RenderHTML(%q) =\n%s\nwant\n%s", content, got, want)141+ }142+ }143+}