Use markdown frontmatter to provide Table of contents, language and frontmatter rendering (#11047)

* Add control for the rendering of the frontmatter
* Add control to include a TOC
* Add control to set language - allows control of ToC header and CJK glyph choice.

Signed-off-by: Andrew Thornton art27@cantab.net

go.mod

@@ -124,6 +124,7 @@ require (
 	gopkg.in/ini.v1 v1.52.0
 	gopkg.in/ldap.v3 v3.0.2
 	gopkg.in/testfixtures.v2 v2.5.0
+	gopkg.in/yaml.v2 v2.2.8
 	mvdan.cc/xurls/v2 v2.1.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.7

modules/markup/html.go

@@ -351,6 +351,27 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
 			visitText = false
 		} else if node.Data == "code" || node.Data == "pre" {
 			return
+		} else if node.Data == "i" {
+			for _, attr := range node.Attr {
+				if attr.Key != "class" {
+					continue
+				}
+				classes := strings.Split(attr.Val, " ")
+				for i, class := range classes {
+					if class == "icon" {
+						classes[0], classes[i] = classes[i], classes[0]
+						attr.Val = strings.Join(classes, " ")
+
+						// Remove all children of icons
+						child := node.FirstChild
+						for child != nil {
+							node.RemoveChild(child)
+							child = node.FirstChild
+						}
+						break
+					}
+				}
+			}
 		}
 		for n := node.FirstChild; n != nil; n = n.NextSibling {
 			ctx.visitNode(n, visitText)

modules/markup/markdown/ast.go

@@ -0,0 +1,107 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markdown
+
+import "github.com/yuin/goldmark/ast"
+
+// Details is a block that contains Summary and details
+type Details struct {
+	ast.BaseBlock
+}
+
+// Dump implements Node.Dump .
+func (n *Details) Dump(source []byte, level int) {
+	ast.DumpHelper(n, source, level, nil, nil)
+}
+
+// KindDetails is the NodeKind for Details
+var KindDetails = ast.NewNodeKind("Details")
+
+// Kind implements Node.Kind.
+func (n *Details) Kind() ast.NodeKind {
+	return KindDetails
+}
+
+// NewDetails returns a new Paragraph node.
+func NewDetails() *Details {
+	return &Details{
+		BaseBlock: ast.BaseBlock{},
+	}
+}
+
+// IsDetails returns true if the given node implements the Details interface,
+// otherwise false.
+func IsDetails(node ast.Node) bool {
+	_, ok := node.(*Details)
+	return ok
+}
+
+// Summary is a block that contains the summary of details block
+type Summary struct {
+	ast.BaseBlock
+}
+
+// Dump implements Node.Dump .
+func (n *Summary) Dump(source []byte, level int) {
+	ast.DumpHelper(n, source, level, nil, nil)
+}
+
+// KindSummary is the NodeKind for Summary
+var KindSummary = ast.NewNodeKind("Summary")
+
+// Kind implements Node.Kind.
+func (n *Summary) Kind() ast.NodeKind {
+	return KindSummary
+}
+
+// NewSummary returns a new Summary node.
+func NewSummary() *Summary {
+	return &Summary{
+		BaseBlock: ast.BaseBlock{},
+	}
+}
+
+// IsSummary returns true if the given node implements the Summary interface,
+// otherwise false.
+func IsSummary(node ast.Node) bool {
+	_, ok := node.(*Summary)
+	return ok
+}
+
+// Icon is an inline for a fomantic icon
+type Icon struct {
+	ast.BaseInline
+	Name []byte
+}
+
+// Dump implements Node.Dump .
+func (n *Icon) Dump(source []byte, level int) {
+	m := map[string]string{}
+	m["Name"] = string(n.Name)
+	ast.DumpHelper(n, source, level, m, nil)
+}
+
+// KindIcon is the NodeKind for Icon
+var KindIcon = ast.NewNodeKind("Icon")
+
+// Kind implements Node.Kind.
+func (n *Icon) Kind() ast.NodeKind {
+	return KindIcon
+}
+
+// NewIcon returns a new Paragraph node.
+func NewIcon(name string) *Icon {
+	return &Icon{
+		BaseInline: ast.BaseInline{},
+		Name:       []byte(name),
+	}
+}
+
+// IsIcon returns true if the given node implements the Icon interface,
+// otherwise false.
+func IsIcon(node ast.Node) bool {
+	_, ok := node.(*Icon)
+	return ok
+}

modules/markup/markdown/goldmark.go

@@ -7,12 +7,16 @@ package markdown
 import (
 	"bytes"
 	"fmt"
+	"regexp"
 	"strings"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/common"
+	"code.gitea.io/gitea/modules/setting"
 	giteautil "code.gitea.io/gitea/modules/util"
 
+	meta "github.com/yuin/goldmark-meta"
 	"github.com/yuin/goldmark/ast"
 	east "github.com/yuin/goldmark/extension/ast"
 	"github.com/yuin/goldmark/parser"
@@ -24,17 +28,56 @@ import (
 
 var byteMailto = []byte("mailto:")
 
-// GiteaASTTransformer is a default transformer of the goldmark tree.
+// Header holds the data about a header.
-type GiteaASTTransformer struct{}
+type Header struct {
+	Level int
+	Text  string
+	ID    string
+}
+
+// ASTTransformer is a default transformer of the goldmark tree.
+type ASTTransformer struct{}
 
 // Transform transforms the given AST tree.
-func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
+func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
+	metaData := meta.GetItems(pc)
+	firstChild := node.FirstChild()
+	createTOC := false
+	var toc = []Header{}
+	rc := &RenderConfig{
+		Meta: "table",
+		Icon: "table",
+		Lang: "",
+	}
+	if metaData != nil {
+		rc.ToRenderConfig(metaData)
+
+		metaNode := rc.toMetaNode(metaData)
+		if metaNode != nil {
+			node.InsertBefore(node, firstChild, metaNode)
+		}
+		createTOC = rc.TOC
+		toc = make([]Header, 0, 100)
+	}
+
 	_ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
 		if !entering {
 			return ast.WalkContinue, nil
 		}
 
 		switch v := n.(type) {
+		case *ast.Heading:
+			if createTOC {
+				text := n.Text(reader.Source())
+				header := Header{
+					Text:  util.BytesToReadOnlyString(text),
+					Level: v.Level,
+				}
+				if id, found := v.AttributeString("id"); found {
+					header.ID = util.BytesToReadOnlyString(id.([]byte))
+				}
+				toc = append(toc, header)
+			}
 		case *ast.Image:
 			// Images need two things:
 			//
@@ -91,6 +134,21 @@ func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader,
 		}
 		return ast.WalkContinue, nil
 	})
+
+	if createTOC && len(toc) > 0 {
+		lang := rc.Lang
+		if len(lang) == 0 {
+			lang = setting.Langs[0]
+		}
+		tocNode := createTOCNode(toc, lang)
+		if tocNode != nil {
+			node.InsertBefore(node, firstChild, tocNode)
+		}
+	}
+
+	if len(rc.Lang) > 0 {
+		node.SetAttributeString("lang", []byte(rc.Lang))
+	}
 }
 
 type prefixedIDs struct {
@@ -139,10 +197,10 @@ func newPrefixedIDs() *prefixedIDs {
 	}
 }
 
-// NewTaskCheckBoxHTMLRenderer creates a TaskCheckBoxHTMLRenderer to render tasklists
+// NewHTMLRenderer creates a HTMLRenderer to render
 // in the gitea form.
-func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer {
+func NewHTMLRenderer(opts ...html.Option) renderer.NodeRenderer {
-	r := &TaskCheckBoxHTMLRenderer{
+	r := &HTMLRenderer{
 		Config: html.NewConfig(),
 	}
 	for _, opt := range opts {
@@ -151,19 +209,109 @@ func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer {
 	return r
 }
 
-// TaskCheckBoxHTMLRenderer is a renderer.NodeRenderer implementation that
+// HTMLRenderer is a renderer.NodeRenderer implementation that
-// renders checkboxes in list items.
+// renders gitea specific features.
-// Overrides the default goldmark one to present the gitea format
+type HTMLRenderer struct {
-type TaskCheckBoxHTMLRenderer struct {
 	html.Config
 }
 
 // RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs.
-func (r *TaskCheckBoxHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+	reg.Register(ast.KindDocument, r.renderDocument)
+	reg.Register(KindDetails, r.renderDetails)
+	reg.Register(KindSummary, r.renderSummary)
+	reg.Register(KindIcon, r.renderIcon)
 	reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox)
 }
 
-func (r *TaskCheckBoxHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	log.Info("renderDocument %v", node)
+	n := node.(*ast.Document)
+
+	if val, has := n.AttributeString("lang"); has {
+		var err error
+		if entering {
+			_, err = w.WriteString("<div")
+			if err == nil {
+				_, err = w.WriteString(fmt.Sprintf(` lang=%q`, val))
+			}
+			if err == nil {
+				_, err = w.WriteRune('>')
+			}
+		} else {
+			_, err = w.WriteString("</div>")
+		}
+
+		if err != nil {
+			return ast.WalkStop, err
+		}
+	}
+
+	return ast.WalkContinue, nil
+}
+
+func (r *HTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	var err error
+	if entering {
+		_, err = w.WriteString("<details>")
+	} else {
+		_, err = w.WriteString("</details>")
+	}
+
+	if err != nil {
+		return ast.WalkStop, err
+	}
+
+	return ast.WalkContinue, nil
+}
+
+func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	var err error
+	if entering {
+		_, err = w.WriteString("<summary>")
+	} else {
+		_, err = w.WriteString("</summary>")
+	}
+
+	if err != nil {
+		return ast.WalkStop, err
+	}
+
+	return ast.WalkContinue, nil
+}
+
+var validNameRE = regexp.MustCompile("^[a-z ]+$")
+
+func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+
+	n := node.(*Icon)
+
+	name := strings.TrimSpace(strings.ToLower(string(n.Name)))
+
+	if len(name) == 0 {
+		// skip this
+		return ast.WalkContinue, nil
+	}
+
+	if !validNameRE.MatchString(name) {
+		// skip this
+		return ast.WalkContinue, nil
+	}
+
+	var err error
+	_, err = w.WriteString(fmt.Sprintf(`<i class="icon %s"></i>`, name))
+
+	if err != nil {
+		return ast.WalkStop, err
+	}
+
+	return ast.WalkContinue, nil
+}
+
+func (r *HTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
 	if !entering {
 		return ast.WalkContinue, nil
 	}

modules/markup/markdown/markdown.go

@@ -54,13 +54,13 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
 						extension.Ellipsis: nil,
 					}),
 				),
-				meta.New(meta.WithTable()),
+				meta.Meta,
 			),
 			goldmark.WithParserOptions(
 				parser.WithAttribute(),
 				parser.WithAutoHeadingID(),
 				parser.WithASTTransformers(
-					util.Prioritized(&GiteaASTTransformer{}, 10000),
+					util.Prioritized(&ASTTransformer{}, 10000),
 				),
 			),
 			goldmark.WithRendererOptions(
@@ -71,7 +71,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
 		// Override the original Tasklist renderer!
 		converter.Renderer().AddOptions(
 			renderer.WithNodeRenderers(
-				util.Prioritized(NewTaskCheckBoxHTMLRenderer(), 1000),
+				util.Prioritized(NewHTMLRenderer(), 10),
 			),
 		)
 
@@ -85,7 +85,6 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
 	if err := converter.Convert(giteautil.NormalizeEOL(body), &buf, parser.WithContext(pc)); err != nil {
 		log.Error("Unable to render: %v", err)
 	}
-
 	return markup.SanitizeReader(&buf).Bytes()
 }
 

modules/markup/markdown/renderconfig.go

@@ -0,0 +1,163 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markdown
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/yuin/goldmark/ast"
+	east "github.com/yuin/goldmark/extension/ast"
+	"gopkg.in/yaml.v2"
+)
+
+// RenderConfig represents rendering configuration for this file
+type RenderConfig struct {
+	Meta string
+	Icon string
+	TOC  bool
+	Lang string
+}
+
+// ToRenderConfig converts a yaml.MapSlice to a RenderConfig
+func (rc *RenderConfig) ToRenderConfig(meta yaml.MapSlice) {
+	if meta == nil {
+		return
+	}
+	found := false
+	var giteaMetaControl yaml.MapItem
+	for _, item := range meta {
+		strKey, ok := item.Key.(string)
+		if !ok {
+			continue
+		}
+		strKey = strings.TrimSpace(strings.ToLower(strKey))
+		switch strKey {
+		case "gitea":
+			giteaMetaControl = item
+			found = true
+		case "include_toc":
+			val, ok := item.Value.(bool)
+			if !ok {
+				continue
+			}
+			rc.TOC = val
+		case "lang":
+			val, ok := item.Value.(string)
+			if !ok {
+				continue
+			}
+			val = strings.TrimSpace(val)
+			if len(val) == 0 {
+				continue
+			}
+			rc.Lang = val
+		}
+	}
+
+	if found {
+		switch v := giteaMetaControl.Value.(type) {
+		case string:
+			switch v {
+			case "none":
+				rc.Meta = "none"
+			case "table":
+				rc.Meta = "table"
+			default: // "details"
+				rc.Meta = "details"
+			}
+		case yaml.MapSlice:
+			for _, item := range v {
+				strKey, ok := item.Key.(string)
+				if !ok {
+					continue
+				}
+				strKey = strings.TrimSpace(strings.ToLower(strKey))
+				switch strKey {
+				case "meta":
+					val, ok := item.Value.(string)
+					if !ok {
+						continue
+					}
+					switch strings.TrimSpace(strings.ToLower(val)) {
+					case "none":
+						rc.Meta = "none"
+					case "table":
+						rc.Meta = "table"
+					default: // "details"
+						rc.Meta = "details"
+					}
+				case "details_icon":
+					val, ok := item.Value.(string)
+					if !ok {
+						continue
+					}
+					rc.Icon = strings.TrimSpace(strings.ToLower(val))
+				case "include_toc":
+					val, ok := item.Value.(bool)
+					if !ok {
+						continue
+					}
+					rc.TOC = val
+				case "lang":
+					val, ok := item.Value.(string)
+					if !ok {
+						continue
+					}
+					val = strings.TrimSpace(val)
+					if len(val) == 0 {
+						continue
+					}
+					rc.Lang = val
+				}
+			}
+		}
+	}
+}
+
+func (rc *RenderConfig) toMetaNode(meta yaml.MapSlice) ast.Node {
+	switch rc.Meta {
+	case "table":
+		return metaToTable(meta)
+	case "details":
+		return metaToDetails(meta, rc.Icon)
+	default:
+		return nil
+	}
+}
+
+func metaToTable(meta yaml.MapSlice) ast.Node {
+	table := east.NewTable()
+	alignments := []east.Alignment{}
+	for range meta {
+		alignments = append(alignments, east.AlignNone)
+	}
+	row := east.NewTableRow(alignments)
+	for _, item := range meta {
+		cell := east.NewTableCell()
+		cell.AppendChild(cell, ast.NewString([]byte(fmt.Sprintf("%v", item.Key))))
+		row.AppendChild(row, cell)
+	}
+	table.AppendChild(table, east.NewTableHeader(row))
+
+	row = east.NewTableRow(alignments)
+	for _, item := range meta {
+		cell := east.NewTableCell()
+		cell.AppendChild(cell, ast.NewString([]byte(fmt.Sprintf("%v", item.Value))))
+		row.AppendChild(row, cell)
+	}
+	table.AppendChild(table, row)
+	return table
+}
+
+func metaToDetails(meta yaml.MapSlice, icon string) ast.Node {
+	details := NewDetails()
+	summary := NewSummary()
+	summary.AppendChild(summary, NewIcon(icon))
+	details.AppendChild(details, summary)
+	details.AppendChild(details, metaToTable(meta))
+
+	return details
+}

modules/markup/markdown/toc.go

@@ -0,0 +1,49 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markdown
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/unknwon/i18n"
+	"github.com/yuin/goldmark/ast"
+)
+
+func createTOCNode(toc []Header, lang string) ast.Node {
+	details := NewDetails()
+	summary := NewSummary()
+
+	summary.AppendChild(summary, ast.NewString([]byte(i18n.Tr(lang, "toc"))))
+	details.AppendChild(details, summary)
+	ul := ast.NewList('-')
+	details.AppendChild(details, ul)
+	currentLevel := 6
+	for _, header := range toc {
+		if header.Level < currentLevel {
+			currentLevel = header.Level
+		}
+	}
+	for _, header := range toc {
+		for currentLevel > header.Level {
+			ul = ul.Parent().(*ast.List)
+			currentLevel--
+		}
+		for currentLevel < header.Level {
+			newL := ast.NewList('-')
+			ul.AppendChild(ul, newL)
+			currentLevel++
+			ul = newL
+		}
+		li := ast.NewListItem(currentLevel * 2)
+		a := ast.NewLink()
+		a.Destination = []byte(fmt.Sprintf("#%s", url.PathEscape(header.ID)))
+		a.AppendChild(a, ast.NewString([]byte(header.Text)))
+		li.AppendChild(li, a)
+		ul.AppendChild(ul, li)
+	}
+
+	return details
+}

modules/markup/sanitizer.go

@@ -56,6 +56,9 @@ func ReplaceSanitizer() {
 	// Allow classes for task lists
 	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`task-list`)).OnElements("ul")
 
+	// Allow icons
+	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^icon(\s+[\p{L}\p{N}_-]+)+$`)).OnElements("i", "span")
+
 	// Allow generally safe attributes
 	generalSafeAttrs := []string{"abbr", "accept", "accept-charset",
 		"accesskey", "action", "align", "alt",

options/locale/locale_en-US.ini

@@ -19,6 +19,7 @@ create_new = Create…
 user_profile_and_more = Profile and Settings…
 signed_in_as = Signed in as
 enable_javascript = This website works better with JavaScript.
+toc = Table of Contents
 
 username = Username
 email = Email Address

vendor/modules.txt

@@ -844,6 +844,7 @@ gopkg.in/toqueteos/substring.v1
 # gopkg.in/warnings.v0 v0.1.2
 gopkg.in/warnings.v0
 # gopkg.in/yaml.v2 v2.2.8
+## explicit
 gopkg.in/yaml.v2
 # mvdan.cc/xurls/v2 v2.1.0
 ## explicit