parser.go

  1/*
  2Package parser implements parser for markdown text that generates AST (abstract syntax tree).
  3*/
  4package parser
  5
  6import (
  7	"bytes"
  8	"fmt"
  9	"strings"
 10	"unicode/utf8"
 11
 12	"github.com/gomarkdown/markdown/ast"
 13)
 14
 15// Extensions is a bitmask of enabled parser extensions.
 16type Extensions int
 17
 18// Bit flags representing markdown parsing extensions.
 19// Use | (or) to specify multiple extensions.
 20const (
 21	NoExtensions           Extensions = 0
 22	NoIntraEmphasis        Extensions = 1 << iota // Ignore emphasis markers inside words
 23	Tables                                        // Parse tables
 24	FencedCode                                    // Parse fenced code blocks
 25	Autolink                                      // Detect embedded URLs that are not explicitly marked
 26	Strikethrough                                 // Strikethrough text using ~~test~~
 27	LaxHTMLBlocks                                 // Loosen up HTML block parsing rules
 28	SpaceHeadings                                 // Be strict about prefix heading rules
 29	HardLineBreak                                 // Translate newlines into line breaks
 30	NonBlockingSpace                              // Translate backspace spaces into line non-blocking spaces
 31	TabSizeEight                                  // Expand tabs to eight spaces instead of four
 32	Footnotes                                     // Pandoc-style footnotes
 33	NoEmptyLineBeforeBlock                        // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
 34	HeadingIDs                                    // specify heading IDs  with {#id}
 35	Titleblock                                    // Titleblock ala pandoc
 36	AutoHeadingIDs                                // Create the heading ID from the text
 37	BackslashLineBreak                            // Translate trailing backslashes into line breaks
 38	DefinitionLists                               // Parse definition lists
 39	MathJax                                       // Parse MathJax
 40	OrderedListStart                              // Keep track of the first number used when starting an ordered list.
 41	Attributes                                    // Block Attributes
 42	SuperSubscript                                // Super- and subscript support: 2^10^, H~2~O.
 43	EmptyLinesBreakList                           // 2 empty lines break out of list
 44	Includes                                      // Support including other files.
 45	Mmark                                         // Support Mmark syntax, see https://mmark.nl/syntax
 46
 47	CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
 48		Autolink | Strikethrough | SpaceHeadings | HeadingIDs |
 49		BackslashLineBreak | DefinitionLists | MathJax
 50)
 51
 52// The size of a tab stop.
 53const (
 54	tabSizeDefault = 4
 55	tabSizeDouble  = 8
 56)
 57
 58// for each character that triggers a response when parsing inline data.
 59type inlineParser func(p *Parser, data []byte, offset int) (int, ast.Node)
 60
 61// ReferenceOverrideFunc is expected to be called with a reference string and
 62// return either a valid Reference type that the reference string maps to or
 63// nil. If overridden is false, the default reference logic will be executed.
 64// See the documentation in Options for more details on use-case.
 65type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool)
 66
 67// Parser is a type that holds extensions and the runtime state used by
 68// Parse, and the renderer. You can not use it directly, construct it with New.
 69type Parser struct {
 70
 71	// ReferenceOverride is an optional function callback that is called every
 72	// time a reference is resolved. It can be set before starting parsing.
 73	//
 74	// In Markdown, the link reference syntax can be made to resolve a link to
 75	// a reference instead of an inline URL, in one of the following ways:
 76	//
 77	//  * [link text][refid]
 78	//  * [refid][]
 79	//
 80	// Usually, the refid is defined at the bottom of the Markdown document. If
 81	// this override function is provided, the refid is passed to the override
 82	// function first, before consulting the defined refids at the bottom. If
 83	// the override function indicates an override did not occur, the refids at
 84	// the bottom will be used to fill in the link details.
 85	ReferenceOverride ReferenceOverrideFunc
 86
 87	Opts Options
 88
 89	// after parsing, this is AST root of parsed markdown text
 90	Doc ast.Node
 91
 92	extensions Extensions
 93
 94	refs           map[string]*reference
 95	refsRecord     map[string]struct{}
 96	inlineCallback [256]inlineParser
 97	nesting        int
 98	maxNesting     int
 99	insideLink     bool
100	indexCnt       int // incremented after every index
101
102	// Footnotes need to be ordered as well as available to quickly check for
103	// presence. If a ref is also a footnote, it's stored both in refs and here
104	// in notes. Slice is nil if footnotes not enabled.
105	notes []*reference
106
107	tip                  ast.Node // = doc
108	oldTip               ast.Node
109	lastMatchedContainer ast.Node // = doc
110	allClosed            bool
111
112	// Attributes are attached to block level elements.
113	attr *ast.Attribute
114
115	includeStack *incStack
116}
117
118// New creates a markdown parser with CommonExtensions.
119//
120// You can then call `doc := p.Parse(markdown)` to parse markdown document
121// and `markdown.Render(doc, renderer)` to convert it to another format with
122// a renderer.
123func New() *Parser {
124	return NewWithExtensions(CommonExtensions)
125}
126
127// NewWithExtensions creates a markdown parser with given extensions.
128func NewWithExtensions(extension Extensions) *Parser {
129	p := Parser{
130		refs:         make(map[string]*reference),
131		refsRecord:   make(map[string]struct{}),
132		maxNesting:   16,
133		insideLink:   false,
134		Doc:          &ast.Document{},
135		extensions:   extension,
136		allClosed:    true,
137		includeStack: newIncStack(),
138	}
139	p.tip = p.Doc
140	p.oldTip = p.Doc
141	p.lastMatchedContainer = p.Doc
142
143	p.inlineCallback[' '] = maybeLineBreak
144	p.inlineCallback['*'] = emphasis
145	p.inlineCallback['_'] = emphasis
146	if p.extensions&Strikethrough != 0 {
147		p.inlineCallback['~'] = emphasis
148	}
149	p.inlineCallback['`'] = codeSpan
150	p.inlineCallback['\n'] = lineBreak
151	p.inlineCallback['['] = link
152	p.inlineCallback['<'] = leftAngle
153	p.inlineCallback['\\'] = escape
154	p.inlineCallback['&'] = entity
155	p.inlineCallback['!'] = maybeImage
156	if p.extensions&Mmark != 0 {
157		p.inlineCallback['('] = maybeShortRefOrIndex
158	}
159	p.inlineCallback['^'] = maybeInlineFootnoteOrSuper
160	if p.extensions&Autolink != 0 {
161		p.inlineCallback['h'] = maybeAutoLink
162		p.inlineCallback['m'] = maybeAutoLink
163		p.inlineCallback['f'] = maybeAutoLink
164		p.inlineCallback['H'] = maybeAutoLink
165		p.inlineCallback['M'] = maybeAutoLink
166		p.inlineCallback['F'] = maybeAutoLink
167	}
168	if p.extensions&MathJax != 0 {
169		p.inlineCallback['$'] = math
170	}
171
172	return &p
173}
174
175func (p *Parser) getRef(refid string) (ref *reference, found bool) {
176	if p.ReferenceOverride != nil {
177		r, overridden := p.ReferenceOverride(refid)
178		if overridden {
179			if r == nil {
180				return nil, false
181			}
182			return &reference{
183				link:     []byte(r.Link),
184				title:    []byte(r.Title),
185				noteID:   0,
186				hasBlock: false,
187				text:     []byte(r.Text)}, true
188		}
189	}
190	// refs are case insensitive
191	ref, found = p.refs[strings.ToLower(refid)]
192	return ref, found
193}
194
195func (p *Parser) isFootnote(ref *reference) bool {
196	_, ok := p.refsRecord[string(ref.link)]
197	return ok
198}
199
200func (p *Parser) finalize(block ast.Node) {
201	p.tip = block.GetParent()
202}
203
204func (p *Parser) addChild(node ast.Node) ast.Node {
205	for !canNodeContain(p.tip, node) {
206		p.finalize(p.tip)
207	}
208	ast.AppendChild(p.tip, node)
209	p.tip = node
210	return node
211}
212
213func canNodeContain(n ast.Node, v ast.Node) bool {
214	switch n.(type) {
215	case *ast.List:
216		return isListItem(v)
217	case *ast.Document, *ast.BlockQuote, *ast.Aside, *ast.ListItem, *ast.CaptionFigure:
218		return !isListItem(v)
219	case *ast.Table:
220		switch v.(type) {
221		case *ast.TableHeader, *ast.TableBody, *ast.TableFooter:
222			return true
223		default:
224			return false
225		}
226	case *ast.TableHeader, *ast.TableBody, *ast.TableFooter:
227		_, ok := v.(*ast.TableRow)
228		return ok
229	case *ast.TableRow:
230		_, ok := v.(*ast.TableCell)
231		return ok
232	}
233	return false
234}
235
236func (p *Parser) closeUnmatchedBlocks() {
237	if p.allClosed {
238		return
239	}
240	for p.oldTip != p.lastMatchedContainer {
241		parent := p.oldTip.GetParent()
242		p.finalize(p.oldTip)
243		p.oldTip = parent
244	}
245	p.allClosed = true
246}
247
248// Reference represents the details of a link.
249// See the documentation in Options for more details on use-case.
250type Reference struct {
251	// Link is usually the URL the reference points to.
252	Link string
253	// Title is the alternate text describing the link in more detail.
254	Title string
255	// Text is the optional text to override the ref with if the syntax used was
256	// [refid][]
257	Text string
258}
259
260// Parse generates AST (abstract syntax tree) representing markdown document.
261//
262// The result is a root of the tree whose underlying type is *ast.Document
263//
264// You can then convert AST to html using html.Renderer, to some other format
265// using a custom renderer or transform the tree.
266func (p *Parser) Parse(input []byte) ast.Node {
267	p.block(input)
268	// Walk the tree and finish up some of unfinished blocks
269	for p.tip != nil {
270		p.finalize(p.tip)
271	}
272	// Walk the tree again and process inline markdown in each block
273	ast.WalkFunc(p.Doc, func(node ast.Node, entering bool) ast.WalkStatus {
274		switch node.(type) {
275		case *ast.Paragraph, *ast.Heading, *ast.TableCell:
276			p.Inline(node, node.AsContainer().Content)
277			node.AsContainer().Content = nil
278		}
279		return ast.GoToNext
280	})
281
282	if p.Opts.Flags&SkipFootnoteList == 0 {
283		p.parseRefsToAST()
284	}
285	return p.Doc
286}
287
288func (p *Parser) parseRefsToAST() {
289	if p.extensions&Footnotes == 0 || len(p.notes) == 0 {
290		return
291	}
292	p.tip = p.Doc
293	list := &ast.List{
294		IsFootnotesList: true,
295		ListFlags:       ast.ListTypeOrdered,
296	}
297	p.addBlock(&ast.Footnotes{})
298	block := p.addBlock(list)
299	flags := ast.ListItemBeginningOfList
300	// Note: this loop is intentionally explicit, not range-form. This is
301	// because the body of the loop will append nested footnotes to p.notes and
302	// we need to process those late additions. Range form would only walk over
303	// the fixed initial set.
304	for i := 0; i < len(p.notes); i++ {
305		ref := p.notes[i]
306		p.addChild(ref.footnote)
307		block := ref.footnote
308		listItem := block.(*ast.ListItem)
309		listItem.ListFlags = flags | ast.ListTypeOrdered
310		listItem.RefLink = ref.link
311		if ref.hasBlock {
312			flags |= ast.ListItemContainsBlock
313			p.block(ref.title)
314		} else {
315			p.Inline(block, ref.title)
316		}
317		flags &^= ast.ListItemBeginningOfList | ast.ListItemContainsBlock
318	}
319	above := list.Parent
320	finalizeList(list)
321	p.tip = above
322
323	ast.WalkFunc(block, func(node ast.Node, entering bool) ast.WalkStatus {
324		switch node.(type) {
325		case *ast.Paragraph, *ast.Heading:
326			p.Inline(node, node.AsContainer().Content)
327			node.AsContainer().Content = nil
328		}
329		return ast.GoToNext
330	})
331}
332
333//
334// Link references
335//
336// This section implements support for references that (usually) appear
337// as footnotes in a document, and can be referenced anywhere in the document.
338// The basic format is:
339//
340//    [1]: http://www.google.com/ "Google"
341//    [2]: http://www.github.com/ "Github"
342//
343// Anywhere in the document, the reference can be linked by referring to its
344// label, i.e., 1 and 2 in this example, as in:
345//
346//    This library is hosted on [Github][2], a git hosting site.
347//
348// Actual footnotes as specified in Pandoc and supported by some other Markdown
349// libraries such as php-markdown are also taken care of. They look like this:
350//
351//    This sentence needs a bit of further explanation.[^note]
352//
353//    [^note]: This is the explanation.
354//
355// Footnotes should be placed at the end of the document in an ordered list.
356// Inline footnotes such as:
357//
358//    Inline footnotes^[Not supported.] also exist.
359//
360// are not yet supported.
361
362// reference holds all information necessary for a reference-style links or
363// footnotes.
364//
365// Consider this markdown with reference-style links:
366//
367//     [link][ref]
368//
369//     [ref]: /url/ "tooltip title"
370//
371// It will be ultimately converted to this HTML:
372//
373//     <p><a href=\"/url/\" title=\"title\">link</a></p>
374//
375// And a reference structure will be populated as follows:
376//
377//     p.refs["ref"] = &reference{
378//         link: "/url/",
379//         title: "tooltip title",
380//     }
381//
382// Alternatively, reference can contain information about a footnote. Consider
383// this markdown:
384//
385//     Text needing a footnote.[^a]
386//
387//     [^a]: This is the note
388//
389// A reference structure will be populated as follows:
390//
391//     p.refs["a"] = &reference{
392//         link: "a",
393//         title: "This is the note",
394//         noteID: <some positive int>,
395//     }
396//
397// TODO: As you can see, it begs for splitting into two dedicated structures
398// for refs and for footnotes.
399type reference struct {
400	link     []byte
401	title    []byte
402	noteID   int // 0 if not a footnote ref
403	hasBlock bool
404	footnote ast.Node // a link to the Item node within a list of footnotes
405
406	text []byte // only gets populated by refOverride feature with Reference.Text
407}
408
409func (r *reference) String() string {
410	return fmt.Sprintf("{link: %q, title: %q, text: %q, noteID: %d, hasBlock: %v}",
411		r.link, r.title, r.text, r.noteID, r.hasBlock)
412}
413
414// Check whether or not data starts with a reference link.
415// If so, it is parsed and stored in the list of references
416// (in the render struct).
417// Returns the number of bytes to skip to move past it,
418// or zero if the first line is not a reference.
419func isReference(p *Parser, data []byte, tabSize int) int {
420	// up to 3 optional leading spaces
421	if len(data) < 4 {
422		return 0
423	}
424	i := 0
425	for i < 3 && data[i] == ' ' {
426		i++
427	}
428
429	noteID := 0
430
431	// id part: anything but a newline between brackets
432	if data[i] != '[' {
433		return 0
434	}
435	i++
436	if p.extensions&Footnotes != 0 {
437		if i < len(data) && data[i] == '^' {
438			// we can set it to anything here because the proper noteIds will
439			// be assigned later during the second pass. It just has to be != 0
440			noteID = 1
441			i++
442		}
443	}
444	idOffset := i
445	for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
446		i++
447	}
448	if i >= len(data) || data[i] != ']' {
449		return 0
450	}
451	idEnd := i
452	// footnotes can have empty ID, like this: [^], but a reference can not be
453	// empty like this: []. Break early if it's not a footnote and there's no ID
454	if noteID == 0 && idOffset == idEnd {
455		return 0
456	}
457	// spacer: colon (space | tab)* newline? (space | tab)*
458	i++
459	if i >= len(data) || data[i] != ':' {
460		return 0
461	}
462	i++
463	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
464		i++
465	}
466	if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
467		i++
468		if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
469			i++
470		}
471	}
472	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
473		i++
474	}
475	if i >= len(data) {
476		return 0
477	}
478
479	var (
480		linkOffset, linkEnd   int
481		titleOffset, titleEnd int
482		lineEnd               int
483		raw                   []byte
484		hasBlock              bool
485	)
486
487	if p.extensions&Footnotes != 0 && noteID != 0 {
488		linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
489		lineEnd = linkEnd
490	} else {
491		linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
492	}
493	if lineEnd == 0 {
494		return 0
495	}
496
497	// a valid ref has been found
498
499	ref := &reference{
500		noteID:   noteID,
501		hasBlock: hasBlock,
502	}
503
504	if noteID > 0 {
505		// reusing the link field for the id since footnotes don't have links
506		ref.link = data[idOffset:idEnd]
507		// if footnote, it's not really a title, it's the contained text
508		ref.title = raw
509	} else {
510		ref.link = data[linkOffset:linkEnd]
511		ref.title = data[titleOffset:titleEnd]
512	}
513
514	// id matches are case-insensitive
515	id := string(bytes.ToLower(data[idOffset:idEnd]))
516
517	p.refs[id] = ref
518
519	return lineEnd
520}
521
522func scanLinkRef(p *Parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
523	// link: whitespace-free sequence, optionally between angle brackets
524	if data[i] == '<' {
525		i++
526	}
527	linkOffset = i
528	for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
529		i++
530	}
531	linkEnd = i
532	if linkEnd < len(data) && data[linkOffset] == '<' && data[linkEnd-1] == '>' {
533		linkOffset++
534		linkEnd--
535	}
536
537	// optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
538	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
539		i++
540	}
541	if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
542		return
543	}
544
545	// compute end-of-line
546	if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
547		lineEnd = i
548	}
549	if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
550		lineEnd++
551	}
552
553	// optional (space|tab)* spacer after a newline
554	if lineEnd > 0 {
555		i = lineEnd + 1
556		for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
557			i++
558		}
559	}
560
561	// optional title: any non-newline sequence enclosed in '"() alone on its line
562	if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
563		i++
564		titleOffset = i
565
566		// look for EOL
567		for i < len(data) && data[i] != '\n' && data[i] != '\r' {
568			i++
569		}
570		if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
571			titleEnd = i + 1
572		} else {
573			titleEnd = i
574		}
575
576		// step back
577		i--
578		for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
579			i--
580		}
581		if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
582			lineEnd = titleEnd
583			titleEnd = i
584		}
585	}
586
587	return
588}
589
590// The first bit of this logic is the same as Parser.listItem, but the rest
591// is much simpler. This function simply finds the entire block and shifts it
592// over by one tab if it is indeed a block (just returns the line if it's not).
593// blockEnd is the end of the section in the input buffer, and contents is the
594// extracted text that was shifted over one tab. It will need to be rendered at
595// the end of the document.
596func scanFootnote(p *Parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
597	if i == 0 || len(data) == 0 {
598		return
599	}
600
601	// skip leading whitespace on first line
602	for i < len(data) && data[i] == ' ' {
603		i++
604	}
605
606	blockStart = i
607
608	// find the end of the line
609	blockEnd = i
610	for i < len(data) && data[i-1] != '\n' {
611		i++
612	}
613
614	// get working buffer
615	var raw bytes.Buffer
616
617	// put the first line into the working buffer
618	raw.Write(data[blockEnd:i])
619	blockEnd = i
620
621	// process the following lines
622	containsBlankLine := false
623
624gatherLines:
625	for blockEnd < len(data) {
626		i++
627
628		// find the end of this line
629		for i < len(data) && data[i-1] != '\n' {
630			i++
631		}
632
633		// if it is an empty line, guess that it is part of this item
634		// and move on to the next line
635		if p.isEmpty(data[blockEnd:i]) > 0 {
636			containsBlankLine = true
637			blockEnd = i
638			continue
639		}
640
641		n := 0
642		if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
643			// this is the end of the block.
644			// we don't want to include this last line in the index.
645			break gatherLines
646		}
647
648		// if there were blank lines before this one, insert a new one now
649		if containsBlankLine {
650			raw.WriteByte('\n')
651			containsBlankLine = false
652		}
653
654		// get rid of that first tab, write to buffer
655		raw.Write(data[blockEnd+n : i])
656		hasBlock = true
657
658		blockEnd = i
659	}
660
661	if data[blockEnd-1] != '\n' {
662		raw.WriteByte('\n')
663	}
664
665	contents = raw.Bytes()
666
667	return
668}
669
670// isPunctuation returns true if c is a punctuation symbol.
671func isPunctuation(c byte) bool {
672	for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
673		if c == r {
674			return true
675		}
676	}
677	return false
678}
679
680// isSpace returns true if c is a white-space charactr
681func isSpace(c byte) bool {
682	return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
683}
684
685// isLetter returns true if c is ascii letter
686func isLetter(c byte) bool {
687	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
688}
689
690// isAlnum returns true if c is a digit or letter
691// TODO: check when this is looking for ASCII alnum and when it should use unicode
692func isAlnum(c byte) bool {
693	return (c >= '0' && c <= '9') || isLetter(c)
694}
695
696// TODO: this is not used
697// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
698// always ends output with a newline
699func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
700	// first, check for common cases: no tabs, or only tabs at beginning of line
701	i, prefix := 0, 0
702	slowcase := false
703	for i = 0; i < len(line); i++ {
704		if line[i] == '\t' {
705			if prefix == i {
706				prefix++
707			} else {
708				slowcase = true
709				break
710			}
711		}
712	}
713
714	// no need to decode runes if all tabs are at the beginning of the line
715	if !slowcase {
716		for i = 0; i < prefix*tabSize; i++ {
717			out.WriteByte(' ')
718		}
719		out.Write(line[prefix:])
720		return
721	}
722
723	// the slow case: we need to count runes to figure out how
724	// many spaces to insert for each tab
725	column := 0
726	i = 0
727	for i < len(line) {
728		start := i
729		for i < len(line) && line[i] != '\t' {
730			_, size := utf8.DecodeRune(line[i:])
731			i += size
732			column++
733		}
734
735		if i > start {
736			out.Write(line[start:i])
737		}
738
739		if i >= len(line) {
740			break
741		}
742
743		for {
744			out.WriteByte(' ')
745			column++
746			if column%tabSize == 0 {
747				break
748			}
749		}
750
751		i++
752	}
753}
754
755// Find if a line counts as indented or not.
756// Returns number of characters the indent is (0 = not indented).
757func isIndented(data []byte, indentSize int) int {
758	if len(data) == 0 {
759		return 0
760	}
761	if data[0] == '\t' {
762		return 1
763	}
764	if len(data) < indentSize {
765		return 0
766	}
767	for i := 0; i < indentSize; i++ {
768		if data[i] != ' ' {
769			return 0
770		}
771	}
772	return indentSize
773}
774
775// Create a url-safe slug for fragments
776func slugify(in []byte) []byte {
777	if len(in) == 0 {
778		return in
779	}
780	out := make([]byte, 0, len(in))
781	sym := false
782
783	for _, ch := range in {
784		if isAlnum(ch) {
785			sym = false
786			out = append(out, ch)
787		} else if sym {
788			continue
789		} else {
790			out = append(out, '-')
791			sym = true
792		}
793	}
794	var a, b int
795	var ch byte
796	for a, ch = range out {
797		if ch != '-' {
798			break
799		}
800	}
801	for b = len(out) - 1; b > 0; b-- {
802		if out[b] != '-' {
803			break
804		}
805	}
806	return out[a : b+1]
807}
808
809func isListItem(d ast.Node) bool {
810	_, ok := d.(*ast.ListItem)
811	return ok
812}