+l3
+l4`,
+ T1: `
+l1
+l2
+
+`,
+ T2: `
+l3
+l4`,
+ },
+ }
+
+ for _, cs := range cases {
+ t.Run(cs.Name, func(t *testing.T) {
+ s := []byte(cs.Source)
+ md := New()
+ n := md.Parser().Parse(text.NewReader(s))
+ c1 := n.FirstChild()
+ c2 := c1.NextSibling().NextSibling()
+ if cs.C {
+ c1 = c1.FirstChild()
+ c2 = c2.FirstChild()
+ }
+ if !bytes.Equal(c1.Text(s), []byte(cs.T1)) {
+ t.Errorf("%s unmatch: %s", cs.Name, testutil.DiffPretty(c1.Text(s), []byte(cs.T1)))
+ }
+ if !bytes.Equal(c2.Text(s), []byte(cs.T2)) {
+ t.Errorf("%s(EOF) unmatch: %s", cs.Name, testutil.DiffPretty(c2.Text(s), []byte(cs.T2)))
+ }
+ })
+ }
+
+}
+
+func TestASTInlineNodeText(t *testing.T) {
+ var cases = []struct {
+ Name string
+ Source string
+ T1 string
+ }{
+ {
+ Name: "CodeSpan",
+ Source: "`c1`",
+ T1: `c1`,
+ },
+ {
+ Name: "Emphasis",
+ Source: `*c1 **c2***`,
+ T1: `c1 c2`,
+ },
+ {
+ Name: "Link",
+ Source: `[label](url)`,
+ T1: `label`,
+ },
+ {
+ Name: "AutoLink",
+ Source: ``,
+ T1: `http://url`,
+ },
+ {
+ Name: "RawHTML",
+ Source: `c1`,
+ T1: ``,
+ },
+ }
+
+ for _, cs := range cases {
+ t.Run(cs.Name, func(t *testing.T) {
+ s := []byte(cs.Source)
+ md := New()
+ n := md.Parser().Parse(text.NewReader(s))
+ c1 := n.FirstChild().FirstChild()
+ if !bytes.Equal(c1.Text(s), []byte(cs.T1)) {
+ t.Errorf("%s unmatch:\n%s", cs.Name, testutil.DiffPretty(c1.Text(s), []byte(cs.T1)))
+ }
+ })
+ }
+
+}
diff --git a/extension/ast_test.go b/extension/ast_test.go
new file mode 100644
index 0000000..e0c24b1
--- /dev/null
+++ b/extension/ast_test.go
@@ -0,0 +1,117 @@
+package extension
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/yuin/goldmark"
+ "github.com/yuin/goldmark/renderer/html"
+ "github.com/yuin/goldmark/testutil"
+ "github.com/yuin/goldmark/text"
+)
+
+func TestASTBlockNodeText(t *testing.T) {
+ var cases = []struct {
+ Name string
+ Source string
+ T1 string
+ T2 string
+ C bool
+ }{
+ {
+ Name: "DefinitionList",
+ Source: `c1
+: c2
+ c3
+
+a
+
+c4
+: c5
+ c6`,
+ T1: `c1c2
+c3`,
+ T2: `c4c5
+c6`,
+ },
+ {
+ Name: "Table",
+ Source: `| h1 | h2 |
+| -- | -- |
+| c1 | c2 |
+
+a
+
+
+| h3 | h4 |
+| -- | -- |
+| c3 | c4 |`,
+
+ T1: `h1h2c1c2`,
+ T2: `h3h4c3c4`,
+ },
+ }
+
+ for _, cs := range cases {
+ t.Run(cs.Name, func(t *testing.T) {
+ s := []byte(cs.Source)
+ md := goldmark.New(
+ goldmark.WithRendererOptions(
+ html.WithUnsafe(),
+ ),
+ goldmark.WithExtensions(
+ DefinitionList,
+ Table,
+ ),
+ )
+ n := md.Parser().Parse(text.NewReader(s))
+ c1 := n.FirstChild()
+ c2 := c1.NextSibling().NextSibling()
+ if cs.C {
+ c1 = c1.FirstChild()
+ c2 = c2.FirstChild()
+ }
+ if !bytes.Equal(c1.Text(s), []byte(cs.T1)) {
+ t.Errorf("%s unmatch:\n%s", cs.Name, testutil.DiffPretty(c1.Text(s), []byte(cs.T1)))
+ }
+ if !bytes.Equal(c2.Text(s), []byte(cs.T2)) {
+ t.Errorf("%s(EOF) unmatch: %s", cs.Name, testutil.DiffPretty(c2.Text(s), []byte(cs.T2)))
+ }
+ })
+ }
+
+}
+
+func TestASTInlineNodeText(t *testing.T) {
+ var cases = []struct {
+ Name string
+ Source string
+ T1 string
+ }{
+ {
+ Name: "Strikethrough",
+ Source: `~c1 *c2*~`,
+ T1: `c1 c2`,
+ },
+ }
+
+ for _, cs := range cases {
+ t.Run(cs.Name, func(t *testing.T) {
+ s := []byte(cs.Source)
+ md := goldmark.New(
+ goldmark.WithRendererOptions(
+ html.WithUnsafe(),
+ ),
+ goldmark.WithExtensions(
+ Strikethrough,
+ ),
+ )
+ n := md.Parser().Parse(text.NewReader(s))
+ c1 := n.FirstChild().FirstChild()
+ if !bytes.Equal(c1.Text(s), []byte(cs.T1)) {
+ t.Errorf("%s unmatch:\n%s", cs.Name, testutil.DiffPretty(c1.Text(s), []byte(cs.T1)))
+ }
+ })
+ }
+
+}
diff --git a/parser/code_block.go b/parser/code_block.go
index 732f18c..d99146c 100644
--- a/parser/code_block.go
+++ b/parser/code_block.go
@@ -35,6 +35,7 @@ func (b *codeBlockParser) Open(parent ast.Node, reader text.Reader, pc Context)
if segment.Padding != 0 {
preserveLeadingTabInCodeBlock(&segment, reader, 0)
}
+ segment.ForceNewline = true
node.Lines().Append(segment)
reader.Advance(segment.Len() - 1)
return node, NoChildren
@@ -59,6 +60,7 @@ func (b *codeBlockParser) Continue(node ast.Node, reader text.Reader, pc Context
preserveLeadingTabInCodeBlock(&segment, reader, 0)
}
+ segment.ForceNewline = true
node.Lines().Append(segment)
reader.Advance(segment.Len() - 1)
return Continue | NoChildren
diff --git a/parser/fcode_block.go b/parser/fcode_block.go
index e51a35a..953b8dc 100644
--- a/parser/fcode_block.go
+++ b/parser/fcode_block.go
@@ -100,6 +100,7 @@ func (b *fencedCodeBlockParser) Continue(node ast.Node, reader text.Reader, pc C
if padding != 0 {
preserveLeadingTabInCodeBlock(&seg, reader, fdata.indent)
}
+ seg.ForceNewline = true // EOF as newline
node.Lines().Append(seg)
reader.AdvanceAndSetPadding(segment.Stop-segment.Start-pos-1, padding)
return Continue | NoChildren
diff --git a/parser/parser.go b/parser/parser.go
index b59666c..b05db13 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -878,12 +878,6 @@ func (p *parser) Parse(reader text.Reader, opts ...ParseOption) ast.Node {
blockReader := text.NewBlockReader(reader.Source(), nil)
p.walkBlock(root, func(node ast.Node) {
p.parseBlock(blockReader, node, pc)
- lines := node.Lines()
- if lines != nil && lines.Len() != 0 {
- s := lines.At(lines.Len() - 1)
- s.EOB = true
- lines.Set(lines.Len()-1, s)
- }
})
for _, at := range p.astTransformers {
at.Transform(root, reader, pc)
diff --git a/text/segment.go b/text/segment.go
index 83c875b..93fbf19 100644
--- a/text/segment.go
+++ b/text/segment.go
@@ -20,8 +20,19 @@ type Segment struct {
// Padding is a padding length of the segment.
Padding int
- // EOB is true if the segment is end of the block.
- EOB bool
+ // ForceNewline is true if the segment should be ended with a newline.
+ // Some elements(i.e. CodeBlock, FencedCodeBlock) does not trim trailing
+ // newlines. Spec defines that EOF is treated as a newline, so we need to
+ // add a newline to the end of the segment if it is not empty.
+ //
+ // i.e.:
+ //
+ // ```go
+ // const test = "test"
+ //
+ // This code does not close the code block and ends with EOF. In this case,
+ // we need to add a newline to the end of the last line like `const test = "test"\n`.
+ ForceNewline bool
}
// NewSegment return a new Segment.
@@ -52,7 +63,7 @@ func (t *Segment) Value(buffer []byte) []byte {
result = append(result, bytes.Repeat(space, t.Padding)...)
result = append(result, buffer[t.Start:t.Stop]...)
}
- if t.EOB && len(result) > 0 && result[len(result)-1] != '\n' {
+ if t.ForceNewline && len(result) > 0 && result[len(result)-1] != '\n' {
result = append(result, '\n')
}
return result
@@ -217,3 +228,12 @@ func (s *Segments) Unshift(v Segment) {
s.values = append(s.values[0:1], s.values[0:]...)
s.values[0] = v
}
+
+// Value returns a string value of the collection.
+func (s *Segments) Value(buffer []byte) []byte {
+ var result []byte
+ for _, v := range s.values {
+ result = append(result, v.Value(buffer)...)
+ }
+ return result
+}