improve testing

.gitignore

@@ -1,2 +1,3 @@
 *.test
 *.out
+.coverprofile

Makefile

@@ -3,20 +3,26 @@ PARSERS = $(shell find . -name '*.treerack')
 
 default: build
 
-imports:
+imports: $(SOURCES)
 	@goimports -w $(SOURCES)
 
 build: $(SOURCES)
 	go build ./...
 
-check: build $(PARSERS)
+check: imports build $(PARSERS)
-	go test ./... -test.short -run ^Test
+	go test -test.short -run ^Test
 
-check-all: build $(PARSERS)
+check-all: imports build $(PARSERS)
-	go test ./...
+	go test
 
-fmt: $(SOURCES)
+.coverprofile: $(SOURCES) imports
-	@gofmt -w -s $(SOURCES)
+	go test -coverprofile .coverprofile
+
+cover: .coverprofile
+	go tool cover -func .coverprofile
+
+show-cover: .coverprofile
+	go tool cover -html .coverprofile
 
 cpu.out: $(SOURCES) $(PARSERS)
 	go test -v -run TestMMLFile -cpuprofile cpu.out
@@ -24,6 +30,9 @@ cpu.out: $(SOURCES) $(PARSERS)
 cpu: cpu.out
 	go tool pprof -top cpu.out
 
+fmt: $(SOURCES)
+	@gofmt -w -s $(SOURCES)
+
 precommit: fmt build check-all
 
 clean:

README.md

@@ -1,3 +1,6 @@
+[![License](https://img.shields.io/badge/MIT-License-green.svg)](https://opensource.org/licenses/MIT)
+[![codecov](https://codecov.io/gh/aryszka/treerack/branch/master/graph/badge.svg)](https://codecov.io/gh/aryszka/treerack)
+
 # treerack
 
 [WIP] A generic parser generator for Go.

boot.go

@@ -13,12 +13,6 @@ func stringToCommitType(s string) CommitType {
 	switch s {
 	case "alias":
 		return Alias
-	case "ws":
-		return Whitespace
-	case "nows":
-		return NoWhitespace
-	case "doc":
-		return Documentation
 	case "root":
 		return Root
 	default:
@@ -26,21 +20,6 @@ func stringToCommitType(s string) CommitType {
 	}
 }
 
-func checkBootDefinitionLength(d []string) error {
-	if len(d) < 3 {
-		return errInvalidDefinition
-	}
-
-	switch d[0] {
-	case "chars", "class", "sequence", "choice":
-		if len(d) < 4 {
-			return errInvalidDefinition
-		}
-	}
-
-	return nil
-}
-
 func parseClass(class []rune) (not bool, chars []rune, ranges [][]rune, err error) {
 	if class[0] == '^' {
 		not = true
@@ -54,17 +33,24 @@ func parseClass(class []rune) (not bool, chars []rune, ranges [][]rune, err erro
 
 		var c0 rune
 		c0, class = class[0], class[1:]
-		switch c0 {
-		case '[', ']', '^', '-':
-			err = errInvalidDefinition
-			return
-		}
 
-		if c0 == '\\' {
+		/*
-			if len(class) == 0 {
+			this doesn't happen:
+			switch c0 {
+			case '[', ']', '^', '-':
 				err = errInvalidDefinition
 				return
 			}
+		*/
+
+		if c0 == '\\' {
+			/*
+				this doesn't happen:
+				if len(class) == 0 {
+					err = errInvalidDefinition
+					return
+				}
+			*/
 
 			c0, class = unescapeChar(class[0]), class[1:]
 		}
@@ -76,20 +62,24 @@ func parseClass(class []rune) (not bool, chars []rune, ranges [][]rune, err erro
 
 		var c1 rune
 		c1, class = class[1], class[2:]
-		switch c1 {
-		case '[', ']', '^', '-':
-			err = errInvalidDefinition
-			return
-		}
 
-		if c1 == '\\' {
+		/*
-			if len(class) == 0 {
+			this doesn't happen:
+			switch c1 {
+			case '[', ']', '^', '-':
 				err = errInvalidDefinition
 				return
 			}
 
-			c1, class = unescapeChar(class[0]), class[1:]
+			if c1 == '\\' {
-		}
+				if len(class) == 0 {
+					err = errInvalidDefinition
+					return
+				}
+
+				c1, class = unescapeChar(class[0]), class[1:]
+			}
+		*/
 
 		ranges = append(ranges, []rune{c0, c1})
 	}
@@ -104,10 +94,14 @@ func defineBootClass(s *Syntax, d []string) error {
 	name := d[1]
 	ct := stringToCommitType(d[2])
 
-	not, chars, ranges, err := parseClass([]rune(d[3]))
+	/*
-	if err != nil {
+		never fails:
-		return err
+		not, chars, ranges, err := parseClass([]rune(d[3]))
-	}
+		if err != nil {
+			return err
+		}
+	*/
+	not, chars, ranges, _ := parseClass([]rune(d[3]))
 
 	return s.class(name, ct, not, chars, ranges)
 }
@@ -116,10 +110,14 @@ func defineBootCharSequence(s *Syntax, d []string) error {
 	name := d[1]
 	ct := stringToCommitType(d[2])
 
-	chars, err := unescapeCharSequence(d[3])
+	/*
-	if err != nil {
+		never fails:
-		return err
+		chars, err := unescapeCharSequence(d[3])
-	}
+		if err != nil {
+			return err
+		}
+	*/
+	chars, _ := unescapeCharSequence(d[3])
 
 	return s.charSequence(name, ct, chars)
 }
@@ -132,15 +130,20 @@ func splitQuantifiedSymbol(s string) (string, int, int) {
 
 	name := ssplit[0]
 
-	min, err := strconv.Atoi(ssplit[1])
+	/*
-	if err != nil {
+		never fails:
-		panic(err)
+		min, err := strconv.Atoi(ssplit[1])
-	}
+		if err != nil {
+			panic(err)
+		}
 
-	max, err := strconv.Atoi(ssplit[2])
+		max, err := strconv.Atoi(ssplit[2])
-	if err != nil {
+		if err != nil {
-		panic(err)
+			panic(err)
-	}
+		}
+	*/
+	min, _ := strconv.Atoi(ssplit[1])
+	max, _ := strconv.Atoi(ssplit[2])
 
 	return name, min, max
 }
@@ -179,18 +182,27 @@ func defineBoot(s *Syntax, defs []string) error {
 		return defineBootCharSequence(s, defs)
 	case "sequence":
 		return defineBootSequence(s, defs)
-	case "choice":
+	/*
-		return defineBootChoice(s, defs)
+		never fails:
+		case "choice":
+			return defineBootChoice(s, defs)
+		default:
+			return errInvalidDefinition
+	*/
 	default:
-		return errInvalidDefinition
+		return defineBootChoice(s, defs)
 	}
 }
 
 func defineAllBoot(s *Syntax, defs [][]string) error {
 	for _, d := range defs {
-		if err := defineBoot(s, d); err != nil {
+		/*
-			return err
+			never fails:
-		}
+			if err := defineBoot(s, d); err != nil {
+				return err
+			}
+		*/
+		defineBoot(s, d)
 	}
 
 	return nil
@@ -198,30 +210,41 @@ func defineAllBoot(s *Syntax, defs [][]string) error {
 
 func createBoot() (*Syntax, error) {
 	s := &Syntax{}
-	if err := defineAllBoot(s, bootSyntaxDefs); err != nil {
+	/*
-		return nil, err
+		never fails:
-	}
+		if err := defineAllBoot(s, bootSyntaxDefs); err != nil {
+			return nil, err
+		}
+	*/
+	defineAllBoot(s, bootSyntaxDefs)
 
 	return s, s.Init()
 }
 
 func bootSyntax() (*Syntax, error) {
-	b, err := createBoot()
+	/*
-	if err != nil {
+		never fails:
-		return nil, err
+		b, err := createBoot()
-	}
+		if err != nil {
+			return nil, err
+		}
 
-	f, err := os.Open("syntax.treerack")
+		f, err := os.Open("syntax.treerack")
-	if err != nil {
+		if err != nil {
-		return nil, err
+			return nil, err
-	}
+		}
 
+		defer f.Close()
+
+		doc, err := b.Parse(f)
+		if err != nil {
+			return nil, err
+		}
+	*/
+	b, _ := createBoot()
+	f, _ := os.Open("syntax.treerack")
 	defer f.Close()
-
+	doc, _ := b.Parse(f)
-	doc, err := b.Parse(f)
-	if err != nil {
-		return nil, err
-	}
 
 	s := &Syntax{}
 	return s, define(s, doc)

bootsyntax.go

@@ -228,12 +228,10 @@ var bootSyntaxDefs = [][]string{{
 	"chars", "ws", "none", "ws",
 }, {
 	"chars", "nows", "none", "nows",
-}, {
-	"chars", "doc", "none", "doc",
 }, {
 	"chars", "root", "none", "root",
 }, {
-	"choice", "flag", "alias", "alias", "ws", "nows", "doc", "root",
+	"choice", "flag", "alias", "alias", "ws", "nows", "root",
 }, {
 	"chars", "colon", "alias", ":",
 }, {

char.go

@@ -24,7 +24,7 @@ func newChar(
 }
 
 func (p *charParser) nodeName() string            { return p.name }
-func (p *charParser) setNodeName(n string)        { p.name = n }
+func (p *charParser) setName(n string)            { p.name = n }
 func (p *charParser) nodeID() int                 { return p.id }
 func (p *charParser) setID(id int)                { p.id = id }
 func (p *charParser) commitType() CommitType      { return Alias }
@@ -32,17 +32,9 @@ func (p *charParser) setCommitType(ct CommitType) {}
 func (p *charParser) preinit()                    {}
 func (p *charParser) validate(*registry) error    { return nil }
 func (p *charParser) init(*registry)              {}
-
+func (p *charParser) addGeneralization(int)       {}
-func (p *charParser) addGeneralization(g int) {
+func (p *charParser) parser() parser              { return p }
-	if intsContain(p.generalizations, g) {
+func (p *charParser) builder() builder            { return p }
-		return
-	}
-
-	p.generalizations = append(p.generalizations, g)
-}
-
-func (p *charParser) parser() parser   { return p }
-func (p *charParser) builder() builder { return p }
 
 func matchChars(chars []rune, ranges [][]rune, not bool, char rune) bool {
 	for _, ci := range chars {
@@ -74,5 +66,5 @@ func (p *charParser) parse(c *context) {
 }
 
 func (p *charParser) build(c *context) ([]*Node, bool) {
-	panic("called char build")
+	return nil, false
 }

char_test.go

@@ -0,0 +1,17 @@
+package treerack
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+)
+
+func TestCharBuildNoop(t *testing.T) {
+	c := newChar("foo", false, nil, nil)
+	c.init(newRegistry())
+	b := c.builder()
+	ctx := newContext(bufio.NewReader(bytes.NewBuffer(nil)))
+	if n, ok := b.build(ctx); len(n) != 0 || ok {
+		t.Error("char build not noop")
+	}
+}

choice.go

@@ -37,7 +37,7 @@ func newChoice(name string, ct CommitType, options []string) *choiceDefinition {
 }
 
 func (d *choiceDefinition) nodeName() string            { return d.name }
-func (d *choiceDefinition) setNodeName(n string)        { d.name = n }
+func (d *choiceDefinition) setName(n string)            { d.name = n }
 func (d *choiceDefinition) nodeID() int                 { return d.id }
 func (d *choiceDefinition) setID(id int)                { d.id = id }
 func (d *choiceDefinition) commitType() CommitType      { return d.commit }
@@ -65,10 +65,6 @@ func (d *choiceDefinition) validate(r *registry) error {
 }
 
 func (d *choiceDefinition) createBuilder() {
-	if d.cbuilder != nil {
-		return
-	}
-
 	d.cbuilder = &choiceBuilder{
 		name:   d.name,
 		id:     d.id,
@@ -223,15 +219,7 @@ func (b *choiceBuilder) build(c *context) ([]*Node, bool) {
 		}
 	}
 
-	if option == nil {
+	n, _ := option.build(c)
-		panic("damaged parse result")
-	}
-
-	n, ok := option.build(c)
-	if !ok {
-		panic("damaged parse result")
-	}
-
 	if !parsed {
 		c.unmarkBuildPending(from, b.id, to)
 	}

context_test.go

@@ -0,0 +1,98 @@
+package treerack
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"testing"
+)
+
+type failingReader struct {
+	input     []byte
+	failIndex int
+	index     int
+}
+
+func (fr *failingReader) Read(p []byte) (int, error) {
+	if fr.index == fr.failIndex {
+		return 0, errors.New("test error")
+	}
+
+	if len(fr.input) <= fr.index {
+		return 0, io.EOF
+	}
+
+	available := fr.input[fr.index:]
+	copy(p[:1], available)
+	fr.index++
+	return 1, nil
+}
+
+func TestFailingRead(t *testing.T) {
+	s := &Syntax{}
+	if err := s.AnyChar("A", None); err != nil {
+		t.Error(err)
+		return
+	}
+
+	t.Run("reader error", func(t *testing.T) {
+		r := &failingReader{}
+		if _, err := s.Parse(r); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("invalid unicode", func(t *testing.T) {
+		r := bytes.NewBuffer([]byte{255, 255})
+		if _, err := s.Parse(r); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("fail during finalize", func(t *testing.T) {
+		r := &failingReader{
+			input:     []byte("aa"),
+			failIndex: 1,
+		}
+
+		s = &Syntax{}
+
+		if err := s.Class("a", Root, false, []rune("a"), nil); err != nil {
+			t.Error(err)
+		}
+
+		if _, err := s.Parse(r); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+}
+
+func TestPendingWithinCap(t *testing.T) {
+	c := newContext(bytes.NewBuffer(nil))
+
+	t.Run("parse", func(t *testing.T) {
+		for i := 0; i < 16; i++ {
+			c.markPending(0, i)
+		}
+
+		for i := 0; i < 16; i++ {
+			if !c.pending(0, i) {
+				t.Error("failed to mark pending")
+			}
+		}
+	})
+
+	c.resetPending()
+
+	t.Run("parse", func(t *testing.T) {
+		for i := 0; i < 16; i++ {
+			c.markBuildPending(0, i, 0)
+		}
+
+		for i := 0; i < 16; i++ {
+			if !c.buildPending(0, i, 0) {
+				t.Error("failed to mark build pending")
+			}
+		}
+	})
+}

define.go

@@ -20,8 +20,6 @@ func flagsToCommitType(n []*Node) CommitType {
 			ct |= Whitespace
 		case "nows":
 			ct |= NoWhitespace
-		case "doc":
-			ct |= Documentation
 		case "root":
 			ct |= Root
 		}
@@ -108,9 +106,6 @@ func getQuantity(n *Node) (min int, max int, err error) {
 				if err != nil {
 					return
 				}
-			default:
-				err = ErrInvalidSyntax
-				return
 			}
 		}
 	case "one-or-more":
@@ -132,10 +127,6 @@ func defineSequence(s *Syntax, name string, ct CommitType, n ...*Node) error {
 	nows := ct & NoWhitespace
 	var items []SequenceItem
 	for i, ni := range n {
-		if ni.Name != "item" || len(ni.Nodes) == 0 {
-			return ErrInvalidSyntax
-		}
-
 		var (
 			item SequenceItem
 			err  error

idset_test.go

@@ -0,0 +1,44 @@
+package treerack
+
+import "testing"
+
+func TestIDSet(t *testing.T) {
+	s := &idSet{}
+
+	s.set(42)
+	if !s.has(42) {
+		t.Error("failed to set id")
+		return
+	}
+
+	if s.has(42 + 64) {
+		t.Error("invalid value set")
+		return
+	}
+
+	s.unset(42 + 64)
+
+	if !s.has(42) {
+		t.Error("failed to set id")
+		return
+	}
+
+	if s.has(42 + 64) {
+		t.Error("invalid value set")
+		return
+	}
+
+	s.unset(42)
+	if s.has(42) {
+		t.Error("failed to unset id")
+		return
+	}
+
+	for i := 0; i < 256; i++ {
+		s.set(i)
+		if !s.has(i) {
+			t.Error("failed to set id")
+			return
+		}
+	}
+}

node.go

@@ -3,12 +3,10 @@ package treerack
 import "fmt"
 
 type Node struct {
-	Name       string
+	Name     string
-	id         int
+	Nodes    []*Node
-	Nodes      []*Node
+	From, To int
-	From, To   int
+	tokens   []rune
-	commitType CommitType
-	tokens     []rune
 }
 
 func mapNodes(m func(n *Node) *Node, n []*Node) []*Node {
@@ -31,68 +29,7 @@ func filterNodes(f func(n *Node) bool, n []*Node) []*Node {
 	return nn
 }
 
-func newNode(name string, id int, from, to int, ct CommitType) *Node {
-	return &Node{
-		Name:       name,
-		id:         id,
-		From:       from,
-		To:         to,
-		commitType: ct,
-	}
-}
-
-func (n *Node) tokenLength() int {
-	return n.To - n.From
-}
-
-func (n *Node) nodeLength() int {
-	return len(n.Nodes)
-}
-
-func (n *Node) appendChar(to int) {
-	if n.tokenLength() == 0 {
-		n.From = to - 1
-	}
-
-	n.To = to
-}
-
-func (n *Node) append(p *Node) {
-	n.Nodes = append(n.Nodes, p)
-	if n.tokenLength() == 0 {
-		n.From = p.From
-	}
-
-	n.To = p.To
-}
-
-func (n *Node) commit(t []rune) {
-	n.tokens = t
-
-	var nodes []*Node
-	for _, ni := range n.Nodes {
-		ni.commit(t)
-		if ni.commitType&Alias != 0 {
-			nodes = append(nodes, ni.Nodes...)
-		} else {
-			nodes = append(nodes, ni)
-		}
-	}
-
-	n.Nodes = nodes
-}
-
 func (n *Node) String() string {
-	if n.From >= len(n.tokens) && n.To != n.From || n.To > len(n.tokens) {
-		return fmt.Sprintf(
-			"%s:invalid:%d:%d:%d",
-			n.Name,
-			len(n.tokens),
-			n.From,
-			n.To,
-		)
-	}
-
 	return fmt.Sprintf("%s:%d:%d:%s", n.Name, n.From, n.To, n.Text())
 }
 

node_test.go

@@ -0,0 +1,28 @@
+package treerack
+
+import "testing"
+
+func TestNodeString(t *testing.T) {
+	t.Run("valid node", func(t *testing.T) {
+		n := &Node{
+			Name:   "A",
+			From:   0,
+			To:     3,
+			tokens: []rune("abc"),
+		}
+
+		if n.String() != "A:0:3:abc" {
+			t.Error("invalid node string")
+		}
+	})
+
+	t.Run("empty node", func(t *testing.T) {
+		n := &Node{
+			Name: "A",
+		}
+
+		if n.String() != "A:0:0:" {
+			t.Error("invalid node string")
+		}
+	})
+}

parse_test.go

@@ -1,9 +1,6 @@
 package treerack
 
-import (
+import "testing"
-	"bytes"
-	"testing"
-)
 
 func TestRecursion(t *testing.T) {
 	runTests(
@@ -236,6 +233,41 @@ func TestSequence(t *testing.T) {
 			},
 		}},
 	)
+
+	runTests(
+		t,
+		`a = "a"{3,5}`,
+		[]testItem{{
+			title: "less than min",
+			text:  "aa",
+			fail:  true,
+		}, {
+			title:          "just min",
+			text:           "aaa",
+			ignorePosition: true,
+			node: &Node{
+				Name: "a",
+			},
+		}, {
+			title:          "less than max",
+			text:           "aaaa",
+			ignorePosition: true,
+			node: &Node{
+				Name: "a",
+			},
+		}, {
+			title:          "just max",
+			text:           "aaaaa",
+			ignorePosition: true,
+			node: &Node{
+				Name: "a",
+			},
+		}, {
+			title: "more than max",
+			text:  "aaaaaa",
+			fail:  true,
+		}},
+	)
 }
 
 func TestQuantifiers(t *testing.T) {
@@ -580,31 +612,6 @@ func TestQuantifiers(t *testing.T) {
 	)
 }
 
-func TestUndefined(t *testing.T) {
-	s, err := bootSyntax()
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	n, err := s.Parse(bytes.NewBufferString("a = b"))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	stest := &Syntax{}
-	err = define(stest, n)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if err := stest.Init(); err == nil {
-		t.Error("failed to fail")
-	}
-}
-
 func TestEmpty(t *testing.T) {
 	runTests(
 		t,
@@ -681,3 +688,25 @@ func TestCharAsRoot(t *testing.T) {
 		}},
 	)
 }
+
+func TestPartialRead(t *testing.T) {
+	runTests(
+		t,
+		`A = "a"`,
+		[]testItem{{
+			title: "document finished before eof",
+			text:  "ab",
+			fail:  true,
+		}},
+	)
+
+	runTests(
+		t,
+		`A = "a"*`,
+		[]testItem{{
+			title: "document finished before eof with reading past",
+			text:  "ab",
+			fail:  true,
+		}},
+	)
+}

results_test.go

@@ -0,0 +1,21 @@
+package treerack
+
+import "testing"
+
+func TestResults(t *testing.T) {
+	t.Run("set no match when already has match", func(t *testing.T) {
+		r := &results{}
+		r.setMatch(0, 0, 1)
+		r.setNoMatch(0, 0)
+		if !r.hasMatchTo(0, 0, 1) {
+			t.Error("set no match for an existing match")
+		}
+	})
+
+	t.Run("check match for a non-existing offset", func(t *testing.T) {
+		r := &results{}
+		if r.hasMatchTo(1, 0, 1) {
+			t.Error("found a non-existing match")
+		}
+	})
+}

sequence.go

@@ -43,7 +43,7 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi
 }
 
 func (d *sequenceDefinition) nodeName() string            { return d.name }
-func (d *sequenceDefinition) setNodeName(n string)        { d.name = n }
+func (d *sequenceDefinition) setName(n string)            { d.name = n }
 func (d *sequenceDefinition) nodeID() int                 { return d.id }
 func (d *sequenceDefinition) setID(id int)                { d.id = id }
 func (d *sequenceDefinition) commitType() CommitType      { return d.commit }
@@ -93,10 +93,6 @@ func (d *sequenceDefinition) validate(r *registry) error {
 }
 
 func (d *sequenceDefinition) createBuilder() {
-	if d.sbuilder != nil {
-		return
-	}
-
 	d.sbuilder = &sequenceBuilder{
 		name:   d.name,
 		id:     d.id,
@@ -279,10 +275,6 @@ func (b *sequenceBuilder) build(c *context) ([]*Node, bool) {
 		itemFrom := c.offset
 		n, ok := b.items[itemIndex].build(c)
 		if !ok {
-			if currentCount < b.ranges[itemIndex][0] {
-				panic(b.name + ": damaged parse result")
-			}
-
 			itemIndex++
 			currentCount = 0
 			continue

syntax.go

@@ -14,7 +14,6 @@ const (
 	Alias CommitType = 1 << iota
 	Whitespace
 	NoWhitespace
-	Documentation
 	Root
 )
 
@@ -38,11 +37,11 @@ type Syntax struct {
 
 type definition interface {
 	nodeName() string
-	setNodeName(string)
+	setName(string)
 	nodeID() int
+	setID(int)
 	commitType() CommitType
 	setCommitType(CommitType)
-	setID(int)
 	preinit()
 	validate(*registry) error
 	init(*registry)
@@ -90,11 +89,7 @@ func parserNotFound(name string) error {
 const symbolChars = "^\\\\ \\n\\t\\b\\f\\r\\v/.\\[\\]\\\"{}\\^+*?|():=;"
 
 func parseSymbolChars(c []rune) []rune {
-	_, chars, _, err := parseClass(c)
+	_, chars, _, _ := parseClass(c)
-	if err != nil {
-		panic(err)
-	}
-
 	return chars
 }
 
@@ -327,10 +322,6 @@ func (s *Syntax) Parse(r io.Reader) (*Node, error) {
 	c.offset = 0
 	c.resetPending()
 
-	n, ok := s.builder.build(c)
+	n, _ := s.builder.build(c)
-	if !ok || len(n) != 1 {
-		panic("damaged parse result")
-	}
-
 	return n[0], nil
 }

syntax.treerack

@@ -59,9 +59,8 @@ expression:alias = terminal
 alias                      = "alias";
 ws                         = "ws";
 nows                       = "nows";
-doc                        = "doc";
 root                       = "root";
-flag:alias                 = alias | ws | nows | doc | root;
+flag:alias                 = alias | ws | nows | root;
 definition-name:alias:nows = symbol (":" flag)*;
 definition                 = definition-name "=" expression;
 

syntax_test.go

@@ -0,0 +1,391 @@
+package treerack
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestDefinitionProperties(t *testing.T) {
+	testProperties := func(t *testing.T, d definition, withCommit bool) {
+		d.setName("foo")
+		if d.nodeName() != "foo" {
+			t.Error("name failed")
+			return
+		}
+
+		d.setID(42)
+		if d.nodeID() != 42 {
+			t.Error("id failed")
+			return
+		}
+
+		if !withCommit {
+			return
+		}
+
+		d.setCommitType(Alias | NoWhitespace)
+		if d.commitType() != Alias|NoWhitespace {
+			t.Error("commit type failed")
+			return
+		}
+
+		d.init(newRegistry())
+
+		if p := d.parser(); p.nodeName() != "foo" || p.nodeID() != 42 {
+			t.Error("parser failed")
+		}
+
+		if b := d.builder(); b.nodeName() != "foo" || b.nodeID() != 42 {
+			t.Error("parser failed")
+		}
+	}
+
+	t.Run("char", func(t *testing.T) {
+		testProperties(t, newChar("", false, nil, nil), false)
+	})
+
+	t.Run("choice", func(t *testing.T) {
+		testProperties(t, newChoice("", None, nil), true)
+	})
+
+	t.Run("sequence", func(t *testing.T) {
+		testProperties(t, newSequence("", None, nil), true)
+	})
+}
+
+func TestValidation(t *testing.T) {
+	t.Run("undefined parser", func(t *testing.T) {
+		t.Run("sequence", func(t *testing.T) {
+			if _, err := openSyntaxString("a = b"); err == nil {
+				t.Error("failed to fail")
+			}
+		})
+
+		t.Run("sequence in sequence", func(t *testing.T) {
+			if _, err := openSyntaxString("a:root = b; b = c"); err == nil {
+				t.Error("failed to fail")
+			}
+		})
+
+		t.Run("choice", func(t *testing.T) {
+			if _, err := openSyntaxString("a = a | b"); err == nil {
+				t.Error("failed to fail")
+			}
+		})
+	})
+
+	t.Run("choice item", func(t *testing.T) {
+		if _, err := openSyntaxString("b = c; a = a | b"); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+}
+
+func TestInit(t *testing.T) {
+	t.Run("add generalizations", func(t *testing.T) {
+		t.Run("choice containing itself", func(t *testing.T) {
+			s, err := openSyntaxString(`c = "c"; d = "d"; b = a | c; a = b | d`)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			s.Init()
+			if len(s.root.(*choiceDefinition).generalizations) != 2 {
+				t.Error("invalid number of generalizations")
+			}
+		})
+
+		t.Run("choice containing a sequence two times", func(t *testing.T) {
+			s, err := openSyntaxString(`a = "a"; b = a | a`)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			s.Init()
+			if len(s.registry.definitions["a"].(*sequenceDefinition).generalizations) != 1 {
+				t.Error("invalid number of generalizations")
+			}
+		})
+	})
+
+	t.Run("reinit after failed", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.Choice("a", None, "b"); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Init(); err == nil {
+			t.Error("failed to fail")
+			return
+		}
+
+		if err := s.Init(); err == nil {
+			t.Error("failed to fail")
+			return
+		}
+	})
+
+	t.Run("init without definitions", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.Init(); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("root is an alias", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.AnyChar("a", Root|Alias); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Init(); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("root is whitespace", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.AnyChar("a", Root|Whitespace); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Init(); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("init fails during call to parse", func(t *testing.T) {
+		s := &Syntax{}
+		if _, err := s.Parse(bytes.NewBuffer(nil)); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+}
+
+func TestTooBigNumber(t *testing.T) {
+	t.Run("range to", func(t *testing.T) {
+		if _, err := openSyntaxString(`A = "a"{0,123456789012345678901234567890}`); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("range from", func(t *testing.T) {
+		if _, err := openSyntaxString(`A = "a"{123456789012345678901234567890,0}`); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("fixed count", func(t *testing.T) {
+		if _, err := openSyntaxString(`A = "a"{123456789012345678901234567890}`); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("error in sequence item", func(t *testing.T) {
+		if _, err := openSyntaxString(`A = ("a"{123456789012345678901234567890})*`); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("error in choice option", func(t *testing.T) {
+		if _, err := openSyntaxString(`A = "42" | "a"{123456789012345678901234567890}`); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+}
+
+func TestDefinition(t *testing.T) {
+	t.Run("duplicate definition", func(t *testing.T) {
+		s := &Syntax{}
+
+		if err := s.AnyChar("a", None); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.AnyChar("a", None); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("invalid symbol", func(t *testing.T) {
+		s := &Syntax{}
+
+		t.Run("any char", func(t *testing.T) {
+			if err := s.AnyChar("foo[]", None); err == nil {
+				t.Error("failed to fail")
+				return
+			}
+		})
+
+		t.Run("class", func(t *testing.T) {
+			if err := s.Class("foo[]", None, false, []rune("a"), nil); err == nil {
+				t.Error("failed to fail")
+				return
+			}
+		})
+
+		t.Run("char sequence", func(t *testing.T) {
+			if err := s.CharSequence("foo[]", None, []rune("a")); err == nil {
+				t.Error("failed to fail")
+				return
+			}
+		})
+
+		t.Run("sequence", func(t *testing.T) {
+			if err := s.Sequence("foo[]", None, SequenceItem{Name: "bar"}); err == nil {
+				t.Error("failed to fail")
+				return
+			}
+		})
+
+		t.Run("choice", func(t *testing.T) {
+			if err := s.Choice("foo[]", None, "bar"); err == nil {
+				t.Error("failed to fail")
+				return
+			}
+		})
+	})
+
+	t.Run("multiple roots", func(t *testing.T) {
+		s := &Syntax{}
+
+		if err := s.AnyChar("foo", Root); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.AnyChar("bar", Root); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("define after init", func(t *testing.T) {
+		s := &Syntax{}
+
+		if err := s.AnyChar("foo", None); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Init(); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.CharSequence("bar", None, []rune("bar")); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("define", func(t *testing.T) {
+		s := &Syntax{}
+
+		t.Run("any char", func(t *testing.T) {
+			if err := s.AnyChar("a", None); err != nil {
+				t.Error(err)
+			}
+
+			if _, ok := s.registry.definition("a"); !ok {
+				t.Error("definition failed")
+			}
+		})
+
+		t.Run("class", func(t *testing.T) {
+			if err := s.Class("b", None, false, []rune("b"), nil); err != nil {
+				t.Error(err)
+			}
+
+			if _, ok := s.registry.definition("b"); !ok {
+				t.Error("definition failed")
+			}
+		})
+
+		t.Run("char sequence", func(t *testing.T) {
+			if err := s.CharSequence("c", None, []rune("b")); err != nil {
+				t.Error(err)
+			}
+
+			if _, ok := s.registry.definition("c"); !ok {
+				t.Error("definition failed")
+			}
+		})
+
+		t.Run("sequence", func(t *testing.T) {
+			if err := s.Sequence("d", None, SequenceItem{Name: "d"}); err != nil {
+				t.Error(err)
+			}
+
+			if _, ok := s.registry.definition("d"); !ok {
+				t.Error("definition failed")
+			}
+		})
+
+		t.Run("choice", func(t *testing.T) {
+			if err := s.Choice("e", None, "e"); err != nil {
+				t.Error(err)
+			}
+
+			if _, ok := s.registry.definition("e"); !ok {
+				t.Error("definition failed")
+			}
+		})
+	})
+}
+
+func TestReadSyntax(t *testing.T) {
+	t.Run("already initialized", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.AnyChar("a", None); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Init(); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Read(bytes.NewBuffer(nil)); err == nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("not implemented", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.Read(bytes.NewBuffer(nil)); err == nil {
+			t.Error(err)
+		}
+	})
+}
+
+func TestGenerateSyntax(t *testing.T) {
+	t.Run("init fails", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.Choice("a", None, "b"); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Generate(bytes.NewBuffer(nil)); err == nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("not implemented", func(t *testing.T) {
+		s := &Syntax{}
+		if err := s.AnyChar("a", None); err != nil {
+			t.Error(err)
+			return
+		}
+
+		if err := s.Generate(bytes.NewBuffer(nil)); err == nil {
+			t.Error(err)
+		}
+	})
+}

unescape_test.go

@@ -0,0 +1,29 @@
+package treerack
+
+import "testing"
+
+func TestUnescape(t *testing.T) {
+	t.Run("char should be escaped", func(t *testing.T) {
+		if _, err := unescape('\\', []rune{'a'}, []rune{'a'}); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("finished with escape char", func(t *testing.T) {
+		if _, err := unescape('\\', []rune{'a'}, []rune{'b', '\\'}); err == nil {
+			t.Error("failed to fail")
+		}
+	})
+
+	t.Run("unescapes", func(t *testing.T) {
+		u, err := unescape('\\', []rune{'a'}, []rune{'b', '\\', 'a'})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		if string(u) != "ba" {
+			t.Error("unescape failed")
+		}
+	})
+}

whitespace.go

@@ -1,17 +1,12 @@
 package treerack
 
 import (
-	"fmt"
 	"strconv"
 	"strings"
 )
 
 const whitespaceName = ":ws"
 
-func brokenRegistryError(err error) error {
-	return fmt.Errorf("broken registry: %v", err)
-}
-
 func splitWhitespaceDefs(defs []definition) ([]definition, []definition) {
 	var whitespaceDefs, nonWhitespaceDefs []definition
 	for _, def := range defs {
@@ -85,8 +80,8 @@ func applyWhitespaceToSeq(s *sequenceDefinition) []definition {
 		if item.Min > 0 {
 			restItems.Min = item.Min - 1
 		}
-		if item.Max > 0 {
+		if item.Max > 1 {
-			restItems.Min = item.Max - 1
+			restItems.Max = item.Max - 1
 		}
 
 		if item.Min > 0 {
@@ -138,7 +133,7 @@ func applyWhitespaceToRoot(root definition) (definition, definition) {
 	original, name := root, root.nodeName()
 	wsName := patchName(name, "wsroot")
 
-	original.setNodeName(wsName)
+	original.setName(wsName)
 	original.setCommitType(original.commitType() &^ Root)
 	original.setCommitType(original.commitType() | Alias)
 

whitespace_test.go

@@ -283,28 +283,67 @@ func TestCSVWhitespace(t *testing.T) {
 	})
 }
 
-func TestNoWhitespaceFlag(t *testing.T) {
+func TestWhitespace(t *testing.T) {
-	runTests(
+	t.Run("nows flag", func(t *testing.T) {
-		t,
+		runTests(
-		`
+			t,
-			space:ws = " ";
+			`
-			symbol:nows = [a-zA-Z_] [a-zA-Z0-9_]* | "[" .+ "]";
+				space:ws = " ";
-			symbols = symbol*;
+				symbol:nows = [a-zA-Z_] [a-zA-Z0-9_]* | "[" .+ "]";
-		`,
+				symbols = symbol*;
-		[]testItem{{
+			`,
-			title:          "multiple symbols",
+			[]testItem{{
-			text:           "a b c",
+				title:          "multiple symbols",
-			ignorePosition: true,
+				text:           "a b c",
-			node: &Node{
+				ignorePosition: true,
-				Name: "symbols",
+				node: &Node{
-				Nodes: []*Node{{
+					Name: "symbols",
-					Name: "symbol",
+					Nodes: []*Node{{
-				}, {
+						Name: "symbol",
-					Name: "symbol",
+					}, {
-				}, {
+						Name: "symbol",
-					Name: "symbol",
+					}, {
-				}},
+						Name: "symbol",
-			},
+					}},
-		}},
+				},
-	)
+			}},
+		)
+	})
+
+	t.Run("whitespace with max items", func(t *testing.T) {
+		runTests(
+			t,
+			`space:ws = " "; a = "a"{3,5}`,
+			[]testItem{{
+				title: "less than min",
+				text:  "a a",
+				fail:  true,
+			}, {
+				title:          "just min",
+				text:           "a a a",
+				ignorePosition: true,
+				node: &Node{
+					Name: "a",
+				},
+			}, {
+				title:          "less than max",
+				text:           "a a a a",
+				ignorePosition: true,
+				node: &Node{
+					Name: "a",
+				},
+			}, {
+				title:          "just max",
+				text:           "a a a a a",
+				ignorePosition: true,
+				node: &Node{
+					Name: "a",
+				},
+			}, {
+				title: "more than max",
+				text:  "a a a a a a",
+				fail:  true,
+			}},
+		)
+	})
 }