fix quantified sequence building

context.go

@@ -61,10 +61,6 @@ func (c *context) token() (rune, bool) {
 		}
 	}
 
-	if len(c.tokens) <= c.offset {
-		println(len(c.tokens), c.offset)
-	}
-
 	return c.tokens[c.offset], true
 }
 

node.go

@@ -64,7 +64,7 @@ func (n *Node) commit(t []rune) {
 
 func (n *Node) String() string {
 	if n.From >= len(n.tokens) || n.To > len(n.tokens) {
-		return n.Name + ":incomplete"
+		return n.Name + ":invalid"
 	}
 
 	return fmt.Sprintf("%s:%d:%d:%s", n.Name, n.From, n.To, n.Text())

notes.txt

@@ -1,3 +1,14 @@
+[whitespace]
+1. merge whitespaces
+2. set ws to alias
+3. apply whitespace to expressions
+- a a -> a ws* a
+- a | b -> a | b
+- a? -> a{0, 1} -> a{0, 1}
+- a+ -> a{1,} -> a (ws* a){,}
+- a* -> a{0,} -> (a (ws* a){,}){,}
+- root -> ws* root ws*
+
 error reporting
 - longest parse
 - count the lines

parse_test.go

@@ -1,6 +1,7 @@
 package treerack
 
 import (
+	"bytes"
 	"testing"
 )
 
@@ -159,7 +160,7 @@ func TestSequence(t *testing.T) {
 		t,
 		`A = "a" | (A?)*`,
 		[]testItem{{
-			title: "sequence in choice with redundant quantifier",
+			title: "recursive sequence in choice with redundant quantifier",
 			text:  "aaa",
 			node: &Node{
 				Name: "A",
@@ -189,6 +190,35 @@ func TestSequence(t *testing.T) {
 	)
 }
 
+func TestSequenceBug(t *testing.T) {
+	runTests(
+		t,
+		`A = "a" | A*`,
+		[]testItem{{
+			title: "BUG: recursive sequence in choice",
+			text:  "aaa",
+			node: &Node{
+				Name: "A",
+				Nodes: []*Node{{
+					Name: "A",
+				}, {
+					Name: "A",
+					Nodes: []*Node{{
+						Name: "A",
+					}, {
+						Name: "A",
+					}, {
+						Name: "A",
+					}},
+				}, {
+					Name: "A",
+				}},
+			},
+			ignorePosition: true,
+		}},
+	)
+}
+
 func TestQuantifiers(t *testing.T) {
 	runTests(
 		t,
@@ -530,3 +560,72 @@ func TestQuantifiers(t *testing.T) {
 		}},
 	)
 }
+
+func TestUndefined(t *testing.T) {
+	s, err := bootSyntax()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	n, err := s.Parse(bytes.NewBufferString("a = b"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	stest := NewSyntax()
+	err = define(stest, n)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if err := stest.Init(); err == nil {
+		t.Error("failed to fail")
+	}
+}
+
+func TestEmpty(t *testing.T) {
+	runTests(
+		t,
+		`A = "1"`,
+		[]testItem{{
+			title: "empty primitive, fail",
+			fail:  true,
+		}},
+	)
+
+	runTests(
+		t,
+		`A = "1"?`,
+		[]testItem{{
+			title: "empty primitive, succeed",
+		}},
+	)
+
+	runTests(
+		t,
+		`a = "1"?; A = a a`,
+		[]testItem{{
+			title: "empty document with quantifiers in the item",
+			node: &Node{
+				Name: "A",
+				Nodes: []*Node{{
+					Name: "a",
+				}, {
+					Name: "a",
+				}},
+			},
+		}},
+	)
+
+	runTests(
+		t,
+		`a = "1"; A = a? a?`,
+		[]testItem{{
+			title: "empty document with quantifiers in the reference",
+			node: &Node{
+				Name: "A",
+			},
+		}},
+	)
+}

sequence.go

@@ -299,6 +299,11 @@ func (b *sequenceBuilder) build(c *context) ([]*Node, bool) {
 		return nil, false
 	}
 
+	// maybe something like this:
+	if to-c.offset == 0 && b.commit&Alias != 0 {
+		return nil, true
+	}
+
 	if b.allChars {
 		from := c.offset
 		c.offset = to
@@ -334,8 +339,10 @@ func (b *sequenceBuilder) build(c *context) ([]*Node, bool) {
 			continue
 		}
 
+		// maybe can handle the commit type differently
+
 		parsed := c.offset > itemFrom
-		if parsed {
+		if parsed || len(n) > 0 {
 			nodes = append(nodes, n...)
 			currentCount++
 		}

store.go

@@ -71,14 +71,15 @@ func (s *store) takeMatch(offset, id int, includedBy *idSet) (int, bool) {
 			continue
 		}
 
-		found = true
-		if s.match[offset][i+1] > to {
+		if s.match[offset][i+1] > to || !found {
 			to = s.match[offset][i+1]
 			index = i
 		}
+
+		found = true
 	}
 
-	if found {
+	if found && to-offset > 0 {
 		s.match[offset][index] = -1
 		for i := 0; i < len(s.match[offset]); i += 2 {
 			if includedBy.has(s.match[offset][i]) && s.match[offset][i+1] == to {