Merge branch 'separate-build-phase'

boot_test.go

@@ -3,6 +3,7 @@ package treerack
 import (
 	"os"
 	"testing"
+	"time"
 )
 
 func TestBoot(t *testing.T) {
@@ -20,11 +21,26 @@ func TestBoot(t *testing.T) {
 
 	defer f.Close()
 
-	n0, err := b.Parse(f)
+	var d time.Duration
+	const n = 120
+	var n0 *Node
+	for i := 0; i < n; i++ {
+		if _, err := f.Seek(0, 0); err != nil {
+			t.Error(err)
+			return
+		}
+
+		start := time.Now()
+		n0, err = b.Parse(f)
+		d += time.Now().Sub(start)
+
 		if err != nil {
 			t.Error(err)
 			return
 		}
+	}
+
+	t.Log("duration:", d/n)
 
 	s0 := NewSyntax()
 	if err := define(s0, n0); err != nil {

char.go

@@ -3,23 +3,20 @@ package treerack
 type charParser struct {
 	name       string
 	id         int
-	commit     CommitType
 	not        bool
 	chars      []rune
 	ranges     [][]rune
-	includedBy []parser
+	includedBy []int
 }
 
 func newChar(
 	name string,
-	ct CommitType,
 	not bool,
 	chars []rune,
 	ranges [][]rune,
 ) *charParser {
 	return &charParser{
 		name:   name,
-		commit: ct,
 		not:    not,
 		chars:  chars,
 		ranges: ranges,
@@ -29,6 +26,14 @@ func newChar(
 func (p *charParser) nodeName() string       { return p.name }
 func (p *charParser) nodeID() int            { return p.id }
 func (p *charParser) setID(id int)           { p.id = id }
+func (p *charParser) commitType() CommitType { return Alias }
+
+func (p *charParser) init(r *registry) error { return nil }
+
+func (p *charParser) setIncludedBy(r *registry, includedBy int, parsers *idSet) error {
+	p.includedBy = appendIfMissing(p.includedBy, includedBy)
+	return nil
+}
 
 func (p *charParser) parser(r *registry, parsers *idSet) (parser, error) {
 	if parsers.has(p.id) {
@@ -43,20 +48,8 @@ func (p *charParser) parser(r *registry, parsers *idSet) (parser, error) {
 	return p, nil
 }
 
-func (p *charParser) commitType() CommitType {
-	return p.commit
-}
-
-func (p *charParser) setIncludedBy(includedBy parser, parsers *idSet) {
-	// if parsers.has(p.id) {
-	// 	panic(cannotIncludeParsers(p.name))
-	// }
-
-	p.includedBy = append(p.includedBy, includedBy)
-}
-
-func (p *charParser) storeIncluded(*context, *Node) {
-	panic(cannotIncludeParsers(p.name))
+func (p *charParser) builder() builder {
+	return p
 }
 
 func (p *charParser) match(t rune) bool {
@@ -77,33 +70,32 @@ func (p *charParser) match(t rune) bool {
 
 func (p *charParser) parse(t Trace, c *context) {
 	// t = t.Extend(p.name)
-	// t.Out1("parsing char", c.offset)
+	// t.Out1("parsing", c.offset)
 
-	// if p.commit&Documentation != 0 {
-	// 	// t.Out1("fail, doc")
-	// 	c.fail(c.offset)
-	// 	return
-	// }
-
-	// if _, ok := c.fromStore(p.id); ok {
-	// 	// t.Out1("found in store, match:", m)
-	// 	return
-	// }
-
-	if tok, ok := c.token(); ok && p.match(tok) {
-		// t.Out1("success", string(tok))
-		// n := newNode(p.name, p.id, c.offset, c.offset+1, p.commit)
-		// c.store.set(c.offset, p.id, n)
-		// for _, includedBy := range p.includedBy {
-		// 	includedBy.storeIncluded(c, n)
-		// }
-
-		c.successChar()
-		return
-	} else {
-		// t.Out1("fail", string(tok))
-		// c.store.set(c.offset, p.id, nil)
+	if tok, ok := c.token(); !ok || !p.match(tok) {
+		// t.Out1("fail")
 		c.fail(c.offset)
 		return
 	}
+
+	// t.Out1("success")
+	c.success(c.offset + 1)
+	for _, includedBy := range p.includedBy {
+		c.store.setMatch(c.offset, includedBy, c.offset+1)
+	}
+}
+
+func (p *charParser) build(c *context) ([]*Node, bool) {
+	t, ok := c.token()
+	if !ok {
+		panic("damaged parser context")
+	}
+
+	if !p.match(t) {
+		return nil, false
+	}
+
+	// always alias
+	c.offset++
+	return nil, true
 }

choice.go

@@ -5,6 +5,8 @@ type choiceDefinition struct {
 	id         int
 	commit     CommitType
 	elements   []string
+	includedBy []int
+	cbuilder   *choiceBuilder
 }
 
 type choiceParser struct {
@@ -12,7 +14,15 @@ type choiceParser struct {
 	id         int
 	commit     CommitType
 	elements   []parser
-	includedBy []parser
+	includedBy []int
+}
+
+type choiceBuilder struct {
+	name       string
+	id         int
+	commit     CommitType
+	elements   []builder
+	includedBy []int
 }
 
 func newChoice(name string, ct CommitType, elements []string) *choiceDefinition {
@@ -26,6 +36,55 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition
 func (d *choiceDefinition) nodeName() string       { return d.name }
 func (d *choiceDefinition) nodeID() int            { return d.id }
 func (d *choiceDefinition) setID(id int)           { d.id = id }
+func (d *choiceDefinition) commitType() CommitType { return d.commit }
+
+func (d *choiceDefinition) init(r *registry) error {
+	if d.cbuilder == nil {
+		d.cbuilder = &choiceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	for _, e := range d.elements {
+		def, ok := r.definition(e)
+		if !ok {
+			return parserNotFound(e)
+		}
+
+		d.cbuilder.elements = append(d.cbuilder.elements, def.builder())
+	}
+
+	parsers := &idSet{}
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, d.elements, d.id, parsers)
+}
+
+func (d *choiceDefinition) setIncludedBy(r *registry, includedBy int, parsers *idSet) error {
+	if parsers.has(d.id) {
+		return nil
+	}
+
+	d.includedBy = appendIfMissing(d.includedBy, includedBy)
+
+	if d.cbuilder == nil {
+		d.cbuilder = &choiceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	d.cbuilder.includedBy = appendIfMissing(d.cbuilder.includedBy, includedBy)
+
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, d.elements, includedBy, parsers)
+}
+
+// TODO:
+// - it may be possible to initialize the parsers non-recursively
+// - maybe the whole definition, parser and builder can be united
 
 func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 	p, ok := r.parser(d.name)
@@ -37,6 +96,7 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 		name:       d.name,
 		id:         d.id,
 		commit:     d.commit,
+		includedBy: d.includedBy,
 	}
 
 	r.setParser(cp)
@@ -48,7 +108,6 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 		element, ok := r.parser(e)
 		if ok {
 			elements = append(elements, element)
-			element.setIncludedBy(cp, parsers)
 			continue
 		}
 
@@ -62,7 +121,6 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 			return nil, err
 		}
 
-		element.setIncludedBy(cp, parsers)
 		elements = append(elements, element)
 	}
 
@@ -70,39 +128,26 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 	return cp, nil
 }
 
-func (d *choiceDefinition) commitType() CommitType {
-	return d.commit
+func (d *choiceDefinition) builder() builder {
+	if d.cbuilder == nil {
+		d.cbuilder = &choiceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	return d.cbuilder
 }
 
 func (p *choiceParser) nodeName() string { return p.name }
 func (p *choiceParser) nodeID() int      { return p.id }
 
-func (p *choiceParser) setIncludedBy(includedBy parser, parsers *idSet) {
-	// if parsers.has(p.id) {
-	// 	return
-	// }
-
-	p.includedBy = append(p.includedBy, includedBy)
-}
-
-func (p *choiceParser) storeIncluded(c *context, n *Node) {
-	if !c.excluded(n.From, p.id) {
-		return
-	}
-
-	nc := newNode(p.name, p.id, n.From, n.To, p.commit)
-	nc.append(n)
-	c.store.set(nc.From, p.id, nc)
-
-	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, nc)
-	}
-}
-
 func (p *choiceParser) parse(t Trace, c *context) {
 	// t = t.Extend(p.name)
 	// t.Out1("parsing choice", c.offset)
 
+	// TODO: don't add documentation
 	if p.commit&Documentation != 0 {
 		// t.Out1("fail, doc")
 		c.fail(c.offset)
@@ -115,38 +160,43 @@ func (p *choiceParser) parse(t Trace, c *context) {
 	}
 
 	if c.excluded(c.offset, p.id) {
-		// t.Out1("excluded")
+		// t.Out1("fail, excluded")
 		c.fail(c.offset)
 		return
 	}
 
 	c.exclude(c.offset, p.id)
-	initialOffset := c.offset
+	from := c.offset
+	to := c.offset
 
-	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
 	var match bool
+	var nextTo int
+	var elementIndex int
 
 	for {
-		elements := p.elements
 		var foundMatch bool
+		elementIndex = 0
 
-		for len(elements) > 0 {
-			elements[0].parse(t, c)
-			elements = elements[1:]
-			c.offset = node.From
+		for elementIndex < len(p.elements) {
+			p.elements[elementIndex].parse(t, c)
+			elementIndex++
+			nextTo = c.offset
+			c.offset = from
 
-			if !c.match || match && c.node.tokenLength() <= node.tokenLength() {
+			if !c.match || match && nextTo <= to {
 				continue
 			}
 
 			match = true
 			foundMatch = true
-			node = newNode(p.name, p.id, c.offset, c.offset, p.commit)
-			node.append(c.node)
+			to = nextTo
 
-			c.store.set(node.From, p.id, node)
+			c.store.setMatch(from, p.id, to)
 			for _, includedBy := range p.includedBy {
-				includedBy.storeIncluded(c, node)
+				if c.excluded(from, includedBy) {
+					// t.Out1("storing included", includedBy)
+					c.store.setMatch(from, includedBy, to)
+				}
 			}
 		}
 
@@ -156,14 +206,60 @@ func (p *choiceParser) parse(t Trace, c *context) {
 	}
 
 	if match {
+		c.success(to)
+		c.include(from, p.id)
 		// t.Out1("choice, success")
-		c.success(node)
-		c.include(initialOffset, p.id)
 		return
 	}
 
 	// t.Out1("fail")
-	c.store.set(node.From, p.id, nil)
-	c.fail(node.From)
-	c.include(initialOffset, p.id)
+	c.store.setNoMatch(from, p.id)
+	c.fail(from)
+	c.include(from, p.id)
+}
+
+func (b *choiceBuilder) nodeName() string { return b.name }
+func (b *choiceBuilder) nodeID() int      { return b.id }
+
+func (b *choiceBuilder) build(c *context) ([]*Node, bool) {
+	to, ok := c.store.takeMatch(c.offset, b.id)
+	if !ok {
+		return nil, false
+	}
+
+	for _, ib := range b.includedBy {
+		c.store.takeMatchLength(c.offset, ib, to)
+	}
+
+	var element builder
+	for _, e := range b.elements {
+		elementTo, match, _ := c.store.getMatch(c.offset, e.nodeID())
+		if match && elementTo == to {
+			element = e
+			break
+		}
+	}
+
+	if element == nil {
+		panic("damaged parse result")
+	}
+
+	from := c.offset
+
+	n, ok := element.build(c)
+	if !ok {
+		panic("damaged parse result")
+	}
+
+	if b.commit&Alias != 0 {
+		return n, true
+	}
+
+	return []*Node{{
+		Name:   b.name,
+		From:   from,
+		To:     to,
+		Nodes:  n,
+		tokens: c.tokens,
+	}}, true
 }

context.go

@@ -15,7 +15,7 @@ type context struct {
 	tokens     []rune
 	match      bool
 	node       *Node
-	isExcluded []*idSet
+	isExcluded [][]int
 }
 
 func newContext(r io.RuneReader) *context {
@@ -65,53 +65,51 @@ func (c *context) token() (rune, bool) {
 }
 
 func (c *context) excluded(offset int, id int) bool {
-	if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
+	if len(c.isExcluded) <= id {
 		return false
 	}
 
-	return c.isExcluded[offset].has(id)
+	for i := range c.isExcluded[id] {
+		if c.isExcluded[id][i] == offset {
+			return true
+		}
+	}
+
+	return false
 }
 
 func (c *context) exclude(offset int, id int) {
-	if c.excluded(offset, id) {
-		return
-	}
-
-	if len(c.isExcluded) <= offset {
-		c.isExcluded = append(c.isExcluded, nil)
-		if cap(c.isExcluded) > offset {
-			c.isExcluded = c.isExcluded[:offset+1]
+	if len(c.isExcluded) <= id {
+		if cap(c.isExcluded) > id {
+			c.isExcluded = c.isExcluded[:id+1]
 		} else {
-			c.isExcluded = append(
-				c.isExcluded[:cap(c.isExcluded)],
-				make([]*idSet, offset+1-cap(c.isExcluded))...,
-			)
+			c.isExcluded = c.isExcluded[:cap(c.isExcluded)]
+			for i := cap(c.isExcluded); i <= id; i++ {
+				c.isExcluded = append(c.isExcluded, nil)
+			}
 		}
 	}
 
-	if c.isExcluded[offset] == nil {
-		c.isExcluded[offset] = &idSet{}
-	}
-
-	c.isExcluded[offset].set(id)
+	c.isExcluded[id] = append(c.isExcluded[id], offset)
 }
 
 func (c *context) include(offset int, id int) {
-	if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
-		return
+	for i := range c.isExcluded[id] {
+		if c.isExcluded[id][i] == offset {
+			c.isExcluded[id] = append(c.isExcluded[id][:i], c.isExcluded[id][i+1:]...)
+			break
+		}
 	}
-
-	c.isExcluded[offset].unset(id)
 }
 
 func (c *context) fromStore(id int) (bool, bool) {
-	n, m, ok := c.store.get(c.offset, id)
+	to, m, ok := c.store.getMatch(c.offset, id)
 	if !ok {
 		return false, false
 	}
 
 	if m {
-		c.success(n)
+		c.success(to)
 	} else {
 		c.fail(c.offset)
 	}
@@ -119,15 +117,8 @@ func (c *context) fromStore(id int) (bool, bool) {
 	return m, true
 }
 
-func (c *context) success(n *Node) {
-	c.node = n
-	c.offset = n.To
-	c.match = true
-}
-
-func (c *context) successChar() {
-	c.node = nil
-	c.offset++
+func (c *context) success(to int) {
+	c.offset = to
 	c.match = true
 }
 
@@ -136,8 +127,10 @@ func (c *context) fail(offset int) {
 	c.match = false
 }
 
-func (c *context) finalize() error {
-	if c.node.To < c.readOffset {
+func (c *context) finalize(root parser) error {
+	rootID := root.nodeID()
+	to, match, found := c.store.getMatch(0, rootID)
+	if !found || !match || to < c.readOffset {
 		return ErrUnexpectedCharacter
 	}
 
@@ -152,6 +145,5 @@ func (c *context) finalize() error {
 		}
 	}
 
-	c.node.commit(c.tokens)
 	return nil
 }

json.parser

@@ -6,7 +6,7 @@ null        = "null";
 string      = "\"" ([^\\"\b\f\n\r\t] | "\\" (["\\/bfnrt] | "u" [0-9a-f]{4}))* "\"";
 number      = "-"? ("0" | [1-9][0-9]*) ("." [0-9]+)? ([eE] [+\-]? [0-9]+)?;
 entry       = string ws* ":" ws* value;
-object      = "{" ws* (entry (ws* "," ws* entry)*)? ws* "}";
-array       = "[" ws* (value (ws* "," ws* value)*)? ws* "]";
+object      = "{" ws* (entry ws* ("," ws* entry)*)? ws* "}";
+array       = "[" ws* (value ws* ("," ws* value)*)? ws* "]";
 value:alias = true | false | null | string | number | object | array;
-json        = value;
+json:root   = value;

notes.txt

@@ -15,8 +15,11 @@ coverage
 custom tokens
 indentation
 streaming
-code generation go
+code generation go:
+- find things that depend on the syntax input
+- char matches can be generated into switches
 code generation js
+ws and nows flags
 
 [problems]
 can the root be an alias? check the commit mechanism

parse.go

@@ -5,19 +5,26 @@ import "fmt"
 type definition interface {
 	nodeName() string
 	nodeID() int
-	setID(int)
-	parser(*registry, *idSet) (parser, error)
 	commitType() CommitType
+	setID(int)
+	init(*registry) error
+	setIncludedBy(*registry, int, *idSet) error
+	parser(*registry, *idSet) (parser, error)
+	builder() builder
 }
 
 type parser interface {
 	nodeName() string
 	nodeID() int
-	setIncludedBy(parser, *idSet)
-	storeIncluded(*context, *Node)
 	parse(Trace, *context)
 }
 
+type builder interface {
+	nodeName() string
+	nodeID() int
+	build(*context) ([]*Node, bool)
+}
+
 func parserNotFound(name string) error {
 	return fmt.Errorf("parser not found: %s", name)
 }
@@ -26,9 +33,9 @@ func cannotIncludeParsers(name string) error {
 	return fmt.Errorf("parser: %s cannot include other parsers", name)
 }
 
-func stringsContainDeprecated(ss []string, s string) bool {
-	for _, si := range ss {
-		if si == s {
+func intsContain(is []int, i int) bool {
+	for _, ii := range is {
+		if ii == i {
 			return true
 		}
 	}
@@ -36,19 +43,59 @@ func stringsContainDeprecated(ss []string, s string) bool {
 	return false
 }
 
-func parse(t Trace, p parser, c *context) (*Node, error) {
+func appendIfMissing(is []int, i int) []int {
+	if intsContain(is, i) {
+		return is
+	}
+
+	return append(is, i)
+}
+
+func setItemsIncludedBy(r *registry, items []string, includedBy int, parsers *idSet) error {
+	for _, item := range items {
+		di, ok := r.definition(item)
+		if !ok {
+			return ErrNoParsersDefined
+		}
+
+		di.setIncludedBy(r, includedBy, parsers)
+	}
+
+	return nil
+}
+
+func sequenceItemNames(items []SequenceItem) []string {
+	names := make([]string, len(items))
+	for i := range items {
+		names[i] = items[i].Name
+	}
+
+	return names
+}
+
+func parse(t Trace, p parser, c *context) error {
 	p.parse(t, c)
 	if c.readErr != nil {
-		return nil, c.readErr
+		return c.readErr
 	}
 
 	if !c.match {
-		return nil, ErrInvalidInput
+		return ErrInvalidInput
 	}
 
-	if err := c.finalize(); err != nil {
-		return nil, err
+	if err := c.finalize(p); err != nil {
+		return err
 	}
 
-	return c.node, nil
+	return nil
+}
+
+func build(b builder, c *context) *Node {
+	c.offset = 0
+	n, ok := b.build(c)
+	if !ok || len(n) != 1 {
+		panic("damaged parse result")
+	}
+
+	return n[0]
 }

sequence.go

@@ -5,6 +5,9 @@ type sequenceDefinition struct {
 	id         int
 	commit     CommitType
 	items      []SequenceItem
+	includedBy []int
+	ranges     [][]int
+	sbuilder   *sequenceBuilder
 }
 
 type sequenceParser struct {
@@ -13,7 +16,16 @@ type sequenceParser struct {
 	commit     CommitType
 	items      []parser
 	ranges     [][]int
-	includedBy []parser
+	includedBy []int
+}
+
+type sequenceBuilder struct {
+	name       string
+	id         int
+	commit     CommitType
+	items      []builder
+	ranges     [][]int
+	includedBy []int
 }
 
 func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefinition {
@@ -27,8 +39,76 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi
 func (d *sequenceDefinition) nodeName() string       { return d.name }
 func (d *sequenceDefinition) nodeID() int            { return d.id }
 func (d *sequenceDefinition) setID(id int)           { d.id = id }
+func (d *sequenceDefinition) commitType() CommitType { return d.commit }
+
+func (d *sequenceDefinition) includeItems() bool {
+	return len(d.items) == 1 && d.items[0].Min == 1 && d.items[0].Max == 1
+}
+
+func (d *sequenceDefinition) init(r *registry) error {
+	if d.sbuilder == nil {
+		d.sbuilder = &sequenceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	for _, item := range d.items {
+		if item.Min == 0 && item.Max == 0 {
+			item.Min, item.Max = 1, 1
+		} else if item.Max == 0 {
+			item.Max = -1
+		}
+
+		d.ranges = append(d.ranges, []int{item.Min, item.Max})
+
+		def, ok := r.definition(item.Name)
+		if !ok {
+			return parserNotFound(item.Name)
+		}
+
+		d.sbuilder.items = append(d.sbuilder.items, def.builder())
+	}
+
+	d.sbuilder.ranges = d.ranges
+
+	if !d.includeItems() {
+		return nil
+	}
+
+	parsers := &idSet{}
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, sequenceItemNames(d.items), d.id, parsers)
+}
+
+func (d *sequenceDefinition) setIncludedBy(r *registry, includedBy int, parsers *idSet) error {
+	if parsers.has(d.id) {
+		return nil
+	}
+
+	d.includedBy = appendIfMissing(d.includedBy, includedBy)
+
+	if d.sbuilder == nil {
+		d.sbuilder = &sequenceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	d.sbuilder.includedBy = appendIfMissing(d.sbuilder.includedBy, includedBy)
+
+	if !d.includeItems() {
+		return nil
+	}
+
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, sequenceItemNames(d.items), includedBy, parsers)
+}
 
 func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
+	// TODO: what is this for? test with sequence containing a sequence through a choice
 	if parsers.has(d.id) {
 		panic(cannotIncludeParsers(d.name))
 	}
@@ -42,28 +122,18 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error)
 		name:       d.name,
 		id:         d.id,
 		commit:     d.commit,
+		includedBy: d.includedBy,
 	}
 
 	r.setParser(sp)
 
-	var (
-		items  []parser
-		ranges [][]int
-	)
-
+	var items []parser
 	parsers.set(d.id)
 	defer parsers.unset(d.id)
 	for _, item := range d.items {
-		if item.Min == 0 && item.Max == 0 {
-			item.Min, item.Max = 1, 1
-		} else if item.Max == 0 {
-			item.Max = -1
-		}
-
 		pi, ok := r.parser(item.Name)
 		if ok {
 			items = append(items, pi)
-			ranges = append(ranges, []int{item.Min, item.Max})
 			continue
 		}
 
@@ -78,48 +148,28 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error)
 		}
 
 		items = append(items, pi)
-		ranges = append(ranges, []int{item.Min, item.Max})
-	}
-
-	// for single items, acts like a choice
-	if len(items) == 1 && ranges[0][0] == 1 && ranges[0][1] == 1 {
-		items[0].setIncludedBy(sp, parsers)
 	}
 
 	sp.items = items
-	sp.ranges = ranges
+	sp.ranges = d.ranges
 	return sp, nil
 }
 
-func (d *sequenceDefinition) commitType() CommitType {
-	return d.commit
+func (d *sequenceDefinition) builder() builder {
+	if d.sbuilder == nil {
+		d.sbuilder = &sequenceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	return d.sbuilder
 }
 
 func (p *sequenceParser) nodeName() string { return p.name }
 func (p *sequenceParser) nodeID() int      { return p.id }
 
-func (p *sequenceParser) setIncludedBy(includedBy parser, parsers *idSet) {
-	// if parsers.has(p.id) {
-	// 	return
-	// }
-
-	p.includedBy = append(p.includedBy, includedBy)
-}
-
-func (p *sequenceParser) storeIncluded(c *context, n *Node) {
-	if !c.excluded(n.From, p.id) {
-		return
-	}
-
-	nc := newNode(p.name, p.id, n.From, n.To, p.commit)
-	nc.append(n)
-	c.store.set(nc.From, p.id, nc)
-
-	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, nc)
-	}
-}
-
 func (p *sequenceParser) parse(t Trace, c *context) {
 	// t = t.Extend(p.name)
 	// t.Out1("parsing sequence", c.offset)
@@ -131,68 +181,119 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 	}
 
 	if c.excluded(c.offset, p.id) {
-		// t.Out1("excluded")
+		// t.Out1("fail, excluded")
 		c.fail(c.offset)
 		return
 	}
 
-	c.exclude(c.offset, p.id)
-	initialOffset := c.offset
-
-	items := p.items
-	ranges := p.ranges
-	var currentCount int
-	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
-
-	for len(items) > 0 {
-		var m bool
-		// var ok bool
-		// m, ok = c.fromStore(items[0].nodeID())
-		// if ok {
-		// 	// t.Out1("sequence item found in store, match:", m, items[0].nodeName(), c.offset)
-		// } else {
-		items[0].parse(t, c)
-		m = c.match
+	// if c.store.hasNoMatch(c.offset, p.id) {
+	// 	c.fail(c.offset)
 	// }
 
-		if !m {
-			if currentCount < ranges[0][0] {
-				// t.Out1("fail, item failed")
-				// c.store.set(node.From, p.id, nil)
-				c.fail(node.From)
-				c.include(initialOffset, p.id)
+	c.exclude(c.offset, p.id)
+
+	itemIndex := 0
+	var currentCount int
+	from := c.offset
+	to := c.offset
+
+	for itemIndex < len(p.items) {
+		// TODO: is it ok to parse before max range check? what if max=0
+		p.items[itemIndex].parse(t, c)
+		if !c.match {
+			if currentCount < p.ranges[itemIndex][0] {
+				// c.store.setNoMatch(from, p.id)
+				c.fail(from)
+				c.include(from, p.id)
+				// t.Out1("fail, not enough items")
 				return
 			}
 
-			items = items[1:]
-			ranges = ranges[1:]
+			itemIndex++
 			currentCount = 0
 			continue
 		}
 
-		// nil as char
-		if c.node == nil {
-			node.appendChar(c.offset)
-			currentCount++
-		} else if c.node.tokenLength() > 0 {
-			node.append(c.node)
+		parsed := c.offset > to
+		if parsed {
 			currentCount++
 		}
 
-		if c.node != nil && c.node.tokenLength() == 0 || ranges[0][1] >= 0 && currentCount == ranges[0][1] {
-			items = items[1:]
-			ranges = ranges[1:]
+		to = c.offset
+
+		if !parsed || p.ranges[itemIndex][1] >= 0 && currentCount == p.ranges[itemIndex][1] {
+			itemIndex++
 			currentCount = 0
 		}
 	}
 
-	// t.Out1("success, items parsed")
-
-	// c.store.set(node.From, p.id, node)
 	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, node)
+		if c.excluded(from, includedBy) {
+			// t.Out1("storing included", includedBy)
+			c.store.setMatch(from, includedBy, to)
+		}
 	}
 
-	c.success(node)
-	c.include(initialOffset, p.id)
+	// t.Out1("success")
+	c.store.setMatch(from, p.id, to)
+	c.success(to)
+	c.include(from, p.id)
+}
+
+func (b *sequenceBuilder) nodeName() string { return b.name }
+func (b *sequenceBuilder) nodeID() int      { return b.id }
+
+func (b *sequenceBuilder) build(c *context) ([]*Node, bool) {
+	to, ok := c.store.takeMatch(c.offset, b.id)
+	if !ok {
+		return nil, false
+	}
+
+	for _, ib := range b.includedBy {
+		c.store.takeMatchLength(c.offset, ib, to)
+	}
+
+	from := c.offset
+	var (
+		itemIndex    int
+		currentCount int
+		nodes        []*Node
+	)
+
+	for itemIndex < len(b.items) {
+		itemFrom := c.offset
+		n, ok := b.items[itemIndex].build(c)
+		if !ok {
+			if currentCount < b.ranges[itemIndex][0] {
+				panic(b.name + ": damaged parse result")
+			}
+
+			itemIndex++
+			currentCount = 0
+			continue
+		}
+
+		parsed := c.offset > itemFrom
+		if parsed {
+			nodes = append(nodes, n...)
+			currentCount++
+		}
+
+		if !parsed || b.ranges[itemIndex][1] >= 0 && currentCount == b.ranges[itemIndex][1] {
+			itemIndex++
+			currentCount = 0
+		}
+	}
+
+	if b.commit&Alias != 0 {
+		return nodes, true
+	}
+
+	return []*Node{{
+		Name:   b.name,
+		From:   from,
+		To:     to,
+		Nodes:  nodes,
+		tokens: c.tokens,
+	}}, true
 }

store.go

@@ -1,270 +1,159 @@
 package treerack
 
-type storeEntry struct {
-	match   *idSet
-	noMatch *idSet
-	nodes   []*Node
-}
+// TODO:
+// - store it similarly to the excluded ones? sorted by offset?
+// - use a helper field for the last accessed position to walk from there? for every offset?
+// - use a helper field to store the largest value and its index, too? for an offset?
 
 type store struct {
-	entries []*storeEntry
+	noMatch []*idSet
+	match   [][]int
 }
 
-func (c *store) get(offset int, id int) (*Node, bool, bool) {
-	if len(c.entries) <= offset {
-		return nil, false, false
+func (s *store) hasNoMatch(offset, id int) bool {
+	if len(s.noMatch) <= offset || s.noMatch[offset] == nil {
+		return false
 	}
 
-	tc := c.entries[offset]
-	if tc == nil {
-		return nil, false, false
+	return s.noMatch[offset].has(id)
 }
 
-	if tc.noMatch.has(id) {
-		return nil, false, true
+func (s *store) getMatch(offset, id int) (int, bool, bool) {
+	if s.hasNoMatch(offset, id) {
+		return 0, false, true
 	}
 
-	if !tc.match.has(id) {
-		return nil, false, false
+	if len(s.match) <= offset {
+		return 0, false, false
 	}
 
-	for _, n := range tc.nodes {
-		if n.id == id {
-			return n, true, true
+	var (
+		found bool
+		to    int
+	)
+
+	for i := 0; i < len(s.match[offset]); i += 2 {
+		if s.match[offset][i] != id {
+			continue
+		}
+
+		found = true
+		if s.match[offset][i+1] > to {
+			to = s.match[offset][i+1]
 		}
 	}
 
-	return nil, false, false
+	return to, found, found
 }
 
-func (c *store) set(offset int, id int, n *Node) {
-	if len(c.entries) <= offset {
-		if cap(c.entries) > offset {
-			c.entries = c.entries[:offset+1]
-		} else {
-			c.entries = c.entries[:cap(c.entries)]
-			for len(c.entries) <= offset {
-				c.entries = append(c.entries, nil)
-			}
-		}
-	}
-
-	tc := c.entries[offset]
-	if tc == nil {
-		tc = &storeEntry{
-			match:   &idSet{},
-			noMatch: &idSet{},
-		}
-
-		c.entries[offset] = tc
-	}
-
-	if n == nil {
-		if tc.match.has(id) {
-			return
-		}
-
-		tc.noMatch.set(id)
-		return
-	}
-
-	tc.match.set(id)
-	for i, ni := range tc.nodes {
-		if ni.id == id {
-			if n.tokenLength() > ni.tokenLength() {
-				tc.nodes[i] = n
-			}
-
-			return
-		}
-	}
-
-	tc.nodes = append(tc.nodes, n)
-}
-
-/*
-
-[][][]int
-
-id, length, where to start in the underlying layer, which list in the layer
-
-attibutes:
-- sequence: length, the items in the layer below
-- choice: the item below
-
-features:
-- there can be sequences or choices under choices
-
-in every position:
-- store the valid choices with the underlying parsed nodes
-
-3D table: layer, choice, sequence
-stored choice identified by: offset, layer, choice index
-
-*/
-
-func (c *store) inc() {
-}
-
-func (c *store) dec() {
-}
-
-func (c *store) get2(offset, id int) (int, bool) {
+func (s *store) takeMatch(offset, id int) (int, bool) {
+	if s.hasNoMatch(offset, id) {
 		return 0, false
 	}
 
-func (c *store) cache(offset, id int, match bool, length int) {
+	if len(s.match) <= offset {
+		return 0, false
 	}
 
-func (c *store) set2(offset, id int, match bool, length int) {
-	/*
-		c.cache(offset, id, match, length)
-		levels := c.offsetLevels[offset]
-		levels[c.currentLevel] = id
-	*/
-}
-
-/*
-package treerack
-
-type storeEntry struct {
-	match   *idSet
-	noMatch *idSet
-	nodes   []*Node
-	offset int
-}
-
-type store struct {
-	current *storeEntry
-	currentIndex int
-	entries []*storeEntry
-}
-
-func (s *store) find(offset int) *storeEntry {
-	if s.current == nil {
-		return nil
-	}
-
-	var seekPrev, seekNext bool
-	for {
-		switch {
-		case s.current.offset == offset:
-			return s.current
-		case s.current.offset < offset:
-			if seekPrev {
-				return nil
-			}
-
-			seekNext = true
-			s.currentIndex++
-			if s.currentIndex == len(s.entries) {
-				s.currentIndex = len(s.entries) - 1
-				return nil
-			}
-
-			s.current = s.entries[s.currentIndex]
-		case s.current.offset > offset:
-			if seekNext {
-				return nil
-			}
-
-			seekPrev = true
-			s.currentIndex--
-			if s.currentIndex == -1 {
-				s.currentIndex = 0
-				return nil
-			}
-
-			s.current = s.entries[s.currentIndex]
-		}
-	}
-}
-
-func (s *store) findCreate(offset int) *storeEntry {
-	entry := s.find(offset)
-	if entry != nil {
-		return entry
-	}
-
-	entry = &storeEntry{
-		match:   &idSet{},
-		noMatch: &idSet{},
-		offset: offset,
-	}
-
-	switch {
-	case s.current != nil && s.current.offset > offset:
-		s.entries = append(
-			s.entries[:s.currentIndex],
-			append([]*storeEntry{entry}, s.entries[s.currentIndex:]...)...,
+	var (
+		found bool
+		to    int
+		index int
 	)
-		s.current = entry
-	case s.current != nil && s.current.offset < offset:
-		s.entries = append(
-			s.entries[:s.currentIndex + 1],
-			append([]*storeEntry{entry}, s.entries[s.currentIndex + 1:]...)...,
+
+	for i := 0; i < len(s.match[offset]); i += 2 {
+		if s.match[offset][i] != id {
+			continue
+		}
+
+		found = true
+		if s.match[offset][i+1] > to {
+			to = s.match[offset][i+1]
+			index = i
+		}
+	}
+
+	if found {
+		s.match[offset][index] = -1
+	}
+
+	return to, found
+}
+
+func (s *store) takeMatchLength(offset, id, to int) (int, bool) {
+	if s.hasNoMatch(offset, id) {
+		return 0, false
+	}
+
+	if len(s.match) <= offset {
+		return 0, false
+	}
+
+	var (
+		found bool
+		// index int
 	)
-		s.current = entry
-		s.currentIndex++
-	default:
-		s.entries = []*storeEntry{entry}
-		s.current = entry
-		s.currentIndex = 0
+
+	for i := 0; i < len(s.match[offset]); i += 2 {
+		if s.match[offset][i] != id {
+			continue
 		}
 
-	return entry
-}
-
-func (s *store) get(offset int, id int) (*Node, bool, bool) {
-	entry := s.find(offset)
-	if entry == nil {
-		return nil, false, false
-	}
-
-	if entry == nil {
-		return nil, false, false
-	}
-
-	if entry.noMatch.has(id) {
-		return nil, false, true
-	}
-
-	if !entry.match.has(id) {
-		return nil, false, false
-	}
-
-	for _, n := range entry.nodes {
-		if n.id == id {
-			return n, true, true
+		found = true
+		if s.match[offset][i+1] == to {
+			s.match[offset][i] = -1
+			return to, true
+			//eindex = i
 		}
 	}
 
-	return nil, false, false
+	if found {
+		// s.match[offset][index] = -1
 	}
 
-func (s *store) set(offset int, id int, n *Node) {
-	entry := s.findCreate(offset)
+	return to, found
+}
 
-	if n == nil {
-		if entry.match.has(id) {
+func (s *store) ensureOffset(offset int) {
+	if len(s.match) > offset {
 		return
 	}
 
-		entry.noMatch.set(id)
+	if cap(s.match) > offset {
+		s.match = s.match[:offset+1]
 		return
 	}
 
-	entry.match.set(id)
-	for i, ni := range entry.nodes {
-		if ni.id == id {
-			if n.tokenLength() > ni.tokenLength() {
-				entry.nodes[i] = n
+	s.match = s.match[:cap(s.match)]
+	for i := len(s.match); i <= offset; i++ {
+		s.match = append(s.match, nil)
+	}
 }
 
+func (s *store) setMatch(offset, id, to int) {
+	if toe, match, ok := s.getMatch(offset, id); ok && match && toe == to {
 		return
 	}
+
+	s.ensureOffset(offset)
+	s.match[offset] = append(s.match[offset], id, to)
 }
 
-	entry.nodes = append(entry.nodes, n)
+func (s *store) setNoMatch(offset, id int) {
+	if len(s.noMatch) <= offset {
+		if cap(s.noMatch) > offset {
+			s.noMatch = s.noMatch[:offset+1]
+		} else {
+			s.noMatch = s.noMatch[:cap(s.noMatch)]
+			for i := cap(s.noMatch); i <= offset; i++ {
+				s.noMatch = append(s.noMatch, nil)
+			}
+		}
+	}
+
+	if s.noMatch[offset] == nil {
+		s.noMatch[offset] = &idSet{}
+	}
+
+	s.noMatch[offset].set(id)
 }
-*/

syntax.go

@@ -29,6 +29,7 @@ type Syntax struct {
 	explicitRoot bool
 	root         definition
 	parser       parser
+	builder      builder
 }
 
 var (
@@ -88,7 +89,7 @@ func childName(name string, childIndex int) string {
 
 func (s *Syntax) Class(name string, ct CommitType, not bool, chars []rune, ranges [][]rune) error {
 	cname := childName(name, 0)
-	if err := s.register(newChar(cname, Alias, not, chars, ranges)); err != nil {
+	if err := s.register(newChar(cname, not, chars, ranges)); err != nil {
 		return err
 	}
 
@@ -100,7 +101,7 @@ func (s *Syntax) CharSequence(name string, ct CommitType, chars []rune) error {
 	for i, ci := range chars {
 		ref := childName(name, i)
 		refs = append(refs, ref)
-		if err := s.register(newChar(ref, Alias, false, []rune{ci}, nil)); err != nil {
+		if err := s.register(newChar(ref, false, []rune{ci}, nil)); err != nil {
 			return err
 		}
 	}
@@ -143,6 +144,10 @@ func (s *Syntax) Init() error {
 		return ErrRootAlias
 	}
 
+	for _, p := range s.registry.definitions {
+		p.init(s.registry)
+	}
+
 	var err error
 	s.parser, err = s.root.parser(s.registry, &idSet{})
 	if err != nil {
@@ -150,6 +155,7 @@ func (s *Syntax) Init() error {
 		return err
 	}
 
+	s.builder = s.root.builder()
 	s.initialized = true
 	return nil
 }
@@ -162,7 +168,7 @@ func (s *Syntax) Generate(w io.Writer) error {
 	return ErrNotImplemented
 }
 
-// TODO: optimize top sequences to save memory
+// TODO: optimize top sequences to save memory, or just support streaming, or combine the two
 
 func (s *Syntax) Parse(r io.Reader) (*Node, error) {
 	if err := s.Init(); err != nil {
@@ -170,5 +176,9 @@ func (s *Syntax) Parse(r io.Reader) (*Node, error) {
 	}
 
 	c := newContext(bufio.NewReader(r))
-	return parse(s.trace, s.parser, c)
+	if err := parse(s.trace, s.parser, c); err != nil {
+		return nil, err
+	}
+
+	return build(s.builder, c), nil
 }