Merge branch 'separate-build-phase' of https://github.com/aryszka/treerack into separate-build-phase

boot_test.go

@@ -13,7 +13,7 @@ func TestBoot(t *testing.T) {
 		return
 	}
 
-	f, err := os.Open("mml.parser")
+	f, err := os.Open("syntax.parser")
 	if err != nil {
 		t.Error(err)
 		return
@@ -21,65 +21,73 @@ func TestBoot(t *testing.T) {
 
 	defer f.Close()
 
-	start := time.Now()
+	var d time.Duration
-	_, err = b.Parse(f)
+	const n = 120
-	t.Log("duration:", time.Now().Sub(start))
+	var n0 *Node
-
+	for i := 0; i < n; i++ {
-	if err != ErrNotImplemented {
+		if _, err := f.Seek(0, 0); err != nil {
 			t.Error(err)
+			return
 		}
 
-	// if err != nil {
+		start := time.Now()
-	// 	t.Error(err)
+		n0, err = b.Parse(f)
-	// 	return
+		d += time.Now().Sub(start)
-	// }
 
-	// s0 := NewSyntax()
+		if err != nil {
-	// if err := define(s0, n0); err != nil {
+			t.Error(err)
-	// 	t.Error(err)
+			return
-	// 	return
+		}
-	// }
+	}
-
+
-	// _, err = f.Seek(0, 0)
+	t.Log("duration:", d/n)
-	// if err != nil {
+
-	// 	t.Error(err)
+	s0 := NewSyntax()
-	// 	return
+	if err := define(s0, n0); err != nil {
-	// }
+		t.Error(err)
-
+		return
-	// err = s0.Init()
+	}
-	// if err != nil {
+
-	// 	t.Error(err)
+	_, err = f.Seek(0, 0)
-	// 	return
+	if err != nil {
-	// }
+		t.Error(err)
-
+		return
-	// n1, err := s0.Parse(f)
+	}
-	// if err != nil {
+
-	// 	t.Error(err)
+	err = s0.Init()
-	// 	return
+	if err != nil {
-	// }
+		t.Error(err)
-
+		return
-	// checkNode(t, n1, n0)
+	}
-	// if t.Failed() {
+
-	// 	return
+	n1, err := s0.Parse(f)
-	// }
+	if err != nil {
-
+		t.Error(err)
-	// s1 := NewSyntax()
+		return
-	// if err := define(s1, n1); err != nil {
+	}
-	// 	t.Error(err)
+
-	// 	return
+	checkNode(t, n1, n0)
-	// }
+	if t.Failed() {
-
+		return
-	// _, err = f.Seek(0, 0)
+	}
-	// if err != nil {
+
-	// 	t.Error(err)
+	s1 := NewSyntax()
-	// 	return
+	if err := define(s1, n1); err != nil {
-	// }
+		t.Error(err)
-
+		return
-	// n2, err := s1.Parse(f)
+	}
-	// if err != nil {
+
-	// 	t.Error(err)
+	_, err = f.Seek(0, 0)
-	// 	return
+	if err != nil {
-	// }
+		t.Error(err)
-
+		return
-	// checkNode(t, n2, n1)
+	}
+
+	n2, err := s1.Parse(f)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	checkNode(t, n2, n1)
 }

char.go

@@ -3,23 +3,20 @@ package treerack
 type charParser struct {
 	name       string
 	id         int
-	commit     CommitType
 	not        bool
 	chars      []rune
 	ranges     [][]rune
-	includedBy []parser
+	includedBy []int
 }
 
 func newChar(
 	name string,
-	ct CommitType,
 	not bool,
 	chars []rune,
 	ranges [][]rune,
 ) *charParser {
 	return &charParser{
 		name:   name,
-		commit: ct,
 		not:    not,
 		chars:  chars,
 		ranges: ranges,
@@ -29,6 +26,14 @@ func newChar(
 func (p *charParser) nodeName() string       { return p.name }
 func (p *charParser) nodeID() int            { return p.id }
 func (p *charParser) setID(id int)           { p.id = id }
+func (p *charParser) commitType() CommitType { return Alias }
+
+func (p *charParser) init(r *registry) error { return nil }
+
+func (p *charParser) setIncludedBy(r *registry, includedBy int, parsers *idSet) error {
+	p.includedBy = appendIfMissing(p.includedBy, includedBy)
+	return nil
+}
 
 func (p *charParser) parser(r *registry, parsers *idSet) (parser, error) {
 	if parsers.has(p.id) {
@@ -43,20 +48,8 @@ func (p *charParser) parser(r *registry, parsers *idSet) (parser, error) {
 	return p, nil
 }
 
-func (p *charParser) commitType() CommitType {
+func (p *charParser) builder() builder {
-	return p.commit
+	return p
-}
-
-func (p *charParser) setIncludedBy(includedBy parser, parsers *idSet) {
-	if parsers.has(p.id) {
-		panic(cannotIncludeParsers(p.name))
-	}
-
-	p.includedBy = append(p.includedBy, includedBy)
-}
-
-func (p *charParser) storeIncluded(*context, int, int) {
-	panic(cannotIncludeParsers(p.name))
 }
 
 func (p *charParser) match(t rune) bool {
@@ -83,6 +76,21 @@ func (p *charParser) parse(t Trace, c *context) {
 
 	c.success(c.offset + 1)
 	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, c.offset, c.offset+1)
+		c.store.setMatch(c.offset, includedBy, c.offset+1)
 	}
 }
+
+func (p *charParser) build(c *context) ([]*Node, bool) {
+	t, ok := c.token()
+	if !ok {
+		panic("damaged parser context")
+	}
+
+	if !p.match(t) {
+		return nil, false
+	}
+
+	// always alias
+	c.offset++
+	return nil, true
+}

choice.go

@@ -5,6 +5,8 @@ type choiceDefinition struct {
 	id         int
 	commit     CommitType
 	elements   []string
+	includedBy []int
+	cbuilder   *choiceBuilder
 }
 
 type choiceParser struct {
@@ -12,7 +14,15 @@ type choiceParser struct {
 	id         int
 	commit     CommitType
 	elements   []parser
-	includedBy []parser
+	includedBy []int
+}
+
+type choiceBuilder struct {
+	name       string
+	id         int
+	commit     CommitType
+	elements   []builder
+	includedBy []int
 }
 
 func newChoice(name string, ct CommitType, elements []string) *choiceDefinition {
@@ -26,6 +36,55 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition
 func (d *choiceDefinition) nodeName() string       { return d.name }
 func (d *choiceDefinition) nodeID() int            { return d.id }
 func (d *choiceDefinition) setID(id int)           { d.id = id }
+func (d *choiceDefinition) commitType() CommitType { return d.commit }
+
+func (d *choiceDefinition) init(r *registry) error {
+	if d.cbuilder == nil {
+		d.cbuilder = &choiceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	for _, e := range d.elements {
+		def, ok := r.definition(e)
+		if !ok {
+			return parserNotFound(e)
+		}
+
+		d.cbuilder.elements = append(d.cbuilder.elements, def.builder())
+	}
+
+	parsers := &idSet{}
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, d.elements, d.id, parsers)
+}
+
+func (d *choiceDefinition) setIncludedBy(r *registry, includedBy int, parsers *idSet) error {
+	if parsers.has(d.id) {
+		return nil
+	}
+
+	d.includedBy = appendIfMissing(d.includedBy, includedBy)
+
+	if d.cbuilder == nil {
+		d.cbuilder = &choiceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	d.cbuilder.includedBy = appendIfMissing(d.cbuilder.includedBy, includedBy)
+
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, d.elements, includedBy, parsers)
+}
+
+// TODO:
+// - it may be possible to initialize the parsers non-recursively
+// - maybe the whole definition, parser and builder can be united
 
 func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 	p, ok := r.parser(d.name)
@@ -37,6 +96,7 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 		name:       d.name,
 		id:         d.id,
 		commit:     d.commit,
+		includedBy: d.includedBy,
 	}
 
 	r.setParser(cp)
@@ -48,7 +108,6 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 		element, ok := r.parser(e)
 		if ok {
 			elements = append(elements, element)
-			element.setIncludedBy(cp, parsers)
 			continue
 		}
 
@@ -62,7 +121,6 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 			return nil, err
 		}
 
-		element.setIncludedBy(cp, parsers)
 		elements = append(elements, element)
 	}
 
@@ -70,33 +128,21 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
 	return cp, nil
 }
 
-func (d *choiceDefinition) commitType() CommitType {
+func (d *choiceDefinition) builder() builder {
-	return d.commit
+	if d.cbuilder == nil {
+		d.cbuilder = &choiceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	return d.cbuilder
 }
 
 func (p *choiceParser) nodeName() string { return p.name }
 func (p *choiceParser) nodeID() int      { return p.id }
 
-func (p *choiceParser) setIncludedBy(includedBy parser, parsers *idSet) {
-	if parsers.has(p.id) {
-		return
-	}
-
-	p.includedBy = append(p.includedBy, includedBy)
-}
-
-func (p *choiceParser) storeIncluded(c *context, from, to int) {
-	if !c.excluded(from, p.id) {
-		return
-	}
-
-	c.store.setMatch(from, p.id, to)
-
-	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, from, to)
-	}
-}
-
 func (p *choiceParser) parse(t Trace, c *context) {
 	if p.commit&Documentation != 0 {
 		c.fail(c.offset)
@@ -140,7 +186,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
 
 			c.store.setMatch(from, p.id, to)
 			for _, includedBy := range p.includedBy {
-				includedBy.storeIncluded(c, from, to)
+				c.store.setMatch(from, includedBy, to)
 			}
 		}
 
@@ -159,3 +205,49 @@ func (p *choiceParser) parse(t Trace, c *context) {
 	c.fail(from)
 	c.include(from, p.id)
 }
+
+func (b *choiceBuilder) nodeName() string { return b.name }
+func (b *choiceBuilder) nodeID() int      { return b.id }
+
+func (b *choiceBuilder) build(c *context) ([]*Node, bool) {
+	to, ok := c.store.takeMatch(c.offset, b.id)
+	if !ok {
+		return nil, false
+	}
+
+	for _, ib := range b.includedBy {
+		c.store.takeMatchLength(c.offset, ib, to)
+	}
+
+	var element builder
+	for _, e := range b.elements {
+		elementTo, match, _ := c.store.getMatch(c.offset, e.nodeID())
+		if match && elementTo == to {
+			element = e
+			break
+		}
+	}
+
+	if element == nil {
+		panic("damaged parse result")
+	}
+
+	from := c.offset
+
+	n, ok := element.build(c)
+	if !ok {
+		panic("damaged parse result")
+	}
+
+	if b.commit&Alias != 0 {
+		return n, true
+	}
+
+	return []*Node{{
+		Name:   b.name,
+		From:   from,
+		To:     to,
+		Nodes:  n,
+		tokens: c.tokens,
+	}}, true
+}

context.go

@@ -127,10 +127,10 @@ func (c *context) fail(offset int) {
 	c.match = false
 }
 
-func (c *context) finalize() error {
+func (c *context) finalize(root parser) error {
-	return ErrNotImplemented
+	rootID := root.nodeID()
-
+	to, match, found := c.store.getMatch(0, rootID)
-	if c.node.To < c.readOffset {
+	if !found || !match || to < c.readOffset {
 		return ErrUnexpectedCharacter
 	}
 
@@ -145,6 +145,5 @@ func (c *context) finalize() error {
 		}
 	}
 
-	c.node.commit(c.tokens)
 	return nil
 }

json.parser

@@ -1,12 +1,12 @@
 // JSON (http://www.json.org)
-ws:ws       = [ \b\f\n\r\t];
+ws:alias    = [ \b\f\n\r\t];
 true        = "true";
 false       = "false";
 null        = "null";
-string:nows = "\"" ([^\\"\b\f\n\r\t] | "\\" (["\\/bfnrt] | "u" [0-9a-f]{4}))* "\"";
+string      = "\"" ([^\\"\b\f\n\r\t] | "\\" (["\\/bfnrt] | "u" [0-9a-f]{4}))* "\"";
-number:nows = "-"? ("0" | [1-9][0-9]*) ("." [0-9]+)? ([eE] [+\-]? [0-9]+)?;
+number      = "-"? ("0" | [1-9][0-9]*) ("." [0-9]+)? ([eE] [+\-]? [0-9]+)?;
-entry       = string ":" value;
+entry       = string ws* ":" ws* value;
-object      = "{" (entry ("," entry)*)? "}";
+object      = "{" ws* (entry ws* ("," ws* entry)*)? ws* "}";
-array       = "[" (value ("," value)*)? "]";
+array       = "[" ws* (value ws* ("," ws* value)*)? ws* "]";
 value:alias = true | false | null | string | number | object | array;
 json:root   = value;

mml_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestMML(t *testing.T) {
-	test(t, "mml.parser", "mml", []testItem{{
+	testTrace(t, "mml.parser", "mml", 1, []testItem{{
 		msg:  "empty",
 		node: &Node{Name: "mml"},
 	}, {

parse.go

@@ -5,24 +5,24 @@ import "fmt"
 type definition interface {
 	nodeName() string
 	nodeID() int
-	setID(int)
-	parser(*registry, *idSet) (parser, error)
 	commitType() CommitType
-	// builder() builder
+	setID(int)
+	init(*registry) error
+	setIncludedBy(*registry, int, *idSet) error
+	parser(*registry, *idSet) (parser, error)
+	builder() builder
 }
 
 type parser interface {
 	nodeName() string
 	nodeID() int
-	setIncludedBy(parser, *idSet)
-	storeIncluded(*context, int, int) // can be just an id set, taking what's excluded from the context
 	parse(Trace, *context)
 }
 
 type builder interface {
 	nodeName() string
 	nodeID() int
-	build(*context) *Node
+	build(*context) ([]*Node, bool)
 }
 
 func parserNotFound(name string) error {
@@ -33,19 +33,69 @@ func cannotIncludeParsers(name string) error {
 	return fmt.Errorf("parser: %s cannot include other parsers", name)
 }
 
-func parse(t Trace, p parser, c *context) (*Node, error) {
+func intsContain(is []int, i int) bool {
+	for _, ii := range is {
+		if ii == i {
+			return true
+		}
+	}
+
+	return false
+}
+
+func appendIfMissing(is []int, i int) []int {
+	if intsContain(is, i) {
+		return is
+	}
+
+	return append(is, i)
+}
+
+func setItemsIncludedBy(r *registry, items []string, includedBy int, parsers *idSet) error {
+	for _, item := range items {
+		di, ok := r.definition(item)
+		if !ok {
+			return ErrNoParsersDefined
+		}
+
+		di.setIncludedBy(r, includedBy, parsers)
+	}
+
+	return nil
+}
+
+func sequenceItemNames(items []SequenceItem) []string {
+	names := make([]string, len(items))
+	for i := range items {
+		names[i] = items[i].Name
+	}
+
+	return names
+}
+
+func parse(t Trace, p parser, c *context) error {
 	p.parse(t, c)
 	if c.readErr != nil {
-		return nil, c.readErr
+		return c.readErr
 	}
 
 	if !c.match {
-		return nil, ErrInvalidInput
+		return ErrInvalidInput
 	}
 
-	if err := c.finalize(); err != nil {
+	if err := c.finalize(p); err != nil {
-		return nil, err
+		return err
 	}
 
-	return c.node, nil
+	return nil
+}
+
+func build(b builder, c *context) *Node {
+	c.offset = 0
+	n, ok := b.build(c)
+	if !ok || len(n) != 1 {
+		panic("damaged parse result")
+	}
+
+	return n[0]
 }

sequence.go

@@ -5,6 +5,9 @@ type sequenceDefinition struct {
 	id         int
 	commit     CommitType
 	items      []SequenceItem
+	includedBy []int
+	ranges     [][]int
+	sbuilder   *sequenceBuilder
 }
 
 type sequenceParser struct {
@@ -13,7 +16,16 @@ type sequenceParser struct {
 	commit     CommitType
 	items      []parser
 	ranges     [][]int
-	includedBy []parser
+	includedBy []int
+}
+
+type sequenceBuilder struct {
+	name       string
+	id         int
+	commit     CommitType
+	items      []builder
+	ranges     [][]int
+	includedBy []int
 }
 
 func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefinition {
@@ -27,8 +39,76 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi
 func (d *sequenceDefinition) nodeName() string       { return d.name }
 func (d *sequenceDefinition) nodeID() int            { return d.id }
 func (d *sequenceDefinition) setID(id int)           { d.id = id }
+func (d *sequenceDefinition) commitType() CommitType { return d.commit }
+
+func (d *sequenceDefinition) includeItems() bool {
+	return len(d.items) == 1 && d.items[0].Min == 1 && d.items[0].Max == 1
+}
+
+func (d *sequenceDefinition) init(r *registry) error {
+	if d.sbuilder == nil {
+		d.sbuilder = &sequenceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	for _, item := range d.items {
+		if item.Min == 0 && item.Max == 0 {
+			item.Min, item.Max = 1, 1
+		} else if item.Max == 0 {
+			item.Max = -1
+		}
+
+		d.ranges = append(d.ranges, []int{item.Min, item.Max})
+
+		def, ok := r.definition(item.Name)
+		if !ok {
+			return parserNotFound(item.Name)
+		}
+
+		d.sbuilder.items = append(d.sbuilder.items, def.builder())
+	}
+
+	d.sbuilder.ranges = d.ranges
+
+	if !d.includeItems() {
+		return nil
+	}
+
+	parsers := &idSet{}
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, sequenceItemNames(d.items), d.id, parsers)
+}
+
+func (d *sequenceDefinition) setIncludedBy(r *registry, includedBy int, parsers *idSet) error {
+	if parsers.has(d.id) {
+		return nil
+	}
+
+	d.includedBy = appendIfMissing(d.includedBy, includedBy)
+
+	if d.sbuilder == nil {
+		d.sbuilder = &sequenceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	d.sbuilder.includedBy = appendIfMissing(d.sbuilder.includedBy, includedBy)
+
+	if !d.includeItems() {
+		return nil
+	}
+
+	parsers.set(d.id)
+	return setItemsIncludedBy(r, sequenceItemNames(d.items), includedBy, parsers)
+}
 
 func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) {
+	// TODO: what is this for? test with sequence containing a sequence through a choice
 	if parsers.has(d.id) {
 		panic(cannotIncludeParsers(d.name))
 	}
@@ -42,28 +122,18 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error)
 		name:       d.name,
 		id:         d.id,
 		commit:     d.commit,
+		includedBy: d.includedBy,
 	}
 
 	r.setParser(sp)
 
-	var (
+	var items []parser
-		items  []parser
-		ranges [][]int
-	)
-
 	parsers.set(d.id)
 	defer parsers.unset(d.id)
 	for _, item := range d.items {
-		if item.Min == 0 && item.Max == 0 {
-			item.Min, item.Max = 1, 1
-		} else if item.Max == 0 {
-			item.Max = -1
-		}
-
 		pi, ok := r.parser(item.Name)
 		if ok {
 			items = append(items, pi)
-			ranges = append(ranges, []int{item.Min, item.Max})
 			continue
 		}
 
@@ -78,46 +148,28 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error)
 		}
 
 		items = append(items, pi)
-		ranges = append(ranges, []int{item.Min, item.Max})
-	}
-
-	// for single items, acts like a choice
-	if len(items) == 1 && ranges[0][0] == 1 && ranges[0][1] == 1 {
-		items[0].setIncludedBy(sp, parsers)
 	}
 
 	sp.items = items
-	sp.ranges = ranges
+	sp.ranges = d.ranges
 	return sp, nil
 }
 
-func (d *sequenceDefinition) commitType() CommitType {
+func (d *sequenceDefinition) builder() builder {
-	return d.commit
+	if d.sbuilder == nil {
+		d.sbuilder = &sequenceBuilder{
+			name:   d.name,
+			id:     d.id,
+			commit: d.commit,
+		}
+	}
+
+	return d.sbuilder
 }
 
 func (p *sequenceParser) nodeName() string { return p.name }
 func (p *sequenceParser) nodeID() int      { return p.id }
 
-func (p *sequenceParser) setIncludedBy(includedBy parser, parsers *idSet) {
-	if parsers.has(p.id) {
-		return
-	}
-
-	p.includedBy = append(p.includedBy, includedBy)
-}
-
-func (p *sequenceParser) storeIncluded(c *context, from, to int) {
-	if !c.excluded(from, p.id) {
-		return
-	}
-
-	c.store.setMatch(from, p.id, to)
-
-	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, from, to)
-	}
-}
-
 func (p *sequenceParser) parse(t Trace, c *context) {
 	if p.commit&Documentation != 0 {
 		c.fail(c.offset)
@@ -141,6 +193,7 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 	to := c.offset
 
 	for itemIndex < len(p.items) {
+		// TODO: is it ok to parse before max range check? what if max=0
 		p.items[itemIndex].parse(t, c)
 		if !c.match {
 			if currentCount < p.ranges[itemIndex][0] {
@@ -169,10 +222,68 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 	}
 
 	for _, includedBy := range p.includedBy {
-		includedBy.storeIncluded(c, from, to)
+		c.store.setMatch(from, includedBy, to)
 	}
 
 	c.store.setMatch(from, p.id, to)
 	c.success(to)
 	c.include(from, p.id)
 }
+
+func (b *sequenceBuilder) nodeName() string { return b.name }
+func (b *sequenceBuilder) nodeID() int      { return b.id }
+
+func (b *sequenceBuilder) build(c *context) ([]*Node, bool) {
+	to, ok := c.store.takeMatch(c.offset, b.id)
+	if !ok {
+		return nil, false
+	}
+
+	for _, ib := range b.includedBy {
+		c.store.takeMatchLength(c.offset, ib, to)
+	}
+
+	from := c.offset
+	var (
+		itemIndex    int
+		currentCount int
+		nodes        []*Node
+	)
+
+	for itemIndex < len(b.items) {
+		itemFrom := c.offset
+		n, ok := b.items[itemIndex].build(c)
+		if !ok {
+			if currentCount < b.ranges[itemIndex][0] {
+				panic(b.name + ": damaged parse result")
+			}
+
+			itemIndex++
+			currentCount = 0
+			continue
+		}
+
+		parsed := c.offset > itemFrom
+		if parsed {
+			nodes = append(nodes, n...)
+			currentCount++
+		}
+
+		if !parsed || b.ranges[itemIndex][1] >= 0 && currentCount == b.ranges[itemIndex][1] {
+			itemIndex++
+			currentCount = 0
+		}
+	}
+
+	if b.commit&Alias != 0 {
+		return nodes, true
+	}
+
+	return []*Node{{
+		Name:   b.name,
+		From:   from,
+		To:     to,
+		Nodes:  nodes,
+		tokens: c.tokens,
+	}}, true
+}

store.go

@@ -29,21 +29,89 @@ func (s *store) getMatch(offset, id int) (int, bool, bool) {
 
 	var (
 		found bool
-		length int
+		to    int
 	)
 
-	for i := 0; i < len(s.match[offset]); i++ {
+	for i := 0; i < len(s.match[offset]); i += 2 {
 		if s.match[offset][i] != id {
 			continue
 		}
 
 		found = true
-		if s.match[offset][i+1] > length {
+		if s.match[offset][i+1] > to {
-			length = s.match[offset][i+1]
+			to = s.match[offset][i+1]
 		}
 	}
 
-	return length, found, found
+	return to, found, found
+}
+
+func (s *store) takeMatch(offset, id int) (int, bool) {
+	if s.hasNoMatch(offset, id) {
+		return 0, false
+	}
+
+	if len(s.match) <= offset {
+		return 0, false
+	}
+
+	var (
+		found bool
+		to    int
+		index int
+	)
+
+	for i := 0; i < len(s.match[offset]); i += 2 {
+		if s.match[offset][i] != id {
+			continue
+		}
+
+		found = true
+		if s.match[offset][i+1] > to {
+			to = s.match[offset][i+1]
+			index = i
+		}
+	}
+
+	if found {
+		s.match[offset][index] = -1
+	}
+
+	return to, found
+}
+
+func (s *store) takeMatchLength(offset, id, to int) (int, bool) {
+	if s.hasNoMatch(offset, id) {
+		return 0, false
+	}
+
+	if len(s.match) <= offset {
+		return 0, false
+	}
+
+	var (
+		found bool
+		// index int
+	)
+
+	for i := 0; i < len(s.match[offset]); i += 2 {
+		if s.match[offset][i] != id {
+			continue
+		}
+
+		found = true
+		if s.match[offset][i+1] == to {
+			s.match[offset][i] = -1
+			return to, true
+			//eindex = i
+		}
+	}
+
+	if found {
+		// s.match[offset][index] = -1
+	}
+
+	return to, found
 }
 
 func (s *store) ensureOffset(offset int) {
@@ -57,12 +125,16 @@ func (s *store) ensureOffset(offset int) {
 	}
 
 	s.match = s.match[:cap(s.match)]
-	for i := cap(s.match); i <= offset; i++ {
+	for i := len(s.match); i <= offset; i++ {
 		s.match = append(s.match, nil)
 	}
 }
 
 func (s *store) setMatch(offset, id, to int) {
+	if toe, match, ok := s.getMatch(offset, id); ok && match && toe == to {
+		return
+	}
+
 	s.ensureOffset(offset)
 	s.match[offset] = append(s.match[offset], id, to)
 }

syntax.go

@@ -29,6 +29,7 @@ type Syntax struct {
 	explicitRoot bool
 	root         definition
 	parser       parser
+	builder      builder
 }
 
 var (
@@ -88,7 +89,7 @@ func childName(name string, childIndex int) string {
 
 func (s *Syntax) Class(name string, ct CommitType, not bool, chars []rune, ranges [][]rune) error {
 	cname := childName(name, 0)
-	if err := s.register(newChar(cname, Alias, not, chars, ranges)); err != nil {
+	if err := s.register(newChar(cname, not, chars, ranges)); err != nil {
 		return err
 	}
 
@@ -100,7 +101,7 @@ func (s *Syntax) CharSequence(name string, ct CommitType, chars []rune) error {
 	for i, ci := range chars {
 		ref := childName(name, i)
 		refs = append(refs, ref)
-		if err := s.register(newChar(ref, Alias, false, []rune{ci}, nil)); err != nil {
+		if err := s.register(newChar(ref, false, []rune{ci}, nil)); err != nil {
 			return err
 		}
 	}
@@ -143,6 +144,10 @@ func (s *Syntax) Init() error {
 		return ErrRootAlias
 	}
 
+	for _, p := range s.registry.definitions {
+		p.init(s.registry)
+	}
+
 	var err error
 	s.parser, err = s.root.parser(s.registry, &idSet{})
 	if err != nil {
@@ -150,6 +155,7 @@ func (s *Syntax) Init() error {
 		return err
 	}
 
+	s.builder = s.root.builder()
 	s.initialized = true
 	return nil
 }
@@ -170,5 +176,9 @@ func (s *Syntax) Parse(r io.Reader) (*Node, error) {
 	}
 
 	c := newContext(bufio.NewReader(r))
-	return parse(s.trace, s.parser, c)
+	if err := parse(s.trace, s.parser, c); err != nil {
+		return nil, err
+	}
+
+	return build(s.builder, c), nil
 }