From 94a21ae7550ec1ec8f5b84a03cca353814cc9dfd Mon Sep 17 00:00:00 2001
From: Arpad Ryszka <arpad@squareroundforest.org>
Date: Sat, 15 Jul 2017 21:49:08 +0200
Subject: [PATCH] track parsers by numeric id

---
 boot.go        |  2 +-
 boot_test.go   |  2 +-
 bootsyntax.go  |  2 +-
 char.go        | 15 +++++++++------
 choice.go      | 36 +++++++++++++++++++++---------------
 context.go     | 33 ++++++++++++++++-----------------
 define.go      |  2 +-
 eskip_test.go  |  2 +-
 idset.go       | 45 +++++++++++++++++++++++++++++++++++++++++++++
 json_test.go   |  2 +-
 keyval_test.go |  2 +-
 mml_test.go    |  2 +-
 node.go        |  6 ++++--
 parse.go       |  7 +++++--
 parse_test.go  |  2 +-
 registry.go    | 13 ++++++++++++-
 scheme_test.go |  2 +-
 sequence.go    | 34 ++++++++++++++++++++--------------
 sexpr_test.go  |  2 +-
 store.go       |  2 +-
 syntax.go      | 20 ++++++++++----------
 trace.go       |  2 +-
 22 files changed, 155 insertions(+), 80 deletions(-)
 create mode 100644 idset.go

diff --git a/boot.go b/boot.go
index b240857..6e21524 100644
--- a/boot.go
+++ b/boot.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"errors"
diff --git a/boot_test.go b/boot_test.go
index 5733efe..f92f347 100644
--- a/boot_test.go
+++ b/boot_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"os"
diff --git a/bootsyntax.go b/bootsyntax.go
index 44471d2..8d86c1f 100644
--- a/bootsyntax.go
+++ b/bootsyntax.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 var bootSyntaxDefs = [][]string{{
 	"chars", "space", "alias", " ",
diff --git a/char.go b/char.go
index e5dcbb9..f60cdd6 100644
--- a/char.go
+++ b/char.go
@@ -1,7 +1,8 @@
-package parse
+package treerack
 
 type charParser struct {
 	name       string
+	id         int
 	commit     CommitType
 	not        bool
 	chars      []rune
@@ -26,9 +27,11 @@ func newChar(
 }
 
 func (p *charParser) nodeName() string { return p.name }
+func (p *charParser) nodeID() int      { return p.id }
+func (p *charParser) setID(id int)     { p.id = p.id }
 
-func (p *charParser) parser(r *registry, path []string) (parser, error) {
-	if stringsContain(path, p.name) {
+func (p *charParser) parser(r *registry, parsers []string) (parser, error) {
+	if stringsContainDeprecated(parsers, p.name) {
 		panic(cannotIncludeParsers(p.name))
 	}
 
@@ -44,8 +47,8 @@ func (p *charParser) commitType() CommitType {
 	return p.commit
 }
 
-func (p *charParser) setIncludedBy(includedBy parser, path []string) {
-	if stringsContain(path, p.name) {
+func (p *charParser) setIncludedBy(includedBy parser, parsers []string) {
+	if stringsContainDeprecated(parsers, p.name) {
 		panic(cannotIncludeParsers(p.name))
 	}
 
@@ -89,7 +92,7 @@ func (p *charParser) parse(t Trace, c *context) {
 
 	if tok, ok := c.token(); ok && p.match(tok) {
 		// t.Out1("success", string(tok))
-		n := newNode(p.name, c.offset, c.offset+1, p.commit)
+		n := newNode(p.name, p.id, c.offset, c.offset+1, p.commit)
 		c.store.set(c.offset, p.name, n)
 		for _, includedBy := range p.includedBy {
 			includedBy.storeIncluded(c, n)
diff --git a/choice.go b/choice.go
index 4b49cee..858cf3a 100644
--- a/choice.go
+++ b/choice.go
@@ -1,13 +1,15 @@
-package parse
+package treerack
 
 type choiceDefinition struct {
 	name     string
+	id       int
 	commit   CommitType
 	elements []string
 }
 
 type choiceParser struct {
 	name       string
+	id         int
 	commit     CommitType
 	elements   []parser
 	includedBy []parser
@@ -22,8 +24,10 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition
 }
 
 func (d *choiceDefinition) nodeName() string { return d.name }
+func (d *choiceDefinition) nodeID() int      { return d.id }
+func (d *choiceDefinition) setID(id int)     { d.id = id }
 
-func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
+func (d *choiceDefinition) parser(r *registry, parsers []string) (parser, error) {
 	p, ok := r.parser(d.name)
 	if ok {
 		return p, nil
@@ -31,18 +35,19 @@ func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
 
 	cp := &choiceParser{
 		name:   d.name,
+		id:     d.id,
 		commit: d.commit,
 	}
 
 	r.setParser(cp)
 
 	var elements []parser
-	path = append(path, d.name)
+	parsers = append(parsers, d.name)
 	for _, e := range d.elements {
 		element, ok := r.parser(e)
 		if ok {
 			elements = append(elements, element)
-			element.setIncludedBy(cp, path)
+			element.setIncludedBy(cp, parsers)
 			continue
 		}
 
@@ -51,12 +56,12 @@ func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
 			return nil, parserNotFound(e)
 		}
 
-		element, err := elementDefinition.parser(r, path)
+		element, err := elementDefinition.parser(r, parsers)
 		if err != nil {
 			return nil, err
 		}
 
-		element.setIncludedBy(cp, path)
+		element.setIncludedBy(cp, parsers)
 		elements = append(elements, element)
 	}
 
@@ -69,9 +74,10 @@ func (d *choiceDefinition) commitType() CommitType {
 }
 
 func (p *choiceParser) nodeName() string { return p.name }
+func (p *choiceParser) nodeID() int      { return p.id }
 
-func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
-	if stringsContain(path, p.name) {
+func (p *choiceParser) setIncludedBy(includedBy parser, parsers []string) {
+	if stringsContainDeprecated(parsers, p.name) {
 		return
 	}
 
@@ -79,11 +85,11 @@ func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
 }
 
 func (p *choiceParser) storeIncluded(c *context, n *Node) {
-	if !c.excluded(n.From, p.name) {
+	if !c.excluded(n.From, p.id) {
 		return
 	}
 
-	nc := newNode(p.name, n.From, n.To, p.commit)
+	nc := newNode(p.name, p.id, n.From, n.To, p.commit)
 	nc.append(n)
 	c.store.set(nc.From, p.name, nc)
 
@@ -107,16 +113,16 @@ func (p *choiceParser) parse(t Trace, c *context) {
 		return
 	}
 
-	if c.excluded(c.offset, p.name) {
+	if c.excluded(c.offset, p.id) {
 		// t.Out1("excluded")
 		c.fail(c.offset)
 		return
 	}
 
-	c.exclude(c.offset, p.name)
-	defer c.include(c.offset, p.name)
+	c.exclude(c.offset, p.id)
+	defer c.include(c.offset, p.id) // TODO: test if can be optimized
 
-	node := newNode(p.name, c.offset, c.offset, p.commit)
+	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
 	var match bool
 
 	for {
@@ -134,7 +140,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
 
 			match = true
 			foundMatch = true
-			node = newNode(p.name, c.offset, c.offset, p.commit)
+			node = newNode(p.name, p.id, c.offset, c.offset, p.commit)
 			node.append(c.node)
 
 			c.store.set(node.From, p.name, node)
diff --git a/context.go b/context.go
index 30e80da..a3c99b5 100644
--- a/context.go
+++ b/context.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"io"
@@ -15,7 +15,7 @@ type context struct {
 	tokens     []rune
 	match      bool
 	node       *Node
-	isExcluded [][]string
+	isExcluded []*idSet
 }
 
 func newContext(r io.RuneReader) *context {
@@ -64,16 +64,16 @@ func (c *context) token() (rune, bool) {
 	return c.tokens[c.offset], true
 }
 
-func (c *context) excluded(offset int, name string) bool {
-	if len(c.isExcluded) <= offset {
+func (c *context) excluded(offset int, id int) bool {
+	if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
 		return false
 	}
 
-	return stringsContain(c.isExcluded[offset], name)
+	return c.isExcluded[offset].has(id)
 }
 
-func (c *context) exclude(offset int, name string) {
-	if c.excluded(offset, name) {
+func (c *context) exclude(offset int, id int) {
+	if c.excluded(offset, id) {
 		return
 	}
 
@@ -84,25 +84,24 @@ func (c *context) exclude(offset int, name string) {
 		} else {
 			c.isExcluded = append(
 				c.isExcluded[:cap(c.isExcluded)],
-				make([][]string, offset+1-cap(c.isExcluded))...,
+				make([]*idSet, offset+1-cap(c.isExcluded))...,
 			)
 		}
 	}
 
-	c.isExcluded[offset] = append(c.isExcluded[offset], name)
+	if c.isExcluded[offset] == nil {
+		c.isExcluded[offset] = &idSet{}
+	}
+
+	c.isExcluded[offset].set(id)
 }
 
-func (c *context) include(offset int, name string) {
-	if len(c.isExcluded) <= offset {
+func (c *context) include(offset int, id int) {
+	if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
 		return
 	}
 
-	for i := len(c.isExcluded[offset]) - 1; i >= 0; i-- {
-		if c.isExcluded[offset][i] == name {
-			c.isExcluded[offset] = append(c.isExcluded[offset][:i], c.isExcluded[offset][i+1:]...)
-			return
-		}
-	}
+	c.isExcluded[offset].unset(id)
 }
 
 func (c *context) fromStore(name string) (bool, bool) {
diff --git a/define.go b/define.go
index b53e8e5..78ed61e 100644
--- a/define.go
+++ b/define.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "strconv"
 
diff --git a/eskip_test.go b/eskip_test.go
index c5e3340..bb23efe 100644
--- a/eskip_test.go
+++ b/eskip_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bytes"
diff --git a/idset.go b/idset.go
new file mode 100644
index 0000000..f9f1237
--- /dev/null
+++ b/idset.go
@@ -0,0 +1,45 @@
+package treerack
+
+import "strconv"
+
+type idSet struct {
+	ids []uint
+}
+
+func divModBits(id int) (int, int) {
+	return id / strconv.IntSize, id % strconv.IntSize
+}
+
+func (s *idSet) set(id int) {
+	d, m := divModBits(id)
+	if d >= len(s.ids) {
+		if d < cap(s.ids) {
+			s.ids = s.ids[:d+1]
+		} else {
+			s.ids = s.ids[:cap(s.ids)]
+			for i := cap(s.ids); i <= d; i++ {
+				s.ids = append(s.ids, 0)
+			}
+		}
+	}
+
+	s.ids[d] |= 1 << uint(m)
+}
+
+func (s *idSet) unset(id int) {
+	d, m := divModBits(id)
+	if d >= len(s.ids) {
+		return
+	}
+
+	s.ids[d] &^= 1 << uint(m)
+}
+
+func (s *idSet) has(id int) bool {
+	d, m := divModBits(id)
+	if d >= len(s.ids) {
+		return false
+	}
+
+	return s.ids[d]&(1<<uint(m)) != 0
+}
diff --git a/json_test.go b/json_test.go
index b5279e4..0610a8e 100644
--- a/json_test.go
+++ b/json_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bytes"
diff --git a/keyval_test.go b/keyval_test.go
index 4198c30..2827561 100644
--- a/keyval_test.go
+++ b/keyval_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 
diff --git a/mml_test.go b/mml_test.go
index ceb7df5..9bae088 100644
--- a/mml_test.go
+++ b/mml_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 
diff --git a/node.go b/node.go
index 539e775..a657d01 100644
--- a/node.go
+++ b/node.go
@@ -1,18 +1,20 @@
-package parse
+package treerack
 
 import "fmt"
 
 type Node struct {
 	Name       string
+	id         int
 	Nodes      []*Node
 	From, To   int
 	commitType CommitType
 	tokens     []rune
 }
 
-func newNode(name string, from, to int, ct CommitType) *Node {
+func newNode(name string, id int, from, to int, ct CommitType) *Node {
 	return &Node{
 		Name:       name,
+		id:         id,
 		From:       from,
 		To:         to,
 		commitType: ct,
diff --git a/parse.go b/parse.go
index 8140e99..d77c31a 100644
--- a/parse.go
+++ b/parse.go
@@ -1,15 +1,18 @@
-package parse
+package treerack
 
 import "fmt"
 
 type definition interface {
 	nodeName() string
+	nodeID() int
+	setID(int)
 	parser(*registry, []string) (parser, error)
 	commitType() CommitType
 }
 
 type parser interface {
 	nodeName() string
+	nodeID() int
 	setIncludedBy(parser, []string)
 	storeIncluded(*context, *Node)
 	parse(Trace, *context)
@@ -23,7 +26,7 @@ func cannotIncludeParsers(name string) error {
 	return fmt.Errorf("parser: %s cannot include other parsers", name)
 }
 
-func stringsContain(ss []string, s string) bool {
+func stringsContainDeprecated(ss []string, s string) bool {
 	for _, si := range ss {
 		if si == s {
 			return true
diff --git a/parse_test.go b/parse_test.go
index b88f752..b68a43e 100644
--- a/parse_test.go
+++ b/parse_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bytes"
diff --git a/registry.go b/registry.go
index 09160ad..497e348 100644
--- a/registry.go
+++ b/registry.go
@@ -1,12 +1,17 @@
-package parse
+package treerack
 
 type registry struct {
+	idSeed      int
+	ids         map[string]int
+	names       map[int]string
 	definitions map[string]definition
 	parsers     map[string]parser
 }
 
 func newRegistry() *registry {
 	return &registry{
+		ids:         make(map[string]int),
+		names:       make(map[int]string),
 		definitions: make(map[string]definition),
 		parsers:     make(map[string]parser),
 	}
@@ -27,6 +32,12 @@ func (r *registry) setDefinition(d definition) error {
 		return duplicateDefinition(d.nodeName())
 	}
 
+	r.idSeed++
+	id := r.idSeed
+	d.setID(id)
+	r.ids[d.nodeName()] = id
+	r.names[id] = d.nodeName()
+
 	r.definitions[d.nodeName()] = d
 	return nil
 }
diff --git a/scheme_test.go b/scheme_test.go
index 15f07a3..7ffd837 100644
--- a/scheme_test.go
+++ b/scheme_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 
diff --git a/sequence.go b/sequence.go
index f334f5a..f96f764 100644
--- a/sequence.go
+++ b/sequence.go
@@ -1,13 +1,15 @@
-package parse
+package treerack
 
 type sequenceDefinition struct {
 	name   string
+	id     int
 	commit CommitType
 	items  []SequenceItem
 }
 
 type sequenceParser struct {
 	name       string
+	id         int
 	commit     CommitType
 	items      []parser
 	ranges     [][]int
@@ -23,9 +25,11 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi
 }
 
 func (d *sequenceDefinition) nodeName() string { return d.name }
+func (d *sequenceDefinition) nodeID() int      { return d.id }
+func (d *sequenceDefinition) setID(id int)     { d.id = id }
 
-func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error) {
-	if stringsContain(path, d.name) {
+func (d *sequenceDefinition) parser(r *registry, parsers []string) (parser, error) {
+	if stringsContainDeprecated(parsers, d.name) {
 		panic(cannotIncludeParsers(d.name))
 	}
 
@@ -36,6 +40,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 
 	sp := &sequenceParser{
 		name:   d.name,
+		id:     d.id,
 		commit: d.commit,
 	}
 
@@ -46,7 +51,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 		ranges [][]int
 	)
 
-	path = append(path, d.name)
+	parsers = append(parsers, d.name)
 	for _, item := range d.items {
 		if item.Min == 0 && item.Max == 0 {
 			item.Min, item.Max = 1, 1
@@ -66,7 +71,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 			return nil, parserNotFound(item.Name)
 		}
 
-		pi, err := itemDefinition.parser(r, path)
+		pi, err := itemDefinition.parser(r, parsers)
 		if err != nil {
 			return nil, err
 		}
@@ -77,7 +82,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 
 	// for single items, acts like a choice
 	if len(items) == 1 && ranges[0][0] == 1 && ranges[0][1] == 1 {
-		items[0].setIncludedBy(sp, path)
+		items[0].setIncludedBy(sp, parsers)
 	}
 
 	sp.items = items
@@ -90,9 +95,10 @@ func (d *sequenceDefinition) commitType() CommitType {
 }
 
 func (p *sequenceParser) nodeName() string { return p.name }
+func (p *sequenceParser) nodeID() int      { return p.id }
 
-func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
-	if stringsContain(path, p.name) {
+func (p *sequenceParser) setIncludedBy(includedBy parser, parsers []string) {
+	if stringsContainDeprecated(parsers, p.name) {
 		return
 	}
 
@@ -100,11 +106,11 @@ func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
 }
 
 func (p *sequenceParser) storeIncluded(c *context, n *Node) {
-	if !c.excluded(n.From, p.name) {
+	if !c.excluded(n.From, p.id) {
 		return
 	}
 
-	nc := newNode(p.name, n.From, n.To, p.commit)
+	nc := newNode(p.name, p.id, n.From, n.To, p.commit)
 	nc.append(n)
 	c.store.set(nc.From, p.name, nc)
 
@@ -123,19 +129,19 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 		return
 	}
 
-	if c.excluded(c.offset, p.name) {
+	if c.excluded(c.offset, p.id) {
 		// t.Out1("excluded")
 		c.fail(c.offset)
 		return
 	}
 
-	c.exclude(c.offset, p.name)
-	defer c.include(c.offset, p.name)
+	c.exclude(c.offset, p.id)
+	defer c.include(c.offset, p.id)
 
 	items := p.items
 	ranges := p.ranges
 	var currentCount int
-	node := newNode(p.name, c.offset, c.offset, p.commit)
+	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
 
 	for len(items) > 0 {
 		m, ok := c.fromStore(items[0].nodeName())
diff --git a/sexpr_test.go b/sexpr_test.go
index 67c16c2..e41f385 100644
--- a/sexpr_test.go
+++ b/sexpr_test.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 
diff --git a/store.go b/store.go
index 473a2b4..6ca99b8 100644
--- a/store.go
+++ b/store.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 type storedItem struct {
 	name string
diff --git a/syntax.go b/syntax.go
index 190cb59..c07a6c3 100644
--- a/syntax.go
+++ b/syntax.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bufio"
@@ -22,13 +22,13 @@ type SequenceItem struct {
 }
 
 type Syntax struct {
-	trace       Trace
-	registry    *registry
-	initialized bool
-	initFailed  bool
-	rootSet     bool
-	root        definition
-	parser      parser
+	trace        Trace
+	registry     *registry
+	initialized  bool
+	initFailed   bool
+	explicitRoot bool
+	root         definition
+	parser       parser
 }
 
 var (
@@ -70,8 +70,8 @@ func (s *Syntax) register(d definition) error {
 
 	if d.commitType()&Root != 0 {
 		s.root = d
-		s.rootSet = true
-	} else if !s.rootSet {
+		s.explicitRoot = true
+	} else if !s.explicitRoot {
 		s.root = d
 	}
 
diff --git a/trace.go b/trace.go
index 223a6a6..a8a1de2 100644
--- a/trace.go
+++ b/trace.go
@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"fmt"