track parsers by numeric id

boot.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"errors"

boot_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"os"

bootsyntax.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 var bootSyntaxDefs = [][]string{{
 	"chars", "space", "alias", " ",

char.go

@@ -1,7 +1,8 @@
-package parse
+package treerack
 
 type charParser struct {
 	name       string
+	id         int
 	commit     CommitType
 	not        bool
 	chars      []rune
@@ -26,9 +27,11 @@ func newChar(
 }
 
 func (p *charParser) nodeName() string { return p.name }
+func (p *charParser) nodeID() int      { return p.id }
+func (p *charParser) setID(id int)     { p.id = p.id }
 
-func (p *charParser) parser(r *registry, path []string) (parser, error) {
+func (p *charParser) parser(r *registry, parsers []string) (parser, error) {
-	if stringsContain(path, p.name) {
+	if stringsContainDeprecated(parsers, p.name) {
 		panic(cannotIncludeParsers(p.name))
 	}
 
@@ -44,8 +47,8 @@ func (p *charParser) commitType() CommitType {
 	return p.commit
 }
 
-func (p *charParser) setIncludedBy(includedBy parser, path []string) {
+func (p *charParser) setIncludedBy(includedBy parser, parsers []string) {
-	if stringsContain(path, p.name) {
+	if stringsContainDeprecated(parsers, p.name) {
 		panic(cannotIncludeParsers(p.name))
 	}
 
@@ -89,7 +92,7 @@ func (p *charParser) parse(t Trace, c *context) {
 
 	if tok, ok := c.token(); ok && p.match(tok) {
 		// t.Out1("success", string(tok))
-		n := newNode(p.name, c.offset, c.offset+1, p.commit)
+		n := newNode(p.name, p.id, c.offset, c.offset+1, p.commit)
 		c.store.set(c.offset, p.name, n)
 		for _, includedBy := range p.includedBy {
 			includedBy.storeIncluded(c, n)

choice.go

@@ -1,13 +1,15 @@
-package parse
+package treerack
 
 type choiceDefinition struct {
 	name     string
+	id       int
 	commit   CommitType
 	elements []string
 }
 
 type choiceParser struct {
 	name       string
+	id         int
 	commit     CommitType
 	elements   []parser
 	includedBy []parser
@@ -22,8 +24,10 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition
 }
 
 func (d *choiceDefinition) nodeName() string { return d.name }
+func (d *choiceDefinition) nodeID() int      { return d.id }
+func (d *choiceDefinition) setID(id int)     { d.id = id }
 
-func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
+func (d *choiceDefinition) parser(r *registry, parsers []string) (parser, error) {
 	p, ok := r.parser(d.name)
 	if ok {
 		return p, nil
@@ -31,18 +35,19 @@ func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
 
 	cp := &choiceParser{
 		name:   d.name,
+		id:     d.id,
 		commit: d.commit,
 	}
 
 	r.setParser(cp)
 
 	var elements []parser
-	path = append(path, d.name)
+	parsers = append(parsers, d.name)
 	for _, e := range d.elements {
 		element, ok := r.parser(e)
 		if ok {
 			elements = append(elements, element)
-			element.setIncludedBy(cp, path)
+			element.setIncludedBy(cp, parsers)
 			continue
 		}
 
@@ -51,12 +56,12 @@ func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
 			return nil, parserNotFound(e)
 		}
 
-		element, err := elementDefinition.parser(r, path)
+		element, err := elementDefinition.parser(r, parsers)
 		if err != nil {
 			return nil, err
 		}
 
-		element.setIncludedBy(cp, path)
+		element.setIncludedBy(cp, parsers)
 		elements = append(elements, element)
 	}
 
@@ -69,9 +74,10 @@ func (d *choiceDefinition) commitType() CommitType {
 }
 
 func (p *choiceParser) nodeName() string { return p.name }
+func (p *choiceParser) nodeID() int      { return p.id }
 
-func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
+func (p *choiceParser) setIncludedBy(includedBy parser, parsers []string) {
-	if stringsContain(path, p.name) {
+	if stringsContainDeprecated(parsers, p.name) {
 		return
 	}
 
@@ -79,11 +85,11 @@ func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
 }
 
 func (p *choiceParser) storeIncluded(c *context, n *Node) {
-	if !c.excluded(n.From, p.name) {
+	if !c.excluded(n.From, p.id) {
 		return
 	}
 
-	nc := newNode(p.name, n.From, n.To, p.commit)
+	nc := newNode(p.name, p.id, n.From, n.To, p.commit)
 	nc.append(n)
 	c.store.set(nc.From, p.name, nc)
 
@@ -107,16 +113,16 @@ func (p *choiceParser) parse(t Trace, c *context) {
 		return
 	}
 
-	if c.excluded(c.offset, p.name) {
+	if c.excluded(c.offset, p.id) {
 		// t.Out1("excluded")
 		c.fail(c.offset)
 		return
 	}
 
-	c.exclude(c.offset, p.name)
+	c.exclude(c.offset, p.id)
-	defer c.include(c.offset, p.name)
+	defer c.include(c.offset, p.id) // TODO: test if can be optimized
 
-	node := newNode(p.name, c.offset, c.offset, p.commit)
+	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
 	var match bool
 
 	for {
@@ -134,7 +140,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
 
 			match = true
 			foundMatch = true
-			node = newNode(p.name, c.offset, c.offset, p.commit)
+			node = newNode(p.name, p.id, c.offset, c.offset, p.commit)
 			node.append(c.node)
 
 			c.store.set(node.From, p.name, node)

context.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"io"
@@ -15,7 +15,7 @@ type context struct {
 	tokens     []rune
 	match      bool
 	node       *Node
-	isExcluded [][]string
+	isExcluded []*idSet
 }
 
 func newContext(r io.RuneReader) *context {
@@ -64,16 +64,16 @@ func (c *context) token() (rune, bool) {
 	return c.tokens[c.offset], true
 }
 
-func (c *context) excluded(offset int, name string) bool {
+func (c *context) excluded(offset int, id int) bool {
-	if len(c.isExcluded) <= offset {
+	if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
 		return false
 	}
 
-	return stringsContain(c.isExcluded[offset], name)
+	return c.isExcluded[offset].has(id)
 }
 
-func (c *context) exclude(offset int, name string) {
+func (c *context) exclude(offset int, id int) {
-	if c.excluded(offset, name) {
+	if c.excluded(offset, id) {
 		return
 	}
 
@@ -84,25 +84,24 @@ func (c *context) exclude(offset int, name string) {
 		} else {
 			c.isExcluded = append(
 				c.isExcluded[:cap(c.isExcluded)],
-				make([][]string, offset+1-cap(c.isExcluded))...,
+				make([]*idSet, offset+1-cap(c.isExcluded))...,
 			)
 		}
 	}
 
-	c.isExcluded[offset] = append(c.isExcluded[offset], name)
+	if c.isExcluded[offset] == nil {
+		c.isExcluded[offset] = &idSet{}
+	}
+
+	c.isExcluded[offset].set(id)
 }
 
-func (c *context) include(offset int, name string) {
+func (c *context) include(offset int, id int) {
-	if len(c.isExcluded) <= offset {
+	if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
 		return
 	}
 
-	for i := len(c.isExcluded[offset]) - 1; i >= 0; i-- {
+	c.isExcluded[offset].unset(id)
-		if c.isExcluded[offset][i] == name {
-			c.isExcluded[offset] = append(c.isExcluded[offset][:i], c.isExcluded[offset][i+1:]...)
-			return
-		}
-	}
 }
 
 func (c *context) fromStore(name string) (bool, bool) {

define.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "strconv"
 

eskip_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bytes"

idset.go

@@ -0,0 +1,45 @@
+package treerack
+
+import "strconv"
+
+type idSet struct {
+	ids []uint
+}
+
+func divModBits(id int) (int, int) {
+	return id / strconv.IntSize, id % strconv.IntSize
+}
+
+func (s *idSet) set(id int) {
+	d, m := divModBits(id)
+	if d >= len(s.ids) {
+		if d < cap(s.ids) {
+			s.ids = s.ids[:d+1]
+		} else {
+			s.ids = s.ids[:cap(s.ids)]
+			for i := cap(s.ids); i <= d; i++ {
+				s.ids = append(s.ids, 0)
+			}
+		}
+	}
+
+	s.ids[d] |= 1 << uint(m)
+}
+
+func (s *idSet) unset(id int) {
+	d, m := divModBits(id)
+	if d >= len(s.ids) {
+		return
+	}
+
+	s.ids[d] &^= 1 << uint(m)
+}
+
+func (s *idSet) has(id int) bool {
+	d, m := divModBits(id)
+	if d >= len(s.ids) {
+		return false
+	}
+
+	return s.ids[d]&(1<<uint(m)) != 0
+}

json_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bytes"

keyval_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 

mml_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 

node.go

@@ -1,18 +1,20 @@
-package parse
+package treerack
 
 import "fmt"
 
 type Node struct {
 	Name       string
+	id         int
 	Nodes      []*Node
 	From, To   int
 	commitType CommitType
 	tokens     []rune
 }
 
-func newNode(name string, from, to int, ct CommitType) *Node {
+func newNode(name string, id int, from, to int, ct CommitType) *Node {
 	return &Node{
 		Name:       name,
+		id:         id,
 		From:       from,
 		To:         to,
 		commitType: ct,

parse.go

@@ -1,15 +1,18 @@
-package parse
+package treerack
 
 import "fmt"
 
 type definition interface {
 	nodeName() string
+	nodeID() int
+	setID(int)
 	parser(*registry, []string) (parser, error)
 	commitType() CommitType
 }
 
 type parser interface {
 	nodeName() string
+	nodeID() int
 	setIncludedBy(parser, []string)
 	storeIncluded(*context, *Node)
 	parse(Trace, *context)
@@ -23,7 +26,7 @@ func cannotIncludeParsers(name string) error {
 	return fmt.Errorf("parser: %s cannot include other parsers", name)
 }
 
-func stringsContain(ss []string, s string) bool {
+func stringsContainDeprecated(ss []string, s string) bool {
 	for _, si := range ss {
 		if si == s {
 			return true

parse_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bytes"

registry.go

@@ -1,12 +1,17 @@
-package parse
+package treerack
 
 type registry struct {
+	idSeed      int
+	ids         map[string]int
+	names       map[int]string
 	definitions map[string]definition
 	parsers     map[string]parser
 }
 
 func newRegistry() *registry {
 	return &registry{
+		ids:         make(map[string]int),
+		names:       make(map[int]string),
 		definitions: make(map[string]definition),
 		parsers:     make(map[string]parser),
 	}
@@ -27,6 +32,12 @@ func (r *registry) setDefinition(d definition) error {
 		return duplicateDefinition(d.nodeName())
 	}
 
+	r.idSeed++
+	id := r.idSeed
+	d.setID(id)
+	r.ids[d.nodeName()] = id
+	r.names[id] = d.nodeName()
+
 	r.definitions[d.nodeName()] = d
 	return nil
 }

scheme_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 

sequence.go

@@ -1,13 +1,15 @@
-package parse
+package treerack
 
 type sequenceDefinition struct {
 	name   string
+	id     int
 	commit CommitType
 	items  []SequenceItem
 }
 
 type sequenceParser struct {
 	name       string
+	id         int
 	commit     CommitType
 	items      []parser
 	ranges     [][]int
@@ -23,9 +25,11 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi
 }
 
 func (d *sequenceDefinition) nodeName() string { return d.name }
+func (d *sequenceDefinition) nodeID() int      { return d.id }
+func (d *sequenceDefinition) setID(id int)     { d.id = id }
 
-func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error) {
+func (d *sequenceDefinition) parser(r *registry, parsers []string) (parser, error) {
-	if stringsContain(path, d.name) {
+	if stringsContainDeprecated(parsers, d.name) {
 		panic(cannotIncludeParsers(d.name))
 	}
 
@@ -36,6 +40,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 
 	sp := &sequenceParser{
 		name:   d.name,
+		id:     d.id,
 		commit: d.commit,
 	}
 
@@ -46,7 +51,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 		ranges [][]int
 	)
 
-	path = append(path, d.name)
+	parsers = append(parsers, d.name)
 	for _, item := range d.items {
 		if item.Min == 0 && item.Max == 0 {
 			item.Min, item.Max = 1, 1
@@ -66,7 +71,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 			return nil, parserNotFound(item.Name)
 		}
 
-		pi, err := itemDefinition.parser(r, path)
+		pi, err := itemDefinition.parser(r, parsers)
 		if err != nil {
 			return nil, err
 		}
@@ -77,7 +82,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
 
 	// for single items, acts like a choice
 	if len(items) == 1 && ranges[0][0] == 1 && ranges[0][1] == 1 {
-		items[0].setIncludedBy(sp, path)
+		items[0].setIncludedBy(sp, parsers)
 	}
 
 	sp.items = items
@@ -90,9 +95,10 @@ func (d *sequenceDefinition) commitType() CommitType {
 }
 
 func (p *sequenceParser) nodeName() string { return p.name }
+func (p *sequenceParser) nodeID() int      { return p.id }
 
-func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
+func (p *sequenceParser) setIncludedBy(includedBy parser, parsers []string) {
-	if stringsContain(path, p.name) {
+	if stringsContainDeprecated(parsers, p.name) {
 		return
 	}
 
@@ -100,11 +106,11 @@ func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
 }
 
 func (p *sequenceParser) storeIncluded(c *context, n *Node) {
-	if !c.excluded(n.From, p.name) {
+	if !c.excluded(n.From, p.id) {
 		return
 	}
 
-	nc := newNode(p.name, n.From, n.To, p.commit)
+	nc := newNode(p.name, p.id, n.From, n.To, p.commit)
 	nc.append(n)
 	c.store.set(nc.From, p.name, nc)
 
@@ -123,19 +129,19 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 		return
 	}
 
-	if c.excluded(c.offset, p.name) {
+	if c.excluded(c.offset, p.id) {
 		// t.Out1("excluded")
 		c.fail(c.offset)
 		return
 	}
 
-	c.exclude(c.offset, p.name)
+	c.exclude(c.offset, p.id)
-	defer c.include(c.offset, p.name)
+	defer c.include(c.offset, p.id)
 
 	items := p.items
 	ranges := p.ranges
 	var currentCount int
-	node := newNode(p.name, c.offset, c.offset, p.commit)
+	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
 
 	for len(items) > 0 {
 		m, ok := c.fromStore(items[0].nodeName())

sexpr_test.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import "testing"
 

store.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 type storedItem struct {
 	name string

syntax.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"bufio"
@@ -22,13 +22,13 @@ type SequenceItem struct {
 }
 
 type Syntax struct {
-	trace       Trace
+	trace        Trace
-	registry    *registry
+	registry     *registry
-	initialized bool
+	initialized  bool
-	initFailed  bool
+	initFailed   bool
-	rootSet     bool
+	explicitRoot bool
-	root        definition
+	root         definition
-	parser      parser
+	parser       parser
 }
 
 var (
@@ -70,8 +70,8 @@ func (s *Syntax) register(d definition) error {
 
 	if d.commitType()&Root != 0 {
 		s.root = d
-		s.rootSet = true
+		s.explicitRoot = true
-	} else if !s.rootSet {
+	} else if !s.explicitRoot {
 		s.root = d
 	}
 

trace.go

@@ -1,4 +1,4 @@
-package parse
+package treerack
 
 import (
 	"fmt"