arpio/treerack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

track parsers by numeric id

Browse Source
This commit is contained in:
Arpad Ryszka 2017-07-15 21:49:08 +02:00
parent 9e30f1512f
commit 94a21ae755
22 changed files with 155 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
boot.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"errors"

2
boot_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"os"

2
bootsyntax.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
var bootSyntaxDefs = [][]string{{
"chars", "space", "alias", " ",

15
char.go
View File

@ -1,7 +1,8 @@
package parse
package treerack
type charParser struct {
name string
id int
commit CommitType
not bool
chars []rune
@ -26,9 +27,11 @@ func newChar(
}
func (p *charParser) nodeName() string { return p.name }
func (p *charParser) nodeID() int { return p.id }
func (p *charParser) setID(id int) { p.id = p.id }
func (p *charParser) parser(r *registry, path []string) (parser, error) {
if stringsContain(path, p.name) {
func (p *charParser) parser(r *registry, parsers []string) (parser, error) {
if stringsContainDeprecated(parsers, p.name) {
panic(cannotIncludeParsers(p.name))
}
@ -44,8 +47,8 @@ func (p *charParser) commitType() CommitType {
return p.commit
}
func (p *charParser) setIncludedBy(includedBy parser, path []string) {
if stringsContain(path, p.name) {
func (p *charParser) setIncludedBy(includedBy parser, parsers []string) {
if stringsContainDeprecated(parsers, p.name) {
panic(cannotIncludeParsers(p.name))
}
@ -89,7 +92,7 @@ func (p *charParser) parse(t Trace, c *context) {
if tok, ok := c.token(); ok && p.match(tok) {
// t.Out1("success", string(tok))
n := newNode(p.name, c.offset, c.offset+1, p.commit)
n := newNode(p.name, p.id, c.offset, c.offset+1, p.commit)
c.store.set(c.offset, p.name, n)
for _, includedBy := range p.includedBy {
includedBy.storeIncluded(c, n)

36
choice.go
View File

@ -1,13 +1,15 @@
package parse
package treerack
type choiceDefinition struct {
name string
id int
commit CommitType
elements []string
}
type choiceParser struct {
name string
id int
commit CommitType
elements []parser
includedBy []parser
@ -22,8 +24,10 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition
}
func (d *choiceDefinition) nodeName() string { return d.name }
func (d *choiceDefinition) nodeID() int { return d.id }
func (d *choiceDefinition) setID(id int) { d.id = id }
func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
func (d *choiceDefinition) parser(r *registry, parsers []string) (parser, error) {
p, ok := r.parser(d.name)
if ok {
return p, nil
@ -31,18 +35,19 @@ func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
cp := &choiceParser{
name: d.name,
id: d.id,
commit: d.commit,
}
r.setParser(cp)
var elements []parser
path = append(path, d.name)
parsers = append(parsers, d.name)
for _, e := range d.elements {
element, ok := r.parser(e)
if ok {
elements = append(elements, element)
element.setIncludedBy(cp, path)
element.setIncludedBy(cp, parsers)
continue
}
@ -51,12 +56,12 @@ func (d *choiceDefinition) parser(r *registry, path []string) (parser, error) {
return nil, parserNotFound(e)
}
element, err := elementDefinition.parser(r, path)
element, err := elementDefinition.parser(r, parsers)
if err != nil {
return nil, err
}
element.setIncludedBy(cp, path)
element.setIncludedBy(cp, parsers)
elements = append(elements, element)
}
@ -69,9 +74,10 @@ func (d *choiceDefinition) commitType() CommitType {
}
func (p *choiceParser) nodeName() string { return p.name }
func (p *choiceParser) nodeID() int { return p.id }
func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
if stringsContain(path, p.name) {
func (p *choiceParser) setIncludedBy(includedBy parser, parsers []string) {
if stringsContainDeprecated(parsers, p.name) {
return
}
@ -79,11 +85,11 @@ func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
}
func (p *choiceParser) storeIncluded(c *context, n *Node) {
if !c.excluded(n.From, p.name) {
if !c.excluded(n.From, p.id) {
return
}
nc := newNode(p.name, n.From, n.To, p.commit)
nc := newNode(p.name, p.id, n.From, n.To, p.commit)
nc.append(n)
c.store.set(nc.From, p.name, nc)
@ -107,16 +113,16 @@ func (p *choiceParser) parse(t Trace, c *context) {
return
}
if c.excluded(c.offset, p.name) {
if c.excluded(c.offset, p.id) {
// t.Out1("excluded")
c.fail(c.offset)
return
}
c.exclude(c.offset, p.name)
defer c.include(c.offset, p.name)
c.exclude(c.offset, p.id)
defer c.include(c.offset, p.id) // TODO: test if can be optimized
node := newNode(p.name, c.offset, c.offset, p.commit)
node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
var match bool
for {
@ -134,7 +140,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
match = true
foundMatch = true
node = newNode(p.name, c.offset, c.offset, p.commit)
node = newNode(p.name, p.id, c.offset, c.offset, p.commit)
node.append(c.node)
c.store.set(node.From, p.name, node)

33
context.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"io"
@ -15,7 +15,7 @@ type context struct {
tokens []rune
match bool
node *Node
isExcluded [][]string
isExcluded []*idSet
}
func newContext(r io.RuneReader) *context {
@ -64,16 +64,16 @@ func (c *context) token() (rune, bool) {
return c.tokens[c.offset], true
}
func (c *context) excluded(offset int, name string) bool {
if len(c.isExcluded) <= offset {
func (c *context) excluded(offset int, id int) bool {
if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
return false
}
return stringsContain(c.isExcluded[offset], name)
return c.isExcluded[offset].has(id)
}
func (c *context) exclude(offset int, name string) {
if c.excluded(offset, name) {
func (c *context) exclude(offset int, id int) {
if c.excluded(offset, id) {
return
}
@ -84,25 +84,24 @@ func (c *context) exclude(offset int, name string) {
} else {
c.isExcluded = append(
c.isExcluded[:cap(c.isExcluded)],
make([][]string, offset+1-cap(c.isExcluded))...,
make([]*idSet, offset+1-cap(c.isExcluded))...,
)
}
}
c.isExcluded[offset] = append(c.isExcluded[offset], name)
if c.isExcluded[offset] == nil {
c.isExcluded[offset] = &idSet{}
}
c.isExcluded[offset].set(id)
}
func (c *context) include(offset int, name string) {
if len(c.isExcluded) <= offset {
func (c *context) include(offset int, id int) {
if len(c.isExcluded) <= offset || c.isExcluded[offset] == nil {
return
}
for i := len(c.isExcluded[offset]) - 1; i >= 0; i-- {
if c.isExcluded[offset][i] == name {
c.isExcluded[offset] = append(c.isExcluded[offset][:i], c.isExcluded[offset][i+1:]...)
return
}
}
c.isExcluded[offset].unset(id)
}
func (c *context) fromStore(name string) (bool, bool) {

2
define.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import "strconv"

2
eskip_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"bytes"

45
idset.go Normal file
View File

@ -0,0 +1,45 @@
package treerack
import "strconv"
type idSet struct {
ids []uint
}
func divModBits(id int) (int, int) {
return id / strconv.IntSize, id % strconv.IntSize
}
func (s *idSet) set(id int) {
d, m := divModBits(id)
if d >= len(s.ids) {
if d < cap(s.ids) {
s.ids = s.ids[:d+1]
} else {
s.ids = s.ids[:cap(s.ids)]
for i := cap(s.ids); i <= d; i++ {
s.ids = append(s.ids, 0)
}
}
}
s.ids[d] |= 1 << uint(m)
}
func (s *idSet) unset(id int) {
d, m := divModBits(id)
if d >= len(s.ids) {
return
}
s.ids[d] &^= 1 << uint(m)
}
func (s *idSet) has(id int) bool {
d, m := divModBits(id)
if d >= len(s.ids) {
return false
}
return s.ids[d]&(1<<uint(m)) != 0
}

2
json_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"bytes"

2
keyval_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import "testing"

2
mml_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import "testing"

6
node.go
View File

@ -1,18 +1,20 @@
package parse
package treerack
import "fmt"
type Node struct {
Name string
id int
Nodes []*Node
From, To int
commitType CommitType
tokens []rune
}
func newNode(name string, from, to int, ct CommitType) *Node {
func newNode(name string, id int, from, to int, ct CommitType) *Node {
return &Node{
Name: name,
id: id,
From: from,
To: to,
commitType: ct,

7
parse.go
View File

@ -1,15 +1,18 @@
package parse
package treerack
import "fmt"
type definition interface {
nodeName() string
nodeID() int
setID(int)
parser(*registry, []string) (parser, error)
commitType() CommitType
}
type parser interface {
nodeName() string
nodeID() int
setIncludedBy(parser, []string)
storeIncluded(*context, *Node)
parse(Trace, *context)
@ -23,7 +26,7 @@ func cannotIncludeParsers(name string) error {
return fmt.Errorf("parser: %s cannot include other parsers", name)
}
func stringsContain(ss []string, s string) bool {
func stringsContainDeprecated(ss []string, s string) bool {
for _, si := range ss {
if si == s {
return true

2
parse_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"bytes"

13
registry.go
View File

@ -1,12 +1,17 @@
package parse
package treerack
type registry struct {
idSeed int
ids map[string]int
names map[int]string
definitions map[string]definition
parsers map[string]parser
}
func newRegistry() *registry {
return &registry{
ids: make(map[string]int),
names: make(map[int]string),
definitions: make(map[string]definition),
parsers: make(map[string]parser),
}
@ -27,6 +32,12 @@ func (r *registry) setDefinition(d definition) error {
return duplicateDefinition(d.nodeName())
}
r.idSeed++
id := r.idSeed
d.setID(id)
r.ids[d.nodeName()] = id
r.names[id] = d.nodeName()
r.definitions[d.nodeName()] = d
return nil
}

2
scheme_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import "testing"

34
sequence.go
View File

@ -1,13 +1,15 @@
package parse
package treerack
type sequenceDefinition struct {
name string
id int
commit CommitType
items []SequenceItem
}
type sequenceParser struct {
name string
id int
commit CommitType
items []parser
ranges [][]int
@ -23,9 +25,11 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi
}
func (d *sequenceDefinition) nodeName() string { return d.name }
func (d *sequenceDefinition) nodeID() int { return d.id }
func (d *sequenceDefinition) setID(id int) { d.id = id }
func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error) {
if stringsContain(path, d.name) {
func (d *sequenceDefinition) parser(r *registry, parsers []string) (parser, error) {
if stringsContainDeprecated(parsers, d.name) {
panic(cannotIncludeParsers(d.name))
}
@ -36,6 +40,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
sp := &sequenceParser{
name: d.name,
id: d.id,
commit: d.commit,
}
@ -46,7 +51,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
ranges [][]int
)
path = append(path, d.name)
parsers = append(parsers, d.name)
for _, item := range d.items {
if item.Min == 0 && item.Max == 0 {
item.Min, item.Max = 1, 1
@ -66,7 +71,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
return nil, parserNotFound(item.Name)
}
pi, err := itemDefinition.parser(r, path)
pi, err := itemDefinition.parser(r, parsers)
if err != nil {
return nil, err
}
@ -77,7 +82,7 @@ func (d *sequenceDefinition) parser(r *registry, path []string) (parser, error)
// for single items, acts like a choice
if len(items) == 1 && ranges[0][0] == 1 && ranges[0][1] == 1 {
items[0].setIncludedBy(sp, path)
items[0].setIncludedBy(sp, parsers)
}
sp.items = items
@ -90,9 +95,10 @@ func (d *sequenceDefinition) commitType() CommitType {
}
func (p *sequenceParser) nodeName() string { return p.name }
func (p *sequenceParser) nodeID() int { return p.id }
func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
if stringsContain(path, p.name) {
func (p *sequenceParser) setIncludedBy(includedBy parser, parsers []string) {
if stringsContainDeprecated(parsers, p.name) {
return
}
@ -100,11 +106,11 @@ func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
}
func (p *sequenceParser) storeIncluded(c *context, n *Node) {
if !c.excluded(n.From, p.name) {
if !c.excluded(n.From, p.id) {
return
}
nc := newNode(p.name, n.From, n.To, p.commit)
nc := newNode(p.name, p.id, n.From, n.To, p.commit)
nc.append(n)
c.store.set(nc.From, p.name, nc)
@ -123,19 +129,19 @@ func (p *sequenceParser) parse(t Trace, c *context) {
return
}
if c.excluded(c.offset, p.name) {
if c.excluded(c.offset, p.id) {
// t.Out1("excluded")
c.fail(c.offset)
return
}
c.exclude(c.offset, p.name)
defer c.include(c.offset, p.name)
c.exclude(c.offset, p.id)
defer c.include(c.offset, p.id)
items := p.items
ranges := p.ranges
var currentCount int
node := newNode(p.name, c.offset, c.offset, p.commit)
node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
for len(items) > 0 {
m, ok := c.fromStore(items[0].nodeName())

2
sexpr_test.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import "testing"

2
store.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
type storedItem struct {
name string

20
syntax.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"bufio"
@ -22,13 +22,13 @@ type SequenceItem struct {
}
type Syntax struct {
trace Trace
registry *registry
initialized bool
initFailed bool
rootSet bool
root definition
parser parser
trace Trace
registry *registry
initialized bool
initFailed bool
explicitRoot bool
root definition
parser parser
}
var (
@ -70,8 +70,8 @@ func (s *Syntax) register(d definition) error {
if d.commitType()&Root != 0 {
s.root = d
s.rootSet = true
} else if !s.rootSet {
s.explicitRoot = true
} else if !s.explicitRoot {
s.root = d
}

2
trace.go
View File

@ -1,4 +1,4 @@
package parse
package treerack
import (
"fmt"