generic cleanup
This commit is contained in:
parent
2d8d1ae4ef
commit
5d38caf222
2
boot.go
2
boot.go
@ -197,7 +197,7 @@ func initBoot(definitions [][]string) (*Syntax, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func bootSyntax() (*Syntax, error) {
|
func bootSyntax() (*Syntax, error) {
|
||||||
b, err := initBoot(bootDefinitions)
|
b, err := initBoot(bootSyntaxDefs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -6,7 +6,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestBoot(t *testing.T) {
|
func TestBoot(t *testing.T) {
|
||||||
b, err := initBoot(bootDefinitions)
|
b, err := initBoot(bootSyntaxDefs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
return
|
return
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
package parse
|
package parse
|
||||||
|
|
||||||
var bootDefinitions = [][]string{{
|
var bootSyntaxDefs = [][]string{{
|
||||||
"chars", "space", "alias", " ",
|
"chars", "space", "alias", " ",
|
||||||
}, {
|
}, {
|
||||||
"chars", "tab", "alias", "\\t",
|
"chars", "tab", "alias", "\\t",
|
||||||
|
5
buzz.txt
Normal file
5
buzz.txt
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
generator, in-process init or command line
|
||||||
|
syntax from file or in-memory
|
||||||
|
simple syntax with recursion
|
||||||
|
no lexer required
|
||||||
|
utf8, 8bit or custom tokens
|
13
cache.go
13
cache.go
@ -1,5 +1,7 @@
|
|||||||
package parse
|
package parse
|
||||||
|
|
||||||
|
// TODO: rename to store
|
||||||
|
|
||||||
type cacheItem struct {
|
type cacheItem struct {
|
||||||
name string
|
name string
|
||||||
node *Node
|
node *Node
|
||||||
@ -39,11 +41,11 @@ func (c *cache) get(offset int, name string) (*Node, bool, bool) {
|
|||||||
return nil, false, false
|
return nil, false, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *cache) setOne(offset int, name string, n *Node) {
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *cache) set(offset int, name string, n *Node) {
|
func (c *cache) set(offset int, name string, n *Node) {
|
||||||
if len(c.tokens) <= offset {
|
var tc *tokenCache
|
||||||
|
if len(c.tokens) > offset {
|
||||||
|
tc = c.tokens[offset]
|
||||||
|
} else {
|
||||||
if cap(c.tokens) > offset {
|
if cap(c.tokens) > offset {
|
||||||
c.tokens = c.tokens[:offset+1]
|
c.tokens = c.tokens[:offset+1]
|
||||||
} else {
|
} else {
|
||||||
@ -52,10 +54,7 @@ func (c *cache) set(offset int, name string, n *Node) {
|
|||||||
c.tokens = append(c.tokens, nil)
|
c.tokens = append(c.tokens, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
tc := c.tokens[offset]
|
|
||||||
if tc == nil {
|
|
||||||
tc = &tokenCache{}
|
tc = &tokenCache{}
|
||||||
c.tokens[offset] = tc
|
c.tokens[offset] = tc
|
||||||
}
|
}
|
||||||
|
2
char.go
2
char.go
@ -89,7 +89,7 @@ func (p *charParser) parse(t Trace, c *context) {
|
|||||||
|
|
||||||
if tok, ok := c.token(); ok && p.match(tok) {
|
if tok, ok := c.token(); ok && p.match(tok) {
|
||||||
// t.Out1("success", string(tok))
|
// t.Out1("success", string(tok))
|
||||||
n := newNode(p.name, p.commit, c.offset, c.offset+1)
|
n := newNode(p.name, c.offset, c.offset+1, p.commit)
|
||||||
c.cache.set(c.offset, p.name, n)
|
c.cache.set(c.offset, p.name, n)
|
||||||
for _, includedBy := range p.includedBy {
|
for _, includedBy := range p.includedBy {
|
||||||
includedBy.cacheIncluded(c, n)
|
includedBy.cacheIncluded(c, n)
|
||||||
|
18
choice.go
18
choice.go
@ -79,13 +79,13 @@ func (p *choiceParser) setIncludedBy(includedBy parser, path []string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *choiceParser) cacheIncluded(c *context, n *Node) {
|
func (p *choiceParser) cacheIncluded(c *context, n *Node) {
|
||||||
if !c.excluded(n.from, p.name) {
|
if !c.excluded(n.From, p.name) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
nc := newNode(p.name, p.commit, n.from, n.to)
|
nc := newNode(p.name, n.From, n.To, p.commit)
|
||||||
nc.append(n)
|
nc.append(n)
|
||||||
c.cache.set(nc.from, p.name, nc)
|
c.cache.set(nc.From, p.name, nc)
|
||||||
|
|
||||||
for _, includedBy := range p.includedBy {
|
for _, includedBy := range p.includedBy {
|
||||||
includedBy.cacheIncluded(c, nc)
|
includedBy.cacheIncluded(c, nc)
|
||||||
@ -116,7 +116,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
|
|||||||
c.exclude(c.offset, p.name)
|
c.exclude(c.offset, p.name)
|
||||||
defer c.include(c.offset, p.name)
|
defer c.include(c.offset, p.name)
|
||||||
|
|
||||||
node := newNode(p.name, p.commit, c.offset, c.offset)
|
node := newNode(p.name, c.offset, c.offset, p.commit)
|
||||||
var match bool
|
var match bool
|
||||||
|
|
||||||
for {
|
for {
|
||||||
@ -126,7 +126,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
|
|||||||
for len(elements) > 0 {
|
for len(elements) > 0 {
|
||||||
elements[0].parse(t, c)
|
elements[0].parse(t, c)
|
||||||
elements = elements[1:]
|
elements = elements[1:]
|
||||||
c.offset = node.from
|
c.offset = node.From
|
||||||
|
|
||||||
if !c.match || match && c.node.tokenLength() <= node.tokenLength() {
|
if !c.match || match && c.node.tokenLength() <= node.tokenLength() {
|
||||||
continue
|
continue
|
||||||
@ -134,10 +134,10 @@ func (p *choiceParser) parse(t Trace, c *context) {
|
|||||||
|
|
||||||
match = true
|
match = true
|
||||||
foundMatch = true
|
foundMatch = true
|
||||||
node = newNode(p.name, p.commit, c.offset, c.offset)
|
node = newNode(p.name, c.offset, c.offset, p.commit)
|
||||||
node.append(c.node)
|
node.append(c.node)
|
||||||
|
|
||||||
c.cache.set(node.from, p.name, node)
|
c.cache.set(node.From, p.name, node)
|
||||||
for _, includedBy := range p.includedBy {
|
for _, includedBy := range p.includedBy {
|
||||||
includedBy.cacheIncluded(c, node)
|
includedBy.cacheIncluded(c, node)
|
||||||
}
|
}
|
||||||
@ -155,6 +155,6 @@ func (p *choiceParser) parse(t Trace, c *context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// t.Out1("fail")
|
// t.Out1("fail")
|
||||||
c.cache.set(node.from, p.name, nil)
|
c.cache.set(node.From, p.name, nil)
|
||||||
c.fail(node.from)
|
c.fail(node.From)
|
||||||
}
|
}
|
||||||
|
16
context.go
16
context.go
@ -73,6 +73,10 @@ func (c *context) excluded(offset int, name string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *context) exclude(offset int, name string) {
|
func (c *context) exclude(offset int, name string) {
|
||||||
|
if c.excluded(offset, name) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if len(c.isExcluded) <= offset {
|
if len(c.isExcluded) <= offset {
|
||||||
c.isExcluded = append(c.isExcluded, nil)
|
c.isExcluded = append(c.isExcluded, nil)
|
||||||
if cap(c.isExcluded) > offset {
|
if cap(c.isExcluded) > offset {
|
||||||
@ -96,6 +100,7 @@ func (c *context) include(offset int, name string) {
|
|||||||
for i := len(c.isExcluded[offset]) - 1; i >= 0; i-- {
|
for i := len(c.isExcluded[offset]) - 1; i >= 0; i-- {
|
||||||
if c.isExcluded[offset][i] == name {
|
if c.isExcluded[offset][i] == name {
|
||||||
c.isExcluded[offset] = append(c.isExcluded[offset][:i], c.isExcluded[offset][i+1:]...)
|
c.isExcluded[offset] = append(c.isExcluded[offset][:i], c.isExcluded[offset][i+1:]...)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -117,7 +122,7 @@ func (c *context) fromCache(name string) (bool, bool) {
|
|||||||
|
|
||||||
func (c *context) success(n *Node) {
|
func (c *context) success(n *Node) {
|
||||||
c.node = n
|
c.node = n
|
||||||
c.offset = n.to
|
c.offset = n.To
|
||||||
c.match = true
|
c.match = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -127,7 +132,7 @@ func (c *context) fail(offset int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *context) finalize() error {
|
func (c *context) finalize() error {
|
||||||
if c.node.to < c.readOffset {
|
if c.node.To < c.readOffset {
|
||||||
return ErrUnexpectedCharacter
|
return ErrUnexpectedCharacter
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -142,11 +147,6 @@ func (c *context) finalize() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
c.node.commit()
|
c.node.commit(c.tokens)
|
||||||
if c.node.commitType&Alias != 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
c.node.applyTokens(c.tokens)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
140
keyval_test.go
140
keyval_test.go
@ -13,13 +13,13 @@ func TestKeyVal(t *testing.T) {
|
|||||||
text: "a key",
|
text: "a key",
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
to: 5,
|
To: 5,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
to: 5,
|
To: 5,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
to: 5,
|
To: 5,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
@ -28,16 +28,16 @@ func TestKeyVal(t *testing.T) {
|
|||||||
text: " a key",
|
text: " a key",
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
from: 1,
|
From: 1,
|
||||||
to: 6,
|
To: 6,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
from: 1,
|
From: 1,
|
||||||
to: 6,
|
To: 6,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 1,
|
From: 1,
|
||||||
to: 6,
|
To: 6,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
@ -50,16 +50,16 @@ func TestKeyVal(t *testing.T) {
|
|||||||
`,
|
`,
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
from: 20,
|
From: 20,
|
||||||
to: 25,
|
To: 25,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
from: 20,
|
From: 20,
|
||||||
to: 25,
|
To: 25,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 20,
|
From: 20,
|
||||||
to: 25,
|
To: 25,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
@ -68,18 +68,18 @@ func TestKeyVal(t *testing.T) {
|
|||||||
text: "a key = a value",
|
text: "a key = a value",
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
to: 15,
|
To: 15,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
to: 5,
|
To: 5,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
to: 5,
|
To: 5,
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
Name: "value",
|
Name: "value",
|
||||||
from: 8,
|
From: 8,
|
||||||
to: 15,
|
To: 15,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
@ -90,39 +90,39 @@ func TestKeyVal(t *testing.T) {
|
|||||||
`,
|
`,
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
from: 11,
|
From: 11,
|
||||||
to: 32,
|
To: 32,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
from: 11,
|
From: 11,
|
||||||
to: 16,
|
To: 16,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 11,
|
From: 11,
|
||||||
to: 16,
|
To: 16,
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
Name: "value",
|
Name: "value",
|
||||||
from: 25,
|
From: 25,
|
||||||
to: 32,
|
To: 32,
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
from: 61,
|
From: 61,
|
||||||
to: 88,
|
To: 88,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
from: 61,
|
From: 61,
|
||||||
to: 72,
|
To: 72,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 61,
|
From: 61,
|
||||||
to: 72,
|
To: 72,
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
Name: "value",
|
Name: "value",
|
||||||
from: 75,
|
From: 75,
|
||||||
to: 88,
|
To: 88,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
@ -130,11 +130,11 @@ func TestKeyVal(t *testing.T) {
|
|||||||
text: "= a value",
|
text: "= a value",
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
to: 9,
|
To: 9,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "value",
|
Name: "value",
|
||||||
from: 2,
|
From: 2,
|
||||||
to: 9,
|
To: 9,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
@ -145,25 +145,25 @@ func TestKeyVal(t *testing.T) {
|
|||||||
`,
|
`,
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
from: 4,
|
From: 4,
|
||||||
to: 34,
|
To: 34,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "comment",
|
Name: "comment",
|
||||||
from: 4,
|
From: 4,
|
||||||
to: 15,
|
To: 15,
|
||||||
}, {
|
}, {
|
||||||
Name: "key",
|
Name: "key",
|
||||||
from: 19,
|
From: 19,
|
||||||
to: 24,
|
To: 24,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 19,
|
From: 19,
|
||||||
to: 24,
|
To: 24,
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
Name: "value",
|
Name: "value",
|
||||||
from: 27,
|
From: 27,
|
||||||
to: 34,
|
To: 34,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
@ -171,32 +171,32 @@ func TestKeyVal(t *testing.T) {
|
|||||||
text: "a key . with.multiple.symbols=a value",
|
text: "a key . with.multiple.symbols=a value",
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "key-val",
|
Name: "key-val",
|
||||||
to: 37,
|
To: 37,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "key",
|
Name: "key",
|
||||||
from: 0,
|
From: 0,
|
||||||
to: 29,
|
To: 29,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 0,
|
From: 0,
|
||||||
to: 5,
|
To: 5,
|
||||||
}, {
|
}, {
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 8,
|
From: 8,
|
||||||
to: 12,
|
To: 12,
|
||||||
}, {
|
}, {
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 13,
|
From: 13,
|
||||||
to: 21,
|
To: 21,
|
||||||
}, {
|
}, {
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 22,
|
From: 22,
|
||||||
to: 29,
|
To: 29,
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
Name: "value",
|
Name: "value",
|
||||||
from: 30,
|
From: 30,
|
||||||
to: 37,
|
To: 37,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
@ -207,20 +207,20 @@ func TestKeyVal(t *testing.T) {
|
|||||||
`,
|
`,
|
||||||
nodes: []*Node{{
|
nodes: []*Node{{
|
||||||
Name: "group-key",
|
Name: "group-key",
|
||||||
from: 4,
|
From: 4,
|
||||||
to: 38,
|
To: 38,
|
||||||
Nodes: []*Node{{
|
Nodes: []*Node{{
|
||||||
Name: "comment",
|
Name: "comment",
|
||||||
from: 4,
|
From: 4,
|
||||||
to: 15,
|
To: 15,
|
||||||
}, {
|
}, {
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 20,
|
From: 20,
|
||||||
to: 31,
|
To: 31,
|
||||||
}, {
|
}, {
|
||||||
Name: "symbol",
|
Name: "symbol",
|
||||||
from: 32,
|
From: 32,
|
||||||
to: 37,
|
To: 37,
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
}, {
|
}, {
|
||||||
|
1054
mml_test.go
1054
mml_test.go
File diff suppressed because it is too large
Load Diff
50
node.go
50
node.go
@ -5,67 +5,43 @@ import "fmt"
|
|||||||
type Node struct {
|
type Node struct {
|
||||||
Name string
|
Name string
|
||||||
Nodes []*Node
|
Nodes []*Node
|
||||||
|
From, To int
|
||||||
commitType CommitType
|
commitType CommitType
|
||||||
from, to int
|
|
||||||
tokens []rune
|
tokens []rune
|
||||||
}
|
}
|
||||||
|
|
||||||
func newNode(name string, ct CommitType, from, to int) *Node {
|
func newNode(name string, from, to int, ct CommitType) *Node {
|
||||||
return &Node{
|
return &Node{
|
||||||
Name: name,
|
Name: name,
|
||||||
|
From: from,
|
||||||
|
To: to,
|
||||||
commitType: ct,
|
commitType: ct,
|
||||||
from: from,
|
|
||||||
to: to,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) tokenLength() int {
|
func (n *Node) tokenLength() int {
|
||||||
return n.to - n.from
|
return n.To - n.From
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) nodeLength() int {
|
func (n *Node) nodeLength() int {
|
||||||
return len(n.Nodes)
|
return len(n.Nodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func findNode(in, n *Node) {
|
|
||||||
if n == in {
|
|
||||||
panic(fmt.Errorf("found self in %s", in.Name))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, ni := range n.Nodes {
|
|
||||||
findNode(in, ni)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Node) append(p *Node) {
|
func (n *Node) append(p *Node) {
|
||||||
findNode(n, p)
|
|
||||||
n.Nodes = append(n.Nodes, p)
|
n.Nodes = append(n.Nodes, p)
|
||||||
// TODO: check rather if n.from <= p.from??? or panic if less? or check rather node length and commit
|
if n.tokenLength() == 0 {
|
||||||
// happens in the end anyway?
|
n.From = p.From
|
||||||
if n.from == 0 && n.to == 0 {
|
|
||||||
n.from = p.from
|
|
||||||
}
|
}
|
||||||
|
|
||||||
n.to = p.to
|
n.To = p.To
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) clear() {
|
func (n *Node) commit(t []rune) {
|
||||||
n.from = 0
|
|
||||||
n.to = 0
|
|
||||||
n.Nodes = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Node) applyTokens(t []rune) {
|
|
||||||
n.tokens = t
|
n.tokens = t
|
||||||
for _, ni := range n.Nodes {
|
|
||||||
ni.applyTokens(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Node) commit() {
|
|
||||||
var nodes []*Node
|
var nodes []*Node
|
||||||
for _, ni := range n.Nodes {
|
for _, ni := range n.Nodes {
|
||||||
ni.commit()
|
ni.commit(t)
|
||||||
if ni.commitType&Alias != 0 {
|
if ni.commitType&Alias != 0 {
|
||||||
nodes = append(nodes, ni.Nodes...)
|
nodes = append(nodes, ni.Nodes...)
|
||||||
} else {
|
} else {
|
||||||
@ -77,13 +53,13 @@ func (n *Node) commit() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) String() string {
|
func (n *Node) String() string {
|
||||||
if n.from >= len(n.tokens) || n.to > len(n.tokens) {
|
if n.From >= len(n.tokens) || n.To > len(n.tokens) {
|
||||||
return n.Name + ":incomplete"
|
return n.Name + ":incomplete"
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%s:%d:%d:%s", n.Name, n.from, n.to, n.Text())
|
return fmt.Sprintf("%s:%d:%d:%s", n.Name, n.From, n.To, n.Text())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) Text() string {
|
func (n *Node) Text() string {
|
||||||
return string(n.tokens[n.from:n.to])
|
return string(n.tokens[n.From:n.To])
|
||||||
}
|
}
|
||||||
|
7
notes.txt
Normal file
7
notes.txt
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
cleanup
|
||||||
|
error reporting
|
||||||
|
custom tokens
|
||||||
|
indentation
|
||||||
|
benchmarking
|
||||||
|
code generation go
|
||||||
|
code generation js
|
17
parse.go
17
parse.go
@ -33,23 +33,6 @@ func stringsContain(ss []string, s string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyIncludes(to, from map[string]CommitType) {
|
|
||||||
if from == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, ct := range from {
|
|
||||||
to[name] = ct
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeIncludes(left, right map[string]CommitType) map[string]CommitType {
|
|
||||||
m := make(map[string]CommitType)
|
|
||||||
copyIncludes(m, left)
|
|
||||||
copyIncludes(m, right)
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
func parse(t Trace, p parser, c *context) (*Node, error) {
|
func parse(t Trace, p parser, c *context) (*Node, error) {
|
||||||
p.parse(t, c)
|
p.parse(t, c)
|
||||||
if c.readErr != nil {
|
if c.readErr != nil {
|
||||||
|
@ -28,7 +28,7 @@ func testSyntaxReader(r io.Reader, traceLevel int) (*Syntax, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var trace Trace = NopTrace{}
|
var trace Trace
|
||||||
if traceLevel >= 0 {
|
if traceLevel >= 0 {
|
||||||
trace = NewTrace(traceLevel)
|
trace = NewTrace(traceLevel)
|
||||||
}
|
}
|
||||||
@ -45,10 +45,6 @@ func testSyntaxReader(r io.Reader, traceLevel int) (*Syntax, error) {
|
|||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func testSyntaxString(s string, traceLevel int) (*Syntax, error) {
|
|
||||||
return testSyntaxReader(bytes.NewBufferString(s), traceLevel)
|
|
||||||
}
|
|
||||||
|
|
||||||
func testSyntax(file string, traceLevel int) (*Syntax, error) {
|
func testSyntax(file string, traceLevel int) (*Syntax, error) {
|
||||||
f, err := os.Open(file)
|
f, err := os.Open(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -90,13 +86,13 @@ func checkNodePosition(t *testing.T, left, right *Node, position bool) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if position && left.from != right.from {
|
if position && left.From != right.From {
|
||||||
t.Error("from doesn't match", left.Name, left.from, right.from)
|
t.Error("from doesn't match", left.Name, left.From, right.From)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if position && left.to != right.to {
|
if position && left.To != right.To {
|
||||||
t.Error("to doesn't match", left.Name, left.to, right.to)
|
t.Error("to doesn't match", left.Name, left.To, right.To)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -177,8 +173,7 @@ func testReaderTrace(t *testing.T, r io.Reader, rootName string, traceLevel int,
|
|||||||
} else {
|
} else {
|
||||||
cn(t, n, &Node{
|
cn(t, n, &Node{
|
||||||
Name: rootName,
|
Name: rootName,
|
||||||
from: 0,
|
To: len(ti.text),
|
||||||
to: len(ti.text),
|
|
||||||
Nodes: ti.nodes,
|
Nodes: ti.nodes,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -294,7 +289,7 @@ func TestRecursion(t *testing.T) {
|
|||||||
text: "aaa",
|
text: "aaa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -307,7 +302,7 @@ func TestRecursion(t *testing.T) {
|
|||||||
text: "aaa",
|
text: "aaa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -320,7 +315,7 @@ func TestRecursion(t *testing.T) {
|
|||||||
text: "aaa",
|
text: "aaa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -333,7 +328,7 @@ func TestRecursion(t *testing.T) {
|
|||||||
text: "aaa",
|
text: "aaa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -348,14 +343,14 @@ func TestSequence(t *testing.T) {
|
|||||||
text: "abb",
|
text: "abb",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "AB",
|
Name: "AB",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "sequence with optional items, none",
|
msg: "sequence with optional items, none",
|
||||||
text: "bb",
|
text: "bb",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "AB",
|
Name: "AB",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -388,7 +383,7 @@ func TestSequence(t *testing.T) {
|
|||||||
text: "aaa",
|
text: "aaa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -403,7 +398,7 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aba",
|
text: "aba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero, fail",
|
msg: "zero, fail",
|
||||||
@ -424,7 +419,7 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aba",
|
text: "aba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "one, too much",
|
msg: "one, too much",
|
||||||
@ -445,7 +440,7 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "abbba",
|
text: "abbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 5,
|
To: 5,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "three, too much",
|
msg: "three, too much",
|
||||||
@ -462,14 +457,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or one explicit",
|
msg: "zero or one explicit",
|
||||||
text: "aba",
|
text: "aba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or one explicit, too much",
|
msg: "zero or one explicit, too much",
|
||||||
@ -486,14 +481,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or one explicit, omit zero",
|
msg: "zero or one explicit, omit zero",
|
||||||
text: "aba",
|
text: "aba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or one explicit, omit zero, too much",
|
msg: "zero or one explicit, omit zero, too much",
|
||||||
@ -510,14 +505,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or one explicit, shortcut",
|
msg: "zero or one explicit, shortcut",
|
||||||
text: "aba",
|
text: "aba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 3,
|
To: 3,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or one explicit, shortcut, too much",
|
msg: "zero or one explicit, shortcut, too much",
|
||||||
@ -534,21 +529,21 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or three",
|
msg: "zero or three",
|
||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or three",
|
msg: "zero or three",
|
||||||
text: "abbba",
|
text: "abbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 5,
|
To: 5,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or three, too much",
|
msg: "zero or three, too much",
|
||||||
@ -565,21 +560,21 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or three, omit zero",
|
msg: "zero or three, omit zero",
|
||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or three, omit zero",
|
msg: "zero or three, omit zero",
|
||||||
text: "abbba",
|
text: "abbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 5,
|
To: 5,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or three, omit zero, too much",
|
msg: "zero or three, omit zero, too much",
|
||||||
@ -600,14 +595,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "one or three",
|
msg: "one or three",
|
||||||
text: "abbba",
|
text: "abbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 5,
|
To: 5,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "one or three, too much",
|
msg: "one or three, too much",
|
||||||
@ -628,14 +623,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "abbbba",
|
text: "abbbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 6,
|
To: 6,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "three or five",
|
msg: "three or five",
|
||||||
text: "abbbbba",
|
text: "abbbbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 7,
|
To: 7,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "three or five, too much",
|
msg: "three or five, too much",
|
||||||
@ -652,14 +647,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or more, explicit",
|
msg: "zero or more, explicit",
|
||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -672,14 +667,14 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "aa",
|
text: "aa",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 2,
|
To: 2,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
msg: "zero or more, shortcut",
|
msg: "zero or more, shortcut",
|
||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -696,7 +691,7 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -713,7 +708,7 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "abba",
|
text: "abba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 4,
|
To: 4,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
@ -730,7 +725,7 @@ func TestQuantifiers(t *testing.T) {
|
|||||||
text: "abbbba",
|
text: "abbbba",
|
||||||
node: &Node{
|
node: &Node{
|
||||||
Name: "A",
|
Name: "A",
|
||||||
to: 6,
|
To: 6,
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
)
|
)
|
||||||
|
14
sequence.go
14
sequence.go
@ -100,13 +100,13 @@ func (p *sequenceParser) setIncludedBy(includedBy parser, path []string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *sequenceParser) cacheIncluded(c *context, n *Node) {
|
func (p *sequenceParser) cacheIncluded(c *context, n *Node) {
|
||||||
if !c.excluded(n.from, p.name) {
|
if !c.excluded(n.From, p.name) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
nc := newNode(p.name, p.commit, n.from, n.to)
|
nc := newNode(p.name, n.From, n.To, p.commit)
|
||||||
nc.append(n)
|
nc.append(n)
|
||||||
c.cache.set(nc.from, p.name, nc)
|
c.cache.set(nc.From, p.name, nc)
|
||||||
|
|
||||||
for _, includedBy := range p.includedBy {
|
for _, includedBy := range p.includedBy {
|
||||||
includedBy.cacheIncluded(c, nc)
|
includedBy.cacheIncluded(c, nc)
|
||||||
@ -135,7 +135,7 @@ func (p *sequenceParser) parse(t Trace, c *context) {
|
|||||||
items := p.items
|
items := p.items
|
||||||
ranges := p.ranges
|
ranges := p.ranges
|
||||||
var currentCount int
|
var currentCount int
|
||||||
node := newNode(p.name, p.commit, c.offset, c.offset)
|
node := newNode(p.name, c.offset, c.offset, p.commit)
|
||||||
|
|
||||||
for len(items) > 0 {
|
for len(items) > 0 {
|
||||||
m, ok := c.fromCache(items[0].nodeName())
|
m, ok := c.fromCache(items[0].nodeName())
|
||||||
@ -149,8 +149,8 @@ func (p *sequenceParser) parse(t Trace, c *context) {
|
|||||||
if !m {
|
if !m {
|
||||||
if currentCount < ranges[0][0] {
|
if currentCount < ranges[0][0] {
|
||||||
// t.Out1("fail, item failed")
|
// t.Out1("fail, item failed")
|
||||||
c.cache.set(node.from, p.name, nil)
|
c.cache.set(node.From, p.name, nil)
|
||||||
c.fail(node.from)
|
c.fail(node.From)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -174,7 +174,7 @@ func (p *sequenceParser) parse(t Trace, c *context) {
|
|||||||
|
|
||||||
// t.Out1("success, items parsed")
|
// t.Out1("success, items parsed")
|
||||||
|
|
||||||
c.cache.set(node.from, p.name, node)
|
c.cache.set(node.From, p.name, node)
|
||||||
for _, includedBy := range p.includedBy {
|
for _, includedBy := range p.includedBy {
|
||||||
includedBy.cacheIncluded(c, node)
|
includedBy.cacheIncluded(c, node)
|
||||||
}
|
}
|
||||||
|
@ -36,10 +36,11 @@ var (
|
|||||||
ErrInitFailed = errors.New("init failed")
|
ErrInitFailed = errors.New("init failed")
|
||||||
ErrNoParsersDefined = errors.New("no parsers defined")
|
ErrNoParsersDefined = errors.New("no parsers defined")
|
||||||
ErrInvalidInput = errors.New("invalid input")
|
ErrInvalidInput = errors.New("invalid input")
|
||||||
ErrInvalidCharacter = errors.New("invalid character") // two use cases: utf8 and boot
|
ErrInvalidCharacter = errors.New("invalid character") // TODO: fix two use cases, utf8 and boot
|
||||||
ErrUnexpectedCharacter = errors.New("unexpected character")
|
ErrUnexpectedCharacter = errors.New("unexpected character")
|
||||||
ErrInvalidSyntax = errors.New("invalid syntax")
|
ErrInvalidSyntax = errors.New("invalid syntax")
|
||||||
ErrRootAlias = errors.New("root node cannot be an alias")
|
ErrRootAlias = errors.New("root node cannot be an alias")
|
||||||
|
ErrNotImplemented = errors.New("not implemented")
|
||||||
)
|
)
|
||||||
|
|
||||||
func duplicateDefinition(name string) error {
|
func duplicateDefinition(name string) error {
|
||||||
@ -114,7 +115,7 @@ func (s *Syntax) Read(r io.Reader) error {
|
|||||||
return ErrSyntaxInitialized
|
return ErrSyntaxInitialized
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return ErrNotImplemented
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Syntax) Init() error {
|
func (s *Syntax) Init() error {
|
||||||
@ -150,7 +151,7 @@ func (s *Syntax) Generate(w io.Writer) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return ErrNotImplemented
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Syntax) Parse(r io.Reader) (*Node, error) {
|
func (s *Syntax) Parse(r io.Reader) (*Node, error) {
|
||||||
|
Loading…
Reference in New Issue
Block a user