From 6f8ffa81610d3d74516bd3345b3a5801af5a3dfb Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Wed, 26 Jul 2017 18:06:11 +0200 Subject: [PATCH 1/7] fix step --- store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store.go b/store.go index cd0e3e0..d3d90ce 100644 --- a/store.go +++ b/store.go @@ -27,7 +27,7 @@ func (s *store) getMatch(offset, id int) (int, bool, bool) { length int ) - for i := 0; i < len(s.match[offset]); i++ { + for i := 0; i < len(s.match[offset]); i += 2 { if s.match[offset][i] != id { continue } From 507299f31821118d3168ff9ee353c26424130af8 Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Wed, 26 Jul 2017 20:03:46 +0200 Subject: [PATCH 2/7] measure multiple parse average --- boot_test.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/boot_test.go b/boot_test.go index 3d24635..7bde6c6 100644 --- a/boot_test.go +++ b/boot_test.go @@ -21,14 +21,26 @@ func TestBoot(t *testing.T) { defer f.Close() - start := time.Now() - _, err = b.Parse(f) - t.Log("duration:", time.Now().Sub(start)) + var d time.Duration + const n = 120 + for i := 0; i < n; i++ { + if _, err := f.Seek(0, 0); err != nil { + t.Error(err) + return + } - if err != ErrNotImplemented { - t.Error(err) + start := time.Now() + _, err = b.Parse(f) + d += time.Now().Sub(start) + + if err != ErrNotImplemented { + t.Error(err) + return + } } + t.Log("duration:", d/n) + // if err != nil { // t.Error(err) // return From 72c8418e45bb16f3391caa96ec0820fc37572a07 Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Thu, 27 Jul 2017 01:48:16 +0200 Subject: [PATCH 3/7] refactore included --- char.go | 19 ++++++------- choice.go | 58 ++++++++++++++++++-------------------- parse.go | 44 +++++++++++++++++++++++++++-- sequence.go | 81 +++++++++++++++++++++++++++++++---------------------- syntax.go | 4 +++ 5 files changed, 129 insertions(+), 77 deletions(-) diff --git a/char.go b/char.go index 3bb0ae6..24dfd9a 100644 --- a/char.go +++ b/char.go @@ -7,7 +7,7 @@ type charParser struct { not bool chars []rune ranges [][]rune - includedBy []parser + includedBy []int } func newChar( @@ -30,6 +30,13 @@ func (p *charParser) nodeName() string { return p.name } func (p *charParser) nodeID() int { return p.id } func (p *charParser) setID(id int) { p.id = id } +func (p *charParser) init(r *registry) error { return nil } + +func (p *charParser) setIncludedBy(r *registry, includedBy int, parsers *idSet) error { + p.includedBy = appendIfMissing(p.includedBy, includedBy) + return nil +} + func (p *charParser) parser(r *registry, parsers *idSet) (parser, error) { if parsers.has(p.id) { panic(cannotIncludeParsers(p.name)) @@ -47,14 +54,6 @@ func (p *charParser) commitType() CommitType { return p.commit } -func (p *charParser) setIncludedBy(includedBy parser, parsers *idSet) { - if parsers.has(p.id) { - panic(cannotIncludeParsers(p.name)) - } - - p.includedBy = append(p.includedBy, includedBy) -} - func (p *charParser) storeIncluded(*context, int, int) { panic(cannotIncludeParsers(p.name)) } @@ -83,6 +82,6 @@ func (p *charParser) parse(t Trace, c *context) { c.success(c.offset + 1) for _, includedBy := range p.includedBy { - includedBy.storeIncluded(c, c.offset, c.offset+1) + c.store.setMatch(c.offset, includedBy, c.offset+1) } } diff --git a/choice.go b/choice.go index 55df20b..a0f9ce0 100644 --- a/choice.go +++ b/choice.go @@ -1,10 +1,11 @@ package treerack type choiceDefinition struct { - name string - id int - commit CommitType - elements []string + name string + id int + commit CommitType + elements []string + includedBy []int } type choiceParser struct { @@ -12,7 +13,7 @@ type choiceParser struct { id int commit CommitType elements []parser - includedBy []parser + includedBy []int } func newChoice(name string, ct CommitType, elements []string) *choiceDefinition { @@ -27,6 +28,22 @@ func (d *choiceDefinition) nodeName() string { return d.name } func (d *choiceDefinition) nodeID() int { return d.id } func (d *choiceDefinition) setID(id int) { d.id = id } +func (d *choiceDefinition) init(r *registry) error { + parsers := &idSet{} + parsers.set(d.id) + return setItemsIncludedBy(r, d.elements, d.id, parsers) +} + +func (d *choiceDefinition) setIncludedBy(r *registry, includedBy int, parsers *idSet) error { + if parsers.has(d.id) { + return nil + } + + d.includedBy = appendIfMissing(d.includedBy, includedBy) + parsers.set(d.id) + return setItemsIncludedBy(r, d.elements, includedBy, parsers) +} + func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { p, ok := r.parser(d.name) if ok { @@ -34,9 +51,10 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { } cp := &choiceParser{ - name: d.name, - id: d.id, - commit: d.commit, + name: d.name, + id: d.id, + commit: d.commit, + includedBy: d.includedBy, } r.setParser(cp) @@ -48,7 +66,6 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { element, ok := r.parser(e) if ok { elements = append(elements, element) - element.setIncludedBy(cp, parsers) continue } @@ -62,7 +79,6 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { return nil, err } - element.setIncludedBy(cp, parsers) elements = append(elements, element) } @@ -77,26 +93,6 @@ func (d *choiceDefinition) commitType() CommitType { func (p *choiceParser) nodeName() string { return p.name } func (p *choiceParser) nodeID() int { return p.id } -func (p *choiceParser) setIncludedBy(includedBy parser, parsers *idSet) { - if parsers.has(p.id) { - return - } - - p.includedBy = append(p.includedBy, includedBy) -} - -func (p *choiceParser) storeIncluded(c *context, from, to int) { - if !c.excluded(from, p.id) { - return - } - - c.store.setMatch(from, p.id, to) - - for _, includedBy := range p.includedBy { - includedBy.storeIncluded(c, from, to) - } -} - func (p *choiceParser) parse(t Trace, c *context) { if p.commit&Documentation != 0 { c.fail(c.offset) @@ -140,7 +136,7 @@ func (p *choiceParser) parse(t Trace, c *context) { c.store.setMatch(from, p.id, to) for _, includedBy := range p.includedBy { - includedBy.storeIncluded(c, from, to) + c.store.setMatch(from, includedBy, to) } } diff --git a/parse.go b/parse.go index 91bb52d..d7d859d 100644 --- a/parse.go +++ b/parse.go @@ -6,6 +6,8 @@ type definition interface { nodeName() string nodeID() int setID(int) + init(*registry) error + setIncludedBy(*registry, int, *idSet) error parser(*registry, *idSet) (parser, error) commitType() CommitType // builder() builder @@ -14,8 +16,6 @@ type definition interface { type parser interface { nodeName() string nodeID() int - setIncludedBy(parser, *idSet) - storeIncluded(*context, int, int) // can be just an id set, taking what's excluded from the context parse(Trace, *context) } @@ -33,6 +33,46 @@ func cannotIncludeParsers(name string) error { return fmt.Errorf("parser: %s cannot include other parsers", name) } +func intsContain(is []int, i int) bool { + for _, ii := range is { + if ii == i { + return true + } + } + + return false +} + +func appendIfMissing(is []int, i int) []int { + if intsContain(is, i) { + return is + } + + return append(is, i) +} + +func setItemsIncludedBy(r *registry, items []string, includedBy int, parsers *idSet) error { + for _, item := range items { + di, ok := r.definition(item) + if !ok { + return ErrNoParsersDefined + } + + di.setIncludedBy(r, includedBy, parsers) + } + + return nil +} + +func sequenceItemNames(items []SequenceItem) []string { + names := make([]string, len(items)) + for i := range items { + names[i] = items[i].Name + } + + return names +} + func parse(t Trace, p parser, c *context) (*Node, error) { p.parse(t, c) if c.readErr != nil { diff --git a/sequence.go b/sequence.go index eecaa1d..c30582a 100644 --- a/sequence.go +++ b/sequence.go @@ -1,10 +1,11 @@ package treerack type sequenceDefinition struct { - name string - id int - commit CommitType - items []SequenceItem + name string + id int + commit CommitType + items []SequenceItem + includedBy []int } type sequenceParser struct { @@ -13,7 +14,7 @@ type sequenceParser struct { commit CommitType items []parser ranges [][]int - includedBy []parser + includedBy []int } func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefinition { @@ -28,6 +29,42 @@ func (d *sequenceDefinition) nodeName() string { return d.name } func (d *sequenceDefinition) nodeID() int { return d.id } func (d *sequenceDefinition) setID(id int) { d.id = id } +func (d *sequenceDefinition) includeItems() bool { + return len(d.items) == 1 && d.items[0].Min == 1 && d.items[0].Max == 1 +} + +func (d *sequenceDefinition) init(r *registry) error { + for _, item := range d.items { + if item.Min == 0 && item.Max == 0 { + item.Min, item.Max = 1, 1 + } else if item.Max == 0 { + item.Max = -1 + } + } + + if !d.includeItems() { + return nil + } + + parsers := &idSet{} + parsers.set(d.id) + return setItemsIncludedBy(r, sequenceItemNames(d.items), d.id, parsers) +} + +func (d *sequenceDefinition) setIncludedBy(r *registry, includedBy int, parsers *idSet) error { + if parsers.has(d.id) { + return nil + } + + d.includedBy = appendIfMissing(d.includedBy, includedBy) + if !d.includeItems() { + return nil + } + + parsers.set(d.id) + return setItemsIncludedBy(r, sequenceItemNames(d.items), includedBy, parsers) +} + func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) { if parsers.has(d.id) { panic(cannotIncludeParsers(d.name)) @@ -39,9 +76,10 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) } sp := &sequenceParser{ - name: d.name, - id: d.id, - commit: d.commit, + name: d.name, + id: d.id, + commit: d.commit, + includedBy: d.includedBy, } r.setParser(sp) @@ -81,11 +119,6 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) ranges = append(ranges, []int{item.Min, item.Max}) } - // for single items, acts like a choice - if len(items) == 1 && ranges[0][0] == 1 && ranges[0][1] == 1 { - items[0].setIncludedBy(sp, parsers) - } - sp.items = items sp.ranges = ranges return sp, nil @@ -98,26 +131,6 @@ func (d *sequenceDefinition) commitType() CommitType { func (p *sequenceParser) nodeName() string { return p.name } func (p *sequenceParser) nodeID() int { return p.id } -func (p *sequenceParser) setIncludedBy(includedBy parser, parsers *idSet) { - if parsers.has(p.id) { - return - } - - p.includedBy = append(p.includedBy, includedBy) -} - -func (p *sequenceParser) storeIncluded(c *context, from, to int) { - if !c.excluded(from, p.id) { - return - } - - c.store.setMatch(from, p.id, to) - - for _, includedBy := range p.includedBy { - includedBy.storeIncluded(c, from, to) - } -} - func (p *sequenceParser) parse(t Trace, c *context) { if p.commit&Documentation != 0 { c.fail(c.offset) @@ -169,7 +182,7 @@ func (p *sequenceParser) parse(t Trace, c *context) { } for _, includedBy := range p.includedBy { - includedBy.storeIncluded(c, from, to) + c.store.setMatch(from, includedBy, to) } c.store.setMatch(from, p.id, to) diff --git a/syntax.go b/syntax.go index 22cef17..424b6b3 100644 --- a/syntax.go +++ b/syntax.go @@ -143,6 +143,10 @@ func (s *Syntax) Init() error { return ErrRootAlias } + for _, p := range s.registry.definitions { + p.init(s.registry) + } + var err error s.parser, err = s.root.parser(s.registry, &idSet{}) if err != nil { From f6761a7e3d654b0cbd5c38f81a3721e4e4c4a306 Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Thu, 27 Jul 2017 02:18:19 +0200 Subject: [PATCH 4/7] fix storage allocation --- store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store.go b/store.go index d3d90ce..515a324 100644 --- a/store.go +++ b/store.go @@ -52,7 +52,7 @@ func (s *store) ensureOffset(offset int) { } s.match = s.match[:cap(s.match)] - for i := cap(s.match); i <= offset; i++ { + for i := len(s.match); i <= offset; i++ { s.match = append(s.match, nil) } } From 1e7c0e8c2897992b5a0864d21fd5b6f7c0a3ef69 Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Sat, 29 Jul 2017 16:25:17 +0200 Subject: [PATCH 5/7] add builder definitions --- char.go | 27 ++++++++++++++++++--------- choice.go | 22 ++++++++++++++++++++-- context.go | 10 ++++++---- parse.go | 8 ++++---- sequence.go | 19 +++++++++++++++++-- store.go | 4 ++++ syntax.go | 4 ++-- 7 files changed, 71 insertions(+), 23 deletions(-) diff --git a/char.go b/char.go index 24dfd9a..4f588dd 100644 --- a/char.go +++ b/char.go @@ -3,7 +3,6 @@ package treerack type charParser struct { name string id int - commit CommitType not bool chars []rune ranges [][]rune @@ -12,14 +11,12 @@ type charParser struct { func newChar( name string, - ct CommitType, not bool, chars []rune, ranges [][]rune, ) *charParser { return &charParser{ name: name, - commit: ct, not: not, chars: chars, ranges: ranges, @@ -29,6 +26,7 @@ func newChar( func (p *charParser) nodeName() string { return p.name } func (p *charParser) nodeID() int { return p.id } func (p *charParser) setID(id int) { p.id = id } +func (p *charParser) commitType() CommitType { return Alias } func (p *charParser) init(r *registry) error { return nil } @@ -50,12 +48,8 @@ func (p *charParser) parser(r *registry, parsers *idSet) (parser, error) { return p, nil } -func (p *charParser) commitType() CommitType { - return p.commit -} - -func (p *charParser) storeIncluded(*context, int, int) { - panic(cannotIncludeParsers(p.name)) +func (p *charParser) builder() builder { + return p } func (p *charParser) match(t rune) bool { @@ -85,3 +79,18 @@ func (p *charParser) parse(t Trace, c *context) { c.store.setMatch(c.offset, includedBy, c.offset+1) } } + +func (p *charParser) build(c *context) ([]*Node, bool) { + t, ok := c.token() + if !ok { + panic("damaged parser context") + } + + if !p.match(t) { + return nil, false + } + + // always alias + c.offset++ + return nil, true +} diff --git a/choice.go b/choice.go index a0f9ce0..4377cf0 100644 --- a/choice.go +++ b/choice.go @@ -16,6 +16,12 @@ type choiceParser struct { includedBy []int } +type choiceBuilder struct { + name string + id int + commit CommitType +} + func newChoice(name string, ct CommitType, elements []string) *choiceDefinition { return &choiceDefinition{ name: name, @@ -27,6 +33,7 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition func (d *choiceDefinition) nodeName() string { return d.name } func (d *choiceDefinition) nodeID() int { return d.id } func (d *choiceDefinition) setID(id int) { d.id = id } +func (d *choiceDefinition) commitType() CommitType { return d.commit } func (d *choiceDefinition) init(r *registry) error { parsers := &idSet{} @@ -44,6 +51,10 @@ func (d *choiceDefinition) setIncludedBy(r *registry, includedBy int, parsers *i return setItemsIncludedBy(r, d.elements, includedBy, parsers) } +// TODO: +// - it may be possible to initialize the parsers non-recursively +// - maybe the whole definition, parser and builder can be united + func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { p, ok := r.parser(d.name) if ok { @@ -86,8 +97,8 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { return cp, nil } -func (d *choiceDefinition) commitType() CommitType { - return d.commit +func (d *choiceDefinition) builder() builder { + return &choiceBuilder{} } func (p *choiceParser) nodeName() string { return p.name } @@ -155,3 +166,10 @@ func (p *choiceParser) parse(t Trace, c *context) { c.fail(from) c.include(from, p.id) } + +func (b *choiceBuilder) nodeName() string { return b.name } +func (b *choiceBuilder) nodeID() int { return b.id } + +func (b *choiceBuilder) build(*context) ([]*Node, bool) { + return nil, false +} diff --git a/context.go b/context.go index 150e7e7..10bdb3f 100644 --- a/context.go +++ b/context.go @@ -127,10 +127,10 @@ func (c *context) fail(offset int) { c.match = false } -func (c *context) finalize() error { - return ErrNotImplemented - - if c.node.To < c.readOffset { +func (c *context) finalize(root parser) error { + rootID := root.nodeID() + to, match, found := c.store.getMatch(0, rootID) + if !found || !match || to < c.readOffset { return ErrUnexpectedCharacter } @@ -145,6 +145,8 @@ func (c *context) finalize() error { } } + return ErrNotImplemented + c.node.commit(c.tokens) return nil } diff --git a/parse.go b/parse.go index d7d859d..e646a68 100644 --- a/parse.go +++ b/parse.go @@ -5,12 +5,12 @@ import "fmt" type definition interface { nodeName() string nodeID() int + commitType() CommitType setID(int) init(*registry) error setIncludedBy(*registry, int, *idSet) error parser(*registry, *idSet) (parser, error) - commitType() CommitType - // builder() builder + builder() builder } type parser interface { @@ -22,7 +22,7 @@ type parser interface { type builder interface { nodeName() string nodeID() int - build(*context) *Node + build(*context) ([]*Node, bool) } func parserNotFound(name string) error { @@ -83,7 +83,7 @@ func parse(t Trace, p parser, c *context) (*Node, error) { return nil, ErrInvalidInput } - if err := c.finalize(); err != nil { + if err := c.finalize(p); err != nil { return nil, err } diff --git a/sequence.go b/sequence.go index c30582a..4daf082 100644 --- a/sequence.go +++ b/sequence.go @@ -17,6 +17,12 @@ type sequenceParser struct { includedBy []int } +type sequenceBuilder struct { + name string + id int + commit CommitType +} + func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefinition { return &sequenceDefinition{ name: name, @@ -28,6 +34,7 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi func (d *sequenceDefinition) nodeName() string { return d.name } func (d *sequenceDefinition) nodeID() int { return d.id } func (d *sequenceDefinition) setID(id int) { d.id = id } +func (d *sequenceDefinition) commitType() CommitType { return d.commit } func (d *sequenceDefinition) includeItems() bool { return len(d.items) == 1 && d.items[0].Min == 1 && d.items[0].Max == 1 @@ -66,6 +73,7 @@ func (d *sequenceDefinition) setIncludedBy(r *registry, includedBy int, parsers } func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) { + // TODO: what is this for? test with sequence containing a sequence through a choice if parsers.has(d.id) { panic(cannotIncludeParsers(d.name)) } @@ -124,8 +132,8 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) return sp, nil } -func (d *sequenceDefinition) commitType() CommitType { - return d.commit +func (d *sequenceDefinition) builder() builder { + return &sequenceBuilder{} } func (p *sequenceParser) nodeName() string { return p.name } @@ -189,3 +197,10 @@ func (p *sequenceParser) parse(t Trace, c *context) { c.success(to) c.include(from, p.id) } + +func (b *sequenceBuilder) nodeName() string { return b.name } +func (b *sequenceBuilder) nodeID() int { return b.id } + +func (b *sequenceBuilder) build(*context) ([]*Node, bool) { + return nil, false +} diff --git a/store.go b/store.go index 515a324..dafeb67 100644 --- a/store.go +++ b/store.go @@ -58,6 +58,10 @@ func (s *store) ensureOffset(offset int) { } func (s *store) setMatch(offset, id, to int) { + if toe, match, ok := s.getMatch(offset, id); ok && match && toe == to { + return + } + s.ensureOffset(offset) s.match[offset] = append(s.match[offset], id, to) } diff --git a/syntax.go b/syntax.go index 424b6b3..2e2d6d8 100644 --- a/syntax.go +++ b/syntax.go @@ -88,7 +88,7 @@ func childName(name string, childIndex int) string { func (s *Syntax) Class(name string, ct CommitType, not bool, chars []rune, ranges [][]rune) error { cname := childName(name, 0) - if err := s.register(newChar(cname, Alias, not, chars, ranges)); err != nil { + if err := s.register(newChar(cname, not, chars, ranges)); err != nil { return err } @@ -100,7 +100,7 @@ func (s *Syntax) CharSequence(name string, ct CommitType, chars []rune) error { for i, ci := range chars { ref := childName(name, i) refs = append(refs, ref) - if err := s.register(newChar(ref, Alias, false, []rune{ci}, nil)); err != nil { + if err := s.register(newChar(ref, false, []rune{ci}, nil)); err != nil { return err } } From 73585dd07db65998c53c8668350e74af5b253b57 Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Sat, 29 Jul 2017 18:40:22 +0200 Subject: [PATCH 6/7] separate build phase for boot --- boot_test.go | 92 ++++++++++++++++++++---------------------- char.go | 6 +-- choice.go | 80 ++++++++++++++++++++++++++++++++----- context.go | 3 -- parse.go | 20 +++++++--- sequence.go | 111 +++++++++++++++++++++++++++++++++++++++++---------- store.go | 44 +++++++++++++++++--- syntax.go | 8 +++- 8 files changed, 268 insertions(+), 96 deletions(-) diff --git a/boot_test.go b/boot_test.go index 7bde6c6..ff5a192 100644 --- a/boot_test.go +++ b/boot_test.go @@ -13,7 +13,7 @@ func TestBoot(t *testing.T) { return } - f, err := os.Open("mml.parser") + f, err := os.Open("syntax.parser") if err != nil { t.Error(err) return @@ -23,6 +23,7 @@ func TestBoot(t *testing.T) { var d time.Duration const n = 120 + var n0 *Node for i := 0; i < n; i++ { if _, err := f.Seek(0, 0); err != nil { t.Error(err) @@ -30,10 +31,10 @@ func TestBoot(t *testing.T) { } start := time.Now() - _, err = b.Parse(f) + n0, err = b.Parse(f) d += time.Now().Sub(start) - if err != ErrNotImplemented { + if err != nil { t.Error(err) return } @@ -41,57 +42,52 @@ func TestBoot(t *testing.T) { t.Log("duration:", d/n) - // if err != nil { - // t.Error(err) - // return - // } + s0 := NewSyntax() + if err := define(s0, n0); err != nil { + t.Error(err) + return + } - // s0 := NewSyntax() - // if err := define(s0, n0); err != nil { - // t.Error(err) - // return - // } + _, err = f.Seek(0, 0) + if err != nil { + t.Error(err) + return + } - // _, err = f.Seek(0, 0) - // if err != nil { - // t.Error(err) - // return - // } + err = s0.Init() + if err != nil { + t.Error(err) + return + } - // err = s0.Init() - // if err != nil { - // t.Error(err) - // return - // } + n1, err := s0.Parse(f) + if err != nil { + t.Error(err) + return + } - // n1, err := s0.Parse(f) - // if err != nil { - // t.Error(err) - // return - // } + checkNode(t, n1, n0) + if t.Failed() { + return + } - // checkNode(t, n1, n0) - // if t.Failed() { - // return - // } + s1 := NewSyntax() + if err := define(s1, n1); err != nil { + t.Error(err) + return + } - // s1 := NewSyntax() - // if err := define(s1, n1); err != nil { - // t.Error(err) - // return - // } + _, err = f.Seek(0, 0) + if err != nil { + t.Error(err) + return + } - // _, err = f.Seek(0, 0) - // if err != nil { - // t.Error(err) - // return - // } + n2, err := s1.Parse(f) + if err != nil { + t.Error(err) + return + } - // n2, err := s1.Parse(f) - // if err != nil { - // t.Error(err) - // return - // } - - // checkNode(t, n2, n1) + checkNode(t, n2, n1) } diff --git a/char.go b/char.go index 4f588dd..54e562c 100644 --- a/char.go +++ b/char.go @@ -23,9 +23,9 @@ func newChar( } } -func (p *charParser) nodeName() string { return p.name } -func (p *charParser) nodeID() int { return p.id } -func (p *charParser) setID(id int) { p.id = id } +func (p *charParser) nodeName() string { return p.name } +func (p *charParser) nodeID() int { return p.id } +func (p *charParser) setID(id int) { p.id = id } func (p *charParser) commitType() CommitType { return Alias } func (p *charParser) init(r *registry) error { return nil } diff --git a/choice.go b/choice.go index 4377cf0..07e4f88 100644 --- a/choice.go +++ b/choice.go @@ -6,6 +6,7 @@ type choiceDefinition struct { commit CommitType elements []string includedBy []int + cbuilder *choiceBuilder } type choiceParser struct { @@ -17,9 +18,10 @@ type choiceParser struct { } type choiceBuilder struct { - name string - id int - commit CommitType + name string + id int + commit CommitType + elements []builder } func newChoice(name string, ct CommitType, elements []string) *choiceDefinition { @@ -30,12 +32,29 @@ func newChoice(name string, ct CommitType, elements []string) *choiceDefinition } } -func (d *choiceDefinition) nodeName() string { return d.name } -func (d *choiceDefinition) nodeID() int { return d.id } -func (d *choiceDefinition) setID(id int) { d.id = id } +func (d *choiceDefinition) nodeName() string { return d.name } +func (d *choiceDefinition) nodeID() int { return d.id } +func (d *choiceDefinition) setID(id int) { d.id = id } func (d *choiceDefinition) commitType() CommitType { return d.commit } func (d *choiceDefinition) init(r *registry) error { + if d.cbuilder == nil { + d.cbuilder = &choiceBuilder{ + name: d.name, + id: d.id, + commit: d.commit, + } + } + + for _, e := range d.elements { + def, ok := r.definition(e) + if !ok { + return parserNotFound(e) + } + + d.cbuilder.elements = append(d.cbuilder.elements, def.builder()) + } + parsers := &idSet{} parsers.set(d.id) return setItemsIncludedBy(r, d.elements, d.id, parsers) @@ -98,7 +117,15 @@ func (d *choiceDefinition) parser(r *registry, parsers *idSet) (parser, error) { } func (d *choiceDefinition) builder() builder { - return &choiceBuilder{} + if d.cbuilder == nil { + d.cbuilder = &choiceBuilder{ + name: d.name, + id: d.id, + commit: d.commit, + } + } + + return d.cbuilder } func (p *choiceParser) nodeName() string { return p.name } @@ -170,6 +197,41 @@ func (p *choiceParser) parse(t Trace, c *context) { func (b *choiceBuilder) nodeName() string { return b.name } func (b *choiceBuilder) nodeID() int { return b.id } -func (b *choiceBuilder) build(*context) ([]*Node, bool) { - return nil, false +func (b *choiceBuilder) build(c *context) ([]*Node, bool) { + to, ok := c.store.takeMatch(c.offset, b.id) + if !ok { + return nil, false + } + + var element builder + for _, e := range b.elements { + elementTo, match, _ := c.store.getMatch(c.offset, e.nodeID()) + if match && elementTo == to { + element = e + break + } + } + + if element == nil { + panic("damaged parse result") + } + + from := c.offset + + n, ok := element.build(c) + if !ok { + panic("damaged parse result") + } + + if b.commit&Alias != 0 { + return n, true + } + + return []*Node{{ + Name: b.name, + From: from, + To: to, + Nodes: n, + tokens: c.tokens, + }}, true } diff --git a/context.go b/context.go index 10bdb3f..b9ea63e 100644 --- a/context.go +++ b/context.go @@ -145,8 +145,5 @@ func (c *context) finalize(root parser) error { } } - return ErrNotImplemented - - c.node.commit(c.tokens) return nil } diff --git a/parse.go b/parse.go index e646a68..958b505 100644 --- a/parse.go +++ b/parse.go @@ -73,19 +73,29 @@ func sequenceItemNames(items []SequenceItem) []string { return names } -func parse(t Trace, p parser, c *context) (*Node, error) { +func parse(t Trace, p parser, c *context) error { p.parse(t, c) if c.readErr != nil { - return nil, c.readErr + return c.readErr } if !c.match { - return nil, ErrInvalidInput + return ErrInvalidInput } if err := c.finalize(p); err != nil { - return nil, err + return err } - return c.node, nil + return nil +} + +func build(b builder, c *context) *Node { + c.offset = 0 + n, ok := b.build(c) + if !ok || len(n) != 1 { + panic("damaged parse result") + } + + return n[0] } diff --git a/sequence.go b/sequence.go index 4daf082..4473644 100644 --- a/sequence.go +++ b/sequence.go @@ -6,6 +6,8 @@ type sequenceDefinition struct { commit CommitType items []SequenceItem includedBy []int + ranges [][]int + sbuilder *sequenceBuilder } type sequenceParser struct { @@ -18,9 +20,11 @@ type sequenceParser struct { } type sequenceBuilder struct { - name string - id int + name string + id int commit CommitType + items []builder + ranges [][]int } func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefinition { @@ -31,9 +35,9 @@ func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefi } } -func (d *sequenceDefinition) nodeName() string { return d.name } -func (d *sequenceDefinition) nodeID() int { return d.id } -func (d *sequenceDefinition) setID(id int) { d.id = id } +func (d *sequenceDefinition) nodeName() string { return d.name } +func (d *sequenceDefinition) nodeID() int { return d.id } +func (d *sequenceDefinition) setID(id int) { d.id = id } func (d *sequenceDefinition) commitType() CommitType { return d.commit } func (d *sequenceDefinition) includeItems() bool { @@ -41,14 +45,33 @@ func (d *sequenceDefinition) includeItems() bool { } func (d *sequenceDefinition) init(r *registry) error { + if d.sbuilder == nil { + d.sbuilder = &sequenceBuilder{ + name: d.name, + id: d.id, + commit: d.commit, + } + } + for _, item := range d.items { if item.Min == 0 && item.Max == 0 { item.Min, item.Max = 1, 1 } else if item.Max == 0 { item.Max = -1 } + + d.ranges = append(d.ranges, []int{item.Min, item.Max}) + + def, ok := r.definition(item.Name) + if !ok { + return parserNotFound(item.Name) + } + + d.sbuilder.items = append(d.sbuilder.items, def.builder()) } + d.sbuilder.ranges = d.ranges + if !d.includeItems() { return nil } @@ -92,24 +115,13 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) r.setParser(sp) - var ( - items []parser - ranges [][]int - ) - + var items []parser parsers.set(d.id) defer parsers.unset(d.id) for _, item := range d.items { - if item.Min == 0 && item.Max == 0 { - item.Min, item.Max = 1, 1 - } else if item.Max == 0 { - item.Max = -1 - } - pi, ok := r.parser(item.Name) if ok { items = append(items, pi) - ranges = append(ranges, []int{item.Min, item.Max}) continue } @@ -124,16 +136,23 @@ func (d *sequenceDefinition) parser(r *registry, parsers *idSet) (parser, error) } items = append(items, pi) - ranges = append(ranges, []int{item.Min, item.Max}) } sp.items = items - sp.ranges = ranges + sp.ranges = d.ranges return sp, nil } func (d *sequenceDefinition) builder() builder { - return &sequenceBuilder{} + if d.sbuilder == nil { + d.sbuilder = &sequenceBuilder{ + name: d.name, + id: d.id, + commit: d.commit, + } + } + + return d.sbuilder } func (p *sequenceParser) nodeName() string { return p.name } @@ -162,6 +181,7 @@ func (p *sequenceParser) parse(t Trace, c *context) { to := c.offset for itemIndex < len(p.items) { + // TODO: is it ok to parse before max range check? what if max=0 p.items[itemIndex].parse(t, c) if !c.match { if currentCount < p.ranges[itemIndex][0] { @@ -201,6 +221,53 @@ func (p *sequenceParser) parse(t Trace, c *context) { func (b *sequenceBuilder) nodeName() string { return b.name } func (b *sequenceBuilder) nodeID() int { return b.id } -func (b *sequenceBuilder) build(*context) ([]*Node, bool) { - return nil, false +func (b *sequenceBuilder) build(c *context) ([]*Node, bool) { + to, ok := c.store.takeMatch(c.offset, b.id) + if !ok { + return nil, false + } + + from := c.offset + var ( + itemIndex int + currentCount int + nodes []*Node + ) + + for itemIndex < len(b.items) { + itemFrom := c.offset + n, ok := b.items[itemIndex].build(c) + if !ok { + if currentCount < b.ranges[itemIndex][0] { + panic("damaged parse result") + } + + itemIndex++ + currentCount = 0 + continue + } + + parsed := c.offset > itemFrom + if parsed { + nodes = append(nodes, n...) + currentCount++ + } + + if !parsed || b.ranges[itemIndex][1] >= 0 && currentCount == b.ranges[itemIndex][1] { + itemIndex++ + currentCount = 0 + } + } + + if b.commit&Alias != 0 { + return nodes, true + } + + return []*Node{{ + Name: b.name, + From: from, + To: to, + Nodes: nodes, + tokens: c.tokens, + }}, true } diff --git a/store.go b/store.go index dafeb67..9d53797 100644 --- a/store.go +++ b/store.go @@ -23,8 +23,8 @@ func (s *store) getMatch(offset, id int) (int, bool, bool) { } var ( - found bool - length int + found bool + to int ) for i := 0; i < len(s.match[offset]); i += 2 { @@ -33,12 +33,46 @@ func (s *store) getMatch(offset, id int) (int, bool, bool) { } found = true - if s.match[offset][i+1] > length { - length = s.match[offset][i+1] + if s.match[offset][i+1] > to { + to = s.match[offset][i+1] } } - return length, found, found + return to, found, found +} + +func (s *store) takeMatch(offset, id int) (int, bool) { + if s.hasNoMatch(offset, id) { + return 0, false + } + + if len(s.match) <= offset { + return 0, false + } + + var ( + found bool + to int + index int + ) + + for i := 0; i < len(s.match[offset]); i += 2 { + if s.match[offset][i] != id { + continue + } + + found = true + if s.match[offset][i+1] > to { + to = s.match[offset][i+1] + index = i + } + } + + if found { + s.match[offset][index] = -1 + } + + return to, found } func (s *store) ensureOffset(offset int) { diff --git a/syntax.go b/syntax.go index 2e2d6d8..d6275a6 100644 --- a/syntax.go +++ b/syntax.go @@ -29,6 +29,7 @@ type Syntax struct { explicitRoot bool root definition parser parser + builder builder } var ( @@ -154,6 +155,7 @@ func (s *Syntax) Init() error { return err } + s.builder = s.root.builder() s.initialized = true return nil } @@ -174,5 +176,9 @@ func (s *Syntax) Parse(r io.Reader) (*Node, error) { } c := newContext(bufio.NewReader(r)) - return parse(s.trace, s.parser, c) + if err := parse(s.trace, s.parser, c); err != nil { + return nil, err + } + + return build(s.builder, c), nil } From b1969e802f3102a2f37f1bcbd306a6b4bc8f7b2d Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Sat, 29 Jul 2017 20:04:22 +0200 Subject: [PATCH 7/7] remove included during build --- choice.go | 24 ++++++++++++++++++++---- json.parser | 12 ++++++------ mml_test.go | 2 +- sequence.go | 28 ++++++++++++++++++++++------ store.go | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 17 deletions(-) diff --git a/choice.go b/choice.go index 07e4f88..e759f80 100644 --- a/choice.go +++ b/choice.go @@ -18,10 +18,11 @@ type choiceParser struct { } type choiceBuilder struct { - name string - id int - commit CommitType - elements []builder + name string + id int + commit CommitType + elements []builder + includedBy []int } func newChoice(name string, ct CommitType, elements []string) *choiceDefinition { @@ -66,6 +67,17 @@ func (d *choiceDefinition) setIncludedBy(r *registry, includedBy int, parsers *i } d.includedBy = appendIfMissing(d.includedBy, includedBy) + + if d.cbuilder == nil { + d.cbuilder = &choiceBuilder{ + name: d.name, + id: d.id, + commit: d.commit, + } + } + + d.cbuilder.includedBy = appendIfMissing(d.cbuilder.includedBy, includedBy) + parsers.set(d.id) return setItemsIncludedBy(r, d.elements, includedBy, parsers) } @@ -203,6 +215,10 @@ func (b *choiceBuilder) build(c *context) ([]*Node, bool) { return nil, false } + for _, ib := range b.includedBy { + c.store.takeMatchLength(c.offset, ib, to) + } + var element builder for _, e := range b.elements { elementTo, match, _ := c.store.getMatch(c.offset, e.nodeID()) diff --git a/json.parser b/json.parser index c7d40e8..de8c34d 100644 --- a/json.parser +++ b/json.parser @@ -1,12 +1,12 @@ // JSON (http://www.json.org) -ws:ws = [ \b\f\n\r\t]; +ws:alias = [ \b\f\n\r\t]; true = "true"; false = "false"; null = "null"; -string:nows = "\"" ([^\\"\b\f\n\r\t] | "\\" (["\\/bfnrt] | "u" [0-9a-f]{4}))* "\""; -number:nows = "-"? ("0" | [1-9][0-9]*) ("." [0-9]+)? ([eE] [+\-]? [0-9]+)?; -entry = string ":" value; -object = "{" (entry ("," entry)*)? "}"; -array = "[" (value ("," value)*)? "]"; +string = "\"" ([^\\"\b\f\n\r\t] | "\\" (["\\/bfnrt] | "u" [0-9a-f]{4}))* "\""; +number = "-"? ("0" | [1-9][0-9]*) ("." [0-9]+)? ([eE] [+\-]? [0-9]+)?; +entry = string ws* ":" ws* value; +object = "{" ws* (entry ws* ("," ws* entry)*)? ws* "}"; +array = "[" ws* (value ws* ("," ws* value)*)? ws* "]"; value:alias = true | false | null | string | number | object | array; json:root = value; diff --git a/mml_test.go b/mml_test.go index ac3cc16..db47e05 100644 --- a/mml_test.go +++ b/mml_test.go @@ -9,7 +9,7 @@ import ( ) func TestMML(t *testing.T) { - test(t, "mml.parser", "mml", []testItem{{ + testTrace(t, "mml.parser", "mml", 1, []testItem{{ msg: "empty", node: &Node{Name: "mml"}, }, { diff --git a/sequence.go b/sequence.go index 4473644..623327f 100644 --- a/sequence.go +++ b/sequence.go @@ -20,11 +20,12 @@ type sequenceParser struct { } type sequenceBuilder struct { - name string - id int - commit CommitType - items []builder - ranges [][]int + name string + id int + commit CommitType + items []builder + ranges [][]int + includedBy []int } func newSequence(name string, ct CommitType, items []SequenceItem) *sequenceDefinition { @@ -87,6 +88,17 @@ func (d *sequenceDefinition) setIncludedBy(r *registry, includedBy int, parsers } d.includedBy = appendIfMissing(d.includedBy, includedBy) + + if d.sbuilder == nil { + d.sbuilder = &sequenceBuilder{ + name: d.name, + id: d.id, + commit: d.commit, + } + } + + d.sbuilder.includedBy = appendIfMissing(d.sbuilder.includedBy, includedBy) + if !d.includeItems() { return nil } @@ -227,6 +239,10 @@ func (b *sequenceBuilder) build(c *context) ([]*Node, bool) { return nil, false } + for _, ib := range b.includedBy { + c.store.takeMatchLength(c.offset, ib, to) + } + from := c.offset var ( itemIndex int @@ -239,7 +255,7 @@ func (b *sequenceBuilder) build(c *context) ([]*Node, bool) { n, ok := b.items[itemIndex].build(c) if !ok { if currentCount < b.ranges[itemIndex][0] { - panic("damaged parse result") + panic(b.name + ": damaged parse result") } itemIndex++ diff --git a/store.go b/store.go index 9d53797..e40b539 100644 --- a/store.go +++ b/store.go @@ -75,6 +75,40 @@ func (s *store) takeMatch(offset, id int) (int, bool) { return to, found } +func (s *store) takeMatchLength(offset, id, to int) (int, bool) { + if s.hasNoMatch(offset, id) { + return 0, false + } + + if len(s.match) <= offset { + return 0, false + } + + var ( + found bool + // index int + ) + + for i := 0; i < len(s.match[offset]); i += 2 { + if s.match[offset][i] != id { + continue + } + + found = true + if s.match[offset][i+1] == to { + s.match[offset][i] = -1 + return to, true + //eindex = i + } + } + + if found { + // s.match[offset][index] = -1 + } + + return to, found +} + func (s *store) ensureOffset(offset int) { if len(s.match) > offset { return