store sequences

choice.go

@@ -90,7 +90,7 @@ func (p *choiceParser) storeIncluded(c *context, from, to int) {
 		return
 	}
 
-	c.store.set(from, p.id, true, to)
+	c.store.setMatch(from, p.id, to)
 
 	for _, includedBy := range p.includedBy {
 		includedBy.storeIncluded(c, from, to)
@@ -136,7 +136,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
 			foundMatch = true
 			to = nextTo
 
-			c.store.set(from, p.id, true, to)
+			c.store.setMatch(from, p.id, to)
 			for _, includedBy := range p.includedBy {
 				includedBy.storeIncluded(c, from, to)
 			}
@@ -153,7 +153,7 @@ func (p *choiceParser) parse(t Trace, c *context) {
 		return
 	}
 
-	c.store.set(from, p.id, false, 0)
+	c.store.setNoMatch(from, p.id)
 	c.fail(from)
 	c.include(from, p.id)
 }

context.go

@@ -105,7 +105,7 @@ func (c *context) include(offset int, id int) {
 }
 
 func (c *context) fromStore(id int) (bool, bool) {
-	to, m, ok := c.store.get(c.offset, id)
+	to, m, ok := c.store.getMatch(c.offset, id)
 	if !ok {
 		return false, false
 	}

sequence.go

@@ -111,7 +111,7 @@ func (p *sequenceParser) storeIncluded(c *context, from, to int) {
 		return
 	}
 
-	c.store.set(from, p.id, true, to)
+	c.store.setMatch(from, p.id, to)
 
 	for _, includedBy := range p.includedBy {
 		includedBy.storeIncluded(c, from, to)
@@ -129,6 +129,10 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 		return
 	}
 
+	if c.store.hasNoMatch(c.offset, p.id) {
+		c.fail(c.offset)
+	}
+
 	c.exclude(c.offset, p.id)
 
 	itemIndex := 0
@@ -140,6 +144,7 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 		p.items[itemIndex].parse(t, c)
 		if !c.match {
 			if currentCount < p.ranges[itemIndex][0] {
+				c.store.setNoMatch(from, p.id)
 				c.fail(from)
 				c.include(from, p.id)
 				return
@@ -167,6 +172,7 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 		includedBy.storeIncluded(c, from, to)
 	}
 
+	c.store.setMatch(from, p.id, to)
 	c.success(to)
 	c.include(from, p.id)
 }

store.go

@@ -3,268 +3,118 @@ package treerack
 type storeEntry struct {
 	match   *idSet
 	noMatch *idSet
-	nodes   []int
+	matches   []int
+	all []int
 }
 
 type store struct {
 	entries []*storeEntry
 }
 
-func (c *store) get(offset int, id int) (int, bool, bool) {
+func (s *store) getEntry(offset int) *storeEntry {
-	if len(c.entries) <= offset {
+	if len(s.entries) <= offset {
+		return nil
+	}
+
+	return s.entries[offset]
+}
+
+func (s *store) hasNoMatch(offset, id int) bool {
+	e := s.getEntry(offset)
+	if e == nil {
+		return false
+	}
+
+	return e.noMatch.has(id)
+}
+
+func (s *store) getMatch(offset, id int) (int, bool, bool) {
+	e := s.getEntry(offset)
+	if e == nil {
 		return 0, false, false
 	}
 
-	tc := c.entries[offset]
+	if e.noMatch.has(id) {
-	if tc == nil {
-		return 0, false, false
-	}
-
-	if tc.noMatch.has(id) {
 		return 0, false, true
 	}
 
-	if !tc.match.has(id) {
+	if !e.match.has(id) {
 		return 0, false, false
 	}
 
-	for i := 0; i < len(tc.nodes); i += 2 {
+	for i := 0; i < len(e.matches); i += 2 {
-		if tc.nodes[i] == id {
+		if e.matches[i] == id {
-			return tc.nodes[i + 1], true, true
+			return e.matches[i + 1], true, true
 		}
 	}
 
 	return 0, false, false
 }
 
-func (c *store) set(offset int, id int, match bool, to int) {
+func (s *store) ensureOffset(offset int) {
-	if len(c.entries) <= offset {
+	if len(s.entries) > offset {
-		if cap(c.entries) > offset {
+		return
-			c.entries = c.entries[:offset+1]
-		} else {
-			c.entries = c.entries[:cap(c.entries)]
-			for len(c.entries) <= offset {
-				c.entries = append(c.entries, nil)
 	}
+
+	if cap(s.entries) > offset {
+		s.entries = s.entries[:offset+1]
+		return
+	}
+
+	s.entries = s.entries[:cap(s.entries)]
+	for len(s.entries) <= offset {
+		s.entries = append(s.entries, nil)
 	}
 }
 
-	tc := c.entries[offset]
+func (s *store) ensureEntry(offset int) *storeEntry {
-	if tc == nil {
+	s.ensureOffset(offset)
-		tc = &storeEntry{
+	e := s.entries[offset]
+	if e != nil {
+		return e
+	}
+
+	e = &storeEntry{
 		match:   &idSet{},
 		noMatch: &idSet{},
 	}
 
-		c.entries[offset] = tc
+	s.entries[offset] = e
+	return e
 }
 
-	if !match {
+func (s *store) setMatch(offset, id, to int) {
-		if tc.match.has(id) {
+	e := s.ensureEntry(offset)
-			return
+
+	e.match.set(id)
+	for i := 0; i < len(e.matches); i += 2 {
+		if e.matches[i] == id {
+			if to > e.matches[i + 1] {
+				e.matches[i + 1] = to
 			}
 
-		tc.noMatch.set(id)
+			if to != e.matches[i + 1] {
-		return
+				e.all = append(e.all, id, to)
-	}
-
-	tc.match.set(id)
-	for i := 0; i < len(tc.nodes); i += 2 {
-		if tc.nodes[i] == id {
-			if to > tc.nodes[i + 1] {
-				tc.nodes[i + 1] = to
 			}
 
 			return
 		}
 	}
 
-	tc.nodes = append(tc.nodes, id, to)
+	e.matches = append(e.matches, id, to)
 }
 
-/*
+func (s *store) setNoMatch(offset, id int) {
+	e := s.ensureEntry(offset)
 
-[][][]int
+	if e.match.has(id) {
-
-id, length, where to start in the underlying layer, which list in the layer
-
-attibutes:
-- sequence: length, the items in the layer below
-- choice: the item below
-
-features:
-- there can be sequences or choices under choices
-
-in every position:
-- store the valid choices with the underlying parsed nodes
-
-3D table: layer, choice, sequence
-stored choice identified by: offset, layer, choice index
-
-*/
-
-func (c *store) inc() {
-}
-
-func (c *store) dec() {
-}
-
-func (c *store) get2(offset, id int) (int, bool) {
-	return 0, false
-}
-
-func (c *store) cache(offset, id int, match bool, length int) {
-}
-
-func (c *store) set2(offset, id int, match bool, length int) {
-	/*
-		c.cache(offset, id, match, length)
-		levels := c.offsetLevels[offset]
-		levels[c.currentLevel] = id
-	*/
-}
-
-/*
-package treerack
-
-type storeEntry struct {
-	match   *idSet
-	noMatch *idSet
-	nodes   []*Node
-	offset int
-}
-
-type store struct {
-	current *storeEntry
-	currentIndex int
-	entries []*storeEntry
-}
-
-func (s *store) find(offset int) *storeEntry {
-	if s.current == nil {
-		return nil
-	}
-
-	var seekPrev, seekNext bool
-	for {
-		switch {
-		case s.current.offset == offset:
-			return s.current
-		case s.current.offset < offset:
-			if seekPrev {
-				return nil
-			}
-
-			seekNext = true
-			s.currentIndex++
-			if s.currentIndex == len(s.entries) {
-				s.currentIndex = len(s.entries) - 1
-				return nil
-			}
-
-			s.current = s.entries[s.currentIndex]
-		case s.current.offset > offset:
-			if seekNext {
-				return nil
-			}
-
-			seekPrev = true
-			s.currentIndex--
-			if s.currentIndex == -1 {
-				s.currentIndex = 0
-				return nil
-			}
-
-			s.current = s.entries[s.currentIndex]
-		}
-	}
-}
-
-func (s *store) findCreate(offset int) *storeEntry {
-	entry := s.find(offset)
-	if entry != nil {
-		return entry
-	}
-
-	entry = &storeEntry{
-		match:   &idSet{},
-		noMatch: &idSet{},
-		offset: offset,
-	}
-
-	switch {
-	case s.current != nil && s.current.offset > offset:
-		s.entries = append(
-			s.entries[:s.currentIndex],
-			append([]*storeEntry{entry}, s.entries[s.currentIndex:]...)...,
-		)
-		s.current = entry
-	case s.current != nil && s.current.offset < offset:
-		s.entries = append(
-			s.entries[:s.currentIndex + 1],
-			append([]*storeEntry{entry}, s.entries[s.currentIndex + 1:]...)...,
-		)
-		s.current = entry
-		s.currentIndex++
-	default:
-		s.entries = []*storeEntry{entry}
-		s.current = entry
-		s.currentIndex = 0
-	}
-
-	return entry
-}
-
-func (s *store) get(offset int, id int) (*Node, bool, bool) {
-	entry := s.find(offset)
-	if entry == nil {
-		return nil, false, false
-	}
-
-	if entry == nil {
-		return nil, false, false
-	}
-
-	if entry.noMatch.has(id) {
-		return nil, false, true
-	}
-
-	if !entry.match.has(id) {
-		return nil, false, false
-	}
-
-	for _, n := range entry.nodes {
-		if n.id == id {
-			return n, true, true
-		}
-	}
-
-	return nil, false, false
-}
-
-func (s *store) set(offset int, id int, n *Node) {
-	entry := s.findCreate(offset)
-
-	if n == nil {
-		if entry.match.has(id) {
 		return
 	}
 
-		entry.noMatch.set(id)
+	e.noMatch.set(id)
-		return
 }
 
-	entry.match.set(id)
+func (s *store) add(offset, id, to int) {
-	for i, ni := range entry.nodes {
+	e := s.ensureEntry(offset)
-		if ni.id == id {
+	e.all = append(e.all, id, to)
-			if n.tokenLength() > ni.tokenLength() {
-				entry.nodes[i] = n
 }
-
-			return
-		}
-	}
-
-	entry.nodes = append(entry.nodes, n)
-}
-*/

syntax.go

@@ -162,7 +162,7 @@ func (s *Syntax) Generate(w io.Writer) error {
 	return ErrNotImplemented
 }
 
-// TODO: optimize top sequences to save memory
+// TODO: optimize top sequences to save memory, or just support streaming, or combine the two
 
 func (s *Syntax) Parse(r io.Reader) (*Node, error) {
 	if err := s.Init(); err != nil {