reduce storage size

char.go

@@ -93,7 +93,7 @@ func (p *charParser) parse(t Trace, c *context) {
 	if tok, ok := c.token(); ok && p.match(tok) {
 		// t.Out1("success", string(tok))
 		n := newNode(p.name, p.id, c.offset, c.offset+1, p.commit)
-		c.store.set(c.offset, p.id, n)
+		// c.store.set(c.offset, p.id, n)
 		for _, includedBy := range p.includedBy {
 			includedBy.storeIncluded(c, n)
 		}
@@ -102,7 +102,7 @@ func (p *charParser) parse(t Trace, c *context) {
 		return
 	} else {
 		// t.Out1("fail", string(tok))
-		c.store.set(c.offset, p.id, nil)
+		// c.store.set(c.offset, p.id, nil)
 		c.fail(c.offset)
 		return
 	}

choice.go

@@ -158,12 +158,12 @@ func (p *choiceParser) parse(t Trace, c *context) {
 	if match {
 		// t.Out1("choice, success")
 		c.success(node)
-		c.include(initialOffset, p.id) // TODO: test if can be optimized
+		c.include(initialOffset, p.id)
 		return
 	}
 
 	// t.Out1("fail")
 	c.store.set(node.From, p.id, nil)
 	c.fail(node.From)
-	c.include(initialOffset, p.id) // TODO: test if can be optimized
+	c.include(initialOffset, p.id)
 }

notes.txt

@@ -7,13 +7,14 @@ error reporting
 read, with error reporting
 
 [next]
-cleanup
+optimization
+why normalization failed
+why normalization was slower?
 error reporting
 coverage
 custom tokens
 indentation
 streaming
-benchmarking
 code generation go
 code generation js
 

parse_test.go

@@ -144,12 +144,13 @@ func testReaderTrace(t *testing.T, r io.Reader, rootName string, traceLevel int,
 		return
 	}
 
-	start := time.Now()
-	defer func() { t.Log("\ntotal duration", time.Since(start)) }()
-
 	for _, ti := range tests {
 		t.Run(ti.msg, func(t *testing.T) {
-			n, err := s.Parse(bytes.NewBufferString(ti.text))
+			b := bytes.NewBufferString(ti.text)
+
+			start := time.Now()
+			n, err := s.Parse(b)
+			t.Log("parse duration:", time.Now().Sub(start))
 
 			if ti.fail && err == nil {
 				t.Error("failed to fail")

sequence.go

@@ -145,18 +145,20 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 	node := newNode(p.name, p.id, c.offset, c.offset, p.commit)
 
 	for len(items) > 0 {
-		m, ok := c.fromStore(items[0].nodeID())
+		var m bool
-		if ok {
+		// var ok bool
-			// t.Out1("sequence item found in store, match:", m, items[0].nodeName(), c.offset)
+		// m, ok = c.fromStore(items[0].nodeID())
-		} else {
+		// if ok {
-			items[0].parse(t, c)
+		// 	// t.Out1("sequence item found in store, match:", m, items[0].nodeName(), c.offset)
-			m = c.match
+		// } else {
-		}
+		items[0].parse(t, c)
+		m = c.match
+		// }
 
 		if !m {
 			if currentCount < ranges[0][0] {
 				// t.Out1("fail, item failed")
-				c.store.set(node.From, p.id, nil)
+				// c.store.set(node.From, p.id, nil)
 				c.fail(node.From)
 				c.include(initialOffset, p.id)
 				return
@@ -182,7 +184,7 @@ func (p *sequenceParser) parse(t Trace, c *context) {
 
 	// t.Out1("success, items parsed")
 
-	c.store.set(node.From, p.id, node)
+	// c.store.set(node.From, p.id, node)
 	for _, includedBy := range p.includedBy {
 		includedBy.storeIncluded(c, node)
 	}

store.go

@@ -38,10 +38,7 @@ func (c *store) get(offset int, id int) (*Node, bool, bool) {
 }
 
 func (c *store) set(offset int, id int, n *Node) {
-	var tc *storeEntry
+	if len(c.entries) <= offset {
-	if len(c.entries) > offset {
-		tc = c.entries[offset]
-	} else {
 		if cap(c.entries) > offset {
 			c.entries = c.entries[:offset+1]
 		} else {
@@ -50,7 +47,10 @@ func (c *store) set(offset int, id int, n *Node) {
 				c.entries = append(c.entries, nil)
 			}
 		}
+	}
 
+	tc := c.entries[offset]
+	if tc == nil {
 		tc = &storeEntry{
 			match:   &idSet{},
 			noMatch: &idSet{},
@@ -81,3 +81,169 @@ func (c *store) set(offset int, id int, n *Node) {
 
 	tc.nodes = append(tc.nodes, n)
 }
+
+func (c *store) inc() {
+}
+
+func (c *store) dec() {
+}
+
+func (c *store) get2(offset, id int) (int, bool) {
+	return 0, false
+}
+
+func (c *store) cache(offset, id int, match bool, length int) {
+}
+
+func (c *store) set2(offset, id int, match bool, length int) {
+	/*
+		c.cache(offset, id, match, length)
+		levels := c.offsetLevels[offset]
+		levels[c.currentLevel] = id
+	*/
+}
+
+/*
+package treerack
+
+type storeEntry struct {
+	match   *idSet
+	noMatch *idSet
+	nodes   []*Node
+	offset int
+}
+
+type store struct {
+	current *storeEntry
+	currentIndex int
+	entries []*storeEntry
+}
+
+func (s *store) find(offset int) *storeEntry {
+	if s.current == nil {
+		return nil
+	}
+
+	var seekPrev, seekNext bool
+	for {
+		switch {
+		case s.current.offset == offset:
+			return s.current
+		case s.current.offset < offset:
+			if seekPrev {
+				return nil
+			}
+
+			seekNext = true
+			s.currentIndex++
+			if s.currentIndex == len(s.entries) {
+				s.currentIndex = len(s.entries) - 1
+				return nil
+			}
+
+			s.current = s.entries[s.currentIndex]
+		case s.current.offset > offset:
+			if seekNext {
+				return nil
+			}
+
+			seekPrev = true
+			s.currentIndex--
+			if s.currentIndex == -1 {
+				s.currentIndex = 0
+				return nil
+			}
+
+			s.current = s.entries[s.currentIndex]
+		}
+	}
+}
+
+func (s *store) findCreate(offset int) *storeEntry {
+	entry := s.find(offset)
+	if entry != nil {
+		return entry
+	}
+
+	entry = &storeEntry{
+		match:   &idSet{},
+		noMatch: &idSet{},
+		offset: offset,
+	}
+
+	switch {
+	case s.current != nil && s.current.offset > offset:
+		s.entries = append(
+			s.entries[:s.currentIndex],
+			append([]*storeEntry{entry}, s.entries[s.currentIndex:]...)...,
+		)
+		s.current = entry
+	case s.current != nil && s.current.offset < offset:
+		s.entries = append(
+			s.entries[:s.currentIndex + 1],
+			append([]*storeEntry{entry}, s.entries[s.currentIndex + 1:]...)...,
+		)
+		s.current = entry
+		s.currentIndex++
+	default:
+		s.entries = []*storeEntry{entry}
+		s.current = entry
+		s.currentIndex = 0
+	}
+
+	return entry
+}
+
+func (s *store) get(offset int, id int) (*Node, bool, bool) {
+	entry := s.find(offset)
+	if entry == nil {
+		return nil, false, false
+	}
+
+	if entry == nil {
+		return nil, false, false
+	}
+
+	if entry.noMatch.has(id) {
+		return nil, false, true
+	}
+
+	if !entry.match.has(id) {
+		return nil, false, false
+	}
+
+	for _, n := range entry.nodes {
+		if n.id == id {
+			return n, true, true
+		}
+	}
+
+	return nil, false, false
+}
+
+func (s *store) set(offset int, id int, n *Node) {
+	entry := s.findCreate(offset)
+
+	if n == nil {
+		if entry.match.has(id) {
+			return
+		}
+
+		entry.noMatch.set(id)
+		return
+	}
+
+	entry.match.set(id)
+	for i, ni := range entry.nodes {
+		if ni.id == id {
+			if n.tokenLength() > ni.tokenLength() {
+				entry.nodes[i] = n
+			}
+
+			return
+		}
+	}
+
+	entry.nodes = append(entry.nodes, n)
+}
+*/

syntax.go

@@ -119,6 +119,8 @@ func (s *Syntax) Read(r io.Reader) error {
 	return ErrNotImplemented
 }
 
+// TODO: why normalization failed?
+
 func (s *Syntax) Init() error {
 	if s.initFailed {
 		return ErrInitFailed
@@ -155,6 +157,8 @@ func (s *Syntax) Generate(w io.Writer) error {
 	return ErrNotImplemented
 }
 
+// TODO: optimize top sequences to save memory
+
 func (s *Syntax) Parse(r io.Reader) (*Node, error) {
 	if err := s.Init(); err != nil {
 		return nil, err

test.mml

@@ -6,58 +6,49 @@ fn defaultCompare(a, b) {
 	}
 }
 
-let undefined {}
+let empty {}
 
 fn treeWithCompare(compare) {
 	compare: compare
-	node: undefined
+	node: empty
 }
 
 let tree treeWithCompare(defaultCompare)
 
+let hasLess hasField("less")
+let hasGreater hasField("greater")
+
 fn nextNode(compare, node, value) {
-	if node == undefined {
+	let greater node != empty && compare(node.value, value) > 0
-		undefined
+	switch {
-	} else {
+	case greater && hasLess(node):
-		let c compare(node.value, value)
+		nextNode(compare, node.less, value)
-		switch {
+	case greater:
-		case c > 0:
+		node
-			match node {
+	case hasGreater(node):
-			case {less}: nextNode(compare, node.less, value)
+		nextNode(compare, node.greater, value)
-			default: node
+	default:
-			}
+		empty
-		default:
-			match node {
-			case {greater}: nextNode(compare, node.greater, value)
-			default: undefined
-			}
-		}
 	}
 }
 
 fn prevNode(compare, node, value) {
-	if node == undefined {
+	let less node != empty && compare(node.value, value) < 0
-		undefined
+	switch {
-	} else {
+	case less && hasGreater(node):
-		let c compare(node.value, value)
+		prevNode(compare, node.greater, value)
-		switch {
+	case less:
-		case c < 0:
+		node
-			match node {
+	case hasLess(node):
-			case {greater}: prevNode(compare, node.greater, value)
+		prevNode(compare, node.less, value)
-			default: node
+	default:
-			}
+		empty
-		default:
-			match node {
-			case {less}: prevNode(compare, node.less, value)
-			default: undefined
-			}
-		}
 	}
 }
 
 fn findNode(compare, node, value) {
 	switch {
-	case node == undefined: false
+	case node == empty: false
 	case node.value == value: true
 	case compare(node.value, value) < 0:
 		findNode(compare, node.greater, value)
@@ -88,7 +79,7 @@ fn delNode(compare, node, value) {
 		case {greater}:
 			{ node...
 			  greater: delNode(compare, node.greater, node.value)
-			} /* -> udpateDepth() */ -> balance()
+			} -> balance()
 		default:
 			node
 		}
@@ -99,7 +90,7 @@ fn delNode(compare, node, value) {
 		case {less}:
 			{ node...
 			  less: delNode(compare, node.less, node.value)
-			} /* -> udpateDepth() */ -> balance()
+			} -> balance()
 		default:
 			node
 		}
@@ -114,28 +105,42 @@ fn delNode(compare, node, value) {
 		}
 	}
 
-	node == undefined ? undefined : delNode()
+	node == empty ? empty : delNode()
 }
 
 fn insertNode(compare, node, value) {
-	if node == undefined {
+	let (
-		{value: value}
+		empty node == empty
-	} else {
+		c empty ? 0 : compare(node.value, value)
-		let c compare(node.value, value)
+		insertGreater fn() {
-		switch {
+			node...
-		case c > 0:
+			greater: insertNode(
-			{node..., greater: insertNode(compare, node.greater, value)} /* -> updateDepth() */ -> balance()
+				compare
-		case c < 0:
+				node.greater
-			{node..., less: insertNode(compare, node.less, value)} /* -> updateDepth() */ -> balance()
+				value
-		default:
+			)
-			node
 		}
+		insertLess fn() {
+			node...
+			less: insertNode(
+				compare
+				node.less
+				value
+			)
+		}
+	)
+
+	switch {
+	case empty: {value: value}
+	case c > 0: insertGreater() -> balance()
+	case c < 0: insertLess() -> balance()
+	default: node
 	}
 }
 
 fn prevNextValue(f, tree, value) {
 	let node f(tree.compare, tree.node, value)
-	node == undefined ? value : node.value
+	node == empty ? value : node.value
 }
 
 fn update(f, tree, value) {
@@ -143,19 +148,19 @@ fn update(f, tree, value) {
 	node: f(tree.compare, tree.node, value)
 }
 
-type alias tree {compare: fn (a, a) int, node: {}}
+type alias tree {compare: fn(a, a) int, node: {}}
-type next fn (tree, a) a
+type next fn(tree, a) a
-type prev fn (tree, a) a
+type prev fn(tree, a) a
-type find fn (tree, a) bool
+type find fn(tree, a) bool
-type delete fn (tree, a) tree
+type delete fn(tree, a) tree
-type insert fn (tree, a) tree
+type insert fn(tree, a) tree
 
 let (
 	next = prevNextValue(prevNode)
 	prev = prevNextValue(nextNode)
 )
 
-fn find(tree, value) findNode(tree.compare, tree.node, value) != undefined
+fn find(tree, value) findNode(tree.compare, tree.node, value) != empty
 
 let (
 	del update(delNode)