create arena structure

arena.go

@@ -0,0 +1,352 @@
+package treerack
+
+type arena struct {
+
+	// indicates fixed size pages from the pool
+	f bool
+
+	// tolerates pages of different size, but expects the page size to be min w + 1
+	p *pool[[]int]
+
+	// the max size of the stored data blocks. Expected to be larger than 0
+	w int
+
+	// pages storing the cursors for the indexes. The cursors point to the position of the first record
+	// for an index in a. The total range of indexes can be larger then the sum of the page lengths,
+	// depending on the offset. Initialized to -1, which means a cursor not pointing to a data item
+	c [][]int
+
+	// pages storing the records. Every record belongs to an index in c. The size of a record is w
+	// + 1, where w fields represent the data, and one field hold the cursor pointing to the next record
+	// belonging to the same index. The cursors point to the cursor position, the data fields are before the
+	// cursor position. This means 0 is not a valid value for a cursor, and represents an empty record
+	// placeholder. The pages in a are initialized to 0. Records are not split between pages
+	a [][]int
+
+	// virtual size of c. The actual size is l - o
+	l int
+
+	// offset in between the total virtual range of the indexes and the first index available in c. By
+	// default it is 0, until one or more pages are removed from the beginning of c as a result of prune.
+	// All the other indicators and the input indexes are interpreted for the total virtual range of the
+	// arena
+	o int
+
+	// available size for records. The virtual total length of the pages in a starting from the original
+	// zero.
+	m int
+
+	// virutal total size occupied by records. Effectively, the next available position in a, unless s >= m - w -
+	// 1
+	s int
+
+	// total size of pages trimmed from the arena
+	t int
+}
+
+func newArena(w int, p *pool[[]int], f bool) *arena {
+	return &arena{
+		f: f,
+		p: p,
+		w: w,
+	}
+}
+
+func (a *arena) findPosition(pages [][]int, p int) ([]int, int) {
+	if a.f {
+		if len(pages) == 0 {
+			return nil, -1
+		}
+
+		s := len(pages[0])
+		if p >= s*len(pages) {
+			return nil, -1
+		}
+
+		return pages[p/s], p % s
+	}
+
+	for i := 0; i < len(pages); i++ {
+		if p < len(pages[i]) {
+			return pages[i], p
+		}
+
+		p -= len(pages[i])
+	}
+
+	return nil, -1
+}
+
+func matchValues(v, q []int) bool {
+	for i := 0; i < len(q); i++ {
+		if v[i] != q[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (a *arena) cursor(index int) *int {
+	index -= a.o
+	if index < 0 {
+		return nil
+	}
+
+	p, i := a.findPosition(a.c, index)
+	if i < 0 {
+		return nil
+	}
+
+	return &p[i]
+}
+
+func (a *arena) stepCursor(c *int) *int {
+	p, i := a.findPosition(a.a, *c-a.t)
+	return &p[i]
+}
+
+func (a *arena) recordAt(c int) []int {
+	p, i := a.findPosition(a.a, c-a.t)
+	return p[i-a.w : i+1]
+}
+
+func (a *arena) allocCursor(index int) {
+	for index >= a.l {
+		p := a.p.get()
+		for i := range p {
+			p[i] = -1
+		}
+
+		a.c = append(a.c, p)
+		a.l += len(p)
+	}
+}
+
+func (a *arena) allocArena() {
+	if a.s+a.w+1 <= a.m {
+		return
+	}
+
+	p := a.p.get()
+	for i := range p {
+		p[i] = 0
+	}
+
+	a.a = append(a.a, p)
+	a.m += len(p)
+}
+
+func (a *arena) append(c int, v ...int) int {
+	p := a.a[len(a.a)-1]
+	i := a.s - a.m + len(p)
+	if i < 0 {
+		i = 0
+		a.s = a.m - len(p)
+	}
+
+	copy(p[i:i+a.w], v)
+	a.s += a.w + 1
+	p[i+a.w] = c
+	return a.s - 1
+}
+
+func (a *arena) clearCursor(c *int, q ...int) {
+	if len(q) > a.w {
+		return
+	}
+
+	for c != nil && *c > 0 {
+		r := a.recordAt(*c)
+		if matchValues(r[:len(r)-1], q) {
+			*c, r[a.w] = r[a.w], 0
+			continue
+		}
+
+		c = a.stepCursor(c)
+	}
+}
+
+func (a *arena) lastActivePosition(p []int) int {
+	for i := len(p) - len(p)%(a.w+1) - 1; i > 0; i -= a.w + 1 {
+		if p[i] == 0 {
+			continue
+		}
+
+		return i
+	}
+
+	return -1
+}
+
+func (a *arena) trim() {
+	for {
+		if len(a.a) == 0 {
+			break
+		}
+
+		if a.lastActivePosition(a.a[0]) > 0 {
+			break
+		}
+
+		a.p.put(a.a[0])
+		a.t += len(a.a[0])
+		a.a = a.a[1:]
+	}
+
+	for {
+		if len(a.a) == 0 {
+			break
+		}
+
+		last := len(a.a) - 1
+		p := a.lastActivePosition(a.a[last])
+		if p > 0 {
+			a.s = a.m - len(a.a[last]) + p + 1
+			break
+		}
+
+		a.p.put(a.a[last])
+		a.m -= len(a.a[last])
+		a.s = a.m
+		a.a = a.a[:last]
+	}
+}
+
+func (a *arena) len() int {
+	return a.l
+}
+
+func (a *arena) has(index int, q ...int) bool {
+	if len(q) == 0 {
+		return true
+	}
+
+	if len(q) > a.w {
+		return false
+	}
+
+	for c := a.cursor(index); c != nil && *c > 0; c = a.stepCursor(c) {
+		r := a.recordAt(*c)
+		if matchValues(r[:len(r)-1], q) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// it does not copy the data
+func (a *arena) get(index int, q ...int) [][]int {
+	if len(q) > a.w {
+		return nil
+	}
+
+	var values [][]int
+	for c := a.cursor(index); c != nil && *c > 0; c = a.stepCursor(c) {
+		r := a.recordAt(*c)
+		v := r[:len(r)-1]
+		if matchValues(v, q) {
+			values = append(values, v)
+		}
+	}
+
+	return values
+}
+
+// it does not copy the data
+func (a *arena) set(index int, v ...int) {
+	if a.has(index, v...) {
+		return
+	}
+
+	a.allocCursor(index)
+	c := a.cursor(index)
+
+	// index can be in pruned range
+	if c == nil {
+		return
+	}
+
+	a.allocArena()
+	*c = a.append(*c, v...)
+}
+
+func (a *arena) del(index int, q ...int) {
+	c := a.cursor(index)
+	a.clearCursor(c, q...)
+	a.trim()
+}
+
+func (a *arena) prune(from, to int) {
+	var p int
+	for _, c := range a.c {
+		if p+len(c) <= from-a.o {
+			p += len(c)
+			continue
+		}
+
+		if p >= to-a.o {
+			break
+		}
+
+		for i := max(0, from-a.o-p); i < min(len(c), to-a.o-p); i++ {
+			a.clearCursor(&c[i])
+		}
+
+		p += len(c)
+	}
+
+	for {
+		if len(a.c) == 0 {
+			break
+		}
+
+		var used bool
+		c := a.c[0]
+		if a.o+len(c) > to {
+			break
+		}
+
+		for _, ci := range c {
+			if ci > 0 {
+				used = true
+				break
+			}
+		}
+
+		if used {
+			break
+		}
+
+		a.o += len(c)
+		a.p.put(c)
+		a.c = a.c[1:]
+	}
+
+	for {
+		if len(a.c) == 0 {
+			break
+		}
+
+		var used bool
+		last := len(a.c) - 1
+		c := a.c[last]
+		for _, ci := range c {
+			if ci > 0 {
+				used = true
+				break
+			}
+		}
+
+		if used {
+			break
+		}
+
+		a.l -= len(c)
+		a.p.put(c)
+		a.c = a.c[:last]
+	}
+
+	a.trim()
+}

choice.go

@@ -136,7 +136,6 @@ func (b *choiceBuilder) build(c *context) ([]Node, bool) {
 
 	from := c.offset
 	parsed := to > from
-
 	if parsed {
 		c.results.dropMatchTo(c.offset, b.id, to)
 		for _, g := range b.generalizations {

mmlexp_test.go

@@ -2980,12 +2980,7 @@ func TestMMLExp(t *testing.T) {
 }
 
 func TestMMLFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip()
-	}
-
 	const n = 180
-
 	s, err := openSyntaxFile("doc/example/mml-exp.treerack")
 	if err != nil {
 		t.Error(err)
@@ -2993,7 +2988,6 @@ func TestMMLFile(t *testing.T) {
 	}
 
 	s.Init()
-
 	b, err := os.ReadFile("doc/example/test.mml")
 	if err != nil {
 		t.Fatal(err)

pool.go

@@ -0,0 +1,25 @@
+package treerack
+
+type pool[T any] struct {
+	create func() T
+	values []T
+}
+
+func newPool[T any](create func() T) *pool[T] {
+	return &pool[T]{create: create}
+}
+
+func (p *pool[T]) get() T {
+	if len(p.values) == 0 {
+		return p.create()
+	}
+
+	last := len(p.values) - 1
+	v := p.values[last]
+	p.values = p.values[:last]
+	return v
+}
+
+func (p *pool[T]) put(v T) {
+	p.values = append(p.values, v)
+}

syntaxhead.go

@@ -216,7 +216,6 @@ func parseInput(r io.Reader, p parser, b builder, kw []parser, maxTraceLength in
 
 	c.offset = 0
 	c.results.resetPending()
-
 	n, _ := b.build(c)
 	return n[0], nil
 }