buffer/readutf8_test.go

package buffer_test

import (
	"testing"
	"code.squareroundforest.org/arpio/buffer"
	"errors"
	"io"
)

func TestReadUTF8(t *testing.T) {
	t.Run("read all after error", func(t *testing.T) {
		g := &gen{
			rng:     utf8Range,
			max:     24,
			fastErr: true,
		}

		o := buffer.Options{Pool: buffer.NoPool(1 << 12)}
		r := buffer.ReaderFrom(g, o)
		runes, n0, err := r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 12 {
			t.Fatal("short read", len(runes))
		}

		if string(runes) != string(generateFrom(utf8Range, n0)) {
			t.Fatal("invalid content")
		}

		runes, n1, err := r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 3 {
			t.Fatal("short read", len(runes))
		}

		if string(runes) != string(generateFrom(utf8Range, n0+n1)[n0:]) {
			t.Fatal("invalid content", string(runes), string(generateFrom(utf8Range, n0+n1)[n0:]))
		}

		runes, n2, err := r.ReadUTF8(12)
		if !errors.Is(err, io.EOF) {
			t.Fatal(err)
		}

		if n2 != 0 {
			t.Fatal("unexpected consumption")
		}

		if len(runes) != 0 {
			t.Fatal("unexpected read", len(runes))
		}
	})

	t.Run("ascii", func(t *testing.T) {
		g := &gen{max: 1 << 15}
		o := buffer.Options{Pool: buffer.NoPool(1 << 12)}
		r := buffer.ReaderFrom(g, o)
		runes, n, err := r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 12 {
			t.Fatal("short read", len(runes))
		}

		if string(runes) != string(generate(n)) {
			t.Fatal("invalid content")
		}
	})

	t.Run("long within segment", func(t *testing.T) {
		g := &gen{
			rng: utf8W2Range,
			max: 1 << 15,
		}

		o := buffer.Options{Pool: buffer.NoPool(1 << 12)}
		r := buffer.ReaderFrom(g, o)
		runes, n, err := r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 12 {
			t.Fatal("short read", len(runes))
		}

		if string(runes) != string(generateFrom(utf8W2Range, n)) {
			t.Fatal("invalid content")
		}
	})

	t.Run("long across segments", func(t *testing.T) {
		g := &gen{
			rng: utf8W2Range,
			max: 1 << 15,
		}

		o := buffer.Options{Pool:        buffer.NoPool(9)}

		r := buffer.ReaderFrom(g, o)
		runes, n, err := r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 12 {
			t.Fatal("short read", len(runes))
		}

		if string(runes) != string(generateFrom(utf8W2Range, n)) {
			t.Fatal("invalid content")
		}
	})

	t.Run("null read", func(t *testing.T) {
		const numRunes = 6
		nullReadAfter := len(string([]rune(string(utf8W2Range))[:numRunes]))
		g := &gen{
			rng:           utf8W2Range,
			max:           1 << 15,
			nullReadAfter: []int{nullReadAfter, nullReadAfter},
		}

		o := buffer.Options{
			Pool:        buffer.NoPool(nullReadAfter),
			ReadSize:    nullReadAfter,
		}

		r := buffer.ReaderFrom(g, o)
		runes, _, err := r.ReadUTF8(numRunes - 2) // -2 for min read in readUTF8
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != numRunes-2 {
			t.Fatal("short read", len(runes))
		}

		if string(runes) != string(generateFrom(utf8W2Range, nullReadAfter-4)) {
			t.Fatal("invalid content")
		}

		runes, _, err = r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 2 {
			t.Fatal("short read 2", len(runes))
		}

		if string(runes) != string(generateFrom(utf8W2Range, nullReadAfter)[nullReadAfter-4:]) {
			t.Fatal("invalid content 2")
		}

		runes, _, err = r.ReadUTF8(12)
		if err != nil {
			t.Fatal(err)
		}

		if len(runes) != 12 {
			t.Fatal("short read 3", len(runes))
		}

		if string(runes) != string(generateFrom(utf8W2Range, 3*nullReadAfter)[nullReadAfter:]) {
			t.Fatal("invalid content 3")
		}
	})

	t.Run("broken unicode at the end", func(t *testing.T) {
		brokenRange := []byte{0xc3, 0xc3, 0xc3}
		g := &gen{
			rng:                utf8Range,
			max:                len(utf8Range) + len(brokenRange),
			customContentAfter: []int{len(utf8Range)},
			customContent:      map[int][]byte{len(utf8Range): brokenRange},
		}

		o := buffer.Options{Pool: buffer.NoPool(1 << 12)}
		r := buffer.ReaderFrom(g, o)
		runes, n, err := r.ReadUTF8(24)
		if err != nil {
			t.Fatal(err)
		}

		if string(runes) != string(generateFrom(utf8Range, n)) {
			t.Fatal("invalid content", string(runes), string(generateFrom(utf8Range, n)))
		}

		if len([]byte(string(runes))) != len(utf8Range) {
			t.Fatal("invalid number of bytes")
		}

		for i := 0; i < 3; i++ {
			runes, n, err = r.ReadUTF8(24)
			if len(runes) != 0 || n != 1 || err != nil {
				t.Fatal("failed to read out broken end")
			}
		}

		runes, n, err = r.ReadUTF8(24)
		if len(runes) != 0 || n != 0 || !errors.Is(err, io.EOF) {
			t.Fatal("failed to read EOF", len(runes), n, err)
		}
	})

	t.Run("immediate err", func(t *testing.T) {
		g := &gen{rng: utf8Range}
		o := buffer.Options{Pool: buffer.NoPool(1 << 12)}
		r := buffer.ReaderFrom(g, o)
		runes, n, err := r.ReadUTF8(12)
		if !errors.Is(err, io.EOF) {
			t.Fatal(err)
		}

		if n != 0 {
			t.Fatal("unexpected read", n)
		}

		if len(runes) != 0 {
			t.Fatal("unexpected content")
		}
	})
}