ignore broken unicode chars
This commit is contained in:
parent
cb3240747d
commit
71640ab28f
9
lib.go
9
lib.go
@ -1,7 +1,10 @@
|
||||
// Package textedit provides a non-regexp, streaming editor to apply basic text manipulation.
|
||||
package textedit
|
||||
|
||||
import "io"
|
||||
import (
|
||||
"io"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Editor instances can be used to edit a text stream. It is expected from the implementations to be reusable
|
||||
// with fresh state after the Flush was called on the enclosing writer.
|
||||
@ -109,6 +112,10 @@ func (w *Writer) write(r []rune) error {
|
||||
}
|
||||
|
||||
for _, ri := range r {
|
||||
if ri == unicode.ReplacementChar {
|
||||
continue
|
||||
}
|
||||
|
||||
rr, s := w.editor.Edit(ri, w.state)
|
||||
if _, err := w.out.Write([]byte(string(rr))); err != nil {
|
||||
w.err = err
|
||||
|
||||
16
lib_test.go
16
lib_test.go
@ -75,6 +75,22 @@ func TestWriteRune(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrokenUnicode(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
w := textedit.New(&b)
|
||||
if _, err := w.Write([]byte("foo \xc2bar baz")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := w.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if b.String() != "foo bar baz" {
|
||||
t.Fatal(b.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailingWriter(t *testing.T) {
|
||||
t.Run("after write", func(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
|
||||
Loading…
Reference in New Issue
Block a user