diff --git a/lib.go b/lib.go index cd6e6be..a0eaf24 100644 --- a/lib.go +++ b/lib.go @@ -1,7 +1,10 @@ // Package textedit provides a non-regexp, streaming editor to apply basic text manipulation. package textedit -import "io" +import ( + "io" + "unicode" +) // Editor instances can be used to edit a text stream. It is expected from the implementations to be reusable // with fresh state after the Flush was called on the enclosing writer. @@ -109,6 +112,10 @@ func (w *Writer) write(r []rune) error { } for _, ri := range r { + if ri == unicode.ReplacementChar { + continue + } + rr, s := w.editor.Edit(ri, w.state) if _, err := w.out.Write([]byte(string(rr))); err != nil { w.err = err diff --git a/lib_test.go b/lib_test.go index 19c60d7..ca723d8 100644 --- a/lib_test.go +++ b/lib_test.go @@ -75,6 +75,22 @@ func TestWriteRune(t *testing.T) { } } +func TestBrokenUnicode(t *testing.T) { + var b bytes.Buffer + w := textedit.New(&b) + if _, err := w.Write([]byte("foo \xc2bar baz")); err != nil { + t.Fatal(err) + } + + if err := w.Flush(); err != nil { + t.Fatal(err) + } + + if b.String() != "foo bar baz" { + t.Fatal(b.String()) + } +} + func TestFailingWriter(t *testing.T) { t.Run("after write", func(t *testing.T) { var b bytes.Buffer