ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2025-08-02 22:09:49 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

token: new test case with Unicode replacement

Browse Source 
Signed-off-by: Alexander Bezzubov <bzz@apache.org>
This commit is contained in:
Alexander Bezzubov
2019-04-17 19:28:06 +02:00
parent 278eaf1c22
commit 8bdc830833
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/tokenizer/tokenize_test.go
View File

@@ -115,6 +115,13 @@ func TestTokenize(t *testing.T) {
} }
} }
func TestTokenizerLatin1AsUtf8(t *testing.T) {
content := []byte("th\xe5 filling") // `th<74> filling`
t.Logf("%v - %q", content, string(content))
tokens := Tokenize(content)
require.Equal(t, 3, len(tokens))
}
func BenchmarkTokenizer_BaselineCopy(b *testing.B) { func BenchmarkTokenizer_BaselineCopy(b *testing.B) {
b.ReportAllocs() b.ReportAllocs()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {