ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2025-08-30 02:07:30 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

refactor to build tags

Browse Source 
Signed-off-by: Alexander Bezzubov <bzz@apache.org>
This commit is contained in:
Alexander Bezzubov
2019-03-24 18:55:05 +01:00
parent 553399ed76
commit 8756fbdcb4
10 changed files with 141 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
internal/tokenizer/tokenize_test.go
View File

@@ -91,7 +91,7 @@ var (
"-", "|", "+", "&&", "<", "<", "-", "!", "!", "!", "=", "=", "!", ":", "=", ":", "=", ",", ",", "=", ">", ">", "=", "=", "=", "=", ">",
"'", ",", ">", "=", ">", "=", "=", ">", "=", ">", ":", ">", "=", ">"}
tests = []struct {
Tests = []struct {
name string
content []byte
expected []string
@@ -101,10 +101,10 @@ var (
)
func TestTokenize(t *testing.T) {
for _, test := range tests {
for _, test := range Tests {
t.Run(test.name, func(t *testing.T) {
before := string(test.content)
tokens := TokenizeFlex(test.content)
tokens := Tokenize(test.content)
after := string(test.content)
require.Equal(t, before, after, "the input slice was modified")
require.Equal(t, len(test.expected), len(tokens), fmt.Sprintf("token' slice length = %v, want %v", len(test.expected), len(tokens)))
@@ -118,39 +118,21 @@ func TestTokenize(t *testing.T) {
func BenchmarkTokenizer_BaselineCopy(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
for _, test := range tests {
for _, test := range Tests {
test.content = append([]byte(nil), test.content...)
}
}
}
func BenchmarkTokenizerGo(b *testing.B) {
func BenchmarkTokenizer(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
for _, test := range tests {
for _, test := range Tests {
Tokenize(test.content)
}
}
}
func BenchmarkTokenizerC(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
for _, test := range tests {
TokenizeC(test.content)
}
}
}
func BenchmarkTokenizerFlex(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
for _, test := range tests {
TokenizeFlex(test.content)
}
}
}
//TODO(bzz): introduce tokenizer benchmark suit
// baseline - just read the files
// RE2