From 169060e1cd54c8f0e2eccf0c6d9bcd52402f32f4 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Tue, 29 Jan 2019 10:03:09 -0800 Subject: [PATCH] Add a test that tokenization does not modify the input. At present this test fails, since the tokenizer replaces text in shared slices of the input. A subsequent commit will fix that. Signed-off-by: M. J. Fromberger --- internal/tokenizer/tokenize_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/tokenizer/tokenize_test.go b/internal/tokenizer/tokenize_test.go index 9736543..34f75db 100644 --- a/internal/tokenizer/tokenize_test.go +++ b/internal/tokenizer/tokenize_test.go @@ -102,7 +102,10 @@ func TestTokenize(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + before := string(test.content) tokens := Tokenize(test.content) + after := string(test.content) + assert.Equal(t, before, after, "the input slice was modified") assert.Equal(t, len(test.expected), len(tokens), fmt.Sprintf("token' slice length = %v, want %v", len(test.expected), len(tokens))) for i, expectedToken := range test.expected { assert.Equal(t, expectedToken, tokens[i], fmt.Sprintf("token = %v, want %v", tokens[i], expectedToken))