From 3ea961e5ab26e6f097523b62aab3376452f477b9 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Sat, 28 Mar 2020 22:47:57 +0100 Subject: [PATCH] generator: change-detector tests on EOL-dependant sample Signed-off-by: Alexander Bezzubov --- internal/code-generator/generator/generator_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/code-generator/generator/generator_test.go b/internal/code-generator/generator/generator_test.go index 4f8b105..b243abf 100644 --- a/internal/code-generator/generator/generator_test.go +++ b/internal/code-generator/generator/generator_test.go @@ -9,7 +9,10 @@ import ( "strings" "testing" + "github.com/go-enry/go-enry/v2/internal/tokenizer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -311,6 +314,16 @@ func (s *GeneratorTestSuite) TestGenerationFiles() { } } +func (s *GeneratorTestSuite) TestTokenizerOnATS() { + const suspiciousSample = "samples/ATS/csv_parse.hats" + sFile := filepath.Join(s.tmpLinguist, suspiciousSample) + content, err := ioutil.ReadFile(sFile) + require.NoError(s.T(), err) + + tokens := tokenizer.Tokenize(content) + assert.Equal(s.T(), 381, len(tokens), "Number of tokens using LF as line endings") +} + // normalizeSpaces returns a copy of str with whitespaces normalized. // We use this to compare generated source as gofmt format may change. // E.g for changes between Go 1.10 and 1.11 see