diff --git a/internal/code-generator/generator/generator_test.go b/internal/code-generator/generator/generator_test.go index 4f8b105..b243abf 100644 --- a/internal/code-generator/generator/generator_test.go +++ b/internal/code-generator/generator/generator_test.go @@ -9,7 +9,10 @@ import ( "strings" "testing" + "github.com/go-enry/go-enry/v2/internal/tokenizer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -311,6 +314,16 @@ func (s *GeneratorTestSuite) TestGenerationFiles() { } } +func (s *GeneratorTestSuite) TestTokenizerOnATS() { + const suspiciousSample = "samples/ATS/csv_parse.hats" + sFile := filepath.Join(s.tmpLinguist, suspiciousSample) + content, err := ioutil.ReadFile(sFile) + require.NoError(s.T(), err) + + tokens := tokenizer.Tokenize(content) + assert.Equal(s.T(), 381, len(tokens), "Number of tokens using LF as line endings") +} + // normalizeSpaces returns a copy of str with whitespaces normalized. // We use this to compare generated source as gofmt format may change. // E.g for changes between Go 1.10 and 1.11 see