Use classes instead of structs to allow properties of the same type

This commit is contained in:
2024-08-22 21:52:59 -03:00
parent 0f3b7fc3c5
commit bd3df10d2c
6 changed files with 57 additions and 41 deletions

View File

@ -72,8 +72,7 @@ end
# Helper that creates lexer and tokenizes
def tokenize(lexer_name, text)
lexer = Tartrazine.lexer(lexer_name)
tokenizer = Tartrazine::Tokenizer.new(lexer, text)
tokenizer = Tartrazine.lexer(lexer_name).tokenizer(text)
Tartrazine::Lexer.collapse_tokens(tokenizer.to_a)
end