This commit is contained in:
2024-08-04 20:00:48 -03:00
parent 696227a935
commit 935e72c18e
3 changed files with 11 additions and 47 deletions

View File

@ -74,7 +74,7 @@ end
# Helper that creates lexer and tokenizes
def tokenize(lexer_name, text)
lexer = Tartrazine::Lexer.from_xml(File.read("lexers/#{lexer_name}.xml"))
lexer = Tartrazine.get_lexer(lexer_name)
collapse_tokens(lexer.tokenize(text))
end