Tokenize via an iterator, makes everything much faster

src/formatters/ansi.cr

@@ -7,19 +7,39 @@ module Tartrazine
     def initialize(@theme : Theme = Tartrazine.theme("default-dark"), @line_numbers : Bool = false)
     end
 
+    private def line_label(i : Int32) : String
+      "#{i + 1}".rjust(4).ljust(5)
+    end
+
     def format(text : String, lexer : Lexer) : String
+      tokenizer = Tokenizer.new(lexer, text)
+      i = 0
       output = String.build do |outp|
-        lexer.group_tokens_in_lines(lexer.tokenize(text)).each_with_index do |line, i|
-          label = line_numbers? ? "#{i + 1}".rjust(4).ljust(5) : ""
-          outp << label
-          line.each do |token|
-            outp << colorize(token[:value], token[:type])
+        outp << line_label(i) if line_numbers?
+        tokenizer.each do |token|
+          outp << colorize(token[:value], token[:type])
+          if token[:value].includes?("\n")
+            i += 1
+            outp << line_label(i) if line_numbers?
           end
         end
       end
       output
     end
 
+    # def format(text : String, lexer : Lexer) : String
+    #   output = String.build do |outp|
+    #     lexer.group_tokens_in_lines(lexer.tokenize(text)).each_with_index do |line, i|
+    #       label = line_numbers? ? "#{i + 1}".rjust(4).ljust(5) : ""
+    #       outp << label
+    #       line.each do |token|
+    #         outp << colorize(token[:value], token[:type])
+    #       end
+    #     end
+    #   end
+    #   output
+    # end
+
     def colorize(text : String, token : String) : String
       style = theme.styles.fetch(token, nil)
       return text if style.nil?