From 7524f4330a1fa3c5012e68d664df9e23cacfbb3c Mon Sep 17 00:00:00 2001
From: Roberto Alsina <roberto.alsina@gmail.com>
Date: Sat, 3 Aug 2024 09:42:32 -0300
Subject: [PATCH] Let's go nuts and try to tokenize ALL THE PYGMENTS TESTS

---
 src/tartrazine.cr | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/tartrazine.cr b/src/tartrazine.cr
index 97c5da3..d419d29 100644
--- a/src/tartrazine.cr
+++ b/src/tartrazine.cr
@@ -227,14 +227,12 @@ end
 lexers = {} of String => Tartrazine::Lexer
 Dir.glob("lexers/*.xml").each do |fname|
   l = Tartrazine::Lexer.from_xml(File.read(fname))
-  lexers[l.config[:name]] = l
+  lexers[l.config[:name].downcase] = l
+  l.config[:aliases].each do |key|
+    lexers[key.downcase] = l
+  end
 end
 
-# Parse some plaintext
-puts lexers["plaintext"].tokenize("Hello, world!\n")
-
-# Now some bash
-puts lexers["Bash"].tokenize("echo 'Hello, world!'\n")
 
 # Convenience macros to parse XML
 macro xml_to_s(node, name)
@@ -248,3 +246,20 @@ end
 macro xml_to_a(node, name)
 {{node}}.children.select{|n| n.name == "{{name}}".lstrip("_")}.map {|n| n.content.to_s}
 end
+
+
+
+# Let's run some tests
+
+Dir.glob("tests/*/") do |lexername|
+  key = File.basename(lexername).downcase
+  next unless lexers.has_key? key
+  lexer = lexers[key]
+
+  Dir.glob("#{lexername}*.txt") do |testname|
+    test = File.read(testname).split("---input---\n").last.split("--tokens---").first
+    tokens = lexer.tokenize(test)
+    puts "Testing #{key} with #{testname}"
+    # puts tokens
+  end
+end
\ No newline at end of file