From 84980459cf7a2cf0966572e664c1b1c579bc4a2a Mon Sep 17 00:00:00 2001 From: Roberto Alsina Date: Mon, 9 Sep 2024 12:22:11 -0300 Subject: [PATCH] feat: use the native crystal highlighter The chroma highlighter for crystal is not great, because the pygments one special cases things like heredocs and that got lost in translation. Since the crystal compiler comes with a highlighter why not use it? --- lexers/crystal.xml | 762 ------------------ spec/tests/crystal/test_annotation.txt | 16 - spec/tests/crystal/test_array_access.txt | 11 - spec/tests/crystal/test_chars.txt | 25 - .../crystal/test_constant_and_module.txt | 14 - .../crystal/test_empty_percent_strings.txt | 27 - .../crystal/test_escaped_bracestring.txt | 19 - .../crystal/test_escaped_interpolation.txt | 9 - .../test_interpolation_nested_curly.txt | 56 -- spec/tests/crystal/test_lib.txt | 58 -- spec/tests/crystal/test_macro.txt | 76 -- spec/tests/crystal/test_numbers.txt | 84 -- spec/tests/crystal/test_operator_methods.txt | 18 - spec/tests/crystal/test_percent_strings.txt | 41 - .../crystal/test_percent_strings_special.txt | 31 - spec/tests/crystal/test_pseudo_builtins.txt | 20 - spec/tests/crystal/test_pseudo_keywords.txt | 50 -- spec/tests/crystal/test_range_syntax1.txt | 8 - spec/tests/crystal/test_range_syntax2.txt | 10 - spec/tests/crystal/test_string_escapes.txt | 58 -- spec/tests/crystal/test_symbols.txt | 20 - src/lexer.cr | 83 ++ src/rules.cr | 4 - src/styles.cr | 21 - 24 files changed, 83 insertions(+), 1438 deletions(-) delete mode 100644 lexers/crystal.xml delete mode 100644 spec/tests/crystal/test_annotation.txt delete mode 100644 spec/tests/crystal/test_array_access.txt delete mode 100644 spec/tests/crystal/test_chars.txt delete mode 100644 spec/tests/crystal/test_constant_and_module.txt delete mode 100644 spec/tests/crystal/test_empty_percent_strings.txt delete mode 100644 spec/tests/crystal/test_escaped_bracestring.txt delete mode 100644 spec/tests/crystal/test_escaped_interpolation.txt delete mode 100644 spec/tests/crystal/test_interpolation_nested_curly.txt delete mode 100644 spec/tests/crystal/test_lib.txt delete mode 100644 spec/tests/crystal/test_macro.txt delete mode 100644 spec/tests/crystal/test_numbers.txt delete mode 100644 spec/tests/crystal/test_operator_methods.txt delete mode 100644 spec/tests/crystal/test_percent_strings.txt delete mode 100644 spec/tests/crystal/test_percent_strings_special.txt delete mode 100644 spec/tests/crystal/test_pseudo_builtins.txt delete mode 100644 spec/tests/crystal/test_pseudo_keywords.txt delete mode 100644 spec/tests/crystal/test_range_syntax1.txt delete mode 100644 spec/tests/crystal/test_range_syntax2.txt delete mode 100644 spec/tests/crystal/test_string_escapes.txt delete mode 100644 spec/tests/crystal/test_symbols.txt diff --git a/lexers/crystal.xml b/lexers/crystal.xml deleted file mode 100644 index 76e299c..0000000 --- a/lexers/crystal.xml +++ /dev/null @@ -1,762 +0,0 @@ - - - Crystal - cr - crystal - *.cr - text/x-crystal - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spec/tests/crystal/test_annotation.txt b/spec/tests/crystal/test_annotation.txt deleted file mode 100644 index b585fa1..0000000 --- a/spec/tests/crystal/test_annotation.txt +++ /dev/null @@ -1,16 +0,0 @@ ----input--- -@[FOO::Bar::Baz(opt: "xx")] - ----tokens--- -'@[' Operator -'FOO::Bar::Baz' Name.Decorator -'(' Punctuation -'opt' Literal.String.Symbol -':' Punctuation -' ' Text.Whitespace -'"' Literal.String.Double -'xx' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -']' Operator -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_array_access.txt b/spec/tests/crystal/test_array_access.txt deleted file mode 100644 index 6b36c51..0000000 --- a/spec/tests/crystal/test_array_access.txt +++ /dev/null @@ -1,11 +0,0 @@ ----input--- -[5][5]? - ----tokens--- -'[' Operator -'5' Literal.Number.Integer -']' Operator -'[' Operator -'5' Literal.Number.Integer -']?' Operator -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_chars.txt b/spec/tests/crystal/test_chars.txt deleted file mode 100644 index c254bfb..0000000 --- a/spec/tests/crystal/test_chars.txt +++ /dev/null @@ -1,25 +0,0 @@ ----input--- -'a' -'я' -'\u{1234}' -' -' -'abc' - ----tokens--- -"'a'" Literal.String.Char -'\n' Text.Whitespace - -"'я'" Literal.String.Char -'\n' Text.Whitespace - -"'\\u{1234}'" Literal.String.Char -'\n' Text.Whitespace - -"'\n'" Literal.String.Char -'\n' Text.Whitespace - -"'" Error -'abc' Name -"'" Error -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_constant_and_module.txt b/spec/tests/crystal/test_constant_and_module.txt deleted file mode 100644 index f8d33ff..0000000 --- a/spec/tests/crystal/test_constant_and_module.txt +++ /dev/null @@ -1,14 +0,0 @@ ----input--- -HTTP -HTTP::Server.new - ----tokens--- -'HTTP' Name.Constant -'\n' Text.Whitespace - -'HTTP' Name -'::' Operator -'Server' Name -'.' Operator -'new' Name -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_empty_percent_strings.txt b/spec/tests/crystal/test_empty_percent_strings.txt deleted file mode 100644 index 3cc0844..0000000 --- a/spec/tests/crystal/test_empty_percent_strings.txt +++ /dev/null @@ -1,27 +0,0 @@ ----input--- -%() -%[] -%{} -%<> -%|| - ----tokens--- -'%(' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace - -'%[' Literal.String.Other -']' Literal.String.Other -'\n' Text.Whitespace - -'%{' Literal.String.Other -'}' Literal.String.Other -'\n' Text.Whitespace - -'%<' Literal.String.Other -'>' Literal.String.Other -'\n' Text.Whitespace - -'%|' Literal.String.Other -'|' Literal.String.Other -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_escaped_bracestring.txt b/spec/tests/crystal/test_escaped_bracestring.txt deleted file mode 100644 index 14718b9..0000000 --- a/spec/tests/crystal/test_escaped_bracestring.txt +++ /dev/null @@ -1,19 +0,0 @@ ----input--- -str.gsub(%r{\\\\}, "/") - ----tokens--- -'str' Name -'.' Operator -'gsub' Name -'(' Punctuation -'%r{' Literal.String.Regex -'\\\\' Literal.String.Regex -'\\\\' Literal.String.Regex -'}' Literal.String.Regex -',' Punctuation -' ' Text.Whitespace -'"' Literal.String.Double -'/' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_escaped_interpolation.txt b/spec/tests/crystal/test_escaped_interpolation.txt deleted file mode 100644 index c623464..0000000 --- a/spec/tests/crystal/test_escaped_interpolation.txt +++ /dev/null @@ -1,9 +0,0 @@ ----input--- -"\#{a + b}" - ----tokens--- -'"' Literal.String.Double -'\\#' Literal.String.Escape -'{a + b}' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_interpolation_nested_curly.txt b/spec/tests/crystal/test_interpolation_nested_curly.txt deleted file mode 100644 index f4a69f7..0000000 --- a/spec/tests/crystal/test_interpolation_nested_curly.txt +++ /dev/null @@ -1,56 +0,0 @@ ----input--- -"A#{ (3..5).group_by { |x| x/2}.map do |k,v| "#{k}" end.join }" + "Z" - ----tokens--- -'"' Literal.String.Double -'A' Literal.String.Double -'#{' Literal.String.Interpol -' ' Text.Whitespace -'(' Punctuation -'3' Literal.Number.Integer -'..' Operator -'5' Literal.Number.Integer -')' Punctuation -'.' Operator -'group_by' Name -' ' Text.Whitespace -'{' Literal.String.Interpol -' ' Text.Whitespace -'|' Operator -'x' Name -'|' Operator -' ' Text.Whitespace -'x' Name -'/' Operator -'2' Literal.Number.Integer -'}' Literal.String.Interpol -'.' Operator -'map' Name -' ' Text.Whitespace -'do' Keyword -' ' Text.Whitespace -'|' Operator -'k' Name -',' Punctuation -'v' Name -'|' Operator -' ' Text.Whitespace -'"' Literal.String.Double -'#{' Literal.String.Interpol -'k' Name -'}' Literal.String.Interpol -'"' Literal.String.Double -' ' Text.Whitespace -'end' Keyword -'.' Operator -'join' Name -' ' Text.Whitespace -'}' Literal.String.Interpol -'"' Literal.String.Double -' ' Text.Whitespace -'+' Operator -' ' Text.Whitespace -'"' Literal.String.Double -'Z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_lib.txt b/spec/tests/crystal/test_lib.txt deleted file mode 100644 index 6f6f107..0000000 --- a/spec/tests/crystal/test_lib.txt +++ /dev/null @@ -1,58 +0,0 @@ ----input--- -@[Link("some")] -lib LibSome -@[CallConvention("X86_StdCall")] -fun foo="some.foo"(thing : Void*) : LibC::Int -end - ----tokens--- -'@[' Operator -'Link' Name.Decorator -'(' Punctuation -'"' Literal.String.Double -'some' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -']' Operator -'\n' Text.Whitespace - -'lib' Keyword -' ' Text.Whitespace -'LibSome' Name.Namespace -'\n' Text.Whitespace - -'@[' Operator -'CallConvention' Name.Decorator -'(' Punctuation -'"' Literal.String.Double -'X86_StdCall' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -']' Operator -'\n' Text.Whitespace - -'fun' Keyword -' ' Text.Whitespace -'foo' Name.Function -'=' Operator -'"' Literal.String.Double -'some.foo' Literal.String.Double -'"' Literal.String.Double -'(' Punctuation -'thing' Name -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'Void' Name -'*' Operator -')' Punctuation -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'LibC' Name -'::' Operator -'Int' Name -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_macro.txt b/spec/tests/crystal/test_macro.txt deleted file mode 100644 index 765caa2..0000000 --- a/spec/tests/crystal/test_macro.txt +++ /dev/null @@ -1,76 +0,0 @@ ----input--- -def<=>(other : self) : Int -{%for field in %w(first_name middle_name last_name)%} -cmp={{field.id}}<=>other.{{field.id}} -return cmp if cmp!=0 -{%end%} -0 -end - ----tokens--- -'def' Keyword -'<=>' Name.Function -'(' Punctuation -'other' Name -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'self' Keyword -')' Punctuation -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'Int' Name -'\n' Text.Whitespace - -'{%' Literal.String.Interpol -'for' Keyword -' ' Text.Whitespace -'field' Name -' ' Text.Whitespace -'in' Keyword -' ' Text.Whitespace -'%w(' Literal.String.Other -'first_name middle_name last_name' Literal.String.Other -')' Literal.String.Other -'%}' Literal.String.Interpol -'\n' Text.Whitespace - -'cmp' Name -'=' Operator -'{{' Literal.String.Interpol -'field' Name -'.' Operator -'id' Name -'}}' Literal.String.Interpol -'<=>' Operator -'other' Name -'.' Operator -'{{' Literal.String.Interpol -'field' Name -'.' Operator -'id' Name -'}}' Literal.String.Interpol -'\n' Text.Whitespace - -'return' Keyword -' ' Text.Whitespace -'cmp' Name -' ' Text.Whitespace -'if' Keyword -' ' Text.Whitespace -'cmp' Name -'!=' Operator -'0' Literal.Number.Integer -'\n' Text.Whitespace - -'{%' Literal.String.Interpol -'end' Keyword -'%}' Literal.String.Interpol -'\n' Text.Whitespace - -'0' Literal.Number.Integer -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_numbers.txt b/spec/tests/crystal/test_numbers.txt deleted file mode 100644 index 0ddfec2..0000000 --- a/spec/tests/crystal/test_numbers.txt +++ /dev/null @@ -1,84 +0,0 @@ ----input--- -# Integers -0 -1 -1_000_000 -1u8 -11231231231121312i64 - -# Floats -0.0 -1.0_f32 -1_f32 -0f64 -1e+4 -1e111 -1_234.567_890 - -# Error -01 -0b2 -0x129g2 -0o12358 - ----tokens--- -'# Integers' Comment.Single -'\n' Text.Whitespace - -'0' Literal.Number.Integer -'\n' Text.Whitespace - -'1' Literal.Number.Integer -'\n' Text.Whitespace - -'1_000_000' Literal.Number.Integer -'\n' Text.Whitespace - -'1u8' Literal.Number.Integer -'\n' Text.Whitespace - -'11231231231121312i64' Literal.Number.Integer -'\n\n' Text.Whitespace - -'# Floats' Comment.Single -'\n' Text.Whitespace - -'0.0' Literal.Number.Float -'\n' Text.Whitespace - -'1.0_f32' Literal.Number.Float -'\n' Text.Whitespace - -'1_f32' Literal.Number.Float -'\n' Text.Whitespace - -'0f64' Literal.Number.Float -'\n' Text.Whitespace - -'1e+4' Literal.Number.Float -'\n' Text.Whitespace - -'1e111' Literal.Number.Float -'\n' Text.Whitespace - -'1_234.567_890' Literal.Number.Float -'\n\n' Text.Whitespace - -'# Error' Comment.Single -'\n' Text.Whitespace - -'0' Error -'1' Literal.Number.Integer -'\n' Text.Whitespace - -'0' Error -'b2' Name -'\n' Text.Whitespace - -'0' Error -'x129g2' Name -'\n' Text.Whitespace - -'0' Error -'o12358' Name -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_operator_methods.txt b/spec/tests/crystal/test_operator_methods.txt deleted file mode 100644 index 084771f..0000000 --- a/spec/tests/crystal/test_operator_methods.txt +++ /dev/null @@ -1,18 +0,0 @@ ----input--- -([] of Int32).[]?(5) - ----tokens--- -'(' Punctuation -'[' Operator -']' Operator -' ' Text.Whitespace -'of' Keyword -' ' Text.Whitespace -'Int32' Name -')' Punctuation -'.' Operator -'[]?' Name.Operator -'(' Punctuation -'5' Literal.Number.Integer -')' Punctuation -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_percent_strings.txt b/spec/tests/crystal/test_percent_strings.txt deleted file mode 100644 index 80f247c..0000000 --- a/spec/tests/crystal/test_percent_strings.txt +++ /dev/null @@ -1,41 +0,0 @@ ----input--- -%(hello ("world")) -%[hello ["world"]] -%{hello "world"} -%> -%|hello "world"| - ----tokens--- -'%(' Literal.String.Other -'hello ' Literal.String.Other -'(' Literal.String.Other -'"world"' Literal.String.Other -')' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace - -'%[' Literal.String.Other -'hello ' Literal.String.Other -'[' Literal.String.Other -'"world"' Literal.String.Other -']' Literal.String.Other -']' Literal.String.Other -'\n' Text.Whitespace - -'%{' Literal.String.Other -'hello "world"' Literal.String.Other -'}' Literal.String.Other -'\n' Text.Whitespace - -'%<' Literal.String.Other -'hello ' Literal.String.Other -'<' Literal.String.Other -'"world"' Literal.String.Other -'>' Literal.String.Other -'>' Literal.String.Other -'\n' Text.Whitespace - -'%|' Literal.String.Other -'hello "world"' Literal.String.Other -'|' Literal.String.Other -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_percent_strings_special.txt b/spec/tests/crystal/test_percent_strings_special.txt deleted file mode 100644 index 4ca1c0b..0000000 --- a/spec/tests/crystal/test_percent_strings_special.txt +++ /dev/null @@ -1,31 +0,0 @@ ----input--- -%Q(hello \n #{name}) -%q(hello \n #{name}) -%w(foo\nbar baz) - ----tokens--- -'%Q(' Literal.String.Other -'hello ' Literal.String.Other -'\\n' Literal.String.Escape -' ' Literal.String.Other -'#{' Literal.String.Interpol -'name' Name -'}' Literal.String.Interpol -')' Literal.String.Other -'\n' Text.Whitespace - -'%q(' Literal.String.Other -'hello ' Literal.String.Other -'\\' Literal.String.Other -'n ' Literal.String.Other -'#' Literal.String.Other -'{name}' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace - -'%w(' Literal.String.Other -'foo' Literal.String.Other -'\\' Literal.String.Other -'nbar baz' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_pseudo_builtins.txt b/spec/tests/crystal/test_pseudo_builtins.txt deleted file mode 100644 index 9c8c917..0000000 --- a/spec/tests/crystal/test_pseudo_builtins.txt +++ /dev/null @@ -1,20 +0,0 @@ ----input--- -record Cls do -def_equals s -end - ----tokens--- -'record' Name.Builtin.Pseudo -' ' Text.Whitespace -'Cls' Name -' ' Text.Whitespace -'do' Keyword -'\n' Text.Whitespace - -'def_equals' Name.Builtin.Pseudo -' ' Text.Whitespace -'s' Name -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_pseudo_keywords.txt b/spec/tests/crystal/test_pseudo_keywords.txt deleted file mode 100644 index 18827ad..0000000 --- a/spec/tests/crystal/test_pseudo_keywords.txt +++ /dev/null @@ -1,50 +0,0 @@ ----input--- -def f(x : T, line = __LINE__) forall T -if x.is_a?(String) -pp! x -end -end - ----tokens--- -'def' Keyword -' ' Text.Whitespace -'f' Name.Function -'(' Punctuation -'x' Name -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'T' Name -',' Punctuation -' ' Text.Whitespace -'line' Name -' ' Text.Whitespace -'=' Operator -' ' Text.Whitespace -'__LINE__' Keyword.Pseudo -')' Punctuation -' ' Text.Whitespace -'forall' Keyword.Pseudo -' ' Text.Whitespace -'T' Name -'\n' Text.Whitespace - -'if' Keyword -' ' Text.Whitespace -'x' Name -'.is_a?' Keyword.Pseudo -'(' Punctuation -'String' Name -')' Punctuation -'\n' Text.Whitespace - -'pp!' Name.Builtin.Pseudo -' ' Text.Whitespace -'x' Name -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_range_syntax1.txt b/spec/tests/crystal/test_range_syntax1.txt deleted file mode 100644 index a3ba24a..0000000 --- a/spec/tests/crystal/test_range_syntax1.txt +++ /dev/null @@ -1,8 +0,0 @@ ----input--- -1...3 - ----tokens--- -'1' Literal.Number.Integer -'...' Operator -'3' Literal.Number.Integer -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_range_syntax2.txt b/spec/tests/crystal/test_range_syntax2.txt deleted file mode 100644 index 08bf4b1..0000000 --- a/spec/tests/crystal/test_range_syntax2.txt +++ /dev/null @@ -1,10 +0,0 @@ ----input--- -1 .. 3 - ----tokens--- -'1' Literal.Number.Integer -' ' Text.Whitespace -'..' Operator -' ' Text.Whitespace -'3' Literal.Number.Integer -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_string_escapes.txt b/spec/tests/crystal/test_string_escapes.txt deleted file mode 100644 index 3ea12f4..0000000 --- a/spec/tests/crystal/test_string_escapes.txt +++ /dev/null @@ -1,58 +0,0 @@ ----input--- -"a\nz" -"a\az" -"a\xffz" -"a\u1234z" -"a\000z" -"a\u{0}z" -"a\u{10AfF9}z" - ----tokens--- -'"' Literal.String.Double -'a' Literal.String.Double -'\\n' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\a' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\xff' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\u1234' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\000' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\u{0}' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\u{10AfF9}' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_symbols.txt b/spec/tests/crystal/test_symbols.txt deleted file mode 100644 index 3d2cb0a..0000000 --- a/spec/tests/crystal/test_symbols.txt +++ /dev/null @@ -1,20 +0,0 @@ ----input--- -:sym_bol -:あ -:question? -:"symbol" - ----tokens--- -':sym_bol' Literal.String.Symbol -'\n' Text.Whitespace - -':あ' Literal.String.Symbol -'\n' Text.Whitespace - -':question?' Literal.String.Symbol -'\n' Text.Whitespace - -':"' Literal.String.Symbol -'symbol' Literal.String.Symbol -'"' Literal.String.Symbol -'\n' Text.Whitespace diff --git a/src/lexer.cr b/src/lexer.cr index 1e6197d..6ec8522 100644 --- a/src/lexer.cr +++ b/src/lexer.cr @@ -1,6 +1,7 @@ require "./constants/lexers" require "./heuristics" require "baked_file_system" +require "crystal/syntax_highlighter" module Tartrazine class LexerFiles @@ -26,6 +27,7 @@ module Tartrazine end private def self.lexer_by_name(name : String) : BaseLexer + return CrystalLexer.new if name == "crystal" lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil) return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+" raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil? @@ -34,6 +36,10 @@ module Tartrazine end private def self.lexer_by_filename(filename : String) : BaseLexer + if filename.ends_with?(".cr") + return CrystalLexer.new + end + candidates = Set(String).new LEXERS_BY_FILENAME.each do |k, v| candidates += v.to_set if File.match?(k, File.basename(filename)) @@ -327,4 +333,81 @@ module Tartrazine new_state end end + + class CustomCrystalHighlighter < Crystal::SyntaxHighlighter + @tokens = [] of Token + + def render_delimiter(&block) + @tokens << {type: "LiteralString", value: block.call.to_s} + end + + def render_interpolation(&block) + @tokens << {type: "LiteralStringInterpol", value: "\#{"} + @tokens << {type: "Text", value: block.call.to_s} + @tokens << {type: "LiteralStringInterpol", value: "}"} + end + + def render_string_array(&block) + @tokens << {type: "LiteralString", value: block.call.to_s} + end + + # ameba:disable Metrics/CyclomaticComplexity + def render(type : TokenType, value : String) + case type + when .comment? + @tokens << {type: "Comment", value: value} + when .number? + @tokens << {type: "LiteralNumber", value: value} + when .char? + @tokens << {type: "LiteralStringChar", value: value} + when .symbol? + @tokens << {type: "LiteralStringSymbol", value: value} + when .const? + @tokens << {type: "NameConstant", value: value} + when .string? + @tokens << {type: "LiteralString", value: value} + when .ident? + @tokens << {type: "NameVariable", value: value} + when .keyword?, .self? + @tokens << {type: "NameKeyword", value: value} + when .primitive_literal? + @tokens << {type: "Literal", value: value} + when .operator? + @tokens << {type: "Operator", value: value} + when Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN, Crystal::SyntaxHighlighter::TokenType::DELIMITER_START, Crystal::SyntaxHighlighter::TokenType::DELIMITER_END + @tokens << {type: "LiteralString", value: value} + else + @tokens << {type: "Text", value: value} + end + end + end + + class CrystalTokenizer < Tartrazine::BaseTokenizer + include Iterator(Token) + @hl = CustomCrystalHighlighter.new + @lexer : BaseLexer + @iter : Iterator(Token) + + # delegate next, to: @iter + + def initialize(@lexer : BaseLexer, text : String, secondary = false) + # Respect the `ensure_nl` config option + if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary + text += "\n" + end + # Just do the tokenizing + @hl.highlight(text) + @iter = @hl.@tokens.each + end + + def next : Iterator::Stop | Token + @iter.next + end + end + + class CrystalLexer < BaseLexer + def tokenizer(text : String, secondary = false) : BaseTokenizer + CrystalTokenizer.new(self, text, secondary) + end + end end diff --git a/src/rules.cr b/src/rules.cr index 6eaa8d7..167ccee 100644 --- a/src/rules.cr +++ b/src/rules.cr @@ -17,7 +17,6 @@ module Tartrazine abstract struct BaseRule abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token)) - abstract def initialize(node : XML::Node) @actions : Array(Action) = [] of Action @@ -40,9 +39,6 @@ module Tartrazine return true, pos + match[0].size, @actions.flat_map(&.emit(match, tokenizer)) end - def initialize(node : XML::Node) - end - def initialize(node : XML::Node, multiline, dotall, ignorecase) pattern = node["pattern"] pattern = "(?m)" + pattern if multiline diff --git a/src/styles.cr b/src/styles.cr index fd52d4a..a0bc70e 100644 --- a/src/styles.cr +++ b/src/styles.cr @@ -84,27 +84,6 @@ module Tartrazine property styles = {} of String => Style - # Get the style for a token. - def style(token) - styles[token] = Style.new unless styles.has_key?(token) - s = styles[token] - - # We already got the data from the style hierarchy - return s if s.complete? - - # Form the hierarchy of parent styles - parents = style_parents(token) - - s = parents.map do |parent| - styles[parent] - end.reduce(s) do |acc, style| - acc + style - end - s.complete = true - styles[token] = s - s - end - def style_parents(token) parents = ["Background"] parts = token.underscore.split("_").map(&.capitalize)