diff --git a/lexers/crystal.xml b/lexers/crystal.xml deleted file mode 100644 index 76e299c..0000000 --- a/lexers/crystal.xml +++ /dev/null @@ -1,762 +0,0 @@ - - - Crystal - cr - crystal - *.cr - text/x-crystal - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spec/tests/crystal/test_annotation.txt b/spec/tests/crystal/test_annotation.txt deleted file mode 100644 index b585fa1..0000000 --- a/spec/tests/crystal/test_annotation.txt +++ /dev/null @@ -1,16 +0,0 @@ ----input--- -@[FOO::Bar::Baz(opt: "xx")] - ----tokens--- -'@[' Operator -'FOO::Bar::Baz' Name.Decorator -'(' Punctuation -'opt' Literal.String.Symbol -':' Punctuation -' ' Text.Whitespace -'"' Literal.String.Double -'xx' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -']' Operator -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_array_access.txt b/spec/tests/crystal/test_array_access.txt deleted file mode 100644 index 6b36c51..0000000 --- a/spec/tests/crystal/test_array_access.txt +++ /dev/null @@ -1,11 +0,0 @@ ----input--- -[5][5]? - ----tokens--- -'[' Operator -'5' Literal.Number.Integer -']' Operator -'[' Operator -'5' Literal.Number.Integer -']?' Operator -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_chars.txt b/spec/tests/crystal/test_chars.txt deleted file mode 100644 index c254bfb..0000000 --- a/spec/tests/crystal/test_chars.txt +++ /dev/null @@ -1,25 +0,0 @@ ----input--- -'a' -'я' -'\u{1234}' -' -' -'abc' - ----tokens--- -"'a'" Literal.String.Char -'\n' Text.Whitespace - -"'я'" Literal.String.Char -'\n' Text.Whitespace - -"'\\u{1234}'" Literal.String.Char -'\n' Text.Whitespace - -"'\n'" Literal.String.Char -'\n' Text.Whitespace - -"'" Error -'abc' Name -"'" Error -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_constant_and_module.txt b/spec/tests/crystal/test_constant_and_module.txt deleted file mode 100644 index f8d33ff..0000000 --- a/spec/tests/crystal/test_constant_and_module.txt +++ /dev/null @@ -1,14 +0,0 @@ ----input--- -HTTP -HTTP::Server.new - ----tokens--- -'HTTP' Name.Constant -'\n' Text.Whitespace - -'HTTP' Name -'::' Operator -'Server' Name -'.' Operator -'new' Name -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_empty_percent_strings.txt b/spec/tests/crystal/test_empty_percent_strings.txt deleted file mode 100644 index 3cc0844..0000000 --- a/spec/tests/crystal/test_empty_percent_strings.txt +++ /dev/null @@ -1,27 +0,0 @@ ----input--- -%() -%[] -%{} -%<> -%|| - ----tokens--- -'%(' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace - -'%[' Literal.String.Other -']' Literal.String.Other -'\n' Text.Whitespace - -'%{' Literal.String.Other -'}' Literal.String.Other -'\n' Text.Whitespace - -'%<' Literal.String.Other -'>' Literal.String.Other -'\n' Text.Whitespace - -'%|' Literal.String.Other -'|' Literal.String.Other -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_escaped_bracestring.txt b/spec/tests/crystal/test_escaped_bracestring.txt deleted file mode 100644 index 14718b9..0000000 --- a/spec/tests/crystal/test_escaped_bracestring.txt +++ /dev/null @@ -1,19 +0,0 @@ ----input--- -str.gsub(%r{\\\\}, "/") - ----tokens--- -'str' Name -'.' Operator -'gsub' Name -'(' Punctuation -'%r{' Literal.String.Regex -'\\\\' Literal.String.Regex -'\\\\' Literal.String.Regex -'}' Literal.String.Regex -',' Punctuation -' ' Text.Whitespace -'"' Literal.String.Double -'/' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_escaped_interpolation.txt b/spec/tests/crystal/test_escaped_interpolation.txt deleted file mode 100644 index c623464..0000000 --- a/spec/tests/crystal/test_escaped_interpolation.txt +++ /dev/null @@ -1,9 +0,0 @@ ----input--- -"\#{a + b}" - ----tokens--- -'"' Literal.String.Double -'\\#' Literal.String.Escape -'{a + b}' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_interpolation_nested_curly.txt b/spec/tests/crystal/test_interpolation_nested_curly.txt deleted file mode 100644 index f4a69f7..0000000 --- a/spec/tests/crystal/test_interpolation_nested_curly.txt +++ /dev/null @@ -1,56 +0,0 @@ ----input--- -"A#{ (3..5).group_by { |x| x/2}.map do |k,v| "#{k}" end.join }" + "Z" - ----tokens--- -'"' Literal.String.Double -'A' Literal.String.Double -'#{' Literal.String.Interpol -' ' Text.Whitespace -'(' Punctuation -'3' Literal.Number.Integer -'..' Operator -'5' Literal.Number.Integer -')' Punctuation -'.' Operator -'group_by' Name -' ' Text.Whitespace -'{' Literal.String.Interpol -' ' Text.Whitespace -'|' Operator -'x' Name -'|' Operator -' ' Text.Whitespace -'x' Name -'/' Operator -'2' Literal.Number.Integer -'}' Literal.String.Interpol -'.' Operator -'map' Name -' ' Text.Whitespace -'do' Keyword -' ' Text.Whitespace -'|' Operator -'k' Name -',' Punctuation -'v' Name -'|' Operator -' ' Text.Whitespace -'"' Literal.String.Double -'#{' Literal.String.Interpol -'k' Name -'}' Literal.String.Interpol -'"' Literal.String.Double -' ' Text.Whitespace -'end' Keyword -'.' Operator -'join' Name -' ' Text.Whitespace -'}' Literal.String.Interpol -'"' Literal.String.Double -' ' Text.Whitespace -'+' Operator -' ' Text.Whitespace -'"' Literal.String.Double -'Z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_lib.txt b/spec/tests/crystal/test_lib.txt deleted file mode 100644 index 6f6f107..0000000 --- a/spec/tests/crystal/test_lib.txt +++ /dev/null @@ -1,58 +0,0 @@ ----input--- -@[Link("some")] -lib LibSome -@[CallConvention("X86_StdCall")] -fun foo="some.foo"(thing : Void*) : LibC::Int -end - ----tokens--- -'@[' Operator -'Link' Name.Decorator -'(' Punctuation -'"' Literal.String.Double -'some' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -']' Operator -'\n' Text.Whitespace - -'lib' Keyword -' ' Text.Whitespace -'LibSome' Name.Namespace -'\n' Text.Whitespace - -'@[' Operator -'CallConvention' Name.Decorator -'(' Punctuation -'"' Literal.String.Double -'X86_StdCall' Literal.String.Double -'"' Literal.String.Double -')' Punctuation -']' Operator -'\n' Text.Whitespace - -'fun' Keyword -' ' Text.Whitespace -'foo' Name.Function -'=' Operator -'"' Literal.String.Double -'some.foo' Literal.String.Double -'"' Literal.String.Double -'(' Punctuation -'thing' Name -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'Void' Name -'*' Operator -')' Punctuation -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'LibC' Name -'::' Operator -'Int' Name -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_macro.txt b/spec/tests/crystal/test_macro.txt deleted file mode 100644 index 765caa2..0000000 --- a/spec/tests/crystal/test_macro.txt +++ /dev/null @@ -1,76 +0,0 @@ ----input--- -def<=>(other : self) : Int -{%for field in %w(first_name middle_name last_name)%} -cmp={{field.id}}<=>other.{{field.id}} -return cmp if cmp!=0 -{%end%} -0 -end - ----tokens--- -'def' Keyword -'<=>' Name.Function -'(' Punctuation -'other' Name -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'self' Keyword -')' Punctuation -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'Int' Name -'\n' Text.Whitespace - -'{%' Literal.String.Interpol -'for' Keyword -' ' Text.Whitespace -'field' Name -' ' Text.Whitespace -'in' Keyword -' ' Text.Whitespace -'%w(' Literal.String.Other -'first_name middle_name last_name' Literal.String.Other -')' Literal.String.Other -'%}' Literal.String.Interpol -'\n' Text.Whitespace - -'cmp' Name -'=' Operator -'{{' Literal.String.Interpol -'field' Name -'.' Operator -'id' Name -'}}' Literal.String.Interpol -'<=>' Operator -'other' Name -'.' Operator -'{{' Literal.String.Interpol -'field' Name -'.' Operator -'id' Name -'}}' Literal.String.Interpol -'\n' Text.Whitespace - -'return' Keyword -' ' Text.Whitespace -'cmp' Name -' ' Text.Whitespace -'if' Keyword -' ' Text.Whitespace -'cmp' Name -'!=' Operator -'0' Literal.Number.Integer -'\n' Text.Whitespace - -'{%' Literal.String.Interpol -'end' Keyword -'%}' Literal.String.Interpol -'\n' Text.Whitespace - -'0' Literal.Number.Integer -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_numbers.txt b/spec/tests/crystal/test_numbers.txt deleted file mode 100644 index 0ddfec2..0000000 --- a/spec/tests/crystal/test_numbers.txt +++ /dev/null @@ -1,84 +0,0 @@ ----input--- -# Integers -0 -1 -1_000_000 -1u8 -11231231231121312i64 - -# Floats -0.0 -1.0_f32 -1_f32 -0f64 -1e+4 -1e111 -1_234.567_890 - -# Error -01 -0b2 -0x129g2 -0o12358 - ----tokens--- -'# Integers' Comment.Single -'\n' Text.Whitespace - -'0' Literal.Number.Integer -'\n' Text.Whitespace - -'1' Literal.Number.Integer -'\n' Text.Whitespace - -'1_000_000' Literal.Number.Integer -'\n' Text.Whitespace - -'1u8' Literal.Number.Integer -'\n' Text.Whitespace - -'11231231231121312i64' Literal.Number.Integer -'\n\n' Text.Whitespace - -'# Floats' Comment.Single -'\n' Text.Whitespace - -'0.0' Literal.Number.Float -'\n' Text.Whitespace - -'1.0_f32' Literal.Number.Float -'\n' Text.Whitespace - -'1_f32' Literal.Number.Float -'\n' Text.Whitespace - -'0f64' Literal.Number.Float -'\n' Text.Whitespace - -'1e+4' Literal.Number.Float -'\n' Text.Whitespace - -'1e111' Literal.Number.Float -'\n' Text.Whitespace - -'1_234.567_890' Literal.Number.Float -'\n\n' Text.Whitespace - -'# Error' Comment.Single -'\n' Text.Whitespace - -'0' Error -'1' Literal.Number.Integer -'\n' Text.Whitespace - -'0' Error -'b2' Name -'\n' Text.Whitespace - -'0' Error -'x129g2' Name -'\n' Text.Whitespace - -'0' Error -'o12358' Name -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_operator_methods.txt b/spec/tests/crystal/test_operator_methods.txt deleted file mode 100644 index 084771f..0000000 --- a/spec/tests/crystal/test_operator_methods.txt +++ /dev/null @@ -1,18 +0,0 @@ ----input--- -([] of Int32).[]?(5) - ----tokens--- -'(' Punctuation -'[' Operator -']' Operator -' ' Text.Whitespace -'of' Keyword -' ' Text.Whitespace -'Int32' Name -')' Punctuation -'.' Operator -'[]?' Name.Operator -'(' Punctuation -'5' Literal.Number.Integer -')' Punctuation -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_percent_strings.txt b/spec/tests/crystal/test_percent_strings.txt deleted file mode 100644 index 80f247c..0000000 --- a/spec/tests/crystal/test_percent_strings.txt +++ /dev/null @@ -1,41 +0,0 @@ ----input--- -%(hello ("world")) -%[hello ["world"]] -%{hello "world"} -%> -%|hello "world"| - ----tokens--- -'%(' Literal.String.Other -'hello ' Literal.String.Other -'(' Literal.String.Other -'"world"' Literal.String.Other -')' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace - -'%[' Literal.String.Other -'hello ' Literal.String.Other -'[' Literal.String.Other -'"world"' Literal.String.Other -']' Literal.String.Other -']' Literal.String.Other -'\n' Text.Whitespace - -'%{' Literal.String.Other -'hello "world"' Literal.String.Other -'}' Literal.String.Other -'\n' Text.Whitespace - -'%<' Literal.String.Other -'hello ' Literal.String.Other -'<' Literal.String.Other -'"world"' Literal.String.Other -'>' Literal.String.Other -'>' Literal.String.Other -'\n' Text.Whitespace - -'%|' Literal.String.Other -'hello "world"' Literal.String.Other -'|' Literal.String.Other -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_percent_strings_special.txt b/spec/tests/crystal/test_percent_strings_special.txt deleted file mode 100644 index 4ca1c0b..0000000 --- a/spec/tests/crystal/test_percent_strings_special.txt +++ /dev/null @@ -1,31 +0,0 @@ ----input--- -%Q(hello \n #{name}) -%q(hello \n #{name}) -%w(foo\nbar baz) - ----tokens--- -'%Q(' Literal.String.Other -'hello ' Literal.String.Other -'\\n' Literal.String.Escape -' ' Literal.String.Other -'#{' Literal.String.Interpol -'name' Name -'}' Literal.String.Interpol -')' Literal.String.Other -'\n' Text.Whitespace - -'%q(' Literal.String.Other -'hello ' Literal.String.Other -'\\' Literal.String.Other -'n ' Literal.String.Other -'#' Literal.String.Other -'{name}' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace - -'%w(' Literal.String.Other -'foo' Literal.String.Other -'\\' Literal.String.Other -'nbar baz' Literal.String.Other -')' Literal.String.Other -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_pseudo_builtins.txt b/spec/tests/crystal/test_pseudo_builtins.txt deleted file mode 100644 index 9c8c917..0000000 --- a/spec/tests/crystal/test_pseudo_builtins.txt +++ /dev/null @@ -1,20 +0,0 @@ ----input--- -record Cls do -def_equals s -end - ----tokens--- -'record' Name.Builtin.Pseudo -' ' Text.Whitespace -'Cls' Name -' ' Text.Whitespace -'do' Keyword -'\n' Text.Whitespace - -'def_equals' Name.Builtin.Pseudo -' ' Text.Whitespace -'s' Name -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_pseudo_keywords.txt b/spec/tests/crystal/test_pseudo_keywords.txt deleted file mode 100644 index 18827ad..0000000 --- a/spec/tests/crystal/test_pseudo_keywords.txt +++ /dev/null @@ -1,50 +0,0 @@ ----input--- -def f(x : T, line = __LINE__) forall T -if x.is_a?(String) -pp! x -end -end - ----tokens--- -'def' Keyword -' ' Text.Whitespace -'f' Name.Function -'(' Punctuation -'x' Name -' ' Text.Whitespace -':' Punctuation -' ' Text.Whitespace -'T' Name -',' Punctuation -' ' Text.Whitespace -'line' Name -' ' Text.Whitespace -'=' Operator -' ' Text.Whitespace -'__LINE__' Keyword.Pseudo -')' Punctuation -' ' Text.Whitespace -'forall' Keyword.Pseudo -' ' Text.Whitespace -'T' Name -'\n' Text.Whitespace - -'if' Keyword -' ' Text.Whitespace -'x' Name -'.is_a?' Keyword.Pseudo -'(' Punctuation -'String' Name -')' Punctuation -'\n' Text.Whitespace - -'pp!' Name.Builtin.Pseudo -' ' Text.Whitespace -'x' Name -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace - -'end' Keyword -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_range_syntax1.txt b/spec/tests/crystal/test_range_syntax1.txt deleted file mode 100644 index a3ba24a..0000000 --- a/spec/tests/crystal/test_range_syntax1.txt +++ /dev/null @@ -1,8 +0,0 @@ ----input--- -1...3 - ----tokens--- -'1' Literal.Number.Integer -'...' Operator -'3' Literal.Number.Integer -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_range_syntax2.txt b/spec/tests/crystal/test_range_syntax2.txt deleted file mode 100644 index 08bf4b1..0000000 --- a/spec/tests/crystal/test_range_syntax2.txt +++ /dev/null @@ -1,10 +0,0 @@ ----input--- -1 .. 3 - ----tokens--- -'1' Literal.Number.Integer -' ' Text.Whitespace -'..' Operator -' ' Text.Whitespace -'3' Literal.Number.Integer -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_string_escapes.txt b/spec/tests/crystal/test_string_escapes.txt deleted file mode 100644 index 3ea12f4..0000000 --- a/spec/tests/crystal/test_string_escapes.txt +++ /dev/null @@ -1,58 +0,0 @@ ----input--- -"a\nz" -"a\az" -"a\xffz" -"a\u1234z" -"a\000z" -"a\u{0}z" -"a\u{10AfF9}z" - ----tokens--- -'"' Literal.String.Double -'a' Literal.String.Double -'\\n' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\a' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\xff' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\u1234' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\000' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\u{0}' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace - -'"' Literal.String.Double -'a' Literal.String.Double -'\\u{10AfF9}' Literal.String.Escape -'z' Literal.String.Double -'"' Literal.String.Double -'\n' Text.Whitespace diff --git a/spec/tests/crystal/test_symbols.txt b/spec/tests/crystal/test_symbols.txt deleted file mode 100644 index 3d2cb0a..0000000 --- a/spec/tests/crystal/test_symbols.txt +++ /dev/null @@ -1,20 +0,0 @@ ----input--- -:sym_bol -:あ -:question? -:"symbol" - ----tokens--- -':sym_bol' Literal.String.Symbol -'\n' Text.Whitespace - -':あ' Literal.String.Symbol -'\n' Text.Whitespace - -':question?' Literal.String.Symbol -'\n' Text.Whitespace - -':"' Literal.String.Symbol -'symbol' Literal.String.Symbol -'"' Literal.String.Symbol -'\n' Text.Whitespace diff --git a/src/lexer.cr b/src/lexer.cr index 1e6197d..6ec8522 100644 --- a/src/lexer.cr +++ b/src/lexer.cr @@ -1,6 +1,7 @@ require "./constants/lexers" require "./heuristics" require "baked_file_system" +require "crystal/syntax_highlighter" module Tartrazine class LexerFiles @@ -26,6 +27,7 @@ module Tartrazine end private def self.lexer_by_name(name : String) : BaseLexer + return CrystalLexer.new if name == "crystal" lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil) return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+" raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil? @@ -34,6 +36,10 @@ module Tartrazine end private def self.lexer_by_filename(filename : String) : BaseLexer + if filename.ends_with?(".cr") + return CrystalLexer.new + end + candidates = Set(String).new LEXERS_BY_FILENAME.each do |k, v| candidates += v.to_set if File.match?(k, File.basename(filename)) @@ -327,4 +333,81 @@ module Tartrazine new_state end end + + class CustomCrystalHighlighter < Crystal::SyntaxHighlighter + @tokens = [] of Token + + def render_delimiter(&block) + @tokens << {type: "LiteralString", value: block.call.to_s} + end + + def render_interpolation(&block) + @tokens << {type: "LiteralStringInterpol", value: "\#{"} + @tokens << {type: "Text", value: block.call.to_s} + @tokens << {type: "LiteralStringInterpol", value: "}"} + end + + def render_string_array(&block) + @tokens << {type: "LiteralString", value: block.call.to_s} + end + + # ameba:disable Metrics/CyclomaticComplexity + def render(type : TokenType, value : String) + case type + when .comment? + @tokens << {type: "Comment", value: value} + when .number? + @tokens << {type: "LiteralNumber", value: value} + when .char? + @tokens << {type: "LiteralStringChar", value: value} + when .symbol? + @tokens << {type: "LiteralStringSymbol", value: value} + when .const? + @tokens << {type: "NameConstant", value: value} + when .string? + @tokens << {type: "LiteralString", value: value} + when .ident? + @tokens << {type: "NameVariable", value: value} + when .keyword?, .self? + @tokens << {type: "NameKeyword", value: value} + when .primitive_literal? + @tokens << {type: "Literal", value: value} + when .operator? + @tokens << {type: "Operator", value: value} + when Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN, Crystal::SyntaxHighlighter::TokenType::DELIMITER_START, Crystal::SyntaxHighlighter::TokenType::DELIMITER_END + @tokens << {type: "LiteralString", value: value} + else + @tokens << {type: "Text", value: value} + end + end + end + + class CrystalTokenizer < Tartrazine::BaseTokenizer + include Iterator(Token) + @hl = CustomCrystalHighlighter.new + @lexer : BaseLexer + @iter : Iterator(Token) + + # delegate next, to: @iter + + def initialize(@lexer : BaseLexer, text : String, secondary = false) + # Respect the `ensure_nl` config option + if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary + text += "\n" + end + # Just do the tokenizing + @hl.highlight(text) + @iter = @hl.@tokens.each + end + + def next : Iterator::Stop | Token + @iter.next + end + end + + class CrystalLexer < BaseLexer + def tokenizer(text : String, secondary = false) : BaseTokenizer + CrystalTokenizer.new(self, text, secondary) + end + end end diff --git a/src/rules.cr b/src/rules.cr index 6eaa8d7..167ccee 100644 --- a/src/rules.cr +++ b/src/rules.cr @@ -17,7 +17,6 @@ module Tartrazine abstract struct BaseRule abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token)) - abstract def initialize(node : XML::Node) @actions : Array(Action) = [] of Action @@ -40,9 +39,6 @@ module Tartrazine return true, pos + match[0].size, @actions.flat_map(&.emit(match, tokenizer)) end - def initialize(node : XML::Node) - end - def initialize(node : XML::Node, multiline, dotall, ignorecase) pattern = node["pattern"] pattern = "(?m)" + pattern if multiline diff --git a/src/styles.cr b/src/styles.cr index fd52d4a..a0bc70e 100644 --- a/src/styles.cr +++ b/src/styles.cr @@ -84,27 +84,6 @@ module Tartrazine property styles = {} of String => Style - # Get the style for a token. - def style(token) - styles[token] = Style.new unless styles.has_key?(token) - s = styles[token] - - # We already got the data from the style hierarchy - return s if s.complete? - - # Form the hierarchy of parent styles - parents = style_parents(token) - - s = parents.map do |parent| - styles[parent] - end.reduce(s) do |acc, style| - acc + style - end - s.complete = true - styles[token] = s - s - end - def style_parents(token) parents = ["Background"] parts = token.underscore.split("_").map(&.capitalize)