diff --git a/.ameba.yml b/.ameba.yml index 766994f..e65029e 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -1,9 +1,9 @@ # This configuration file was generated by `ameba --gen-config` -# on 2024-09-11 00:56:14 UTC using Ameba version 1.6.1. +# on 2024-09-21 14:59:30 UTC using Ameba version 1.6.1. # The point is for the user to remove these configuration records # one by one as the reported problems are removed from the code base. -# Problems found: 4 +# Problems found: 3 # Run `ameba --only Documentation/DocumentationAdmonition` for details Documentation/DocumentationAdmonition: Description: Reports documentation admonitions @@ -11,10 +11,24 @@ Documentation/DocumentationAdmonition: Excluded: - src/lexer.cr - src/actions.cr - - spec/examples/crystal/lexer_spec.cr Admonitions: - TODO - FIXME - BUG Enabled: true Severity: Warning + +# Problems found: 1 +# Run `ameba --only Lint/SpecFilename` for details +Lint/SpecFilename: + Description: Enforces spec filenames to have `_spec` suffix + Excluded: + - spec/examples/crystal/hello.cr + IgnoredDirs: + - spec/support + - spec/fixtures + - spec/data + IgnoredFilenames: + - spec_helper + Enabled: true + Severity: Warning diff --git a/spec/examples/crystal/hello.cr b/spec/examples/crystal/hello.cr new file mode 100644 index 0000000..287bc3f --- /dev/null +++ b/spec/examples/crystal/hello.cr @@ -0,0 +1 @@ +puts "Hello Crystal!" diff --git a/spec/examples/crystal/hello.cr.json b/spec/examples/crystal/hello.cr.json new file mode 100644 index 0000000..4719d00 --- /dev/null +++ b/spec/examples/crystal/hello.cr.json @@ -0,0 +1 @@ +[{"type":"Text","value":"puts "},{"type":"LiteralString","value":"\"Hello Crystal!\""},{"type":"Text","value":"\n"}] diff --git a/spec/examples/crystal/lexer.cr b/spec/examples/crystal/lexer.cr deleted file mode 100644 index 6ec8522..0000000 --- a/spec/examples/crystal/lexer.cr +++ /dev/null @@ -1,413 +0,0 @@ -require "./constants/lexers" -require "./heuristics" -require "baked_file_system" -require "crystal/syntax_highlighter" - -module Tartrazine - class LexerFiles - extend BakedFileSystem - bake_folder "../lexers", __DIR__ - end - - # Get the lexer object for a language name - # FIXME: support mimetypes - def self.lexer(name : String? = nil, filename : String? = nil, mimetype : String? = nil) : BaseLexer - return lexer_by_name(name) if name && name != "autodetect" - return lexer_by_filename(filename) if filename - return lexer_by_mimetype(mimetype) if mimetype - - RegexLexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end) - end - - private def self.lexer_by_mimetype(mimetype : String) : BaseLexer - lexer_file_name = LEXERS_BY_MIMETYPE.fetch(mimetype, nil) - raise Exception.new("Unknown mimetype: #{mimetype}") if lexer_file_name.nil? - - RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end) - end - - private def self.lexer_by_name(name : String) : BaseLexer - return CrystalLexer.new if name == "crystal" - lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil) - return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+" - raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil? - - RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end) - end - - private def self.lexer_by_filename(filename : String) : BaseLexer - if filename.ends_with?(".cr") - return CrystalLexer.new - end - - candidates = Set(String).new - LEXERS_BY_FILENAME.each do |k, v| - candidates += v.to_set if File.match?(k, File.basename(filename)) - end - - case candidates.size - when 0 - lexer_file_name = LEXERS_BY_NAME["plaintext"] - when 1 - lexer_file_name = candidates.first - else - lexer_file_name = self.lexer_by_content(filename) - begin - return self.lexer(lexer_file_name) - rescue ex : Exception - raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}, heuristics suggest #{lexer_file_name} but there is no matching lexer.") - end - end - - RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end) - end - - private def self.lexer_by_content(fname : String) : String? - h = Linguist::Heuristic.from_yaml(LexerFiles.get("/heuristics.yml").gets_to_end) - result = h.run(fname, File.read(fname)) - case result - when Nil - raise Exception.new "No lexer found for #{fname}" - when String - result.as(String) - when Array(String) - result.first - end - end - - private def self.create_delegating_lexer(name : String) : BaseLexer - language, root = name.split("+", 2) - language_lexer = lexer(language) - root_lexer = lexer(root) - DelegatingLexer.new(language_lexer, root_lexer) - end - - # Return a list of all lexers - def self.lexers : Array(String) - LEXERS_BY_NAME.keys.sort! - end - - # A token, the output of the tokenizer - alias Token = NamedTuple(type: String, value: String) - - abstract class BaseTokenizer - end - - class Tokenizer < BaseTokenizer - include Iterator(Token) - property lexer : BaseLexer - property text : Bytes - property pos : Int32 = 0 - @dq = Deque(Token).new - property state_stack = ["root"] - - def initialize(@lexer : BaseLexer, text : String, secondary = false) - # Respect the `ensure_nl` config option - if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary - text += "\n" - end - @text = text.to_slice - end - - def next : Iterator::Stop | Token - if @dq.size > 0 - return @dq.shift - end - if pos == @text.size - return stop - end - - matched = false - while @pos < @text.size - @lexer.states[@state_stack.last].rules.each do |rule| - matched, new_pos, new_tokens = rule.match(@text, @pos, self) - if matched - @pos = new_pos - split_tokens(new_tokens).each { |token| @dq << token } - break - end - end - if !matched - if @text[@pos] == 10u8 - @dq << {type: "Text", value: "\n"} - @state_stack = ["root"] - else - @dq << {type: "Error", value: String.new(@text[@pos..@pos])} - end - @pos += 1 - break - end - end - self.next - end - - # If a token contains a newline, split it into two tokens - def split_tokens(tokens : Array(Token)) : Array(Token) - split_tokens = [] of Token - tokens.each do |token| - if token[:value].includes?("\n") - values = token[:value].split("\n") - values.each_with_index do |value, index| - value += "\n" if index < values.size - 1 - split_tokens << {type: token[:type], value: value} - end - else - split_tokens << token - end - end - split_tokens - end - end - - alias BaseLexer = Lexer - - abstract class Lexer - property config = { - name: "", - priority: 0.0, - case_insensitive: false, - dot_all: false, - not_multiline: false, - ensure_nl: false, - } - property states = {} of String => State - - def tokenizer(text : String, secondary = false) : BaseTokenizer - Tokenizer.new(self, text, secondary) - end - end - - # This implements a lexer for Pygments RegexLexers as expressed - # in Chroma's XML serialization. - # - # For explanations on what actions and states do - # the Pygments documentation is a good place to start. - # https://pygments.org/docs/lexerdevelopment/ - class RegexLexer < BaseLexer - # Collapse consecutive tokens of the same type for easier comparison - # and smaller output - def self.collapse_tokens(tokens : Array(Tartrazine::Token)) : Array(Tartrazine::Token) - result = [] of Tartrazine::Token - tokens = tokens.reject { |token| token[:value] == "" } - tokens.each do |token| - if result.empty? - result << token - next - end - last = result.last - if last[:type] == token[:type] - new_token = {type: last[:type], value: last[:value] + token[:value]} - result.pop - result << new_token - else - result << token - end - end - result - end - - def self.from_xml(xml : String) : Lexer - l = RegexLexer.new - lexer = XML.parse(xml).first_element_child - if lexer - config = lexer.children.find { |node| - node.name == "config" - } - if config - l.config = { - name: xml_to_s(config, name) || "", - priority: xml_to_f(config, priority) || 0.0, - not_multiline: xml_to_s(config, not_multiline) == "true", - dot_all: xml_to_s(config, dot_all) == "true", - case_insensitive: xml_to_s(config, case_insensitive) == "true", - ensure_nl: xml_to_s(config, ensure_nl) == "true", - } - end - - rules = lexer.children.find { |node| - node.name == "rules" - } - if rules - # Rules contains states 🤷 - rules.children.select { |node| - node.name == "state" - }.each do |state_node| - state = State.new - state.name = state_node["name"] - if l.states.has_key?(state.name) - raise Exception.new("Duplicate state: #{state.name}") - else - l.states[state.name] = state - end - # And states contain rules 🤷 - state_node.children.select { |node| - node.name == "rule" - }.each do |rule_node| - case rule_node["pattern"]? - when nil - if rule_node.first_element_child.try &.name == "include" - rule = IncludeStateRule.new(rule_node) - else - rule = UnconditionalRule.new(rule_node) - end - else - rule = Rule.new(rule_node, - multiline: !l.config[:not_multiline], - dotall: l.config[:dot_all], - ignorecase: l.config[:case_insensitive]) - end - state.rules << rule - end - end - end - end - l - end - end - - # A lexer that takes two lexers as arguments. A root lexer - # and a language lexer. Everything is scalled using the - # language lexer, afterwards all `Other` tokens are lexed - # using the root lexer. - # - # This is useful for things like template languages, where - # you have Jinja + HTML or Jinja + CSS and so on. - class DelegatingLexer < Lexer - property language_lexer : BaseLexer - property root_lexer : BaseLexer - - def initialize(@language_lexer : BaseLexer, @root_lexer : BaseLexer) - end - - def tokenizer(text : String, secondary = false) : DelegatingTokenizer - DelegatingTokenizer.new(self, text, secondary) - end - end - - # This Tokenizer works with a DelegatingLexer. It first tokenizes - # using the language lexer, and "Other" tokens are tokenized using - # the root lexer. - class DelegatingTokenizer < BaseTokenizer - include Iterator(Token) - @dq = Deque(Token).new - @language_tokenizer : BaseTokenizer - - def initialize(@lexer : DelegatingLexer, text : String, secondary = false) - # Respect the `ensure_nl` config option - if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary - text += "\n" - end - @language_tokenizer = @lexer.language_lexer.tokenizer(text, true) - end - - def next : Iterator::Stop | Token - if @dq.size > 0 - return @dq.shift - end - token = @language_tokenizer.next - if token.is_a? Iterator::Stop - return stop - elsif token.as(Token).[:type] == "Other" - root_tokenizer = @lexer.root_lexer.tokenizer(token.as(Token).[:value], true) - root_tokenizer.each do |root_token| - @dq << root_token - end - else - @dq << token.as(Token) - end - self.next - end - end - - # A Lexer state. A state has a name and a list of rules. - # The state machine has a state stack containing references - # to states to decide which rules to apply. - struct State - property name : String = "" - property rules = [] of BaseRule - - def +(other : State) - new_state = State.new - new_state.name = Random.base58(8) - new_state.rules = rules + other.rules - new_state - end - end - - class CustomCrystalHighlighter < Crystal::SyntaxHighlighter - @tokens = [] of Token - - def render_delimiter(&block) - @tokens << {type: "LiteralString", value: block.call.to_s} - end - - def render_interpolation(&block) - @tokens << {type: "LiteralStringInterpol", value: "\#{"} - @tokens << {type: "Text", value: block.call.to_s} - @tokens << {type: "LiteralStringInterpol", value: "}"} - end - - def render_string_array(&block) - @tokens << {type: "LiteralString", value: block.call.to_s} - end - - # ameba:disable Metrics/CyclomaticComplexity - def render(type : TokenType, value : String) - case type - when .comment? - @tokens << {type: "Comment", value: value} - when .number? - @tokens << {type: "LiteralNumber", value: value} - when .char? - @tokens << {type: "LiteralStringChar", value: value} - when .symbol? - @tokens << {type: "LiteralStringSymbol", value: value} - when .const? - @tokens << {type: "NameConstant", value: value} - when .string? - @tokens << {type: "LiteralString", value: value} - when .ident? - @tokens << {type: "NameVariable", value: value} - when .keyword?, .self? - @tokens << {type: "NameKeyword", value: value} - when .primitive_literal? - @tokens << {type: "Literal", value: value} - when .operator? - @tokens << {type: "Operator", value: value} - when Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN, Crystal::SyntaxHighlighter::TokenType::DELIMITER_START, Crystal::SyntaxHighlighter::TokenType::DELIMITER_END - @tokens << {type: "LiteralString", value: value} - else - @tokens << {type: "Text", value: value} - end - end - end - - class CrystalTokenizer < Tartrazine::BaseTokenizer - include Iterator(Token) - @hl = CustomCrystalHighlighter.new - @lexer : BaseLexer - @iter : Iterator(Token) - - # delegate next, to: @iter - - def initialize(@lexer : BaseLexer, text : String, secondary = false) - # Respect the `ensure_nl` config option - if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary - text += "\n" - end - # Just do the tokenizing - @hl.highlight(text) - @iter = @hl.@tokens.each - end - - def next : Iterator::Stop | Token - @iter.next - end - end - - class CrystalLexer < BaseLexer - def tokenizer(text : String, secondary = false) : BaseTokenizer - CrystalTokenizer.new(self, text, secondary) - end - end -end diff --git a/spec/examples/crystal/lexer.cr.json b/spec/examples/crystal/lexer.cr.json deleted file mode 100644 index 7d86139..0000000 --- a/spec/examples/crystal/lexer.cr.json +++ /dev/null @@ -1 +0,0 @@ -[{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"./constants/lexers\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"./heuristics\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"baked_file_system\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"crystal/syntax_highlighter\""},{"type":"Text","value":"\n\n"},{"type":"NameKeyword","value":"module"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"extend"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BakedFileSystem"},{"type":"Text","value":"\n bake_folder "},{"type":"LiteralString","value":"\"../lexers\""},{"type":"Text","value":", __DIR__\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# Get the lexer object for a language name"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# FIXME: support mimetypes"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":", filename : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":", mimetype : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_name(name) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" name "},{"type":"Operator","value":"&&"},{"type":"Text","value":" name "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"autodetect\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_filename(filename) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" filename\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_mimetype(mimetype) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" mimetype\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":"["},{"type":"LiteralString","value":"\"plaintext\""},{"type":"Text","value":"]"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_mimetype(mimetype : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_MIMETYPE"},{"type":"Text","value":".fetch(mimetype, "},{"type":"Literal","value":"nil"},{"type":"Text","value":")\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Unknown mimetype: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"mimetype"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":") "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":"\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_name(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":".new "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"crystal\""},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":".fetch(name.downcase, "},{"type":"Literal","value":"nil"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" create_delegating_lexer(name) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" name.includes? "},{"type":"LiteralString","value":"\"+\""},{"type":"Text","value":"\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Unknown lexer: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":") "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":"\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_filename(filename : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" filename.ends_with?("},{"type":"LiteralString","value":"\".cr\""},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":".new\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n candidates "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Set"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":").new\n "},{"type":"NameConstant","value":"LEXERS_BY_FILENAME"},{"type":"Text","value":".each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"k, v"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n candidates "},{"type":"Operator","value":"+="},{"type":"Text","value":" v.to_set "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".match?(k, "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".basename(filename))\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" candidates.size\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":"["},{"type":"LiteralString","value":"\"plaintext\""},{"type":"Text","value":"]\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" candidates.first\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":".lexer_by_content(filename)\n "},{"type":"NameKeyword","value":"begin"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":".lexer(lexer_file_name)\n "},{"type":"NameKeyword","value":"rescue"},{"type":"Text","value":" ex : "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":"\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Multiple lexers match the filename: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"candidates.to_a.join("},{"type":"LiteralString","value":"\", \""},{"type":"Text","value":")"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":", heuristics suggest "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":" but there is no matching lexer.\""},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_content(fname : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"?\n h "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Linguist::Heuristic"},{"type":"Text","value":".from_yaml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/heuristics.yml\""},{"type":"Text","value":").gets_to_end)\n result "},{"type":"Operator","value":"="},{"type":"Text","value":" h.run(fname, "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".read(fname))\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" result\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Nil"},{"type":"Text","value":"\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new "},{"type":"LiteralString","value":"\"No lexer found for "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"fname"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"\n result."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n result.first\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".create_delegating_lexer(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n language, root "},{"type":"Operator","value":"="},{"type":"Text","value":" name.split("},{"type":"LiteralString","value":"\"+\""},{"type":"Text","value":", "},{"type":"LiteralNumber","value":"2"},{"type":"Text","value":")\n language_lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer(language)\n root_lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer(root)\n "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":".new(language_lexer, root_lexer)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# Return a list of all lexers"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexers : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":".keys.sort!\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# A token, the output of the tokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"alias"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"NamedTuple"},{"type":"Text","value":"("},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", value: "},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n\n "},{"type":"NameKeyword","value":"abstract"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n property lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n property text : "},{"type":"NameConstant","value":"Bytes"},{"type":"Text","value":"\n property pos : "},{"type":"NameConstant","value":"Int32"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Deque"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").new\n property state_stack "},{"type":"Operator","value":"="},{"type":"Text","value":" ["},{"type":"LiteralString","value":"\"root\""},{"type":"Text","value":"]\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n @text "},{"type":"Operator","value":"="},{"type":"Text","value":" text.to_slice\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @dq.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" @dq.shift\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" pos "},{"type":"Operator","value":"=="},{"type":"Text","value":" @text.size\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" stop\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n matched "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"while"},{"type":"Text","value":" @pos "},{"type":"Operator","value":"<"},{"type":"Text","value":" @text.size\n @lexer.states[@state_stack.last].rules.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"rule"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n matched, new_pos, new_tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" rule.match(@text, @pos, "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" matched\n @pos "},{"type":"Operator","value":"="},{"type":"Text","value":" new_pos\n split_tokens(new_tokens).each { "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":" @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token }\n "},{"type":"NameKeyword","value":"break"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"matched\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @text[@pos] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"10u8"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"}\n @state_stack "},{"type":"Operator","value":"="},{"type":"Text","value":" ["},{"type":"LiteralString","value":"\"root\""},{"type":"Text","value":"]\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Error\""},{"type":"Text","value":", value: "},{"type":"NameConstant","value":"String"},{"type":"Text","value":".new(@text[@pos..@pos])}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n @pos "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"break"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":"."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# If a token contains a newline, split it into two tokens"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"split_tokens"},{"type":"Text","value":"(tokens : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")) : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n split_tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n tokens.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"].includes?("},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":")\n values "},{"type":"Operator","value":"="},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"].split("},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":")\n values.each_with_index "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"value, index"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n value "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":" "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" index "},{"type":"Operator","value":"<"},{"type":"Text","value":" values.size "},{"type":"Operator","value":"-"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n split_tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": token["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"], value: value}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n split_tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n split_tokens\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"alias"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"abstract"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n property config "},{"type":"Operator","value":"="},{"type":"Text","value":" {\n name: "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":",\n priority: "},{"type":"LiteralNumber","value":"0.0"},{"type":"Text","value":",\n case_insensitive: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n dot_all: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n not_multiline: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n ensure_nl: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n }\n property states "},{"type":"Operator","value":"="},{"type":"Text","value":" {} "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"String"},{"type":"Text","value":" "},{"type":"Operator","value":"=>"},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameConstant","value":"Tokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# This implements a lexer for Pygments RegexLexers as expressed"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# in Chroma's XML serialization."},{"type":"Text","value":"\n "},{"type":"Comment","value":"#"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# For explanations on what actions and states do"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# the Pygments documentation is a good place to start."},{"type":"Text","value":"\n "},{"type":"Comment","value":"# https://pygments.org/docs/lexerdevelopment/"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# Collapse consecutive tokens of the same type for easier comparison"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# and smaller output"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".collapse_tokens(tokens : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":")) : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":")\n result "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":"\n tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" tokens.reject { "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":" }\n tokens.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" result.empty?\n result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n "},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n last "},{"type":"Operator","value":"="},{"type":"Text","value":" result.last\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" last["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"]\n new_token "},{"type":"Operator","value":"="},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": last["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"], value: last["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"] "},{"type":"Operator","value":"+"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"]}\n result.pop\n result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" new_token\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n result\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".from_xml(xml : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n l "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".new\n lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"XML"},{"type":"Text","value":".parse(xml).first_element_child\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer\n config "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer.children.find { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"config\""},{"type":"Text","value":"\n }\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" config\n l.config "},{"type":"Operator","value":"="},{"type":"Text","value":" {\n name: xml_to_s(config, name) "},{"type":"Operator","value":"||"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":",\n priority: xml_to_f(config, priority) "},{"type":"Operator","value":"||"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0.0"},{"type":"Text","value":",\n not_multiline: xml_to_s(config, not_multiline) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n dot_all: xml_to_s(config, dot_all) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n case_insensitive: xml_to_s(config, case_insensitive) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n ensure_nl: xml_to_s(config, ensure_nl) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n }\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n rules "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer.children.find { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"rules\""},{"type":"Text","value":"\n }\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" rules\n "},{"type":"Comment","value":"# Rules contains states 🤷"},{"type":"Text","value":"\n rules.children."},{"type":"NameKeyword","value":"select"},{"type":"Text","value":" { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"state\""},{"type":"Text","value":"\n }.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"state_node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n state "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":".new\n state.name "},{"type":"Operator","value":"="},{"type":"Text","value":" state_node["},{"type":"LiteralString","value":"\"name\""},{"type":"Text","value":"]\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" l.states.has_key?(state.name)\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Duplicate state: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"state.name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n l.states[state.name] "},{"type":"Operator","value":"="},{"type":"Text","value":" state\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# And states contain rules 🤷"},{"type":"Text","value":"\n state_node.children."},{"type":"NameKeyword","value":"select"},{"type":"Text","value":" { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"rule\""},{"type":"Text","value":"\n }.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"rule_node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" rule_node["},{"type":"LiteralString","value":"\"pattern\""},{"type":"Text","value":"]?\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" rule_node.first_element_child.try "},{"type":"Operator","value":"&"},{"type":"Text","value":".name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"include\""},{"type":"Text","value":"\n rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"IncludeStateRule"},{"type":"Text","value":".new(rule_node)\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"UnconditionalRule"},{"type":"Text","value":".new(rule_node)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Rule"},{"type":"Text","value":".new(rule_node,\n multiline: "},{"type":"Operator","value":"!"},{"type":"Text","value":"l.config["},{"type":"LiteralStringSymbol","value":":not_multiline"},{"type":"Text","value":"],\n dotall: l.config["},{"type":"LiteralStringSymbol","value":":dot_all"},{"type":"Text","value":"],\n ignorecase: l.config["},{"type":"LiteralStringSymbol","value":":case_insensitive"},{"type":"Text","value":"])\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n state.rules "},{"type":"Operator","value":"<<"},{"type":"Text","value":" rule\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n l\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# A lexer that takes two lexers as arguments. A root lexer"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# and a language lexer. Everything is scalled using the"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# language lexer, afterwards all `Other` tokens are lexed"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# using the root lexer."},{"type":"Text","value":"\n "},{"type":"Comment","value":"#"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# This is useful for things like template languages, where"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# you have Jinja + HTML or Jinja + CSS and so on."},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n property language_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n property root_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@language_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", @root_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":"\n "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# This Tokenizer works with a DelegatingLexer. It first tokenizes"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# using the language lexer, and \"Other\" tokens are tokenized using"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# the root lexer."},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n @dq "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Deque"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").new\n @language_tokenizer : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n @language_tokenizer "},{"type":"Operator","value":"="},{"type":"Text","value":" @lexer.language_lexer.tokenizer(text, "},{"type":"Literal","value":"true"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @dq.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" @dq.shift\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n token "},{"type":"Operator","value":"="},{"type":"Text","value":" @language_tokenizer."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"is_a?"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" stop\n "},{"type":"NameKeyword","value":"elsif"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"Other\""},{"type":"Text","value":"\n root_tokenizer "},{"type":"Operator","value":"="},{"type":"Text","value":" @lexer.root_lexer.tokenizer(token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"], "},{"type":"Literal","value":"true"},{"type":"Text","value":")\n root_tokenizer.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"root_token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" root_token\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":"."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# A Lexer state. A state has a name and a list of rules."},{"type":"Text","value":"\n "},{"type":"Comment","value":"# The state machine has a state stack containing references"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# to states to decide which rules to apply."},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"struct"},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":"\n property name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":"\n property rules "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseRule"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"+"},{"type":"Text","value":"(other : "},{"type":"NameConstant","value":"State"},{"type":"Text","value":")\n new_state "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":".new\n new_state.name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Random"},{"type":"Text","value":".base58("},{"type":"LiteralNumber","value":"8"},{"type":"Text","value":")\n new_state.rules "},{"type":"Operator","value":"="},{"type":"Text","value":" rules "},{"type":"Operator","value":"+"},{"type":"Text","value":" other.rules\n new_state\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CustomCrystalHighlighter"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter"},{"type":"Text","value":"\n @tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_delimiter"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: block.call.to_s}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_interpolation"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringInterpol\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"\\#{\""},{"type":"Text","value":"}\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: block.call.to_s}\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringInterpol\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"}\""},{"type":"Text","value":"}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_string_array"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: block.call.to_s}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# ameba:disable Metrics/CyclomaticComplexity"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render"},{"type":"Text","value":"("},{"type":"NameKeyword","value":"type"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"TokenType"},{"type":"Text","value":", value : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"type"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .comment?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Comment\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .number?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralNumber\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .char?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringChar\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .symbol?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringSymbol\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .const?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameConstant\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .string?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .ident?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameVariable\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .keyword?, .self?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameKeyword\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .primitive_literal?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Literal\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .operator?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Operator\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN"},{"type":"Text","value":", "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITER_START"},{"type":"Text","value":", "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITER_END"},{"type":"Text","value":"\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalTokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine::BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n @hl "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"CustomCrystalHighlighter"},{"type":"Text","value":".new\n @lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n @iter : "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n\n "},{"type":"Comment","value":"# delegate next, to: @iter"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# Just do the tokenizing"},{"type":"Text","value":"\n @hl.highlight(text)\n @iter "},{"type":"Operator","value":"="},{"type":"Text","value":" @hl.@tokens.each\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n @iter."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameConstant","value":"CrystalTokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n"}] diff --git a/src/formatter.cr b/src/formatter.cr index 3203078..c718e8d 100644 --- a/src/formatter.cr +++ b/src/formatter.cr @@ -17,12 +17,19 @@ module Tartrazine end def format(text : String, lexer : Lexer) : String - raise Exception.new("Not implemented") + outp = String::Builder.new("") + format(text, lexer, outp) + outp.to_s end # Return the styles, if the formatter supports it. def style_defs : String raise Exception.new("Not implemented") end + + # Is this line in the highlighted ranges? + def highlighted?(line : Int) : Bool + highlight_lines.any?(&.includes?(line)) + end end end diff --git a/src/formatters/ansi.cr b/src/formatters/ansi.cr index 4273906..52b589c 100644 --- a/src/formatters/ansi.cr +++ b/src/formatters/ansi.cr @@ -20,12 +20,6 @@ module Tartrazine "#{i + 1}".rjust(4).ljust(5) end - def format(text : String, lexer : BaseLexer) : String - outp = String::Builder.new("") - format(text, lexer, outp) - outp.to_s - end - def format(text : String, lexer : BaseLexer, outp : IO) : Nil tokenizer = lexer.tokenizer(text) i = 0 diff --git a/src/formatters/html.cr b/src/formatters/html.cr index 002b567..8495205 100644 --- a/src/formatters/html.cr +++ b/src/formatters/html.cr @@ -134,10 +134,5 @@ module Tartrazine end class_prefix + Abbreviations[token] end - - # Is this line in the highlighted ranges? - def highlighted?(line : Int) : Bool - highlight_lines.any?(&.includes?(line)) - end end end diff --git a/src/formatters/svg.cr b/src/formatters/svg.cr new file mode 100644 index 0000000..5bb3a77 --- /dev/null +++ b/src/formatters/svg.cr @@ -0,0 +1,129 @@ +require "../constants/token_abbrevs.cr" +require "../formatter" +require "html" + +module Tartrazine + def self.to_svg(text : String, language : String, + theme : String = "default-dark", + standalone : Bool = true, + line_numbers : Bool = false) : String + Tartrazine::Svg.new( + theme: Tartrazine.theme(theme), + standalone: standalone, + line_numbers: line_numbers + ).format(text, Tartrazine.lexer(name: language)) + end + + class Svg < Formatter + property highlight_lines : Array(Range(Int32, Int32)) = [] of Range(Int32, Int32) + property line_number_id_prefix : String = "line-" + property line_number_start : Int32 = 1 + property tab_width = 8 + property? line_numbers : Bool = false + property? linkable_line_numbers : Bool = true + property? standalone : Bool = false + property weight_of_bold : Int32 = 600 + property fs : Int32 + property ystep : Int32 + + property theme : Theme + + def initialize(@theme : Theme = Tartrazine.theme("default-dark"), *, + @highlight_lines = [] of Range(Int32, Int32), + @class_prefix : String = "", + @line_number_id_prefix = "line-", + @line_number_start = 1, + @tab_width = 8, + @line_numbers : Bool = false, + @linkable_line_numbers : Bool = true, + @standalone : Bool = false, + @weight_of_bold : Int32 = 600, + @font_family : String = "monospace", + @font_size : String = "14px") + if font_size.ends_with? "px" + @fs = font_size[0...-2].to_i + else + @fs = font_size.to_i + end + @ystep = @fs + 5 + end + + def format(text : String, lexer : BaseLexer, io : IO) : Nil + pre, post = wrap_standalone + io << pre if standalone? + format_text(text, lexer, io) + io << post if standalone? + end + + # Wrap text into a full HTML document, including the CSS for the theme + def wrap_standalone + output = String.build do |outp| + outp << %( + + + ) + end + {output.to_s, ""} + end + + private def line_label(i : Int32, x : Int32, y : Int32) : String + line_label = "#{i + 1}".rjust(4).ljust(5) + line_style = highlighted?(i + 1) ? "font-weight=\"#{@weight_of_bold}\"" : "" + line_id = linkable_line_numbers? ? "id=\"#{line_number_id_prefix}#{i + 1}\"" : "" + %(#{line_label}) + end + + def format_text(text : String, lexer : BaseLexer, outp : IO) + x = 0 + y = ystep + i = 0 + line_x = x + line_x += 5 * ystep if line_numbers? + tokenizer = lexer.tokenizer(text) + outp << line_label(i, x, y) if line_numbers? + outp << %() + tokenizer.each do |token| + if token[:value].ends_with? "\n" + outp << "#{HTML.escape(token[:value][0...-1])}" + outp << "" + x = 0 + y += ystep + i += 1 + outp << line_label(i, x, y) if line_numbers? + outp << %() + else + outp << "#{HTML.escape(token[:value])}" + x += token[:value].size * ystep + end + end + outp << "" + end + + # Given a token type, return the style. + def get_style(token : String) : String + if !theme.styles.has_key? token + # Themes don't contain information for each specific + # token type. However, they may contain information + # for a parent style. Worst case, we go to the root + # (Background) style. + parent = theme.style_parents(token).reverse.find { |dad| + theme.styles.has_key?(dad) + } + theme.styles[token] = theme.styles[parent] + end + output = String.build do |outp| + style = theme.styles[token] + outp << " fill=\"##{style.color.try &.hex}\"" if style.color + # No support for background color or border in SVG + + outp << " font-weight=\"#{@weight_of_bold}\"" if style.bold + outp << " font-weight=\"normal\"" if style.bold == false + outp << " font-style=\"italic\"" if style.italic + outp << " font-style=\"normal\"" if style.italic == false + outp << " text-decoration=\"underline\"" if style.underline + outp << " text-decoration=\"none" if style.underline == false + end + output + end + end +end diff --git a/src/main.cr b/src/main.cr index 75cd713..78c4ac7 100644 --- a/src/main.cr +++ b/src/main.cr @@ -11,6 +11,8 @@ Usage: tartrazine -f html -t theme --css tartrazine FILE -f terminal [-t theme][-l lexer][--line-numbers] [-o output] + tartrazine FILE -f svg [-t theme][--standalone][--line-numbers] + [-l lexer][-o output] tartrazine FILE -f json [-o output] tartrazine --list-themes tartrazine --list-lexers @@ -18,7 +20,7 @@ Usage: tartrazine --version Options: - -f Format to use (html, terminal, json) + -f Format to use (html, terminal, json, svg) -t Theme to use, see --list-themes [default: default-dark] -l Lexer (language) to use, see --list-lexers. Use more than one lexer with "+" (e.g. jinja+yaml) [default: autodetect] @@ -71,6 +73,11 @@ if options["-f"] formatter.theme = theme when "json" formatter = Tartrazine::Json.new + when "svg" + formatter = Tartrazine::Svg.new + formatter.standalone = options["--standalone"] != nil + formatter.line_numbers = options["--line-numbers"] != nil + formatter.theme = theme else puts "Invalid formatter: #{formatter}" exit 1