[{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"./constants/lexers\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"./heuristics\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"baked_file_system\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"crystal/syntax_highlighter\""},{"type":"Text","value":"\n\n"},{"type":"NameKeyword","value":"module"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"extend"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BakedFileSystem"},{"type":"Text","value":"\n    bake_folder "},{"type":"LiteralString","value":"\"../lexers\""},{"type":"Text","value":", __DIR__\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# Get the lexer object for a language name"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# FIXME: support mimetypes"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":", filename : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":", mimetype : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_name(name) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" name "},{"type":"Operator","value":"&&"},{"type":"Text","value":" name "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"autodetect\""},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_filename(filename) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" filename\n    "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_mimetype(mimetype) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" mimetype\n\n    "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":"["},{"type":"LiteralString","value":"\"plaintext\""},{"type":"Text","value":"]"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_mimetype(mimetype : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_MIMETYPE"},{"type":"Text","value":".fetch(mimetype, "},{"type":"Literal","value":"nil"},{"type":"Text","value":")\n    raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Unknown mimetype: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"mimetype"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":") "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":"\n\n    "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_name(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":".new "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"crystal\""},{"type":"Text","value":"\n    lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":".fetch(name.downcase, "},{"type":"Literal","value":"nil"},{"type":"Text","value":")\n    "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" create_delegating_lexer(name) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" name.includes? "},{"type":"LiteralString","value":"\"+\""},{"type":"Text","value":"\n    raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Unknown lexer: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":") "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":"\n\n    "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_filename(filename : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" filename.ends_with?("},{"type":"LiteralString","value":"\".cr\""},{"type":"Text","value":")\n      "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":".new\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    candidates "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Set"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":").new\n    "},{"type":"NameConstant","value":"LEXERS_BY_FILENAME"},{"type":"Text","value":".each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"k, v"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n      candidates "},{"type":"Operator","value":"+="},{"type":"Text","value":" v.to_set "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".match?(k, "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".basename(filename))\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" candidates.size\n    "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n      lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":"["},{"type":"LiteralString","value":"\"plaintext\""},{"type":"Text","value":"]\n    "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n      lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" candidates.first\n    "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n      lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":".lexer_by_content(filename)\n      "},{"type":"NameKeyword","value":"begin"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":".lexer(lexer_file_name)\n      "},{"type":"NameKeyword","value":"rescue"},{"type":"Text","value":" ex : "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":"\n        raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Multiple lexers match the filename: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"candidates.to_a.join("},{"type":"LiteralString","value":"\", \""},{"type":"Text","value":")"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":", heuristics suggest "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":" but there is no matching lexer.\""},{"type":"Text","value":")\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_content(fname : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"?\n    h "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Linguist::Heuristic"},{"type":"Text","value":".from_yaml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/heuristics.yml\""},{"type":"Text","value":").gets_to_end)\n    result "},{"type":"Operator","value":"="},{"type":"Text","value":" h.run(fname, "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".read(fname))\n    "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" result\n    "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Nil"},{"type":"Text","value":"\n      raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new "},{"type":"LiteralString","value":"\"No lexer found for "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"fname"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"\n      result."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n    "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n      result.first\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".create_delegating_lexer(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    language, root "},{"type":"Operator","value":"="},{"type":"Text","value":" name.split("},{"type":"LiteralString","value":"\"+\""},{"type":"Text","value":", "},{"type":"LiteralNumber","value":"2"},{"type":"Text","value":")\n    language_lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer(language)\n    root_lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer(root)\n    "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":".new(language_lexer, root_lexer)\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# Return a list of all lexers"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexers : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n    "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":".keys.sort!\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# A token, the output of the tokenizer"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"alias"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"NamedTuple"},{"type":"Text","value":"("},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", value: "},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n\n  "},{"type":"NameKeyword","value":"abstract"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n    property lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    property text : "},{"type":"NameConstant","value":"Bytes"},{"type":"Text","value":"\n    property pos : "},{"type":"NameConstant","value":"Int32"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n    @dq "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Deque"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").new\n    property state_stack "},{"type":"Operator","value":"="},{"type":"Text","value":" ["},{"type":"LiteralString","value":"\"root\""},{"type":"Text","value":"]\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n      "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n        text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      @text "},{"type":"Operator","value":"="},{"type":"Text","value":" text.to_slice\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @dq.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" @dq.shift\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" pos "},{"type":"Operator","value":"=="},{"type":"Text","value":" @text.size\n        "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" stop\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n      matched "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"while"},{"type":"Text","value":" @pos "},{"type":"Operator","value":"<"},{"type":"Text","value":" @text.size\n        @lexer.states[@state_stack.last].rules.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"rule"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n          matched, new_pos, new_tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" rule.match(@text, @pos, "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":")\n          "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" matched\n            @pos "},{"type":"Operator","value":"="},{"type":"Text","value":" new_pos\n            split_tokens(new_tokens).each { "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":" @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token }\n            "},{"type":"NameKeyword","value":"break"},{"type":"Text","value":"\n          "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"matched\n          "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @text[@pos] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"10u8"},{"type":"Text","value":"\n            @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"}\n            @state_stack "},{"type":"Operator","value":"="},{"type":"Text","value":" ["},{"type":"LiteralString","value":"\"root\""},{"type":"Text","value":"]\n          "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n            @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Error\""},{"type":"Text","value":", value: "},{"type":"NameConstant","value":"String"},{"type":"Text","value":".new(@text[@pos..@pos])}\n          "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n          @pos "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n          "},{"type":"NameKeyword","value":"break"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":"."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"Comment","value":"# If a token contains a newline, split it into two tokens"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"split_tokens"},{"type":"Text","value":"(tokens : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")) : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n      split_tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n      tokens.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"].includes?("},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":")\n          values "},{"type":"Operator","value":"="},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"].split("},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":")\n          values.each_with_index "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"value, index"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n            value "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":" "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" index "},{"type":"Operator","value":"<"},{"type":"Text","value":" values.size "},{"type":"Operator","value":"-"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n            split_tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": token["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"], value: value}\n          "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n          split_tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      split_tokens\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"alias"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"abstract"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n    property config "},{"type":"Operator","value":"="},{"type":"Text","value":" {\n      name:             "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":",\n      priority:         "},{"type":"LiteralNumber","value":"0.0"},{"type":"Text","value":",\n      case_insensitive: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n      dot_all:          "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n      not_multiline:    "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n      ensure_nl:        "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n    }\n    property states "},{"type":"Operator","value":"="},{"type":"Text","value":" {} "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"String"},{"type":"Text","value":" "},{"type":"Operator","value":"=>"},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n      "},{"type":"NameConstant","value":"Tokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# This implements a lexer for Pygments RegexLexers as expressed"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# in Chroma's XML serialization."},{"type":"Text","value":"\n  "},{"type":"Comment","value":"#"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# For explanations on what actions and states do"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# the Pygments documentation is a good place to start."},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# https://pygments.org/docs/lexerdevelopment/"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    "},{"type":"Comment","value":"# Collapse consecutive tokens of the same type for easier comparison"},{"type":"Text","value":"\n    "},{"type":"Comment","value":"# and smaller output"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".collapse_tokens(tokens : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":")) : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":")\n      result "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":"\n      tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" tokens.reject { "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":" }\n      tokens.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" result.empty?\n          result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n          "},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n        last "},{"type":"Operator","value":"="},{"type":"Text","value":" result.last\n        "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" last["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"]\n          new_token "},{"type":"Operator","value":"="},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": last["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"], value: last["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"] "},{"type":"Operator","value":"+"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"]}\n          result.pop\n          result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" new_token\n        "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n          result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      result\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".from_xml(xml : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n      l "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".new\n      lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"XML"},{"type":"Text","value":".parse(xml).first_element_child\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer\n        config "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer.children.find { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n          node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"config\""},{"type":"Text","value":"\n        }\n        "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" config\n          l.config "},{"type":"Operator","value":"="},{"type":"Text","value":" {\n            name:             xml_to_s(config, name) "},{"type":"Operator","value":"||"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":",\n            priority:         xml_to_f(config, priority) "},{"type":"Operator","value":"||"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0.0"},{"type":"Text","value":",\n            not_multiline:    xml_to_s(config, not_multiline) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n            dot_all:          xml_to_s(config, dot_all) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n            case_insensitive: xml_to_s(config, case_insensitive) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n            ensure_nl:        xml_to_s(config, ensure_nl) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n          }\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n        rules "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer.children.find { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n          node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"rules\""},{"type":"Text","value":"\n        }\n        "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" rules\n          "},{"type":"Comment","value":"# Rules contains states 🤷"},{"type":"Text","value":"\n          rules.children."},{"type":"NameKeyword","value":"select"},{"type":"Text","value":" { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n            node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"state\""},{"type":"Text","value":"\n          }.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"state_node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n            state "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":".new\n            state.name "},{"type":"Operator","value":"="},{"type":"Text","value":" state_node["},{"type":"LiteralString","value":"\"name\""},{"type":"Text","value":"]\n            "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" l.states.has_key?(state.name)\n              raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Duplicate state: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"state.name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":")\n            "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n              l.states[state.name] "},{"type":"Operator","value":"="},{"type":"Text","value":" state\n            "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n            "},{"type":"Comment","value":"# And states contain rules 🤷"},{"type":"Text","value":"\n            state_node.children."},{"type":"NameKeyword","value":"select"},{"type":"Text","value":" { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n              node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"rule\""},{"type":"Text","value":"\n            }.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"rule_node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n              "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" rule_node["},{"type":"LiteralString","value":"\"pattern\""},{"type":"Text","value":"]?\n              "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":"\n                "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" rule_node.first_element_child.try "},{"type":"Operator","value":"&"},{"type":"Text","value":".name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"include\""},{"type":"Text","value":"\n                  rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"IncludeStateRule"},{"type":"Text","value":".new(rule_node)\n                "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n                  rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"UnconditionalRule"},{"type":"Text","value":".new(rule_node)\n                "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n              "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n                rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Rule"},{"type":"Text","value":".new(rule_node,\n                  multiline: "},{"type":"Operator","value":"!"},{"type":"Text","value":"l.config["},{"type":"LiteralStringSymbol","value":":not_multiline"},{"type":"Text","value":"],\n                  dotall: l.config["},{"type":"LiteralStringSymbol","value":":dot_all"},{"type":"Text","value":"],\n                  ignorecase: l.config["},{"type":"LiteralStringSymbol","value":":case_insensitive"},{"type":"Text","value":"])\n              "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n              state.rules "},{"type":"Operator","value":"<<"},{"type":"Text","value":" rule\n            "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n          "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      l\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# A lexer that takes two lexers as arguments. A root lexer"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# and a language lexer. Everything is scalled using the"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# language lexer, afterwards all `Other` tokens are lexed"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# using the root lexer."},{"type":"Text","value":"\n  "},{"type":"Comment","value":"#"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# This is useful for things like template languages, where"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# you have Jinja + HTML or Jinja + CSS and so on."},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n    property language_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    property root_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@language_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", @root_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":")\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":"\n      "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# This Tokenizer works with a DelegatingLexer. It first tokenizes"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# using the language lexer, and \"Other\" tokens are tokenized using"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# the root lexer."},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n    @dq "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Deque"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").new\n    @language_tokenizer : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n      "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n        text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      @language_tokenizer "},{"type":"Operator","value":"="},{"type":"Text","value":" @lexer.language_lexer.tokenizer(text, "},{"type":"Literal","value":"true"},{"type":"Text","value":")\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @dq.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" @dq.shift\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      token "},{"type":"Operator","value":"="},{"type":"Text","value":" @language_tokenizer."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"is_a?"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":"\n        "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" stop\n      "},{"type":"NameKeyword","value":"elsif"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"Other\""},{"type":"Text","value":"\n        root_tokenizer "},{"type":"Operator","value":"="},{"type":"Text","value":" @lexer.root_lexer.tokenizer(token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"], "},{"type":"Literal","value":"true"},{"type":"Text","value":")\n        root_tokenizer.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"root_token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n          @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" root_token\n        "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n        @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":"."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"Comment","value":"# A Lexer state. A state has a name and a list of rules."},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# The state machine has a state stack containing references"},{"type":"Text","value":"\n  "},{"type":"Comment","value":"# to states to decide which rules to apply."},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"struct"},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":"\n    property name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":"\n    property rules "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseRule"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"+"},{"type":"Text","value":"(other : "},{"type":"NameConstant","value":"State"},{"type":"Text","value":")\n      new_state "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":".new\n      new_state.name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Random"},{"type":"Text","value":".base58("},{"type":"LiteralNumber","value":"8"},{"type":"Text","value":")\n      new_state.rules "},{"type":"Operator","value":"="},{"type":"Text","value":" rules "},{"type":"Operator","value":"+"},{"type":"Text","value":" other.rules\n      new_state\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CustomCrystalHighlighter"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter"},{"type":"Text","value":"\n    @tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_delimiter"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n      @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: block.call.to_s}\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_interpolation"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n      @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringInterpol\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"\\#{\""},{"type":"Text","value":"}\n      @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: block.call.to_s}\n      @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringInterpol\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"}\""},{"type":"Text","value":"}\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_string_array"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n      @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: block.call.to_s}\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"Comment","value":"# ameba:disable Metrics/CyclomaticComplexity"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render"},{"type":"Text","value":"("},{"type":"NameKeyword","value":"type"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"TokenType"},{"type":"Text","value":", value : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n      "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"type"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .comment?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Comment\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .number?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralNumber\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .char?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringChar\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .symbol?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringSymbol\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .const?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameConstant\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .string?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .ident?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameVariable\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .keyword?, .self?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameKeyword\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .primitive_literal?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Literal\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .operator?\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Operator\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN"},{"type":"Text","value":", "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITER_START"},{"type":"Text","value":", "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITER_END"},{"type":"Text","value":"\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n        @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: value}\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalTokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine::BaseTokenizer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n    @hl "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"CustomCrystalHighlighter"},{"type":"Text","value":".new\n    @lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    @iter : "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n\n    "},{"type":"Comment","value":"# delegate next, to: @iter"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n      "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n        text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n      "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n      "},{"type":"Comment","value":"# Just do the tokenizing"},{"type":"Text","value":"\n      @hl.highlight(text)\n      @iter "},{"type":"Operator","value":"="},{"type":"Text","value":" @hl.@tokens.each\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n      @iter."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n  "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n    "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n      "},{"type":"NameConstant","value":"CrystalTokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n    "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n  "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n"}]