From e288a558128b6edf7f4cfd3542f19ef9f1311cf7 Mon Sep 17 00:00:00 2001 From: Roberto Alsina Date: Tue, 10 Sep 2024 21:47:39 -0300 Subject: [PATCH] test: add basic tests for crystal and delegating lexers --- .ameba.yml | 107 +------------------- spec/examples/crystal/lexer.cr.json | 1 + spec/examples/jinja+python/funko.py.j2 | 11 ++ spec/examples/jinja+python/funko.py.j2.json | 1 + spec/tartrazine_spec.cr | 11 ++ 5 files changed, 27 insertions(+), 104 deletions(-) create mode 100644 spec/examples/crystal/lexer.cr.json create mode 100644 spec/examples/jinja+python/funko.py.j2 create mode 100644 spec/examples/jinja+python/funko.py.j2.json diff --git a/.ameba.yml b/.ameba.yml index 11462c5..766994f 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -1,9 +1,9 @@ # This configuration file was generated by `ameba --gen-config` -# on 2024-08-12 22:00:49 UTC using Ameba version 1.6.1. +# on 2024-09-11 00:56:14 UTC using Ameba version 1.6.1. # The point is for the user to remove these configuration records # one by one as the reported problems are removed from the code base. -# Problems found: 2 +# Problems found: 4 # Run `ameba --only Documentation/DocumentationAdmonition` for details Documentation/DocumentationAdmonition: Description: Reports documentation admonitions @@ -11,111 +11,10 @@ Documentation/DocumentationAdmonition: Excluded: - src/lexer.cr - src/actions.cr + - spec/examples/crystal/lexer_spec.cr Admonitions: - TODO - FIXME - BUG Enabled: true Severity: Warning - -# Problems found: 22 -# Run `ameba --only Lint/MissingBlockArgument` for details -Lint/MissingBlockArgument: - Description: Disallows yielding method definitions without block argument - Excluded: - - pygments/tests/examplefiles/cr/test.cr - Enabled: true - Severity: Warning - -# Problems found: 1 -# Run `ameba --only Lint/NotNil` for details -Lint/NotNil: - Description: Identifies usage of `not_nil!` calls - Excluded: - - pygments/tests/examplefiles/cr/test.cr - Enabled: true - Severity: Warning - -# Problems found: 34 -# Run `ameba --only Lint/ShadowingOuterLocalVar` for details -Lint/ShadowingOuterLocalVar: - Description: Disallows the usage of the same name as outer local variables for block - or proc arguments - Excluded: - - pygments/tests/examplefiles/cr/test.cr - Enabled: true - Severity: Warning - -# Problems found: 1 -# Run `ameba --only Lint/UnreachableCode` for details -Lint/UnreachableCode: - Description: Reports unreachable code - Excluded: - - pygments/tests/examplefiles/cr/test.cr - Enabled: true - Severity: Warning - -# Problems found: 6 -# Run `ameba --only Lint/UselessAssign` for details -Lint/UselessAssign: - Description: Disallows useless variable assignments - ExcludeTypeDeclarations: false - Excluded: - - pygments/tests/examplefiles/cr/test.cr - Enabled: true - Severity: Warning - -# Problems found: 3 -# Run `ameba --only Naming/BlockParameterName` for details -Naming/BlockParameterName: - Description: Disallows non-descriptive block parameter names - MinNameLength: 3 - AllowNamesEndingInNumbers: true - Excluded: - - pygments/tests/examplefiles/cr/test.cr - AllowedNames: - - _ - - e - - i - - j - - k - - v - - x - - y - - ex - - io - - ws - - op - - tx - - id - - ip - - k1 - - k2 - - v1 - - v2 - ForbiddenNames: [] - Enabled: true - Severity: Convention - -# Problems found: 1 -# Run `ameba --only Naming/RescuedExceptionsVariableName` for details -Naming/RescuedExceptionsVariableName: - Description: Makes sure that rescued exceptions variables are named as expected - Excluded: - - pygments/tests/examplefiles/cr/test.cr - AllowedNames: - - e - - ex - - exception - - error - Enabled: true - Severity: Convention - -# Problems found: 6 -# Run `ameba --only Naming/TypeNames` for details -Naming/TypeNames: - Description: Enforces type names in camelcase manner - Excluded: - - pygments/tests/examplefiles/cr/test.cr - Enabled: true - Severity: Convention diff --git a/spec/examples/crystal/lexer.cr.json b/spec/examples/crystal/lexer.cr.json new file mode 100644 index 0000000..7d86139 --- /dev/null +++ b/spec/examples/crystal/lexer.cr.json @@ -0,0 +1 @@ +[{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"./constants/lexers\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"./heuristics\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"baked_file_system\""},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"require"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"crystal/syntax_highlighter\""},{"type":"Text","value":"\n\n"},{"type":"NameKeyword","value":"module"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"extend"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BakedFileSystem"},{"type":"Text","value":"\n bake_folder "},{"type":"LiteralString","value":"\"../lexers\""},{"type":"Text","value":", __DIR__\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# Get the lexer object for a language name"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# FIXME: support mimetypes"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":", filename : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":", mimetype : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"? "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_name(name) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" name "},{"type":"Operator","value":"&&"},{"type":"Text","value":" name "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"autodetect\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_filename(filename) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" filename\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" lexer_by_mimetype(mimetype) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" mimetype\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":"["},{"type":"LiteralString","value":"\"plaintext\""},{"type":"Text","value":"]"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_mimetype(mimetype : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_MIMETYPE"},{"type":"Text","value":".fetch(mimetype, "},{"type":"Literal","value":"nil"},{"type":"Text","value":")\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Unknown mimetype: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"mimetype"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":") "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":"\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_name(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":".new "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"crystal\""},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":".fetch(name.downcase, "},{"type":"Literal","value":"nil"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" create_delegating_lexer(name) "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" name.includes? "},{"type":"LiteralString","value":"\"+\""},{"type":"Text","value":"\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Unknown lexer: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":") "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer_file_name."},{"type":"NameKeyword","value":"nil?"},{"type":"Text","value":"\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_filename(filename : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" filename.ends_with?("},{"type":"LiteralString","value":"\".cr\""},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":".new\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n candidates "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Set"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":").new\n "},{"type":"NameConstant","value":"LEXERS_BY_FILENAME"},{"type":"Text","value":".each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"k, v"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n candidates "},{"type":"Operator","value":"+="},{"type":"Text","value":" v.to_set "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".match?(k, "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".basename(filename))\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" candidates.size\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":"["},{"type":"LiteralString","value":"\"plaintext\""},{"type":"Text","value":"]\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" candidates.first\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n lexer_file_name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":".lexer_by_content(filename)\n "},{"type":"NameKeyword","value":"begin"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":".lexer(lexer_file_name)\n "},{"type":"NameKeyword","value":"rescue"},{"type":"Text","value":" ex : "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":"\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Multiple lexers match the filename: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"candidates.to_a.join("},{"type":"LiteralString","value":"\", \""},{"type":"Text","value":")"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":", heuristics suggest "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":" but there is no matching lexer.\""},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".from_xml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/"},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"lexer_file_name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":".xml\""},{"type":"Text","value":").gets_to_end)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexer_by_content(fname : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"?\n h "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Linguist::Heuristic"},{"type":"Text","value":".from_yaml("},{"type":"NameConstant","value":"LexerFiles"},{"type":"Text","value":".get("},{"type":"LiteralString","value":"\"/heuristics.yml\""},{"type":"Text","value":").gets_to_end)\n result "},{"type":"Operator","value":"="},{"type":"Text","value":" h.run(fname, "},{"type":"NameConstant","value":"File"},{"type":"Text","value":".read(fname))\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" result\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Nil"},{"type":"Text","value":"\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new "},{"type":"LiteralString","value":"\"No lexer found for "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"fname"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"String"},{"type":"Text","value":"\n result."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n result.first\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"private"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".create_delegating_lexer(name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n language, root "},{"type":"Operator","value":"="},{"type":"Text","value":" name.split("},{"type":"LiteralString","value":"\"+\""},{"type":"Text","value":", "},{"type":"LiteralNumber","value":"2"},{"type":"Text","value":")\n language_lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer(language)\n root_lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer(root)\n "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":".new(language_lexer, root_lexer)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# Return a list of all lexers"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".lexers : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n "},{"type":"NameConstant","value":"LEXERS_BY_NAME"},{"type":"Text","value":".keys.sort!\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# A token, the output of the tokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"alias"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"NamedTuple"},{"type":"Text","value":"("},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", value: "},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n\n "},{"type":"NameKeyword","value":"abstract"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n property lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n property text : "},{"type":"NameConstant","value":"Bytes"},{"type":"Text","value":"\n property pos : "},{"type":"NameConstant","value":"Int32"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Deque"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").new\n property state_stack "},{"type":"Operator","value":"="},{"type":"Text","value":" ["},{"type":"LiteralString","value":"\"root\""},{"type":"Text","value":"]\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n @text "},{"type":"Operator","value":"="},{"type":"Text","value":" text.to_slice\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @dq.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" @dq.shift\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" pos "},{"type":"Operator","value":"=="},{"type":"Text","value":" @text.size\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" stop\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n matched "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"while"},{"type":"Text","value":" @pos "},{"type":"Operator","value":"<"},{"type":"Text","value":" @text.size\n @lexer.states[@state_stack.last].rules.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"rule"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n matched, new_pos, new_tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" rule.match(@text, @pos, "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" matched\n @pos "},{"type":"Operator","value":"="},{"type":"Text","value":" new_pos\n split_tokens(new_tokens).each { "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":" @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token }\n "},{"type":"NameKeyword","value":"break"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"matched\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @text[@pos] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"10u8"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"}\n @state_stack "},{"type":"Operator","value":"="},{"type":"Text","value":" ["},{"type":"LiteralString","value":"\"root\""},{"type":"Text","value":"]\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Error\""},{"type":"Text","value":", value: "},{"type":"NameConstant","value":"String"},{"type":"Text","value":".new(@text[@pos..@pos])}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n @pos "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"break"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":"."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# If a token contains a newline, split it into two tokens"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"split_tokens"},{"type":"Text","value":"(tokens : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")) : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n split_tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n tokens.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"].includes?("},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":")\n values "},{"type":"Operator","value":"="},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"].split("},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":")\n values.each_with_index "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"value, index"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n value "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":" "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" index "},{"type":"Operator","value":"<"},{"type":"Text","value":" values.size "},{"type":"Operator","value":"-"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"1"},{"type":"Text","value":"\n split_tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": token["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"], value: value}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n split_tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n split_tokens\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"alias"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"abstract"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n property config "},{"type":"Operator","value":"="},{"type":"Text","value":" {\n name: "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":",\n priority: "},{"type":"LiteralNumber","value":"0.0"},{"type":"Text","value":",\n case_insensitive: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n dot_all: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n not_multiline: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n ensure_nl: "},{"type":"Literal","value":"false"},{"type":"Text","value":",\n }\n property states "},{"type":"Operator","value":"="},{"type":"Text","value":" {} "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"String"},{"type":"Text","value":" "},{"type":"Operator","value":"=>"},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameConstant","value":"Tokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# This implements a lexer for Pygments RegexLexers as expressed"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# in Chroma's XML serialization."},{"type":"Text","value":"\n "},{"type":"Comment","value":"#"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# For explanations on what actions and states do"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# the Pygments documentation is a good place to start."},{"type":"Text","value":"\n "},{"type":"Comment","value":"# https://pygments.org/docs/lexerdevelopment/"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# Collapse consecutive tokens of the same type for easier comparison"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# and smaller output"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".collapse_tokens(tokens : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":")) : "},{"type":"NameConstant","value":"Array"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":")\n result "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine::Token"},{"type":"Text","value":"\n tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" tokens.reject { "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":" }\n tokens.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" result.empty?\n result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n "},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n last "},{"type":"Operator","value":"="},{"type":"Text","value":" result.last\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" last["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"]\n new_token "},{"type":"Operator","value":"="},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": last["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"], value: last["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"] "},{"type":"Operator","value":"+"},{"type":"Text","value":" token["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"]}\n result.pop\n result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" new_token\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n result "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n result\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"self"},{"type":"Text","value":".from_xml(xml : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n l "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"RegexLexer"},{"type":"Text","value":".new\n lexer "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"XML"},{"type":"Text","value":".parse(xml).first_element_child\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" lexer\n config "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer.children.find { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"config\""},{"type":"Text","value":"\n }\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" config\n l.config "},{"type":"Operator","value":"="},{"type":"Text","value":" {\n name: xml_to_s(config, name) "},{"type":"Operator","value":"||"},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":",\n priority: xml_to_f(config, priority) "},{"type":"Operator","value":"||"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0.0"},{"type":"Text","value":",\n not_multiline: xml_to_s(config, not_multiline) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n dot_all: xml_to_s(config, dot_all) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n case_insensitive: xml_to_s(config, case_insensitive) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n ensure_nl: xml_to_s(config, ensure_nl) "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"true\""},{"type":"Text","value":",\n }\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n rules "},{"type":"Operator","value":"="},{"type":"Text","value":" lexer.children.find { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"rules\""},{"type":"Text","value":"\n }\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" rules\n "},{"type":"Comment","value":"# Rules contains states 🤷"},{"type":"Text","value":"\n rules.children."},{"type":"NameKeyword","value":"select"},{"type":"Text","value":" { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"state\""},{"type":"Text","value":"\n }.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"state_node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n state "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":".new\n state.name "},{"type":"Operator","value":"="},{"type":"Text","value":" state_node["},{"type":"LiteralString","value":"\"name\""},{"type":"Text","value":"]\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" l.states.has_key?(state.name)\n raise "},{"type":"NameConstant","value":"Exception"},{"type":"Text","value":".new("},{"type":"LiteralString","value":"\"Duplicate state: "},{"type":"LiteralStringInterpol","value":"#{"},{"type":"Text","value":"state.name"},{"type":"LiteralStringInterpol","value":"}"},{"type":"LiteralString","value":"\""},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n l.states[state.name] "},{"type":"Operator","value":"="},{"type":"Text","value":" state\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# And states contain rules 🤷"},{"type":"Text","value":"\n state_node.children."},{"type":"NameKeyword","value":"select"},{"type":"Text","value":" { "},{"type":"Operator","value":"|"},{"type":"Text","value":"node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n node.name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"rule\""},{"type":"Text","value":"\n }.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"rule_node"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" rule_node["},{"type":"LiteralString","value":"\"pattern\""},{"type":"Text","value":"]?\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"Literal","value":"nil"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" rule_node.first_element_child.try "},{"type":"Operator","value":"&"},{"type":"Text","value":".name "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"include\""},{"type":"Text","value":"\n rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"IncludeStateRule"},{"type":"Text","value":".new(rule_node)\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"UnconditionalRule"},{"type":"Text","value":".new(rule_node)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n rule "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Rule"},{"type":"Text","value":".new(rule_node,\n multiline: "},{"type":"Operator","value":"!"},{"type":"Text","value":"l.config["},{"type":"LiteralStringSymbol","value":":not_multiline"},{"type":"Text","value":"],\n dotall: l.config["},{"type":"LiteralStringSymbol","value":":dot_all"},{"type":"Text","value":"],\n ignorecase: l.config["},{"type":"LiteralStringSymbol","value":":case_insensitive"},{"type":"Text","value":"])\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n state.rules "},{"type":"Operator","value":"<<"},{"type":"Text","value":" rule\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n l\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# A lexer that takes two lexers as arguments. A root lexer"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# and a language lexer. Everything is scalled using the"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# language lexer, afterwards all `Other` tokens are lexed"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# using the root lexer."},{"type":"Text","value":"\n "},{"type":"Comment","value":"#"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# This is useful for things like template languages, where"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# you have Jinja + HTML or Jinja + CSS and so on."},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Lexer"},{"type":"Text","value":"\n property language_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n property root_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@language_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", @root_lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":"\n "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# This Tokenizer works with a DelegatingLexer. It first tokenizes"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# using the language lexer, and \"Other\" tokens are tokenized using"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# the root lexer."},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"DelegatingTokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n @dq "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Deque"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").new\n @language_tokenizer : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"DelegatingLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n @language_tokenizer "},{"type":"Operator","value":"="},{"type":"Text","value":" @lexer.language_lexer.tokenizer(text, "},{"type":"Literal","value":"true"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" @dq.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" @dq.shift\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n token "},{"type":"Operator","value":"="},{"type":"Text","value":" @language_tokenizer."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"is_a?"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"return"},{"type":"Text","value":" stop\n "},{"type":"NameKeyword","value":"elsif"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").["},{"type":"LiteralStringSymbol","value":":type"},{"type":"Text","value":"] "},{"type":"Operator","value":"=="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"Other\""},{"type":"Text","value":"\n root_tokenizer "},{"type":"Operator","value":"="},{"type":"Text","value":" @lexer.root_lexer.tokenizer(token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":").["},{"type":"LiteralStringSymbol","value":":value"},{"type":"Text","value":"], "},{"type":"Literal","value":"true"},{"type":"Text","value":")\n root_tokenizer.each "},{"type":"NameKeyword","value":"do"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":"root_token"},{"type":"Operator","value":"|"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" root_token\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n @dq "},{"type":"Operator","value":"<<"},{"type":"Text","value":" token."},{"type":"NameKeyword","value":"as"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"self"},{"type":"Text","value":"."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# A Lexer state. A state has a name and a list of rules."},{"type":"Text","value":"\n "},{"type":"Comment","value":"# The state machine has a state stack containing references"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# to states to decide which rules to apply."},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"struct"},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":"\n property name : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\""},{"type":"Text","value":"\n property rules "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseRule"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"+"},{"type":"Text","value":"(other : "},{"type":"NameConstant","value":"State"},{"type":"Text","value":")\n new_state "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"State"},{"type":"Text","value":".new\n new_state.name "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"Random"},{"type":"Text","value":".base58("},{"type":"LiteralNumber","value":"8"},{"type":"Text","value":")\n new_state.rules "},{"type":"Operator","value":"="},{"type":"Text","value":" rules "},{"type":"Operator","value":"+"},{"type":"Text","value":" other.rules\n new_state\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CustomCrystalHighlighter"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter"},{"type":"Text","value":"\n @tokens "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Operator","value":"[]"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"of"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_delimiter"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: block.call.to_s}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_interpolation"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringInterpol\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"\\#{\""},{"type":"Text","value":"}\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: block.call.to_s}\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringInterpol\""},{"type":"Text","value":", value: "},{"type":"LiteralString","value":"\"}\""},{"type":"Text","value":"}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render_string_array"},{"type":"Text","value":"("},{"type":"Operator","value":"&"},{"type":"Text","value":"block)\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: block.call.to_s}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"Comment","value":"# ameba:disable Metrics/CyclomaticComplexity"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"render"},{"type":"Text","value":"("},{"type":"NameKeyword","value":"type"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"TokenType"},{"type":"Text","value":", value : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":")\n "},{"type":"NameKeyword","value":"case"},{"type":"Text","value":" "},{"type":"NameKeyword","value":"type"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .comment?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Comment\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .number?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralNumber\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .char?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringChar\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .symbol?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralStringSymbol\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .const?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameConstant\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .string?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .ident?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameVariable\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .keyword?, .self?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"NameKeyword\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .primitive_literal?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Literal\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" .operator?\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Operator\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"when"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN"},{"type":"Text","value":", "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITER_START"},{"type":"Text","value":", "},{"type":"NameConstant","value":"Crystal::SyntaxHighlighter::TokenType::DELIMITER_END"},{"type":"Text","value":"\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"LiteralString\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"else"},{"type":"Text","value":"\n @tokens "},{"type":"Operator","value":"<<"},{"type":"Text","value":" {"},{"type":"NameKeyword","value":"type"},{"type":"Text","value":": "},{"type":"LiteralString","value":"\"Text\""},{"type":"Text","value":", value: value}\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalTokenizer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Tartrazine::BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"include"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n @hl "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"NameConstant","value":"CustomCrystalHighlighter"},{"type":"Text","value":".new\n @lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n @iter : "},{"type":"NameConstant","value":"Iterator"},{"type":"Text","value":"("},{"type":"NameConstant","value":"Token"},{"type":"Text","value":")\n\n "},{"type":"Comment","value":"# delegate next, to: @iter"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"initialize"},{"type":"Text","value":"(@lexer : "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":", text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":")\n "},{"type":"Comment","value":"# Respect the `ensure_nl` config option"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"if"},{"type":"Text","value":" text.size "},{"type":"Operator","value":">"},{"type":"Text","value":" "},{"type":"LiteralNumber","value":"0"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" text["},{"type":"LiteralNumber","value":"-1"},{"type":"Text","value":"] "},{"type":"Operator","value":"!="},{"type":"Text","value":" "},{"type":"LiteralStringChar","value":"'\\n'"},{"type":"Text","value":" "},{"type":"Operator","value":"&&"},{"type":"Text","value":" @lexer.config["},{"type":"LiteralStringSymbol","value":":ensure_nl"},{"type":"Text","value":"] "},{"type":"Operator","value":"&&"},{"type":"Text","value":" "},{"type":"Operator","value":"!"},{"type":"Text","value":"secondary\n text "},{"type":"Operator","value":"+="},{"type":"Text","value":" "},{"type":"LiteralString","value":"\"\\n\""},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"Comment","value":"# Just do the tokenizing"},{"type":"Text","value":"\n @hl.highlight(text)\n @iter "},{"type":"Operator","value":"="},{"type":"Text","value":" @hl.@tokens.each\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"next"},{"type":"Text","value":" : "},{"type":"NameConstant","value":"Iterator::Stop"},{"type":"Text","value":" "},{"type":"Operator","value":"|"},{"type":"Text","value":" "},{"type":"NameConstant","value":"Token"},{"type":"Text","value":"\n @iter."},{"type":"NameKeyword","value":"next"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n\n "},{"type":"NameKeyword","value":"class"},{"type":"Text","value":" "},{"type":"NameConstant","value":"CrystalLexer"},{"type":"Text","value":" "},{"type":"Operator","value":"<"},{"type":"Text","value":" "},{"type":"NameConstant","value":"BaseLexer"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"def"},{"type":"Text","value":" "},{"type":"NameVariable","value":"tokenizer"},{"type":"Text","value":"(text : "},{"type":"NameConstant","value":"String"},{"type":"Text","value":", secondary "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Literal","value":"false"},{"type":"Text","value":") : "},{"type":"NameConstant","value":"BaseTokenizer"},{"type":"Text","value":"\n "},{"type":"NameConstant","value":"CrystalTokenizer"},{"type":"Text","value":".new("},{"type":"NameKeyword","value":"self"},{"type":"Text","value":", text, secondary)\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n "},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n"},{"type":"NameKeyword","value":"end"},{"type":"Text","value":"\n"}] diff --git a/spec/examples/jinja+python/funko.py.j2 b/spec/examples/jinja+python/funko.py.j2 new file mode 100644 index 0000000..a5b73a1 --- /dev/null +++ b/spec/examples/jinja+python/funko.py.j2 @@ -0,0 +1,11 @@ +from flask import Flask, request + +app = Flask("{{name}}") + +@app.route('/') +def handle(): + return "Hello World from Flask!" + +@app.route('/ping') +def ping(): + return "OK" diff --git a/spec/examples/jinja+python/funko.py.j2.json b/spec/examples/jinja+python/funko.py.j2.json new file mode 100644 index 0000000..627c417 --- /dev/null +++ b/spec/examples/jinja+python/funko.py.j2.json @@ -0,0 +1 @@ +[{"type":"KeywordNamespace","value":"from"},{"type":"Text","value":" "},{"type":"NameNamespace","value":"flask"},{"type":"Text","value":" "},{"type":"KeywordNamespace","value":"import"},{"type":"Text","value":" "},{"type":"Name","value":"Flask"},{"type":"Punctuation","value":","},{"type":"Text","value":" "},{"type":"Name","value":"request"},{"type":"Text","value":"\n\n"},{"type":"Name","value":"app"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Name","value":"Flask"},{"type":"Punctuation","value":"("},{"type":"LiteralStringDouble","value":"\""},{"type":"CommentPreproc","value":"{{"},{"type":"NameVariable","value":"name"},{"type":"CommentPreproc","value":"}}"},{"type":"LiteralStringDouble","value":"\")"},{"type":"Text","value":"\n\n"},{"type":"NameDecorator","value":"@app.route"},{"type":"Punctuation","value":"("},{"type":"LiteralStringSingle","value":"'/'"},{"type":"Punctuation","value":")"},{"type":"Text","value":"\n"},{"type":"Keyword","value":"def"},{"type":"Text","value":" "},{"type":"NameFunction","value":"handle"},{"type":"Punctuation","value":"():"},{"type":"Text","value":"\n "},{"type":"Keyword","value":"return"},{"type":"Text","value":" "},{"type":"LiteralStringDouble","value":"\"Hello World from Flask!\""},{"type":"Text","value":"\n\n"},{"type":"NameDecorator","value":"@app.route"},{"type":"Punctuation","value":"("},{"type":"LiteralStringSingle","value":"'/ping'"},{"type":"Punctuation","value":")"},{"type":"Text","value":"\n"},{"type":"Keyword","value":"def"},{"type":"Text","value":" "},{"type":"NameFunction","value":"ping"},{"type":"Punctuation","value":"():"},{"type":"Text","value":"\n "},{"type":"Keyword","value":"return"},{"type":"Text","value":" "},{"type":"LiteralStringDouble","value":"\"OK\""},{"type":"Text","value":"\n"}] diff --git a/spec/tartrazine_spec.cr b/spec/tartrazine_spec.cr index 7e8d8c7..ce29075 100644 --- a/spec/tartrazine_spec.cr +++ b/spec/tartrazine_spec.cr @@ -3,6 +3,9 @@ require "./spec_helper" # These are the testcases from Pygments testcases = Dir.glob("#{__DIR__}/tests/**/*txt").sort +# These are custom testcases +examples = Dir.glob("#{__DIR__}/examples/**/*.*").reject(&.ends_with? ".json").sort! + # These lexers don't load because of parsing issues failing_lexers = { "webgpu_shading_language", @@ -51,6 +54,14 @@ not_my_fault = { describe Tartrazine do describe "Lexer" do + examples.each do |example| + it "parses #{example}".split("/")[-2...].join("/") do + lexer = Tartrazine.lexer(name: File.basename(File.dirname(example)).downcase) + text = File.read(example) + expected = Array(Tartrazine::Token).from_json(File.read("#{example}.json")) + Tartrazine::RegexLexer.collapse_tokens(lexer.tokenizer(text).to_a).should eq expected + end + end testcases.each do |testcase| if known_bad.includes?(testcase) pending "parses #{testcase}".split("/")[-2...].join("/") do