mirror of
https://github.com/ralsina/tartrazine.git
synced 2024-09-20 07:21:22 +00:00
113 lines
3.6 KiB
Crystal
113 lines
3.6 KiB
Crystal
require "./spec_helper"
|
|
|
|
# These are the testcases from Pygments
|
|
testcases = Dir.glob("#{__DIR__}/tests/**/*txt").sort
|
|
|
|
# These lexers don't load because of parsing issues
|
|
failing_lexers = {
|
|
"webgpu_shading_language",
|
|
}
|
|
|
|
# These testcases fail because of differences in the way chroma and tartrazine
|
|
# represent unicode, but they are actually correct
|
|
unicode_problems = {
|
|
"#{__DIR__}/tests/java/test_string_literals.txt",
|
|
"#{__DIR__}/tests/json/test_strings.txt",
|
|
"#{__DIR__}/tests/systemd/example1.txt",
|
|
}
|
|
|
|
# These testcases fail because of differences in the way chroma and tartrazine tokenize
|
|
# but tartrazine is correct
|
|
bad_in_chroma = {
|
|
"#{__DIR__}/tests/bash_session/test_comment_after_prompt.txt",
|
|
"#{__DIR__}/tests/java/test_default.txt",
|
|
"#{__DIR__}/tests/java/test_multiline_string.txt",
|
|
"#{__DIR__}/tests/java/test_numeric_literals.txt",
|
|
"#{__DIR__}/tests/php/test_string_escaping_run.txt",
|
|
"#{__DIR__}/tests/python_2/test_cls_builtin.txt",
|
|
}
|
|
|
|
known_bad = {
|
|
"#{__DIR__}/tests/bash_session/fake_ps2_prompt.txt",
|
|
"#{__DIR__}/tests/bash_session/prompt_in_output.txt",
|
|
"#{__DIR__}/tests/bash_session/test_newline_in_echo_no_ps2.txt",
|
|
"#{__DIR__}/tests/bash_session/test_newline_in_ls_ps2.txt",
|
|
"#{__DIR__}/tests/bash_session/ps2_prompt.txt",
|
|
"#{__DIR__}/tests/bash_session/test_newline_in_ls_no_ps2.txt",
|
|
"#{__DIR__}/tests/bash_session/test_virtualenv.txt",
|
|
"#{__DIR__}/tests/bash_session/test_newline_in_echo_ps2.txt",
|
|
"#{__DIR__}/tests/c/test_string_resembling_decl_end.txt",
|
|
"#{__DIR__}/tests/html/css_backtracking.txt",
|
|
"#{__DIR__}/tests/mcfunction/data.txt",
|
|
"#{__DIR__}/tests/mcfunction/selectors.txt",
|
|
"#{__DIR__}/tests/php/anonymous_class.txt",
|
|
"#{__DIR__}/tests/html/javascript_unclosed.txt",
|
|
|
|
}
|
|
|
|
# Tests that fail because of a limitation in PCRE2
|
|
not_my_fault = {
|
|
"#{__DIR__}/tests/fortran/test_string_cataback.txt",
|
|
}
|
|
|
|
describe Tartrazine do
|
|
describe "Lexer" do
|
|
testcases.each do |testcase|
|
|
if known_bad.includes?(testcase)
|
|
pending "parses #{testcase}".split("/")[-2...].join("/") do
|
|
end
|
|
else
|
|
it "parses #{testcase}".split("/")[-2...].join("/") do
|
|
text = File.read(testcase).split("---input---\n").last.split("---tokens---").first
|
|
lexer_name = File.basename(File.dirname(testcase)).downcase
|
|
unless failing_lexers.includes?(lexer_name) ||
|
|
unicode_problems.includes?(testcase) ||
|
|
bad_in_chroma.includes?(testcase) ||
|
|
not_my_fault.includes?(testcase)
|
|
tokenize(lexer_name, text).should eq(chroma_tokenize(lexer_name, text))
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
# Helper that creates lexer and tokenizes
|
|
def tokenize(lexer_name, text)
|
|
lexer = Tartrazine::Lexer.from_xml(File.read("lexers/#{lexer_name}.xml"))
|
|
collapse_tokens(lexer.tokenize(text))
|
|
end
|
|
|
|
# Helper that tokenizes using chroma to validate the lexer
|
|
def chroma_tokenize(lexer_name, text)
|
|
output = IO::Memory.new
|
|
input = IO::Memory.new(text)
|
|
Process.run(
|
|
"chroma",
|
|
["-f", "json", "-l", lexer_name],
|
|
input: input, output: output
|
|
)
|
|
collapse_tokens(Array(Tartrazine::Token).from_json(output.to_s))
|
|
end
|
|
|
|
# Collapse consecutive tokens of the same type for easier comparison
|
|
def collapse_tokens(tokens : Array(Tartrazine::Token))
|
|
result = [] of Tartrazine::Token
|
|
|
|
tokens.each do |token|
|
|
if result.empty?
|
|
result << token
|
|
next
|
|
end
|
|
last = result.last
|
|
if last[:type] == token[:type]
|
|
new_token = {type: last[:type], value: last[:value] + token[:value]}
|
|
result.pop
|
|
result << new_token
|
|
else
|
|
result << token
|
|
end
|
|
end
|
|
result
|
|
end
|