diff --git a/TODO.md b/TODO.md index 51cf8e9..71e22bf 100644 --- a/TODO.md +++ b/TODO.md @@ -9,4 +9,4 @@ * ✅ Implement lexer loader by file extension * ✅ Add --line-numbers to terminal formatter * Implement lexer loader by mime type -* Implement Delegating lexers \ No newline at end of file +* ✅ Implement Delegating lexers diff --git a/lexers/markdown.xml b/lexers/markdown.xml index 058f9b6..e415ebf 100644 --- a/lexers/markdown.xml +++ b/lexers/markdown.xml @@ -18,19 +18,21 @@ - - - - + + + + + + + + + + + + + + + diff --git a/src/actions.cr b/src/actions.cr index 74ac190..9ae536d 100644 --- a/src/actions.cr +++ b/src/actions.cr @@ -16,13 +16,16 @@ module Tartrazine Push Token Using + Usingbygroup Usingself end struct Action property actions : Array(Action) = [] of Action + @content_index : Int32 = 0 @depth : Int32 = 0 + @lexer_index : Int32 = 0 @lexer_name : String = "" @states : Array(String) = [] of String @states_to_push : Array(String) = [] of String @@ -62,6 +65,9 @@ module Tartrazine @states = xml.attributes.select { |attrib| attrib.name == "state" }.map &.content + when ActionType::Usingbygroup + @lexer_index = xml["lexer"].to_i + @content_index = xml["content"].to_i end end @@ -134,6 +140,12 @@ module Tartrazine tokenizer.lexer.states[new_state.name] = new_state tokenizer.state_stack << new_state.name [] of Token + when ActionType::Usingbygroup + # Shunt to content-specified lexer + return [] of Token if match.empty? + Tartrazine.lexer(String.new(match[@lexer_index].value)).tokenizer( + String.new(match[@content_index].value), + secondary: true).to_a else raise Exception.new("Unknown action type: #{@type}") end diff --git a/src/lexer.cr b/src/lexer.cr index a515baf..e38a261 100644 --- a/src/lexer.cr +++ b/src/lexer.cr @@ -12,24 +12,24 @@ module Tartrazine def self.lexer(name : String? = nil, filename : String? = nil) : BaseLexer return lexer_by_name(name) if name && name != "autodetect" return lexer_by_filename(filename) if filename - + Lexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end) end - + private def self.lexer_by_name(name : String) : BaseLexer lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil) return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+" raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil? - + Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end) end - + private def self.lexer_by_filename(filename : String) : BaseLexer candidates = Set(String).new LEXERS_BY_FILENAME.each do |k, v| candidates += v.to_set if File.match?(k, File.basename(filename)) end - + case candidates.size when 0 lexer_file_name = LEXERS_BY_NAME["plaintext"] @@ -38,16 +38,17 @@ module Tartrazine else raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}") end - + Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end) end - + private def self.create_delegating_lexer(name : String) : BaseLexer language, root = name.split("+", 2) language_lexer = lexer(language) root_lexer = lexer(root) DelegatingLexer.new(language_lexer, root_lexer) end + # Return a list of all lexers def self.lexers : Array(String) LEXERS_BY_NAME.keys.sort!