mirror of
https://github.com/ralsina/tartrazine.git
synced 2024-11-10 05:22:23 +00:00
Make usingself MUCH cheaper, since it was called many times when parsing C
This commit is contained in:
parent
f72a40f095
commit
58e8dac038
@ -109,7 +109,7 @@ module Tartrazine
|
|||||||
when "usingself"
|
when "usingself"
|
||||||
# Shunt to another copy of this lexer
|
# Shunt to another copy of this lexer
|
||||||
return [] of Token if match.empty?
|
return [] of Token if match.empty?
|
||||||
new_lexer = Lexer.from_xml(lexer.xml)
|
new_lexer = lexer.copy
|
||||||
new_lexer.tokenize(String.new(match[match_group].value), usingself: true)
|
new_lexer.tokenize(String.new(match[match_group].value), usingself: true)
|
||||||
when "combined"
|
when "combined"
|
||||||
# Combine two states into one anonymous state
|
# Combine two states into one anonymous state
|
||||||
|
13
src/lexer.cr
13
src/lexer.cr
@ -56,10 +56,18 @@ module Tartrazine
|
|||||||
not_multiline: false,
|
not_multiline: false,
|
||||||
ensure_nl: false,
|
ensure_nl: false,
|
||||||
}
|
}
|
||||||
property xml : String = ""
|
# property xml : String = ""
|
||||||
property states = {} of String => State
|
property states = {} of String => State
|
||||||
property state_stack = ["root"]
|
property state_stack = ["root"]
|
||||||
|
|
||||||
|
def copy : Lexer
|
||||||
|
new_lexer = Lexer.new
|
||||||
|
new_lexer.config = config
|
||||||
|
new_lexer.states = states
|
||||||
|
new_lexer.state_stack = state_stack[0..-1]
|
||||||
|
new_lexer
|
||||||
|
end
|
||||||
|
|
||||||
# Turn the text into a list of tokens. The `usingself` parameter
|
# Turn the text into a list of tokens. The `usingself` parameter
|
||||||
# is true when the lexer is being used to tokenize a string
|
# is true when the lexer is being used to tokenize a string
|
||||||
# from a larger text that is already being tokenized.
|
# from a larger text that is already being tokenized.
|
||||||
@ -85,12 +93,10 @@ module Tartrazine
|
|||||||
if matched
|
if matched
|
||||||
# Move position forward, save the tokens,
|
# Move position forward, save the tokens,
|
||||||
# tokenize from the new position
|
# tokenize from the new position
|
||||||
# Log.trace { "MATCHED: #{rule.xml}" }
|
|
||||||
pos = new_pos
|
pos = new_pos
|
||||||
tokens += new_tokens
|
tokens += new_tokens
|
||||||
break
|
break
|
||||||
end
|
end
|
||||||
# Log.trace { "NOT MATCHED: #{rule.xml}" }
|
|
||||||
end
|
end
|
||||||
# If no rule matches, emit an error token
|
# If no rule matches, emit an error token
|
||||||
unless matched
|
unless matched
|
||||||
@ -156,7 +162,6 @@ module Tartrazine
|
|||||||
# ameba:disable Metrics/CyclomaticComplexity
|
# ameba:disable Metrics/CyclomaticComplexity
|
||||||
def self.from_xml(xml : String) : Lexer
|
def self.from_xml(xml : String) : Lexer
|
||||||
l = Lexer.new
|
l = Lexer.new
|
||||||
l.xml = xml
|
|
||||||
lexer = XML.parse(xml).first_element_child
|
lexer = XML.parse(xml).first_element_child
|
||||||
if lexer
|
if lexer
|
||||||
config = lexer.children.find { |node|
|
config = lexer.children.find { |node|
|
||||||
|
Loading…
Reference in New Issue
Block a user