ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-10 05:22:23 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Nicer tokenize flow

Browse Source
This commit is contained in:
Roberto Alsina 2024-08-05 07:54:14 -03:00
parent da963e8a31
commit 03b81c2e54
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/tartrazine.cr
View File

@ -67,13 +67,13 @@ module Tartrazine
Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" } Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
state.rules.each do |rule| state.rules.each do |rule|
matched, new_pos, new_tokens = rule.match(text, pos, self) matched, new_pos, new_tokens = rule.match(text, pos, self)
if matched
Log.trace { "MATCHED: #{rule.xml}" }
pos = new_pos
tokens += new_tokens
break # We go back to processing with current state
end
Log.trace { "NOT MATCHED: #{rule.xml}" } Log.trace { "NOT MATCHED: #{rule.xml}" }
next unless matched
Log.trace { "MATCHED: #{rule.xml}" }
pos = new_pos
tokens += new_tokens
break # We go back to processing with current state
end end
# If no rule matches, emit an error token # If no rule matches, emit an error token
unless matched unless matched