class Liquid::Lexer2
Constants
- CLOSE_ROUND
- CLOSE_SQUARE
- COLON
- COMMA
- COMPARISION_NOT_EQUAL
- COMPARISON_CONTAINS
- COMPARISON_EQUAL
- COMPARISON_GREATER_THAN
- COMPARISON_GREATER_THAN_OR_EQUAL
- COMPARISON_JUMP_TABLE
- COMPARISON_LESS_THAN
- COMPARISON_LESS_THAN_OR_EQUAL
- COMPARISON_NOT_EQUAL_ALT
- DASH
- DOT
- DOTDOT
- DOT_ORD
- DOUBLE_STRING_LITERAL
- EOS
- IDENTIFIER
- NEXT_MATCHER_JUMP_TABLE
- NUMBER_LITERAL
- NUMBER_TABLE
- OPEN_ROUND
- OPEN_SQUARE
- PIPE
- QUESTION
- RUBY_WHITESPACE
- SINGLE_COMPARISON_TOKENS
- SINGLE_STRING_LITERAL
- SPECIAL_TABLE
- TWO_CHARS_COMPARISON_JUMP_TABLE
- WHITESPACE_OR_NOTHING
Public Class Methods
new(input)
click to toggle source
# File lib/liquid/lexer.rb, line 160 def initialize(input) @ss = StringScanner.new(input) end
Public Instance Methods
raise_syntax_error(start_pos)
click to toggle source
# File lib/liquid/lexer.rb, line 229 def raise_syntax_error(start_pos) @ss.pos = start_pos # the character could be a UTF-8 character, use getch to get all the bytes raise SyntaxError, "Unexpected character #{@ss.getch}" end
tokenize()
click to toggle source
rubocop:disable Metrics/BlockNesting
# File lib/liquid/lexer.rb, line 165 def tokenize @output = [] until @ss.eos? @ss.skip(WHITESPACE_OR_NOTHING) break if @ss.eos? start_pos = @ss.pos peeked = @ss.peek_byte if (special = SPECIAL_TABLE[peeked]) @ss.scan_byte # Special case for ".." if special == DOT && @ss.peek_byte == DOT_ORD @ss.scan_byte @output << DOTDOT elsif special == DASH # Special case for negative numbers if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte] @ss.pos -= 1 @output << [:number, @ss.scan(NUMBER_LITERAL)] else @output << special end else @output << special end elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked]) @ss.scan_byte if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte]) @output << found @ss.scan_byte else raise_syntax_error(start_pos) end elsif (sub_table = COMPARISON_JUMP_TABLE[peeked]) @ss.scan_byte if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte]) @output << found @ss.scan_byte else @output << SINGLE_COMPARISON_TOKENS[peeked] end else type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked] if type && (t = @ss.scan(pattern)) # Special case for "contains" @output << if type == :id && t == "contains" && @output.last&.first != :dot COMPARISON_CONTAINS else [type, t] end else raise_syntax_error(start_pos) end end end # rubocop:enable Metrics/BlockNesting @output << EOS end