class Liquid::Lexer2

Constants

CLOSE_ROUND
CLOSE_SQUARE
COLON
COMMA
COMPARISION_NOT_EQUAL
COMPARISON_CONTAINS
COMPARISON_EQUAL
COMPARISON_GREATER_THAN
COMPARISON_GREATER_THAN_OR_EQUAL
COMPARISON_JUMP_TABLE
COMPARISON_LESS_THAN
COMPARISON_LESS_THAN_OR_EQUAL
COMPARISON_NOT_EQUAL_ALT
DASH
DOT
DOTDOT
DOT_ORD
DOUBLE_STRING_LITERAL
EOS
IDENTIFIER
NEXT_MATCHER_JUMP_TABLE
NUMBER_LITERAL
NUMBER_TABLE
OPEN_ROUND
OPEN_SQUARE
PIPE
QUESTION
RUBY_WHITESPACE
SINGLE_COMPARISON_TOKENS
SINGLE_STRING_LITERAL
SPECIAL_TABLE
TWO_CHARS_COMPARISON_JUMP_TABLE
WHITESPACE_OR_NOTHING

Public Class Methods

new(input) click to toggle source
# File lib/liquid/lexer.rb, line 160
def initialize(input)
  @ss = StringScanner.new(input)
end

Public Instance Methods

raise_syntax_error(start_pos) click to toggle source
# File lib/liquid/lexer.rb, line 229
def raise_syntax_error(start_pos)
  @ss.pos = start_pos
  # the character could be a UTF-8 character, use getch to get all the bytes
  raise SyntaxError, "Unexpected character #{@ss.getch}"
end
tokenize() click to toggle source

rubocop:disable Metrics/BlockNesting

# File lib/liquid/lexer.rb, line 165
def tokenize
  @output = []

  until @ss.eos?
    @ss.skip(WHITESPACE_OR_NOTHING)

    break if @ss.eos?

    start_pos = @ss.pos
    peeked = @ss.peek_byte

    if (special = SPECIAL_TABLE[peeked])
      @ss.scan_byte
      # Special case for ".."
      if special == DOT && @ss.peek_byte == DOT_ORD
        @ss.scan_byte
        @output << DOTDOT
      elsif special == DASH
        # Special case for negative numbers
        if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
          @ss.pos -= 1
          @output << [:number, @ss.scan(NUMBER_LITERAL)]
        else
          @output << special
        end
      else
        @output << special
      end
    elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
      @ss.scan_byte
      if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
        @output << found
        @ss.scan_byte
      else
        raise_syntax_error(start_pos)
      end
    elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
      @ss.scan_byte
      if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
        @output << found
        @ss.scan_byte
      else
        @output << SINGLE_COMPARISON_TOKENS[peeked]
      end
    else
      type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]

      if type && (t = @ss.scan(pattern))
        # Special case for "contains"
        @output << if type == :id && t == "contains" && @output.last&.first != :dot
          COMPARISON_CONTAINS
        else
          [type, t]
        end
      else
        raise_syntax_error(start_pos)
      end
    end
  end
  # rubocop:enable Metrics/BlockNesting

  @output << EOS
end