class Liquid::Tokenizer

Attributes

for_liquid_tag[R]
line_number[R]

Public Class Methods

new(source, line_numbers = false, line_number: nil, for_liquid_tag: false) click to toggle source
# File lib/liquid/tokenizer.rb, line 7
def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
  @source         = source.to_s.to_str
  @line_number    = line_number || (line_numbers ? 1 : nil)
  @for_liquid_tag = for_liquid_tag
  @offset         = 0
  @tokens         = tokenize
end

Public Instance Methods

shift() click to toggle source
# File lib/liquid/tokenizer.rb, line 15
def shift
  token = @tokens[@offset]
  return nil unless token

  @offset += 1

  if @line_number
    @line_number += @for_liquid_tag ? 1 : token.count("\n")
  end

  token
end

Private Instance Methods

tokenize() click to toggle source
# File lib/liquid/tokenizer.rb, line 30
def tokenize
  return [] if @source.empty?

  return @source.split("\n") if @for_liquid_tag

  tokens = @source.split(TemplateParser)

  # removes the rogue empty element at the beginning of the array
  if tokens[0]&.empty?
    @offset += 1
  end

  tokens
end