class Gem::RequestSet::Lockfile::Tokenizer
Constants
- EOF
- Token
Public Class Methods
from_file(file)
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 10 def self.from_file(file) new File.read(file), file end
new(input, filename = nil, line = 0, pos = 0)
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 14 def initialize(input, filename = nil, line = 0, pos = 0) @line = line @line_pos = pos @tokens = [] @filename = filename tokenize input end
Public Instance Methods
empty?()
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 42 def empty? @tokens.empty? end
make_parser(set, platforms)
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 22 def make_parser(set, platforms) Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename end
next_token()
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 50 def next_token @tokens.shift end
Also aliased as: shift
peek()
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 55 def peek @tokens.first || EOF end
skip(type)
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 30 def skip(type) @tokens.shift while !@tokens.empty? && peek.type == type end
to_a()
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 26 def to_a @tokens.map {|token| [token.type, token.value, token.column, token.line] } end
unshift(token)
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 46 def unshift(token) @tokens.unshift token end
Private Instance Methods
tokenize(input)
click to toggle source
# File rubygems/request_set/lockfile/tokenizer.rb, line 61 def tokenize(input) require "strscan" s = StringScanner.new input until s.eos? do pos = s.pos pos = s.pos if leading_whitespace = s.scan(/ +/) if s.scan(/[<|=>]{7}/) message = "your #{@filename} contains merge conflict markers" column, line = token_pos pos raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename end @tokens << if s.scan(/\r?\n/) token = Token.new(:newline, nil, *token_pos(pos)) @line_pos = s.pos @line += 1 token elsif s.scan(/[A-Z]+/) if leading_whitespace text = s.matched text += s.scan(/[^\s)]*/).to_s # in case of no match Token.new(:text, text, *token_pos(pos)) else Token.new(:section, s.matched, *token_pos(pos)) end elsif s.scan(/([a-z]+):\s/) s.pos -= 1 # rewind for possible newline Token.new(:entry, s[1], *token_pos(pos)) elsif s.scan(/\(/) Token.new(:l_paren, nil, *token_pos(pos)) elsif s.scan(/\)/) Token.new(:r_paren, nil, *token_pos(pos)) elsif s.scan(/<=|>=|=|~>|<|>|!=/) Token.new(:requirement, s.matched, *token_pos(pos)) elsif s.scan(/,/) Token.new(:comma, nil, *token_pos(pos)) elsif s.scan(/!/) Token.new(:bang, nil, *token_pos(pos)) elsif s.scan(/[^\s),!]*/) Token.new(:text, s.matched, *token_pos(pos)) else raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}" end end @tokens end