module Rapidito
  class Tokenizer

    def initialize( *delimiters )
      @regexp = Regexp.union( *delimiters  + [/$/] )
    end

    attr_accessor :source

    def has_next?
      ! @source.empty?
    end

    def next_token
      p = (@source =~ @regexp)
      if p == 0 #delimiter
        token = nil
        @source.sub!( @regexp ) { |match| token=match.to_sym; "" }
        token
      else #text
        token = @source[0,p]
        @source = @source[p,@source.length]
        token
      end
    end

    def all_tokens
      tokens = []
      while has_next?
        tokens << next_token
      end
      tokens
    end

  end
end