# File lib/rdoc/ruby_lex.rb, line 65
  def self.tokenize ruby, options
    tokens = []

    scanner = RDoc::RubyLex.new ruby, options
    scanner.exception_on_syntax_error = true

    while token = scanner.token do
      tokens << token
    end

    tokens
  end