process_token (RubyLexer)

    # File lib/ruby_lexer.rb, line 1232
      def process_token(command_state)

        token << src.matched if token =~ /^\w/ && src.scan(/[\!\?](?!=)/)

        result = nil
        last_state = lex_state

        case token
        when /^\$/ then
          self.lex_state, result = :expr_end, :tGVAR
        when /^@@/ then
          self.lex_state, result = :expr_end, :tCVAR
        when /^@/ then
          self.lex_state, result = :expr_end, :tIVAR
        else
          if token =~ /[!?]$/ then
            result = :tFID
          else
            if lex_state == :expr_fname then
              # ident=, not =~ => == or followed by =>
              # TODO test lexing of a=>b vs a==>b
              if src.scan(/=(?:(?![~>=])|(?==>))/) then
                result = :tIDENTIFIER
                token << src.matched
              end
            end

            result ||= if token =~ /^[A-Z]/ then
                         :tCONSTANT
                       else
                         :tIDENTIFIER
                       end
          end

          unless lex_state == :expr_dot then
            # See if it is a reserved word.
            keyword = Keyword.keyword token

            if keyword then
              state           = lex_state
              self.lex_state  = keyword.state
              self.yacc_value = token

              if keyword.id0 == :kDO then
                self.command_start = true
                return :kDO_COND  if cond.is_in_state
                return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
                return :kDO_BLOCK if state == :expr_endarg
                return :kDO
              end

              return keyword.id0 if state == :expr_beg

              self.lex_state = :expr_beg if keyword.id0 != keyword.id1

              return keyword.id1
            end
          end

          if (lex_state == :expr_beg || lex_state == :expr_mid ||
              lex_state == :expr_dot || lex_state == :expr_arg ||
              lex_state == :expr_cmdarg) then
            if command_state then
              self.lex_state = :expr_cmdarg
            else
              self.lex_state = :expr_arg
            end
          else
            self.lex_state = :expr_end
          end
        end

        self.yacc_value = token

        self.lex_state = :expr_end if
          last_state != :expr_dot && self.parser.env[token.to_sym] == :lvar

        return result
      end