yylex (RubyLexer)

    # File lib/ruby_lexer.rb, line 618
      def yylex # 826 lines

        c = ''
        space_seen = false
        command_state = false
        src = self.src

        self.token = nil
        self.yacc_value = nil

        return yylex_string if lex_strterm

        command_state = self.command_start
        self.command_start = false

        last_state = lex_state

        loop do # START OF CASE
          if src.scan(/\ |\t|\r|\f|\13/) then # white spaces, 13 = '\v
            space_seen = true
            next
          elsif src.check(/[^a-zA-Z]/) then
            if src.scan(/\n|#/) then
              self.lineno = nil
              c = src.matched
              if c == '#' then
                src.unread c # ok

                while src.scan(/\s*#.*(\n+|\z)/) do
                  @comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
                end

                if src.eos? then
                  return RubyLexer::EOF
                end
              end

              # Replace a string of newlines with a single one
              src.scan(/\n+/)

              if [:expr_beg, :expr_fname,
                  :expr_dot, :expr_class].include? lex_state then
                next
              end

              self.command_start = true
              self.lex_state = :expr_beg
              return :tNL
            elsif src.scan(/[\]\)\}]/) then
              cond.lexpop
              cmdarg.lexpop
              self.lex_state = :expr_end
              self.yacc_value = src.matched
              result = {
                ")" => :tRPAREN,
                "]" => :tRBRACK,
                "}" => :tRCURLY
              }[src.matched]
              return result
            elsif src.check(/\./) then
              if src.scan(/\.\.\./) then
                self.lex_state = :expr_beg
                self.yacc_value = "..."
                return :tDOT3
              elsif src.scan(/\.\./) then
                self.lex_state = :expr_beg
                self.yacc_value = ".."
                return :tDOT2
              elsif src.scan(/\.\d/) then
                rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
              elsif src.scan(/\./) then
                self.lex_state = :expr_dot
                self.yacc_value = "."
                return :tDOT
              end
            elsif src.scan(/\,/) then
              self.lex_state = :expr_beg
              self.yacc_value = ","
              return :tCOMMA
            elsif src.scan(/\(/) then
              result = :tLPAREN2
              self.command_start = true
              if lex_state == :expr_beg || lex_state == :expr_mid then
                result = :tLPAREN
              elsif space_seen then
                if lex_state == :expr_cmdarg then
                  result = :tLPAREN_ARG
                elsif lex_state == :expr_arg then
                  warning("don't put space before argument parentheses")
                  result = :tLPAREN2
                end
              end

              self.expr_beg_push "("

              return result
            elsif src.check(/\=/) then
              if src.scan(/\=\=\=/) then
                self.fix_arg_lex_state
                self.yacc_value = "==="
                return :tEQQ
              elsif src.scan(/\=\=/) then
                self.fix_arg_lex_state
                self.yacc_value = "=="
                return :tEQ
              elsif src.scan(/\=~/) then
                self.fix_arg_lex_state
                self.yacc_value = "=~"
                return :tMATCH
              elsif src.scan(/\=>/) then
                self.fix_arg_lex_state
                self.yacc_value = "=>"
                return :tASSOC
              elsif src.scan(/\=/) then
                if src.was_begin_of_line and src.scan(/begin(?=\s)/) then
                  @comments << '=' << src.matched

                  unless src.scan(/.*?\n=end\s*(\n|\z)/m) then
                    @comments.clear
                    rb_compile_error("embedded document meets end of file")
                  end

                  @comments << src.matched

                  next
                else
                  self.fix_arg_lex_state
                  self.yacc_value = '='
                  return :tEQL
                end
              end
            elsif src.scan(/\"(#{ESC_RE}|#(#{ESC_RE}|[^\{\#\@\$\"\\])|[^\"\\\#])*\"/o) then
              self.yacc_value = src.matched[1..-2].gsub(ESC_RE) { unescape $1 }
              self.lex_state = :expr_end
              return :tSTRING
            elsif src.scan(/\"/) then # FALLBACK
              self.lex_strterm = [:strterm, STR_DQUOTE, '"', "\0"] # TODO: question this
              self.yacc_value = "\""
              return :tSTRING_BEG
            elsif src.scan(/\@\@?\w*/) then
              self.token = src.matched

              rb_compile_error "`#{token}` is not allowed as a variable name" if
                token =~ /\@\d/

              return process_token(command_state)
            elsif src.scan(/\:\:/) then
              if (lex_state == :expr_beg ||
                  lex_state == :expr_mid ||
                  lex_state == :expr_class ||
                  (lex_state.is_argument && space_seen)) then
                self.lex_state = :expr_beg
                self.yacc_value = "::"
                return :tCOLON3
              end

              self.lex_state = :expr_dot
              self.yacc_value = "::"
              return :tCOLON2
            elsif lex_state != :expr_end && lex_state != :expr_endarg && src.scan(/:([a-zA-Z_]\w*(?:[?!]|=(?!>))?)/) then
              self.yacc_value = src[1]
              self.lex_state = :expr_end
              return :tSYMBOL
            elsif src.scan(/\:/) then
              # ?: / then / when
              if (lex_state == :expr_end || lex_state == :expr_endarg||
                  src.check(/\s/)) then
                self.lex_state = :expr_beg
                self.yacc_value = ":"
                return :tCOLON
              end

              case
              when src.scan(/\'/) then
                self.lex_strterm = [:strterm, STR_SSYM, src.matched, "\0"]
              when src.scan(/\"/) then
                self.lex_strterm = [:strterm, STR_DSYM, src.matched, "\0"]
              end

              self.lex_state = :expr_fname
              self.yacc_value = ":"
              return :tSYMBEG
            elsif src.check(/[0-9]/) then
              return parse_number
            elsif src.scan(/\[/) then
              result = src.matched

              if lex_state == :expr_fname || lex_state == :expr_dot then
                self.lex_state = :expr_arg
                case
                when src.scan(/\]\=/) then
                  self.yacc_value = "[]="
                  return :tASET
                when src.scan(/\]/) then
                  self.yacc_value = "[]"
                  return :tAREF
                else
                  rb_compile_error "unexpected '['"
                end
              elsif lex_state == :expr_beg || lex_state == :expr_mid then
                result = :tLBRACK
              elsif lex_state.is_argument && space_seen then
                result = :tLBRACK
              end

              self.expr_beg_push "["

              return result
            elsif src.scan(/\'(\\.|[^\'])*\'/) then
              self.yacc_value = src.matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'")
              self.lex_state = :expr_end
              return :tSTRING
            elsif src.check(/\|/) then
              if src.scan(/\|\|\=/) then
                self.lex_state = :expr_beg
                self.yacc_value = "||"
                return :tOP_ASGN
              elsif src.scan(/\|\|/) then
                self.lex_state = :expr_beg
                self.yacc_value = "||"
                return :tOROP
              elsif src.scan(/\|\=/) then
                self.lex_state = :expr_beg
                self.yacc_value = "|"
                return :tOP_ASGN
              elsif src.scan(/\|/) then
                self.fix_arg_lex_state
                self.yacc_value = "|"
                return :tPIPE
              end
            elsif src.scan(/\{/) then
              result = if lex_state.is_argument || lex_state == :expr_end then
                         :tLCURLY      #  block (primary)
                       elsif lex_state == :expr_endarg then
                         :tLBRACE_ARG  #  block (expr)
                       else
                         :tLBRACE      #  hash
                       end

              self.expr_beg_push "{"

              return result
            elsif src.scan(/[+-]/) then
              sign = src.matched
              utype, type = if sign == "+" then
                              [:tUPLUS, :tPLUS]
                            else
                              [:tUMINUS, :tMINUS]
                            end

              if lex_state == :expr_fname || lex_state == :expr_dot then
                self.lex_state = :expr_arg
                if src.scan(/@/) then
                  self.yacc_value = "#{sign}@"
                  return utype
                else
                  self.yacc_value = sign
                  return type
                end
              end

              if src.scan(/\=/) then
                self.lex_state = :expr_beg
                self.yacc_value = sign
                return :tOP_ASGN
              end

              if (lex_state == :expr_beg || lex_state == :expr_mid ||
                  (lex_state.is_argument && space_seen && !src.check(/\s/))) then
                if lex_state.is_argument then
                  arg_ambiguous
                end

                self.lex_state = :expr_beg
                self.yacc_value = sign

                if src.check(/\d/) then
                  if utype == :tUPLUS then
                    return self.parse_number
                  else
                    return :tUMINUS_NUM
                  end
                end

                return utype
              end

              self.lex_state = :expr_beg
              self.yacc_value = sign
              return type
            elsif src.check(/\*/) then
              if src.scan(/\*\*=/) then
                self.lex_state = :expr_beg
                self.yacc_value = "**"
                return :tOP_ASGN
              elsif src.scan(/\*\*/) then
                self.yacc_value = "**"
                self.fix_arg_lex_state
                return :tPOW
              elsif src.scan(/\*\=/) then
                self.lex_state = :expr_beg
                self.yacc_value = "*"
                return :tOP_ASGN
              elsif src.scan(/\*/) then
                result = if lex_state.is_argument && space_seen && src.check(/\S/) then
                           warning("`*' interpreted as argument prefix")
                           :tSTAR
                         elsif lex_state == :expr_beg || lex_state == :expr_mid then
                           :tSTAR
                         else
                           :tSTAR2
                         end
                self.yacc_value = "*"
                self.fix_arg_lex_state

                return result
              end
            elsif src.check(/\!/) then
              if src.scan(/\!\=/) then
                self.lex_state = :expr_beg
                self.yacc_value = "!="
                return :tNEQ
              elsif src.scan(/\!~/) then
                self.lex_state = :expr_beg
                self.yacc_value = "!~"
                return :tNMATCH
              elsif src.scan(/\!/) then
                self.lex_state = :expr_beg
                self.yacc_value = "!"
                return :tBANG
              end
            elsif src.check(/\</) then
              if src.scan(/\<\=\>/) then
                self.fix_arg_lex_state
                self.yacc_value = "<=>"
                return :tCMP
              elsif src.scan(/\<\=/) then
                self.fix_arg_lex_state
                self.yacc_value = "<="
                return :tLEQ
              elsif src.scan(/\<\<\=/) then
                self.fix_arg_lex_state
                self.lex_state = :expr_beg
                self.yacc_value = "\<\<"
                return :tOP_ASGN
              elsif src.scan(/\<\</) then
                if (! [:expr_end,    :expr_dot,
                       :expr_endarg, :expr_class].include?(lex_state) &&
                    (!lex_state.is_argument || space_seen)) then
                  tok = self.heredoc_identifier
                  if tok then
                    return tok
                  end
                end

                self.fix_arg_lex_state
                self.yacc_value = "\<\<"
                return :tLSHFT
              elsif src.scan(/\</) then
                self.fix_arg_lex_state
                self.yacc_value = "<"
                return :tLT
              end
            elsif src.check(/\>/) then
              if src.scan(/\>\=/) then
                self.fix_arg_lex_state
                self.yacc_value = ">="
                return :tGEQ
              elsif src.scan(/\>\>=/) then
                self.fix_arg_lex_state
                self.lex_state = :expr_beg
                self.yacc_value = ">>"
                return :tOP_ASGN
              elsif src.scan(/\>\>/) then
                self.fix_arg_lex_state
                self.yacc_value = ">>"
                return :tRSHFT
              elsif src.scan(/\>/) then
                self.fix_arg_lex_state
                self.yacc_value = ">"
                return :tGT
              end
            elsif src.scan(/\`/) then
              self.yacc_value = "`"
              case lex_state
              when :expr_fname then
                self.lex_state = :expr_end
                return :tBACK_REF2
              when :expr_dot then
                self.lex_state = if command_state then
                                   :expr_cmdarg
                                 else
                                   :expr_arg
                                 end
                return :tBACK_REF2
              end
              self.lex_strterm = [:strterm, STR_XQUOTE, '`', "\0"]
              return :tXSTRING_BEG
            elsif src.scan(/\?/) then
              if lex_state == :expr_end || lex_state == :expr_endarg then
                self.lex_state = :expr_beg
                self.yacc_value = "?"
                return :tEH
              end

              if src.eos? then
                rb_compile_error "incomplete character syntax"
              end

              if src.check(/\s|\v/) then
                unless lex_state.is_argument then
                  c2 = { " " => 's',
                        "\n" => 'n',
                        "\t" => 't',
                        "\v" => 'v',
                        "\r" => 'r',
                        "\f" => 'f' }[src.matched]

                  if c2 then
                    warning("invalid character syntax; use ?\\" + c2)
                  end
                end

                # ternary
                self.lex_state = :expr_beg
                self.yacc_value = "?"
                return :tEH
              elsif src.check(/\w(?=\w)/) then # ternary, also
                self.lex_state = :expr_beg
                self.yacc_value = "?"
                return :tEH
              end

              c = if src.scan(/\\/) then
                    self.read_escape
                  else
                    src.getch
                  end
              self.lex_state = :expr_end
              self.yacc_value = c[0].ord & 0xff
              return :tINTEGER
            elsif src.check(/\&/) then
              if src.scan(/\&\&\=/) then
                self.yacc_value = "&&"
                self.lex_state = :expr_beg
                return :tOP_ASGN
              elsif src.scan(/\&\&/) then
                self.lex_state = :expr_beg
                self.yacc_value = "&&"
                return :tANDOP
              elsif src.scan(/\&\=/) then
                self.yacc_value = "&"
                self.lex_state = :expr_beg
                return :tOP_ASGN
              elsif src.scan(/&/) then
                result = if lex_state.is_argument && space_seen &&
                             !src.check(/\s/) then
                           warning("`&' interpreted as argument prefix")
                           :tAMPER
                         elsif lex_state == :expr_beg || lex_state == :expr_mid then
                           :tAMPER
                         else
                           :tAMPER2
                         end

                self.fix_arg_lex_state
                self.yacc_value = "&"
                return result
              end
            elsif src.scan(/\//) then
              if lex_state == :expr_beg || lex_state == :expr_mid then
                self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
                self.yacc_value = "/"
                return :tREGEXP_BEG
              end

              if src.scan(/\=/) then
                self.yacc_value = "/"
                self.lex_state = :expr_beg
                return :tOP_ASGN
              end

              if lex_state.is_argument && space_seen then
                unless src.scan(/\s/) then
                  arg_ambiguous
                  self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
                  self.yacc_value = "/"
                  return :tREGEXP_BEG
                end
              end

              self.fix_arg_lex_state
              self.yacc_value = "/"

              return :tDIVIDE
            elsif src.scan(/\^=/) then
              self.lex_state = :expr_beg
              self.yacc_value = "^"
              return :tOP_ASGN
            elsif src.scan(/\^/) then
              self.fix_arg_lex_state
              self.yacc_value = "^"
              return :tCARET
            elsif src.scan(/\;/) then
              self.command_start = true
              self.lex_state = :expr_beg
              self.yacc_value = ";"
              return :tSEMI
            elsif src.scan(/\~/) then
              if lex_state == :expr_fname || lex_state == :expr_dot then
                src.scan(/@/)
              end

              self.fix_arg_lex_state
              self.yacc_value = "~"

              return :tTILDE
            elsif src.scan(/\\/) then
              if src.scan(/\n/) then
                self.lineno = nil
                space_seen = true
                next
              end
              rb_compile_error "bare backslash only allowed before newline"
            elsif src.scan(/\%/) then
              if lex_state == :expr_beg || lex_state == :expr_mid then
                return parse_quote
              end

              if src.scan(/\=/) then
                self.lex_state = :expr_beg
                self.yacc_value = "%"
                return :tOP_ASGN
              end

              if lex_state.is_argument && space_seen && ! src.check(/\s/) then
                return parse_quote
              end

              self.fix_arg_lex_state
              self.yacc_value = "%"

              return :tPERCENT
            elsif src.check(/\$/) then
              if src.scan(/(\$_)(\w+)/) then
                self.lex_state = :expr_end
                self.token = src.matched
                return process_token(command_state)
              elsif src.scan(/\$_/) then
                self.lex_state = :expr_end
                self.token = src.matched
                self.yacc_value = src.matched
                return :tGVAR
              elsif src.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
                self.lex_state = :expr_end
                self.yacc_value = src.matched
                return :tGVAR
              elsif src.scan(/\$([\&\`\'\+])/) then
                self.lex_state = :expr_end
                # Explicit reference to these vars as symbols...
                if last_state == :expr_fname then
                  self.yacc_value = src.matched
                  return :tGVAR
                else
                  self.yacc_value = src[1].to_sym
                  return :tBACK_REF
                end
              elsif src.scan(/\$([1-9]\d*)/) then
                self.lex_state = :expr_end
                if last_state == :expr_fname then
                  self.yacc_value = src.matched
                  return :tGVAR
                else
                  self.yacc_value = src[1].to_i
                  return :tNTH_REF
                end
              elsif src.scan(/\$0/) then
                self.lex_state = :expr_end
                self.token = src.matched
                return process_token(command_state)
              elsif src.scan(/\$\W|\$\z/) then # TODO: remove?
                self.lex_state = :expr_end
                self.yacc_value = "$"
                return "$"
              elsif src.scan(/\$\w+/)
                self.lex_state = :expr_end
                self.token = src.matched
                return process_token(command_state)
              end
            elsif src.check(/\_/) then
              if src.beginning_of_line? && src.scan(/\__END__(\n|\Z)/) then
                self.lineno = nil
                return RubyLexer::EOF
              elsif src.scan(/\_\w*/) then
                self.token = src.matched
                return process_token(command_state)
              end
            end
          end # END OF CASE

          if src.scan(/\004|\032|\000/) || src.eos? then # ^D, ^Z, EOF
            return RubyLexer::EOF
          else # alpha check
            if src.scan(/\W/) then
              rb_compile_error "Invalid char #{src.matched.inspect} in expression"
            end
          end

          self.token = src.matched if self.src.scan(/\w+/)

          return process_token(command_state)
        end
      end