def lex(input)
      @input = input

      return enum_for(:lex, input) unless block_given?

      if debug
        logger.info "lex: tokens   = #{@dsl.lex_tokens}"
        logger.info "lex: states   = #{@dsl.state_info}"
        logger.info "lex: ignore   = #{@dsl.state_ignore}"
        logger.info "lex: error    = #{@dsl.state_error}"
      end

      stream_tokens(input) do |token|
        yield token
      end
    end
def lex(input)
      @input = input

      return enum_for(:lex, input) unless block_given?

      if debug
        logger.info "lex: tokens   = #{@dsl.lex_tokens}"
        logger.info "lex: states   = #{@dsl.state_info}"
        logger.info "lex: ignore   = #{@dsl.state_ignore}"
        logger.info "lex: error    = #{@dsl.state_error}"
      end

      stream_tokens(input) do |token|
        yield token
      end
    end
Tokenizes input and returns all tokens

 @param [String] input

 @return [Enumerator]
   the tokens found

 @api public