def stream_tokens(input, &block)
      scanner = StringScanner.new(input)
      while !scanner.eos?
        current_char = scanner.peek(1)
        if @dsl.state_ignore[current_state].include?(current_char)
          scanner.pos += current_char.size
          @char_pos_in_line += current_char.size
          next
        end

        if debug
          logger.info "lex: [#{current_state}]: lexemes = #{@dsl.state_lexemes[current_state].map(&:name)}"
        end
        # Look for regex match
        longest_token = nil
        @dsl.state_lexemes[current_state].each do |lexeme|
          match = lexeme.match(scanner)
          next if match.nil?
          longest_token = match if longest_token.nil?
          next if longest_token.value.length >= match.value.length
          longest_token = match
        end

        if longest_token
          if longest_token.action
            new_token = longest_token.action.call(self, longest_token)
            # No value returned from action move to the next token
            if new_token.nil? || !new_token.is_a?(Token)
              chars_to_skip = longest_token.value.to_s.length
              scanner.pos += chars_to_skip
              unless longest_token.name == :newline
                @char_pos_in_line += chars_to_skip
              end
              next
            end
          end
          move_by = longest_token.value.to_s.length
          start_char_pos_in_token = @char_pos_in_line + current_char.size
          longest_token.update_line(current_line, start_char_pos_in_token)
          advance_column(move_by)
          scanner.pos += move_by
        end

        # No match
        if longest_token.nil?
          # Check in errors
          if @dsl.state_error[current_state]
            token = Token.new(:error, current_char)
            start_char_pos_in_token = @char_pos_in_line + current_char.size
            token.update_line(current_line, start_char_pos_in_token)
            new_token = @dsl.state_error[current_state].call(self, token)
            advance_column(current_char.length)
            scanner.pos += current_char.length
            if new_token.is_a?(Token) || !new_token.nil?
              longest_token = new_token
            else
              next
            end
          end

          if longest_token.nil?
            complain("Illegal character `#{current_char}`")
          end
        end

        logger.info "lex: #{longest_token}" if debug
        block.call(longest_token)
      end
    end
def stream_tokens(input, &block)
      scanner = StringScanner.new(input)
      while !scanner.eos?
        current_char = scanner.peek(1)
        if @dsl.state_ignore[current_state].include?(current_char)
          scanner.pos += current_char.size
          @char_pos_in_line += current_char.size
          next
        end

        if debug
          logger.info "lex: [#{current_state}]: lexemes = #{@dsl.state_lexemes[current_state].map(&:name)}"
        end
        # Look for regex match
        longest_token = nil
        @dsl.state_lexemes[current_state].each do |lexeme|
          match = lexeme.match(scanner)
          next if match.nil?
          longest_token = match if longest_token.nil?
          next if longest_token.value.length >= match.value.length
          longest_token = match
        end

        if longest_token
          if longest_token.action
            new_token = longest_token.action.call(self, longest_token)
            # No value returned from action move to the next token
            if new_token.nil? || !new_token.is_a?(Token)
              chars_to_skip = longest_token.value.to_s.length
              scanner.pos += chars_to_skip
              unless longest_token.name == :newline
                @char_pos_in_line += chars_to_skip
              end
              next
            end
          end
          move_by = longest_token.value.to_s.length
          start_char_pos_in_token = @char_pos_in_line + current_char.size
          longest_token.update_line(current_line, start_char_pos_in_token)
          advance_column(move_by)
          scanner.pos += move_by
        end

        # No match
        if longest_token.nil?
          # Check in errors
          if @dsl.state_error[current_state]
            token = Token.new(:error, current_char)
            start_char_pos_in_token = @char_pos_in_line + current_char.size
            token.update_line(current_line, start_char_pos_in_token)
            new_token = @dsl.state_error[current_state].call(self, token)
            advance_column(current_char.length)
            scanner.pos += current_char.length
            if new_token.is_a?(Token) || !new_token.nil?
              longest_token = new_token
            else
              next
            end
          end

          if longest_token.nil?
            complain("Illegal character `#{current_char}`")
          end
        end

        logger.info "lex: #{longest_token}" if debug
        block.call(longest_token)
      end
    end
Advances through input and streams tokens

 @param [String] input

 @yield [Lex::Token]

 @api public