def extract(data)
            data.force_encoding(@encoding)
            @input << data

            entities = []

            loop do
                found = false

                last = if @indicator
                    check = @input.partition(@indicator)
                    break unless check[1].length > 0

                    check[2]
                else
                    @input
                end

                result = @callback.call(last)

                if result
                    found = true

                    # Check for multi-byte indicator edge case
                    case result
                    when Integer
                        entities << last[0...result]
                        @input = last[result..-1]
                    else
                        entities << last
                        reset
                    end
                end

                break if not found
            end

            # Check to see if the buffer has exceeded capacity, if we're imposing a limit
            if @size_limit && @input.size > @size_limit
                if @indicator.respond_to?(:length) # check for regex
                    # save enough of the buffer that if one character of the indicator were
                    # missing we would match on next extract (very much an edge case) and
                    # best we can do with a full buffer.
                    @input = @input[-(@indicator.length - 1)..-1]
                else
                    reset
                end
                raise 'input buffer exceeded limit' if @verbose
            end

            return entities
        end
def extract(data)
            data.force_encoding(@encoding)
            @input << data

            entities = []

            loop do
                found = false

                last = if @indicator
                    check = @input.partition(@indicator)
                    break unless check[1].length > 0

                    check[2]
                else
                    @input
                end

                result = @callback.call(last)

                if result
                    found = true

                    # Check for multi-byte indicator edge case
                    case result
                    when Integer
                        entities << last[0...result]
                        @input = last[result..-1]
                    else
                        entities << last
                        reset
                    end
                end

                break if not found
            end

            # Check to see if the buffer has exceeded capacity, if we're imposing a limit
            if @size_limit && @input.size > @size_limit
                if @indicator.respond_to?(:length) # check for regex
                    # save enough of the buffer that if one character of the indicator were
                    # missing we would match on next extract (very much an edge case) and
                    # best we can do with a full buffer.
                    @input = @input[-(@indicator.length - 1)..-1]
                else
                    reset
                end
                raise 'input buffer exceeded limit' if @verbose
            end

            return entities
        end
@param [Hash] options
 Extract takes an arbitrary string of input data and returns an array of
 tokenized entities using a message start indicator

 @example

     tokenizer.extract(data).
         map { |entity| Decode(entity) }.each { ... }

 @param [String] data