streamcorpus_pipeline/_kvlayer.py
def parse_keys_and_ranges(i_str, keyfunc, rangefunc):
    '''Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.

    '''
    while i_str:
        m = _STREAM_ID_RE.match(i_str)
        if m:
            # old style text stream_id
            for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):
                yield retval
            i_str = i_str[m.end():]
            while i_str and ((i_str[0] == ',') or (i_str[0] == ';')):
                i_str = i_str[1:]
            continue

        if len(i_str) == SI_KEY_LENGTH:
            # one key, get it.
            key = parse_si_key(i_str)
            for retval in keyfunc(key):
                yield retval
            return

        keya = i_str[:SI_KEY_LENGTH]
        splitc = i_str[SI_KEY_LENGTH]
        if splitc == '<':
            # range
            keyb = i_str[SI_KEY_LENGTH+1:SI_KEY_LENGTH+1+SI_KEY_LENGTH]
            i_str = i_str[SI_KEY_LENGTH+1+SI_KEY_LENGTH:]
            keya = parse_si_key(keya)
            keyb = parse_si_key(keyb)
            for retval in rangefunc(keya, keyb):
                yield retval
        elif splitc == ';':
            # keya is single key to load
            keya = parse_si_key(keya)
            for retval in keyfunc(keya):
                yield retval
            i_str = i_str[SI_KEY_LENGTH+1+1:]
        else:
            logger.error('bogus key splitter %s, %r', splitc, i_str)
            return
def parse_keys_and_ranges(i_str, keyfunc, rangefunc):
    '''Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.

    '''
    while i_str:
        m = _STREAM_ID_RE.match(i_str)
        if m:
            # old style text stream_id
            for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):
                yield retval
            i_str = i_str[m.end():]
            while i_str and ((i_str[0] == ',') or (i_str[0] == ';')):
                i_str = i_str[1:]
            continue

        if len(i_str) == SI_KEY_LENGTH:
            # one key, get it.
            key = parse_si_key(i_str)
            for retval in keyfunc(key):
                yield retval
            return

        keya = i_str[:SI_KEY_LENGTH]
        splitc = i_str[SI_KEY_LENGTH]
        if splitc == '<':
            # range
            keyb = i_str[SI_KEY_LENGTH+1:SI_KEY_LENGTH+1+SI_KEY_LENGTH]
            i_str = i_str[SI_KEY_LENGTH+1+SI_KEY_LENGTH:]
            keya = parse_si_key(keya)
            keyb = parse_si_key(keyb)
            for retval in rangefunc(keya, keyb):
                yield retval
        elif splitc == ';':
            # keya is single key to load
            keya = parse_si_key(keya)
            for retval in keyfunc(keya):
                yield retval
            i_str = i_str[SI_KEY_LENGTH+1+1:]
        else:
            logger.error('bogus key splitter %s, %r', splitc, i_str)
            return
Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.