splearn/rdd.py
def _block_tuple(iterator, dtypes, bsize=-1):
    """Pack rdd of tuples as tuples of arrays or scipy.sparse matrices."""
    i = 0
    blocked_tuple = None
    for tuple_i in iterator:
        if blocked_tuple is None:
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))

        if (bsize > 0) and (i >= bsize):
            yield tuple(_pack_accumulated(x, dtype)
                        for x, dtype in zip(blocked_tuple, dtypes))
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))
            i = 0

        for x_j, x in zip(tuple_i, blocked_tuple):
            x.append(x_j)
        i += 1
    if i > 0:
        yield tuple(_pack_accumulated(x, dtype)
                    for x, dtype in zip(blocked_tuple, dtypes))
def _block_tuple(iterator, dtypes, bsize=-1):
    """Pack rdd of tuples as tuples of arrays or scipy.sparse matrices."""
    i = 0
    blocked_tuple = None
    for tuple_i in iterator:
        if blocked_tuple is None:
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))

        if (bsize > 0) and (i >= bsize):
            yield tuple(_pack_accumulated(x, dtype)
                        for x, dtype in zip(blocked_tuple, dtypes))
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))
            i = 0

        for x_j, x in zip(tuple_i, blocked_tuple):
            x.append(x_j)
        i += 1
    if i > 0:
        yield tuple(_pack_accumulated(x, dtype)
                    for x, dtype in zip(blocked_tuple, dtypes))
Pack rdd of tuples as tuples of arrays or scipy.sparse matrices.