packages/vaex-core/vaex/__init__.py
def read_csv_and_convert(path, shuffle=False, copy_index=True, **kwargs):
    '''Convert a path (or glob pattern) to a single hdf5 file, will open the hdf5 file if exists.

    Example:
            >>> vaex.read_csv_and_convert('test-*.csv', shuffle=True)  # this may take a while
            >>> vaex.read_csv_and_convert('test-*.csv', shuffle=True)  # 2nd time it is instant

    :param str path: path of file or glob pattern for multiple files
    :param bool shuffle: shuffle DataFrame when converting to hdf5
    :param bool copy_index: by default pandas will create an index (row number), set to false if you want to drop that
    :param kwargs: parameters passed to pandas' read_cvs

    '''
    from concurrent.futures import ProcessPoolExecutor
    import pandas as pd
    filenames = glob.glob(path)
    if len(filenames) > 1:
        filename_hdf5 = _convert_name(filenames, shuffle=shuffle)
        filename_hdf5_noshuffle = _convert_name(filenames, shuffle=False)
        if not os.path.exists(filename_hdf5):
            if not os.path.exists(filename_hdf5_noshuffle):
                # with ProcessPoolExecutor() as executor:
                # executor.submit(read_csv_and_convert, filenames, shuffle=shuffle, **kwargs)
                for filename in filenames:
                    read_csv_and_convert(filename, shuffle=shuffle, copy_index=copy_index, **kwargs)
                ds = open_many([_convert_name(k, shuffle=shuffle) for k in filenames])
            else:
                ds = open(filename_hdf5_noshuffle)
            ds.export_hdf5(filename_hdf5, shuffle=shuffle)
        return open(filename_hdf5)
    else:
        filename = filenames[0]
        filename_hdf5 = _convert_name(filename, shuffle=shuffle)
        filename_hdf5_noshuffle = _convert_name(filename, shuffle=False)
        if not os.path.exists(filename_hdf5):
            if not os.path.exists(filename_hdf5_noshuffle):
                df = pd.read_csv(filename, **kwargs)
                ds = from_pandas(df, copy_index=copy_index)
            else:
                ds = open(filename_hdf5_noshuffle)
            ds.export_hdf5(filename_hdf5, shuffle=shuffle)
        return open(filename_hdf5)
def read_csv_and_convert(path, shuffle=False, copy_index=True, **kwargs):
    '''Convert a path (or glob pattern) to a single hdf5 file, will open the hdf5 file if exists.

    Example:
            >>> vaex.read_csv_and_convert('test-*.csv', shuffle=True)  # this may take a while
            >>> vaex.read_csv_and_convert('test-*.csv', shuffle=True)  # 2nd time it is instant

    :param str path: path of file or glob pattern for multiple files
    :param bool shuffle: shuffle DataFrame when converting to hdf5
    :param bool copy_index: by default pandas will create an index (row number), set to false if you want to drop that
    :param kwargs: parameters passed to pandas' read_cvs

    '''
    from concurrent.futures import ProcessPoolExecutor
    import pandas as pd
    filenames = glob.glob(path)
    if len(filenames) > 1:
        filename_hdf5 = _convert_name(filenames, shuffle=shuffle)
        filename_hdf5_noshuffle = _convert_name(filenames, shuffle=False)
        if not os.path.exists(filename_hdf5):
            if not os.path.exists(filename_hdf5_noshuffle):
                # with ProcessPoolExecutor() as executor:
                # executor.submit(read_csv_and_convert, filenames, shuffle=shuffle, **kwargs)
                for filename in filenames:
                    read_csv_and_convert(filename, shuffle=shuffle, copy_index=copy_index, **kwargs)
                ds = open_many([_convert_name(k, shuffle=shuffle) for k in filenames])
            else:
                ds = open(filename_hdf5_noshuffle)
            ds.export_hdf5(filename_hdf5, shuffle=shuffle)
        return open(filename_hdf5)
    else:
        filename = filenames[0]
        filename_hdf5 = _convert_name(filename, shuffle=shuffle)
        filename_hdf5_noshuffle = _convert_name(filename, shuffle=False)
        if not os.path.exists(filename_hdf5):
            if not os.path.exists(filename_hdf5_noshuffle):
                df = pd.read_csv(filename, **kwargs)
                ds = from_pandas(df, copy_index=copy_index)
            else:
                ds = open(filename_hdf5_noshuffle)
            ds.export_hdf5(filename_hdf5, shuffle=shuffle)
        return open(filename_hdf5)
Convert a path (or glob pattern) to a single hdf5 file, will open the hdf5 file if exists.

    Example:
            >>> vaex.read_csv_and_convert('test-*.csv', shuffle=True)  # this may take a while
            >>> vaex.read_csv_and_convert('test-*.csv', shuffle=True)  # 2nd time it is instant

    :param str path: path of file or glob pattern for multiple files
    :param bool shuffle: shuffle DataFrame when converting to hdf5
    :param bool copy_index: by default pandas will create an index (row number), set to false if you want to drop that
    :param kwargs: parameters passed to pandas' read_cvs