packages/vaex-core/vaex/ext/readcol.py
def readcol(filename,skipline=0,skipafter=0,names=False,fsep=None,twod=True,
        fixedformat=None,asdict=False,comment='#',verbose=True,nullval=None,
        asStruct=False,namecomment=True,removeblanks=False,header_badchars=None,
        asRecArray=False):
    """
    The default return is a two dimensional float array.  If you want a list of
    columns output instead of a 2D array, pass 'twod=False'.  In this case,
    each column's data type will be automatically detected.
    
    Example usage:
    CASE 1) a table has the format:
     X    Y    Z
    0.0  2.4  8.2
    1.0  3.4  5.6
    0.7  3.2  2.1
    ...
    names,(x,y,z)=readcol("myfile.tbl",names=True,twod=False)
    or
    x,y,z=readcol("myfile.tbl",skipline=1,twod=False)
    or 
    names,xx = readcol("myfile.tbl",names=True)
    or
    xxdict = readcol("myfile.tbl",asdict=True)
    or
    xxstruct = readcol("myfile.tbl",asStruct=True)

    CASE 2) no title is contained into the table, then there is
    no need to skipline:
    x,y,z=readcol("myfile.tbl")
    
    CASE 3) there is a names column and then more descriptive text:
     X      Y     Z
    (deg) (deg) (km/s) 
    0.0    2.4   8.2
    1.0    3.4.  5.6
    ...
    then use:
    names,x,y,z=readcol("myfile.tbl",names=True,skipline=1,twod=False)
    or
    x,y,z=readcol("myfile.tbl",skipline=2,twod=False)

    INPUTS:
        fsep - field separator, e.g. for comma separated value (csv) files
        skipline - number of lines to ignore at the start of the file
        names - read / don't read in the first line as a list of column names
                can specify an integer line number too, though it will be 
                the line number after skipping lines
        twod - two dimensional or one dimensional output
        nullval - if specified, all instances of this value will be replaced
           with a floating NaN
        asdict - zips names with data to create a dict with column headings 
            tied to column data.  If asdict=True, names will be set to True
        asStruct - same as asdict, but returns a structure instead of a dictionary
            (i.e. you call struct.key instead of struct['key'])
        fixedformat - if you have a fixed format file, this is a python list of 
            column lengths.  e.g. the first table above would be [3,5,5].  Note
            that if you specify the wrong fixed format, you will get junk; if your
            format total is greater than the line length, the last entries will all
            be blank but readcol will not report an error.
        namecomment - assumed that "Name" row is on a comment line.  If it is not - 
            e.g., it is the first non-comment line, change this to False
        removeblanks - remove all blank entries from split lines.  This can cause lost
            data if you have blank entries on some lines.
        header_badchars - remove these characters from a header before parsing it
            (helpful for IPAC tables that are delimited with | )

    If you get this error: "scipy could not be imported.  Your table must have
    full rows." it means readcol cannot automatically guess which columns
    contain data.  If you have scipy and columns of varying length, readcol will
    read in all of the rows with length=mode(row lengths).
    """
    with open(filename,'r') as f:
        f = f.readlines()
    
        null=[f.pop(0) for i in range(skipline)]

        commentfilter = make_commentfilter(comment)

        if not asStruct:
            asStruct = asRecArray

        if namecomment is False and (names or asdict or asStruct):
            while 1:
                line = f.pop(0)
                if line[0] != comment:
                    nameline = line
                    if header_badchars:
                        for c in header_badchars:
                            nameline = nameline.replace(c,' ')
                    nms=nameline.split(fsep)
                    break
                elif len(f) == 0:
                    raise Exception("No uncommented lines found.")
        else:
            if names or asdict or asStruct:
                # can specify name line
                if type(names) == type(1):
                    nameline = f.pop(names)
                else:
                    nameline = f.pop(0)
                if nameline[0]==comment:
                    nameline = nameline[1:]
                if header_badchars:
                    for c in header_badchars:
                        nameline = nameline.replace(c,' ')
                nms=list([name.strip() for name in nameline.split(fsep)])

        null=[f.pop(0) for i in range(skipafter)]

        if fixedformat:
            myreadff = lambda x: readff(x,fixedformat)
            splitarr = list(map(myreadff,f))
            splitarr = list(filter(commentfilter,splitarr))
        else:
            fstrip = list(map(str.strip,f))
            fseps = [ fsep for i in range(len(f)) ]
            splitarr = list(map(str.split,fstrip,fseps))
            if removeblanks:
                for i in range(splitarr.count([''])):
                    splitarr.remove([''])

            splitarr = list(filter(commentfilter,splitarr))

            # check to make sure each line has the same number of columns to avoid
            # "ValueError: setting an array element with a sequence."
            nperline = list(map(len,splitarr))
            if hasmode:
                ncols,nrows = mode(nperline)
                if nrows != len(splitarr):
                    if verbose:
                        print("Removing %i rows that don't match most common length %i.  \
                         \n%i rows read into array." % (len(splitarr) - nrows,ncols,nrows))
                    for i in range(len(splitarr)-1,-1,-1):  # need to go backwards
                        if nperline[i] != ncols:
                            splitarr.pop(i)

        try:
            x = numpy.asarray( splitarr , dtype='float')
        except ValueError:
            if verbose:
                print("WARNING: reading as string array because %s array failed" % 'float')
            try:
                x = numpy.asarray( splitarr , dtype='S')
            except ValueError:
                if hasmode:
                    raise Exception( "ValueError when converting data to array." + \
                            "  You have scipy.mode on your system, so this is " + \
                            "probably not an issue of differing row lengths." )
                else:
                    raise Exception( "Conversion to array error.  You probably " + \
                            "have different row lengths and scipy.mode was not " + \
                            "imported." )

        if nullval is not None:
            x[x==nullval] = numpy.nan
            x = get_autotype(x)

        if asdict or asStruct:
            mydict = OrderedDict(zip(nms,x.T))
            for k,v in mydict.items():
                mydict[k] = get_autotype(v)
            if asdict:
                return mydict
            elif asRecArray:
                return Struct(mydict).as_recarray()
            elif asStruct:
                return Struct(mydict)
        elif names and twod:
            return nms,x
        elif names:
            # if not returning a twod array, try to return each vector as the spec. type
            return nms,[ get_autotype(x.T[i]) for i in range(x.shape[1]) ]
        else:
            if twod:
                return x
            else:
                return [ get_autotype(x.T[i]) for i in range(x.shape[1]) ]
def readcol(filename,skipline=0,skipafter=0,names=False,fsep=None,twod=True,
        fixedformat=None,asdict=False,comment='#',verbose=True,nullval=None,
        asStruct=False,namecomment=True,removeblanks=False,header_badchars=None,
        asRecArray=False):
    """
    The default return is a two dimensional float array.  If you want a list of
    columns output instead of a 2D array, pass 'twod=False'.  In this case,
    each column's data type will be automatically detected.
    
    Example usage:
    CASE 1) a table has the format:
     X    Y    Z
    0.0  2.4  8.2
    1.0  3.4  5.6
    0.7  3.2  2.1
    ...
    names,(x,y,z)=readcol("myfile.tbl",names=True,twod=False)
    or
    x,y,z=readcol("myfile.tbl",skipline=1,twod=False)
    or 
    names,xx = readcol("myfile.tbl",names=True)
    or
    xxdict = readcol("myfile.tbl",asdict=True)
    or
    xxstruct = readcol("myfile.tbl",asStruct=True)

    CASE 2) no title is contained into the table, then there is
    no need to skipline:
    x,y,z=readcol("myfile.tbl")
    
    CASE 3) there is a names column and then more descriptive text:
     X      Y     Z
    (deg) (deg) (km/s) 
    0.0    2.4   8.2
    1.0    3.4.  5.6
    ...
    then use:
    names,x,y,z=readcol("myfile.tbl",names=True,skipline=1,twod=False)
    or
    x,y,z=readcol("myfile.tbl",skipline=2,twod=False)

    INPUTS:
        fsep - field separator, e.g. for comma separated value (csv) files
        skipline - number of lines to ignore at the start of the file
        names - read / don't read in the first line as a list of column names
                can specify an integer line number too, though it will be 
                the line number after skipping lines
        twod - two dimensional or one dimensional output
        nullval - if specified, all instances of this value will be replaced
           with a floating NaN
        asdict - zips names with data to create a dict with column headings 
            tied to column data.  If asdict=True, names will be set to True
        asStruct - same as asdict, but returns a structure instead of a dictionary
            (i.e. you call struct.key instead of struct['key'])
        fixedformat - if you have a fixed format file, this is a python list of 
            column lengths.  e.g. the first table above would be [3,5,5].  Note
            that if you specify the wrong fixed format, you will get junk; if your
            format total is greater than the line length, the last entries will all
            be blank but readcol will not report an error.
        namecomment - assumed that "Name" row is on a comment line.  If it is not - 
            e.g., it is the first non-comment line, change this to False
        removeblanks - remove all blank entries from split lines.  This can cause lost
            data if you have blank entries on some lines.
        header_badchars - remove these characters from a header before parsing it
            (helpful for IPAC tables that are delimited with | )

    If you get this error: "scipy could not be imported.  Your table must have
    full rows." it means readcol cannot automatically guess which columns
    contain data.  If you have scipy and columns of varying length, readcol will
    read in all of the rows with length=mode(row lengths).
    """
    with open(filename,'r') as f:
        f = f.readlines()
    
        null=[f.pop(0) for i in range(skipline)]

        commentfilter = make_commentfilter(comment)

        if not asStruct:
            asStruct = asRecArray

        if namecomment is False and (names or asdict or asStruct):
            while 1:
                line = f.pop(0)
                if line[0] != comment:
                    nameline = line
                    if header_badchars:
                        for c in header_badchars:
                            nameline = nameline.replace(c,' ')
                    nms=nameline.split(fsep)
                    break
                elif len(f) == 0:
                    raise Exception("No uncommented lines found.")
        else:
            if names or asdict or asStruct:
                # can specify name line
                if type(names) == type(1):
                    nameline = f.pop(names)
                else:
                    nameline = f.pop(0)
                if nameline[0]==comment:
                    nameline = nameline[1:]
                if header_badchars:
                    for c in header_badchars:
                        nameline = nameline.replace(c,' ')
                nms=list([name.strip() for name in nameline.split(fsep)])

        null=[f.pop(0) for i in range(skipafter)]

        if fixedformat:
            myreadff = lambda x: readff(x,fixedformat)
            splitarr = list(map(myreadff,f))
            splitarr = list(filter(commentfilter,splitarr))
        else:
            fstrip = list(map(str.strip,f))
            fseps = [ fsep for i in range(len(f)) ]
            splitarr = list(map(str.split,fstrip,fseps))
            if removeblanks:
                for i in range(splitarr.count([''])):
                    splitarr.remove([''])

            splitarr = list(filter(commentfilter,splitarr))

            # check to make sure each line has the same number of columns to avoid
            # "ValueError: setting an array element with a sequence."
            nperline = list(map(len,splitarr))
            if hasmode:
                ncols,nrows = mode(nperline)
                if nrows != len(splitarr):
                    if verbose:
                        print("Removing %i rows that don't match most common length %i.  \
                         \n%i rows read into array." % (len(splitarr) - nrows,ncols,nrows))
                    for i in range(len(splitarr)-1,-1,-1):  # need to go backwards
                        if nperline[i] != ncols:
                            splitarr.pop(i)

        try:
            x = numpy.asarray( splitarr , dtype='float')
        except ValueError:
            if verbose:
                print("WARNING: reading as string array because %s array failed" % 'float')
            try:
                x = numpy.asarray( splitarr , dtype='S')
            except ValueError:
                if hasmode:
                    raise Exception( "ValueError when converting data to array." + \
                            "  You have scipy.mode on your system, so this is " + \
                            "probably not an issue of differing row lengths." )
                else:
                    raise Exception( "Conversion to array error.  You probably " + \
                            "have different row lengths and scipy.mode was not " + \
                            "imported." )

        if nullval is not None:
            x[x==nullval] = numpy.nan
            x = get_autotype(x)

        if asdict or asStruct:
            mydict = OrderedDict(zip(nms,x.T))
            for k,v in mydict.items():
                mydict[k] = get_autotype(v)
            if asdict:
                return mydict
            elif asRecArray:
                return Struct(mydict).as_recarray()
            elif asStruct:
                return Struct(mydict)
        elif names and twod:
            return nms,x
        elif names:
            # if not returning a twod array, try to return each vector as the spec. type
            return nms,[ get_autotype(x.T[i]) for i in range(x.shape[1]) ]
        else:
            if twod:
                return x
            else:
                return [ get_autotype(x.T[i]) for i in range(x.shape[1]) ]
The default return is a two dimensional float array.  If you want a list of
    columns output instead of a 2D array, pass 'twod=False'.  In this case,
    each column's data type will be automatically detected.
    
    Example usage:
    CASE 1) a table has the format:
     X    Y    Z
    0.0  2.4  8.2
    1.0  3.4  5.6
    0.7  3.2  2.1
    ...
    names,(x,y,z)=readcol("myfile.tbl",names=True,twod=False)
    or
    x,y,z=readcol("myfile.tbl",skipline=1,twod=False)
    or 
    names,xx = readcol("myfile.tbl",names=True)
    or
    xxdict = readcol("myfile.tbl",asdict=True)
    or
    xxstruct = readcol("myfile.tbl",asStruct=True)

    CASE 2) no title is contained into the table, then there is
    no need to skipline:
    x,y,z=readcol("myfile.tbl")
    
    CASE 3) there is a names column and then more descriptive text:
     X      Y     Z
    (deg) (deg) (km/s) 
    0.0    2.4   8.2
    1.0    3.4.  5.6
    ...
    then use:
    names,x,y,z=readcol("myfile.tbl",names=True,skipline=1,twod=False)
    or
    x,y,z=readcol("myfile.tbl",skipline=2,twod=False)

    INPUTS:
        fsep - field separator, e.g. for comma separated value (csv) files
        skipline - number of lines to ignore at the start of the file
        names - read / don't read in the first line as a list of column names
                can specify an integer line number too, though it will be 
                the line number after skipping lines
        twod - two dimensional or one dimensional output
        nullval - if specified, all instances of this value will be replaced
           with a floating NaN
        asdict - zips names with data to create a dict with column headings 
            tied to column data.  If asdict=True, names will be set to True
        asStruct - same as asdict, but returns a structure instead of a dictionary
            (i.e. you call struct.key instead of struct['key'])
        fixedformat - if you have a fixed format file, this is a python list of 
            column lengths.  e.g. the first table above would be [3,5,5].  Note
            that if you specify the wrong fixed format, you will get junk; if your
            format total is greater than the line length, the last entries will all
            be blank but readcol will not report an error.
        namecomment - assumed that "Name" row is on a comment line.  If it is not - 
            e.g., it is the first non-comment line, change this to False
        removeblanks - remove all blank entries from split lines.  This can cause lost
            data if you have blank entries on some lines.
        header_badchars - remove these characters from a header before parsing it
            (helpful for IPAC tables that are delimited with | )

    If you get this error: "scipy could not be imported.  Your table must have
    full rows." it means readcol cannot automatically guess which columns
    contain data.  If you have scipy and columns of varying length, readcol will
    read in all of the rows with length=mode(row lengths).