luminoso_api/v4_json_stream.py
def open_json_or_csv_somehow(filename, date_format=None):
    """
    Deduce the format of a file, within reason.

    - If the filename ends with .csv or .txt, it's csv.
    - If the filename ends with .jsons, it's a JSON stream (conveniently the
      format we want to output).
    - If the filename ends with .json, it could be a legitimate JSON file, or
      it could be a JSON stream, following a nonstandard convention that many
      people including us are guilty of. In that case:
      - If the first line is a complete JSON document, and there is more in the
        file besides the first line, then it is a JSON stream.
      - Otherwise, it is probably really JSON.
    - If the filename does not end with .json, .jsons, or .csv, we have to guess
      whether it's still CSV or tab-separated values or something like that.
      If it's JSON, the first character would almost certainly have to be a
      bracket or a brace. If it isn't, assume it's CSV or similar.
    """
    fileformat = None
    if filename.endswith('.csv'):
        fileformat = 'csv'
    elif filename.endswith('.jsons'):
        fileformat = 'jsons'
    else:
        with open(filename) as opened:
            line = opened.readline()
            if line[0] not in '{[' and not filename.endswith('.json'):
                fileformat = 'csv'
            else:
                if (line.count('{') == line.count('}') and
                    line.count('[') == line.count(']')):
                    # This line contains a complete JSON document. This probably
                    # means it's in linewise JSON ('.jsons') format, unless the
                    # whole file is on one line.
                    char = ' '
                    while char.isspace():
                        char = opened.read()
                        if char == '':
                            fileformat = 'json'
                            break
                    if fileformat is None:
                        fileformat = 'jsons'
                else:
                    fileformat = 'json'

    if fileformat == 'json':
        stream = json.load(open(filename), encoding='utf-8')
    elif fileformat == 'csv':
        stream = open_csv_somehow(filename)
    else:
        stream = stream_json_lines(filename)

    return _normalize_data(stream, date_format=date_format)
def open_json_or_csv_somehow(filename, date_format=None):
    """
    Deduce the format of a file, within reason.

    - If the filename ends with .csv or .txt, it's csv.
    - If the filename ends with .jsons, it's a JSON stream (conveniently the
      format we want to output).
    - If the filename ends with .json, it could be a legitimate JSON file, or
      it could be a JSON stream, following a nonstandard convention that many
      people including us are guilty of. In that case:
      - If the first line is a complete JSON document, and there is more in the
        file besides the first line, then it is a JSON stream.
      - Otherwise, it is probably really JSON.
    - If the filename does not end with .json, .jsons, or .csv, we have to guess
      whether it's still CSV or tab-separated values or something like that.
      If it's JSON, the first character would almost certainly have to be a
      bracket or a brace. If it isn't, assume it's CSV or similar.
    """
    fileformat = None
    if filename.endswith('.csv'):
        fileformat = 'csv'
    elif filename.endswith('.jsons'):
        fileformat = 'jsons'
    else:
        with open(filename) as opened:
            line = opened.readline()
            if line[0] not in '{[' and not filename.endswith('.json'):
                fileformat = 'csv'
            else:
                if (line.count('{') == line.count('}') and
                    line.count('[') == line.count(']')):
                    # This line contains a complete JSON document. This probably
                    # means it's in linewise JSON ('.jsons') format, unless the
                    # whole file is on one line.
                    char = ' '
                    while char.isspace():
                        char = opened.read()
                        if char == '':
                            fileformat = 'json'
                            break
                    if fileformat is None:
                        fileformat = 'jsons'
                else:
                    fileformat = 'json'

    if fileformat == 'json':
        stream = json.load(open(filename), encoding='utf-8')
    elif fileformat == 'csv':
        stream = open_csv_somehow(filename)
    else:
        stream = stream_json_lines(filename)

    return _normalize_data(stream, date_format=date_format)
Deduce the format of a file, within reason.

    - If the filename ends with .csv or .txt, it's csv.
    - If the filename ends with .jsons, it's a JSON stream (conveniently the
      format we want to output).
    - If the filename ends with .json, it could be a legitimate JSON file, or
      it could be a JSON stream, following a nonstandard convention that many
      people including us are guilty of. In that case:
      - If the first line is a complete JSON document, and there is more in the
        file besides the first line, then it is a JSON stream.
      - Otherwise, it is probably really JSON.
    - If the filename does not end with .json, .jsons, or .csv, we have to guess
      whether it's still CSV or tab-separated values or something like that.
      If it's JSON, the first character would almost certainly have to be a
      bracket or a brace. If it isn't, assume it's CSV or similar.