luminoso_api/v4_json_stream.py
def _read_csv(reader, header, encode_fn):
    """
    Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
    """
    for row in reader:
        if len(row) == 0:
            continue
        row = [encode_fn(cell) for cell in row]
        row_list = zip(header, row)
        row_dict = dict(row_list)
        if len(row_dict['text']) == 0:
            continue
        row_dict['text'] = unicodedata.normalize(
            'NFKC', row_dict['text'].strip()
        )
        if row_dict.get('title') == '':
            del row_dict['title']
        if 'date' in row_dict:
            # We handle dates further in open_json_or_csv_somehow
            if row_dict['date'] == '':
                del row_dict['date']
        if 'subset' in row_dict:
            subsets = [cell[1] for cell in row_list
                       if cell[1] != '' and cell[0] == 'subset']
            if subsets:
                row_dict['subsets'] = subsets
            if 'subset' in row_dict:
                del row_dict['subset']
        yield row_dict
def _read_csv(reader, header, encode_fn):
    """
    Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
    """
    for row in reader:
        if len(row) == 0:
            continue
        row = [encode_fn(cell) for cell in row]
        row_list = zip(header, row)
        row_dict = dict(row_list)
        if len(row_dict['text']) == 0:
            continue
        row_dict['text'] = unicodedata.normalize(
            'NFKC', row_dict['text'].strip()
        )
        if row_dict.get('title') == '':
            del row_dict['title']
        if 'date' in row_dict:
            # We handle dates further in open_json_or_csv_somehow
            if row_dict['date'] == '':
                del row_dict['date']
        if 'subset' in row_dict:
            subsets = [cell[1] for cell in row_list
                       if cell[1] != '' and cell[0] == 'subset']
            if subsets:
                row_dict['subsets'] = subsets
            if 'subset' in row_dict:
                del row_dict['subset']
        yield row_dict
Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.