luminoso_api/v5_download.py
def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get('docs', offset=offset, limit=DOCS_PER_BATCH)
            docs = response['result']
            for doc in docs:
                # Get the appropriate set of fields for each document
                if expanded:
                    for field in UNNECESSARY_FIELDS:
                        doc.pop(field, None)
                else:
                    doc = {field: doc[field] for field in CONCISE_FIELDS}

                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()
def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get('docs', offset=offset, limit=DOCS_PER_BATCH)
            docs = response['result']
            for doc in docs:
                # Get the appropriate set of fields for each document
                if expanded:
                    for field in UNNECESSARY_FIELDS:
                        doc.pop(field, None)
                else:
                    doc = {field: doc[field] for field in CONCISE_FIELDS}

                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()
Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.