snowplow_analytics_sdk/run_manifests.py
def list_runids(s3_client, full_path):
    """Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive
    """
    listing_finished = False                 # last response was not truncated
    run_ids_buffer = []
    last_continuation_token = None

    (bucket, prefix) = split_full_path(full_path)

    while not listing_finished:
        options = clean_dict({
            'Bucket': bucket,
            'Prefix': prefix,
            'Delimiter': '/',
            'ContinuationToken': last_continuation_token
        })

        response = s3_client.list_objects_v2(**options)
        keys = [extract_run_id(key['Prefix']) for key
                in response.get('CommonPrefixes', [])]
        run_ids_buffer.extend([key for key in keys if key is not None])
        last_continuation_token = response.get('NextContinuationToken', None)

        if not response['IsTruncated']:
            listing_finished = True

    non_archived_run_ids = [run_id for run_id in run_ids_buffer
                            if not is_glacier(s3_client, bucket, run_id)]
    return non_archived_run_ids
def list_runids(s3_client, full_path):
    """Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive
    """
    listing_finished = False                 # last response was not truncated
    run_ids_buffer = []
    last_continuation_token = None

    (bucket, prefix) = split_full_path(full_path)

    while not listing_finished:
        options = clean_dict({
            'Bucket': bucket,
            'Prefix': prefix,
            'Delimiter': '/',
            'ContinuationToken': last_continuation_token
        })

        response = s3_client.list_objects_v2(**options)
        keys = [extract_run_id(key['Prefix']) for key
                in response.get('CommonPrefixes', [])]
        run_ids_buffer.extend([key for key in keys if key is not None])
        last_continuation_token = response.get('NextContinuationToken', None)

        if not response['IsTruncated']:
            listing_finished = True

    non_archived_run_ids = [run_id for run_id in run_ids_buffer
                            if not is_glacier(s3_client, bucket, run_id)]
    return non_archived_run_ids
Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive