src/toil_lib/urls.py
def _s3am_with_retry(job, num_cores, file_path, s3_url, mode='upload', s3_key_path=None):
    """
    Run s3am with 3 retries

    :param toil.job.Job job: Toil job that is calling this function
    :param int num_cores: Number of cores to pass to upload/download slots
    :param str file_path: Full path to the file
    :param str s3_url: S3 URL
    :param str mode: Mode to run s3am in. Either "upload" or "download"
    :param str s3_key_path: Path to the SSE-C key if using encryption
    """
    container_key_file = None
    # try to find suitable credentials
    base_boto = '.boto'
    base_aws = '.aws/credentials'
    docker_home_dir = '/root'
    # map existing credential paths to their mount point within the container
    credentials_to_mount = {os.path.join(os.path.expanduser("~"), path): os.path.join(docker_home_dir, path)
                            for path in [base_aws, base_boto]
                            if os.path.exists(os.path.join(os.path.expanduser("~"), path))}
    require(os.path.isabs(file_path), "'file_path' parameter must be an absolute path")
    dir_path, file_name = file_path.rsplit('/', 1)
    # Mirror user specified paths to simplify debugging
    container_dir_path = '/data' + dir_path
    container_file = os.path.join(container_dir_path, file_name)
    mounts = {dir_path: container_dir_path}
    if s3_key_path:
        require(os.path.isabs(s3_key_path), "'s3_key_path' parameter must be an absolute path")
        key_dir_path, key_name = s3_key_path.rsplit('/', 1)
        container_key_dir_path = '/data' + key_dir_path
        container_key_file = os.path.join(container_key_dir_path, key_name)
        # if the key directory is identical to the file directory this assignment is idempotent
        mounts[key_dir_path] = container_key_dir_path
    for k, v in credentials_to_mount.iteritems():
        mounts[k] = v
    arguments = []
    url_arguments = []
    if mode == 'upload':
        arguments.extend(['upload', '--force', '--upload-slots=%s' % num_cores, '--exists=overwrite'])
        url_arguments.extend(['file://' + container_file, s3_url])
    elif mode == 'download':
        arguments.extend(['download', '--file-exists=overwrite', '--download-exists=discard'])
        url_arguments.extend([s3_url, 'file://' + container_file])
    else:
        raise ValueError('Improper mode specified. mode must be equal to "upload" or "download".')
    if s3_key_path:
        arguments.extend(['--sse-key-is-master', '--sse-key-file', container_key_file])
    arguments.extend(['--part-size=50M', '--download-slots=%s' % num_cores])
    # finally, add the url path arguments after all the tool parameters are set
    arguments.extend(url_arguments)
    # Pass credential-related environment variables into container
    env = {}
    if 'AWS_PROFILE' in os.environ:
        env['AWS_PROFILE'] = os.environ['AWS_PROFILE']
    # Create parameters to pass to Docker
    docker_parameters = ['--rm', '--log-driver', 'none']
    if mounts:
        for k, v in mounts.iteritems():
            docker_parameters.extend(['-v', k + ':' + v])
    if env:
        for e, v in env.iteritems():
            docker_parameters.extend(['-e', '{}={}'.format(e, v)])
    # Run s3am with retries
    retry_count = 3
    for i in xrange(retry_count):
        try:
            dockerCall(job=job, tool='quay.io/ucsc_cgl/s3am:2.0--fed932897e7fd40f4ec878362e5dd6afe15caaf0',
                       parameters=arguments, dockerParameters=docker_parameters)
        except subprocess.CalledProcessError:
            _log.debug('S3AM %s failed', mode, exc_info=True)
        else:
            _log.debug('S3AM %s succeeded', mode)
            return
    raise RuntimeError("S3AM failed to %s after %i retries with arguments %s. Enable 'debug' "
                       "level logging to see more information about the failed attempts." %
                       (mode, retry_count, arguments))
def _s3am_with_retry(job, num_cores, file_path, s3_url, mode='upload', s3_key_path=None):
    """
    Run s3am with 3 retries

    :param toil.job.Job job: Toil job that is calling this function
    :param int num_cores: Number of cores to pass to upload/download slots
    :param str file_path: Full path to the file
    :param str s3_url: S3 URL
    :param str mode: Mode to run s3am in. Either "upload" or "download"
    :param str s3_key_path: Path to the SSE-C key if using encryption
    """
    container_key_file = None
    # try to find suitable credentials
    base_boto = '.boto'
    base_aws = '.aws/credentials'
    docker_home_dir = '/root'
    # map existing credential paths to their mount point within the container
    credentials_to_mount = {os.path.join(os.path.expanduser("~"), path): os.path.join(docker_home_dir, path)
                            for path in [base_aws, base_boto]
                            if os.path.exists(os.path.join(os.path.expanduser("~"), path))}
    require(os.path.isabs(file_path), "'file_path' parameter must be an absolute path")
    dir_path, file_name = file_path.rsplit('/', 1)
    # Mirror user specified paths to simplify debugging
    container_dir_path = '/data' + dir_path
    container_file = os.path.join(container_dir_path, file_name)
    mounts = {dir_path: container_dir_path}
    if s3_key_path:
        require(os.path.isabs(s3_key_path), "'s3_key_path' parameter must be an absolute path")
        key_dir_path, key_name = s3_key_path.rsplit('/', 1)
        container_key_dir_path = '/data' + key_dir_path
        container_key_file = os.path.join(container_key_dir_path, key_name)
        # if the key directory is identical to the file directory this assignment is idempotent
        mounts[key_dir_path] = container_key_dir_path
    for k, v in credentials_to_mount.iteritems():
        mounts[k] = v
    arguments = []
    url_arguments = []
    if mode == 'upload':
        arguments.extend(['upload', '--force', '--upload-slots=%s' % num_cores, '--exists=overwrite'])
        url_arguments.extend(['file://' + container_file, s3_url])
    elif mode == 'download':
        arguments.extend(['download', '--file-exists=overwrite', '--download-exists=discard'])
        url_arguments.extend([s3_url, 'file://' + container_file])
    else:
        raise ValueError('Improper mode specified. mode must be equal to "upload" or "download".')
    if s3_key_path:
        arguments.extend(['--sse-key-is-master', '--sse-key-file', container_key_file])
    arguments.extend(['--part-size=50M', '--download-slots=%s' % num_cores])
    # finally, add the url path arguments after all the tool parameters are set
    arguments.extend(url_arguments)
    # Pass credential-related environment variables into container
    env = {}
    if 'AWS_PROFILE' in os.environ:
        env['AWS_PROFILE'] = os.environ['AWS_PROFILE']
    # Create parameters to pass to Docker
    docker_parameters = ['--rm', '--log-driver', 'none']
    if mounts:
        for k, v in mounts.iteritems():
            docker_parameters.extend(['-v', k + ':' + v])
    if env:
        for e, v in env.iteritems():
            docker_parameters.extend(['-e', '{}={}'.format(e, v)])
    # Run s3am with retries
    retry_count = 3
    for i in xrange(retry_count):
        try:
            dockerCall(job=job, tool='quay.io/ucsc_cgl/s3am:2.0--fed932897e7fd40f4ec878362e5dd6afe15caaf0',
                       parameters=arguments, dockerParameters=docker_parameters)
        except subprocess.CalledProcessError:
            _log.debug('S3AM %s failed', mode, exc_info=True)
        else:
            _log.debug('S3AM %s succeeded', mode)
            return
    raise RuntimeError("S3AM failed to %s after %i retries with arguments %s. Enable 'debug' "
                       "level logging to see more information about the failed attempts." %
                       (mode, retry_count, arguments))
Run s3am with 3 retries

    :param toil.job.Job job: Toil job that is calling this function
    :param int num_cores: Number of cores to pass to upload/download slots
    :param str file_path: Full path to the file
    :param str s3_url: S3 URL
    :param str mode: Mode to run s3am in. Either "upload" or "download"
    :param str s3_key_path: Path to the SSE-C key if using encryption