streamcorpus_pipeline/_s3_storage.py
def get_bucket(config, bucket_name=None):
    '''This function is mostly about managing configuration, and then
    finally returns a boto.Bucket object.

    AWS credentials come first from config keys
    aws_access_key_id_path, aws_secret_access_key_path (paths to one
    line files); secondly from environment variables
    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY; also from $HOME/.aws/credentials
    or the magic Amazon http://169.254.169.254/ service.  If credentials
    are not set in the config then behavior is the same as other AWS-based
    command-line tools.

    '''
    if not bucket_name:
        if 'bucket' not in config:
            raise ConfigurationError(
                'The "bucket" parameter is required for the s3 stages.')
        bucket_name = config['bucket']

    # get AWS credentials. first, from config; else, from env vars.
    # (boto will read environment variables and other normal places.)
    aws_access_key_id_path = config.get('aws_access_key_id_path')
    aws_secret_access_key_path = config.get('aws_secret_access_key_path')

    params = ()
    if aws_access_key_id_path and aws_secret_access_key_path:
        try:
            access = open(aws_access_key_id_path).read().strip()
            secret = open(aws_secret_access_key_path).read().strip()
            params = (access, secret)
        except:
            logger.error('failed reading aws credentials from configured file', exc_info=True)
            raise

    conn = S3Connection(*params)
    bucket = conn.get_bucket(bucket_name)
    return bucket
def get_bucket(config, bucket_name=None):
    '''This function is mostly about managing configuration, and then
    finally returns a boto.Bucket object.

    AWS credentials come first from config keys
    aws_access_key_id_path, aws_secret_access_key_path (paths to one
    line files); secondly from environment variables
    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY; also from $HOME/.aws/credentials
    or the magic Amazon http://169.254.169.254/ service.  If credentials
    are not set in the config then behavior is the same as other AWS-based
    command-line tools.

    '''
    if not bucket_name:
        if 'bucket' not in config:
            raise ConfigurationError(
                'The "bucket" parameter is required for the s3 stages.')
        bucket_name = config['bucket']

    # get AWS credentials. first, from config; else, from env vars.
    # (boto will read environment variables and other normal places.)
    aws_access_key_id_path = config.get('aws_access_key_id_path')
    aws_secret_access_key_path = config.get('aws_secret_access_key_path')

    params = ()
    if aws_access_key_id_path and aws_secret_access_key_path:
        try:
            access = open(aws_access_key_id_path).read().strip()
            secret = open(aws_secret_access_key_path).read().strip()
            params = (access, secret)
        except:
            logger.error('failed reading aws credentials from configured file', exc_info=True)
            raise

    conn = S3Connection(*params)
    bucket = conn.get_bucket(bucket_name)
    return bucket
This function is mostly about managing configuration, and then
    finally returns a boto.Bucket object.

    AWS credentials come first from config keys
    aws_access_key_id_path, aws_secret_access_key_path (paths to one
    line files); secondly from environment variables
    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY; also from $HOME/.aws/credentials
    or the magic Amazon http://169.254.169.254/ service.  If credentials
    are not set in the config then behavior is the same as other AWS-based
    command-line tools.