streamcorpus_pipeline/_s3_storage.py
def get_chunk(self, bucket_name, key_path):
        '''return Chunk object full of records
        bucket_name may be None'''
        bucket = get_bucket(self.config, bucket_name=bucket_name)
        key = bucket.get_key(key_path)
        if key is None:
            raise FailedExtraction('Key "%s" does not exist.' % key_path)

        fh = StringIO()
        key.get_contents_to_file(fh)
        data = fh.getvalue()
        if not data:
            raise FailedExtraction('%s: no data (does the key exist?)'
                                   % key.key)

        chunk_type, compression, encryption = parse_file_extensions(key_path)
        if encryption == 'gpg':
            if not self.gpg_decryption_key_path:
                raise FailedExtraction('%s ends with ".gpg" but gpg_decryption_key_path=%s'
                                       % (key.key, self.gpg_decryption_key_path))

        _errors = []
        if compression or encryption:
            _errors, data = decrypt_and_uncompress(
                data, 
                self.gpg_decryption_key_path,
                tmp_dir=self.config.get('tmp_dir_path'),
                compression=compression,
                )
            if not data:
                msg = 'decrypt_and_uncompress got no data for {0!r}, from {1} bytes' \
                    + ' downloaded, errors: {2}' \
                        .format(key_path, len(data), '\n'.join(_errors))
                logger.error(msg)
                raise FailedExtraction(msg)
            logger.info( '\n'.join(_errors) )

        if not self.config['compare_md5_in_file_name']:
            logger.warn('not checking md5 in file name, consider setting '
                        'from_s3_chunks:compare_md5_in_file_name')
        else:
            logger.info('Verifying md5 for "%s"...' % key.key)

            # The regex hammer.
            m = re.search('([a-z0-9]{32})(?:\.|$)', key.key)
            if m is None:
                raise FailedExtraction(
                    'Could not extract md5 from key "%s". '
                    'Perhaps you should disable compare_md5_in_file_name?'
                    % key.key)

            i_content_md5 = m.group(1)
            #import pdb; pdb.set_trace()
            verify_md5(i_content_md5, data, other_errors=_errors)
        return self._decode(data)
def get_chunk(self, bucket_name, key_path):
        '''return Chunk object full of records
        bucket_name may be None'''
        bucket = get_bucket(self.config, bucket_name=bucket_name)
        key = bucket.get_key(key_path)
        if key is None:
            raise FailedExtraction('Key "%s" does not exist.' % key_path)

        fh = StringIO()
        key.get_contents_to_file(fh)
        data = fh.getvalue()
        if not data:
            raise FailedExtraction('%s: no data (does the key exist?)'
                                   % key.key)

        chunk_type, compression, encryption = parse_file_extensions(key_path)
        if encryption == 'gpg':
            if not self.gpg_decryption_key_path:
                raise FailedExtraction('%s ends with ".gpg" but gpg_decryption_key_path=%s'
                                       % (key.key, self.gpg_decryption_key_path))

        _errors = []
        if compression or encryption:
            _errors, data = decrypt_and_uncompress(
                data, 
                self.gpg_decryption_key_path,
                tmp_dir=self.config.get('tmp_dir_path'),
                compression=compression,
                )
            if not data:
                msg = 'decrypt_and_uncompress got no data for {0!r}, from {1} bytes' \
                    + ' downloaded, errors: {2}' \
                        .format(key_path, len(data), '\n'.join(_errors))
                logger.error(msg)
                raise FailedExtraction(msg)
            logger.info( '\n'.join(_errors) )

        if not self.config['compare_md5_in_file_name']:
            logger.warn('not checking md5 in file name, consider setting '
                        'from_s3_chunks:compare_md5_in_file_name')
        else:
            logger.info('Verifying md5 for "%s"...' % key.key)

            # The regex hammer.
            m = re.search('([a-z0-9]{32})(?:\.|$)', key.key)
            if m is None:
                raise FailedExtraction(
                    'Could not extract md5 from key "%s". '
                    'Perhaps you should disable compare_md5_in_file_name?'
                    % key.key)

            i_content_md5 = m.group(1)
            #import pdb; pdb.set_trace()
            verify_md5(i_content_md5, data, other_errors=_errors)
        return self._decode(data)
return Chunk object full of records
        bucket_name may be None