examples/verify_kba2014.py
def attempt_fetch(work_unit, fpath):
     '''attempt a fetch and iteration over a work_unit.key path in s3
     '''
     url = 'http://s3.amazonaws.com/aws-publicdatasets/' + work_unit.key.strip()

     ## cheapest way to iterate over the corpus is a few stages of
     ## streamed child processes.  Note that stderr needs to go
     ## separately to a file so that reading the stdin doesn't get
     ## blocked:
     cmd = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % (url, fpath)
     print cmd
     child = Popen(cmd, stdout=PIPE, shell=True)
     print 'child launched'
     sys.stdout.flush()

     si_count = 0
     serif_count = 0
     exc = ''
     stream_ids = list()
     clean_visible_bytes = 0
     clean_visible_count = 0
     try:
         for si in Chunk(file_obj=child.stdout):
             print si.stream_id, si.abs_url
             if si.body.language:
                 lang = si.body.language.code
             else:
                 lang = ''
             stream_ids.append((lang, si.stream_id))
             if si.body.clean_visible:
                 clean_visible_count += 1
                 clean_visible_bytes += len(si.body.clean_visible)
             si_count += 1
             if 'serif' in si.body.sentences:
                 serif_count += 1
     except Exception, exc:
         exc = re.sub('\s+', ' ', str(exc)).strip()

     child.terminate()
     child.wait()
     child.stdout.close()
     return exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids
def attempt_fetch(work_unit, fpath):
     '''attempt a fetch and iteration over a work_unit.key path in s3
     '''
     url = 'http://s3.amazonaws.com/aws-publicdatasets/' + work_unit.key.strip()

     ## cheapest way to iterate over the corpus is a few stages of
     ## streamed child processes.  Note that stderr needs to go
     ## separately to a file so that reading the stdin doesn't get
     ## blocked:
     cmd = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % (url, fpath)
     print cmd
     child = Popen(cmd, stdout=PIPE, shell=True)
     print 'child launched'
     sys.stdout.flush()

     si_count = 0
     serif_count = 0
     exc = ''
     stream_ids = list()
     clean_visible_bytes = 0
     clean_visible_count = 0
     try:
         for si in Chunk(file_obj=child.stdout):
             print si.stream_id, si.abs_url
             if si.body.language:
                 lang = si.body.language.code
             else:
                 lang = ''
             stream_ids.append((lang, si.stream_id))
             if si.body.clean_visible:
                 clean_visible_count += 1
                 clean_visible_bytes += len(si.body.clean_visible)
             si_count += 1
             if 'serif' in si.body.sentences:
                 serif_count += 1
     except Exception, exc:
         exc = re.sub('\s+', ' ', str(exc)).strip()

     child.terminate()
     child.wait()
     child.stdout.close()
     return exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids
attempt a fetch and iteration over a work_unit.key path in s3