streamcorpus_pipeline/_guess_media_type.py
def file_type_stats(config):
    '''
    returns a kba.pipeline "transform" function that generates file
    type stats from the stream_items that it sees.  Currently, these
    stats are just the first five non-whitespace characters.
    '''
    ## make a closure around config
    def _file_type_stats(stream_item, context):
        if stream_item.body and stream_item.body.raw:
            #print repr(stream_item.body.raw[:250])
            #sys.stdout.flush()
            #doctype_m = doctype_re.match(stream_item.body.raw[:250])
            #if doctype_m:
                #print 'DOCTYPE: %s' % repr(doctype_m.group('doctype').lower())
            if 'doctype html' in stream_item.body.raw[:250].lower():
                print 'DOCTYPE: html'
            else:
                #if probably_html.search(stream_item.body.raw):
                if has_tags(stream_item.body.raw[:400]):
                    print 'PROBABLY_HTML'
                else:
                    xml = xml_ish.search(stream_item.body.raw)
                    if xml:
                        print 'XML: %s' % repr(xml.group('intro'))
                    else:
                        pdf = pdf_start.search(stream_item.body.raw)
                        if pdf:
                            print 'PDF %s' % repr(pdf.group('version'))
                        else:
                            ext = stream_item.abs_url.split('.')[-1]
                            if len(ext) < 6:
                                print 'UNK ext: %s' % repr(ext)
                            else:
                                first = first_letters.match(stream_item.body.raw)
                                if first and False:
                                    print 'UNK letters: %s' % repr(first.group('first_letters'))
                                else:
                                    print 'UNK first bytes: %s' % repr(stream_item.body.raw[:50])
                    #m = first_three_letters.search(stream_item.body.raw)
                    #if m:
                    #    print repr(m.group('first_three_letters')).lower().strip()
                    #else:
                    #    print repr(stream_item.body.raw[:50]).lower().strip()
        return stream_item

    return _file_type_stats
def file_type_stats(config):
    '''
    returns a kba.pipeline "transform" function that generates file
    type stats from the stream_items that it sees.  Currently, these
    stats are just the first five non-whitespace characters.
    '''
    ## make a closure around config
    def _file_type_stats(stream_item, context):
        if stream_item.body and stream_item.body.raw:
            #print repr(stream_item.body.raw[:250])
            #sys.stdout.flush()
            #doctype_m = doctype_re.match(stream_item.body.raw[:250])
            #if doctype_m:
                #print 'DOCTYPE: %s' % repr(doctype_m.group('doctype').lower())
            if 'doctype html' in stream_item.body.raw[:250].lower():
                print 'DOCTYPE: html'
            else:
                #if probably_html.search(stream_item.body.raw):
                if has_tags(stream_item.body.raw[:400]):
                    print 'PROBABLY_HTML'
                else:
                    xml = xml_ish.search(stream_item.body.raw)
                    if xml:
                        print 'XML: %s' % repr(xml.group('intro'))
                    else:
                        pdf = pdf_start.search(stream_item.body.raw)
                        if pdf:
                            print 'PDF %s' % repr(pdf.group('version'))
                        else:
                            ext = stream_item.abs_url.split('.')[-1]
                            if len(ext) < 6:
                                print 'UNK ext: %s' % repr(ext)
                            else:
                                first = first_letters.match(stream_item.body.raw)
                                if first and False:
                                    print 'UNK letters: %s' % repr(first.group('first_letters'))
                                else:
                                    print 'UNK first bytes: %s' % repr(stream_item.body.raw[:50])
                    #m = first_three_letters.search(stream_item.body.raw)
                    #if m:
                    #    print repr(m.group('first_three_letters')).lower().strip()
                    #else:
                    #    print repr(stream_item.body.raw[:50]).lower().strip()
        return stream_item

    return _file_type_stats
returns a kba.pipeline "transform" function that generates file
    type stats from the stream_items that it sees.  Currently, these
    stats are just the first five non-whitespace characters.