streamcorpus_pipeline/_run_lingpipe.py
def cleanse(span):
    '''Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string
    '''
    try:
        ## attempt to force it to utf8, which might fail
        span = span.encode('utf8', 'ignore')
    except:
        pass
    ## lowercase, strip punctuation, and shrink all whitespace
    span = span.lower()
    span = span.translate(strip_punctuation)
    span = whitespace.sub(' ', span)
    ## trim any leading or trailing whitespace
    return span.strip()
def cleanse(span):
    '''Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string
    '''
    try:
        ## attempt to force it to utf8, which might fail
        span = span.encode('utf8', 'ignore')
    except:
        pass
    ## lowercase, strip punctuation, and shrink all whitespace
    span = span.lower()
    span = span.translate(strip_punctuation)
    span = whitespace.sub(' ', span)
    ## trim any leading or trailing whitespace
    return span.strip()
Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string