streamcorpus_pipeline/_clean_visible.py
def make_clean_visible(_html, tag_replacement_char=' '):
    '''
    Takes an HTML-like Unicode string as input and returns a UTF-8
    encoded string with all tags replaced by whitespace. In particular,
    all Unicode characters inside HTML are replaced with a single
    whitespace character.

    This does not detect comments, style, script, link.  It also does
    do anything with HTML-escaped characters.  All of these are
    handled by the clean_html pre-cursor step.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    This is a simple state machine iterator without regexes
    '''
    def non_tag_chars(html):
        n = 0
        while n < len(html):
            angle = html.find('<', n)
            if angle == -1:
                yield html[n:]
                n = len(html)
                break
            yield html[n:angle]
            n = angle

            while n < len(html):
                nl = html.find('\n', n)
                angle = html.find('>', n)
                if angle == -1:
                    yield ' ' * (len(html) - n)
                    n = len(html)
                    break
                elif nl == -1 or angle < nl:
                    yield ' ' * (angle + 1 - n)
                    n = angle + 1
                    break
                else:
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                    # do not break

    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    # Protect emails by substituting with unique key
    _html = fix_emails(_html)

    #Strip tags with previous logic
    non_tag = ''.join(non_tag_chars(_html))

    return non_tag.encode('utf-8')
def make_clean_visible(_html, tag_replacement_char=' '):
    '''
    Takes an HTML-like Unicode string as input and returns a UTF-8
    encoded string with all tags replaced by whitespace. In particular,
    all Unicode characters inside HTML are replaced with a single
    whitespace character.

    This does not detect comments, style, script, link.  It also does
    do anything with HTML-escaped characters.  All of these are
    handled by the clean_html pre-cursor step.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    This is a simple state machine iterator without regexes
    '''
    def non_tag_chars(html):
        n = 0
        while n < len(html):
            angle = html.find('<', n)
            if angle == -1:
                yield html[n:]
                n = len(html)
                break
            yield html[n:angle]
            n = angle

            while n < len(html):
                nl = html.find('\n', n)
                angle = html.find('>', n)
                if angle == -1:
                    yield ' ' * (len(html) - n)
                    n = len(html)
                    break
                elif nl == -1 or angle < nl:
                    yield ' ' * (angle + 1 - n)
                    n = angle + 1
                    break
                else:
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                    # do not break

    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    # Protect emails by substituting with unique key
    _html = fix_emails(_html)

    #Strip tags with previous logic
    non_tag = ''.join(non_tag_chars(_html))

    return non_tag.encode('utf-8')
Takes an HTML-like Unicode string as input and returns a UTF-8
    encoded string with all tags replaced by whitespace. In particular,
    all Unicode characters inside HTML are replaced with a single
    whitespace character.

    This does not detect comments, style, script, link.  It also does
    do anything with HTML-escaped characters.  All of these are
    handled by the clean_html pre-cursor step.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    This is a simple state machine iterator without regexes