streamcorpus_pipeline/_clean_visible.py
def non_tag_chars_from_raw(html):
    '''generator that yields clean visible as it transitions through
    states in the raw `html`

    '''
    n = 0
    while n < len(html):
        # find start of tag
        angle = html.find('<', n)
        if angle == -1:
            yield html[n:]
            n = len(html)
            break
        yield html[n:angle]
        n = angle

        # find the end of the tag string
        space = html.find(' ',  n, n + longest_extended_tag + 2)
        angle = html.find('>',  n, n + longest_extended_tag + 2)
        nl    = html.find('\n', n, n + longest_extended_tag + 2)
        tab   = html.find('\t', n, n + longest_extended_tag + 2)
        ends = filter(lambda end: end > -1, [tab, nl, space, angle])
        if ends:
            tag = html[n + 1 : min(ends)]
            if tag == '!--':
                # whiteout comment except newlines
                end = html.find('-->', n)
                while n < end:
                    nl = html.find('\n', n, end)
                    if nl != -1:
                        yield ' ' * (nl - n) + '\n'
                        n = nl + 1
                    else:
                        yield ' ' * (end - n + 3)
                        break
                n = end + 3
                continue
            is_extended = tag.lower() in extended_tags
        else:
            is_extended = False

        # find end of tag even if on a lower line
        while n < len(html):
            squote = html.find("'", n)
            dquote = html.find('"', n)
            nl = html.find('\n', n)
            angle = html.find('>', n)
            if angle == -1:
                # hits end of doc before end of tag
                yield ' ' * (len(html) - n)
                n = len(html)
                break
            elif -1 < squote < angle or -1 < dquote < angle:
                if squote != -1 and dquote != -1:
                    if squote < dquote: 
                        open_quote = squote
                        quote = "'"
                    else:
                        open_quote = dquote
                        quote = '"'
                elif dquote != -1:
                    open_quote = dquote
                    quote = '"'
                else:
                    open_quote = squote
                    quote = "'"
                close_quote = html.find(quote, open_quote + 1)
                while n < close_quote:
                    nl = html.find('\n', n, close_quote)
                    if nl == -1: break
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                yield ' ' * (close_quote + 1 - n)
                n = close_quote + 1
                continue

            elif nl == -1 or angle < nl:
                # found close before either newline or end of doc
                yield ' ' * (angle + 1 - n)
                n = angle + 1
                if is_extended and html[angle - 1] != '/':
                    # find matching closing tag. JavaScript can
                    # include HTML *strings* within it, and in
                    # principle, that HTML could contain a closing
                    # script tag in it; ignoring for now.
                    while n < len(html):
                        nl = html.find('\n', n)
                        close = html.find('</', n)
                        close2 = html.find('</', close + 2)
                        angle = html.find('>', close + 2)
                        if nl != -1 and nl < close:
                            yield ' ' * (nl - n) + '\n'
                            n = nl + 1
                        elif close == -1 or angle == -1:
                            # end of doc before matching close tag
                            yield ' ' * (len(html) - n)
                            n = len(html)
                            break
                        elif close2 != -1 and close2 < angle:
                            # broken tag inside current tag
                            yield ' ' * (close + 2 - n)
                            n = close + 2
                        elif html[close + 2:angle].lower() == tag.lower():
                            yield ' ' * (angle + 1 - n)
                            n = angle + 1
                            break
                        else:
                            yield ' ' * (angle + 1 - n)
                            n = angle + 1
                            # do not break
                # finished with tag
                break
            else:
                # found a newline within the current tag
                yield ' ' * (nl - n) + '\n'
                n = nl + 1
def non_tag_chars_from_raw(html):
    '''generator that yields clean visible as it transitions through
    states in the raw `html`

    '''
    n = 0
    while n < len(html):
        # find start of tag
        angle = html.find('<', n)
        if angle == -1:
            yield html[n:]
            n = len(html)
            break
        yield html[n:angle]
        n = angle

        # find the end of the tag string
        space = html.find(' ',  n, n + longest_extended_tag + 2)
        angle = html.find('>',  n, n + longest_extended_tag + 2)
        nl    = html.find('\n', n, n + longest_extended_tag + 2)
        tab   = html.find('\t', n, n + longest_extended_tag + 2)
        ends = filter(lambda end: end > -1, [tab, nl, space, angle])
        if ends:
            tag = html[n + 1 : min(ends)]
            if tag == '!--':
                # whiteout comment except newlines
                end = html.find('-->', n)
                while n < end:
                    nl = html.find('\n', n, end)
                    if nl != -1:
                        yield ' ' * (nl - n) + '\n'
                        n = nl + 1
                    else:
                        yield ' ' * (end - n + 3)
                        break
                n = end + 3
                continue
            is_extended = tag.lower() in extended_tags
        else:
            is_extended = False

        # find end of tag even if on a lower line
        while n < len(html):
            squote = html.find("'", n)
            dquote = html.find('"', n)
            nl = html.find('\n', n)
            angle = html.find('>', n)
            if angle == -1:
                # hits end of doc before end of tag
                yield ' ' * (len(html) - n)
                n = len(html)
                break
            elif -1 < squote < angle or -1 < dquote < angle:
                if squote != -1 and dquote != -1:
                    if squote < dquote: 
                        open_quote = squote
                        quote = "'"
                    else:
                        open_quote = dquote
                        quote = '"'
                elif dquote != -1:
                    open_quote = dquote
                    quote = '"'
                else:
                    open_quote = squote
                    quote = "'"
                close_quote = html.find(quote, open_quote + 1)
                while n < close_quote:
                    nl = html.find('\n', n, close_quote)
                    if nl == -1: break
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                yield ' ' * (close_quote + 1 - n)
                n = close_quote + 1
                continue

            elif nl == -1 or angle < nl:
                # found close before either newline or end of doc
                yield ' ' * (angle + 1 - n)
                n = angle + 1
                if is_extended and html[angle - 1] != '/':
                    # find matching closing tag. JavaScript can
                    # include HTML *strings* within it, and in
                    # principle, that HTML could contain a closing
                    # script tag in it; ignoring for now.
                    while n < len(html):
                        nl = html.find('\n', n)
                        close = html.find('</', n)
                        close2 = html.find('</', close + 2)
                        angle = html.find('>', close + 2)
                        if nl != -1 and nl < close:
                            yield ' ' * (nl - n) + '\n'
                            n = nl + 1
                        elif close == -1 or angle == -1:
                            # end of doc before matching close tag
                            yield ' ' * (len(html) - n)
                            n = len(html)
                            break
                        elif close2 != -1 and close2 < angle:
                            # broken tag inside current tag
                            yield ' ' * (close + 2 - n)
                            n = close + 2
                        elif html[close + 2:angle].lower() == tag.lower():
                            yield ' ' * (angle + 1 - n)
                            n = angle + 1
                            break
                        else:
                            yield ' ' * (angle + 1 - n)
                            n = angle + 1
                            # do not break
                # finished with tag
                break
            else:
                # found a newline within the current tag
                yield ' ' * (nl - n) + '\n'
                n = nl + 1
generator that yields clean visible as it transitions through
    states in the raw `html`