streamcorpus_pipeline/_hyperlink_labels.py
def byte_href_anchors_state_machine(self):
        '''
        byte-based state machine extractor of anchor tags, so we can
        compute byte offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
        '''
        tag_depth = 0
        a_tag_depth = 0
        vals = []
        href = None
        idx_bytes = enumerate( self.clean_html )
        while 1:
            end_idx, val, next_b = read_to( idx_bytes, '<' )
            tag_depth += 1

            if href:
                ## must be inside an anchor tag, so accumulate the
                ## whole anchor
                assert a_tag_depth > 0, (href, self.clean_html)
                vals.append(val)

            ## figure out if start of an "A" anchor tag or close
            ## of a previous tag
            idx, next_b1 = idx_bytes.next()
            if next_b1.lower() == 'a':
                ## could be start of "A" tag
                idx, next_b2 = idx_bytes.next()
                if next_b2 == ' ':
                    a_tag_depth += 1

                    href = None
                    for idx, attr_name, attr_val in iter_attrs( idx_bytes ):
                        if attr_name.lower() == 'href':
                            href = attr_val
                        if idx is None:
                            ## doc ended mid tag, so invalid HTML--> just bail
                            return

                    first = idx + 1

                    ## if we got an href, then we want to keep the
                    ## first byte idx of the anchor:
                    if href:
                        ## Someone could nest an A tag inside another
                        ## A tag, which is invalid (even in HTML5), so
                        ## vals could be nonempty.  We only generate
                        ## the leaf-level A tags in these rare cases
                        ## of nested A tags, so reset it:
                        vals = []

            elif next_b1 == '/':
                idx, next_b1 = idx_bytes.next()
                if next_b1 == 'a':
                    ## could be end of "A" tag
                    idx, next_b2 = idx_bytes.next()
                    if next_b2 == '>':
                        a_tag_depth -= 1
                        if href:
                            ## join is much faster than using += above
                            anchor = b''.join(vals)
                            length = len(anchor)

                            ## yield the data
                            yield href, first, len(anchor), anchor

                            ## reset, no yield again in a nested A tag
                            href = None

            else:
                ## the next_b was not part of </a> or a nested <a tag,
                ## so keep it in the output
                vals.append(next_b)
def byte_href_anchors_state_machine(self):
        '''
        byte-based state machine extractor of anchor tags, so we can
        compute byte offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
        '''
        tag_depth = 0
        a_tag_depth = 0
        vals = []
        href = None
        idx_bytes = enumerate( self.clean_html )
        while 1:
            end_idx, val, next_b = read_to( idx_bytes, '<' )
            tag_depth += 1

            if href:
                ## must be inside an anchor tag, so accumulate the
                ## whole anchor
                assert a_tag_depth > 0, (href, self.clean_html)
                vals.append(val)

            ## figure out if start of an "A" anchor tag or close
            ## of a previous tag
            idx, next_b1 = idx_bytes.next()
            if next_b1.lower() == 'a':
                ## could be start of "A" tag
                idx, next_b2 = idx_bytes.next()
                if next_b2 == ' ':
                    a_tag_depth += 1

                    href = None
                    for idx, attr_name, attr_val in iter_attrs( idx_bytes ):
                        if attr_name.lower() == 'href':
                            href = attr_val
                        if idx is None:
                            ## doc ended mid tag, so invalid HTML--> just bail
                            return

                    first = idx + 1

                    ## if we got an href, then we want to keep the
                    ## first byte idx of the anchor:
                    if href:
                        ## Someone could nest an A tag inside another
                        ## A tag, which is invalid (even in HTML5), so
                        ## vals could be nonempty.  We only generate
                        ## the leaf-level A tags in these rare cases
                        ## of nested A tags, so reset it:
                        vals = []

            elif next_b1 == '/':
                idx, next_b1 = idx_bytes.next()
                if next_b1 == 'a':
                    ## could be end of "A" tag
                    idx, next_b2 = idx_bytes.next()
                    if next_b2 == '>':
                        a_tag_depth -= 1
                        if href:
                            ## join is much faster than using += above
                            anchor = b''.join(vals)
                            length = len(anchor)

                            ## yield the data
                            yield href, first, len(anchor), anchor

                            ## reset, no yield again in a nested A tag
                            href = None

            else:
                ## the next_b was not part of </a> or a nested <a tag,
                ## so keep it in the output
                vals.append(next_b)
byte-based state machine extractor of anchor tags, so we can
        compute byte offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)