streamcorpus_pipeline/_hyperlink_labels.py
def byte_href_anchors(self, chars=False):
        '''
        simple, regex-based extractor of anchor tags, so we can
        compute BYTE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
        '''
        input_buffer = self.clean_html
        if chars:
            input_buffer = input_buffer.decode('utf8')
        idx = 0
        ## split doc up into pieces that end on an anchor tag
        parts = input_buffer.split('</a>')
        assert len('</a>'.join(parts) ) == len(input_buffer)
        for part in parts:
            ## try to get an A tag out:
            m = anchors_re.match(part)

            if not m:
                idx += len(part) + 4
                continue

            before = m.group('before')
            ahref = m.group('ahref')

            ## increment the index to get line number for the anchor
            idx += len(before) + len(ahref)
            first = idx

            length = len(m.group('anchor'))

            ## update the index for the next loop
            # include anchor plus the </a>
            idx += length + 4

            if chars:
                yield m.group('href').encode('utf8'), first, length, m.group('anchor').encode('utf8')
            else:
                yield m.group('href'), first, length, m.group('anchor')

        assert idx - 4 == len(input_buffer)
def byte_href_anchors(self, chars=False):
        '''
        simple, regex-based extractor of anchor tags, so we can
        compute BYTE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
        '''
        input_buffer = self.clean_html
        if chars:
            input_buffer = input_buffer.decode('utf8')
        idx = 0
        ## split doc up into pieces that end on an anchor tag
        parts = input_buffer.split('</a>')
        assert len('</a>'.join(parts) ) == len(input_buffer)
        for part in parts:
            ## try to get an A tag out:
            m = anchors_re.match(part)

            if not m:
                idx += len(part) + 4
                continue

            before = m.group('before')
            ahref = m.group('ahref')

            ## increment the index to get line number for the anchor
            idx += len(before) + len(ahref)
            first = idx

            length = len(m.group('anchor'))

            ## update the index for the next loop
            # include anchor plus the </a>
            idx += length + 4

            if chars:
                yield m.group('href').encode('utf8'), first, length, m.group('anchor').encode('utf8')
            else:
                yield m.group('href'), first, length, m.group('anchor')

        assert idx - 4 == len(input_buffer)
simple, regex-based extractor of anchor tags, so we can
        compute BYTE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)