streamcorpus_pipeline/_hyperlink_labels.py
def line_href_anchors(self):
        '''
        simple, regex-based extractor of anchor tags, so we can
        compute LINE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)

        Also, this mangles the body.clean_html so that LINE offsets
        uniquely identify tokens in anchor tags -- quite a kludge.
        '''
        idx = 0
        new_clean_html = ''
        newlines_added = 0
        ## split doc up into pieces that end on an anchor tag
        parts = self.clean_html.split('</a>')
        assert len('</a>'.join(parts) ) == len(self.clean_html)
        for i in range(len(parts)):
            part = parts[i]

            ## try to get an A tag out:
            m = anchors_re.match(part)

            ## if not, then just keep going
            if not m:
                new_clean_html += part
                if i < len(parts) - 1:
                    new_clean_html += '</a>'
                continue

            before = m.group('before')
            ahref = m.group('ahref')

            ## construct a text containing bytes up to the anchor text
            pre_anchor_increment = before + ahref

            ## increment the index to get line number for the anchor
            idx += len( pre_anchor_increment.splitlines() )
            first = idx

            ## usually this will be one, but it could be more than
            ## that when an anchor text contains newlines
            length = len(m.group('anchor').split('\n'))

            ## construct new clean_html with these newlines inserted
            new_clean_html += pre_anchor_increment + '\n' + m.group('anchor') + '\n</a>'

            newlines_added += 2

            ## update the index for the next loop
            idx += length - 1

            yield m.group('href'), first, length, m.group('anchor')

        ## replace clean_html with our new one that has newlines inserted
        assert len(self.clean_html) == len(new_clean_html) - newlines_added
        self.clean_html = new_clean_html
def line_href_anchors(self):
        '''
        simple, regex-based extractor of anchor tags, so we can
        compute LINE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)

        Also, this mangles the body.clean_html so that LINE offsets
        uniquely identify tokens in anchor tags -- quite a kludge.
        '''
        idx = 0
        new_clean_html = ''
        newlines_added = 0
        ## split doc up into pieces that end on an anchor tag
        parts = self.clean_html.split('</a>')
        assert len('</a>'.join(parts) ) == len(self.clean_html)
        for i in range(len(parts)):
            part = parts[i]

            ## try to get an A tag out:
            m = anchors_re.match(part)

            ## if not, then just keep going
            if not m:
                new_clean_html += part
                if i < len(parts) - 1:
                    new_clean_html += '</a>'
                continue

            before = m.group('before')
            ahref = m.group('ahref')

            ## construct a text containing bytes up to the anchor text
            pre_anchor_increment = before + ahref

            ## increment the index to get line number for the anchor
            idx += len( pre_anchor_increment.splitlines() )
            first = idx

            ## usually this will be one, but it could be more than
            ## that when an anchor text contains newlines
            length = len(m.group('anchor').split('\n'))

            ## construct new clean_html with these newlines inserted
            new_clean_html += pre_anchor_increment + '\n' + m.group('anchor') + '\n</a>'

            newlines_added += 2

            ## update the index for the next loop
            idx += length - 1

            yield m.group('href'), first, length, m.group('anchor')

        ## replace clean_html with our new one that has newlines inserted
        assert len(self.clean_html) == len(new_clean_html) - newlines_added
        self.clean_html = new_clean_html
simple, regex-based extractor of anchor tags, so we can
        compute LINE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)

        Also, this mangles the body.clean_html so that LINE offsets
        uniquely identify tokens in anchor tags -- quite a kludge.