streamcorpus_pipeline/_hyperlink_labels.py
def href_filter(self, href):
        '''
        Test whether an href string meets criteria specified by
        configuration parameters 'require_abs_url', which means "does
        it look like it is probably an absolute URL?" and
        'domain_substrings'.  It searches for each of the
        domain_substrings in the href individually, and if any match,
        then returns True.

        :param: href string
        :returns bool:
        '''
        if self.config['require_abs_url']:
            if not href.lower().startswith(('http://', 'https://')):
                return False
        if self.config['all_domains']:
            ## blanket accept all domains as labels
            return True

        if self.config['domain_substrings']:
            parts = href.split('/')
            if len(parts) < 3:
                return False
            domain = parts[2].lower()
            for substring in self.config['domain_substrings']:
                try:
                    if substring in domain:
                        return True
                except Exception, exc:
                    logger.warn('%r in %r raised', substring, domain, exc_info=True)
                    return False
def href_filter(self, href):
        '''
        Test whether an href string meets criteria specified by
        configuration parameters 'require_abs_url', which means "does
        it look like it is probably an absolute URL?" and
        'domain_substrings'.  It searches for each of the
        domain_substrings in the href individually, and if any match,
        then returns True.

        :param: href string
        :returns bool:
        '''
        if self.config['require_abs_url']:
            if not href.lower().startswith(('http://', 'https://')):
                return False
        if self.config['all_domains']:
            ## blanket accept all domains as labels
            return True

        if self.config['domain_substrings']:
            parts = href.split('/')
            if len(parts) < 3:
                return False
            domain = parts[2].lower()
            for substring in self.config['domain_substrings']:
                try:
                    if substring in domain:
                        return True
                except Exception, exc:
                    logger.warn('%r in %r raised', substring, domain, exc_info=True)
                    return False
Test whether an href string meets criteria specified by
        configuration parameters 'require_abs_url', which means "does
        it look like it is probably an absolute URL?" and
        'domain_substrings'.  It searches for each of the
        domain_substrings in the href individually, and if any match,
        then returns True.

        :param: href string
        :returns bool: