perceval/backends/core/googlehits.py
def __parse_hits(self, hit_raw):
        """Parse the hits returned by the Google Search API"""

        # Create the soup and get the desired div
        bs_result = bs4.BeautifulSoup(hit_raw, 'html.parser')
        hit_string = bs_result.find("div", id="resultStats").text

        # Remove commas or dots
        hit_string = hit_string.replace(',', u'')
        hit_string = hit_string.replace('.', u'')

        fetched_on = datetime_utcnow().timestamp()
        id_args = self.keywords[:]
        id_args.append(str(fetched_on))

        hits_json = {
            'fetched_on': fetched_on,
            'id': uuid(*id_args),
            'keywords': self.keywords,
            'type': 'googleSearchHits'
        }

        if not hit_string:
            logger.warning("No hits for %s", self.keywords)
            hits_json['hits'] = 0

            return hits_json

        str_hits = re.search(r'\d+', hit_string).group(0)
        hits = int(str_hits)
        hits_json['hits'] = hits

        return hits_json
def __parse_hits(self, hit_raw):
        """Parse the hits returned by the Google Search API"""

        # Create the soup and get the desired div
        bs_result = bs4.BeautifulSoup(hit_raw, 'html.parser')
        hit_string = bs_result.find("div", id="resultStats").text

        # Remove commas or dots
        hit_string = hit_string.replace(',', u'')
        hit_string = hit_string.replace('.', u'')

        fetched_on = datetime_utcnow().timestamp()
        id_args = self.keywords[:]
        id_args.append(str(fetched_on))

        hits_json = {
            'fetched_on': fetched_on,
            'id': uuid(*id_args),
            'keywords': self.keywords,
            'type': 'googleSearchHits'
        }

        if not hit_string:
            logger.warning("No hits for %s", self.keywords)
            hits_json['hits'] = 0

            return hits_json

        str_hits = re.search(r'\d+', hit_string).group(0)
        hits = int(str_hits)
        hits_json['hits'] = hits

        return hits_json
Parse the hits returned by the Google Search API