perceval/backends/core/mediawiki.py
def __fetch_pre1_27(self, from_date=None):
        """Fetch the pages from the backend url.

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        def fetch_incremental_changes(namespaces_contents):
            # Use recent changes API to get the pages from date
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            rccontinue = ''
            hole_created = True  # To detect that incremental is not complete
            while rccontinue is not None:
                raw_pages = self.client.get_recent_pages(namespaces_contents, rccontinue)
                data_json = json.loads(raw_pages)

                if 'query-continue' in data_json:
                    # < 1.27
                    rccontinue = data_json['query-continue']['recentchanges']['rccontinue']
                elif 'continue' in data_json:
                    # >= 1.27
                    rccontinue = data_json['continue']['rccontinue']
                else:
                    rccontinue = None

                pages_json = data_json['query']['recentchanges']
                for page in pages_json:

                    page_ts = dateutil.parser.parse(page['timestamp'])
                    if from_date >= page_ts:
                        # The rest of recent changes are older than from_date
                        logger.debug("All recent changes newer than %s processed.", from_date)
                        rccontinue = None
                        hole_created = False
                        break

                    if page['pageid'] in pages_done:
                        logger.debug("Page %s already processed; skipped", page['pageid'])
                        continue

                    tpages += 1
                    pages_done.append(page['pageid'])
                    page_reviews = self.__get_page_reviews(page)

                    if not page_reviews:
                        logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                       page['title'], page['pageid'])
                        continue

                    yield page_reviews
                    npages += 1
            if hole_created:
                logger.error("Incremental update NOT completed. Hole in history created.")
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        def fetch_all_pages(namespaces_contents):
            # Use get all pages API to get pages
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            for ns in namespaces_contents:
                apcontinue = ''  # pagination for getting pages
                logger.debug("Getting pages for namespace: %s", ns)
                while apcontinue is not None:
                    raw_pages = self.client.get_pages(ns, apcontinue)
                    data_json = json.loads(raw_pages)
                    if 'query-continue' in data_json:
                        # < 1.27
                        apcontinue = data_json['query-continue']['allpages']['apcontinue']
                    elif 'continue' in data_json:
                        # >= 1.27
                        apcontinue = data_json['continue']['apcontinue']
                    else:
                        apcontinue = None
                    pages_json = data_json['query']['allpages']
                    for page in pages_json:

                        if page['pageid'] in pages_done:
                            logger.debug("Page %s already processed; skipped", page['pageid'])
                            continue

                        tpages += 1
                        pages_done.append(page['pageid'])
                        page_reviews = self.__get_page_reviews(page)

                        if not page_reviews:
                            logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                           page['title'], page['pageid'])
                            continue

                        yield page_reviews
                        npages += 1
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        logger.info("Looking for pages at url '%s'", self.url)

        # from_date can not be older than MAX_RECENT_DAYS days ago
        if from_date:
            if (datetime_utcnow() - from_date).days >= MAX_RECENT_DAYS:
                cause = "Can't get incremental pages older than %i days." % MAX_RECENT_DAYS
                cause += " Do a complete analysis without from_date for older changes."
                raise BackendError(cause=cause)

        namespaces_contents = self.__get_namespaces_contents()

        if not from_date:
            return fetch_all_pages(namespaces_contents)
        else:
            return fetch_incremental_changes(namespaces_contents)
def __fetch_pre1_27(self, from_date=None):
        """Fetch the pages from the backend url.

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        def fetch_incremental_changes(namespaces_contents):
            # Use recent changes API to get the pages from date
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            rccontinue = ''
            hole_created = True  # To detect that incremental is not complete
            while rccontinue is not None:
                raw_pages = self.client.get_recent_pages(namespaces_contents, rccontinue)
                data_json = json.loads(raw_pages)

                if 'query-continue' in data_json:
                    # < 1.27
                    rccontinue = data_json['query-continue']['recentchanges']['rccontinue']
                elif 'continue' in data_json:
                    # >= 1.27
                    rccontinue = data_json['continue']['rccontinue']
                else:
                    rccontinue = None

                pages_json = data_json['query']['recentchanges']
                for page in pages_json:

                    page_ts = dateutil.parser.parse(page['timestamp'])
                    if from_date >= page_ts:
                        # The rest of recent changes are older than from_date
                        logger.debug("All recent changes newer than %s processed.", from_date)
                        rccontinue = None
                        hole_created = False
                        break

                    if page['pageid'] in pages_done:
                        logger.debug("Page %s already processed; skipped", page['pageid'])
                        continue

                    tpages += 1
                    pages_done.append(page['pageid'])
                    page_reviews = self.__get_page_reviews(page)

                    if not page_reviews:
                        logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                       page['title'], page['pageid'])
                        continue

                    yield page_reviews
                    npages += 1
            if hole_created:
                logger.error("Incremental update NOT completed. Hole in history created.")
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        def fetch_all_pages(namespaces_contents):
            # Use get all pages API to get pages
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            for ns in namespaces_contents:
                apcontinue = ''  # pagination for getting pages
                logger.debug("Getting pages for namespace: %s", ns)
                while apcontinue is not None:
                    raw_pages = self.client.get_pages(ns, apcontinue)
                    data_json = json.loads(raw_pages)
                    if 'query-continue' in data_json:
                        # < 1.27
                        apcontinue = data_json['query-continue']['allpages']['apcontinue']
                    elif 'continue' in data_json:
                        # >= 1.27
                        apcontinue = data_json['continue']['apcontinue']
                    else:
                        apcontinue = None
                    pages_json = data_json['query']['allpages']
                    for page in pages_json:

                        if page['pageid'] in pages_done:
                            logger.debug("Page %s already processed; skipped", page['pageid'])
                            continue

                        tpages += 1
                        pages_done.append(page['pageid'])
                        page_reviews = self.__get_page_reviews(page)

                        if not page_reviews:
                            logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                           page['title'], page['pageid'])
                            continue

                        yield page_reviews
                        npages += 1
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        logger.info("Looking for pages at url '%s'", self.url)

        # from_date can not be older than MAX_RECENT_DAYS days ago
        if from_date:
            if (datetime_utcnow() - from_date).days >= MAX_RECENT_DAYS:
                cause = "Can't get incremental pages older than %i days." % MAX_RECENT_DAYS
                cause += " Do a complete analysis without from_date for older changes."
                raise BackendError(cause=cause)

        namespaces_contents = self.__get_namespaces_contents()

        if not from_date:
            return fetch_all_pages(namespaces_contents)
        else:
            return fetch_incremental_changes(namespaces_contents)
Fetch the pages from the backend url.

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages