perceval/backends/core/mediawiki.py
def __fetch_1_27(self, from_date=None):
        """Fetch the pages from the backend url for MediaWiki >=1.27

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        logger.info("Looking for pages at url '%s'", self.url)

        npages = 0  # number of pages processed
        tpages = 0  # number of total pages
        pages_done = []  # pages already retrieved in reviews API

        namespaces_contents = self.__get_namespaces_contents()

        arvcontinue = ''  # pagination for getting revisions and their pages
        while arvcontinue is not None:
            raw_pages = self.client.get_pages_from_allrevisions(namespaces_contents, from_date, arvcontinue)
            data_json = json.loads(raw_pages)
            arvcontinue = data_json['continue']['arvcontinue'] if 'continue' in data_json else None
            pages_json = data_json['query']['allrevisions']
            for page in pages_json:

                if page['pageid'] in pages_done:
                    logger.debug("Page %s already processed; skipped", page['pageid'])
                    continue

                tpages += 1
                pages_done.append(page['pageid'])
                page_reviews = self.__get_page_reviews(page)

                if not page_reviews:
                    logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                   page['title'], page['pageid'])
                    continue

                yield page_reviews
                npages += 1

        logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)
def __fetch_1_27(self, from_date=None):
        """Fetch the pages from the backend url for MediaWiki >=1.27

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        logger.info("Looking for pages at url '%s'", self.url)

        npages = 0  # number of pages processed
        tpages = 0  # number of total pages
        pages_done = []  # pages already retrieved in reviews API

        namespaces_contents = self.__get_namespaces_contents()

        arvcontinue = ''  # pagination for getting revisions and their pages
        while arvcontinue is not None:
            raw_pages = self.client.get_pages_from_allrevisions(namespaces_contents, from_date, arvcontinue)
            data_json = json.loads(raw_pages)
            arvcontinue = data_json['continue']['arvcontinue'] if 'continue' in data_json else None
            pages_json = data_json['query']['allrevisions']
            for page in pages_json:

                if page['pageid'] in pages_done:
                    logger.debug("Page %s already processed; skipped", page['pageid'])
                    continue

                tpages += 1
                pages_done.append(page['pageid'])
                page_reviews = self.__get_page_reviews(page)

                if not page_reviews:
                    logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                   page['title'], page['pageid'])
                    continue

                yield page_reviews
                npages += 1

        logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)
Fetch the pages from the backend url for MediaWiki >=1.27

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages