perceval/backends/mozilla/crates.py
def __fetch_items(self, path, page=1):
        """Return the items from Crates.io API using pagination"""

        fetch_data = True
        parsed_crates = 0
        total_crates = 0

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                payload = {'sort': 'alphabetical', 'page': page}
                raw_content = self.fetch(path, payload=payload)
                content = json.loads(raw_content)

                parsed_crates += len(content['crates'])

                if not total_crates:
                    total_crates = content['meta']['total']

            except requests.exceptions.HTTPError as e:
                logger.error("HTTP exception raised - %s", e.response.text)
                raise e

            yield raw_content
            page += 1

            if parsed_crates >= total_crates:
                fetch_data = False
def __fetch_items(self, path, page=1):
        """Return the items from Crates.io API using pagination"""

        fetch_data = True
        parsed_crates = 0
        total_crates = 0

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                payload = {'sort': 'alphabetical', 'page': page}
                raw_content = self.fetch(path, payload=payload)
                content = json.loads(raw_content)

                parsed_crates += len(content['crates'])

                if not total_crates:
                    total_crates = content['meta']['total']

            except requests.exceptions.HTTPError as e:
                logger.error("HTTP exception raised - %s", e.response.text)
                raise e

            yield raw_content
            page += 1

            if parsed_crates >= total_crates:
                fetch_data = False
Return the items from Crates.io API using pagination