scraper/github/queryManager.py
def queryGitHub(self, gitquery, gitvars={}, verbosity=0, paginate=False, cursorVar=None, keysToList=[], rest=False, requestCount=0, pageNum=0):
        """Submit a GitHub query.

        Args:
            gitquery (str): The query or endpoint itself.
                Examples:
                       query: 'query { viewer { login } }'
                    endpoint: '/user'
            gitvars (Optional[Dict]): All query variables.
                Defaults to empty.
                GraphQL Only.
            verbosity (Optional[int]): Changes output verbosity levels.
                If < 0, all extra printouts are suppressed.
                If == 0, normal print statements are displayed.
                If > 0, additional status print statements are displayed.
                Defaults to 0.
            paginate (Optional[bool]): Pagination will be completed
                automatically if True. Defaults to False.
            cursorVar (Optional[str]): Key in 'gitvars' that represents the
                pagination cursor. Defaults to None.
                GraphQL Only.
            keysToList (Optional[List[str]]): Ordered list of keys needed to
                retrieve the list in the query results to be extended by
                pagination. Defaults to empty.
                Example:
                    ['data', 'viewer', 'repositories', 'nodes']
                GraphQL Only.
            rest (Optional[bool]): If True, uses the REST API instead
                of GraphQL. Defaults to False.
            requestCount (Optional[int]): Counter for repeated requests.
            pageNum (Optional[int]): Counter for pagination.
                For user readable log messages only, does not affect data.

        Returns:
            Dict: A JSON style dictionary.

        """
        requestCount += 1
        if pageNum < 0:  # no negative page numbers
            pageNum = 0
        pageNum += 1

        if paginate:
            _vPrint((verbosity >= 0), "Page %d" % (pageNum))
        _vPrint((verbosity >= 0), "Sending %s query..." % ("REST" if rest else "GraphQL"))
        response = self._submitQuery(gitquery, gitvars=gitvars, verbose=(verbosity > 0), rest=rest)
        _vPrint((verbosity >= 0), "Checking response...")
        _vPrint((verbosity >= 0), response["headDict"]["http"])
        statusNum = response["statusNum"]

        # Decrement page count before error checks to properly reflect any repeated queries
        pageNum -= 1

        # Make sure the query limit didn't run out
        try:
            apiStatus = {
                "limit": int(response["headDict"]["X-RateLimit-Limit"]),
                "remaining": int(response["headDict"]["X-RateLimit-Remaining"]),
                "reset": int(response["headDict"]["X-RateLimit-Reset"])
            }
            _vPrint((verbosity >= 0), "API Status %s" % (json.dumps(apiStatus)))
            if not apiStatus["remaining"] > 0:
                _vPrint((verbosity >= 0), "API usage limit reached during query.")
                self._awaitReset(apiStatus["reset"])
                _vPrint((verbosity >= 0), "Repeating query...")
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=(requestCount - 1), pageNum=pageNum)
        except KeyError:
            # Handles error cases that don't return X-RateLimit data
            _vPrint((verbosity >= 0), "Failed to check API Status.")

        # Check for accepted but not yet processed, usually due to un-cached data
        if statusNum == 202:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Query accepted but not yet processed. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for server error responses
        if statusNum == 502 or statusNum == 503:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Server error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for other error responses
        if statusNum >= 400 or statusNum == 204:
            raise RuntimeError("Request got an Error response.\n%s\n%s" % (response["headDict"]["http"], response["result"]))

        _vPrint((verbosity >= 0), "Data received!")
        outObj = json.loads(response["result"])

        # Check for GraphQL API errors (e.g. repo not found)
        if not rest and "errors" in outObj:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            elif len(outObj["errors"]) == 1 and len(outObj["errors"][0]) == 1:
                # Poorly defined error type, usually intermittent, try again.
                _vPrint((verbosity >= 0), "GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))
                self._countdown(self.__retryDelay, printString="Unknown API error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
            else:
                raise RuntimeError("GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))

        # Re-increment page count before the next page query
        pageNum += 1

        # Pagination
        if paginate:
            if rest and response["linkDict"]:
                if "next" in response["linkDict"]:
                    nextObj = self.queryGitHub(response["linkDict"]["next"], gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    outObj.extend(nextObj)
            elif not rest:
                if not cursorVar:
                    raise ValueError("Must specify argument 'cursorVar' to use GraphQL auto-pagination.")
                if not len(keysToList) > 0:
                    raise ValueError("Must specify argument 'keysToList' as a non-empty list to use GraphQL auto-pagination.")
                aPage = outObj
                for key in keysToList[0:-1]:
                    aPage = aPage[key]
                gitvars[cursorVar] = aPage["pageInfo"]["endCursor"]
                if aPage["pageInfo"]["hasNextPage"]:
                    nextObj = self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    newPage = nextObj
                    for key in keysToList[0:-1]:
                        newPage = newPage[key]
                    aPage[keysToList[-1]].extend(newPage[keysToList[-1]])
                aPage.pop("pageInfo", None)

        return outObj
def queryGitHub(self, gitquery, gitvars={}, verbosity=0, paginate=False, cursorVar=None, keysToList=[], rest=False, requestCount=0, pageNum=0):
        """Submit a GitHub query.

        Args:
            gitquery (str): The query or endpoint itself.
                Examples:
                       query: 'query { viewer { login } }'
                    endpoint: '/user'
            gitvars (Optional[Dict]): All query variables.
                Defaults to empty.
                GraphQL Only.
            verbosity (Optional[int]): Changes output verbosity levels.
                If < 0, all extra printouts are suppressed.
                If == 0, normal print statements are displayed.
                If > 0, additional status print statements are displayed.
                Defaults to 0.
            paginate (Optional[bool]): Pagination will be completed
                automatically if True. Defaults to False.
            cursorVar (Optional[str]): Key in 'gitvars' that represents the
                pagination cursor. Defaults to None.
                GraphQL Only.
            keysToList (Optional[List[str]]): Ordered list of keys needed to
                retrieve the list in the query results to be extended by
                pagination. Defaults to empty.
                Example:
                    ['data', 'viewer', 'repositories', 'nodes']
                GraphQL Only.
            rest (Optional[bool]): If True, uses the REST API instead
                of GraphQL. Defaults to False.
            requestCount (Optional[int]): Counter for repeated requests.
            pageNum (Optional[int]): Counter for pagination.
                For user readable log messages only, does not affect data.

        Returns:
            Dict: A JSON style dictionary.

        """
        requestCount += 1
        if pageNum < 0:  # no negative page numbers
            pageNum = 0
        pageNum += 1

        if paginate:
            _vPrint((verbosity >= 0), "Page %d" % (pageNum))
        _vPrint((verbosity >= 0), "Sending %s query..." % ("REST" if rest else "GraphQL"))
        response = self._submitQuery(gitquery, gitvars=gitvars, verbose=(verbosity > 0), rest=rest)
        _vPrint((verbosity >= 0), "Checking response...")
        _vPrint((verbosity >= 0), response["headDict"]["http"])
        statusNum = response["statusNum"]

        # Decrement page count before error checks to properly reflect any repeated queries
        pageNum -= 1

        # Make sure the query limit didn't run out
        try:
            apiStatus = {
                "limit": int(response["headDict"]["X-RateLimit-Limit"]),
                "remaining": int(response["headDict"]["X-RateLimit-Remaining"]),
                "reset": int(response["headDict"]["X-RateLimit-Reset"])
            }
            _vPrint((verbosity >= 0), "API Status %s" % (json.dumps(apiStatus)))
            if not apiStatus["remaining"] > 0:
                _vPrint((verbosity >= 0), "API usage limit reached during query.")
                self._awaitReset(apiStatus["reset"])
                _vPrint((verbosity >= 0), "Repeating query...")
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=(requestCount - 1), pageNum=pageNum)
        except KeyError:
            # Handles error cases that don't return X-RateLimit data
            _vPrint((verbosity >= 0), "Failed to check API Status.")

        # Check for accepted but not yet processed, usually due to un-cached data
        if statusNum == 202:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Query accepted but not yet processed. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for server error responses
        if statusNum == 502 or statusNum == 503:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Server error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for other error responses
        if statusNum >= 400 or statusNum == 204:
            raise RuntimeError("Request got an Error response.\n%s\n%s" % (response["headDict"]["http"], response["result"]))

        _vPrint((verbosity >= 0), "Data received!")
        outObj = json.loads(response["result"])

        # Check for GraphQL API errors (e.g. repo not found)
        if not rest and "errors" in outObj:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            elif len(outObj["errors"]) == 1 and len(outObj["errors"][0]) == 1:
                # Poorly defined error type, usually intermittent, try again.
                _vPrint((verbosity >= 0), "GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))
                self._countdown(self.__retryDelay, printString="Unknown API error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
            else:
                raise RuntimeError("GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))

        # Re-increment page count before the next page query
        pageNum += 1

        # Pagination
        if paginate:
            if rest and response["linkDict"]:
                if "next" in response["linkDict"]:
                    nextObj = self.queryGitHub(response["linkDict"]["next"], gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    outObj.extend(nextObj)
            elif not rest:
                if not cursorVar:
                    raise ValueError("Must specify argument 'cursorVar' to use GraphQL auto-pagination.")
                if not len(keysToList) > 0:
                    raise ValueError("Must specify argument 'keysToList' as a non-empty list to use GraphQL auto-pagination.")
                aPage = outObj
                for key in keysToList[0:-1]:
                    aPage = aPage[key]
                gitvars[cursorVar] = aPage["pageInfo"]["endCursor"]
                if aPage["pageInfo"]["hasNextPage"]:
                    nextObj = self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    newPage = nextObj
                    for key in keysToList[0:-1]:
                        newPage = newPage[key]
                    aPage[keysToList[-1]].extend(newPage[keysToList[-1]])
                aPage.pop("pageInfo", None)

        return outObj
Submit a GitHub query.

        Args:
            gitquery (str): The query or endpoint itself.
                Examples:
                       query: 'query { viewer { login } }'
                    endpoint: '/user'
            gitvars (Optional[Dict]): All query variables.
                Defaults to empty.
                GraphQL Only.
            verbosity (Optional[int]): Changes output verbosity levels.
                If < 0, all extra printouts are suppressed.
                If == 0, normal print statements are displayed.
                If > 0, additional status print statements are displayed.
                Defaults to 0.
            paginate (Optional[bool]): Pagination will be completed
                automatically if True. Defaults to False.
            cursorVar (Optional[str]): Key in 'gitvars' that represents the
                pagination cursor. Defaults to None.
                GraphQL Only.
            keysToList (Optional[List[str]]): Ordered list of keys needed to
                retrieve the list in the query results to be extended by
                pagination. Defaults to empty.
                Example:
                    ['data', 'viewer', 'repositories', 'nodes']
                GraphQL Only.
            rest (Optional[bool]): If True, uses the REST API instead
                of GraphQL. Defaults to False.
            requestCount (Optional[int]): Counter for repeated requests.
            pageNum (Optional[int]): Counter for pagination.
                For user readable log messages only, does not affect data.

        Returns:
            Dict: A JSON style dictionary.