PyFunceble/mining.py
def mine(self):  # pragma: no cover
        """
        Search for domain or URL related to the original URL or domain.

        :return: The mined domains or URL.
        :rtype: dict
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            try:
                # We get the history.
                history = PyFunceble.requests.get(
                    self.to_get,
                    timeout=PyFunceble.CONFIGURATION["seconds_before_http_timeout"],
                    headers=self.headers,
                ).history

                # We initiate a dictionnary which will save the
                # list of mined links.
                mined = {self.to_get_bare: []}

                for element in history:
                    # We loop through the history.

                    # We update the element.
                    element = element.url

                    if PyFunceble.INTERN["to_test_type"] == "url":
                        # We are testing a full url.

                        # We get the element to append.
                        to_append = Check().is_url_valid(element, return_base=False)
                    elif PyFunceble.INTERN["to_test_type"] == "domain":
                        # We are testing a domain.

                        # We get the element to append.
                        to_append = Check().is_url_valid(element, return_base=True)
                    else:
                        raise Exception("Unknown tested.")

                    if to_append:
                        # There is something to append.

                        if to_append.endswith(":80"):
                            # The port is present.

                            # We get rid of it.
                            to_append = to_append[:-3]

                        if to_append != self.to_get_bare:
                            # The element to append is different as
                            # the element we are globally testing.

                            # We append the element to append to the
                            # list of mined links.
                            mined[self.to_get_bare].append(to_append)

                if mined[self.to_get_bare]:
                    # There is something in the list of mined links.

                    # We return the whole element.
                    return mined

                # There is nothing in the list of mined links.

                # We return None.
                return None

            except (
                PyFunceble.requests.ConnectionError,
                PyFunceble.requests.exceptions.Timeout,
                PyFunceble.requests.exceptions.InvalidURL,
                PyFunceble.socket.timeout,
                urllib3_exceptions.InvalidHeader,
                UnicodeDecodeError,  # The probability that this happend in production is minimal.
            ):
                # Something went wrong.

                # We return None.
                return None
        return None
def mine(self):  # pragma: no cover
        """
        Search for domain or URL related to the original URL or domain.

        :return: The mined domains or URL.
        :rtype: dict
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            try:
                # We get the history.
                history = PyFunceble.requests.get(
                    self.to_get,
                    timeout=PyFunceble.CONFIGURATION["seconds_before_http_timeout"],
                    headers=self.headers,
                ).history

                # We initiate a dictionnary which will save the
                # list of mined links.
                mined = {self.to_get_bare: []}

                for element in history:
                    # We loop through the history.

                    # We update the element.
                    element = element.url

                    if PyFunceble.INTERN["to_test_type"] == "url":
                        # We are testing a full url.

                        # We get the element to append.
                        to_append = Check().is_url_valid(element, return_base=False)
                    elif PyFunceble.INTERN["to_test_type"] == "domain":
                        # We are testing a domain.

                        # We get the element to append.
                        to_append = Check().is_url_valid(element, return_base=True)
                    else:
                        raise Exception("Unknown tested.")

                    if to_append:
                        # There is something to append.

                        if to_append.endswith(":80"):
                            # The port is present.

                            # We get rid of it.
                            to_append = to_append[:-3]

                        if to_append != self.to_get_bare:
                            # The element to append is different as
                            # the element we are globally testing.

                            # We append the element to append to the
                            # list of mined links.
                            mined[self.to_get_bare].append(to_append)

                if mined[self.to_get_bare]:
                    # There is something in the list of mined links.

                    # We return the whole element.
                    return mined

                # There is nothing in the list of mined links.

                # We return None.
                return None

            except (
                PyFunceble.requests.ConnectionError,
                PyFunceble.requests.exceptions.Timeout,
                PyFunceble.requests.exceptions.InvalidURL,
                PyFunceble.socket.timeout,
                urllib3_exceptions.InvalidHeader,
                UnicodeDecodeError,  # The probability that this happend in production is minimal.
            ):
                # Something went wrong.

                # We return None.
                return None
        return None
Search for domain or URL related to the original URL or domain.

        :return: The mined domains or URL.
        :rtype: dict