parsedmarc/__init__.py
def watch_inbox(host, username, password, callback, port=None, ssl=True,
                ssl_context=None, reports_folder="INBOX",
                archive_folder="Archive", delete=False, test=False, wait=30,
                nameservers=None, dns_timeout=6.0,
                strip_attachment_payloads=False):
    """
    Use an IDLE IMAP connection to parse incoming emails, and pass the results
    to a callback function

    Args:
        host: The mail server hostname or IP address
        username: The mail server username
        password: The mail server password
        callback: The callback function to receive the parsing results
        port: The mail server port
        ssl (bool): Use SSL/TLS
        ssl_context (SSLContext): A SSL context
        reports_folder: The IMAP folder where reports can be found
        archive_folder: The folder to move processed mail to
        delete (bool): Delete  messages after processing them
        test (bool): Do not move or delete messages after processing them
        wait (int): Number of seconds to wait for a IMAP IDLE response
        nameservers (list): A list of one or more nameservers to use
        (Cloudflare's public DNS resolvers by default)
        dns_timeout (float): Set the DNS query timeout
        strip_attachment_payloads (bool): Replace attachment payloads in
        forensic report samples with None
    """
    rf = reports_folder
    af = archive_folder
    ns = nameservers
    dt = dns_timeout
    if ssl_context is None:
        ssl_context = create_default_context()
    server = imapclient.IMAPClient(host, port=port, ssl=ssl,
                                   ssl_context=ssl_context,
                                   use_uid=True)

    try:
        server.login(username, password)
        imap_capabilities = get_imap_capabilities(server)
        if "IDLE" not in imap_capabilities:
            raise IMAPError("Cannot watch inbox: IMAP server does not support "
                            "the IDLE command")

        ms = "MOVE" in imap_capabilities
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        server.idle()

    except imapclient.exceptions.IMAPClientError as error:
        error = error.__str__().replace("b'", "").replace("'", "")
        # Workaround for random Exchange/Office365 IMAP errors
        if "unexpected response" in error or "BAD" in error:
            sleep_minutes = 5
            logger.debug(
                "{0}. "
                "Waiting {1} minutes before trying again".format(
                    error,
                    sleep_minutes))
            logger.debug("Reconnecting watcher")
            try:
                server.logout()
            except Exception as e:
                logger.debug("Failed to log out: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            ms = "MOVE" in get_imap_capabilities(server)
            sa = strip_attachment_payloads
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt,
                                               strip_attachment_payloads=sa)
            callback(res)
            server.idle()
        else:
            raise IMAPError(error)
    except socket.gaierror:
        raise IMAPError("DNS resolution failed")
    except ConnectionRefusedError:
        raise IMAPError("Connection refused")
    except ConnectionResetError:
        logger.debug("IMAP error: Connection reset")
        logger.debug("Reconnecting watcher")
        try:
            server.shutdown()
        except Exception as e:
            logger.debug("Failed to disconnect: {0}".format(e.__str__()))
        server = imapclient.IMAPClient(host)
        server.login(username, password)
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        ms = "MOVE" in get_imap_capabilities(server)
        res = get_dmarc_reports_from_inbox(connection=server,
                                           move_supported=ms,
                                           reports_folder=rf,
                                           archive_folder=af,
                                           delete=delete,
                                           test=test,
                                           nameservers=ns,
                                           dns_timeout=dt)
        callback(res)
        server.idle()
    except KeyError:
        logger.debug("IMAP error: Server returned unexpected result")
        logger.debug("Reconnecting watcher")
        try:
            server.logout()
        except Exception as e:
            logger.debug("Failed to log out: {0}".format(e.__str__()))
        server = imapclient.IMAPClient(host)
        server.login(username, password)
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        ms = "MOVE" in get_imap_capabilities(server)
        res = get_dmarc_reports_from_inbox(connection=server,
                                           move_supported=ms,
                                           reports_folder=rf,
                                           archive_folder=af,
                                           delete=delete,
                                           test=test,
                                           nameservers=ns,
                                           dns_timeout=dt)
        callback(res)
        server.idle()
    except ConnectionAbortedError:
        raise IMAPError("Connection aborted")
    except TimeoutError:
        raise IMAPError("Connection timed out")
    except SSLError as error:
        raise IMAPError("SSL error: {0}".format(error.__str__()))
    except CertificateError as error:
        raise IMAPError("Certificate error: {0}".format(error.__str__()))
    except BrokenPipeError:
        logger.debug("IMAP error: Broken pipe")
        logger.debug("Reconnecting watcher")
        try:
            server.shutdown()
        except Exception as e:
            logger.debug("Failed to disconnect: {0}".format(e.__str__()))
        server = imapclient.IMAPClient(host)
        server.login(username, password)
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        ms = "MOVE" in get_imap_capabilities(server)
        res = get_dmarc_reports_from_inbox(connection=server,
                                           move_supported=ms,
                                           reports_folder=rf,
                                           archive_folder=af,
                                           delete=delete,
                                           test=test,
                                           nameservers=ns,
                                           dns_timeout=dt)
        callback(res)
        server.idle()

    while True:
        try:
            # Refresh the IDLE session every 5 minutes to stay connected
            if time.monotonic() - idle_start_time > 5 * 60:
                logger.debug("IMAP: Refreshing IDLE session")
                server.idle_done()
                server.idle()
                idle_start_time = time.monotonic()
            responses = server.idle_check(timeout=wait)
            if responses is not None:
                if len(responses) == 0:
                    # Gmail/G-Suite does not generate anything in the responses
                    server.idle_done()
                    res = get_dmarc_reports_from_inbox(connection=server,
                                                       move_supported=ms,
                                                       reports_folder=rf,
                                                       archive_folder=af,
                                                       delete=delete,
                                                       test=test,
                                                       nameservers=ns,
                                                       dns_timeout=dt)
                    callback(res)
                    server.idle()
                    idle_start_time = time.monotonic()
                for response in responses:
                    logging.debug("Received response: {0}".format(response))
                    if response[0] > 0 and response[1] == b'RECENT':
                        server.idle_done()
                        res = get_dmarc_reports_from_inbox(connection=server,
                                                           move_supported=ms,
                                                           reports_folder=rf,
                                                           archive_folder=af,
                                                           delete=delete,
                                                           test=test,
                                                           nameservers=ns,
                                                           dns_timeout=dt)
                        callback(res)
                        server.idle()
                        idle_start_time = time.monotonic()
                        break
        except imapclient.exceptions.IMAPClientError as error:
            error = error.__str__().replace("b'", "").replace("'", "")
            # Workaround for random Exchange/Office365 IMAP errors
            if "unexpected response" in error or "BAD" in error:
                sleep_minutes = 5
                logger.debug(
                    "{0}. "
                    "Waiting {1} minutes before trying again".format(
                        error,
                        sleep_minutes))
                logger.debug("Reconnecting watcher")
                try:
                    server.logout()
                except Exception as e:
                    logger.debug("Failed to disconnect: {0}".format(
                        e.__str__()))
                server = imapclient.IMAPClient(host)
                server.login(username, password)
                server.select_folder(rf)
                idle_start_time = time.monotonic()
                ms = "MOVE" in get_imap_capabilities(server)
                res = get_dmarc_reports_from_inbox(connection=server,
                                                   move_supported=ms,
                                                   reports_folder=rf,
                                                   archive_folder=af,
                                                   delete=delete,
                                                   test=test,
                                                   nameservers=ns,
                                                   dns_timeout=dt)
                callback(res)
                server.idle()
            else:
                raise IMAPError(error)
        except socket.gaierror:
            raise IMAPError("DNS resolution failed")
        except ConnectionRefusedError:
            raise IMAPError("Connection refused")
        except (KeyError, socket.error, BrokenPipeError, ConnectionResetError):
            logger.debug("IMAP error: Connection reset")
            logger.debug("Reconnecting watcher")
            try:
                server.logout()
            except Exception as e:
                logger.debug("Failed to disconnect: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            ms = "MOVE" in get_imap_capabilities(server)
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt)
            callback(res)
            server.idle()
        except KeyError:
            logger.debug("IMAP error: Server returned unexpected result")
            logger.debug("Reconnecting watcher")
            try:
                server.logout()
            except Exception as e:
                logger.debug("Failed to log out: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            ms = "MOVE" in get_imap_capabilities(server)
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt)
            callback(res)
            server.idle()
        except ConnectionAbortedError:
            raise IMAPError("Connection aborted")
        except TimeoutError:
            raise IMAPError("Connection timed out")
        except SSLError as error:
            raise IMAPError("SSL error: {0}".format(error.__str__()))
        except CertificateError as error:
            raise IMAPError("Certificate error: {0}".format(error.__str__()))
        except BrokenPipeError:
            logger.debug("IMAP error: Broken pipe")
            logger.debug("Reconnecting watcher")
            try:
                server.shutdown()
            except Exception as e:
                logger.debug("Failed to disconnect: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt)
            callback(res)
            server.idle()
        except KeyboardInterrupt:
            break

    try:
        server.idle_done()
    except BrokenPipeError:
        pass
def watch_inbox(host, username, password, callback, port=None, ssl=True,
                ssl_context=None, reports_folder="INBOX",
                archive_folder="Archive", delete=False, test=False, wait=30,
                nameservers=None, dns_timeout=6.0,
                strip_attachment_payloads=False):
    """
    Use an IDLE IMAP connection to parse incoming emails, and pass the results
    to a callback function

    Args:
        host: The mail server hostname or IP address
        username: The mail server username
        password: The mail server password
        callback: The callback function to receive the parsing results
        port: The mail server port
        ssl (bool): Use SSL/TLS
        ssl_context (SSLContext): A SSL context
        reports_folder: The IMAP folder where reports can be found
        archive_folder: The folder to move processed mail to
        delete (bool): Delete  messages after processing them
        test (bool): Do not move or delete messages after processing them
        wait (int): Number of seconds to wait for a IMAP IDLE response
        nameservers (list): A list of one or more nameservers to use
        (Cloudflare's public DNS resolvers by default)
        dns_timeout (float): Set the DNS query timeout
        strip_attachment_payloads (bool): Replace attachment payloads in
        forensic report samples with None
    """
    rf = reports_folder
    af = archive_folder
    ns = nameservers
    dt = dns_timeout
    if ssl_context is None:
        ssl_context = create_default_context()
    server = imapclient.IMAPClient(host, port=port, ssl=ssl,
                                   ssl_context=ssl_context,
                                   use_uid=True)

    try:
        server.login(username, password)
        imap_capabilities = get_imap_capabilities(server)
        if "IDLE" not in imap_capabilities:
            raise IMAPError("Cannot watch inbox: IMAP server does not support "
                            "the IDLE command")

        ms = "MOVE" in imap_capabilities
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        server.idle()

    except imapclient.exceptions.IMAPClientError as error:
        error = error.__str__().replace("b'", "").replace("'", "")
        # Workaround for random Exchange/Office365 IMAP errors
        if "unexpected response" in error or "BAD" in error:
            sleep_minutes = 5
            logger.debug(
                "{0}. "
                "Waiting {1} minutes before trying again".format(
                    error,
                    sleep_minutes))
            logger.debug("Reconnecting watcher")
            try:
                server.logout()
            except Exception as e:
                logger.debug("Failed to log out: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            ms = "MOVE" in get_imap_capabilities(server)
            sa = strip_attachment_payloads
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt,
                                               strip_attachment_payloads=sa)
            callback(res)
            server.idle()
        else:
            raise IMAPError(error)
    except socket.gaierror:
        raise IMAPError("DNS resolution failed")
    except ConnectionRefusedError:
        raise IMAPError("Connection refused")
    except ConnectionResetError:
        logger.debug("IMAP error: Connection reset")
        logger.debug("Reconnecting watcher")
        try:
            server.shutdown()
        except Exception as e:
            logger.debug("Failed to disconnect: {0}".format(e.__str__()))
        server = imapclient.IMAPClient(host)
        server.login(username, password)
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        ms = "MOVE" in get_imap_capabilities(server)
        res = get_dmarc_reports_from_inbox(connection=server,
                                           move_supported=ms,
                                           reports_folder=rf,
                                           archive_folder=af,
                                           delete=delete,
                                           test=test,
                                           nameservers=ns,
                                           dns_timeout=dt)
        callback(res)
        server.idle()
    except KeyError:
        logger.debug("IMAP error: Server returned unexpected result")
        logger.debug("Reconnecting watcher")
        try:
            server.logout()
        except Exception as e:
            logger.debug("Failed to log out: {0}".format(e.__str__()))
        server = imapclient.IMAPClient(host)
        server.login(username, password)
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        ms = "MOVE" in get_imap_capabilities(server)
        res = get_dmarc_reports_from_inbox(connection=server,
                                           move_supported=ms,
                                           reports_folder=rf,
                                           archive_folder=af,
                                           delete=delete,
                                           test=test,
                                           nameservers=ns,
                                           dns_timeout=dt)
        callback(res)
        server.idle()
    except ConnectionAbortedError:
        raise IMAPError("Connection aborted")
    except TimeoutError:
        raise IMAPError("Connection timed out")
    except SSLError as error:
        raise IMAPError("SSL error: {0}".format(error.__str__()))
    except CertificateError as error:
        raise IMAPError("Certificate error: {0}".format(error.__str__()))
    except BrokenPipeError:
        logger.debug("IMAP error: Broken pipe")
        logger.debug("Reconnecting watcher")
        try:
            server.shutdown()
        except Exception as e:
            logger.debug("Failed to disconnect: {0}".format(e.__str__()))
        server = imapclient.IMAPClient(host)
        server.login(username, password)
        server.select_folder(rf)
        idle_start_time = time.monotonic()
        ms = "MOVE" in get_imap_capabilities(server)
        res = get_dmarc_reports_from_inbox(connection=server,
                                           move_supported=ms,
                                           reports_folder=rf,
                                           archive_folder=af,
                                           delete=delete,
                                           test=test,
                                           nameservers=ns,
                                           dns_timeout=dt)
        callback(res)
        server.idle()

    while True:
        try:
            # Refresh the IDLE session every 5 minutes to stay connected
            if time.monotonic() - idle_start_time > 5 * 60:
                logger.debug("IMAP: Refreshing IDLE session")
                server.idle_done()
                server.idle()
                idle_start_time = time.monotonic()
            responses = server.idle_check(timeout=wait)
            if responses is not None:
                if len(responses) == 0:
                    # Gmail/G-Suite does not generate anything in the responses
                    server.idle_done()
                    res = get_dmarc_reports_from_inbox(connection=server,
                                                       move_supported=ms,
                                                       reports_folder=rf,
                                                       archive_folder=af,
                                                       delete=delete,
                                                       test=test,
                                                       nameservers=ns,
                                                       dns_timeout=dt)
                    callback(res)
                    server.idle()
                    idle_start_time = time.monotonic()
                for response in responses:
                    logging.debug("Received response: {0}".format(response))
                    if response[0] > 0 and response[1] == b'RECENT':
                        server.idle_done()
                        res = get_dmarc_reports_from_inbox(connection=server,
                                                           move_supported=ms,
                                                           reports_folder=rf,
                                                           archive_folder=af,
                                                           delete=delete,
                                                           test=test,
                                                           nameservers=ns,
                                                           dns_timeout=dt)
                        callback(res)
                        server.idle()
                        idle_start_time = time.monotonic()
                        break
        except imapclient.exceptions.IMAPClientError as error:
            error = error.__str__().replace("b'", "").replace("'", "")
            # Workaround for random Exchange/Office365 IMAP errors
            if "unexpected response" in error or "BAD" in error:
                sleep_minutes = 5
                logger.debug(
                    "{0}. "
                    "Waiting {1} minutes before trying again".format(
                        error,
                        sleep_minutes))
                logger.debug("Reconnecting watcher")
                try:
                    server.logout()
                except Exception as e:
                    logger.debug("Failed to disconnect: {0}".format(
                        e.__str__()))
                server = imapclient.IMAPClient(host)
                server.login(username, password)
                server.select_folder(rf)
                idle_start_time = time.monotonic()
                ms = "MOVE" in get_imap_capabilities(server)
                res = get_dmarc_reports_from_inbox(connection=server,
                                                   move_supported=ms,
                                                   reports_folder=rf,
                                                   archive_folder=af,
                                                   delete=delete,
                                                   test=test,
                                                   nameservers=ns,
                                                   dns_timeout=dt)
                callback(res)
                server.idle()
            else:
                raise IMAPError(error)
        except socket.gaierror:
            raise IMAPError("DNS resolution failed")
        except ConnectionRefusedError:
            raise IMAPError("Connection refused")
        except (KeyError, socket.error, BrokenPipeError, ConnectionResetError):
            logger.debug("IMAP error: Connection reset")
            logger.debug("Reconnecting watcher")
            try:
                server.logout()
            except Exception as e:
                logger.debug("Failed to disconnect: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            ms = "MOVE" in get_imap_capabilities(server)
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt)
            callback(res)
            server.idle()
        except KeyError:
            logger.debug("IMAP error: Server returned unexpected result")
            logger.debug("Reconnecting watcher")
            try:
                server.logout()
            except Exception as e:
                logger.debug("Failed to log out: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            ms = "MOVE" in get_imap_capabilities(server)
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt)
            callback(res)
            server.idle()
        except ConnectionAbortedError:
            raise IMAPError("Connection aborted")
        except TimeoutError:
            raise IMAPError("Connection timed out")
        except SSLError as error:
            raise IMAPError("SSL error: {0}".format(error.__str__()))
        except CertificateError as error:
            raise IMAPError("Certificate error: {0}".format(error.__str__()))
        except BrokenPipeError:
            logger.debug("IMAP error: Broken pipe")
            logger.debug("Reconnecting watcher")
            try:
                server.shutdown()
            except Exception as e:
                logger.debug("Failed to disconnect: {0}".format(e.__str__()))
            server = imapclient.IMAPClient(host)
            server.login(username, password)
            server.select_folder(rf)
            idle_start_time = time.monotonic()
            res = get_dmarc_reports_from_inbox(connection=server,
                                               move_supported=ms,
                                               reports_folder=rf,
                                               archive_folder=af,
                                               delete=delete,
                                               test=test,
                                               nameservers=ns,
                                               dns_timeout=dt)
            callback(res)
            server.idle()
        except KeyboardInterrupt:
            break

    try:
        server.idle_done()
    except BrokenPipeError:
        pass
Use an IDLE IMAP connection to parse incoming emails, and pass the results
    to a callback function

    Args:
        host: The mail server hostname or IP address
        username: The mail server username
        password: The mail server password
        callback: The callback function to receive the parsing results
        port: The mail server port
        ssl (bool): Use SSL/TLS
        ssl_context (SSLContext): A SSL context
        reports_folder: The IMAP folder where reports can be found
        archive_folder: The folder to move processed mail to
        delete (bool): Delete  messages after processing them
        test (bool): Do not move or delete messages after processing them
        wait (int): Number of seconds to wait for a IMAP IDLE response
        nameservers (list): A list of one or more nameservers to use
        (Cloudflare's public DNS resolvers by default)
        dns_timeout (float): Set the DNS query timeout
        strip_attachment_payloads (bool): Replace attachment payloads in
        forensic report samples with None