connectordb/logger.py
def sync(self):
        """Attempt to sync with the ConnectorDB server"""
        logging.debug("Logger: Syncing...")
        failed = False
        try:
            # Get the connectordb object
            cdb = self.connectordb

            # Ping the database - most connection errors will happen here
            cdb.ping()

            with self.synclock:
                c = self.database.cursor()
                for stream in self.streams:
                    s = cdb[stream]

                    c.execute(
                        "SELECT * FROM cache WHERE stream=? ORDER BY timestamp ASC;",
                        (stream, ))
                    datapointArray = []
                    for dp in c.fetchall():
                        datapointArray.append(
                            {"t": dp[1],
                             "d": json.loads(dp[2])})

                    # First, check if the data already inserted has newer timestamps,
                    # and in that case, assume that there was an error, and remove the datapoints
                    # with an older timestamp, so that we don't have an error when syncing
                    if len(s) > 0:
                        newtime = s[-1]["t"]
                        while (len(datapointArray) > 0 and datapointArray[0]["t"] < newtime):
                            logging.debug("Datapoint exists with older timestamp. Removing the datapoint.")
                            datapointArray = datapointArray[1:]

                    if len(datapointArray) > 0:
                        logging.debug("%s: syncing %i datapoints" %
                                      (stream, len(datapointArray)))

                        while (len(datapointArray) > DATAPOINT_INSERT_LIMIT):
                            # We insert datapoints in chunks of a couple
                            # thousand so that they fit in the insert size
                            # limit of ConnectorDB
                            s.insert_array(
                                datapointArray[:DATAPOINT_INSERT_LIMIT])

                            # Clear the written datapoints
                            datapointArray = datapointArray[
                                DATAPOINT_INSERT_LIMIT:]

                            # If there was no error inserting, delete the
                            # datapoints from the cache
                            c.execute(
                                "DELETE FROM cache WHERE stream=? AND timestamp <?",
                                (stream, datapointArray[0]["t"]))

                        s.insert_array(datapointArray)

                        # If there was no error inserting, delete the
                        # datapoints from the cache
                        c.execute(
                            "DELETE FROM cache WHERE stream=? AND timestamp <=?",
                            (stream, datapointArray[-1]["t"]))
                self.lastsynctime = time.time()

                if self.onsync is not None:
                    self.onsync()
        except Exception as e:
            # Handle the sync failure callback
            falied = True
            reraise = self.syncraise
            if self.onsyncfail is not None:
                reraise = self.onsyncfail(e)
            if reraise:
                raise
def sync(self):
        """Attempt to sync with the ConnectorDB server"""
        logging.debug("Logger: Syncing...")
        failed = False
        try:
            # Get the connectordb object
            cdb = self.connectordb

            # Ping the database - most connection errors will happen here
            cdb.ping()

            with self.synclock:
                c = self.database.cursor()
                for stream in self.streams:
                    s = cdb[stream]

                    c.execute(
                        "SELECT * FROM cache WHERE stream=? ORDER BY timestamp ASC;",
                        (stream, ))
                    datapointArray = []
                    for dp in c.fetchall():
                        datapointArray.append(
                            {"t": dp[1],
                             "d": json.loads(dp[2])})

                    # First, check if the data already inserted has newer timestamps,
                    # and in that case, assume that there was an error, and remove the datapoints
                    # with an older timestamp, so that we don't have an error when syncing
                    if len(s) > 0:
                        newtime = s[-1]["t"]
                        while (len(datapointArray) > 0 and datapointArray[0]["t"] < newtime):
                            logging.debug("Datapoint exists with older timestamp. Removing the datapoint.")
                            datapointArray = datapointArray[1:]

                    if len(datapointArray) > 0:
                        logging.debug("%s: syncing %i datapoints" %
                                      (stream, len(datapointArray)))

                        while (len(datapointArray) > DATAPOINT_INSERT_LIMIT):
                            # We insert datapoints in chunks of a couple
                            # thousand so that they fit in the insert size
                            # limit of ConnectorDB
                            s.insert_array(
                                datapointArray[:DATAPOINT_INSERT_LIMIT])

                            # Clear the written datapoints
                            datapointArray = datapointArray[
                                DATAPOINT_INSERT_LIMIT:]

                            # If there was no error inserting, delete the
                            # datapoints from the cache
                            c.execute(
                                "DELETE FROM cache WHERE stream=? AND timestamp <?",
                                (stream, datapointArray[0]["t"]))

                        s.insert_array(datapointArray)

                        # If there was no error inserting, delete the
                        # datapoints from the cache
                        c.execute(
                            "DELETE FROM cache WHERE stream=? AND timestamp <=?",
                            (stream, datapointArray[-1]["t"]))
                self.lastsynctime = time.time()

                if self.onsync is not None:
                    self.onsync()
        except Exception as e:
            # Handle the sync failure callback
            falied = True
            reraise = self.syncraise
            if self.onsyncfail is not None:
                reraise = self.onsyncfail(e)
            if reraise:
                raise
Attempt to sync with the ConnectorDB server