arthur/jobs.py
def run(self, backend_args, archive_args=None, resume=False):
        """Run the backend with the given parameters.

        The method will run the backend assigned to this job,
        storing the fetched items in a Redis queue. The ongoing
        status of the job, can be accessed through the property
        `result`. When `resume` is set, the job will start from
        the last execution, overwriting 'from_date' and 'offset'
        parameters, if needed.

        Setting to `True` the parameter `fetch_from_archive`, items can
        be fetched from the archive assigned to this job.

        Any exception during the execution of the process will
        be raised.

        :param backend_args: parameters used to un the backend
        :param archive_args: archive arguments
        :param resume: fetch items starting where the last
            execution stopped
        """
        args = backend_args.copy()

        if archive_args:
            self.initialize_archive_manager(archive_args['archive_path'])

        if not resume:
            max_date = backend_args.get('from_date', None)
            offset = backend_args.get('offset', None)

            if max_date:
                max_date = datetime_to_utc(max_date).timestamp()

            self._result = JobResult(self.job_id, self.task_id, self.backend, self.category,
                                     None, max_date, 0, offset=offset,
                                     nresumed=0)
        else:
            if self.result.max_date:
                args['from_date'] = unixtime_to_datetime(self.result.max_date)
            if self.result.offset:
                args['offset'] = self.result.offset
            self._result.nresumed += 1

        for item in self._execute(args, archive_args):
            self.conn.rpush(self.qitems, pickle.dumps(item))

            self._result.nitems += 1
            self._result.last_uuid = item['uuid']

            if not self.result.max_date or self.result.max_date < item['updated_on']:
                self._result.max_date = item['updated_on']
            if 'offset' in item:
                self._result.offset = item['offset']
def run(self, backend_args, archive_args=None, resume=False):
        """Run the backend with the given parameters.

        The method will run the backend assigned to this job,
        storing the fetched items in a Redis queue. The ongoing
        status of the job, can be accessed through the property
        `result`. When `resume` is set, the job will start from
        the last execution, overwriting 'from_date' and 'offset'
        parameters, if needed.

        Setting to `True` the parameter `fetch_from_archive`, items can
        be fetched from the archive assigned to this job.

        Any exception during the execution of the process will
        be raised.

        :param backend_args: parameters used to un the backend
        :param archive_args: archive arguments
        :param resume: fetch items starting where the last
            execution stopped
        """
        args = backend_args.copy()

        if archive_args:
            self.initialize_archive_manager(archive_args['archive_path'])

        if not resume:
            max_date = backend_args.get('from_date', None)
            offset = backend_args.get('offset', None)

            if max_date:
                max_date = datetime_to_utc(max_date).timestamp()

            self._result = JobResult(self.job_id, self.task_id, self.backend, self.category,
                                     None, max_date, 0, offset=offset,
                                     nresumed=0)
        else:
            if self.result.max_date:
                args['from_date'] = unixtime_to_datetime(self.result.max_date)
            if self.result.offset:
                args['offset'] = self.result.offset
            self._result.nresumed += 1

        for item in self._execute(args, archive_args):
            self.conn.rpush(self.qitems, pickle.dumps(item))

            self._result.nitems += 1
            self._result.last_uuid = item['uuid']

            if not self.result.max_date or self.result.max_date < item['updated_on']:
                self._result.max_date = item['updated_on']
            if 'offset' in item:
                self._result.offset = item['offset']
Run the backend with the given parameters.

        The method will run the backend assigned to this job,
        storing the fetched items in a Redis queue. The ongoing
        status of the job, can be accessed through the property
        `result`. When `resume` is set, the job will start from
        the last execution, overwriting 'from_date' and 'offset'
        parameters, if needed.

        Setting to `True` the parameter `fetch_from_archive`, items can
        be fetched from the archive assigned to this job.

        Any exception during the execution of the process will
        be raised.

        :param backend_args: parameters used to un the backend
        :param archive_args: archive arguments
        :param resume: fetch items starting where the last
            execution stopped