streamcorpus_pipeline/_pipeline.py
def _process_task(self, work_unit):
        '''Process a :class:`coordinate.WorkUnit`.

        The work unit's key is taken as the input file name.  The
        data should have ``start_count`` and ``start_chunk_time``
        values, which are passed on to :meth:`run`.

        :param work_unit: work unit to process
        :paramtype work_unit: :class:`coordinate.WorkUnit`
        :return: number of stream items processed

        '''
        self.work_unit = work_unit
        i_str = work_unit.key
        start_count = work_unit.data['start_count']
        start_chunk_time = work_unit.data['start_chunk_time']
        self.run(i_str, start_count, start_chunk_time)
def _process_task(self, work_unit):
        '''Process a :class:`coordinate.WorkUnit`.

        The work unit's key is taken as the input file name.  The
        data should have ``start_count`` and ``start_chunk_time``
        values, which are passed on to :meth:`run`.

        :param work_unit: work unit to process
        :paramtype work_unit: :class:`coordinate.WorkUnit`
        :return: number of stream items processed

        '''
        self.work_unit = work_unit
        i_str = work_unit.key
        start_count = work_unit.data['start_count']
        start_chunk_time = work_unit.data['start_chunk_time']
        self.run(i_str, start_count, start_chunk_time)
Process a :class:`coordinate.WorkUnit`.

        The work unit's key is taken as the input file name.  The
        data should have ``start_count`` and ``start_chunk_time``
        values, which are passed on to :meth:`run`.

        :param work_unit: work unit to process
        :paramtype work_unit: :class:`coordinate.WorkUnit`
        :return: number of stream items processed