libplanarradpy/planrad.py
def run(self):
        """Distributes the work across the CPUs.  It actually uses _run()"""
        done = False
        dir_list = []
        tic = time.clock()
        lg.info('Starting batch run at :: ' + str(tic))

        if self.run_params.num_cpus == -1:  # user hasn't set a throttle
            self.run_params.num_cpus = os.sysconf("SC_NPROCESSORS_ONLN")
            lg.info('Found ' + str(self.run_params.num_cpus) + ' CPUs')

        # --------------------------------------------------#
        # COUNT THE NUMBER OF DIRECTORIES TO ITERATE THROUGH
        # --------------------------------------------------#
        tmp_dir_list = os.listdir(self.batch_output)
        for direc in tmp_dir_list:
            dir_list.append(os.path.join(self.batch_output, direc))

        num_dirs = len(dir_list)
        lg.info('Found ' + str(num_dirs) + ' directories to process in ' + self.batch_output)

        sub = scipy.floor(num_dirs / self.run_params.num_cpus)
        remainder = num_dirs - (sub * self.run_params.num_cpus)
        if remainder > 0:
            lg.warning('Number of variations not evenly divisible by number of CPUs')
            lg.warning('This is not a problem, last block will not use all available CPUs')
            lg.warning('The remainder is :: ' + str(remainder))

        while not done:
            for l in range(0, int(sub)):
                lg.info('Starting processing block of :: ' + str(self.run_params.num_cpus) + ' processes')

                for m in range(0, self.run_params.num_cpus):
                    #row = (m * sub) + l
                    _dir = dir_list.pop()

                    #--------------------------------------------------#
                    # CHECK TO SEE IF REPORT HAS BEEN GENERATED AND DON'T
                    # BOTHER RUNNING AGAIN IF THEY DO EXIST
                    #--------------------------------------------------#

                    report_dir, report_file_name = os.path.split(self.run_params.report_file)
                    lg.debug(report_file_name)
                    lg.debug(os.path.join(_dir, report_file_name))

                    try:
                        rep_size = os.path.getsize(os.path.join(_dir, report_file_name.strip('\n')))
                        lg.debug('report size is :: ' + str(rep_size))
                    except:
                        rep_size = 0

                    if rep_size < 1.0:  # TODO this is a spoof!
                        lg.info('No report file found, running process')
                        p = Process(target=self._run, args=(_dir,))
                    else:
                        lg.warning('Report file found :: ' + os.path.join(_dir, report_file_name.strip(
                            '\n')) + ' not redoing run ')
                        p = Process(target=self._dummy, args=(_dir,))

                    # !! for testing
                    #p = Process(target=self._dummy, args=(_dir,))
                    p.start()
                    lg.info('Starting Process :: Process ID :: ' + str(p.pid))

                p.join()

            self.run_params.num_cpus = remainder
            remainder = 0
            lg.info('Processing remainder')
            sub = 1
            if remainder == 0:
                done = True

        toc = time.clock()  # this isn't working
        lg.info('Ending batch run at :: ' + str(toc))
        timeTaken = toc - tic
        lg.info('Time taken ::' + str(timeTaken))
def run(self):
        """Distributes the work across the CPUs.  It actually uses _run()"""
        done = False
        dir_list = []
        tic = time.clock()
        lg.info('Starting batch run at :: ' + str(tic))

        if self.run_params.num_cpus == -1:  # user hasn't set a throttle
            self.run_params.num_cpus = os.sysconf("SC_NPROCESSORS_ONLN")
            lg.info('Found ' + str(self.run_params.num_cpus) + ' CPUs')

        # --------------------------------------------------#
        # COUNT THE NUMBER OF DIRECTORIES TO ITERATE THROUGH
        # --------------------------------------------------#
        tmp_dir_list = os.listdir(self.batch_output)
        for direc in tmp_dir_list:
            dir_list.append(os.path.join(self.batch_output, direc))

        num_dirs = len(dir_list)
        lg.info('Found ' + str(num_dirs) + ' directories to process in ' + self.batch_output)

        sub = scipy.floor(num_dirs / self.run_params.num_cpus)
        remainder = num_dirs - (sub * self.run_params.num_cpus)
        if remainder > 0:
            lg.warning('Number of variations not evenly divisible by number of CPUs')
            lg.warning('This is not a problem, last block will not use all available CPUs')
            lg.warning('The remainder is :: ' + str(remainder))

        while not done:
            for l in range(0, int(sub)):
                lg.info('Starting processing block of :: ' + str(self.run_params.num_cpus) + ' processes')

                for m in range(0, self.run_params.num_cpus):
                    #row = (m * sub) + l
                    _dir = dir_list.pop()

                    #--------------------------------------------------#
                    # CHECK TO SEE IF REPORT HAS BEEN GENERATED AND DON'T
                    # BOTHER RUNNING AGAIN IF THEY DO EXIST
                    #--------------------------------------------------#

                    report_dir, report_file_name = os.path.split(self.run_params.report_file)
                    lg.debug(report_file_name)
                    lg.debug(os.path.join(_dir, report_file_name))

                    try:
                        rep_size = os.path.getsize(os.path.join(_dir, report_file_name.strip('\n')))
                        lg.debug('report size is :: ' + str(rep_size))
                    except:
                        rep_size = 0

                    if rep_size < 1.0:  # TODO this is a spoof!
                        lg.info('No report file found, running process')
                        p = Process(target=self._run, args=(_dir,))
                    else:
                        lg.warning('Report file found :: ' + os.path.join(_dir, report_file_name.strip(
                            '\n')) + ' not redoing run ')
                        p = Process(target=self._dummy, args=(_dir,))

                    # !! for testing
                    #p = Process(target=self._dummy, args=(_dir,))
                    p.start()
                    lg.info('Starting Process :: Process ID :: ' + str(p.pid))

                p.join()

            self.run_params.num_cpus = remainder
            remainder = 0
            lg.info('Processing remainder')
            sub = 1
            if remainder == 0:
                done = True

        toc = time.clock()  # this isn't working
        lg.info('Ending batch run at :: ' + str(toc))
        timeTaken = toc - tic
        lg.info('Time taken ::' + str(timeTaken))
Distributes the work across the CPUs.  It actually uses _run()