gbdx_cloud_harness/controller.py
def _run_app(self):
        """
        Method for running a custom Application Templates.
        NOTES:
            * The default name of the application is app.py. So this function is going to look
            for app.py, unless the --file option is provide with a different file name.
            * The generated source bundle will package everything in the work_path. If large files
            not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
            to identify folders and files to ignore.
        USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]
        """
        is_remote_run = self._arguments.get('--remote')
        filename = self._arguments.get('<file_name>')
        upload_ports = self._arguments.get('--upload')
        download_ports = self._arguments.get('--download')
        is_verbose = self._arguments.get('--verbose')
        # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring.
        is_dry_run = self._arguments.get('--dry-run')

        if download_ports:  # TODO temporary until implemented.
            raise NotImplementedError("Downloading of output ports is not implemented yet.")

        # Check if the filename passed is actually a class object (gbdxtools functionality)
        if not isinstance(filename, str) and issubclass(filename, TaskTemplate):
            template_class = filename
            template_file = inspect.getfile(template_class)
            config_file = self._write_config_file(template_file)

        else:
            template_file = self._get_template_abs_path(filename)

            if not os.path.isfile(template_file):
                raise ValueError('The location %s does not exist' % template_file)

            config_file = self._write_config_file(template_file)

            template_class = self._get_class(template_file)

        with template_class() as template:
            if is_remote_run:  # Means the user is running with --remote, push to S3 and submit workflow.

                task = template.task

                # Set the source bundle directory to where the template_file is.
                task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4()))

                task.run_name = '{task_name}_src'.format(task_name=task.name)

                src_bundle_dir = task.source_bundle.value

                # Create source bundle to be executed on the GBDX platform
                self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir)

                port_service = PortService(task)

                if upload_ports:
                    # Push all port data to S3
                    port_service.upload_input_ports()
                else:
                    # Only push source bundle port
                    port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT])

                # Delete source bundle directory and config after upload.
                shutil.rmtree(src_bundle_dir)
                os.remove(config_file)

                # Get the new task object with uploaded port locations.
                task = port_service.task

                # Validate task
                task.is_valid(remote=True)

                workflow = Workflow(task)

                if is_verbose:
                    temp_wf = workflow.json
                    printer(temp_wf)

                if is_dry_run:
                    return task

                try:
                    workflow.execute()
                    printer(workflow.id)
                except Exception as e:
                    printer(e.message)
                    template.reason = "Execution Failed: %s" % e.message
                    return

                # Monitor events of workflow
                is_done = workflow.monitor_run()

                if not is_done:
                    template.reason = "Execution Failed during Run"

                if download_ports:
                    # TODO port_service.download_output_port()
                    pass

            else:
                # For local and Docker container execution.

                # Check that all output locations exist.
                template.check_and_create_outputs()

                # Validate task
                template.task.is_valid()

                if is_verbose:
                    printer(template.task.json())
                    all_ports = template.task.ports[0] + template.task.ports[1]
                    printer([port.__str__() for port in all_ports])

                if is_dry_run:
                    template.reason = "Execution Skipped"
                    return

                # Run Task Locally
                try:
                    template.invoke()
                except Exception as e:
                    template.reason = "Failed Exception: %s" % e

                if template.reason is None or template.reason == '':
                    template.reason = "Execution Completed"
def _run_app(self):
        """
        Method for running a custom Application Templates.
        NOTES:
            * The default name of the application is app.py. So this function is going to look
            for app.py, unless the --file option is provide with a different file name.
            * The generated source bundle will package everything in the work_path. If large files
            not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
            to identify folders and files to ignore.
        USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]
        """
        is_remote_run = self._arguments.get('--remote')
        filename = self._arguments.get('<file_name>')
        upload_ports = self._arguments.get('--upload')
        download_ports = self._arguments.get('--download')
        is_verbose = self._arguments.get('--verbose')
        # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring.
        is_dry_run = self._arguments.get('--dry-run')

        if download_ports:  # TODO temporary until implemented.
            raise NotImplementedError("Downloading of output ports is not implemented yet.")

        # Check if the filename passed is actually a class object (gbdxtools functionality)
        if not isinstance(filename, str) and issubclass(filename, TaskTemplate):
            template_class = filename
            template_file = inspect.getfile(template_class)
            config_file = self._write_config_file(template_file)

        else:
            template_file = self._get_template_abs_path(filename)

            if not os.path.isfile(template_file):
                raise ValueError('The location %s does not exist' % template_file)

            config_file = self._write_config_file(template_file)

            template_class = self._get_class(template_file)

        with template_class() as template:
            if is_remote_run:  # Means the user is running with --remote, push to S3 and submit workflow.

                task = template.task

                # Set the source bundle directory to where the template_file is.
                task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4()))

                task.run_name = '{task_name}_src'.format(task_name=task.name)

                src_bundle_dir = task.source_bundle.value

                # Create source bundle to be executed on the GBDX platform
                self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir)

                port_service = PortService(task)

                if upload_ports:
                    # Push all port data to S3
                    port_service.upload_input_ports()
                else:
                    # Only push source bundle port
                    port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT])

                # Delete source bundle directory and config after upload.
                shutil.rmtree(src_bundle_dir)
                os.remove(config_file)

                # Get the new task object with uploaded port locations.
                task = port_service.task

                # Validate task
                task.is_valid(remote=True)

                workflow = Workflow(task)

                if is_verbose:
                    temp_wf = workflow.json
                    printer(temp_wf)

                if is_dry_run:
                    return task

                try:
                    workflow.execute()
                    printer(workflow.id)
                except Exception as e:
                    printer(e.message)
                    template.reason = "Execution Failed: %s" % e.message
                    return

                # Monitor events of workflow
                is_done = workflow.monitor_run()

                if not is_done:
                    template.reason = "Execution Failed during Run"

                if download_ports:
                    # TODO port_service.download_output_port()
                    pass

            else:
                # For local and Docker container execution.

                # Check that all output locations exist.
                template.check_and_create_outputs()

                # Validate task
                template.task.is_valid()

                if is_verbose:
                    printer(template.task.json())
                    all_ports = template.task.ports[0] + template.task.ports[1]
                    printer([port.__str__() for port in all_ports])

                if is_dry_run:
                    template.reason = "Execution Skipped"
                    return

                # Run Task Locally
                try:
                    template.invoke()
                except Exception as e:
                    template.reason = "Failed Exception: %s" % e

                if template.reason is None or template.reason == '':
                    template.reason = "Execution Completed"
Method for running a custom Application Templates.
        NOTES:
            * The default name of the application is app.py. So this function is going to look
            for app.py, unless the --file option is provide with a different file name.
            * The generated source bundle will package everything in the work_path. If large files
            not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
            to identify folders and files to ignore.
        USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]