invenio_migrator/cli.py
def loadcommon(sources, load_task, asynchronous=True, predicate=None,
               task_args=None, task_kwargs=None):
    """Common helper function for load simple objects.

    .. note::

      Keyword arguments ``task_args`` and ``task_kwargs`` are passed to the
      ``load_task`` function as ``*task_args`` and ``**task_kwargs``.

    .. note::

      The `predicate` argument is used as a predicate function to load only
      a *single* item from across all dumps (this CLI function will return
      after loading the item). This is primarily used for debugging of
      the *dirty* data within the dump. The `predicate` should be a function
      with a signature ``f(dict) -> bool``, i.e. taking a single parameter
      (an item from the dump) and return ``True`` if the item
      should be loaded. See the ``loaddeposit`` for a concrete example.

    :param sources: JSON source files with dumps
    :type sources: list of str (filepaths)
    :param load_task: Shared task which loads the dump.
    :type load_task: function
    :param asynchronous: Flag for serial or asynchronous execution of the task.
    :type asynchronous: bool
    :param predicate: Predicate for selecting only a single item from the dump.
    :type predicate: function
    :param task_args: positional arguments passed to the task.
    :type task_args: tuple
    :param task_kwargs: named arguments passed to the task.
    :type task_kwargs: dict
    """
    # resolve the defaults for task_args and task_kwargs
    task_args = tuple() if task_args is None else task_args
    task_kwargs = dict() if task_kwargs is None else task_kwargs
    click.echo('Loading dumps started.')
    for idx, source in enumerate(sources, 1):
        click.echo('Opening dump file {0} of {1} ({2})'.format(
            idx, len(sources), source.name))
        data = json.load(source)
        with click.progressbar(data) as data_bar:
            for d in data_bar:
                # Load a single item from the dump
                if predicate is not None:
                    if predicate(d):
                        load_task.s(d, *task_args, **task_kwargs).apply(
                            throw=True)
                        click.echo("Loaded a single record.")
                        return
                # Load dumps normally
                else:
                    if asynchronous:
                        load_task.s(d, *task_args, **task_kwargs).apply_async()
                    else:
                        load_task.s(d, *task_args, **task_kwargs).apply(
                            throw=True)
def loadcommon(sources, load_task, asynchronous=True, predicate=None,
               task_args=None, task_kwargs=None):
    """Common helper function for load simple objects.

    .. note::

      Keyword arguments ``task_args`` and ``task_kwargs`` are passed to the
      ``load_task`` function as ``*task_args`` and ``**task_kwargs``.

    .. note::

      The `predicate` argument is used as a predicate function to load only
      a *single* item from across all dumps (this CLI function will return
      after loading the item). This is primarily used for debugging of
      the *dirty* data within the dump. The `predicate` should be a function
      with a signature ``f(dict) -> bool``, i.e. taking a single parameter
      (an item from the dump) and return ``True`` if the item
      should be loaded. See the ``loaddeposit`` for a concrete example.

    :param sources: JSON source files with dumps
    :type sources: list of str (filepaths)
    :param load_task: Shared task which loads the dump.
    :type load_task: function
    :param asynchronous: Flag for serial or asynchronous execution of the task.
    :type asynchronous: bool
    :param predicate: Predicate for selecting only a single item from the dump.
    :type predicate: function
    :param task_args: positional arguments passed to the task.
    :type task_args: tuple
    :param task_kwargs: named arguments passed to the task.
    :type task_kwargs: dict
    """
    # resolve the defaults for task_args and task_kwargs
    task_args = tuple() if task_args is None else task_args
    task_kwargs = dict() if task_kwargs is None else task_kwargs
    click.echo('Loading dumps started.')
    for idx, source in enumerate(sources, 1):
        click.echo('Opening dump file {0} of {1} ({2})'.format(
            idx, len(sources), source.name))
        data = json.load(source)
        with click.progressbar(data) as data_bar:
            for d in data_bar:
                # Load a single item from the dump
                if predicate is not None:
                    if predicate(d):
                        load_task.s(d, *task_args, **task_kwargs).apply(
                            throw=True)
                        click.echo("Loaded a single record.")
                        return
                # Load dumps normally
                else:
                    if asynchronous:
                        load_task.s(d, *task_args, **task_kwargs).apply_async()
                    else:
                        load_task.s(d, *task_args, **task_kwargs).apply(
                            throw=True)
Common helper function for load simple objects.

    .. note::

      Keyword arguments ``task_args`` and ``task_kwargs`` are passed to the
      ``load_task`` function as ``*task_args`` and ``**task_kwargs``.

    .. note::

      The `predicate` argument is used as a predicate function to load only
      a *single* item from across all dumps (this CLI function will return
      after loading the item). This is primarily used for debugging of
      the *dirty* data within the dump. The `predicate` should be a function
      with a signature ``f(dict) -> bool``, i.e. taking a single parameter
      (an item from the dump) and return ``True`` if the item
      should be loaded. See the ``loaddeposit`` for a concrete example.

    :param sources: JSON source files with dumps
    :type sources: list of str (filepaths)
    :param load_task: Shared task which loads the dump.
    :type load_task: function
    :param asynchronous: Flag for serial or asynchronous execution of the task.
    :type asynchronous: bool
    :param predicate: Predicate for selecting only a single item from the dump.
    :type predicate: function
    :param task_args: positional arguments passed to the task.
    :type task_args: tuple
    :param task_kwargs: named arguments passed to the task.
    :type task_kwargs: dict