deeppavlov/models/elmo/bilm_model.py
def _build_loss(self, lstm_outputs):
        """
        Create:
            self.total_loss: total loss op for training
            self.softmax_W, softmax_b: the softmax variables
            self.next_token_id / _reverse: placeholders for gold input

        """
        batch_size = self.options['batch_size']
        unroll_steps = self.options['unroll_steps']

        n_tokens_vocab = self.options['n_tokens_vocab']

        # DEFINE next_token_id and *_reverse placeholders for the gold input
        def _get_next_token_placeholders(suffix):
            name = 'next_token_id' + suffix
            id_placeholder = tf.placeholder(DTYPE_INT,
                                            shape=(batch_size, unroll_steps),
                                            name=name)
            return id_placeholder

        # get the window and weight placeholders
        self.next_token_id = _get_next_token_placeholders('')
        if self.bidirectional:
            self.next_token_id_reverse = _get_next_token_placeholders(
                '_reverse')

        # DEFINE THE SOFTMAX VARIABLES
        # get the dimension of the softmax weights
        # softmax dimension is the size of the output projection_dim
        softmax_dim = self.options['lstm']['projection_dim']

        # the output softmax variables -- they are shared if bidirectional
        if self.share_embedding_softmax:
            # softmax_W is just the embedding layer
            self.softmax_W = self.embedding_weights

        with tf.variable_scope('softmax'), tf.device('/cpu:0'):
            # Glorit init (std=(1.0 / sqrt(fan_in))
            softmax_init = tf.random_normal_initializer(0.0, 1.0 / np.sqrt(softmax_dim))
            if not self.share_embedding_softmax:
                self.softmax_W = tf.get_variable(
                    'W', [n_tokens_vocab, softmax_dim],
                    dtype=DTYPE,
                    initializer=softmax_init
                )
            self.softmax_b = tf.get_variable(
                'b', [n_tokens_vocab],
                dtype=DTYPE,
                initializer=tf.constant_initializer(0.0))

        # now calculate losses
        # loss for each direction of the LSTM
        self.individual_train_losses = []
        self.individual_eval_losses = []

        if self.bidirectional:
            next_ids = [self.next_token_id, self.next_token_id_reverse]
        else:
            next_ids = [self.next_token_id]

        for id_placeholder, lstm_output_flat in zip(next_ids, lstm_outputs):
            # flatten the LSTM output and next token id gold to shape:
            # (batch_size * unroll_steps, softmax_dim)
            # Flatten and reshape the token_id placeholders
            next_token_id_flat = tf.reshape(id_placeholder, [-1, 1])

            with tf.control_dependencies([lstm_output_flat]):
                sampled_losses = tf.nn.sampled_softmax_loss(self.softmax_W, self.softmax_b,
                                                            next_token_id_flat, lstm_output_flat,
                                                            self.options['n_negative_samples_batch'],
                                                            self.options['n_tokens_vocab'],
                                                            num_true=1)

                # get the full softmax loss
                output_scores = tf.matmul(
                    lstm_output_flat,
                    tf.transpose(self.softmax_W)
                ) + self.softmax_b
                # NOTE: tf.nn.sparse_softmax_cross_entropy_with_logits
                #   expects unnormalized output since it performs the
                #   softmax internally
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                    logits=output_scores,
                    labels=tf.squeeze(next_token_id_flat, squeeze_dims=[1])
                )
            sampled_losses = tf.reshape(sampled_losses, [self.options['batch_size'], -1])
            losses = tf.reshape(losses, [self.options['batch_size'], -1])
            self.individual_train_losses.append(tf.reduce_mean(sampled_losses, axis=1))
            self.individual_eval_losses.append(tf.reduce_mean(losses, axis=1))

        # now make the total loss -- it's the train of the individual losses
        if self.bidirectional:
            self.total_train_loss = 0.5 * (self.individual_train_losses[0] + self.individual_train_losses[1])
            self.total_eval_loss = 0.5 * (self.individual_eval_losses[0] + self.individual_eval_losses[1])
        else:
            self.total_train_loss = self.individual_train_losses[0]
            self.total_eval_loss = self.individual_eval_losses[0]
def _build_loss(self, lstm_outputs):
        """
        Create:
            self.total_loss: total loss op for training
            self.softmax_W, softmax_b: the softmax variables
            self.next_token_id / _reverse: placeholders for gold input

        """
        batch_size = self.options['batch_size']
        unroll_steps = self.options['unroll_steps']

        n_tokens_vocab = self.options['n_tokens_vocab']

        # DEFINE next_token_id and *_reverse placeholders for the gold input
        def _get_next_token_placeholders(suffix):
            name = 'next_token_id' + suffix
            id_placeholder = tf.placeholder(DTYPE_INT,
                                            shape=(batch_size, unroll_steps),
                                            name=name)
            return id_placeholder

        # get the window and weight placeholders
        self.next_token_id = _get_next_token_placeholders('')
        if self.bidirectional:
            self.next_token_id_reverse = _get_next_token_placeholders(
                '_reverse')

        # DEFINE THE SOFTMAX VARIABLES
        # get the dimension of the softmax weights
        # softmax dimension is the size of the output projection_dim
        softmax_dim = self.options['lstm']['projection_dim']

        # the output softmax variables -- they are shared if bidirectional
        if self.share_embedding_softmax:
            # softmax_W is just the embedding layer
            self.softmax_W = self.embedding_weights

        with tf.variable_scope('softmax'), tf.device('/cpu:0'):
            # Glorit init (std=(1.0 / sqrt(fan_in))
            softmax_init = tf.random_normal_initializer(0.0, 1.0 / np.sqrt(softmax_dim))
            if not self.share_embedding_softmax:
                self.softmax_W = tf.get_variable(
                    'W', [n_tokens_vocab, softmax_dim],
                    dtype=DTYPE,
                    initializer=softmax_init
                )
            self.softmax_b = tf.get_variable(
                'b', [n_tokens_vocab],
                dtype=DTYPE,
                initializer=tf.constant_initializer(0.0))

        # now calculate losses
        # loss for each direction of the LSTM
        self.individual_train_losses = []
        self.individual_eval_losses = []

        if self.bidirectional:
            next_ids = [self.next_token_id, self.next_token_id_reverse]
        else:
            next_ids = [self.next_token_id]

        for id_placeholder, lstm_output_flat in zip(next_ids, lstm_outputs):
            # flatten the LSTM output and next token id gold to shape:
            # (batch_size * unroll_steps, softmax_dim)
            # Flatten and reshape the token_id placeholders
            next_token_id_flat = tf.reshape(id_placeholder, [-1, 1])

            with tf.control_dependencies([lstm_output_flat]):
                sampled_losses = tf.nn.sampled_softmax_loss(self.softmax_W, self.softmax_b,
                                                            next_token_id_flat, lstm_output_flat,
                                                            self.options['n_negative_samples_batch'],
                                                            self.options['n_tokens_vocab'],
                                                            num_true=1)

                # get the full softmax loss
                output_scores = tf.matmul(
                    lstm_output_flat,
                    tf.transpose(self.softmax_W)
                ) + self.softmax_b
                # NOTE: tf.nn.sparse_softmax_cross_entropy_with_logits
                #   expects unnormalized output since it performs the
                #   softmax internally
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                    logits=output_scores,
                    labels=tf.squeeze(next_token_id_flat, squeeze_dims=[1])
                )
            sampled_losses = tf.reshape(sampled_losses, [self.options['batch_size'], -1])
            losses = tf.reshape(losses, [self.options['batch_size'], -1])
            self.individual_train_losses.append(tf.reduce_mean(sampled_losses, axis=1))
            self.individual_eval_losses.append(tf.reduce_mean(losses, axis=1))

        # now make the total loss -- it's the train of the individual losses
        if self.bidirectional:
            self.total_train_loss = 0.5 * (self.individual_train_losses[0] + self.individual_train_losses[1])
            self.total_eval_loss = 0.5 * (self.individual_eval_losses[0] + self.individual_eval_losses[1])
        else:
            self.total_train_loss = self.individual_train_losses[0]
            self.total_eval_loss = self.individual_eval_losses[0]
Create:
            self.total_loss: total loss op for training
            self.softmax_W, softmax_b: the softmax variables
            self.next_token_id / _reverse: placeholders for gold input