deeppavlov/core/layers/tf_csoftmax_attention.py
def attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, key, cum_att):
    """ It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]
    """
    with tf.name_scope('attention_step'):
        sketch_dims = hidden_for_sketch.get_shape().as_list()
        batch_size = sketch_dims[0]
        num_tokens = sketch_dims[1]
        hidden_size = sketch_dims[2]
        attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list()
        attn_alignment_hidden_size = attn_alignment_dims[2]

        repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1,num_tokens, 1))
        concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-1)


        concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2*hidden_size]) # dirty trick
        reduce_mem = tf.layers.dense(concat_mem, hidden_size)

        projected_key = tf.layers.dense(key, hidden_size)
        t_key = tf.reshape(projected_key,[-1, hidden_size, 1])

        score = tf.reshape(tf.matmul(reduce_mem, t_key), [-1, num_tokens])

        inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-1, num_tokens])
        att = csoftmax(score, inv_cum_att)

        t_reduce_mem = tf.transpose(reduce_mem, [0,2,1])
        t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0,2,1])

        r_att = tf.reshape(att, [-1, num_tokens, 1])

        next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-1)
        aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-1)
    return next_sketch, att, aligned_hidden_sketch
def attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, key, cum_att):
    """ It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]
    """
    with tf.name_scope('attention_step'):
        sketch_dims = hidden_for_sketch.get_shape().as_list()
        batch_size = sketch_dims[0]
        num_tokens = sketch_dims[1]
        hidden_size = sketch_dims[2]
        attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list()
        attn_alignment_hidden_size = attn_alignment_dims[2]

        repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1,num_tokens, 1))
        concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-1)


        concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2*hidden_size]) # dirty trick
        reduce_mem = tf.layers.dense(concat_mem, hidden_size)

        projected_key = tf.layers.dense(key, hidden_size)
        t_key = tf.reshape(projected_key,[-1, hidden_size, 1])

        score = tf.reshape(tf.matmul(reduce_mem, t_key), [-1, num_tokens])

        inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-1, num_tokens])
        att = csoftmax(score, inv_cum_att)

        t_reduce_mem = tf.transpose(reduce_mem, [0,2,1])
        t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0,2,1])

        r_att = tf.reshape(att, [-1, num_tokens, 1])

        next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-1)
        aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-1)
    return next_sketch, att, aligned_hidden_sketch
It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]