# 【Tensorflow】自主实现部分连接层（Partial Connect Layer）

## 0x00 前言

（即使增加一个mask把非计算部分都设为 $0$$0$，乘以 $0$$0$ 的乘法也不要做比较好），

（因为懒得更新TF，各位如果新配的TF环境完全可以试着去官方文档里搜搜 sparse_ 开头的各种方法）

## 0x01 构造思路

class EmbLayer(object):
"""embedding layer"""
def __init__(self, word_emb_shape, word_embeddings_path=None, voc_path=None, partition_strategy='mod', validate_indices=True, max_norm=None, weight_decay=None, stop_gradient=False, show_no_word=True, name='emb'):
# params
self.partition_strategy = partition_strategy
self.validate_indices = validate_indices
self.word_embeddings = None
self.max_norm = max_norm
self.name = name

with tf.name_scope('{}_def'.format(self.name)):
scale = math.sqrt(2.0 / np.prod(word_emb_shape[-1]))
self.word_embeddings = scale * np.random.standard_normal(size=word_emb_shape)

if word_embeddings_path:
assert voc_path is not None
for idx in range(word_emb_shape[0]):
word = idx2word[idx]
if word in word2vec:
self.word_embeddings[idx] = scale * 0.1 * word2vec[word][:word_emb_shape[1]]
elif show_no_word:
print('word2vec no word {}: {}'.format(idx, word.encode('utf-8')))

self.word_embeddings = tf.Variable(
initial_value=self.word_embeddings, dtype=tf.float32, name='word_embeddings')

if weight_decay:
'losses', tf.multiply(tf.nn.l2_loss(self.word_embeddings),
weight_decay, name='weight_decay_loss'))

def __call__(self, ids):
with tf.name_scope('{}_cal'.format(self.name)):
outputs = tf.nn.embedding_lookup(self.word_embeddings, ids,
partition_strategy=self.partition_strategy,
validate_indices=self.validate_indices,
max_norm=self.max_norm,
name=self.name)
return outputs
class FulLayer(object):
"""Full Connect Layer"""
def __init__(self, input_dim, output_dim, activation_fn=tf.sigmoid, weight_decay=None, name="ful"):
weight_shape = (input_dim, output_dim)
self.activation_fn = activation_fn
self.name = name

with tf.name_scope('{}_def'.format(self.name)):
# weight matrix
scale = math.sqrt(2.0 / np.prod(weight_shape[:-1]))
init_value = scale * np.random.standard_normal(size=weight_shape)
self.weight = tf.Variable(init_value, dtype=tf.float32, name='weight')

if weight_decay:
'losses', tf.multiply(tf.nn.l2_loss(self.weight),
weight_decay, name='weight_decay_loss_w'))

# bias vector
self.bias = tf.Variable(
initial_value=tf.constant(0.0, shape=[output_dim]),
dtype=tf.float32, name='bias')
if weight_decay:
'losses', tf.multiply(tf.nn.l2_loss(self.bias),
weight_decay, name='weight_decay_loss_b'))

def __call__(self, inputs):
with tf.name_scope('{}_cal'.format(self.name)):
shape = tf.concat([tf.shape(inputs)[:-1], [tf.shape(self.weight)[-1]]], 0)
# shape = tf.concat([tf.shape(inputs)[:-1], [self.weight.shape[-1]]], 0)

inputs = tf.reshape(inputs, [-1, tf.shape(inputs)[-1]])
if self.activation_fn is not None:
outputs = self.activation_fn(outputs)

outputs = tf.reshape(outputs, shape)
return outputs

Full Connect Layer 则是维护一个Weight矩阵和一个bias向量，就是常说的 $y=Wx+b$$y=Wx+b$ 中的那个 $W$$W$ $b$$b$

## 0x02 Source Code

class PartialLayer(object):
"""Partial Connect Layer"""
def __init__(self, input_dim, output_dim, partial_dim, activation_fn=tf.sigmoid, weight_decay=None, name="par"):
self.partial_dim = partial_dim
self.activation_fn = activation_fn
self.weight_shape = (input_dim, output_dim)
self.name = name

with tf.name_scope('{}_def'.format(self.name)):
# weight matrix
scale = math.sqrt(2.0 / np.prod(self.weight_shape[:-1]))
init_value = scale * np.random.standard_normal(size=self.weight_shape)
self.weight = tf.Variable(init_value, dtype=tf.float32, name='weight')

# bias vector
self.bias = tf.Variable(
initial_value=tf.constant(0.0, shape=[output_dim]),
dtype=tf.float32, name='bias')

if weight_decay:
'losses', tf.multiply(tf.nn.l2_loss(self.weight),
weight_decay, name='weight_decay_loss_w'))
'losses', tf.multiply(tf.nn.l2_loss(self.bias),
weight_decay, name='weight_decay_loss_b'))

self.transposed_weight = tf.transpose(self.weight)
self.transposed_bias = tf.expand_dims(self.bias, -1)

def get_partial_weight(self, targets):
return tf.nn.embedding_lookup(
self.transposed_weight, targets,
partition_strategy='mod',
validate_indices=True,
max_norm=None,
name='partial_weight'
)

def get_partial_bias(self, targets):
return tf.nn.embedding_lookup(
self.transposed_bias, targets,
partition_strategy='mod',
validate_indices=True,
max_norm=None,
name='partial_bias'
)

def __call__(self, inputs, targets):
""" global weight is lstm_dim*2, n_words :param inputs: batch, seg_len, lstm_dim*2 :param targets: batch, seg_len, can_len :return: batch, seg_len, can_len """
with tf.name_scope('{}_cal'.format(self.name)):
inputs = tf.expand_dims(inputs, -1)
# print(inputs.shape)

partial_weight = self.get_partial_weight(targets)
partial_bias = self.get_partial_bias(targets)

# batch*seg_len, can_len
# print (inputs.shape, partial_weight.shape, partial_bias.shape)
# print (type(inputs), type(partial_weight), type(partial_bias))

if self.activation_fn is not None:
outputs = self.activation_fn(outputs)

# batch, seg_len, can_len
outputs = tf.reshape(outputs, tf.shape(outputs)[:-1])
return outputs
# layer initial in network's __init__()
from path.to.my.utils import options
self.partial_layer = PartialLayer(input_dim=2 * options.get('lstm_dim'),
output_dim=options.get('n_words'),
partial_dim=options.get('max_can_len'),
activation_fn=None,
weight_decay=self.options.get('weight_decay'),
name='pc_layer')
# example network construction.
def get_network(self):
# [batch, seg_len + 2] -> [batch, seg_len + 2, emb_dim]
word_emb = self.emb_layer(self.input_data)

# [batch, seg_len(+2), emb_dim] -> [batch, seg_len, lstm_dim*2]
forward_hidden, backward_hidden = self.lstm_layer(word_emb)
context_hidden = tf.concat([forward_hidden, backward_hidden])

# [batch, seg_len, lstm_dim*2] -> [batch, seg_len, can_len]
partial_hidden = self.partial_layer(context_hidden, self.candidates)