參考原文鏈接
Making new layers and models via subclassing
1. 準備工作
import tensorflow as tf
from tensorflow import keras
2. The Layer class: 結合了權重和一些計算
- Layer類整合了Layer的權重和該層的前向傳播
- 下面舉一個線性全連接的例子，並且我們可以使用一個tensor來使用該layer進行計算
class Linear(keras.layers.Layer):
def __init__(self, units=32, input_dim=32):
super(Linear, self).__init__()
# 參數w
w_init = tf.random_normal_initializer()
self.w = tf.Variable(
initial_value=w_init(shape=(input_dim, units), dtype="float32"),
trainable=True,
)
# 參數b
b_init = tf.zeros_initializer()
self.b = tf.Variable(
initial_value=b_init(shape=(units,), dtype="float32"), trainable=True
)
def call(self, inputs):
# 返回前向傳播的結果
return tf.matmul(inputs, self.w) + self.b
# use it
x = tf.ones((2, 2))
linear_layer = Linear(4, 2) # 實例化該類
y = linear_layer(x) # 調用（進行前向傳播）
print(y)
# result
tf.Tensor(
[[0.04719363 0.01185325 0.08139521 0.03705199]
[0.04719363 0.01185325 0.08139521 0.03705199]], shape=(2, 4), dtype=float32)
- 注意，權值w和b被設置爲層屬性後，層會自動跟蹤該參數
- 當然，你也可以選擇自己更加快捷的方式來添加權重：self.add_weight()
class Linear(keras.layers.Layer):
def __init__(self, units=32, input_dim=32):
super(Linear, self).__init__()
# 相比前一個聲明參數更方便
self.w = self.add_weight(
shape=(input_dim, units), initializer="random_normal", trainable=True
)
self.b = self.add_weight(shape=(units,), initializer="zeros", trainable=True)
def call(self, inputs):
return tf.matmul(inputs, self.w) + self.b
x = tf.ones((2, 2))
linear_layer = Linear(4, 2)
y = linear_layer(x)
print(y)
3. Layers 可以有non-trainable的參數
- 除了可訓練的權重，也可以向Layer添加非訓練的權重，這個權重不會參與到反向傳播的過程(一般爲常量)，代碼如下：
class ComputeSum(keras.layers.Layer):
def __init__(self, input_dim):
super(ComputeSum, self).__init__()
self.total = tf.Variable(initial_value=tf.zeros((input_dim,)), trainable=False)
def call(self, inputs):
self.total.assign_add(tf.reduce_sum(inputs, axis=0))
return self.total
x = tf.ones((2, 2))
my_sum = ComputeSum(2)
y = my_sum(x)
print(y.numpy())
y = my_sum(x)
print(y.numpy())
- 非訓練的權重也會作爲Layer.weight的一部分，但是它會被分爲非訓練權重的類別
print("weights:", len(my_sum.weights))
print("non-trainable weights:", len(my_sum.non_trainable_weights))
# It's not included in the trainable weights:
print("trainable_weights:", my_sum.trainable_weights)
# resultda
weights: 1
non-trainable weights: 1
trainable_weights: []
4. Best practice: 延遲創建權重，直到知道輸入的形狀
-
既然Keras文檔強力推薦，那麼，我們在以後寫代碼時，儘量遵循該原則
-
在上面的代碼裏，我們類中都會有一個參數input_dim，它用來計算權重的shape。但是在很多時候，我們事先並不知道input_dim，這時候就要考慮延遲創建權重，並直到input的value變爲已知的時候再去創建，例如實例化玩一個Layer後，將tensor傳入後，就會自動創建。
-
在KerasAPI中，推薦使用在Layer類中的bulid(self,input_shape)方法中實現參數的創建，就像下面的代碼一樣：
class Linear(keras.layers.Layer):
def __init__(self, units=32):
super(Linear, self).__init__()
self.units = units
def build(self, input_shape):
self.w = self.add_weight(
shape=(input_shape[-1], self.units),
initializer="random_normal",
trainable=True,
)
self.b = self.add_weight(
shape=(self.units,), initializer="random_normal", trainable=True
)
def call(self, inputs):
return tf.matmul(inputs, self.w) + self.b
- Layer中的__call__()方法會在你第一次調用Layer時運行build方法
# At instantiation, we don't know on what inputs this is going to get called
linear_layer = Linear(32) # 32 爲units
# The layer's weights are created dynamically the first time the layer is called
y = linear_layer(x)
5. The add_loss() method
# A layer that creates an activity regularization loss
class ActivityRegularizationLayer(keras.layers.Layer):
def __init__(self, rate=1e-2):
super(ActivityRegularizationLayer, self).__init__()
self.rate = rate
def call(self, inputs):
self.add_loss(self.rate * tf.reduce_sum(inputs))
return inputs
- 這些loss(包括由任何內層創建的損失)可以通過layer.losses進行檢索，這個屬性會在每一層的開始處重置到頂層，所以，損耗總是包含上次向前傳遞時創建的損耗值(loss的累加)，這相當有用，尤其是在寫VAE時。
class OuterLayer(keras.layers.Layer):
def __init__(self):
super(OuterLayer, self).__init__()
self.activity_reg = ActivityRegularizationLayer(1e-2)
def call(self, inputs):
return self.activity_reg(inputs)
layer = OuterLayer()
assert len(layer.losses) == 0 # No losses yet since the layer has never been called
_ = layer(tf.zeros(1, 1))
assert len(layer.losses) == 1 # We created one loss value
# `layer.losses` gets reset at the start of each __call__
_ = layer(tf.zeros(1, 1))
assert len(layer.losses) == 1 # This is the loss created during the call above
- loss也可以爲權重的正則化
class OuterLayerWithKernelRegularizer(keras.layers.Layer):
def __init__(self):
super(OuterLayerWithKernelRegularizer, self).__init__()
self.dense = keras.layers.Dense(
32, kernel_regularizer=tf.keras.regularizers.l2(1e-3)
)
def call(self, inputs):
return self.dense(inputs)
layer = OuterLayerWithKernelRegularizer()
_ = layer(tf.zeros((1, 1)))
# This is `1e-3 * sum(layer.dense.kernel ** 2)`,
# created by the `kernel_regularizer` above.
print(layer.losses)
# result
[<tf.Tensor: shape=(), dtype=float32, numpy=0.0016333066>]
- 在編寫訓練循環時應該考慮這些損失，如下所示，這表明我們在Layer上定義的損失會被加入主損失函數中一起進行梯度下降。
# Instantiate an optimizer.
optimizer = tf.keras.optimizers.SGD(learning_rate=1e-3)
loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
# Iterate over the batches of a dataset.
for x_batch_train, y_batch_train in train_dataset:
with tf.GradientTape() as tape:
logits = layer(x_batch_train) # Logits for this minibatch
# Loss value for this minibatch
loss_value = loss_fn(y_batch_train, logits)
# Add extra losses created during this forward pass:
loss_value += sum(model.losses)
grads = tape.gradient(loss_value, model.trainable_weights)
optimizer.apply_gradients(zip(grads, model.trainable_weights))
- 這些損失還可以與fit()無縫地協同工作(如果有損失，它們會自動彙總並添加到主要損失中):
import numpy as np
inputs = keras.Input(shape=(3,))
outputs = ActivityRegularizationLayer()(inputs)
model = keras.Model(inputs, outputs)
# If there is a loss passed in `compile`, thee regularization
# losses get added to it
model.compile(optimizer="adam", loss="mse")
model.fit(np.random.random((2, 3)), np.random.random((2, 3)))
# It's also possible not to pass any loss in `compile`,
# since the model already has a loss to minimize, via the `add_loss`
# call during the forward pass!
model.compile(optimizer="adam")
model.fit(np.random.random((2, 3)), np.random.random((2, 3)))
# result
1/1 [==============================] - 0s 658us/step - loss: 0.1063
1/1 [==============================] - 0s 873us/step - loss: 0.0202
<tensorflow.python.keras.callbacks.History at 0x147cd9410>
6. The add_metric() method
7. You can optionally enable serialization on your layers
class Linear(keras.layers.Layer):
def __init__(self, units=32, **kwargs):
super(Linear, self).__init__(**kwargs)
self.units = units
def build(self, input_shape):
self.w = self.add_weight(
shape=(input_shape[-1], self.units),
initializer="random_normal",
trainable=True,
)
self.b = self.add_weight(
shape=(self.units,), initializer="random_normal", trainable=True
)
def call(self, inputs):
return tf.matmul(inputs, self.w) + self.b
def get_config(self):
config = super(Linear, self).get_config()
config.update({"units": self.units})
return config
layer = Linear(64)
config = layer.get_config()
print(config)
new_layer = Linear.from_config(config)
# result
{'name': 'linear_8', 'trainable': True, 'dtype': 'float32', 'units': 64}
8. The Model class
-
通常，您將使用Layer類來定義內部計算塊，並使用Model類來定義外部模型——您將訓練的對象。
-
因此，如果您想知道，“我應該使用Layer類還是Model類?”，您可以問自己:我需要在它上面調用fit()嗎?我需要調用save()嗎?如果是這樣，那就選擇模型。如果不是(因爲你的類在一個更大的系統中只是一個塊，或者因爲你自己在寫訓練和保存代碼)，使用Layer
9. 總結
from tensorflow.keras import layers
class Sampling(layers.Layer):
"""Uses (z_mean, z_log_var) to sample z, the vector encoding a digit."""
def call(self, inputs):
z_mean, z_log_var = inputs
batch = tf.shape(z_mean)[0]
dim = tf.shape(z_mean)[1]
epsilon = tf.keras.backend.random_normal(shape=(batch, dim))
return z_mean + tf.exp(0.5 * z_log_var) * epsilon
class Encoder(layers.Layer):
"""Maps MNIST digits to a triplet (z_mean, z_log_var, z)."""
def __init__(self, latent_dim=32, intermediate_dim=64, name="encoder", **kwargs):
super(Encoder, self).__init__(name=name, **kwargs)
self.dense_proj = layers.Dense(intermediate_dim, activation="relu")
self.dense_mean = layers.Dense(latent_dim)
self.dense_log_var = layers.Dense(latent_dim)
self.sampling = Sampling()
def call(self, inputs):
x = self.dense_proj(inputs)
z_mean = self.dense_mean(x)
z_log_var = self.dense_log_var(x)
z = self.sampling((z_mean, z_log_var))
return z_mean, z_log_var, z
class Decoder(layers.Layer):
"""Converts z, the encoded digit vector, back into a readable digit."""
def __init__(self, original_dim, intermediate_dim=64, name="decoder", **kwargs):
super(Decoder, self).__init__(name=name, **kwargs)
self.dense_proj = layers.Dense(intermediate_dim, activation="relu")
self.dense_output = layers.Dense(original_dim, activation="sigmoid")
def call(self, inputs):
x = self.dense_proj(inputs)
return self.dense_output(x)
class VariationalAutoEncoder(keras.Model):
"""Combines the encoder and decoder into an end-to-end model for training."""
def __init__(
self,
original_dim,
intermediate_dim=64,
latent_dim=32,
name="autoencoder",
**kwargs
):
super(VariationalAutoEncoder, self).__init__(name=name, **kwargs)
self.original_dim = original_dim
self.encoder = Encoder(latent_dim=latent_dim, intermediate_dim=intermediate_dim)
self.decoder = Decoder(original_dim, intermediate_dim=intermediate_dim)
def call(self, inputs):
z_mean, z_log_var, z = self.encoder(inputs)
reconstructed = self.decoder(z)
# Add KL divergence regularization loss.
kl_loss = -0.5 * tf.reduce_mean(
z_log_var - tf.square(z_mean) - tf.exp(z_log_var) + 1
)
self.add_loss(kl_loss)
return reconstructed
(x_train, _), _ = tf.keras.datasets.mnist.load_data()
x_train = x_train.reshape(60000, 784).astype("float32") / 255
train_dataset = tf.data.Dataset.from_tensor_slices(x_train)
train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)
- 定義fit
vae = VariationalAutoEncoder(784, 64, 32)
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
vae.compile(optimizer, loss=tf.keras.losses.MeanSquaredError())
vae.fit(x_train, x_train, epochs=2, batch_size=64)
# result
Epoch 1/2
938/938 [==============================] - 1s 1ms/step - loss: 0.0746
Epoch 2/2
938/938 [==============================] - 1s 1ms/step - loss: 0.0676
<tensorflow.python.keras.callbacks.History at 0x138a5ccd0>
10. 一點建議
- 既然選擇了python，就要用面向對象，面向過程的思想就往後稍一稍。