tf.nn.nce_loss 函數應用案例

tf.nn.nce_loss(
    weights,
    biases,
    labels,
    inputs,
    num_sampled,
    num_classes,
    num_true=1,
    sampled_values=None,
    remove_accidental_hits=False,
    partition_strategy='mod',
    name='nce_loss'
)

 對於 nce_loss 的瞭解源於 word2vec，主要是通過 負採樣的方式減少 softmax 函數的計算，具體函數值講解，可以字節看疼我tensorflow，這裏提供一個簡單的案例，說明怎麼使用 nce_loss。

import numpy as np
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import math

train_loss_lst = []
train_accuracy_lst = []
test_accuracy_lst = []

def deepn(x):
    """
    Args:
        x: an input tensor with the dimensions (N_examples, 784), where 784 is the
        number of pixels in a standard MNIST image.
    Returns:
        y: (N_examples, 64)
    X(N_examples, 784) x w1(784, 64) + b1(64,) = y(N_examples, 64)
    """
    w1 = tf.Variable(initial_value=tf.truncated_normal(shape=[784, 64]),name="w1")
    b1 = tf.Variable(initial_value=tf.random_uniform(shape=[64,], minval=0, maxval=1))

    fc1 = tf.matmul(x, w1) + b1
    keep_prob = tf.placeholder(tf.float32)
    fc1_drop = tf.nn.dropout(fc1, keep_prob)
    return fc1_drop, keep_prob


def main():
    # Import data
    mnist = input_data.read_data_sets("/mnist_data", one_hot=True)
    X = tf.placeholder(tf.float32, [None, 784])
    y_ = tf.placeholder(tf.float32, [None, 10])
    y_idx = tf.placeholder(tf.float32, [None, 1])

    fc1_drop, keep_prob = deepn(X)

    num_sampled = 1
    vocabulary_size = 10
    embedding_size = 64

    nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size],
                                            stddev=1.0/math.sqrt(embedding_size)),
                        name="embed"
                        )
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
    loss = tf.reduce_mean(
        tf.nn.nce_loss(weights=nce_weights,
                       biases=nce_biases,
                       labels=y_idx,
                       inputs=fc1_drop,
                       num_sampled=num_sampled,
                       num_classes=vocabulary_size),
    )
    train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)

    output = tf.matmul(fc1_drop, tf.transpose(nce_weights)) + nce_biases
    correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(20000):
            print("num: %d"%i)
            batch = mnist.train.next_batch(20)
            idx_ = np.argmax(batch[1], axis=1)[:, np.newaxis].astype("float32")
            train_accuracy, train_loss, _ = sess.run([accuracy, loss, train_step],
                                                     feed_dict={X: batch[0],
                                                                y_: batch[1],
                                                                y_idx: idx_,
                                                                keep_prob: 1.0})
            print("loss: ", train_loss)
            print("train accuracy: ", train_accuracy)

            idx_ = np.argmax(mnist.test.labels, axis=1)[:, np.newaxis].astype("float32")
            test_accuracy = sess.run(accuracy,
                                     feed_dict={X: mnist.test.images,
                                                y_: mnist.test.labels,
                                                y_idx: idx_,
                                                keep_prob: 1.0})
            print("test accuracy: ", test_accuracy)
            train_loss_lst.append(train_loss)
            train_accuracy_lst.append(train_accuracy)
            test_accuracy_lst.append(test_accuracy)

def test():
    mnist = input_data.read_data_sets("./mnist_data", one_hot=True)
    X = tf.placeholder(tf.float32, [None, 784])
    fc1_drop, keep_prob = deepn(X)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(2):
            batch = mnist.train.next_batch(20)
            idx_ = np.argmax(batch[1], axis=1)[:, np.newaxis].astype("float32")
            fc1 = sess.run([fc1_drop], feed_dict={X: batch[0], keep_prob:1.0})
            print(np.array(fc1).shape)

main()
def summary(x, tag, path):
    """
    根據提供的 x 列表繪製曲線
    """
    print(x)
    loss = 0.0
    # tf.summary模塊的定義位於summary.py文件中，該文件中主要定義了在進行可視化將要用到的各種函數
    loss_summary = tf.Summary()
    # 調用tf.summary.Summary.Value子類
    loss_summary.value.add(tag=tag, simple_value=loss)  # tag就是待會產生的圖標名稱

    with tf.Session() as sess:
    # 生成一個寫日誌的writer，將當前tensorflow計算圖寫入日誌。
        summary_writer1 = tf.summary.FileWriter(path, sess.graph)
        tf.global_variables_initializer().run()
        for i in range(len(x)):
            # 固定用法，具體爲什麼我也不懂
            loss_summary.value[0].simple_value = x[i]
            summary_writer1.add_summary(loss_summary, i)

summary(train_loss_lst, tag="loss", path="./train_loss")
summary(train_accuracy_lst, tag="accuracy", path="./train_accuracy")
summary(test_accuracy_lst, tag="accuracy", path="./test_accuracy")