tensorflow深度學習代碼筆記

01、手寫數字識別初體驗

# 01手寫數字識別初體驗

import os

# ---------------準備部分--------------------------------------------------------


# 將日誌等級設置爲2，減少無關日誌輸出
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, datasets

# 從互聯網下載mnist數據集，分爲 訓練集：(x, y),驗證集： (x_val, y_val)
# x: [60k, 28, 28]；  x_val： [10k, 28, 28]
# y: [60k]； y_val： [10k]
(x, y), (x_val, y_val) = datasets.mnist.load_data()
x = tf.convert_to_tensor(x, dtype=tf.float32) / 255.  # 將x轉化爲tensor，同時x: [0~255] => [0~1.]
y = tf.convert_to_tensor(y, dtype=tf.int32)  # 將y轉化爲tensor
y = tf.one_hot(y, depth=10)  # 將y進行one_hot編碼 如：[3]-->[0 0 0 1 0 0 0 0 0 0]
print(x.shape, y.shape)
train_dataset = tf.data.Dataset.from_tensor_slices((x, y))  # 創建一個數據集，元素爲(x, y)，按照第一個維度進行切片
train_dataset = train_dataset.batch(200)  # 將每200個切片組合爲一個batch，以便後續並行訓練
# 創建網絡結構 512-relu->256-relu->10
model = keras.Sequential([
    layers.Dense(512, activation='relu'),
    layers.Dense(256, activation='relu'),
    layers.Dense(10)])
# 創建優化器，使用隨機梯度下降優化器，學習率0.001
optimizer = optimizers.SGD(learning_rate=0.001)


# ---------------準備部分--------------------------------------------------------

# ---------------訓練部分---------------------------------------------------------

def train_epoch(epoch):
    # Step4.loop，每次循環取出一個batch [batch,28,28]
    for step, (x, y) in enumerate(train_dataset):
        # 創建梯度帶，來鏈接需要求取梯度的變量和函數
        with tf.GradientTape() as tape:
            # [b, 28, 28] => [b, 784]
            x = tf.reshape(x, (-1, 28 * 28))
            # Step1. compute output
            # [b, 784] => [b, 10]
            out = model(x)
            # Step2. compute loss
            loss = tf.reduce_sum(tf.square(out - y)) / x.shape[0]

        # Step3. optimize and update w1, w2, w3, b1, b2, b3
        grads = tape.gradient(loss, model.trainable_variables)
        # w' = w - lr * grad
        optimizer.apply_gradients(zip(grads, model.trainable_variables))

        if step % 100 == 0:
            print(epoch, step, 'loss:', loss.numpy())


def train():
    # 循環訓練30輪後結束
    for epoch in range(30):
        train_epoch(epoch)


if __name__ == '__main__':
    train()

輸出：

C:\Users\Administrator\anaconda3\python.exe "C:\Program Files\JetBrains\PyCharm 2020.1.1\plugins\python\helpers\pydev\pydevconsole.py" --mode=client --port=55240
import sys; print('Python %s on %s' % (sys.version, sys.platform))
sys.path.extend(['E:\\桌面文件\\WorkStation', 'E:/桌面文件/WorkStation'])
Python 3.7.6 (default, Jan  8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)]
Type 'copyright', 'credits' or 'license' for more information
IPython 7.12.0 -- An enhanced Interactive Python. Type '?' for help.
PyDev console: using IPython 7.12.0
Python 3.7.6 (default, Jan  8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)] on win32
In[2]: runfile('E:/桌面文件/WorkStation/DL/01_mnist.py', wdir='E:/桌面文件/WorkStation/DL')
(60000, 28, 28) (60000, 10)
0 0 loss: 1.490922
0 100 loss: 0.9255758
0 200 loss: 0.7562221
1 0 loss: 0.64366394
1 100 loss: 0.68554175
1 200 loss: 0.59353197
2 0 loss: 0.54019254
2 100 loss: 0.598027
2 200 loss: 0.5216136
3 0 loss: 0.4867071
3 100 loss: 0.5463192
3 200 loss: 0.4771847
4 0 loss: 0.4508601
4 100 loss: 0.51056063
Process finished with exit code 0

02、前向傳播

# 前向傳播
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets

# x: [60k, 28, 28], [10, 28, 28]
# y: [60k], [10k]
(x, y), (x_test, y_test) = datasets.mnist.load_data()
# x: [0~255] => [0~1.]
x = tf.convert_to_tensor(x, dtype=tf.float32) / 255.
y = tf.convert_to_tensor(y, dtype=tf.int32)
x_test = tf.convert_to_tensor(x_test, dtype=tf.float32) / 255.
y_test = tf.convert_to_tensor(y_test, dtype=tf.int32)

print(x.shape, y.shape, x.dtype, y.dtype)
# 打印x,y的取值範圍
print(tf.reduce_min(x), tf.reduce_max(x))
print(tf.reduce_min(y), tf.reduce_max(y))

train_db = tf.data.Dataset.from_tensor_slices((x, y)).batch(128)  # 訓練數據集切片，batch爲128
test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(128)  # 測試數據集切片，batch爲128

train_iter = iter(train_db)  # 創建一個迭代器
sample = next(train_iter)  # 從迭代器返回下一個項目
print('batch:', sample[0].shape, sample[1].shape)  # 打印數據集中x,y的形狀，x，y對應sample[0].shape, sample[1].shape

# [b, 784] => [b, 256] => [b, 128] => [b, 10]
# [dim_in, dim_out], [dim_out]
w1 = tf.Variable(tf.random.truncated_normal([784, 256], stddev=0.1))  # 使用正態分佈截斷隨機數初始化w1
b1 = tf.Variable(tf.zeros([256]))
w2 = tf.Variable(tf.random.truncated_normal([256, 128], stddev=0.1))
b2 = tf.Variable(tf.zeros([128]))
w3 = tf.Variable(tf.random.truncated_normal([128, 10], stddev=0.1))
b3 = tf.Variable(tf.zeros([10]))

lr = 1e-3

for epoch in range(10):  # iterate db for 10
    for step, (x, y) in enumerate(train_db):  # for every batch
        # x:[128, 28, 28]
        # y: [128]

        # [b, 28, 28] => [b, 28*28]
        x = tf.reshape(x, [-1, 28 * 28])

        with tf.GradientTape() as tape:  # tf.Variable
            # x: [b, 28*28]
            # h1 = x@w1 + b1
            # [b, 784]@[784, 256] + [256] => [b, 256] + [256] => [b, 256] + [b, 256]
            h1 = x @ w1 + tf.broadcast_to(b1, [x.shape[0], 256])
            h1 = tf.nn.relu(h1)
            # [b, 256] => [b, 128]
            h2 = h1 @ w2 + b2
            h2 = tf.nn.relu(h2)
            # [b, 128] => [b, 10]
            out = h2 @ w3 + b3

            # compute loss
            # out: [b, 10]
            # y: [b] => [b, 10]
            y_onehot = tf.one_hot(y, depth=10)

            # mse = mean(sum(y-out)^2)
            # [b, 10]
            loss = tf.square(y_onehot - out)
            # mean: scalar
            loss = tf.reduce_mean(loss)

        # compute gradients
        grads = tape.gradient(loss, [w1, b1, w2, b2, w3, b3])
        # print(grads)
        # w1 = w1 - lr * w1_grad
        w1.assign_sub(lr * grads[0])
        b1.assign_sub(lr * grads[1])
        w2.assign_sub(lr * grads[2])
        b2.assign_sub(lr * grads[3])
        w3.assign_sub(lr * grads[4])
        b3.assign_sub(lr * grads[5])

        if step % 100 == 0:
            print(epoch, step, 'loss:', float(loss))

    # test/evluation
    # [w1, b1, w2, b2, w3, b3]
    total_correct, total_num = 0, 0
    for step, (x, y) in enumerate(test_db):
        # [b, 28, 28] => [b, 28*28]
        x = tf.reshape(x, [-1, 28 * 28])

        # [b, 784] => [b, 256] => [b, 128] => [b, 10]
        h1 = tf.nn.relu(x @ w1 + b1)
        h2 = tf.nn.relu(h1 @ w2 + b2)
        out = h2 @ w3 + b3

        # out: [b, 10] ~ R
        # prob: [b, 10] ~ [0, 1]
        prob = tf.nn.softmax(out, axis=1) # softmax將輸出值映射到[0,1]，同時保證所有輸出之和=1
        # [b, 10] => [b]
        # int64!!!
        pred = tf.argmax(prob, axis=1) # 默認返回axis=1上最大值的下標,即最大值的位置(0-9)  
        pred = tf.cast(pred, dtype=tf.int32) # 數據類型轉換
        # y: [b]
        # [b], int32
        # print(pred.dtype, y.dtype)
        correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) # tf.equal，逐元素比較，返回True or False
        correct = tf.reduce_sum(correct) # 求得correct所有維度元素的和

        total_correct += int(correct)# 累加識別正確的圖片個數
        total_num += x.shape[0]# 累加訓練圖片的個數

    acc = total_correct / total_num # 識別精度
    print('test acc:', acc)

OUT輸出：

C:\Users\Administrator\anaconda3\python.exe E:/桌面文件/WorkStation/DL/02_forward.py
(60000, 28, 28) (60000,) <dtype: 'float32'> <dtype: 'int32'>
tf.Tensor(0.0, shape=(), dtype=float32) tf.Tensor(1.0, shape=(), dtype=float32)
tf.Tensor(0, shape=(), dtype=int32) tf.Tensor(9, shape=(), dtype=int32)
batch: (128, 28, 28) (128,)
0 0 loss: 0.3498707711696625
0 100 loss: 0.21532559394836426
0 200 loss: 0.19365337491035461
0 300 loss: 0.16339325904846191
0 400 loss: 0.16480329632759094
test acc: 0.1829
1 0 loss: 0.15368646383285522
1 100 loss: 0.15009422600269318
1 200 loss: 0.14896038174629211
1 300 loss: 0.13545574247837067
1 400 loss: 0.1392597258090973
test acc: 0.235
2 0 loss: 0.13064464926719666
2 100 loss: 0.12948887050151825
2 200 loss: 0.12855729460716248
2 300 loss: 0.11955704540014267
2 400 loss: 0.12419591099023819
test acc: 0.2771
3 0 loss: 0.11611510813236237
3 100 loss: 0.1166902557015419
3 200 loss: 0.11538122594356537
3 300 loss: 0.10896680504083633
3 400 loss: 0.11403544247150421
test acc: 0.3193
4 0 loss: 0.10608117282390594
4 100 loss: 0.10784359276294708
4 200 loss: 0.106026291847229
4 300 loss: 0.10129009187221527
4 400 loss: 0.10656853020191193
test acc: 0.3542
5 0 loss: 0.0986492857336998
5 100 loss: 0.10125251859426498
5 200 loss: 0.09900902211666107
5 300 loss: 0.09544717520475388
5 400 loss: 0.10088589042425156
test acc: 0.3892
6 0 loss: 0.09289230406284332
6 100 loss: 0.09609203040599823
6 200 loss: 0.09353049099445343
6 300 loss: 0.0908334031701088
6 400 loss: 0.09633533656597137
test acc: 0.4165
7 0 loss: 0.08833207190036774
7 100 loss: 0.09188317507505417
7 200 loss: 0.08912365138530731
7 300 loss: 0.08707904070615768
7 400 loss: 0.09258817881345749
test acc: 0.4424
8 0 loss: 0.08455916494131088
8 100 loss: 0.08837546408176422
8 200 loss: 0.08549362421035767
8 300 loss: 0.08389302343130112
8 400 loss: 0.08944382518529892
test acc: 0.4656
9 0 loss: 0.081412173807621
9 100 loss: 0.0853889137506485
9 200 loss: 0.08242954313755035
9 300 loss: 0.0811467319726944
9 400 loss: 0.08672622591257095
test acc: 0.4903

Process finished with exit code 0

03、張量實戰

import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets, layers, optimizers

print(tf.__version__)


def preprocess(x, y):
    """

    :param x:
    :param y:
    :return:
    """
    # [b, 28, 28], [b]
    x = tf.cast(x, dtype=tf.float32) / 255.
    x = tf.reshape(x, [-1, 28 * 28])
    y = tf.cast(y, dtype=tf.int32)
    y = tf.one_hot(y, depth=10)

    return x, y


(x, y), (x_test, y_test) = datasets.mnist.load_data()
print('x:', x.shape, 'y:', y.shape, 'x test:', x_test.shape, 'y test:', y_test)
train_db = tf.data.Dataset.from_tensor_slices((x, y))
train_db = train_db.shuffle(60000).batch(128).map(preprocess).repeat(30)
# 其中map(preprocess)對每個train_db進行preprocess處理；repeat 重複次數，相當於訓練集疊加30次成爲新的訓練集

test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test))
test_db = test_db.shuffle(10000).batch(128).map(preprocess)
x, y = next(iter(train_db))
print('train sample:', x.shape, y.shape)


# print(x[0], y[0])


def main():
    # learning rate
    lr = 1e-3

    # 784 => 512
    w1, b1 = tf.Variable(tf.random.truncated_normal([784, 512], stddev=0.1)), tf.Variable(tf.zeros([512]))
    # 512 => 256
    w2, b2 = tf.Variable(tf.random.truncated_normal([512, 256], stddev=0.1)), tf.Variable(tf.zeros([256]))
    # 256 => 10
    w3, b3 = tf.Variable(tf.random.truncated_normal([256, 10], stddev=0.1)), tf.Variable(tf.zeros([10]))

    for step, (x, y) in enumerate(train_db):

        # [b, 28, 28] => [b, 784]
        x = tf.reshape(x, (-1, 784))

        with tf.GradientTape() as tape:

            # layer1.
            h1 = x @ w1 + b1
            h1 = tf.nn.relu(h1)
            # layer2
            h2 = h1 @ w2 + b2
            h2 = tf.nn.relu(h2)
            # output
            out = h2 @ w3 + b3
            # out = tf.nn.relu(out)

            # compute loss
            # [b, 10] - [b, 10]
            loss = tf.square(y - out)
            # [b, 10] => [b]
            loss = tf.reduce_mean(loss, axis=1)
            # [b] => scalar
            loss = tf.reduce_mean(loss)

        # compute gradient
        grads = tape.gradient(loss, [w1, b1, w2, b2, w3, b3])
        # for g in grads:
        #     print(tf.norm(g))
        # update w' = w - lr*grad
        # zip()將元素合併爲新序列
        for p, g in zip([w1, b1, w2, b2, w3, b3], grads):
            p.assign_sub(lr * g)

        # print
        if step % 100 == 0:
            print(step, 'loss:', float(loss))

        # evaluate
        if step % 500 == 0:
            total, total_correct = 0., 0

            for step, (x, y) in enumerate(test_db):
                # layer1.
                h1 = x @ w1 + b1
                h1 = tf.nn.relu(h1)
                # layer2
                h2 = h1 @ w2 + b2
                h2 = tf.nn.relu(h2)
                # output
                out = h2 @ w3 + b3
                # [b, 10] => [b]
                pred = tf.argmax(out, axis=1)
                # convert one_hot y to number y
                y = tf.argmax(y, axis=1)
                # bool type
                correct = tf.equal(pred, y)
                # bool tensor => int tensor => numpy
                total_correct += tf.reduce_sum(tf.cast(correct, dtype=tf.int32)).numpy()
                total += x.shape[0]

            print(step, 'Evaluate Acc:', total_correct / total)


if __name__ == '__main__':
    main()

OUT：

C:\Users\Administrator\anaconda3\python.exe "C:\Program Files\JetBrains\PyCharm 2020.1.1\plugins\python\helpers\pydev\pydevconsole.py" --mode=client --port=59947
import sys; print('Python %s on %s' % (sys.version, sys.platform))
sys.path.extend(['E:\\桌面文件\\WorkStation', 'E:/桌面文件/WorkStation'])
Python 3.7.6 (default, Jan  8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)]
Type 'copyright', 'credits' or 'license' for more information
IPython 7.12.0 -- An enhanced Interactive Python. Type '?' for help.
PyDev console: using IPython 7.12.0
Python 3.7.6 (default, Jan  8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)] on win32
In[2]: runfile('E:/桌面文件/WorkStation/DL/mnist_tensor.py', wdir='E:/桌面文件/WorkStation/DL')
2.1.0
x: (60000, 28, 28) y: (60000,) x test: (10000, 28, 28) y test: [7 2 1 ... 4 5 6]
train sample: (128, 784) (128, 10)
0 loss: 0.9876465797424316
78 Evaluate Acc: 0.0895
100 loss: 0.4969977140426636
200 loss: 0.3850606083869934
300 loss: 0.3805661201477051
400 loss: 0.3378632664680481
500 loss: 0.2951761484146118
78 Evaluate Acc: 0.1955
600 loss: 0.2852081060409546
700 loss: 0.2568434476852417
800 loss: 0.2508309483528137
900 loss: 0.2366759181022644
1000 loss: 0.2451413869857788
78 Evaluate Acc: 0.2847
1100 loss: 0.23686105012893677
1200 loss: 0.21285539865493774
1300 loss: 0.23249076306819916
1400 loss: 0.20346932113170624
1500 loss: 0.19869862496852875
78 Evaluate Acc: 0.3541
1600 loss: 0.1912011355161667
1700 loss: 0.2020815759897232
1800 loss: 0.191738560795784
1900 loss: 0.17610424757003784
2000 loss: 0.17332768440246582
78 Evaluate Acc: 0.4002
2100 loss: 0.1790955364704132
2200 loss: 0.15455955266952515
2300 loss: 0.1675846129655838
2400 loss: 0.1618402600288391
2500 loss: 0.15379498898983002
78 Evaluate Acc: 0.4364
2600 loss: 0.15517598390579224
2700 loss: 0.1556098461151123
2800 loss: 0.14552998542785645
2900 loss: 0.144424706697464
3000 loss: 0.1474607139825821
78 Evaluate Acc: 0.4644
3100 loss: 0.15520796179771423
3200 loss: 0.1579381823539734
3300 loss: 0.13556203246116638
3400 loss: 0.14220654964447021
3500 loss: 0.12580065429210663
78 Evaluate Acc: 0.489
3600 loss: 0.12723295390605927
3700 loss: 0.12825264036655426
3800 loss: 0.11863202601671219
3900 loss: 0.12218555808067322
4000 loss: 0.12584573030471802
78 Evaluate Acc: 0.5158
4100 loss: 0.13002924621105194
4200 loss: 0.12499353289604187
4300 loss: 0.1220541000366211
4400 loss: 0.1257769763469696
4500 loss: 0.12774376571178436
78 Evaluate Acc: 0.5346
4600 loss: 0.12365836650133133
4700 loss: 0.1333412081003189
4800 loss: 0.12293830513954163
4900 loss: 0.11675341427326202
5000 loss: 0.10786174237728119
78 Evaluate Acc: 0.5529
5100 loss: 0.1217687800526619
5200 loss: 0.10872087627649307
5300 loss: 0.1188330128788948
5400 loss: 0.11162948608398438
5500 loss: 0.11060798168182373
78 Evaluate Acc: 0.568
5600 loss: 0.11962589621543884
5700 loss: 0.10351087152957916
5800 loss: 0.10827428847551346
5900 loss: 0.09387043118476868
6000 loss: 0.10035699605941772
78 Evaluate Acc: 0.5855
6100 loss: 0.10813583433628082
6200 loss: 0.10126964747905731
6300 loss: 0.10712935775518417
6400 loss: 0.09689076244831085
6500 loss: 0.0929778665304184
78 Evaluate Acc: 0.6008
6600 loss: 0.10015600919723511
6700 loss: 0.09518805146217346
6800 loss: 0.09388855844736099
6900 loss: 0.09701965004205704
7000 loss: 0.09947608411312103
78 Evaluate Acc: 0.6138
7100 loss: 0.09438510239124298
7200 loss: 0.10315567255020142
7300 loss: 0.09819212555885315
7400 loss: 0.0889538824558258
7500 loss: 0.08628912270069122
78 Evaluate Acc: 0.6258
7600 loss: 0.0940876305103302
7700 loss: 0.08304577320814133
7800 loss: 0.10224409401416779
7900 loss: 0.09642487019300461
8000 loss: 0.09523174911737442
78 Evaluate Acc: 0.6375
8100 loss: 0.08897297829389572
8200 loss: 0.08897951245307922
8300 loss: 0.09685827791690826
8400 loss: 0.09143991768360138
8500 loss: 0.08935728669166565
78 Evaluate Acc: 0.6464
8600 loss: 0.09057440608739853
8700 loss: 0.08604606986045837
8800 loss: 0.08302024751901627
8900 loss: 0.0898996964097023
9000 loss: 0.0871603712439537
78 Evaluate Acc: 0.6563
9100 loss: 0.09336507320404053
9200 loss: 0.08167578279972076
9300 loss: 0.09012483805418015
9400 loss: 0.08132807165384293
9500 loss: 0.08673285692930222
78 Evaluate Acc: 0.6678
9600 loss: 0.08347520232200623
9700 loss: 0.08610023558139801
9800 loss: 0.07766544818878174
9900 loss: 0.07913652062416077
10000 loss: 0.07526805996894836
78 Evaluate Acc: 0.6766
10100 loss: 0.07633819431066513
10200 loss: 0.08054754883050919
10300 loss: 0.08082559704780579
10400 loss: 0.08259840309619904
10500 loss: 0.07666351646184921
78 Evaluate Acc: 0.6843
10600 loss: 0.0742708370089531
10700 loss: 0.08520179986953735
10800 loss: 0.07355630397796631
10900 loss: 0.07557353377342224
11000 loss: 0.07521995902061462
78 Evaluate Acc: 0.6926
11100 loss: 0.08090036362409592
11200 loss: 0.0800117701292038
11300 loss: 0.08282358199357986
11400 loss: 0.07431307435035706
11500 loss: 0.07889096438884735
78 Evaluate Acc: 0.698
11600 loss: 0.0732647180557251
11700 loss: 0.08019772171974182
11800 loss: 0.07657134532928467
11900 loss: 0.0773920863866806
12000 loss: 0.0727095752954483
78 Evaluate Acc: 0.7054
12100 loss: 0.07267148047685623
12200 loss: 0.07532171159982681
12300 loss: 0.07941843569278717
12400 loss: 0.07809314131736755
12500 loss: 0.06977152079343796
78 Evaluate Acc: 0.7123
12600 loss: 0.07761208713054657
12700 loss: 0.06975089758634567
12800 loss: 0.06090889126062393
12900 loss: 0.07141058146953583
13000 loss: 0.07252379506826401
78 Evaluate Acc: 0.7169
13100 loss: 0.07300286740064621
13200 loss: 0.07118942588567734
13300 loss: 0.07131950557231903
13400 loss: 0.06739509105682373
13500 loss: 0.06946778297424316
78 Evaluate Acc: 0.7235
13600 loss: 0.07059964537620544
13700 loss: 0.06858604401350021
13800 loss: 0.07205026596784592
13900 loss: 0.07423532754182816
14000 loss: 0.06674812734127045
78 Evaluate Acc: 0.7298
Process finished with exit code 0