python3+TensorFlow 2.x（四）反向传播

反向传播算法

反向传播算法基本步骤：

反向中的参数变化

总结

反向传播算法

反向传播算法（Backpropagation）是训练人工神经网络时使用的一个重要算法，它是通过计算梯度并优化神经网络的权重来最小化误差。反向传播算法的核心是基于链式法则的梯度下降优化方法，通过计算误差对每个权重的偏导数来更新网络中的参数。

反向传播算法基本步骤：

前向传播：将输入数据传递通过神经网络的各层，计算每一层的输出。
计算损失：根据输出和实际标签计算损失（通常使用均方误差或交叉熵等作为损失函数）。
反向传播：根据损失函数对每个参数（如权重、偏置）计算梯度。梯度的计算通过链式法则进行反向传播，直到达到输入层。
更新权重：使用梯度下降算法来更新每一层的权重和偏置，使得损失函数最小化。

链式推到：https://blog.csdn.net/dingyahui123/category_6945552.html?spm=1001.2014.3001.5482

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.datasets import mnist
# 加载 MNIST 数据集
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

# 归一化数据并将其形状调整为 (N, 784)，因为每张图片是 28x28 像素
train_images = train_images.reshape(-1, 28*28) / 255.0
test_images = test_images.reshape(-1, 28*28) / 255.0

# 转换标签为 one-hot 编码
train_labels = np.eye(10)[train_labels]
test_labels = np.eye(10)[test_labels]
# 定义激活函数
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

# 定义激活函数的导数
def sigmoid_derivative(x):
    return x * (1 - x)

# 网络架构参数
input_size = 28 * 28  # 输入层的大小
hidden_size = 128     # 隐藏层的大小
output_size = 10      # 输出层的大小

# 初始化权重和偏置
W1 = np.random.randn(input_size, hidden_size)  # 输入层到隐藏层的权重
b1 = np.zeros((1, hidden_size))  # 隐藏层的偏置
W2 = np.random.randn(hidden_size, output_size)  # 隐藏层到输出层的权重
b2 = np.zeros((1, output_size))  # 输出层的偏置
# 设置超参数
epochs = 20
learning_rate = 0.1
batch_size = 64

# 训练过程
for epoch in range(epochs):
    for i in range(0, len(train_images), batch_size):
        # 选择当前batch的数据
        X_batch = train_images[i:i+batch_size]
        y_batch = train_labels[i:i+batch_size]
        
        # 前向传播
        z1 = np.dot(X_batch, W1) + b1
        a1 = sigmoid(z1)
        z2 = np.dot(a1, W2) + b2
        a2 = sigmoid(z2)
        
        # 计算损失的梯度
        output_error = a2 - y_batch  # 损失函数的梯度
        output_delta = output_error * sigmoid_derivative(a2)
        
        hidden_error = output_delta.dot(W2.T)
        hidden_delta = hidden_error * sigmoid_derivative(a1)
        
        # 更新权重和偏置
        W2 -= learning_rate * a1.T.dot(output_delta)
        b2 -= learning_rate * np.sum(output_delta, axis=0, keepdims=True)
        W1 -= learning_rate * X_batch.T.dot(hidden_delta)
        b1 -= learning_rate * np.sum(hidden_delta, axis=0, keepdims=True)
    
    # 每10轮输出一次损失
    if epoch % 10 == 0:
        loss = np.mean(np.square(a2 - y_batch))
        print(f"Epoch {epoch}, Loss: {loss}")
# 测试模型
z1 = np.dot(test_images, W1) + b1
a1 = sigmoid(z1)
z2 = np.dot(a1, W2) + b2
a2 = sigmoid(z2)

# 计算准确率
predictions = np.argmax(a2, axis=1)
true_labels = np.argmax(test_labels, axis=1)
accuracy = np.mean(predictions == true_labels)

print(f"Test Accuracy: {accuracy * 100:.2f}%")
# 可视化前5个测试图像及其预测结果
for i in range(5):
    plt.imshow(test_images[i].reshape(28, 28), cmap='gray')
    plt.title(f"Predicted: {predictions[i]}, Actual: {true_labels[i]}")
    plt.show()

反向中的参数变化

import numpy as np
import matplotlib.pyplot as plt
import imageio

# 激活函数和其导数
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
    return x * (1 - x)

# 生成一些示例数据
np.random.seed(0)
X = np.array([[0, 0],
              [0, 1],
              [1, 0],
              [1, 1]])
y = np.array([[0], [1], [1], [0]])  # XOR 问题

# 初始化参数
input_layer_neurons = 2
hidden_layer_neurons = 2
output_neurons = 1
learning_rate = 0.5
epochs = 10000

# 初始化权重
weights_input_hidden = np.random.uniform(size=(input_layer_neurons, hidden_layer_neurons))
weights_hidden_output = np.random.uniform(size=(hidden_layer_neurons, output_neurons))

# 存储权重和图像
weights_history = []
losses = []
images = []

# 训练过程
for epoch in range(epochs):
    # 前向传播
    hidden_layer_input = np.dot(X, weights_input_hidden)
    hidden_layer_output = sigmoid(hidden_layer_input)
    
    output_layer_input = np.dot(hidden_layer_output, weights_hidden_output)
    predicted_output = sigmoid(output_layer_input)
    
    loss = np.mean((y - predicted_output) ** 2)
    losses.append(loss)

    # 反向传播
    error = y - predicted_output
    d_predicted_output = error * sigmoid_derivative(predicted_output)
    
    error_hidden_layer = d_predicted_output.dot(weights_hidden_output.T)
    d_hidden_layer = error_hidden_layer * sigmoid_derivative(hidden_layer_output)
    
    # 更新权重
    weights_hidden_output += hidden_layer_output.T.dot(d_predicted_output) * learning_rate
    weights_input_hidden += X.T.dot(d_hidden_layer) * learning_rate
    
    # 保存权重
    weights_history.append((weights_input_hidden.copy(), weights_hidden_output.copy()))
    
    # 每1000次迭代保存一次图像
    if epoch % 1000 == 0:
        plt.figure(figsize=(8, 6))
        plt.subplot(1, 2, 1)
        plt.title('Weights Input-Hidden')
        plt.imshow(weights_input_hidden, cmap='viridis', aspect='auto')
        plt.colorbar()
        
        plt.subplot(1, 2, 2)
        plt.title('Weights Hidden-Output')
        plt.imshow(weights_hidden_output, cmap='viridis', aspect='auto')
        plt.colorbar()
        
        # 保存图像
        plt.savefig(f'weights_epoch_{epoch}.png')
        plt.close()
    if epoch % 1000 == 0:
        plt.figure(figsize=(8, 6))
        plt.plot(losses, label='Loss')
        plt.title('Loss over epochs')
        plt.xlabel('Epochs')
        plt.ylabel('Loss')
        plt.xlim(0, epochs)
        plt.ylim(0, np.max(losses))
        plt.grid()
        plt.legend()
        
        # 保存图像
        plt.savefig(f'loss_epoch_{epoch}.png')
        plt.close()
# 创建 GIF
with imageio.get_writer('weights_update.gif', mode='I', duration=0.5) as writer:
    for epoch in range(0, epochs, 1000):
        image = imageio.imread(f'weights_epoch_{epoch}.png')
        writer.append_data(image)
# 创建 GIF
with imageio.get_writer('training_loss.gif', mode='I', duration=0.5) as writer:
    for epoch in range(0, epochs, 1000):
        image = imageio.imread(f'loss_epoch_{epoch}.png')
        writer.append_data(image)
# 清理生成的图像文件
import os
for epoch in range(0, epochs, 1000):
    os.remove(f'weights_epoch_{epoch}.png')
    os.remove(f'loss_epoch_{epoch}.png')

print("GIF 已生成：training_loss.gif")
print("GIF 已生成：weights_update.gif")