Transformer - 注意⼒机制
flyfish
计算过程
flyfish
# -*- coding: utf-8 -*-
import torch
import torch.nn as nn
import torch.nn.functional as F
import os
import math
def attention(query, key, value, mask=None, dropout=None):
# query的最后⼀维的⼤⼩, ⼀般情况下就等同于词嵌⼊维度, 命名为d_k
d_k = query.size(-1)
scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)
print("scores.shape:",scores.shape)#scores.shape: torch.Size([1, 12, 12])
if mask is not None:
scores = scores.masked_fill(mask == 0, -1e9)
p_attn = F.softmax(scores, dim = -1)
if dropout is not None:
p_attn = dropout(p_attn)
return torch.matmul(p_attn, value), p_attn
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout, max_len=5000):
super(PositionalEncoding, self).__init__()
self.dropout = nn.Dropout(p=dropout)
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len).unsqueeze(1)
div_term = torch.exp(
torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model)
)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer("pe", pe)
def forward(self, x):
x = x + self.pe[:, : x.size(1)].requires_grad_(False)
return self.dropout(x)
#在测试attention的时候需要位置编码PositionalEncoding
# 词嵌⼊维度是8维
d_model = 8
# 置0⽐率为0.1
dropout = 0.1
# 句⼦最⼤⻓度
max_len=12
x = torch.zeros(1, max_len, d_model)
pe = PositionalEncoding(d_model, dropout, max_len)
pe_result = pe(x)
print("pe_result:", pe_result)
query = key = value = pe_result
print("pe_result.shape:",pe_result.shape)
#没有mask的输出情况
#pe_result.shape: torch.Size([1, 12, 8])
attn, p_attn = attention(query, key, value)
print("no mask\n")
print("attn:", attn)
print("p_attn:", p_attn)
#scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)
# 除以math.sqrt(d_k) 表示这个注意力就是 缩放点积注意力，如果没有，那么就是 点积注意力
#当Q=K=V时,又叫⾃注意⼒机制
#有mask的输出情况
print("mask\n")
mask = torch.zeros(1, max_len, max_len)
attn, p_attn = attention(query, key, value, mask=mask)
print("attn:", attn)
print("p_attn:", p_attn)
pe_result: tensor([[[ 0.0000e+00, 1.1111e+00, 0.0000e+00, 1.1111e+00, 0.0000e+00,
1.1111e+00, 0.0000e+00, 1.1111e+00],
[ 9.3497e-01, 6.0034e-01, 1.1093e-01, 1.1056e+00, 1.1111e-02,
1.1111e+00, 1.1111e-03, 1.1111e+00],
[ 1.0103e+00, -4.6239e-01, 2.2074e-01, 1.0890e+00, 2.2221e-02,
0.0000e+00, 2.2222e-03, 1.1111e+00],
[ 1.5680e-01, -1.1000e+00, 0.0000e+00, 1.0615e+00, 3.3328e-02,
0.0000e+00, 3.3333e-03, 1.1111e+00],
[-8.4089e-01, -7.2627e-01, 4.3269e-01, 1.0234e+00, 4.4433e-02,
1.1102e+00, 4.4444e-03, 1.1111e+00],
[-1.0655e+00, 3.1518e-01, 5.3270e-01, 0.0000e+00, 5.5532e-02,
1.1097e+00, 5.5555e-03, 1.1111e+00],
[-3.1046e-01, 1.0669e+00, 6.2738e-01, 9.1704e-01, 0.0000e+00,
1.1091e+00, 6.6666e-03, 0.0000e+00],
[ 7.2999e-01, 8.3767e-01, 7.1580e-01, 8.4982e-01, 7.7714e-02,
1.1084e+00, 7.7777e-03, 1.1111e+00],
[ 1.0993e+00, -1.6167e-01, 7.9706e-01, 7.7412e-01, 8.8794e-02,
1.1076e+00, 8.8888e-03, 1.1111e+00],
[ 4.5791e-01, -0.0000e+00, 8.7036e-01, 6.9068e-01, 9.9865e-02,
1.1066e+00, 9.9999e-03, 1.1111e+00],
[-6.0447e-01, -9.3230e-01, 9.3497e-01, 6.0034e-01, 1.1093e-01,
1.1056e+00, 1.1111e-02, 1.1111e+00],
[-1.1111e+00, 4.9174e-03, 9.9023e-01, 5.0400e-01, 1.2198e-01,
1.1044e+00, 1.2222e-02, 1.1110e+00]]])
pe_result.shape: torch.Size([1, 12, 8])
scores.shape: torch.Size([1, 12, 12])
no mask
attn: tensor([[[ 1.0590e-01, 2.7361e-01, 4.9333e-01, 8.3999e-01, 5.0599e-02,
1.0079e+00, 5.6491e-03, 1.0138e+00],
[ 2.7554e-01, 2.0916e-01, 4.9203e-01, 8.6593e-01, 5.2177e-02,
9.7066e-01, 5.6513e-03, 1.0398e+00],
[ 2.8765e-01, -3.8825e-02, 4.7812e-01, 8.7535e-01, 5.4246e-02,
8.4157e-01, 5.7015e-03, 1.0659e+00],
[ 9.3666e-02, -1.8286e-01, 4.8727e-01, 8.5124e-01, 5.7070e-02,
8.2547e-01, 5.9523e-03, 1.0712e+00],
[-1.6747e-01, -1.0274e-01, 5.6960e-01, 7.7584e-01, 6.3699e-02,
9.6958e-01, 6.7169e-03, 1.0546e+00],
[-2.2646e-01, 6.8462e-02, 5.8668e-01, 7.2227e-01, 6.3119e-02,
1.0233e+00, 6.8004e-03, 1.0310e+00],
[ 8.8945e-04, 2.7654e-01, 5.3750e-01, 8.0958e-01, 5.2289e-02,
1.0259e+00, 6.1360e-03, 9.6094e-01],
[ 2.2231e-01, 2.2832e-01, 5.2263e-01, 8.4111e-01, 5.4828e-02,
9.9655e-01, 5.9765e-03, 1.0298e+00],
[ 2.6388e-01, 7.2239e-02, 5.3800e-01, 8.4070e-01, 5.8958e-02,
9.5033e-01, 6.2306e-03, 1.0564e+00],
[ 1.2822e-01, 7.4518e-02, 5.5305e-01, 8.1381e-01, 6.0125e-02,
9.7442e-01, 6.4089e-03, 1.0462e+00],
[-1.5757e-01, -1.3194e-01, 5.9562e-01, 7.6069e-01, 6.7079e-02,
9.7264e-01, 7.0187e-03, 1.0607e+00],
[-2.3505e-01, 5.6245e-03, 6.0160e-01, 7.3040e-01, 6.5491e-02,
1.0176e+00, 7.0038e-03, 1.0367e+00]]])
p_attn: tensor([[[0.1488, 0.1215, 0.0514, 0.0396, 0.0698, 0.0703, 0.0875, 0.1205,
0.0790, 0.0814, 0.0544, 0.0757],
[0.1170, 0.1434, 0.0757, 0.0489, 0.0590, 0.0460, 0.0642, 0.1304,
0.1161, 0.0943, 0.0527, 0.0524],
[0.0716, 0.1094, 0.1341, 0.1067, 0.0716, 0.0379, 0.0407, 0.0930,
0.1221, 0.0921, 0.0713, 0.0494],
[0.0597, 0.0765, 0.1155, 0.1397, 0.1127, 0.0506, 0.0359, 0.0627,
0.0918, 0.0806, 0.1056, 0.0688],
[0.0692, 0.0607, 0.0509, 0.0740, 0.1475, 0.0846, 0.0509, 0.0607,
0.0692, 0.0788, 0.1342, 0.1194],
[0.0887, 0.0601, 0.0343, 0.0423, 0.1076, 0.1341, 0.0721, 0.0748,
0.0591, 0.0777, 0.1057, 0.1435],
[0.1232, 0.0938, 0.0411, 0.0335, 0.0722, 0.0804, 0.1351, 0.1103,
0.0722, 0.0814, 0.0633, 0.0935],
[0.1124, 0.1263, 0.0623, 0.0388, 0.0571, 0.0553, 0.0731, 0.1388,
0.1134, 0.1001, 0.0571, 0.0652],
[0.0758, 0.1157, 0.0841, 0.0584, 0.0670, 0.0450, 0.0492, 0.1166,
0.1429, 0.1101, 0.0763, 0.0588],
[0.0822, 0.0989, 0.0668, 0.0540, 0.0803, 0.0622, 0.0584, 0.1084,
0.1158, 0.1046, 0.0879, 0.0804],
[0.0548, 0.0551, 0.0515, 0.0705, 0.1364, 0.0845, 0.0454, 0.0617,
0.0801, 0.0877, 0.1499, 0.1224],
[0.0763, 0.0548, 0.0357, 0.0459, 0.1213, 0.1146, 0.0669, 0.0703,
0.0616, 0.0802, 0.1224, 0.1499]]])
mask
scores.shape: torch.Size([1, 12, 12])
attn: tensor([[[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185],
[0.0381, 0.0461, 0.5194, 0.8105, 0.0555, 0.9236, 0.0061, 1.0185]]])
p_attn: tensor([[[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833],
[0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833, 0.0833,
0.0833, 0.0833, 0.0833, 0.0833]]])
