import torch import torch.nn as nn import torch.nn.functional as F class LuongAttention(nn.Module): “””Luong Attention（General）””” def __init__(self, hidden_dim): super(LuongAttention, self).__init__() self.W = nn.Linear(hidden_dim, hidden_dim, bias=False) def forward(self, decoder_hidden, encoder_outputs): # ステップ1: スコア計算 transformed = self.W(decoder_hidden) attention_scores = torch.bmm( transformed.unsqueeze(1), encoder_outputs.transpose(1, 2) ).squeeze(1) # ステップ2: Softmaxで正規化 attention_weights = F.softmax(attention_scores, dim=1) # ステップ3: Context Vector計算 context = torch.bmm( attention_weights.unsqueeze(1), encoder_outputs ).squeeze(1) return context, attention_weights # テスト hidden_dim = 512 attention = LuongAttention(hidden_dim) # ダミーデータ batch_size = 4 src_length = 6 # “The cat is on the mat” (6単語) decoder_hidden = torch.randn(batch_size, hidden_dim) encoder_outputs = torch.randn(batch_size, src_length, hidden_dim) # Attention計算 context, weights = attention(decoder_hidden, encoder_outputs) print(f”Context Vector shape: {context.shape}”) print(f”Attention Weights shape: {weights.shape}”) print(f”Attention Weights sum: {weights[0].sum().item():.4f}”) print(f”\n1つ目のサンプルのAttention Weights:”) print(weights[0].detach().numpy().round(3))

class BahdanauAttention(nn.Module): “””Bahdanau Attention（Additive）””” def __init__(self, hidden_dim): super(BahdanauAttention, self).__init__() # W₁: Decoder状態を変換 self.W1 = nn.Linear(hidden_dim, hidden_dim) # W₂: Encoder状態を変換 self.W2 = nn.Linear(hidden_dim, hidden_dim) # v: スカラー値を出力 self.V = nn.Linear(hidden_dim, 1) def forward(self, decoder_hidden, encoder_outputs): batch_size = encoder_outputs.shape[0] src_length = encoder_outputs.shape[1] # decoder_hiddenを拡張して繰り返す # (batch_size, hidden_dim) → (batch_size, src_length, hidden_dim) decoder_hidden = decoder_hidden.unsqueeze(1).repeat(1, src_length, 1) # スコア計算: v^T × tanh(W₁×s + W₂×h) energy = torch.tanh( self.W1(decoder_hidden) + self.W2(encoder_outputs) ) attention_scores = self.V(energy).squeeze(2) # Softmax attention_weights = F.softmax(attention_scores, dim=1) # Context Vector context = torch.bmm( attention_weights.unsqueeze(1), encoder_outputs ).squeeze(1) return context, attention_weights # テスト bahdanau = BahdanauAttention(hidden_dim) context_b, weights_b = bahdanau(decoder_hidden, encoder_outputs) print(f”Bahdanau Context shape: {context_b.shape}”) print(f”Bahdanau Weights sum: {weights_b[0].sum().item():.4f}”)

class AttentionDecoder(nn.Module): “””Attention付きDecoder””” def __init__(self, output_dim, embedding_dim, hidden_dim, n_layers=1, dropout=0.5): super(AttentionDecoder, self).__init__() self.output_dim = output_dim self.hidden_dim = hidden_dim # Embedding層 self.embedding = nn.Embedding(output_dim, embedding_dim) # Attention層 self.attention = LuongAttention(hidden_dim) # LSTM層 # 入力: embedding + context（Attentionの結果） self.lstm = nn.LSTM( embedding_dim + hidden_dim, # contextを結合するので +hidden_dim hidden_dim, n_layers, dropout=dropout if n_layers > 1 else 0, batch_first=True ) # 出力層 self.fc_out = nn.Linear(hidden_dim, output_dim) self.dropout = nn.Dropout(dropout) def forward(self, input, hidden, cell, encoder_outputs): “”” 1単語を生成 Args: input: 現在の入力単語 (batch_size, 1) hidden: 前の隠れ状態 (n_layers, batch_size, hidden_dim) cell: 前のセル状態 (n_layers, batch_size, hidden_dim) encoder_outputs: Encoderの全出力 (batch_size, src_length, hidden_dim) “”” # 埋め込み embedded = self.dropout(self.embedding(input)) # (batch_size, 1, embedding_dim) # Attention計算（最上層のhiddenを使用） decoder_state = hidden[-1] # (batch_size, hidden_dim) context, attention_weights = self.attention(decoder_state, encoder_outputs) # context: (batch_size, hidden_dim) # embeddedとcontextを結合 context = context.unsqueeze(1) # (batch_size, 1, hidden_dim) lstm_input = torch.cat([embedded, context], dim=2) # (batch_size, 1, embedding_dim + hidden_dim) # LSTM output, (hidden, cell) = self.lstm(lstm_input, (hidden, cell)) # 予測 prediction = self.fc_out(output.squeeze(1)) # (batch_size, output_dim) return prediction, hidden, cell, attention_weights

import matplotlib.pyplot as plt import numpy as np # 日本語フォント設定（Google Colabの場合） # !pip install japanize-matplotlib # import japanize_matplotlib def visualize_attention(src_words, trg_words, attention_weights): “”” Attention Weightsをヒートマップで可視化 Args: src_words: 入力文の単語リスト（例: [“I”, “love”, “you”]） trg_words: 出力文の単語リスト（例: [“私”, “は”, “愛し”, “ます”]） attention_weights: (trg_length, src_length) のテンソル “”” # NumPyに変換 if torch.is_tensor(attention_weights): attention_weights = attention_weights.cpu().detach().numpy() # ヒートマップ作成 fig, ax = plt.subplots(figsize=(8, 6)) im = ax.imshow(attention_weights, cmap=’YlOrRd’) # 軸ラベル ax.set_xticks(range(len(src_words))) ax.set_yticks(range(len(trg_words))) ax.set_xticklabels(src_words) ax.set_yticklabels(trg_words) # 軸の説明 ax.set_xlabel(‘Input (English)’) ax.set_ylabel(‘Output (Japanese)’) ax.set_title(‘Attention Weights’) # カラーバー plt.colorbar(im) # 値を表示 for i in range(len(trg_words)): for j in range(len(src_words)): value = attention_weights[i, j] color = ‘white’ if value > 0.5 else ‘black’ ax.text(j, i, f'{value:.2f}’, ha=’center’, va=’center’, color=color) plt.tight_layout() plt.show() # ダミーデータでテスト src_words = [‘I’, ‘love’, ‘you’, ‘<eos>’] trg_words = [‘私’, ‘は’, ‘あなた’, ‘を’, ‘愛します’, ‘<eos>’] # 理想的なAttention Weights（例） attention_weights = np.array([ [0.85, 0.05, 0.05, 0.05], # “私” → “I” [0.10, 0.10, 0.10, 0.70], # “は” → 文法 [0.05, 0.05, 0.85, 0.05], # “あなた” → “you” [0.10, 0.10, 0.10, 0.70], # “を” → 文法 [0.05, 0.85, 0.05, 0.05], # “愛します” → “love” [0.05, 0.05, 0.05, 0.85], # “<eos>” → “<eos>” ]) visualize_attention(src_words, trg_words, attention_weights)