{ "cells": [ { "cell_type": "markdown", "id": "e2820a58", "metadata": {}, "source": [ "# Lesson 41: Machine translation and sequence-to-sequence demonstration\n", "\n", "This notebook demonstrates sequence preprocessing and encoder-decoder architecture for machine translation.\n", "\n", "**1. Sequence preprocessing**\n", "- 1.1. Tokenization and vocabulary\n", "- 1.2. Padding and sequencing\n", "\n", "**2. Encoder-decoder architecture**\n", "- 2.1. Model architecture\n", "- 2.2. Training\n", "- 2.3. Inference\n", "\n", "\n", "## Notebook set up\n", "\n", "### Imports" ] }, { "cell_type": "code", "execution_count": null, "id": "e194f819", "metadata": {}, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import pandas as pd\n", "\n", "# Set environment variables for TensorFlow (must be done before import)\n", "os.environ['CUDA_VISIBLE_DEVICES'] = '1'\n", "os.environ['TF_USE_CUDNN_RNN'] = '0'\n", "\n", "import tensorflow as tf\n", "from tensorflow.keras.models import Model\n", "from tensorflow.keras.layers import Input, LSTM, Dense, Embedding\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "from tensorflow.keras.utils import to_categorical" ] }, { "cell_type": "markdown", "id": "25b96aa9", "metadata": {}, "source": [ "### Configuration" ] }, { "cell_type": "code", "execution_count": null, "id": "3e09c51a", "metadata": {}, "outputs": [], "source": [ "gpus = tf.config.list_physical_devices('GPU')\n", "\n", "if gpus:\n", " for gpu in gpus:\n", " tf.config.experimental.set_memory_growth(gpu, True)" ] }, { "cell_type": "markdown", "id": "852abe4f", "metadata": {}, "source": [ "### Parallel corpus\n", "\n", "A small English-French parallel corpus for demonstration." ] }, { "cell_type": "code", "execution_count": null, "id": "34a5f65a", "metadata": {}, "outputs": [], "source": [ "# Simple English-French parallel corpus\n", "english_sentences = [\n", " 'hello',\n", " 'how are you',\n", " 'thank you',\n", " 'good morning',\n", " 'good night',\n", " 'i love you',\n", " 'goodbye',\n", " 'yes',\n", " 'no',\n", " 'please'\n", "]\n", "\n", "french_sentences = [\n", " 'bonjour',\n", " 'comment allez vous',\n", " 'merci',\n", " 'bonjour',\n", " 'bonne nuit',\n", " 'je vous aime',\n", " 'au revoir',\n", " 'oui',\n", " 'non',\n", " 'sil vous plait'\n", "]\n", "\n", "# Add start and end tokens to target sentences\n", "french_input = ['startseq ' + s for s in french_sentences]\n", "french_output = [s + ' endseq' for s in french_sentences]\n", "\n", "print(f'Parallel corpus size: {len(english_sentences)} sentence pairs')\n", "pd.DataFrame({'English': english_sentences, 'French': french_sentences})" ] }, { "cell_type": "markdown", "id": "677cf73a", "metadata": {}, "source": [ "## 1. Sequence preprocessing\n", "\n", "### 1.1. Tokenization and vocabulary\n", "\n", "Convert text to integer sequences using a vocabulary index.\n", "\n", "Keras [Tokenizer](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer) documentation" ] }, { "cell_type": "code", "execution_count": 14, "id": "ad5a8dce", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "English vocabulary size: 15\n", "French vocabulary size: 18\n", "\n", "English vocabulary: {'you': 1, 'good': 2, 'hello': 3, 'how': 4, 'are': 5, 'thank': 6, 'morning': 7, 'night': 8, 'i': 9, 'love': 10, 'goodbye': 11, 'yes': 12, 'no': 13, 'please': 14}\n", "\n", "French vocabulary: {'start': 1, 'end': 2, 'vous': 3, 'bonjour': 4, 'comment': 5, 'allez': 6, 'merci': 7, 'bonne': 8, 'nuit': 9, 'je': 10, 'aime': 11, 'au': 12, 'revoir': 13, 'oui': 14, 'non': 15, 'sil': 16, 'plait': 17}\n" ] } ], "source": [ "# Create tokenizers for source and target languages\n", "eng_tokenizer = Tokenizer()\n", "eng_tokenizer.fit_on_texts(english_sentences)\n", "eng_vocab_size = len(eng_tokenizer.word_index) + 1\n", "\n", "fra_tokenizer = Tokenizer()\n", "fra_tokenizer.fit_on_texts(french_input + french_output)\n", "fra_vocab_size = len(fra_tokenizer.word_index) + 1\n", "\n", "print(f'English vocabulary size: {eng_vocab_size}')\n", "print(f'French vocabulary size: {fra_vocab_size}')\n", "print(f'\\nEnglish vocabulary: {eng_tokenizer.word_index}')\n", "print(f'\\nFrench vocabulary: {fra_tokenizer.word_index}')" ] }, { "cell_type": "markdown", "id": "e303615c", "metadata": {}, "source": [ "### 1.2. Padding and sequencing\n", "\n", "Sequences must have the same length for batch processing. Padding adds zeros to shorter sequences.\n", "\n", "Keras [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) documentation" ] }, { "cell_type": "code", "execution_count": null, "id": "82c9a10a", "metadata": {}, "outputs": [], "source": [ "# Convert to sequences\n", "eng_sequences = eng_tokenizer.texts_to_sequences(english_sentences)\n", "fra_input_sequences = fra_tokenizer.texts_to_sequences(french_input)\n", "fra_output_sequences = fra_tokenizer.texts_to_sequences(french_output)\n", "\n", "# Find max sequence lengths\n", "max_eng_len = max(len(seq) for seq in eng_sequences)\n", "max_fra_len = max(len(seq) for seq in fra_input_sequences)\n", "\n", "# Pad sequences\n", "encoder_input = pad_sequences(eng_sequences, maxlen=max_eng_len, padding='post')\n", "decoder_input = pad_sequences(fra_input_sequences, maxlen=max_fra_len, padding='post')\n", "decoder_output = pad_sequences(fra_output_sequences, maxlen=max_fra_len, padding='post')\n", "\n", "print(f'Encoder input shape: {encoder_input.shape}')\n", "print(f'Decoder input shape: {decoder_input.shape}')\n", "print(f'\\nExample (first sentence):')\n", "print(f' English: {english_sentences[0]} -> {encoder_input[0]}')\n", "print(f' French input: {french_input[0]} -> {decoder_input[0]}')\n", "print(f' French output: {french_output[0]} -> {decoder_output[0]}')" ] }, { "cell_type": "markdown", "id": "588facee", "metadata": {}, "source": [ "## 2. Encoder-decoder architecture\n", "\n", "### 2.1. Model architecture (LSTM)\n", "\n", "The encoder processes the input sequence and produces a context vector. The decoder uses this context to generate the output sequence.\n", "\n", "```text\n", " ENCODER DECODER\n", " \n", " Input sequence Target sequence (shifted) \n", " │ │ \n", " ▼ ▼ \n", "┌─────────────┐ ┌─────────────┐ \n", "│ Embedding │ │ Embedding │ \n", "└──────┬──────┘ └──────┬──────┘ \n", " │ │ \n", " ▼ ▼ \n", "┌─────────────┐ Context vector (h, c) ┌─────────────┐ \n", "│ LSTM │ ─────────────────────────────────► │ LSTM │ \n", "└─────────────┘ └──────┬──────┘ \n", " │ \n", " ▼ \n", " ┌─────────────┐ \n", " │ Dense │ \n", " │ (softmax) │ \n", " └──────┬──────┘ \n", " │ \n", " ▼ \n", " Output sequence \n", "```\n", "\n", "**Context vector (h, c):** The LSTM produces two state vectors that form the context:\n", "- **h (hidden state):** The current output representation encoding what the LSTM has \"decided\" to output\n", "- **c (cell state):** The long-term memory that carries information across the sequence\n", "\n", "**Target sequence shifting:** During training, we use \"teacher forcing\" where the decoder input is shifted by one position relative to the expected output:\n", "- Decoder input: `startseq bonjour` (what the model sees)\n", "- Decoder target: `bonjour endseq` (what the model learns to predict)\n", "\n", "This teaches the model to predict the next token given all previous tokens.\n", "\n", "Keras [LSTM](https://www.tensorflow.org/api_docs/python/tf/keras/layers/LSTM) documentation" ] }, { "cell_type": "code", "execution_count": null, "id": "4f178b54", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:Layer encoder_lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n", "WARNING:tensorflow:Layer decoder_lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n", "Model: \"model_3\"\n", "__________________________________________________________________________________________________\n", " Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", " encoder_input (InputLayer) [(None, 3)] 0 [] \n", " \n", " decoder_input (InputLayer) [(None, 4)] 0 [] \n", " \n", " encoder_embedding (Embeddi (None, 3, 64) 960 ['encoder_input[0][0]'] \n", " ng) \n", " \n", " decoder_embedding (Embeddi (None, 4, 64) 1152 ['decoder_input[0][0]'] \n", " ng) \n", " \n", " encoder_lstm (LSTM) [(None, 128), 98816 ['encoder_embedding[0][0]'] \n", " (None, 128), \n", " (None, 128)] \n", " \n", " decoder_lstm (LSTM) [(None, 4, 128), 98816 ['decoder_embedding[0][0]', \n", " (None, 128), 'encoder_lstm[0][1]', \n", " (None, 128)] 'encoder_lstm[0][2]'] \n", " \n", " decoder_output (Dense) (None, 4, 18) 2322 ['decoder_lstm[0][0]'] \n", " \n", "==================================================================================================\n", "Total params: 202066 (789.32 KB)\n", "Trainable params: 202066 (789.32 KB)\n", "Non-trainable params: 0 (0.00 Byte)\n", "__________________________________________________________________________________________________\n" ] } ], "source": [ "# Model parameters\n", "embedding_dim = 64\n", "hidden_units = 128\n", "\n", "# Encoder (recurrent_dropout forces non-CuDNN implementation)\n", "encoder_inputs = Input(shape=(max_eng_len,), name='encoder_input')\n", "encoder_embedding = Embedding(eng_vocab_size, embedding_dim, name='encoder_embedding')(encoder_inputs)\n", "\n", "encoder_lstm, state_h, state_c = LSTM(\n", " hidden_units, return_state=True, recurrent_dropout=1e-5, name='encoder_lstm'\n", ")(encoder_embedding)\n", "\n", "encoder_states = [state_h, state_c]\n", "\n", "# Decoder\n", "decoder_inputs = Input(shape=(max_fra_len,), name='decoder_input')\n", "decoder_embedding = Embedding(fra_vocab_size, embedding_dim, name='decoder_embedding')(decoder_inputs)\n", "\n", "decoder_lstm = LSTM(\n", " hidden_units, return_sequences=True, return_state=True, recurrent_dropout=1e-5, name='decoder_lstm'\n", ")\n", "\n", "decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)\n", "decoder_dense = Dense(fra_vocab_size, activation='softmax', name='decoder_output')\n", "decoder_outputs = decoder_dense(decoder_outputs)\n", "\n", "# Build model\n", "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n", "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n", "\n", "model.summary()" ] }, { "cell_type": "markdown", "id": "e21e2bc4", "metadata": {}, "source": [ "### 2.2. Training" ] }, { "cell_type": "code", "execution_count": 11, "id": "a41e22da", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Final training loss: 0.0875\n", "Final training accuracy: 1.0000\n" ] } ], "source": [ "# Prepare target data (add dimension for sparse categorical crossentropy)\n", "decoder_target = np.expand_dims(decoder_output, -1)\n", "\n", "# Train model\n", "history = model.fit(\n", " [encoder_input, decoder_input],\n", " decoder_target,\n", " epochs=100,\n", " batch_size=5,\n", " verbose=0\n", ")\n", "\n", "print(f'Final training loss: {history.history[\"loss\"][-1]:.4f}')\n", "print(f'Final training accuracy: {history.history[\"accuracy\"][-1]:.4f}')" ] }, { "cell_type": "markdown", "id": "5c638e34", "metadata": {}, "source": [ "### 2.3. Inference\n", "\n", "For inference, we need separate encoder and decoder models to generate translations step by step." ] }, { "cell_type": "code", "execution_count": null, "id": "487a7c9e", "metadata": {}, "outputs": [], "source": [ "# Create inference encoder model\n", "encoder_model = Model(encoder_inputs, encoder_states)\n", "\n", "# Create inference decoder model\n", "decoder_state_input_h = Input(shape=(hidden_units,))\n", "decoder_state_input_c = Input(shape=(hidden_units,))\n", "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n", "\n", "decoder_outputs_inf, state_h_inf, state_c_inf = decoder_lstm(\n", " decoder_embedding,\n", " initial_state=decoder_states_inputs\n", ")\n", "\n", "decoder_states_inf = [state_h_inf, state_c_inf]\n", "decoder_outputs_inf = decoder_dense(decoder_outputs_inf)\n", "\n", "decoder_model = Model(\n", " [decoder_inputs] + decoder_states_inputs,\n", " [decoder_outputs_inf] + decoder_states_inf\n", ")" ] }, { "cell_type": "markdown", "id": "9d7cea3f", "metadata": {}, "source": [ "### Translation process\n", "\n", "The `translate` function generates output tokens one at a time:\n", "\n", "1. Encode the input sentence to get the initial hidden states (h, c)\n", "2. Start decoder with the `startseq` token\n", "3. Loop: predict next token, append to output, feed prediction back as next input\n", "4. Stop when `endseq` is predicted or max length reached" ] }, { "cell_type": "code", "execution_count": 15, "id": "7d7b7e57", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Translation examples:\n", " hello -> bonjour\n", " how are you -> comment\n", " thank you -> merci\n", " good morning -> bonjour\n", " good night -> bonne\n" ] } ], "source": [ "# Build reverse vocabulary for decoding\n", "fra_index_to_word = {v: k for k, v in fra_tokenizer.word_index.items()}\n", "\n", "# Get start/end token keys from vocabulary\n", "start_token = 'startseq' if 'startseq' in fra_tokenizer.word_index else 'start'\n", "end_token = 'endseq' if 'endseq' in fra_tokenizer.word_index else 'end'\n", "\n", "def translate(input_sentence):\n", "\n", " # Encode input\n", " input_seq = eng_tokenizer.texts_to_sequences([input_sentence])\n", " input_seq = pad_sequences(input_seq, maxlen=max_eng_len, padding='post')\n", "\n", " # Get encoder states\n", " states = encoder_model.predict(input_seq, verbose=0)\n", "\n", " # Start with start token\n", " target_seq = np.zeros((1, max_fra_len))\n", " target_seq[0, 0] = fra_tokenizer.word_index[start_token]\n", "\n", " # Generate translation\n", " translation = []\n", "\n", " for i in range(max_fra_len - 1):\n", "\n", " output, h, c = decoder_model.predict([target_seq] + states, verbose=0)\n", " predicted_id = np.argmax(output[0, i, :])\n", "\n", " if predicted_id == 0 or fra_index_to_word.get(predicted_id) == end_token:\n", " break\n", "\n", " word = fra_index_to_word.get(predicted_id, '')\n", "\n", " if word and word != start_token:\n", " translation.append(word)\n", "\n", " target_seq[0, i + 1] = predicted_id\n", " states = [h, c]\n", "\n", " return ' '.join(translation)\n", "\n", "# Test translations\n", "print('Translation examples:')\n", "\n", "for eng in english_sentences[:5]:\n", " fra = translate(eng)\n", " print(f' {eng} -> {fra}')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }