{ "cells": [ { "cell_type": "markdown", "id": "ed18addc", "metadata": {}, "source": [ "# Lesson 23: recommendation systems demonstration\n", "\n", "This notebook demonstrates key concepts for recommendation systems\n", "\n", "1. Collaborative filtering\n", " - Memory based\n", " - Model based\n", "\n", "2. Content-based filtering\n", "3. Hybrid filtering\n", "\n", "## Notebook set up\n", "\n", "### Imports" ] }, { "cell_type": "code", "execution_count": null, "id": "97e3fe09", "metadata": {}, "outputs": [], "source": [ "import pandas as pd # For data manipulation\n", "import numpy as np # For numerical operations\n", "from sklearn.metrics.pairwise import cosine_similarity # To compute similarity scores\n", "from sklearn.decomposition import TruncatedSVD # For matrix factorization" ] }, { "cell_type": "markdown", "id": "5b162041", "metadata": {}, "source": [ "### Dataset\n", "\n", "Load animes & ratings data." ] }, { "cell_type": "code", "execution_count": null, "id": "3dd318b6", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
anime_idnamegenretypeepisodesratingmembers
032281Kimi no Na wa.Drama, Romance, School, SupernaturalMovie19.37200630
15114Fullmetal Alchemist: BrotherhoodAction, Adventure, Drama, Fantasy, Magic, Mili...TV649.26793665
228977Gintama°Action, Comedy, Historical, Parody, Samurai, S...TV519.25114262
39253Steins;GateSci-Fi, ThrillerTV249.17673572
49969Gintama'Action, Comedy, Historical, Parody, Samurai, S...TV519.16151266
\n", "
" ], "text/plain": [ " anime_id name \\\n", "0 32281 Kimi no Na wa. \n", "1 5114 Fullmetal Alchemist: Brotherhood \n", "2 28977 Gintama° \n", "3 9253 Steins;Gate \n", "4 9969 Gintama' \n", "\n", " genre type episodes rating \\\n", "0 Drama, Romance, School, Supernatural Movie 1 9.37 \n", "1 Action, Adventure, Drama, Fantasy, Magic, Mili... TV 64 9.26 \n", "2 Action, Comedy, Historical, Parody, Samurai, S... TV 51 9.25 \n", "3 Sci-Fi, Thriller TV 24 9.17 \n", "4 Action, Comedy, Historical, Parody, Samurai, S... TV 51 9.16 \n", "\n", " members \n", "0 200630 \n", "1 793665 \n", "2 114262 \n", "3 673572 \n", "4 151266 " ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load anime information from CSV file\n", "animes = pd.read_csv('anime.csv')\n", "animes.head()" ] }, { "cell_type": "code", "execution_count": 38, "id": "08dac3b4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 12294 entries, 0 to 12293\n", "Data columns (total 7 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 anime_id 12294 non-null int64 \n", " 1 name 12294 non-null object \n", " 2 genre 12232 non-null object \n", " 3 type 12269 non-null object \n", " 4 episodes 12294 non-null object \n", " 5 rating 12064 non-null float64\n", " 6 members 12294 non-null int64 \n", "dtypes: float64(1), int64(2), object(4)\n", "memory usage: 672.5+ KB\n" ] } ], "source": [ "animes.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "3f7ddb1f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
user_idanime_idrating
0120-1
1124-1
2179-1
31226-1
41241-1
\n", "
" ], "text/plain": [ " user_id anime_id rating\n", "0 1 20 -1\n", "1 1 24 -1\n", "2 1 79 -1\n", "3 1 226 -1\n", "4 1 241 -1" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load user ratings from CSV file\n", "ratings = pd.read_csv('rating.csv')\n", "ratings.head()" ] }, { "cell_type": "markdown", "id": "b577264c", "metadata": {}, "source": [ "Check the size and structure of the ratings dataset to understand the data volume." ] }, { "cell_type": "code", "execution_count": 40, "id": "9c706aec", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 7813737 entries, 0 to 7813736\n", "Data columns (total 3 columns):\n", " # Column Dtype\n", "--- ------ -----\n", " 0 user_id int64\n", " 1 anime_id int64\n", " 2 rating int64\n", "dtypes: int64(3)\n", "memory usage: 178.8 MB\n" ] } ], "source": [ "ratings.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "597c6fe7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Index: 50000 entries, 5012406 to 6767779\n", "Data columns (total 3 columns):\n", " # Column Non-Null Count Dtype\n", "--- ------ -------------- -----\n", " 0 user_id 50000 non-null int64\n", " 1 anime_id 50000 non-null int64\n", " 2 rating 50000 non-null int64\n", "dtypes: int64(3)\n", "memory usage: 1.5 MB\n" ] } ], "source": [ "# Randomly sample 50,000 ratings for faster computation\n", "sample_ratings = ratings.sample(n=50000, random_state=315)\n", "sample_ratings.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "6dd75400", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of unique users in sample: 26940\n", "Number of unique animes in sample: 4863\n" ] } ], "source": [ "# Count unique users and animes in the sample\n", "num_users = sample_ratings['user_id'].nunique()\n", "num_animes = sample_ratings['anime_id'].nunique()\n", "\n", "print(f\"Number of unique users in sample: {num_users}\")\n", "print(f\"Number of unique animes in sample: {num_animes}\")" ] }, { "cell_type": "markdown", "id": "10117302", "metadata": {}, "source": [ "## 1. Collaborative filtering\n", "\n", "### 1.1. Memory based collaborative filtering" ] }, { "cell_type": "markdown", "id": "a4648ac3", "metadata": {}, "source": [ "Create a user-item matrix where each row is a user, each column is an anime, and values are ratings." ] }, { "cell_type": "code", "execution_count": null, "id": "39baff1e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "User-Item Matrix shape: (26940, 4863)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
anime_id156781516171819...33524335583356933606337403374133798339643410334240
user_id
30.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
40.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
50.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
70.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
110.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n", "

5 rows × 4863 columns

\n", "
" ], "text/plain": [ "anime_id 1 5 6 7 8 15 16 17 18 \\\n", "user_id \n", "3 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "7 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "11 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "\n", "anime_id 19 ... 33524 33558 33569 33606 33740 33741 33798 33964 \\\n", "user_id ... \n", "3 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "4 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "5 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "7 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "11 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "\n", "anime_id 34103 34240 \n", "user_id \n", "3 0.0 0.0 \n", "4 0.0 0.0 \n", "5 0.0 0.0 \n", "7 0.0 0.0 \n", "11 0.0 0.0 \n", "\n", "[5 rows x 4863 columns]" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Create a user-item matrix using pivot table\n", "user_item_matrix = sample_ratings.pivot_table(\n", " index='user_id', # Users as rows\n", " columns='anime_id', # Animes as columns\n", " values='rating' # Ratings as values\n", ")\n", "\n", "# Fill missing values (unrated animes) with 0\n", "user_item_filled = user_item_matrix.fillna(0)\n", "\n", "print('User-Item Matrix shape:', user_item_filled.shape)\n", "user_item_filled.head()" ] }, { "cell_type": "markdown", "id": "86d3392e", "metadata": {}, "source": [ "Compute cosine similarity between all pairs of animes to find which animes have similar rating patterns across users." ] }, { "cell_type": "code", "execution_count": 48, "id": "79630a7f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
user_id3457111314172126...73490734917349573499735007350173503735077351073515
anime_id
10.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
50.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
60.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
70.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
80.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
..................................................................
337410.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
337980.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
339640.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
341030.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
342400.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n", "

4863 rows × 26940 columns

\n", "
" ], "text/plain": [ "user_id 3 4 5 7 11 13 14 17 21 \\\n", "anime_id \n", "1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "6 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "7 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "8 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "... ... ... ... ... ... ... ... ... ... \n", "33741 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "33798 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "33964 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "34103 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "34240 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "\n", "user_id 26 ... 73490 73491 73495 73499 73500 73501 73503 73507 \\\n", "anime_id ... \n", "1 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "5 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "6 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "7 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "8 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "... ... ... ... ... ... ... ... ... ... ... \n", "33741 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "33798 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "33964 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "34103 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "34240 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "\n", "user_id 73510 73515 \n", "anime_id \n", "1 0.0 0.0 \n", "5 0.0 0.0 \n", "6 0.0 0.0 \n", "7 0.0 0.0 \n", "8 0.0 0.0 \n", "... ... ... \n", "33741 0.0 0.0 \n", "33798 0.0 0.0 \n", "33964 0.0 0.0 \n", "34103 0.0 0.0 \n", "34240 0.0 0.0 \n", "\n", "[4863 rows x 26940 columns]" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "user_item_filled.T" ] }, { "cell_type": "code", "execution_count": null, "id": "252d8945", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Item similarity matrix shape: (4863, 4863)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
anime_id156781516171819...33524335583356933606337403374133798339643410334240
anime_id
11.0000000.0263860.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
50.0263861.0000000.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
60.0000000.0000001.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
70.0000000.0000000.01.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
80.0000000.0000000.00.01.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n", "

5 rows × 4863 columns

\n", "
" ], "text/plain": [ "anime_id 1 5 6 7 8 15 16 17 18 \\\n", "anime_id \n", "1 1.000000 0.026386 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "5 0.026386 1.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "6 0.000000 0.000000 1.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "7 0.000000 0.000000 0.0 1.0 0.0 0.0 0.0 0.0 0.0 \n", "8 0.000000 0.000000 0.0 0.0 1.0 0.0 0.0 0.0 0.0 \n", "\n", "anime_id 19 ... 33524 33558 33569 33606 33740 33741 33798 33964 \\\n", "anime_id ... \n", "1 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "5 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "6 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "7 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "8 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", "\n", "anime_id 34103 34240 \n", "anime_id \n", "1 0.0 0.0 \n", "5 0.0 0.0 \n", "6 0.0 0.0 \n", "7 0.0 0.0 \n", "8 0.0 0.0 \n", "\n", "[5 rows x 4863 columns]" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Compute cosine similarity between animes (transpose to compare columns)\n", "item_similarity = cosine_similarity(user_item_filled.T)\n", "\n", "# Convert to DataFrame with anime IDs as row and column labels\n", "item_similarity_df = pd.DataFrame(\n", " item_similarity,\n", " index=user_item_matrix.columns,\n", " columns=user_item_matrix.columns\n", ")\n", "\n", "print('Item similarity matrix shape:', item_similarity_df.shape)\n", "item_similarity_df.head()" ] }, { "cell_type": "markdown", "id": "7eeff8cf", "metadata": {}, "source": [ "Create helper functions to convert between anime IDs and names for more readable output." ] }, { "cell_type": "code", "execution_count": null, "id": "430a9e69", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Anime ID 1: Cowboy Bebop\n" ] } ], "source": [ "def get_anime_name(anime_id):\n", " \"\"\"Get anime name from ID\"\"\"\n", " result = animes[animes['anime_id'] == anime_id]['name']\n", " return result.values[0] if len(result) > 0 else f'Unknown (ID: {anime_id})'\n", "\n", "def get_anime_id(anime_name):\n", " \"\"\"Get anime ID from name\"\"\"\n", " result = animes[animes['name'] == anime_name]['anime_id']\n", " return result.values[0] if len(result) > 0 else None\n", "\n", "# Test the helper function\n", "print(f\"Anime ID 1: {get_anime_name(1)}\")" ] }, { "cell_type": "markdown", "id": "920fabc1", "metadata": {}, "source": [ "Demonstrate memory-based collaborative filtering by finding the top 5 animes most similar to a target anime based on user rating patterns." ] }, { "cell_type": "code", "execution_count": null, "id": "41195f2e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Top 5 animes similar to \"Cowboy Bebop\":\n", "\n", "Koi☆Sento: 0.1218\n", "Hikyou Tanken Fam & Ihrlie: 0.1218\n", "Houkago 2: Saiyuri: 0.1218\n", "Lupin III: Ikiteita Majutsushi: 0.1206\n", "Gyakuten Majo Saiban: Chijo na Majo ni Sabakarechau The Animation: 0.1200\n" ] } ], "source": [ "# Select target anime\n", "anime_id = 1\n", "\n", "# Get similarity scores and sort in descending order\n", "similar_animes = item_similarity_df[anime_id].sort_values(ascending=False)\n", "\n", "print(f'Top 5 animes similar to \"{get_anime_name(anime_id)}\":')\n", "print()\n", "\n", "# Display top 5 (skip first one since it's the anime itself)\n", "for anime_id_similar, score in similar_animes[1:6].items():\n", " print(f'{get_anime_name(anime_id_similar)}: {score:.4f}')" ] }, { "cell_type": "markdown", "id": "c1e0143d", "metadata": {}, "source": [ "### 1.2. Model based collaborative filtering" ] }, { "cell_type": "markdown", "id": "197457e8", "metadata": {}, "source": [ "Use matrix factorization (SVD) to reduce dimensionality and fill in missing ratings by learning latent features of users and animes." ] }, { "cell_type": "code", "execution_count": null, "id": "d0cfc633", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Predicted ratings matrix shape: (26940, 4863)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
anime_id156781516171819...33524335583356933606337403374133798339643410334240
user_id
3-0.1235580.0013470.015531-0.0013601.970482e-05-0.0006530.0062066.840352e-04-6.027962e-05-0.007176...-0.0065080.000537-0.0014960.000805-1.090123e-04-5.796410e-050.0006186.392568e-050.0000470.008004
4-0.000143-0.0000300.000164-0.000010-2.217778e-07-0.000014-0.000014-8.815549e-07-5.654995e-07-0.000016...-0.0000100.000002-0.000002-0.000010-3.455666e-072.782329e-08-0.000001-2.231423e-08-0.0000030.000010
5-0.002929-0.000565-0.0004390.000008-9.950065e-070.0002180.0005133.954197e-052.285818e-05-0.000223...-0.0001150.0001070.0000140.000066-2.426508e-06-6.658181e-070.0000479.848932e-06-0.0000830.000776
7-0.005738-0.000151-0.0005070.0000059.891335e-070.0000870.0000568.833031e-065.095776e-060.000141...-0.0000170.0000100.000010-0.000019-1.102061e-06-1.011350e-070.0000043.406460e-070.000005-0.000029
110.2929690.0434700.075774-0.001745-4.406893e-050.001446-0.012093-6.346332e-041.169022e-030.003752...-0.0065070.004725-0.0058960.002050-1.419158e-05-9.158018e-05-0.0011236.495072e-040.000187-0.022802
\n", "

5 rows × 4863 columns

\n", "
" ], "text/plain": [ "anime_id 1 5 6 7 8 15 \\\n", "user_id \n", "3 -0.123558 0.001347 0.015531 -0.001360 1.970482e-05 -0.000653 \n", "4 -0.000143 -0.000030 0.000164 -0.000010 -2.217778e-07 -0.000014 \n", "5 -0.002929 -0.000565 -0.000439 0.000008 -9.950065e-07 0.000218 \n", "7 -0.005738 -0.000151 -0.000507 0.000005 9.891335e-07 0.000087 \n", "11 0.292969 0.043470 0.075774 -0.001745 -4.406893e-05 0.001446 \n", "\n", "anime_id 16 17 18 19 ... 33524 \\\n", "user_id ... \n", "3 0.006206 6.840352e-04 -6.027962e-05 -0.007176 ... -0.006508 \n", "4 -0.000014 -8.815549e-07 -5.654995e-07 -0.000016 ... -0.000010 \n", "5 0.000513 3.954197e-05 2.285818e-05 -0.000223 ... -0.000115 \n", "7 0.000056 8.833031e-06 5.095776e-06 0.000141 ... -0.000017 \n", "11 -0.012093 -6.346332e-04 1.169022e-03 0.003752 ... -0.006507 \n", "\n", "anime_id 33558 33569 33606 33740 33741 33798 \\\n", "user_id \n", "3 0.000537 -0.001496 0.000805 -1.090123e-04 -5.796410e-05 0.000618 \n", "4 0.000002 -0.000002 -0.000010 -3.455666e-07 2.782329e-08 -0.000001 \n", "5 0.000107 0.000014 0.000066 -2.426508e-06 -6.658181e-07 0.000047 \n", "7 0.000010 0.000010 -0.000019 -1.102061e-06 -1.011350e-07 0.000004 \n", "11 0.004725 -0.005896 0.002050 -1.419158e-05 -9.158018e-05 -0.001123 \n", "\n", "anime_id 33964 34103 34240 \n", "user_id \n", "3 6.392568e-05 0.000047 0.008004 \n", "4 -2.231423e-08 -0.000003 0.000010 \n", "5 9.848932e-06 -0.000083 0.000776 \n", "7 3.406460e-07 0.000005 -0.000029 \n", "11 6.495072e-04 0.000187 -0.022802 \n", "\n", "[5 rows x 4863 columns]" ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Create SVD model with 50 latent features\n", "svd_model = TruncatedSVD(n_components=50, random_state=315)\n", "\n", "# Fit model and transform user-item matrix to user features\n", "user_features = svd_model.fit_transform(user_item_filled)\n", "\n", "# Reconstruct ratings matrix by multiplying user and item features\n", "predicted_ratings = np.dot(user_features, svd_model.components_)\n", "\n", "# Convert back to DataFrame with original indices\n", "predicted_ratings_df = pd.DataFrame(\n", " predicted_ratings,\n", " index=user_item_matrix.index,\n", " columns=user_item_matrix.columns\n", ")\n", "\n", "print('Predicted ratings matrix shape:', predicted_ratings_df.shape)\n", "predicted_ratings_df.head()" ] }, { "cell_type": "markdown", "id": "fc4e48a1", "metadata": {}, "source": [ "Demonstrate model-based collaborative filtering by recommending unwatched animes to a user based on predicted ratings from SVD." ] }, { "cell_type": "code", "execution_count": null, "id": "d097343c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Top 5 recommended animes for user 3:\n", "anime_id\n", "18679 0.843389\n", "2236 0.649259\n", "223 0.471859\n", "9989 0.449235\n", "6880 0.438066\n", "Name: 3, dtype: float64\n" ] } ], "source": [ "# Select first user from the matrix\n", "user_id = user_item_matrix.index[0]\n", "\n", "# Get predicted ratings for this user\n", "user_predictions = predicted_ratings_df.loc[user_id]\n", "\n", "# Find animes the user hasn't rated (missing values in original matrix)\n", "unrated_animes = user_item_matrix.loc[user_id][user_item_matrix.loc[user_id].isna()]\n", "\n", "# Get predictions for unrated animes and sort by predicted rating\n", "recommendations = user_predictions[unrated_animes.index].sort_values(ascending=False)\n", "\n", "print(f'Top 5 recommended animes for user {user_id}:')\n", "print(recommendations.head())" ] }, { "cell_type": "markdown", "id": "3550de28", "metadata": {}, "source": [ "## 2. Content-based filtering" ] }, { "cell_type": "markdown", "id": "e2bfac3e", "metadata": {}, "source": [ "Examine the content features (genre, type) available for each anime to use in content-based filtering." ] }, { "cell_type": "code", "execution_count": null, "id": "f17f98a0", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
anime_idnamegenretype
032281Kimi no Na wa.Drama, Romance, School, SupernaturalMovie
15114Fullmetal Alchemist: BrotherhoodAction, Adventure, Drama, Fantasy, Magic, Mili...TV
228977Gintama°Action, Comedy, Historical, Parody, Samurai, S...TV
39253Steins;GateSci-Fi, ThrillerTV
49969Gintama'Action, Comedy, Historical, Parody, Samurai, S...TV
532935Haikyuu!!: Karasuno Koukou VS Shiratorizawa Ga...Comedy, Drama, School, Shounen, SportsTV
611061Hunter x Hunter (2011)Action, Adventure, Shounen, Super PowerTV
7820Ginga Eiyuu DensetsuDrama, Military, Sci-Fi, SpaceOVA
815335Gintama Movie: Kanketsu-hen - Yorozuya yo Eien...Action, Comedy, Historical, Parody, Samurai, S...Movie
915417Gintama': EnchousenAction, Comedy, Historical, Parody, Samurai, S...TV
\n", "
" ], "text/plain": [ " anime_id name \\\n", "0 32281 Kimi no Na wa. \n", "1 5114 Fullmetal Alchemist: Brotherhood \n", "2 28977 Gintama° \n", "3 9253 Steins;Gate \n", "4 9969 Gintama' \n", "5 32935 Haikyuu!!: Karasuno Koukou VS Shiratorizawa Ga... \n", "6 11061 Hunter x Hunter (2011) \n", "7 820 Ginga Eiyuu Densetsu \n", "8 15335 Gintama Movie: Kanketsu-hen - Yorozuya yo Eien... \n", "9 15417 Gintama': Enchousen \n", "\n", " genre type \n", "0 Drama, Romance, School, Supernatural Movie \n", "1 Action, Adventure, Drama, Fantasy, Magic, Mili... TV \n", "2 Action, Comedy, Historical, Parody, Samurai, S... TV \n", "3 Sci-Fi, Thriller TV \n", "4 Action, Comedy, Historical, Parody, Samurai, S... TV \n", "5 Comedy, Drama, School, Shounen, Sports TV \n", "6 Action, Adventure, Shounen, Super Power TV \n", "7 Drama, Military, Sci-Fi, Space OVA \n", "8 Action, Comedy, Historical, Parody, Samurai, S... Movie \n", "9 Action, Comedy, Historical, Parody, Samurai, S... TV " ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Display relevant features for content-based filtering\n", "animes[['anime_id', 'name', 'genre', 'type']].head(10)" ] }, { "cell_type": "markdown", "id": "9610f83c", "metadata": {}, "source": [ "Create a function to calculate similarity between animes based on their genres using Jaccard similarity (intersection over union)." ] }, { "cell_type": "code", "execution_count": 49, "id": "f1080bef", "metadata": {}, "outputs": [], "source": [ "# Convert genre strings to sets for easier comparison\n", "animes['genre_set'] = animes['genre'].fillna('').apply(lambda x: set(x.split(', ')))\n", "\n", "def genre_similarity(genres1, genres2):\n", " \"\"\"Calculate Jaccard similarity between two genre sets\"\"\"\n", "\n", " # Return 0 if either set is empty\n", " if len(genres1) == 0 or len(genres2) == 0:\n", " return 0\n", "\n", " # Calculate intersection (common genres) and union (all unique genres)\n", " intersection = len(genres1.intersection(genres2))\n", " union = len(genres1.union(genres2))\n", "\n", " # Jaccard similarity = intersection / union\n", " return intersection / union if union > 0 else 0" ] }, { "cell_type": "markdown", "id": "8e1a45fd", "metadata": {}, "source": [ "Select a target anime to demonstrate content-based filtering using genre similarity." ] }, { "cell_type": "code", "execution_count": 50, "id": "4f4e926c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Target anime: Cowboy Bebop\n", "Genres: Action, Adventure, Comedy, Drama, Sci-Fi, Space\n" ] } ], "source": [ "# Choose anime to find similar content for\n", "target_anime_id = 1\n", "target_anime = animes[animes['anime_id'] == target_anime_id].iloc[0]\n", "target_genres = target_anime['genre_set']\n", "\n", "print(f\"Target anime: {target_anime['name']}\")\n", "print(f\"Genres: {target_anime['genre']}\")" ] }, { "cell_type": "markdown", "id": "0b7ab7a1", "metadata": {}, "source": [ "Demonstrate content-based filtering by finding animes with the most similar genres to the target anime." ] }, { "cell_type": "code", "execution_count": 51, "id": "5778c65d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Top 5 similar animes based on genre:\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
namegenresimilarity
1465Cowboy Bebop: Yose Atsume BluesAction, Adventure, Comedy, Drama, Sci-Fi, Space1.000000
6568Seihou Tenshi Angel LinksAction, Adventure, Comedy, Drama, Romance, Sci...0.857143
2735Uchuu Kaizoku Captain Harlock: Arcadia-gou no ...Action, Adventure, Drama, Sci-Fi, Space0.833333
5721Kaitei Choutokkyuu: Marine ExpressAction, Adventure, Comedy, Drama, Sci-Fi0.833333
1073Waga Seishun no ArcadiaAction, Adventure, Drama, Sci-Fi, Space0.833333
\n", "
" ], "text/plain": [ " name \\\n", "1465 Cowboy Bebop: Yose Atsume Blues \n", "6568 Seihou Tenshi Angel Links \n", "2735 Uchuu Kaizoku Captain Harlock: Arcadia-gou no ... \n", "5721 Kaitei Choutokkyuu: Marine Express \n", "1073 Waga Seishun no Arcadia \n", "\n", " genre similarity \n", "1465 Action, Adventure, Comedy, Drama, Sci-Fi, Space 1.000000 \n", "6568 Action, Adventure, Comedy, Drama, Romance, Sci... 0.857143 \n", "2735 Action, Adventure, Drama, Sci-Fi, Space 0.833333 \n", "5721 Action, Adventure, Comedy, Drama, Sci-Fi 0.833333 \n", "1073 Action, Adventure, Drama, Sci-Fi, Space 0.833333 " ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Calculate genre similarity for all animes\n", "animes['similarity'] = animes['genre_set'].apply(\n", " lambda x: genre_similarity(target_genres, x)\n", ")\n", "\n", "# Find top similar animes (excluding the target itself)\n", "similar_animes = animes[animes['anime_id'] != target_anime_id].sort_values(\n", " 'similarity', \n", " ascending=False\n", ")[['name', 'genre', 'similarity']].head(5)\n", "\n", "print('Top 5 similar animes based on genre:')\n", "similar_animes.head()" ] }, { "cell_type": "markdown", "id": "3a7a3dfb", "metadata": {}, "source": [ "## 3. Hybrid filtering" ] }, { "cell_type": "markdown", "id": "80dd4fc8", "metadata": {}, "source": [ "Combine collaborative filtering and content-based filtering using a weighted average to leverage both user behavior and content features." ] }, { "cell_type": "code", "execution_count": 54, "id": "a04a94c9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Top 5 hybrid recommendations for anime_id 1:\n", "Cowboy Bebop: Yose Atsume Blues: 0.5000\n", "Seihou Tenshi Angel Links: 0.4294\n", "Seihou Bukyou Outlaw Star: 0.4167\n", "Ginga Tetsudou Monogatari: 0.4167\n", "Waga Seishun no Arcadia: Mugen Kidou SSX: 0.4167\n" ] } ], "source": [ "# Get collaborative filtering scores (based on user ratings)\n", "collab_score = item_similarity_df[target_anime_id]\n", "\n", "# Get content-based scores (based on genre similarity)\n", "content_score = animes.set_index('anime_id')['similarity']\n", "\n", "# Find animes that exist in both scoring methods\n", "common_animes = collab_score.index.intersection(content_score.index)\n", "\n", "# Combine scores with equal weights (50% each)\n", "hybrid_score = (\n", " 0.5 * collab_score[common_animes] + \n", " 0.5 * content_score[common_animes]\n", ")\n", "\n", "# Sort and get top 5 (excluding the target anime itself)print(hybrid_recommendations)\n", "hybrid_recommendations = hybrid_score.sort_values(ascending=False)[1:6]\n", "print(f'Top 5 hybrid recommendations for anime_id {target_anime_id}:')\n", "\n", "for rec in hybrid_recommendations.items():\n", " anime_id_rec, score = rec\n", " print(f'{get_anime_name(anime_id_rec)}: {score:.4f}')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.12" } }, "nbformat": 4, "nbformat_minor": 5 }