# Build index mappings for sparse matrices
user_ids = np.sort(train_df["UserID"].unique())
movie_ids = np.sort(train_df["MovieID"].unique())
u2i = {u: i for i, u in enumerate(user_ids)}
m2i = {m: i for i, m in enumerate(movie_ids)}
train_idx = train_df[train_df["UserID"].isin(u2i) & train_df["MovieID"].isin(m2i)].copy()
rows = train_idx["UserID"].map(u2i).values
cols = train_idx["MovieID"].map(m2i).values
vals = train_idx["Rating"].astype(float).values
R = csr_matrix((vals, (rows, cols)), shape=(len(user_ids), len(movie_ids)))
R_dense = R.toarray()
# Item-based Pearson with overlap-aware shrinkage
rated_mask = (R_dense > 0).astype(np.uint8)
co_counts = rated_mask.T @ rated_mask
user_mean_vec = np.where(
rated_mask.sum(axis=1) > 0,
R_dense.sum(axis=1) / np.maximum(rated_mask.sum(axis=1), 1),
global_mean
)
R_centered = R_dense.copy()
for i in range(R_centered.shape[0]):
mask = R_centered[i] != 0
R_centered[i, mask] = R_centered[i, mask] - user_mean_vec[i]
item_corr_raw = np.corrcoef(R_centered.T)
item_corr_raw = np.nan_to_num(item_corr_raw, nan=0.0, posinf=0.0, neginf=0.0)
shrink_lambda = 25.0
item_corr = item_corr_raw * (co_counts / (co_counts + shrink_lambda))
np.fill_diagonal(item_corr, 1.0)
# User-based Cosine similarity
user_cos = cosine_similarity(R)
np.fill_diagonal(user_cos, 0.0)
print("Matrices ready:")
print("R shape:", R.shape, "Item corr:", item_corr.shape, "User cosine:", user_cos.shape)
def predict_item_pearson(user_id, movie_id, k=40):
if user_id not in u2i or movie_id not in m2i:
return global_mean
u = u2i[user_id]
m = m2i[movie_id]
user_ratings = R_dense[u]
rated_idx = np.where(user_ratings > 0)[0]
if len(rated_idx) == 0:
return global_mean
sims = item_corr[m, rated_idx]
top_idx = np.argsort(np.abs(sims))[::-1][:k]
sims_top = sims[top_idx]
ratings_top = user_ratings[rated_idx[top_idx]]
denom = np.sum(np.abs(sims_top))
if denom == 0:
return user_mean_vec[u]
pred = np.dot(sims_top, ratings_top) / denom
return float(clip_rating(pred))
def predict_user_cosine(user_id, movie_id, k=40):
if user_id not in u2i or movie_id not in m2i:
return global_mean
u = u2i[user_id]
m = m2i[movie_id]
movie_col = R_dense[:, m]
raters = np.where(movie_col > 0)[0]
if len(raters) == 0:
return global_mean
sims = user_cos[u, raters]
top_idx = np.argsort(np.abs(sims))[::-1][:k]
sims_top = sims[top_idx]
ratings_top = movie_col[raters[top_idx]]
denom = np.sum(np.abs(sims_top))
if denom == 0:
return user_mean_vec[u]
pred = np.dot(sims_top, ratings_top) / denom
return float(clip_rating(pred))
def batch_predict(df_eval, predictor, k=40):
return np.array([predictor(u, m, k=k) for u, m in zip(df_eval["UserID"], df_eval["MovieID"])])
k_grid = [20, 40, 80, 120]
item_scores = []
user_scores = []
for k in k_grid:
v_item = val_df.copy()
v_item["pred"] = batch_predict(v_item, predict_item_pearson, k=k)
item_scores.append({"k": k, **evaluate_predictions(v_item)})
v_user = val_df.copy()
v_user["pred"] = batch_predict(v_user, predict_user_cosine, k=k)
user_scores.append({"k": k, **evaluate_predictions(v_user)})
item_scores_df = pd.DataFrame(item_scores).sort_values("RMSE")
user_scores_df = pd.DataFrame(user_scores).sort_values("RMSE")
display(item_scores_df)
display(user_scores_df)
best_k_item = int(item_scores_df.iloc[0]["k"])
best_k_user = int(user_scores_df.iloc[0]["k"])
test_item = test_df.copy()
test_item["pred"] = batch_predict(test_item, predict_item_pearson, k=best_k_item)
item_test_metrics = evaluate_predictions(test_item)
test_user = test_df.copy()
test_user["pred"] = batch_predict(test_user, predict_user_cosine, k=best_k_user)
user_test_metrics = evaluate_predictions(test_user)
print(f"Best item-based Pearson k: {best_k_item}, test metrics: {item_test_metrics}")
print(f"Best user-based Cosine k: {best_k_user}, test metrics: {user_test_metrics}")
# Pearson diagnostics: overlap and shrinkage effects
tri_i, tri_j = np.triu_indices(item_corr.shape[0], k=1)
overlap_vals = co_counts[tri_i, tri_j]
raw_vals = item_corr_raw[tri_i, tri_j]
shrunk_vals = item_corr[tri_i, tri_j]
print("Pearson diagnostics summary:")
print("Median overlap across item pairs:", float(np.median(overlap_vals)))
print("Share of pairs with overlap >= 20:", round(float((overlap_vals >= 20).mean()), 4))
print("Mean |raw corr|:", round(float(np.mean(np.abs(raw_vals))), 4))
print("Mean |shrunk corr|:", round(float(np.mean(np.abs(shrunk_vals))), 4))
valid = overlap_vals >= 20
id_to_title_local = movies.drop_duplicates("MovieID").set_index("MovieID")["Title"].to_dict()
if valid.any():
valid_idx = np.where(valid)[0]
top_idx = valid_idx[np.argsort(shrunk_vals[valid_idx])[::-1][:10]]
top_pairs = []
for idx in top_idx:
mid_1 = movie_ids[tri_i[idx]]
mid_2 = movie_ids[tri_j[idx]]
top_pairs.append({
"Movie1": id_to_title_local.get(mid_1, str(mid_1)),
"Movie2": id_to_title_local.get(mid_2, str(mid_2)),
"OverlapUsers": int(overlap_vals[idx]),
"RawPearson": float(raw_vals[idx]),
"ShrunkPearson": float(shrunk_vals[idx]),
})
print("Top high-confidence item correlations (overlap >= 20):")
display(pd.DataFrame(top_pairs))
else:
print("No item pairs met overlap >= 20 threshold in current train matrix.")
