From da752b3b0a22b0f8fcd38121f5caa5ad11944a8c Mon Sep 17 00:00:00 2001 From: Giuseppe Spillo <44213842+giuspillo@users.noreply.github.com> Date: Thu, 25 Jun 2026 11:42:21 +0200 Subject: [PATCH 1/3] MMGCF implementation --- src/configs/model/MMGCF.yaml | 39 +++++ src/models/mmgcf.py | 289 +++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 src/configs/model/MMGCF.yaml create mode 100644 src/models/mmgcf.py diff --git a/src/configs/model/MMGCF.yaml b/src/configs/model/MMGCF.yaml new file mode 100644 index 0000000..e438460 --- /dev/null +++ b/src/configs/model/MMGCF.yaml @@ -0,0 +1,39 @@ +# MMGCF – Multimodal Graph Collaborative Filtering +# ────────────────────────────────────────────────────────────────────────── +# Model hyperparameters +# ────────────────────────────────────────────────────────────────────────── + +# Dimension of user/item ID embeddings (LightGCN output size). +embedding_size: 64 + +# Output dimension of the visual/text feature projection layers. +# For non-concat fusion modes (mean, sum), this MUST equal +# embedding_size so that ID and modal embeddings can be combined +# element-wise. Only concat mode handles differing dimensions natively. +feat_embed_dim: 64 + +# Number of LightGCN propagation layers over the user-item graph. +n_ui_layers: [1,2,3] + +# L2 regularisation weight applied to initial (un-propagated) ID embeddings. +reg_weight: [1e-1, 1e-2, 1e-3, 1e-4] + +# How to combine the ID embedding with multimodal feature embeddings. +# Options: mean | sum | concat +fusion_mode: ['mean','sum','concat'] + +# How to weight ID vs. modal embeddings before/during fusion. +# equal – fuse modalities first (mean/sum/prod/concat), then fuse +# with ID embedding at equal weight [recommended default] +# alpha – learnable sigmoid gate α; ID emb scaled by α, +# each modal feat scaled by (1−α), then jointly fused +# normalized – L2-normalise every embedding before fusing; ID emb is +# additionally scaled by n_modalities for balance +weighting: ['equal', 'alpha', 'normalized'] + +# Edge dropout probability applied before each training epoch. +# Edges are sampled proportional to their normalised adjacency weight +# (degree-sensitive pruning). Set to 0.0 to disable. +dropout: [0.2, 0.5, 0.8] + +hyper_parameters: ['n_ui_layers', 'reg_weight', 'fusion_mode', 'weighting', 'dropout'] \ No newline at end of file diff --git a/src/models/mmgcf.py b/src/models/mmgcf.py new file mode 100644 index 0000000..207831c --- /dev/null +++ b/src/models/mmgcf.py @@ -0,0 +1,289 @@ +# coding: utf-8 +""" +MMGCF: Multimodal Graph Collaborative Filtering +Late-fusion of LightGCN ID embeddings with multimodal item features. + +Fusion modes : mean | sum | concat +Weighting modes: equal | alpha | normalized + +Adapted for the MMRec framework. +NOTE: For non-concat fusion modes, feat_embed_dim must equal embedding_size + so that ID and modal embeddings can be stacked/added element-wise. +""" + +import numpy as np +import scipy.sparse as sp +import torch +import torch.nn as nn +import torch.nn.functional as F + +from common.abstract_recommender import GeneralRecommender + + +class MMGCF(GeneralRecommender): + def __init__(self, config, dataset): + super(MMGCF, self).__init__(config, dataset) + + self.embedding_dim = config['embedding_size'] + self.feat_embed_dim = config['feat_embed_dim'] + self.n_ui_layers = config['n_ui_layers'] + self.reg_weight = config['reg_weight'] + self.fusion_mode = config['fusion_mode'] # mean | sum | concat + self.weighting = config['weighting'] # equal | alpha | normalized + self.dropout = config['dropout'] + + self.n_nodes = self.n_users + self.n_items + + # ── interaction matrix & adjacency ──────────────────────────────── + self.interaction_matrix = dataset.inter_matrix(form='coo').astype(np.float32) + self.norm_adj = self.get_norm_adj_mat().to(self.device) + self.masked_adj = None + self.edge_indices, self.edge_values = self.get_edge_info() + self.edge_indices = self.edge_indices.to(self.device) + self.edge_values = self.edge_values.to(self.device) + + # ── ID embeddings ───────────────────────────────────────────────── + self.user_embedding = nn.Embedding(self.n_users, self.embedding_dim) + self.item_id_embedding = nn.Embedding(self.n_items, self.embedding_dim) + nn.init.xavier_uniform_(self.user_embedding.weight) + nn.init.xavier_uniform_(self.item_id_embedding.weight) + + # ── multimodal feature embeddings & projection layers ───────────── + # Raw pretrained features are stored in freezed Embeddings; + # a learnable Linear layer projects them to feat_embed_dim on every forward pass. + self.n_modalities = 0 + if self.v_feat is not None: + self.image_embedding = nn.Embedding.from_pretrained(self.v_feat, freeze=True) + self.image_trs = nn.Linear(self.v_feat.shape[1], self.feat_embed_dim) + self.n_modalities += 1 + if self.t_feat is not None: + self.text_embedding = nn.Embedding.from_pretrained(self.t_feat, freeze=True) + self.text_trs = nn.Linear(self.t_feat.shape[1], self.feat_embed_dim) + self.n_modalities += 1 + + # ── weighting: learnable α scalar (only used when weighting='alpha') ── + if self.weighting == 'alpha': + # α = sigmoid(mm_alpha); item_emb weighted by α, mm feats by (1-α) + self.mm_alpha = nn.Parameter(torch.tensor(0.0)) + + # ── concat fusion layers ────────────────────────────────────────── + # Three separate Linear layers are used so each concat step has its own + # fixed input size, avoiding the LazyLinear shape-collision bug in the + # original code when equal+concat are combined. + if self.fusion_mode == 'concat' and self.n_modalities > 0: + # Used by 'alpha' and 'normalized': concat [id_emb, mm1, ..., mmN] + all_in = self.embedding_dim + self.n_modalities * self.feat_embed_dim + self.all_concat_layer = nn.Linear(all_in, self.embedding_dim) + + # Used by 'equal' step-1: concat all modality feats (only when >1 mod) + if self.n_modalities > 1: + mm_in = self.n_modalities * self.feat_embed_dim + self.mm_concat_layer = nn.Linear(mm_in, self.feat_embed_dim) + + # Used by 'equal' step-2: concat [id_emb, fused_mm] + id_mm_in = self.embedding_dim + self.feat_embed_dim + self.id_mm_concat_layer = nn.Linear(id_mm_in, self.embedding_dim) + + # ══════════════════════════════════════════════════════════════════════ + # Graph helpers + # ══════════════════════════════════════════════════════════════════════ + + def get_norm_adj_mat(self): + """Build symmetric normalised Laplacian for the user-item bipartite graph.""" + A = sp.dok_matrix( + (self.n_users + self.n_items, self.n_users + self.n_items), + dtype=np.float32, + ) + inter_M = self.interaction_matrix + inter_M_t = self.interaction_matrix.transpose() + # fill upper-right and lower-left blocks + data_dict = dict(zip(zip(inter_M.row, inter_M.col + self.n_users), + [1] * inter_M.nnz)) + data_dict.update(dict(zip(zip(inter_M_t.row + self.n_users, inter_M_t.col), + [1] * inter_M_t.nnz))) + # A._update(data_dict) + for (row, col), value in data_dict.items(): + A[row, col] = value + # D^{-1/2} A D^{-1/2} + sumArr = (A > 0).sum(axis=1) + diag = np.power(np.array(sumArr.flatten())[0] + 1e-7, -0.5) + D = sp.diags(diag) + L = sp.coo_matrix(D * A * D) + i = torch.LongTensor(np.array([L.row, L.col])) + data = torch.FloatTensor(L.data) + return torch.sparse.FloatTensor(i, data, torch.Size((self.n_nodes, self.n_nodes))) + + def _normalize_adj_m(self, indices, adj_size): + """Compute D_u^{-1/2} * D_i^{-1/2} edge weights for a bipartite sub-graph.""" + adj = torch.sparse.FloatTensor( + indices, torch.ones_like(indices[0], dtype=torch.float32), adj_size + ) + row_sum = 1e-7 + torch.sparse.sum(adj, -1).to_dense() + col_sum = 1e-7 + torch.sparse.sum(adj.t(), -1).to_dense() + return torch.pow(row_sum, -0.5)[indices[0]] * torch.pow(col_sum, -0.5)[indices[1]] + + def get_edge_info(self): + rows = torch.from_numpy(self.interaction_matrix.row) + cols = torch.from_numpy(self.interaction_matrix.col) + edges = torch.stack([rows, cols]).type(torch.LongTensor) + vals = self._normalize_adj_m(edges, torch.Size((self.n_users, self.n_items))) + return edges, vals + + def pre_epoch_processing(self): + """ + Degree-sensitive edge dropout (disabled when dropout <= 0). + Edges with higher normalised weight are more likely to be kept, + encouraging the model to retain informative interactions. + """ + if self.dropout <= 0.0: + self.masked_adj = self.norm_adj + return + degree_len = int(self.edge_values.size(0) * (1.0 - self.dropout)) + degree_idx = torch.multinomial(self.edge_values, degree_len) + keep_idx = self.edge_indices[:, degree_idx] + keep_vals = self._normalize_adj_m(keep_idx, torch.Size((self.n_users, self.n_items))) + all_vals = torch.cat((keep_vals, keep_vals)) + keep_idx[1] += self.n_users # shift item indices into the joint space + all_idx = torch.cat((keep_idx, torch.flip(keep_idx, [0])), dim=1) + self.masked_adj = torch.sparse.FloatTensor( + all_idx, all_vals, self.norm_adj.shape + ).to(self.device) + + # ══════════════════════════════════════════════════════════════════════ + # LightGCN propagation + # ══════════════════════════════════════════════════════════════════════ + + def lightgcn_propagate(self, adj): + """ + Standard LightGCN: layer-wise neighbourhood aggregation over the + user-item graph, followed by mean-pooling across all layer outputs. + Returns (user_emb, item_emb) each of shape (N, embedding_dim). + """ + ego = torch.cat( + [self.user_embedding.weight, self.item_id_embedding.weight], dim=0 + ) + layers = [ego] + for _ in range(self.n_ui_layers): + ego = torch.sparse.mm(adj, ego) + layers.append(ego) + # mean of all layer embeddings (LightGCN aggregation) + out = torch.stack(layers, dim=1).mean(dim=1) + return torch.split(out, [self.n_users, self.n_items], dim=0) + + # ══════════════════════════════════════════════════════════════════════ + # Multimodal fusion + # ══════════════════════════════════════════════════════════════════════ + + def _get_mm_feats(self): + """Project raw pretrained features into feat_embed_dim.""" + feats = [] + if self.v_feat is not None: + feats.append(self.image_trs(self.image_embedding.weight)) + if self.t_feat is not None: + feats.append(self.text_trs(self.text_embedding.weight)) + return feats # list of (n_items, feat_embed_dim) tensors + + def _apply_fusion(self, tensors, concat_layer=None): + """ + Fuse a list of tensors according to self.fusion_mode. + + mean / sum — element-wise ops; all tensors must share shape. + concat — cat along last dim, then project via concat_layer. + """ + if self.fusion_mode == 'mean': + return torch.stack(tensors).mean(dim=0) + elif self.fusion_mode == 'sum': + return torch.stack(tensors).sum(dim=0) + else: # concat + return concat_layer(torch.cat(tensors, dim=-1)) + + def fuse_item_embeddings(self, item_emb): + """ + Late-fuse LightGCN item embeddings with multimodal features. + + Weighting strategies + -------------------- + alpha : learnable sigmoid gate; item_emb scaled by α, + each modal feat scaled by (1−α), then jointly fused. + normalized : L2-normalise every embedding before fusing; the ID + embedding is additionally scaled by n_modalities so + all modalities contribute roughly equally. + equal : two-stage — first fuse modalities together, then + fuse the result with the ID embedding (equal weight). + """ + mm_feats = self._get_mm_feats() + if not mm_feats: + return item_emb + + if self.weighting == 'alpha': + alpha = torch.sigmoid(self.mm_alpha) + tensors = [item_emb * alpha] + [f * (1.0 - alpha) for f in mm_feats] + return self._apply_fusion( + tensors, + concat_layer=self.all_concat_layer if self.fusion_mode == 'concat' else None, + ) + + elif self.weighting == 'normalized': + tensors = ( + [F.normalize(item_emb) * self.n_modalities] + + [F.normalize(f) for f in mm_feats] + ) + return self._apply_fusion( + tensors, + concat_layer=self.all_concat_layer if self.fusion_mode == 'concat' else None, + ) + + else: # equal + # Step 1: reduce multiple modalities to a single mm embedding + if len(mm_feats) > 1: + mm_fused = self._apply_fusion( + mm_feats, + concat_layer=self.mm_concat_layer if self.fusion_mode == 'concat' else None, + ) + else: + mm_fused = mm_feats[0] # single modality — no reduction needed + + # Step 2: combine ID embedding with the fused modal embedding + return self._apply_fusion( + [item_emb, mm_fused], + concat_layer=self.id_mm_concat_layer if self.fusion_mode == 'concat' else None, + ) + + # ══════════════════════════════════════════════════════════════════════ + # Forward pass + # ══════════════════════════════════════════════════════════════════════ + + def forward(self, adj): + user_emb, item_emb = self.lightgcn_propagate(adj) + item_emb = self.fuse_item_embeddings(item_emb) + return user_emb, item_emb + + # ══════════════════════════════════════════════════════════════════════ + # Loss & prediction + # ══════════════════════════════════════════════════════════════════════ + + def bpr_loss(self, users, pos_items, neg_items): + pos_scores = (users * pos_items).sum(dim=1) + neg_scores = (users * neg_items).sum(dim=1) + return -F.logsigmoid(pos_scores - neg_scores).mean() + + def calculate_loss(self, interaction): + users, pos_items, neg_items = interaction[0], interaction[1], interaction[2] + + ua_emb, ia_emb = self.forward(self.masked_adj) + mf_loss = self.bpr_loss(ua_emb[users], ia_emb[pos_items], ia_emb[neg_items]) + + # L2 regularisation on the initial (un-propagated) ID embeddings only, + # consistent with the LightGCN / BPR-MF convention. + reg_loss = ( + self.user_embedding.weight[users].norm(2).pow(2) + + self.item_id_embedding.weight[pos_items].norm(2).pow(2) + + self.item_id_embedding.weight[neg_items].norm(2).pow(2) + ) / (2 * len(users)) + + return mf_loss + self.reg_weight * reg_loss + + def full_sort_predict(self, interaction): + user = interaction[0] + user_emb, item_emb = self.forward(self.norm_adj) + return torch.matmul(user_emb[user], item_emb.t()) \ No newline at end of file From 5a388444f31865f8270dec180a98e4ffe7e92b19 Mon Sep 17 00:00:00 2001 From: Giuseppe Spillo <44213842+giuspillo@users.noreply.github.com> Date: Fri, 26 Jun 2026 01:21:30 +0200 Subject: [PATCH 2/3] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a733c35..1708e62 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ source code at: `src\models` | DA-MRS | [Improving Multi-modal Recommender Systems by Denoising and Aligning Multi-modal Content and User Feedback](https://dl.acm.org/doi/10.1145/3637528.3671703) | KDD'24 | damrs.py | | SMORE | [Spectrum-based Modality Representation Fusion Graph Convolutional Network for Multimodal Recommendation](https://arxiv.org/abs/2412.14978) | WSDM'25 | smore.py | | PGL | [Mind Individual Information! Principal Graph Learning for Multimedia Recommendation](https://ojs.aaai.org/index.php/AAAI/article/view/33429) | AAAI'25 | pgl.py | +| MMGCF | [Multimodal Graph Collaborative Filtering for Recommendation with Graph Convolutional Networks](https://dl.acm.org/doi/abs/10.1145/3774935.3806158) | UMAP'26 | mmgcf.py | #### Please consider to cite our paper if this framework helps you, thanks: From 99c370db78b51073decd7bc1fc44b742e96a6bae Mon Sep 17 00:00:00 2001 From: Giuseppe Spillo <44213842+giuspillo@users.noreply.github.com> Date: Fri, 11 Sep 2026 18:13:55 +0200 Subject: [PATCH 3/3] added beyond-accuracy metrics --- src/configs/overall.yaml | 8 +- src/utils/metrics.py | 181 +++++++++++++++++++++++++++++++++++- src/utils/topk_evaluator.py | 96 ++++++++++++++++--- 3 files changed, 267 insertions(+), 18 deletions(-) diff --git a/src/configs/overall.yaml b/src/configs/overall.yaml index 328146f..127219d 100644 --- a/src/configs/overall.yaml +++ b/src/configs/overall.yaml @@ -39,10 +39,16 @@ field_separator: "\t" # evaluation settings -metrics: ["Recall", "NDCG", "Precision", "MAP"] +# accuracy metrics: Recall, Recall2, Precision, NDCG, MAP +# beyond-accuracy (item-distribution) metrics: GiniIndex, AveragePopularity, TailPercentage, ItemCoverage, ShannonEntropy +metrics: ["Recall", "NDCG", "Precision", "MAP", "GiniIndex", "AveragePopularity", "TailPercentage", "ItemCoverage", "ShannonEntropy"] topk: [5, 10, 20, 50] valid_metric: Recall@20 eval_batch_size: 4096 +# threshold defining the "long tail" item set for TailPercentage: a fraction +# in (0, 1] of the least-popular training items, or an absolute popularity +# count if > 1 +tail_ratio: 0.1 # use_raw_features: False diff --git a/src/utils/metrics.py b/src/utils/metrics.py index d76d09e..ac66bc0 100644 --- a/src/utils/metrics.py +++ b/src/utils/metrics.py @@ -48,13 +48,13 @@ def ndcg_(pos_index, pos_len): len_rank = np.full_like(pos_len, pos_index.shape[1]) idcg_len = np.where(pos_len > len_rank, len_rank, pos_len) - iranks = np.zeros_like(pos_index, dtype=np.float) + iranks = np.zeros_like(pos_index, dtype=np.float64) iranks[:, :] = np.arange(1, pos_index.shape[1] + 1) idcg = np.cumsum(1.0 / np.log2(iranks + 1), axis=1) for row, idx in enumerate(idcg_len): idcg[row, idx:] = idcg[row, idx - 1] - ranks = np.zeros_like(pos_index, dtype=np.float) + ranks = np.zeros_like(pos_index, dtype=np.float64) ranks[:, :] = np.arange(1, pos_index.shape[1] + 1) dcg = 1.0 / np.log2(ranks + 1) dcg = np.cumsum(np.where(pos_index, dcg, 0), axis=1) @@ -78,10 +78,10 @@ def map_(pos_index, pos_len): \end{align*} """ pre = pos_index.cumsum(axis=1) / np.arange(1, pos_index.shape[1] + 1) - sum_pre = np.cumsum(pre * pos_index.astype(np.float), axis=1) + sum_pre = np.cumsum(pre * pos_index.astype(np.float64), axis=1) len_rank = np.full_like(pos_len, pos_index.shape[1]) actual_len = np.where(pos_len > len_rank, len_rank, pos_len) - result = np.zeros_like(pos_index, dtype=np.float) + result = np.zeros_like(pos_index, dtype=np.float64) for row, lens in enumerate(actual_len): ranges = np.arange(1, pos_index.shape[1]+1) ranges[lens:] = ranges[lens - 1] @@ -116,3 +116,176 @@ def precision_(pos_index, pos_len): 'precision': precision_, 'map': map_, } + + +""" +############################ +Beyond-accuracy (item-distribution) metrics. + +Unlike the metrics above, these do not measure whether recommended items are +relevant to a user. They measure properties of the *distribution* of items +recommended across the whole user base, so they need the actual recommended +item ids (not just a relevance mask) plus item popularity computed from the +training interactions. Implementations follow RecBole's definitions +(recbole.evaluator.metrics) so that results are directly comparable. +""" + + +def giniindex_(item_matrix, num_items): + r"""GiniIndex_ measures the inequality of the item recommendation frequency + distribution: 0 means every item is recommended equally often, 1 means a + single item absorbs every recommendation slot. Lower is more equitable. + + .. _GiniIndex: https://en.wikipedia.org/wiki/Gini_coefficient + + .. math:: + \mathrm {GiniIndex@K}=\frac{\sum_{i=1}^{|I|}(2i-|I|-1) P_{(i)}}{|I| \sum_{i=1}^{|I|} P_{(i)}} + + :math:`P_{(i)}` is the number of times item :math:`i` appears across all + users' top-K lists, items indexed in non-decreasing order of :math:`P`. + Items never recommended contribute 0 to the count but still occupy the + lowest ranks of the distribution. + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param num_items: total number of items in the catalog (|I|). + """ + max_k = item_matrix.shape[1] + result = np.zeros(max_k) + cum_counts = np.zeros(num_items, dtype=np.int64) + idx = np.arange(1, num_items + 1) + for k in range(max_k): + cum_counts += np.bincount(item_matrix[:, k], minlength=num_items) + sorted_count = np.sort(cum_counts) + total_num = sorted_count.sum() + result[k] = np.sum((2 * idx - num_items - 1) * sorted_count) / (num_items * total_num) + return result + + +def averagepopularity_(item_matrix, item_count, num_items): + r"""AveragePopularity_ computes the average training-set popularity of the + recommended items, averaged over users. Higher means the model leans on + already-popular items. + + .. math:: + \mathrm {AveragePopularity@K}=\frac{1}{|U|} \sum_{u \in U} \frac{\sum_{i \in R_u} \phi(i)}{|R_u|} + + :math:`\phi(i)` is the number of interactions of item :math:`i` in the + training data (0 if the item never appeared in training). + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param item_count: (num_items,) array, training-set interaction count per item. + :param num_items: total number of items in the catalog (unused, kept for a + uniform signature with the other item-distribution metrics). + """ + pop = item_count[item_matrix].astype(np.float64) + cum_avg = pop.cumsum(axis=1) / np.arange(1, item_matrix.shape[1] + 1) + return cum_avg.mean(axis=0) + + +def tailpercentage_(item_matrix, item_count, num_items, tail_ratio=0.1): + r"""TailPercentage_ (a.k.a. Average Percentage of Long-tail items, APLT) + computes the fraction of recommended items that belong to the long tail, + averaged over users. Higher means the model surfaces more long-tail items. + + .. _TailPercentage: https://en.wikipedia.org/wiki/Long_tail#Criticisms + + .. math:: + \mathrm {TailPercentage@K}=\frac{1}{|U|} \sum_{u \in U} \frac{\sum_{i \in R_u} \delta(i \in T)}{|R_u|} + + :math:`T` is the set of long-tail items: the least-popular items in the + training data. If ``tail_ratio`` is in (0, 1], :math:`T` is the bottom + ``tail_ratio`` fraction (by training popularity, ties broken by item id) + of items that were seen at least once in training. If ``tail_ratio`` > 1, + :math:`T` is every item whose training popularity is <= ``tail_ratio``. + Items never seen in training are not counted as long-tail (they are a + cold-start/coverage issue, not a popularity-bias one). + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param item_count: (num_items,) array, training-set interaction count per item. + :param num_items: total number of items in the catalog. + :param tail_ratio: threshold defining the long tail, see above. Default 0.1. + """ + observed = np.nonzero(item_count > 0)[0] + if tail_ratio > 1: + tail_mask = (item_count > 0) & (item_count <= tail_ratio) + else: + order = np.lexsort((observed, item_count[observed])) + cut = max(int(len(observed) * tail_ratio), 1) + tail_items = observed[order][:cut] + tail_mask = np.zeros(num_items, dtype=bool) + tail_mask[tail_items] = True + is_tail = tail_mask[item_matrix].astype(np.float64) + cum_avg = is_tail.cumsum(axis=1) / np.arange(1, item_matrix.shape[1] + 1) + return cum_avg.mean(axis=0) + + +def itemcoverage_(item_matrix, num_items): + r"""ItemCoverage_ measures the fraction of the item catalog that has been + recommended to at least one user within the top-K. Higher means the + model surfaces a larger share of the catalog. + + .. _ItemCoverage: https://en.wikipedia.org/wiki/Long_tail#Criticisms + + .. math:: + \mathrm {ItemCoverage@K}=\frac{|\bigcup_{u \in U} R_u@K|}{|I|} + + :math:`R_u@K` is the set of items recommended to user :math:`u` within + the top-K, and :math:`|I|` is the catalog size. + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param num_items: total number of items in the catalog (|I|). + """ + max_k = item_matrix.shape[1] + result = np.zeros(max_k) + seen = np.zeros(num_items, dtype=bool) + for k in range(max_k): + seen[item_matrix[:, k]] = True + result[k] = seen.sum() / num_items + return result + + +def shannonentropy_(item_matrix, num_items): + r"""ShannonEntropy_ measures the diversity of the item recommendation + frequency distribution across users: 0 means every recommendation slot + is filled by a single item, :math:`\ln(|I|)` is the maximum, reached when + every item is recommended equally often. Higher is more diverse. + + .. _ShannonEntropy: https://en.wikipedia.org/wiki/Entropy_(information_theory) + + .. math:: + \mathrm {ShannonEntropy@K}=-\sum_{i=1}^{|I|} p_{(i)} \log p_{(i)} + + :math:`p_{(i)}=P_{(i)}/\sum_{j} P_{(j)}` is the fraction of all top-K + recommendation slots occupied by item :math:`i`. Items never recommended + (:math:`p_{(i)}=0`) contribute 0. Natural log is used, so results are in + nats. + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param num_items: total number of items in the catalog (|I|). + """ + max_k = item_matrix.shape[1] + result = np.zeros(max_k) + cum_counts = np.zeros(num_items, dtype=np.int64) + for k in range(max_k): + cum_counts += np.bincount(item_matrix[:, k], minlength=num_items) + probs = cum_counts[cum_counts > 0] / cum_counts.sum() + result[k] = -np.sum(probs * np.log(probs)) + return result + + +"""Function name and function mapper for beyond-accuracy (item-distribution) +metrics. These take a different signature than `metrics_dict` above (they +need recommended item ids + item popularity, not a relevance mask), so they +are kept in a separate registry. +""" +item_metrics_dict = { + 'giniindex': giniindex_, + 'averagepopularity': averagepopularity_, + 'tailpercentage': tailpercentage_, + 'itemcoverage': itemcoverage_, + 'shannonentropy': shannonentropy_, +} + +# item-distribution metrics whose implementation only needs the recommended +# item ids + catalog size (not per-item training popularity) +num_items_only_item_metrics = frozenset(('giniindex', 'itemcoverage', 'shannonentropy')) diff --git a/src/utils/topk_evaluator.py b/src/utils/topk_evaluator.py index a36adab..0674fcc 100644 --- a/src/utils/topk_evaluator.py +++ b/src/utils/topk_evaluator.py @@ -7,13 +7,19 @@ import numpy as np import pandas as pd import torch -from utils.metrics import metrics_dict +from utils.metrics import metrics_dict, item_metrics_dict, num_items_only_item_metrics from torch.nn.utils.rnn import pad_sequence from utils.utils import get_local_time -# These metrics are typical in topk recommendations +# These metrics are typical in topk recommendations, evaluated against user relevance topk_metrics = {metric.lower(): metric for metric in ['Recall', 'Recall2', 'Precision', 'NDCG', 'MAP']} +# These "beyond-accuracy" metrics evaluate the distribution of recommended items +# across users (using item popularity computed from the training set) rather +# than their relevance to any single user +item_metrics = {metric.lower(): metric for metric in + ['GiniIndex', 'AveragePopularity', 'TailPercentage', 'ItemCoverage', 'ShannonEntropy']} +all_metrics = {**topk_metrics, **item_metrics} class TopKEvaluator(object): @@ -31,7 +37,11 @@ def __init__(self, config): self.metrics = config['metrics'] self.topk = config['topk'] self.save_recom_result = config['save_recommended_topk'] + self.tail_ratio = config['tail_ratio'] if config['tail_ratio'] else 0.1 self._check_args() + # cache of (item_count, num_items) per training dataset, keyed by id() since + # the training dataset is fixed for the lifetime of a trainer/evaluator + self._item_popularity_cache = {} def collect(self, interaction, scores_tensor, full=False): """collect the topk intermediate result of one batch, this function mainly @@ -86,21 +96,55 @@ def evaluate(self, batch_matrix_list, eval_data, is_test=False, idx=0): x_df = x_df.astype(int) x_df.to_csv(file_path, sep='\t', index=False) assert len(pos_len_list) == len(topk_index) - # if recom right? - bool_rec_matrix = [] - for m, n in zip(pos_items, topk_index): - bool_rec_matrix.append([True if i in m else False for i in n]) - bool_rec_matrix = np.asarray(bool_rec_matrix) - # get metrics + # compute each metric's result array (shape (max_k,)) keyed by metric + # name first, then assemble metric_dict below following the order + # metrics were declared in the config, regardless of which of the two + # groups (relevance-based vs. item-distribution) they belong to + results = {} + + # relevance-based (accuracy) metrics + if self.topk_metrics: + # if recom right? + bool_rec_matrix = [] + for m, n in zip(pos_items, topk_index): + bool_rec_matrix.append([True if i in m else False for i in n]) + bool_rec_matrix = np.asarray(bool_rec_matrix) + + result_list = self._calculate_metrics(pos_len_list, bool_rec_matrix) + results.update(zip(self.topk_metrics, result_list)) + + # beyond-accuracy (item-distribution) metrics + if self.item_metrics: + item_count, num_items = self._get_item_popularity(eval_data) + result_list = self._calculate_item_metrics(topk_index, item_count, num_items) + results.update(zip(self.item_metrics, result_list)) + metric_dict = {} - result_list = self._calculate_metrics(pos_len_list, bool_rec_matrix) - for metric, value in zip(self.metrics, result_list): + for metric in self.metrics: + value = results[metric] for k in self.topk: key = '{}@{}'.format(metric, k) metric_dict[key] = round(value[k - 1], 4) + return metric_dict + def _get_item_popularity(self, eval_data): + """Item popularity (interaction count) computed from the training set, + as a dense (num_items,) array, plus the catalog size. Cached per + training dataset since it does not change across evaluation calls. + """ + train_dataset = eval_data.additional_dataset + key = id(train_dataset) + if key not in self._item_popularity_cache: + iid_field = train_dataset.iid_field + num_items = train_dataset.item_num + counts = train_dataset.df[iid_field].value_counts() + item_count = np.zeros(num_items, dtype=np.int64) + item_count[counts.index.values.astype(int)] = counts.values + self._item_popularity_cache[key] = (item_count, num_items) + return self._item_popularity_cache[key] + def _check_args(self): # Check metrics if isinstance(self.metrics, (str, list)): @@ -111,9 +155,13 @@ def _check_args(self): # Convert metric to lowercase for m in self.metrics: - if m.lower() not in topk_metrics: + if m.lower() not in all_metrics: raise ValueError("There is no user grouped topk metric named {}!".format(m)) self.metrics = [metric.lower() for metric in self.metrics] + # split into relevance-based vs. item-distribution (beyond-accuracy) metrics, + # since they need different inputs to compute + self.topk_metrics = [m for m in self.metrics if m in topk_metrics] + self.item_metrics = [m for m in self.metrics if m in item_metrics] # Check topk: if isinstance(self.topk, (int, list)): @@ -136,14 +184,36 @@ def _calculate_metrics(self, pos_len_list, topk_index): np.ndarray: a matrix which contains the metrics result """ result_list = [] - for metric in self.metrics: + for metric in self.topk_metrics: metric_fuc = metrics_dict[metric.lower()] result = metric_fuc(topk_index, pos_len_list) result_list.append(result) return np.stack(result_list, axis=0) + def _calculate_item_metrics(self, topk_index, item_count, num_items): + """calculate the beyond-accuracy (item-distribution) metrics + + Args: + topk_index (np.ndarray): matrix of recommended item ids, (n_users, max_k) + item_count (np.ndarray): training-set interaction count per item, (num_items,) + num_items (int): total number of items in the catalog + Returns: + list: one np.ndarray per metric, each of shape (max_k,) + """ + result_list = [] + for metric in self.item_metrics: + metric_fuc = item_metrics_dict[metric] + if metric in num_items_only_item_metrics: + result = metric_fuc(topk_index, num_items) + elif metric == 'tailpercentage': + result = metric_fuc(topk_index, item_count, num_items, self.tail_ratio) + else: + result = metric_fuc(topk_index, item_count, num_items) + result_list.append(result) + return result_list + def __str__(self): mesg = 'The TopK Evaluator Info:\n' + '\tMetrics:[' + ', '.join( - [topk_metrics[metric.lower()] for metric in self.metrics]) \ + [all_metrics[metric.lower()] for metric in self.metrics]) \ + '], TopK:[' + ', '.join(map(str, self.topk)) + ']' return mesg