Added beyond-accuracy evaluation metrics

src/configs/overall.yaml

@@ -39,10 +39,16 @@ field_separator: "\t"
 
 
 # evaluation settings
-metrics: ["Recall", "NDCG", "Precision", "MAP"]
+# accuracy metrics: Recall, Recall2, Precision, NDCG, MAP
+# beyond-accuracy (item-distribution) metrics: GiniIndex, AveragePopularity, TailPercentage, ItemCoverage, ShannonEntropy
+metrics: ["Recall", "NDCG", "Precision", "MAP", "GiniIndex", "AveragePopularity", "TailPercentage", "ItemCoverage", "ShannonEntropy"]
 topk: [5, 10, 20, 50]
 valid_metric: Recall@20
 eval_batch_size: 4096
+# threshold defining the "long tail" item set for TailPercentage: a fraction
+# in (0, 1] of the least-popular training items, or an absolute popularity
+# count if > 1
+tail_ratio: 0.1
 
 #
 use_raw_features: False

src/utils/metrics.py

@@ -48,13 +48,13 @@ def ndcg_(pos_index, pos_len):
     len_rank = np.full_like(pos_len, pos_index.shape[1])
     idcg_len = np.where(pos_len > len_rank, len_rank, pos_len)
 
-    iranks = np.zeros_like(pos_index, dtype=np.float)
+    iranks = np.zeros_like(pos_index, dtype=np.float64)
     iranks[:, :] = np.arange(1, pos_index.shape[1] + 1)
     idcg = np.cumsum(1.0 / np.log2(iranks + 1), axis=1)
     for row, idx in enumerate(idcg_len):
         idcg[row, idx:] = idcg[row, idx - 1]
 
-    ranks = np.zeros_like(pos_index, dtype=np.float)
+    ranks = np.zeros_like(pos_index, dtype=np.float64)
     ranks[:, :] = np.arange(1, pos_index.shape[1] + 1)
     dcg = 1.0 / np.log2(ranks + 1)
     dcg = np.cumsum(np.where(pos_index, dcg, 0), axis=1)
@@ -78,10 +78,10 @@ def map_(pos_index, pos_len):
         \end{align*}
     """
     pre = pos_index.cumsum(axis=1) / np.arange(1, pos_index.shape[1] + 1)
-    sum_pre = np.cumsum(pre * pos_index.astype(np.float), axis=1)
+    sum_pre = np.cumsum(pre * pos_index.astype(np.float64), axis=1)
     len_rank = np.full_like(pos_len, pos_index.shape[1])
     actual_len = np.where(pos_len > len_rank, len_rank, pos_len)
-    result = np.zeros_like(pos_index, dtype=np.float)
+    result = np.zeros_like(pos_index, dtype=np.float64)
     for row, lens in enumerate(actual_len):
         ranges = np.arange(1, pos_index.shape[1]+1)
         ranges[lens:] = ranges[lens - 1]
@@ -116,3 +116,176 @@ def precision_(pos_index, pos_len):
     'precision': precision_,
     'map': map_,
 }
+
+
+"""
+############################
+Beyond-accuracy (item-distribution) metrics.
+
+Unlike the metrics above, these do not measure whether recommended items are
+relevant to a user. They measure properties of the *distribution* of items
+recommended across the whole user base, so they need the actual recommended
+item ids (not just a relevance mask) plus item popularity computed from the
+training interactions. Implementations follow RecBole's definitions
+(recbole.evaluator.metrics) so that results are directly comparable.
+"""
+
+
+def giniindex_(item_matrix, num_items):
+    r"""GiniIndex_ measures the inequality of the item recommendation frequency
+    distribution: 0 means every item is recommended equally often, 1 means a
+    single item absorbs every recommendation slot. Lower is more equitable.
+
+    .. _GiniIndex: https://en.wikipedia.org/wiki/Gini_coefficient
+
+    .. math::
+        \mathrm {GiniIndex@K}=\frac{\sum_{i=1}^{|I|}(2i-|I|-1) P_{(i)}}{|I| \sum_{i=1}^{|I|} P_{(i)}}
+
+    :math:`P_{(i)}` is the number of times item :math:`i` appears across all
+    users' top-K lists, items indexed in non-decreasing order of :math:`P`.
+    Items never recommended contribute 0 to the count but still occupy the
+    lowest ranks of the distribution.
+
+    :param item_matrix: (n_users, max_k) matrix of recommended item ids.
+    :param num_items: total number of items in the catalog (|I|).
+    """
+    max_k = item_matrix.shape[1]
+    result = np.zeros(max_k)
+    cum_counts = np.zeros(num_items, dtype=np.int64)
+    idx = np.arange(1, num_items + 1)
+    for k in range(max_k):
+        cum_counts += np.bincount(item_matrix[:, k], minlength=num_items)
+        sorted_count = np.sort(cum_counts)
+        total_num = sorted_count.sum()
+        result[k] = np.sum((2 * idx - num_items - 1) * sorted_count) / (num_items * total_num)
+    return result
+
+
+def averagepopularity_(item_matrix, item_count, num_items):
+    r"""AveragePopularity_ computes the average training-set popularity of the
+    recommended items, averaged over users. Higher means the model leans on
+    already-popular items.
+
+    .. math::
+        \mathrm {AveragePopularity@K}=\frac{1}{|U|} \sum_{u \in U} \frac{\sum_{i \in R_u} \phi(i)}{|R_u|}
+
+    :math:`\phi(i)` is the number of interactions of item :math:`i` in the
+    training data (0 if the item never appeared in training).
+
+    :param item_matrix: (n_users, max_k) matrix of recommended item ids.
+    :param item_count: (num_items,) array, training-set interaction count per item.
+    :param num_items: total number of items in the catalog (unused, kept for a
+        uniform signature with the other item-distribution metrics).
+    """
+    pop = item_count[item_matrix].astype(np.float64)
+    cum_avg = pop.cumsum(axis=1) / np.arange(1, item_matrix.shape[1] + 1)
+    return cum_avg.mean(axis=0)
+
+
+def tailpercentage_(item_matrix, item_count, num_items, tail_ratio=0.1):
+    r"""TailPercentage_ (a.k.a. Average Percentage of Long-tail items, APLT)
+    computes the fraction of recommended items that belong to the long tail,
+    averaged over users. Higher means the model surfaces more long-tail items.
+
+    .. _TailPercentage: https://en.wikipedia.org/wiki/Long_tail#Criticisms
+
+    .. math::
+        \mathrm {TailPercentage@K}=\frac{1}{|U|} \sum_{u \in U} \frac{\sum_{i \in R_u} \delta(i \in T)}{|R_u|}
+
+    :math:`T` is the set of long-tail items: the least-popular items in the
+    training data. If ``tail_ratio`` is in (0, 1], :math:`T` is the bottom
+    ``tail_ratio`` fraction (by training popularity, ties broken by item id)
+    of items that were seen at least once in training. If ``tail_ratio`` > 1,
+    :math:`T` is every item whose training popularity is <= ``tail_ratio``.
+    Items never seen in training are not counted as long-tail (they are a
+    cold-start/coverage issue, not a popularity-bias one).
+
+    :param item_matrix: (n_users, max_k) matrix of recommended item ids.
+    :param item_count: (num_items,) array, training-set interaction count per item.
+    :param num_items: total number of items in the catalog.
+    :param tail_ratio: threshold defining the long tail, see above. Default 0.1.
+    """
+    observed = np.nonzero(item_count > 0)[0]
+    if tail_ratio > 1:
+        tail_mask = (item_count > 0) & (item_count <= tail_ratio)
+    else:
+        order = np.lexsort((observed, item_count[observed]))
+        cut = max(int(len(observed) * tail_ratio), 1)
+        tail_items = observed[order][:cut]
+        tail_mask = np.zeros(num_items, dtype=bool)
+        tail_mask[tail_items] = True
+    is_tail = tail_mask[item_matrix].astype(np.float64)
+    cum_avg = is_tail.cumsum(axis=1) / np.arange(1, item_matrix.shape[1] + 1)
+    return cum_avg.mean(axis=0)
+
+
+def itemcoverage_(item_matrix, num_items):
+    r"""ItemCoverage_ measures the fraction of the item catalog that has been
+    recommended to at least one user within the top-K. Higher means the
+    model surfaces a larger share of the catalog.
+
+    .. _ItemCoverage: https://en.wikipedia.org/wiki/Long_tail#Criticisms
+
+    .. math::
+        \mathrm {ItemCoverage@K}=\frac{|\bigcup_{u \in U} R_u@K|}{|I|}
+
+    :math:`R_u@K` is the set of items recommended to user :math:`u` within
+    the top-K, and :math:`|I|` is the catalog size.
+
+    :param item_matrix: (n_users, max_k) matrix of recommended item ids.
+    :param num_items: total number of items in the catalog (|I|).
+    """
+    max_k = item_matrix.shape[1]
+    result = np.zeros(max_k)
+    seen = np.zeros(num_items, dtype=bool)
+    for k in range(max_k):
+        seen[item_matrix[:, k]] = True
+        result[k] = seen.sum() / num_items
+    return result
+
+
+def shannonentropy_(item_matrix, num_items):
+    r"""ShannonEntropy_ measures the diversity of the item recommendation
+    frequency distribution across users: 0 means every recommendation slot
+    is filled by a single item, :math:`\ln(|I|)` is the maximum, reached when
+    every item is recommended equally often. Higher is more diverse.
+
+    .. _ShannonEntropy: https://en.wikipedia.org/wiki/Entropy_(information_theory)
+
+    .. math::
+        \mathrm {ShannonEntropy@K}=-\sum_{i=1}^{|I|} p_{(i)} \log p_{(i)}
+
+    :math:`p_{(i)}=P_{(i)}/\sum_{j} P_{(j)}` is the fraction of all top-K
+    recommendation slots occupied by item :math:`i`. Items never recommended
+    (:math:`p_{(i)}=0`) contribute 0. Natural log is used, so results are in
+    nats.
+
+    :param item_matrix: (n_users, max_k) matrix of recommended item ids.
+    :param num_items: total number of items in the catalog (|I|).
+    """
+    max_k = item_matrix.shape[1]
+    result = np.zeros(max_k)
+    cum_counts = np.zeros(num_items, dtype=np.int64)
+    for k in range(max_k):
+        cum_counts += np.bincount(item_matrix[:, k], minlength=num_items)
+        probs = cum_counts[cum_counts > 0] / cum_counts.sum()
+        result[k] = -np.sum(probs * np.log(probs))
+    return result
+
+
+"""Function name and function mapper for beyond-accuracy (item-distribution)
+metrics. These take a different signature than `metrics_dict` above (they
+need recommended item ids + item popularity, not a relevance mask), so they
+are kept in a separate registry.
+"""
+item_metrics_dict = {
+    'giniindex': giniindex_,
+    'averagepopularity': averagepopularity_,
+    'tailpercentage': tailpercentage_,
+    'itemcoverage': itemcoverage_,
+    'shannonentropy': shannonentropy_,
+}
+
+# item-distribution metrics whose implementation only needs the recommended
+# item ids + catalog size (not per-item training popularity)
+num_items_only_item_metrics = frozenset(('giniindex', 'itemcoverage', 'shannonentropy'))

src/utils/topk_evaluator.py

@@ -7,13 +7,19 @@
 import numpy as np
 import pandas as pd
 import torch
-from utils.metrics import metrics_dict
+from utils.metrics import metrics_dict, item_metrics_dict, num_items_only_item_metrics
 from torch.nn.utils.rnn import pad_sequence
 from utils.utils import get_local_time
 
 
-# These metrics are typical in topk recommendations
+# These metrics are typical in topk recommendations, evaluated against user relevance
 topk_metrics = {metric.lower(): metric for metric in ['Recall', 'Recall2', 'Precision', 'NDCG', 'MAP']}
+# These "beyond-accuracy" metrics evaluate the distribution of recommended items
+# across users (using item popularity computed from the training set) rather
+# than their relevance to any single user
+item_metrics = {metric.lower(): metric for metric in
+                 ['GiniIndex', 'AveragePopularity', 'TailPercentage', 'ItemCoverage', 'ShannonEntropy']}
+all_metrics = {**topk_metrics, **item_metrics}
 
 
 class TopKEvaluator(object):
@@ -31,7 +37,11 @@ def __init__(self, config):
         self.metrics = config['metrics']
         self.topk = config['topk']
         self.save_recom_result = config['save_recommended_topk']
+        self.tail_ratio = config['tail_ratio'] if config['tail_ratio'] else 0.1
         self._check_args()
+        # cache of (item_count, num_items) per training dataset, keyed by id() since
+        # the training dataset is fixed for the lifetime of a trainer/evaluator
+        self._item_popularity_cache = {}
 
     def collect(self, interaction, scores_tensor, full=False):
         """collect the topk intermediate result of one batch, this function mainly
@@ -86,21 +96,55 @@ def evaluate(self, batch_matrix_list, eval_data, is_test=False, idx=0):
             x_df = x_df.astype(int)
             x_df.to_csv(file_path, sep='\t', index=False)
         assert len(pos_len_list) == len(topk_index)
-        # if recom right?
-        bool_rec_matrix = []
-        for m, n in zip(pos_items, topk_index):
-            bool_rec_matrix.append([True if i in m else False for i in n])
-        bool_rec_matrix = np.asarray(bool_rec_matrix)
 
-        # get metrics
+        # compute each metric's result array (shape (max_k,)) keyed by metric
+        # name first, then assemble metric_dict below following the order
+        # metrics were declared in the config, regardless of which of the two
+        # groups (relevance-based vs. item-distribution) they belong to
+        results = {}
+
+        # relevance-based (accuracy) metrics
+        if self.topk_metrics:
+            # if recom right?
+            bool_rec_matrix = []
+            for m, n in zip(pos_items, topk_index):
+                bool_rec_matrix.append([True if i in m else False for i in n])
+            bool_rec_matrix = np.asarray(bool_rec_matrix)
+
+            result_list = self._calculate_metrics(pos_len_list, bool_rec_matrix)
+            results.update(zip(self.topk_metrics, result_list))
+
+        # beyond-accuracy (item-distribution) metrics
+        if self.item_metrics:
+            item_count, num_items = self._get_item_popularity(eval_data)
+            result_list = self._calculate_item_metrics(topk_index, item_count, num_items)
+            results.update(zip(self.item_metrics, result_list))
+
         metric_dict = {}
-        result_list = self._calculate_metrics(pos_len_list, bool_rec_matrix)
-        for metric, value in zip(self.metrics, result_list):
+        for metric in self.metrics:
+            value = results[metric]
             for k in self.topk:
                 key = '{}@{}'.format(metric, k)
                 metric_dict[key] = round(value[k - 1], 4)
+
         return metric_dict
 
+    def _get_item_popularity(self, eval_data):
+        """Item popularity (interaction count) computed from the training set,
+        as a dense (num_items,) array, plus the catalog size. Cached per
+        training dataset since it does not change across evaluation calls.
+        """
+        train_dataset = eval_data.additional_dataset
+        key = id(train_dataset)
+        if key not in self._item_popularity_cache:
+            iid_field = train_dataset.iid_field
+            num_items = train_dataset.item_num
+            counts = train_dataset.df[iid_field].value_counts()
+            item_count = np.zeros(num_items, dtype=np.int64)
+            item_count[counts.index.values.astype(int)] = counts.values
+            self._item_popularity_cache[key] = (item_count, num_items)
+        return self._item_popularity_cache[key]
+
     def _check_args(self):
         # Check metrics
         if isinstance(self.metrics, (str, list)):
@@ -111,9 +155,13 @@ def _check_args(self):
 
         # Convert metric to lowercase
         for m in self.metrics:
-            if m.lower() not in topk_metrics:
+            if m.lower() not in all_metrics:
                 raise ValueError("There is no user grouped topk metric named {}!".format(m))
         self.metrics = [metric.lower() for metric in self.metrics]
+        # split into relevance-based vs. item-distribution (beyond-accuracy) metrics,
+        # since they need different inputs to compute
+        self.topk_metrics = [m for m in self.metrics if m in topk_metrics]
+        self.item_metrics = [m for m in self.metrics if m in item_metrics]
 
         # Check topk:
         if isinstance(self.topk, (int, list)):
@@ -136,14 +184,36 @@ def _calculate_metrics(self, pos_len_list, topk_index):
             np.ndarray: a matrix which contains the metrics result
         """
         result_list = []
-        for metric in self.metrics:
+        for metric in self.topk_metrics:
             metric_fuc = metrics_dict[metric.lower()]
             result = metric_fuc(topk_index, pos_len_list)
             result_list.append(result)
         return np.stack(result_list, axis=0)
 
+    def _calculate_item_metrics(self, topk_index, item_count, num_items):
+        """calculate the beyond-accuracy (item-distribution) metrics
+
+        Args:
+            topk_index (np.ndarray): matrix of recommended item ids, (n_users, max_k)
+            item_count (np.ndarray): training-set interaction count per item, (num_items,)
+            num_items (int): total number of items in the catalog
+        Returns:
+            list: one np.ndarray per metric, each of shape (max_k,)
+        """
+        result_list = []
+        for metric in self.item_metrics:
+            metric_fuc = item_metrics_dict[metric]
+            if metric in num_items_only_item_metrics:
+                result = metric_fuc(topk_index, num_items)
+            elif metric == 'tailpercentage':
+                result = metric_fuc(topk_index, item_count, num_items, self.tail_ratio)
+            else:
+                result = metric_fuc(topk_index, item_count, num_items)
+            result_list.append(result)
+        return result_list
+
     def __str__(self):
         mesg = 'The TopK Evaluator Info:\n' + '\tMetrics:[' + ', '.join(
-            [topk_metrics[metric.lower()] for metric in self.metrics]) \
+            [all_metrics[metric.lower()] for metric in self.metrics]) \
                + '], TopK:[' + ', '.join(map(str, self.topk)) + ']'
         return mesg