diff --git a/src/configs/overall.yaml b/src/configs/overall.yaml index 328146f..127219d 100644 --- a/src/configs/overall.yaml +++ b/src/configs/overall.yaml @@ -39,10 +39,16 @@ field_separator: "\t" # evaluation settings -metrics: ["Recall", "NDCG", "Precision", "MAP"] +# accuracy metrics: Recall, Recall2, Precision, NDCG, MAP +# beyond-accuracy (item-distribution) metrics: GiniIndex, AveragePopularity, TailPercentage, ItemCoverage, ShannonEntropy +metrics: ["Recall", "NDCG", "Precision", "MAP", "GiniIndex", "AveragePopularity", "TailPercentage", "ItemCoverage", "ShannonEntropy"] topk: [5, 10, 20, 50] valid_metric: Recall@20 eval_batch_size: 4096 +# threshold defining the "long tail" item set for TailPercentage: a fraction +# in (0, 1] of the least-popular training items, or an absolute popularity +# count if > 1 +tail_ratio: 0.1 # use_raw_features: False diff --git a/src/utils/metrics.py b/src/utils/metrics.py index d76d09e..ac66bc0 100644 --- a/src/utils/metrics.py +++ b/src/utils/metrics.py @@ -48,13 +48,13 @@ def ndcg_(pos_index, pos_len): len_rank = np.full_like(pos_len, pos_index.shape[1]) idcg_len = np.where(pos_len > len_rank, len_rank, pos_len) - iranks = np.zeros_like(pos_index, dtype=np.float) + iranks = np.zeros_like(pos_index, dtype=np.float64) iranks[:, :] = np.arange(1, pos_index.shape[1] + 1) idcg = np.cumsum(1.0 / np.log2(iranks + 1), axis=1) for row, idx in enumerate(idcg_len): idcg[row, idx:] = idcg[row, idx - 1] - ranks = np.zeros_like(pos_index, dtype=np.float) + ranks = np.zeros_like(pos_index, dtype=np.float64) ranks[:, :] = np.arange(1, pos_index.shape[1] + 1) dcg = 1.0 / np.log2(ranks + 1) dcg = np.cumsum(np.where(pos_index, dcg, 0), axis=1) @@ -78,10 +78,10 @@ def map_(pos_index, pos_len): \end{align*} """ pre = pos_index.cumsum(axis=1) / np.arange(1, pos_index.shape[1] + 1) - sum_pre = np.cumsum(pre * pos_index.astype(np.float), axis=1) + sum_pre = np.cumsum(pre * pos_index.astype(np.float64), axis=1) len_rank = np.full_like(pos_len, pos_index.shape[1]) actual_len = np.where(pos_len > len_rank, len_rank, pos_len) - result = np.zeros_like(pos_index, dtype=np.float) + result = np.zeros_like(pos_index, dtype=np.float64) for row, lens in enumerate(actual_len): ranges = np.arange(1, pos_index.shape[1]+1) ranges[lens:] = ranges[lens - 1] @@ -116,3 +116,176 @@ def precision_(pos_index, pos_len): 'precision': precision_, 'map': map_, } + + +""" +############################ +Beyond-accuracy (item-distribution) metrics. + +Unlike the metrics above, these do not measure whether recommended items are +relevant to a user. They measure properties of the *distribution* of items +recommended across the whole user base, so they need the actual recommended +item ids (not just a relevance mask) plus item popularity computed from the +training interactions. Implementations follow RecBole's definitions +(recbole.evaluator.metrics) so that results are directly comparable. +""" + + +def giniindex_(item_matrix, num_items): + r"""GiniIndex_ measures the inequality of the item recommendation frequency + distribution: 0 means every item is recommended equally often, 1 means a + single item absorbs every recommendation slot. Lower is more equitable. + + .. _GiniIndex: https://en.wikipedia.org/wiki/Gini_coefficient + + .. math:: + \mathrm {GiniIndex@K}=\frac{\sum_{i=1}^{|I|}(2i-|I|-1) P_{(i)}}{|I| \sum_{i=1}^{|I|} P_{(i)}} + + :math:`P_{(i)}` is the number of times item :math:`i` appears across all + users' top-K lists, items indexed in non-decreasing order of :math:`P`. + Items never recommended contribute 0 to the count but still occupy the + lowest ranks of the distribution. + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param num_items: total number of items in the catalog (|I|). + """ + max_k = item_matrix.shape[1] + result = np.zeros(max_k) + cum_counts = np.zeros(num_items, dtype=np.int64) + idx = np.arange(1, num_items + 1) + for k in range(max_k): + cum_counts += np.bincount(item_matrix[:, k], minlength=num_items) + sorted_count = np.sort(cum_counts) + total_num = sorted_count.sum() + result[k] = np.sum((2 * idx - num_items - 1) * sorted_count) / (num_items * total_num) + return result + + +def averagepopularity_(item_matrix, item_count, num_items): + r"""AveragePopularity_ computes the average training-set popularity of the + recommended items, averaged over users. Higher means the model leans on + already-popular items. + + .. math:: + \mathrm {AveragePopularity@K}=\frac{1}{|U|} \sum_{u \in U} \frac{\sum_{i \in R_u} \phi(i)}{|R_u|} + + :math:`\phi(i)` is the number of interactions of item :math:`i` in the + training data (0 if the item never appeared in training). + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param item_count: (num_items,) array, training-set interaction count per item. + :param num_items: total number of items in the catalog (unused, kept for a + uniform signature with the other item-distribution metrics). + """ + pop = item_count[item_matrix].astype(np.float64) + cum_avg = pop.cumsum(axis=1) / np.arange(1, item_matrix.shape[1] + 1) + return cum_avg.mean(axis=0) + + +def tailpercentage_(item_matrix, item_count, num_items, tail_ratio=0.1): + r"""TailPercentage_ (a.k.a. Average Percentage of Long-tail items, APLT) + computes the fraction of recommended items that belong to the long tail, + averaged over users. Higher means the model surfaces more long-tail items. + + .. _TailPercentage: https://en.wikipedia.org/wiki/Long_tail#Criticisms + + .. math:: + \mathrm {TailPercentage@K}=\frac{1}{|U|} \sum_{u \in U} \frac{\sum_{i \in R_u} \delta(i \in T)}{|R_u|} + + :math:`T` is the set of long-tail items: the least-popular items in the + training data. If ``tail_ratio`` is in (0, 1], :math:`T` is the bottom + ``tail_ratio`` fraction (by training popularity, ties broken by item id) + of items that were seen at least once in training. If ``tail_ratio`` > 1, + :math:`T` is every item whose training popularity is <= ``tail_ratio``. + Items never seen in training are not counted as long-tail (they are a + cold-start/coverage issue, not a popularity-bias one). + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param item_count: (num_items,) array, training-set interaction count per item. + :param num_items: total number of items in the catalog. + :param tail_ratio: threshold defining the long tail, see above. Default 0.1. + """ + observed = np.nonzero(item_count > 0)[0] + if tail_ratio > 1: + tail_mask = (item_count > 0) & (item_count <= tail_ratio) + else: + order = np.lexsort((observed, item_count[observed])) + cut = max(int(len(observed) * tail_ratio), 1) + tail_items = observed[order][:cut] + tail_mask = np.zeros(num_items, dtype=bool) + tail_mask[tail_items] = True + is_tail = tail_mask[item_matrix].astype(np.float64) + cum_avg = is_tail.cumsum(axis=1) / np.arange(1, item_matrix.shape[1] + 1) + return cum_avg.mean(axis=0) + + +def itemcoverage_(item_matrix, num_items): + r"""ItemCoverage_ measures the fraction of the item catalog that has been + recommended to at least one user within the top-K. Higher means the + model surfaces a larger share of the catalog. + + .. _ItemCoverage: https://en.wikipedia.org/wiki/Long_tail#Criticisms + + .. math:: + \mathrm {ItemCoverage@K}=\frac{|\bigcup_{u \in U} R_u@K|}{|I|} + + :math:`R_u@K` is the set of items recommended to user :math:`u` within + the top-K, and :math:`|I|` is the catalog size. + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param num_items: total number of items in the catalog (|I|). + """ + max_k = item_matrix.shape[1] + result = np.zeros(max_k) + seen = np.zeros(num_items, dtype=bool) + for k in range(max_k): + seen[item_matrix[:, k]] = True + result[k] = seen.sum() / num_items + return result + + +def shannonentropy_(item_matrix, num_items): + r"""ShannonEntropy_ measures the diversity of the item recommendation + frequency distribution across users: 0 means every recommendation slot + is filled by a single item, :math:`\ln(|I|)` is the maximum, reached when + every item is recommended equally often. Higher is more diverse. + + .. _ShannonEntropy: https://en.wikipedia.org/wiki/Entropy_(information_theory) + + .. math:: + \mathrm {ShannonEntropy@K}=-\sum_{i=1}^{|I|} p_{(i)} \log p_{(i)} + + :math:`p_{(i)}=P_{(i)}/\sum_{j} P_{(j)}` is the fraction of all top-K + recommendation slots occupied by item :math:`i`. Items never recommended + (:math:`p_{(i)}=0`) contribute 0. Natural log is used, so results are in + nats. + + :param item_matrix: (n_users, max_k) matrix of recommended item ids. + :param num_items: total number of items in the catalog (|I|). + """ + max_k = item_matrix.shape[1] + result = np.zeros(max_k) + cum_counts = np.zeros(num_items, dtype=np.int64) + for k in range(max_k): + cum_counts += np.bincount(item_matrix[:, k], minlength=num_items) + probs = cum_counts[cum_counts > 0] / cum_counts.sum() + result[k] = -np.sum(probs * np.log(probs)) + return result + + +"""Function name and function mapper for beyond-accuracy (item-distribution) +metrics. These take a different signature than `metrics_dict` above (they +need recommended item ids + item popularity, not a relevance mask), so they +are kept in a separate registry. +""" +item_metrics_dict = { + 'giniindex': giniindex_, + 'averagepopularity': averagepopularity_, + 'tailpercentage': tailpercentage_, + 'itemcoverage': itemcoverage_, + 'shannonentropy': shannonentropy_, +} + +# item-distribution metrics whose implementation only needs the recommended +# item ids + catalog size (not per-item training popularity) +num_items_only_item_metrics = frozenset(('giniindex', 'itemcoverage', 'shannonentropy')) diff --git a/src/utils/topk_evaluator.py b/src/utils/topk_evaluator.py index a36adab..0674fcc 100644 --- a/src/utils/topk_evaluator.py +++ b/src/utils/topk_evaluator.py @@ -7,13 +7,19 @@ import numpy as np import pandas as pd import torch -from utils.metrics import metrics_dict +from utils.metrics import metrics_dict, item_metrics_dict, num_items_only_item_metrics from torch.nn.utils.rnn import pad_sequence from utils.utils import get_local_time -# These metrics are typical in topk recommendations +# These metrics are typical in topk recommendations, evaluated against user relevance topk_metrics = {metric.lower(): metric for metric in ['Recall', 'Recall2', 'Precision', 'NDCG', 'MAP']} +# These "beyond-accuracy" metrics evaluate the distribution of recommended items +# across users (using item popularity computed from the training set) rather +# than their relevance to any single user +item_metrics = {metric.lower(): metric for metric in + ['GiniIndex', 'AveragePopularity', 'TailPercentage', 'ItemCoverage', 'ShannonEntropy']} +all_metrics = {**topk_metrics, **item_metrics} class TopKEvaluator(object): @@ -31,7 +37,11 @@ def __init__(self, config): self.metrics = config['metrics'] self.topk = config['topk'] self.save_recom_result = config['save_recommended_topk'] + self.tail_ratio = config['tail_ratio'] if config['tail_ratio'] else 0.1 self._check_args() + # cache of (item_count, num_items) per training dataset, keyed by id() since + # the training dataset is fixed for the lifetime of a trainer/evaluator + self._item_popularity_cache = {} def collect(self, interaction, scores_tensor, full=False): """collect the topk intermediate result of one batch, this function mainly @@ -86,21 +96,55 @@ def evaluate(self, batch_matrix_list, eval_data, is_test=False, idx=0): x_df = x_df.astype(int) x_df.to_csv(file_path, sep='\t', index=False) assert len(pos_len_list) == len(topk_index) - # if recom right? - bool_rec_matrix = [] - for m, n in zip(pos_items, topk_index): - bool_rec_matrix.append([True if i in m else False for i in n]) - bool_rec_matrix = np.asarray(bool_rec_matrix) - # get metrics + # compute each metric's result array (shape (max_k,)) keyed by metric + # name first, then assemble metric_dict below following the order + # metrics were declared in the config, regardless of which of the two + # groups (relevance-based vs. item-distribution) they belong to + results = {} + + # relevance-based (accuracy) metrics + if self.topk_metrics: + # if recom right? + bool_rec_matrix = [] + for m, n in zip(pos_items, topk_index): + bool_rec_matrix.append([True if i in m else False for i in n]) + bool_rec_matrix = np.asarray(bool_rec_matrix) + + result_list = self._calculate_metrics(pos_len_list, bool_rec_matrix) + results.update(zip(self.topk_metrics, result_list)) + + # beyond-accuracy (item-distribution) metrics + if self.item_metrics: + item_count, num_items = self._get_item_popularity(eval_data) + result_list = self._calculate_item_metrics(topk_index, item_count, num_items) + results.update(zip(self.item_metrics, result_list)) + metric_dict = {} - result_list = self._calculate_metrics(pos_len_list, bool_rec_matrix) - for metric, value in zip(self.metrics, result_list): + for metric in self.metrics: + value = results[metric] for k in self.topk: key = '{}@{}'.format(metric, k) metric_dict[key] = round(value[k - 1], 4) + return metric_dict + def _get_item_popularity(self, eval_data): + """Item popularity (interaction count) computed from the training set, + as a dense (num_items,) array, plus the catalog size. Cached per + training dataset since it does not change across evaluation calls. + """ + train_dataset = eval_data.additional_dataset + key = id(train_dataset) + if key not in self._item_popularity_cache: + iid_field = train_dataset.iid_field + num_items = train_dataset.item_num + counts = train_dataset.df[iid_field].value_counts() + item_count = np.zeros(num_items, dtype=np.int64) + item_count[counts.index.values.astype(int)] = counts.values + self._item_popularity_cache[key] = (item_count, num_items) + return self._item_popularity_cache[key] + def _check_args(self): # Check metrics if isinstance(self.metrics, (str, list)): @@ -111,9 +155,13 @@ def _check_args(self): # Convert metric to lowercase for m in self.metrics: - if m.lower() not in topk_metrics: + if m.lower() not in all_metrics: raise ValueError("There is no user grouped topk metric named {}!".format(m)) self.metrics = [metric.lower() for metric in self.metrics] + # split into relevance-based vs. item-distribution (beyond-accuracy) metrics, + # since they need different inputs to compute + self.topk_metrics = [m for m in self.metrics if m in topk_metrics] + self.item_metrics = [m for m in self.metrics if m in item_metrics] # Check topk: if isinstance(self.topk, (int, list)): @@ -136,14 +184,36 @@ def _calculate_metrics(self, pos_len_list, topk_index): np.ndarray: a matrix which contains the metrics result """ result_list = [] - for metric in self.metrics: + for metric in self.topk_metrics: metric_fuc = metrics_dict[metric.lower()] result = metric_fuc(topk_index, pos_len_list) result_list.append(result) return np.stack(result_list, axis=0) + def _calculate_item_metrics(self, topk_index, item_count, num_items): + """calculate the beyond-accuracy (item-distribution) metrics + + Args: + topk_index (np.ndarray): matrix of recommended item ids, (n_users, max_k) + item_count (np.ndarray): training-set interaction count per item, (num_items,) + num_items (int): total number of items in the catalog + Returns: + list: one np.ndarray per metric, each of shape (max_k,) + """ + result_list = [] + for metric in self.item_metrics: + metric_fuc = item_metrics_dict[metric] + if metric in num_items_only_item_metrics: + result = metric_fuc(topk_index, num_items) + elif metric == 'tailpercentage': + result = metric_fuc(topk_index, item_count, num_items, self.tail_ratio) + else: + result = metric_fuc(topk_index, item_count, num_items) + result_list.append(result) + return result_list + def __str__(self): mesg = 'The TopK Evaluator Info:\n' + '\tMetrics:[' + ', '.join( - [topk_metrics[metric.lower()] for metric in self.metrics]) \ + [all_metrics[metric.lower()] for metric in self.metrics]) \ + '], TopK:[' + ', '.join(map(str, self.topk)) + ']' return mesg