diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index 95385936c..b2704cbe7 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -162,6 +162,7 @@ def _train_model_fn(self, features, labels, run_config): is_training=True) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() + model.build_summary_graph() regularization_losses = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index f2408ba47..0575c45b5 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -178,6 +178,104 @@ def build_loss_graph(self): def build_metric_graph(self, eval_config): return self._metric_dict + def _get_summary_label_name(self, label_name=None): + if label_name: + return label_name + if hasattr(self, '_label_name'): + return self._label_name + if self._base_model_config.HasField('label_name'): + return self._base_model_config.label_name + if self._labels: + return list(self._labels.keys())[0] + raise ValueError( + 'summaries pcoc requires labels; set pcoc.label_name or model_config.label_name' + ) + + def _resolve_pred_tensor(self, pred_name): + if pred_name not in self._prediction_dict: + raise ValueError( + 'summaries pred_name "%s" not found in prediction_dict keys: %s' % + (pred_name, sorted(self._prediction_dict.keys()))) + pred = tf.to_float(self._prediction_dict[pred_name]) + if pred_name == 'logits' or pred_name.startswith('logits'): + pred = tf.sigmoid(pred) + if pred.shape.ndims is not None and pred.shape.ndims > 1: + if pred.shape.ndims == 2 and int(pred.shape[-1]) == 2: + pred = pred[:, 1] + else: + pred = tf.reshape(pred, [-1]) + return pred + + def _build_feature_mask(self, feature_name, feature_value): + if feature_name not in self._feature_dict: + raise ValueError( + 'summaries feature_name "%s" not found in feature_dict keys: %s' % + (feature_name, sorted(self._feature_dict.keys()))) + feat = self._feature_dict[feature_name] + if isinstance(feat, tf.SparseTensor): + dense = tf.sparse_to_dense( + feat.indices, + feat.dense_shape, + feat.values, + default_value='' if feat.values.dtype == tf.string else 0) + feat = tf.reshape(dense, [-1]) + else: + feat = tf.reshape(feat, [-1]) + if feat.dtype in (tf.float32, tf.float64, tf.int32, tf.int64): + try: + target = float(feature_value) + except (TypeError, ValueError): + target = None + if target is not None: + return tf.equal(tf.to_float(feat), tf.constant(target, dtype=tf.float32)) + if feat.dtype != tf.string: + feat = tf.as_string(feat) + return tf.equal(feat, str(feature_value)) + + def _masked_mean(self, values, mask): + values = tf.reshape(tf.to_float(values), [-1]) + masked = tf.boolean_mask(values, mask) + count = tf.size(masked) + return tf.cond( + count > 0, + lambda: tf.reduce_mean(masked), + lambda: tf.constant(0.0, dtype=tf.float32)) + + def _build_summary_impl(self, summary): + summary_type = summary.WhichOneof('summary') + if summary_type == 'pcoc': + if self._labels is None: + return + pred_name = summary.pcoc.pred_name or 'probs' + preds = self._resolve_pred_tensor(pred_name) + label_name = self._get_summary_label_name( + summary.pcoc.label_name if summary.pcoc.HasField('label_name') else None) + label = tf.to_float(self._labels[label_name]) + label = tf.reshape(label, [-1]) + predicted_ctr = tf.reduce_mean(preds) + observed_ctr = tf.reduce_mean(label) + epsilon = summary.pcoc.epsilon + pcoc = predicted_ctr / (observed_ctr + epsilon) + tf.summary.scalar('summary/predicted_ctr', predicted_ctr) + tf.summary.scalar('summary/observed_ctr', observed_ctr) + tf.summary.scalar('summary/pcoc', pcoc) + elif summary_type == 'scalars': + cfg = summary.scalars + pred_name = cfg.pred_name or 'probs' + preds = self._resolve_pred_tensor(pred_name) + if cfg.HasField('feature_name') and cfg.HasField('feature_value'): + mask = self._build_feature_mask(cfg.feature_name, cfg.feature_value) + value = self._masked_mean(preds, mask) + else: + value = tf.reduce_mean(preds) + tf.summary.scalar('summary/%s' % cfg.name, value) + elif summary_type is not None: + raise ValueError('unsupported summary type: %s' % summary_type) + + def build_summary_graph(self): + for summary in self._base_model_config.summaries_set: + self._build_summary_impl(summary) + @abstractmethod def get_outputs(self): pass diff --git a/easy_rec/python/protos/easy_rec_model.proto b/easy_rec/python/protos/easy_rec_model.proto index 87cd6754d..8b47c596a 100644 --- a/easy_rec/python/protos/easy_rec_model.proto +++ b/easy_rec/python/protos/easy_rec_model.proto @@ -30,6 +30,7 @@ import "easy_rec/python/protos/pdn.proto"; import "easy_rec/python/protos/dssm_senet.proto"; import "easy_rec/python/protos/simi.proto"; import "easy_rec/python/protos/dat.proto"; +import "easy_rec/python/protos/summary.proto"; // for input performance test message DummyModel { } @@ -158,4 +159,9 @@ message EasyRecModel { // label name for rank_model to select one label between multiple labels optional string label_name = 18; + + // Training TensorBoard summaries, e.g.: + // summaries_set { pcoc {} } + // summaries_set { scalars { name: "c1_1005_pred" feature_name: "c1" feature_value: "1005" } } + repeated ModelSummaries summaries_set = 19; } diff --git a/easy_rec/python/protos/summary.proto b/easy_rec/python/protos/summary.proto new file mode 100644 index 000000000..f07d77b46 --- /dev/null +++ b/easy_rec/python/protos/summary.proto @@ -0,0 +1,23 @@ +syntax = "proto2"; +package protos; + +message PCOC { + optional float epsilon = 1 [default = 1e-7]; + optional string pred_name = 2 [default = "probs"]; + // optional; default model_config.label_name or the first label + optional string label_name = 3; +} + +message Scalars { + required string name = 1; + optional string pred_name = 2 [default = "probs"]; + optional string feature_name = 3; + optional string feature_value = 4; +} + +message ModelSummaries { + oneof summary { + PCOC pcoc = 1; + Scalars scalars = 2; + } +} diff --git a/easy_rec/python/test/eval_summary_test.py b/easy_rec/python/test/eval_summary_test.py new file mode 100644 index 000000000..d28da958f --- /dev/null +++ b/easy_rec/python/test/eval_summary_test.py @@ -0,0 +1,127 @@ +# -*- encoding:utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +from __future__ import division + +import logging +import unittest + +from google.protobuf import text_format + +from easy_rec.python.model.easy_rec_model import EasyRecModel +from easy_rec.python.protos.easy_rec_model_pb2 import EasyRecModel as EasyRecModelProto +from easy_rec.python.protos.summary_pb2 import ModelSummaries + +if __name__ == '__main__': + import tensorflow as tf + if tf.__version__ >= '2.0': + tf = tf.compat.v1 + tf.disable_eager_execution() + + +def _tf_v1(): + import tensorflow as tf + return tf.compat.v1 if tf.__version__ >= '2.0' else tf + + +class _FakeEasyRecModel(object): + _resolve_pred_tensor = EasyRecModel._resolve_pred_tensor + _build_feature_mask = EasyRecModel._build_feature_mask + _masked_mean = EasyRecModel._masked_mean + _get_summary_label_name = EasyRecModel._get_summary_label_name + + +def _run_summary_tags(model, summary_text): + tf = _tf_v1() + tf.reset_default_graph() + model = model() + summary = ModelSummaries() + text_format.Parse(summary_text, summary) + EasyRecModel._build_summary_impl(model, summary) + with tf.Session() as sess: + summary_str = sess.run(tf.summary.merge_all()) + summary_proto = tf.Summary() + summary_proto.ParseFromString(summary_str) + return {v.tag: v.simple_value for v in summary_proto.value} + + +class SummariesSetTest(unittest.TestCase): + + def setUp(self): + logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) + + def test_proto_and_pcoc_graph(self): + model = EasyRecModelProto() + text_format.Parse( + 'model_class: "DeepFM" deepfm {} summaries_set { pcoc {} }', model) + self.assertEqual(model.summaries_set[0].WhichOneof('summary'), 'pcoc') + + class _M(_FakeEasyRecModel): + + def __init__(self): + tf = _tf_v1() + self._labels = {'label': tf.constant([1, 0, 1, 0], dtype=tf.float32)} + self._prediction_dict = { + 'probs': tf.constant([0.8, 0.4, 0.6, 0.2], dtype=tf.float32) + } + self._feature_dict = {} + self._label_name = 'label' + self._base_model_config = EasyRecModelProto() + + values = _run_summary_tags(_M, 'pcoc { epsilon: 1e-7 }') + self.assertAlmostEqual(values['summary/pcoc'], 1.0, places=5) + + def test_scalars_feature_slice(self): + class _M(_FakeEasyRecModel): + + def __init__(self): + tf = _tf_v1() + self._labels = {'label': tf.constant([1, 0], dtype=tf.float32)} + self._prediction_dict = { + 'probs': tf.constant([0.9, 0.1], dtype=tf.float32) + } + self._feature_dict = {'c1': tf.constant([1005.0, 1002.0], dtype=tf.float32)} + self._label_name = 'label' + self._base_model_config = EasyRecModelProto() + + values = _run_summary_tags( + _M, + 'scalars { name: "c1_1005_pred" feature_name: "c1" feature_value: "1005" }') + self.assertAlmostEqual(values['summary/c1_1005_pred'], 0.9, places=5) + + def test_pcoc_without_rank_label_name(self): + """MatchModel 等无 _label_name 时,回退到 model_config.label_name。""" + + class _M(_FakeEasyRecModel): + + def __init__(self): + tf = _tf_v1() + self._labels = {'clk': tf.constant([1, 0], dtype=tf.float32)} + self._prediction_dict = { + 'probs': tf.constant([0.6, 0.4], dtype=tf.float32) + } + self._feature_dict = {} + self._base_model_config = EasyRecModelProto() + self._base_model_config.label_name = 'clk' + + values = _run_summary_tags(_M, 'pcoc {}') + self.assertAlmostEqual(values['summary/pcoc'], 1.0, places=5) + + def test_pcoc_custom_pred_name(self): + class _M(_FakeEasyRecModel): + + def __init__(self): + tf = _tf_v1() + self._labels = {'label': tf.constant([1, 0], dtype=tf.float32)} + self._prediction_dict = { + 'similarity': tf.constant([0.75, 0.25], dtype=tf.float32) + } + self._feature_dict = {} + self._label_name = 'label' + self._base_model_config = EasyRecModelProto() + + values = _run_summary_tags(_M, 'pcoc { pred_name: "similarity" }') + self.assertAlmostEqual(values['summary/pcoc'], 1.0, places=5) + + +if __name__ == '__main__': + unittest.main() diff --git a/samples/model_config/deepfm_combo_on_avazu_ctr.config b/samples/model_config/deepfm_combo_on_avazu_ctr.config index a2a7fa765..ea3501fab 100644 --- a/samples/model_config/deepfm_combo_on_avazu_ctr.config +++ b/samples/model_config/deepfm_combo_on_avazu_ctr.config @@ -365,6 +365,25 @@ model_config:{ l2_regularization: 1e-5 } # embedding_regularization: 1e-7 + + # Training TensorBoard summaries (model_config.summaries_set, not eval_config) + # PCOC = mean(pred)/mean(label); tags: summary/pcoc, summary/predicted_ctr, summary/observed_ctr + summaries_set { + pcoc { + # pred_name: "probs" # prediction_dict key; "logits" applies sigmoid first + # label_name: "label" # optional; for multi-task use e.g. probs_ctr + matching label + # epsilon: 1e-7 # divisor stabilizer when observed ctr is near zero + } + } + # Feature-sliced scalar (mean pred on c1=="1005"); tag: summary/c1_1005_pred + summaries_set { + scalars { + name: "c1_1005_pred" + pred_name: "probs" + feature_name: "c1" + feature_value: "1005" + } + } } export_config {