feat: model_config 支持 summaries_set（PCOC / scalars）训练期 TensorBoard 监控

easy_rec/python/model/easy_rec_estimator.py

@@ -162,6 +162,7 @@ def _train_model_fn(self, features, labels, run_config):
         is_training=True)
     predict_dict = model.build_predict_graph()
     loss_dict = model.build_loss_graph()
+    model.build_summary_graph()
 
     regularization_losses = tf.get_collection(
         tf.GraphKeys.REGULARIZATION_LOSSES)

easy_rec/python/model/easy_rec_model.py

@@ -178,6 +178,104 @@ def build_loss_graph(self):
   def build_metric_graph(self, eval_config):
     return self._metric_dict
 
+  def _get_summary_label_name(self, label_name=None):
+    if label_name:
+      return label_name
+    if hasattr(self, '_label_name'):
+      return self._label_name
+    if self._base_model_config.HasField('label_name'):
+      return self._base_model_config.label_name
+    if self._labels:
+      return list(self._labels.keys())[0]
+    raise ValueError(
+        'summaries pcoc requires labels; set pcoc.label_name or model_config.label_name'
+    )
+
+  def _resolve_pred_tensor(self, pred_name):
+    if pred_name not in self._prediction_dict:
+      raise ValueError(
+          'summaries pred_name "%s" not found in prediction_dict keys: %s' %
+          (pred_name, sorted(self._prediction_dict.keys())))
+    pred = tf.to_float(self._prediction_dict[pred_name])
+    if pred_name == 'logits' or pred_name.startswith('logits'):
+      pred = tf.sigmoid(pred)
+    if pred.shape.ndims is not None and pred.shape.ndims > 1:
+      if pred.shape.ndims == 2 and int(pred.shape[-1]) == 2:
+        pred = pred[:, 1]
+      else:
+        pred = tf.reshape(pred, [-1])
+    return pred
+
+  def _build_feature_mask(self, feature_name, feature_value):
+    if feature_name not in self._feature_dict:
+      raise ValueError(
+          'summaries feature_name "%s" not found in feature_dict keys: %s' %
+          (feature_name, sorted(self._feature_dict.keys())))
+    feat = self._feature_dict[feature_name]
+    if isinstance(feat, tf.SparseTensor):
+      dense = tf.sparse_to_dense(
+          feat.indices,
+          feat.dense_shape,
+          feat.values,
+          default_value='' if feat.values.dtype == tf.string else 0)
+      feat = tf.reshape(dense, [-1])
+    else:
+      feat = tf.reshape(feat, [-1])
+    if feat.dtype in (tf.float32, tf.float64, tf.int32, tf.int64):
+      try:
+        target = float(feature_value)
+      except (TypeError, ValueError):
+        target = None
+      if target is not None:
+        return tf.equal(tf.to_float(feat), tf.constant(target, dtype=tf.float32))
+    if feat.dtype != tf.string:
+      feat = tf.as_string(feat)
+    return tf.equal(feat, str(feature_value))
+
+  def _masked_mean(self, values, mask):
+    values = tf.reshape(tf.to_float(values), [-1])
+    masked = tf.boolean_mask(values, mask)
+    count = tf.size(masked)
+    return tf.cond(
+        count > 0,
+        lambda: tf.reduce_mean(masked),
+        lambda: tf.constant(0.0, dtype=tf.float32))
+
+  def _build_summary_impl(self, summary):
+    summary_type = summary.WhichOneof('summary')
+    if summary_type == 'pcoc':
+      if self._labels is None:
+        return
+      pred_name = summary.pcoc.pred_name or 'probs'
+      preds = self._resolve_pred_tensor(pred_name)
+      label_name = self._get_summary_label_name(
+          summary.pcoc.label_name if summary.pcoc.HasField('label_name') else None)
+      label = tf.to_float(self._labels[label_name])
+      label = tf.reshape(label, [-1])
+      predicted_ctr = tf.reduce_mean(preds)
+      observed_ctr = tf.reduce_mean(label)
+      epsilon = summary.pcoc.epsilon
+      pcoc = predicted_ctr / (observed_ctr + epsilon)
+      tf.summary.scalar('summary/predicted_ctr', predicted_ctr)
+      tf.summary.scalar('summary/observed_ctr', observed_ctr)
+      tf.summary.scalar('summary/pcoc', pcoc)
+    elif summary_type == 'scalars':
+      cfg = summary.scalars
+      pred_name = cfg.pred_name or 'probs'
+      preds = self._resolve_pred_tensor(pred_name)
+      if cfg.HasField('feature_name') and cfg.HasField('feature_value'):
+        mask = self._build_feature_mask(cfg.feature_name, cfg.feature_value)
+        value = self._masked_mean(preds, mask)
+      else:
+        value = tf.reduce_mean(preds)
+      tf.summary.scalar('summary/%s' % cfg.name, value)
+    elif summary_type is not None:
+      raise ValueError('unsupported summary type: %s' % summary_type)
+
+  def build_summary_graph(self):
+    for summary in self._base_model_config.summaries_set:
+      self._build_summary_impl(summary)
+
   @abstractmethod
   def get_outputs(self):
     pass

easy_rec/python/protos/easy_rec_model.proto

@@ -30,6 +30,7 @@ import "easy_rec/python/protos/pdn.proto";
 import "easy_rec/python/protos/dssm_senet.proto";
 import "easy_rec/python/protos/simi.proto";
 import "easy_rec/python/protos/dat.proto";
+import "easy_rec/python/protos/summary.proto";
 // for input performance test
 message DummyModel {
 }
@@ -158,4 +159,9 @@ message EasyRecModel {
 
     // label name for rank_model to select one label between multiple labels
     optional string label_name = 18;
+
+    // Training TensorBoard summaries, e.g.:
+    //   summaries_set { pcoc {} }
+    //   summaries_set { scalars { name: "c1_1005_pred" feature_name: "c1" feature_value: "1005" } }
+    repeated ModelSummaries summaries_set = 19;
 }

easy_rec/python/protos/summary.proto

@@ -0,0 +1,23 @@
+syntax = "proto2";
+package protos;
+
+message PCOC {
+  optional float epsilon = 1 [default = 1e-7];
+  optional string pred_name = 2 [default = "probs"];
+  // optional; default model_config.label_name or the first label
+  optional string label_name = 3;
+}
+
+message Scalars {
+  required string name = 1;
+  optional string pred_name = 2 [default = "probs"];
+  optional string feature_name = 3;
+  optional string feature_value = 4;
+}
+
+message ModelSummaries {
+  oneof summary {
+    PCOC pcoc = 1;
+    Scalars scalars = 2;
+  }
+}

easy_rec/python/test/eval_summary_test.py

@@ -0,0 +1,127 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from __future__ import division
+
+import logging
+import unittest
+
+from google.protobuf import text_format
+
+from easy_rec.python.model.easy_rec_model import EasyRecModel
+from easy_rec.python.protos.easy_rec_model_pb2 import EasyRecModel as EasyRecModelProto
+from easy_rec.python.protos.summary_pb2 import ModelSummaries
+
+if __name__ == '__main__':
+  import tensorflow as tf
+  if tf.__version__ >= '2.0':
+    tf = tf.compat.v1
+  tf.disable_eager_execution()
+
+
+def _tf_v1():
+  import tensorflow as tf
+  return tf.compat.v1 if tf.__version__ >= '2.0' else tf
+
+
+class _FakeEasyRecModel(object):
+  _resolve_pred_tensor = EasyRecModel._resolve_pred_tensor
+  _build_feature_mask = EasyRecModel._build_feature_mask
+  _masked_mean = EasyRecModel._masked_mean
+  _get_summary_label_name = EasyRecModel._get_summary_label_name
+
+
+def _run_summary_tags(model, summary_text):
+  tf = _tf_v1()
+  tf.reset_default_graph()
+  model = model()
+  summary = ModelSummaries()
+  text_format.Parse(summary_text, summary)
+  EasyRecModel._build_summary_impl(model, summary)
+  with tf.Session() as sess:
+    summary_str = sess.run(tf.summary.merge_all())
+  summary_proto = tf.Summary()
+  summary_proto.ParseFromString(summary_str)
+  return {v.tag: v.simple_value for v in summary_proto.value}
+
+
+class SummariesSetTest(unittest.TestCase):
+
+  def setUp(self):
+    logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName))
+
+  def test_proto_and_pcoc_graph(self):
+    model = EasyRecModelProto()
+    text_format.Parse(
+        'model_class: "DeepFM" deepfm {} summaries_set { pcoc {} }', model)
+    self.assertEqual(model.summaries_set[0].WhichOneof('summary'), 'pcoc')
+
+    class _M(_FakeEasyRecModel):
+
+      def __init__(self):
+        tf = _tf_v1()
+        self._labels = {'label': tf.constant([1, 0, 1, 0], dtype=tf.float32)}
+        self._prediction_dict = {
+            'probs': tf.constant([0.8, 0.4, 0.6, 0.2], dtype=tf.float32)
+        }
+        self._feature_dict = {}
+        self._label_name = 'label'
+        self._base_model_config = EasyRecModelProto()
+
+    values = _run_summary_tags(_M, 'pcoc { epsilon: 1e-7 }')
+    self.assertAlmostEqual(values['summary/pcoc'], 1.0, places=5)
+
+  def test_scalars_feature_slice(self):
+    class _M(_FakeEasyRecModel):
+
+      def __init__(self):
+        tf = _tf_v1()
+        self._labels = {'label': tf.constant([1, 0], dtype=tf.float32)}
+        self._prediction_dict = {
+            'probs': tf.constant([0.9, 0.1], dtype=tf.float32)
+        }
+        self._feature_dict = {'c1': tf.constant([1005.0, 1002.0], dtype=tf.float32)}
+        self._label_name = 'label'
+        self._base_model_config = EasyRecModelProto()
+
+    values = _run_summary_tags(
+        _M,
+        'scalars { name: "c1_1005_pred" feature_name: "c1" feature_value: "1005" }')
+    self.assertAlmostEqual(values['summary/c1_1005_pred'], 0.9, places=5)
+
+  def test_pcoc_without_rank_label_name(self):
+    """MatchModel 等无 _label_name 时，回退到 model_config.label_name。"""
+
+    class _M(_FakeEasyRecModel):
+
+      def __init__(self):
+        tf = _tf_v1()
+        self._labels = {'clk': tf.constant([1, 0], dtype=tf.float32)}
+        self._prediction_dict = {
+            'probs': tf.constant([0.6, 0.4], dtype=tf.float32)
+        }
+        self._feature_dict = {}
+        self._base_model_config = EasyRecModelProto()
+        self._base_model_config.label_name = 'clk'
+
+    values = _run_summary_tags(_M, 'pcoc {}')
+    self.assertAlmostEqual(values['summary/pcoc'], 1.0, places=5)
+
+  def test_pcoc_custom_pred_name(self):
+    class _M(_FakeEasyRecModel):
+
+      def __init__(self):
+        tf = _tf_v1()
+        self._labels = {'label': tf.constant([1, 0], dtype=tf.float32)}
+        self._prediction_dict = {
+            'similarity': tf.constant([0.75, 0.25], dtype=tf.float32)
+        }
+        self._feature_dict = {}
+        self._label_name = 'label'
+        self._base_model_config = EasyRecModelProto()
+
+    values = _run_summary_tags(_M, 'pcoc { pred_name: "similarity" }')
+    self.assertAlmostEqual(values['summary/pcoc'], 1.0, places=5)
+
+
+if __name__ == '__main__':
+  unittest.main()

samples/model_config/deepfm_combo_on_avazu_ctr.config

@@ -365,6 +365,25 @@ model_config:{
     l2_regularization: 1e-5
   }
   # embedding_regularization: 1e-7
+
+  # Training TensorBoard summaries (model_config.summaries_set, not eval_config)
+  # PCOC = mean(pred)/mean(label); tags: summary/pcoc, summary/predicted_ctr, summary/observed_ctr
+  summaries_set {
+    pcoc {
+      # pred_name: "probs"   # prediction_dict key; "logits" applies sigmoid first
+      # label_name: "label"  # optional; for multi-task use e.g. probs_ctr + matching label
+      # epsilon: 1e-7        # divisor stabilizer when observed ctr is near zero
+    }
+  }
+  # Feature-sliced scalar (mean pred on c1=="1005"); tag: summary/c1_1005_pred
+  summaries_set {
+    scalars {
+      name: "c1_1005_pred"
+      pred_name: "probs"
+      feature_name: "c1"
+      feature_value: "1005"
+    }
+  }
 }
 
 export_config {