DM-54696: Add analysis_tools task and tooling to aggregate metadata metrics on-the-fly

config/aggregateCalibrateImageMetadata.py

@@ -0,0 +1,82 @@
+# Configuration for lsst.analysis.tools.tasks.AggregatedTaskMetadataAnalysisTask
+# when aggregating calibrateImage task metadata across detectors to produce
+# per-visit aggregate statistics.
+
+from lsst.analysis.tools.actions.scalar import CountAction, MaxAction, MedianAction, MinAction, SigmaMadAction
+from lsst.analysis.tools.atools import AggregatedTaskMetadataMetricTool
+
+config.atools.calibrateImageMetadataMetrics = AggregatedTaskMetadataMetricTool
+config.atools.calibrateImageMetadataMetrics.taskName = "calibrateImage"
+config.atools.calibrateImageMetadataMetrics.metrics = {
+    "initial_psf_positive_footprint_count": "ct",
+    "initial_psf_negative_footprint_count": "ct",
+    "initial_psf_positive_peak_count": "ct",
+    "initial_psf_negative_peak_count": "ct",
+    "simple_psf_positive_footprint_count": "ct",
+    "simple_psf_negative_footprint_count": "ct",
+    "simple_psf_positive_peak_count": "ct",
+    "simple_psf_negative_peak_count": "ct",
+    "bad_mask_fraction": "",
+    "cr_mask_fraction": "",
+    "crosstalk_mask_fraction": "",
+    "detected_mask_fraction": "",
+    "detected_negative_mask_fraction": "",
+    "edge_mask_fraction": "",
+    "intrp_mask_fraction": "",
+    "no_data_mask_fraction": "",
+    "sat_mask_fraction": "",
+    "suspect_mask_fraction": "",
+    "unmaskednan_mask_fraction": "",
+    "numAvailStars": "ct",
+    "numGoodStars": "ct",
+    "sky_footprint_count": "ct",
+    "post_deblend_source_count": "ct",
+    "star_count": "ct",
+    "saturated_source_count": "ct",
+    "bad_source_count": "ct",
+    "cosmic_ray_count": "ct",
+    "matched_psf_star_count": "ct",
+    "final_psf_sigma": "pixel",
+    "astrometry_matches_count": "ct",
+    "photometry_matches_count": "ct",
+    "bg_subtracted_skyPixel_instFlux_median": "adu",
+    "bg_subtracted_skyPixel_instFlux_stdev": "adu",
+    "bg_subtracted_skySource_flux_median": "nJy",
+    "bg_subtracted_skySource_flux_stdev": "nJy",
+    "adaptive_threshold_value": "",
+    "initial_to_final_wcs": "arcsec",
+    "astrom_offset_mean": "arcsec",
+    "astrom_offset_std": "arcsec",
+    "astrom_offset_median": "arcsec",
+    "failed_deblend_source_count": "ct",
+}
+config.atools.calibrateImageMetadataMetrics.subTaskNames = {
+    "numAvailStars": "psf_measure_psf",
+    "numGoodStars": "psf_measure_psf",
+    "sky_footprint_count": "star_sky_sources",
+    "cosmic_ray_count": "psf_repair",
+}
+config.atools.calibrateImageMetadataMetrics.newNames = {
+    "numAvailStars": "psf_available_star_count",
+    "numGoodStars": "psf_good_star_count",
+    "adaptive_threshold_value": "final_adaptive_threshold_value",
+}
+
+aggregations = {
+    "min": MinAction,
+    "max": MaxAction,
+    "median": MedianAction,
+    "mad": SigmaMadAction,
+    "ct": CountAction,
+}
+config.atools.calibrateImageMetadataMetrics.aggregationUnits = {"ct": "ct"}
+
+for metric in config.atools.calibrateImageMetadataMetrics.metrics:
+    for agg_name, agg_cls in aggregations.items():
+        action = agg_cls()
+        action.vectorKey = metric
+        setattr(
+            config.atools.calibrateImageMetadataMetrics.process.calculateActions,
+            f"{metric}_{agg_name}",
+            action,
+        )

pipelines/visitQualityCore.yaml

@@ -38,66 +38,19 @@ tasks:
     config:
       atools.calexpSummaryMetrics: CalexpSummaryMetrics
       python: from lsst.analysis.tools.atools import *
-  analyzeCalibrateImageMetadata:
-    class: lsst.analysis.tools.tasks.TaskMetadataAnalysisTask
+  aggregateCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.AggregatedTaskMetadataAnalysisTask
     config:
       connections.inputName: calibrateImage_metadata
-      connections.outputName: calibrateImage_metadata  # Will be appended with "_metrics"
-      connections.storageClass: TaskMetadata
-      raiseNoWorkFoundOnIncompleteMetadata: true
-      inputDimensions: ["instrument", "visit", "detector"]
-      atools.calibrateImageMetadataMetrics: TaskMetadataMetricTool
-      atools.calibrateImageMetadataMetrics.taskName: calibrateImage
-      atools.calibrateImageMetadataMetrics.metrics:
-        initial_psf_positive_footprint_count: ct
-        initial_psf_negative_footprint_count: ct
-        initial_psf_positive_peak_count: ct
-        initial_psf_negative_peak_count: ct
-        simple_psf_positive_footprint_count: ct
-        simple_psf_negative_footprint_count: ct
-        simple_psf_positive_peak_count: ct
-        simple_psf_negative_peak_count: ct
-        bad_mask_fraction: ""
-        cr_mask_fraction: ""
-        crosstalk_mask_fraction: ""
-        detected_mask_fraction: ""
-        detected_negative_mask_fraction: ""
-        edge_mask_fraction: ""
-        intrp_mask_fraction: ""
-        no_data_mask_fraction: ""
-        sat_mask_fraction: ""
-        suspect_mask_fraction: ""
-        unmaskednan_mask_fraction: ""
-        numAvailStars: ct
-        numGoodStars: ct
-        sky_footprint_count: ct
-        post_deblend_source_count: ct
-        star_count: ct
-        saturated_source_count: ct
-        bad_source_count: ct
-        cosmic_ray_count: ct
-        matched_psf_star_count: ct
-        final_psf_sigma: pixel
-        astrometry_matches_count: ct
-        photometry_matches_count: ct
-        bg_subtracted_skyPixel_instFlux_median: adu
-        bg_subtracted_skyPixel_instFlux_stdev: adu
-        bg_subtracted_skySource_flux_median: nJy
-        bg_subtracted_skySource_flux_stdev: nJy
-        adaptive_threshold_value: ""
-        initial_to_final_wcs: arcsec
-        astrom_offset_mean: arcsec
-        astrom_offset_std: arcsec
-        astrom_offset_median: arcsec
-        failed_deblend_source_count: ct
-      atools.calibrateImageMetadataMetrics.subTaskNames:
-        numAvailStars: psf_measure_psf
-        numGoodStars: psf_measure_psf
-        sky_footprint_count: star_sky_sources
-        cosmic_ray_count: psf_repair
-      atools.calibrateImageMetadataMetrics.newNames:
-        numAvailStars: psf_available_star_count
-        numGoodStars: psf_good_star_count
-        adaptive_threshold_value: final_adaptive_threshold_value
-      python: |
-        from lsst.analysis.tools.atools import TaskMetadataMetricTool
+      connections.outputName: calibrateImage_metadata_aggregate
+      inputDataDimensions: ["instrument", "visit", "detector"]
+      outputDimensions: ["instrument", "visit"]
+      file: $ANALYSIS_TOOLS_DIR/config/aggregateCalibrateImageMetadata.py
+  makeMetricTableAggregateCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.MakeMetricTableTask
+    config:
+      connections.metricBundleName: "calibrateImage_metadata_aggregate_metrics"
+      connections.outputTableName: "calibrateImage_metadata_aggregate_metricsTable"
+      inputDataDimensions: ["instrument", "visit"]
+      outputTableDimensions: ["instrument"]
+      dataIdFieldsToIncludeAsColumns: ["visit", "band"]

pipelines/visitQualityExtended.yaml

@@ -1,8 +1,6 @@
 description: |
   Tier2 plots and metrics to assess visit quality
 tasks:
-  catalogMatchVisit:
-    class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
   refCatSourceVisit:
     class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
   deltaSkyCorrHist:
@@ -13,3 +11,66 @@ tasks:
       atools.deltaSkyCorr: DeltaSkyCorrXYPlot
       python: |
         from lsst.analysis.tools.atools import *
+  analyzeCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.TaskMetadataAnalysisTask
+    config:
+      connections.inputName: calibrateImage_metadata
+      connections.outputName: calibrateImage_metadata  # Will be appended with "_metrics"
+      connections.storageClass: TaskMetadata
+      raiseNoWorkFoundOnIncompleteMetadata: true
+      inputDimensions: ["instrument", "visit", "detector"]
+      atools.calibrateImageMetadataMetrics: TaskMetadataMetricTool
+      atools.calibrateImageMetadataMetrics.taskName: calibrateImage
+      atools.calibrateImageMetadataMetrics.metrics:
+        initial_psf_positive_footprint_count: ct
+        initial_psf_negative_footprint_count: ct
+        initial_psf_positive_peak_count: ct
+        initial_psf_negative_peak_count: ct
+        simple_psf_positive_footprint_count: ct
+        simple_psf_negative_footprint_count: ct
+        simple_psf_positive_peak_count: ct
+        simple_psf_negative_peak_count: ct
+        bad_mask_fraction: ""
+        cr_mask_fraction: ""
+        crosstalk_mask_fraction: ""
+        detected_mask_fraction: ""
+        detected_negative_mask_fraction: ""
+        edge_mask_fraction: ""
+        intrp_mask_fraction: ""
+        no_data_mask_fraction: ""
+        sat_mask_fraction: ""
+        suspect_mask_fraction: ""
+        unmaskednan_mask_fraction: ""
+        numAvailStars: ct
+        numGoodStars: ct
+        sky_footprint_count: ct
+        post_deblend_source_count: ct
+        star_count: ct
+        saturated_source_count: ct
+        bad_source_count: ct
+        cosmic_ray_count: ct
+        matched_psf_star_count: ct
+        final_psf_sigma: pixel
+        astrometry_matches_count: ct
+        photometry_matches_count: ct
+        bg_subtracted_skyPixel_instFlux_median: adu
+        bg_subtracted_skyPixel_instFlux_stdev: adu
+        bg_subtracted_skySource_flux_median: nJy
+        bg_subtracted_skySource_flux_stdev: nJy
+        adaptive_threshold_value: ""
+        initial_to_final_wcs: arcsec
+        astrom_offset_mean: arcsec
+        astrom_offset_std: arcsec
+        astrom_offset_median: arcsec
+        failed_deblend_source_count: ct
+      atools.calibrateImageMetadataMetrics.subTaskNames:
+        numAvailStars: psf_measure_psf
+        numGoodStars: psf_measure_psf
+        sky_footprint_count: star_sky_sources
+        cosmic_ray_count: psf_repair
+      atools.calibrateImageMetadataMetrics.newNames:
+        numAvailStars: psf_available_star_count
+        numGoodStars: psf_good_star_count
+        adaptive_threshold_value: final_adaptive_threshold_value
+      python: |
+        from lsst.analysis.tools.atools import TaskMetadataMetricTool

python/lsst/analysis/tools/atools/metadataMetrics.py

@@ -21,6 +21,7 @@
 from __future__ import annotations
 
 __all__ = (
+    "AggregatedTaskMetadataMetricTool",
     "DatasetMetadataMetricTool",
     "TaskMetadataMetricTool",
 )
@@ -117,3 +118,57 @@ def finalize(self):
 
         if self.newNames is not None:
             self.produce.metric.newNames = dict(self.newNames.items())
+
+
+class AggregatedTaskMetadataMetricTool(MetadataMetricTool):
+    """Tool to compute aggregate statistics from task metadata across
+    multiple inputs.
+    """
+
+    taskName = Field[str](
+        doc="The name of the task to extract metadata from.",
+        default=None,
+    )
+
+    subTaskNames = DictField[str, str](
+        doc="The names of subtasks to extract metadata from. "
+        "If the metric name is identified as one of the keys, then "
+        "the corresponding value is taken as the subTask metadata "
+        "from which to extract metadata.",
+        default=None,
+        optional=True,
+    )
+
+    aggregationUnits = DictField[str, str](
+        doc="Fixed units for specific aggregations, keyed by aggregation name "
+        "(the suffix after the metric name, e.g. 'ct' or 'frac'). "
+        "Overrides the source metric's unit for that aggregation.",
+        default={},
+        optional=True,
+    )
+
+    def finalize(self):
+        # Attribute names can't contain spaces, but metadata metric names
+        # might. Remove any spaces from names.
+        valid_metric_names = {self.makeValidAttributeName(k): v for k, v in self.metrics.items()}
+        units = {}
+        # This is the same as looping over the metric names, but it also
+        # provides the aggregator name at the same time which is needed
+        # for post-aggregation unit lookup.
+        for name, action in self.process.calculateActions.items():
+            vector_key = getattr(action, "vectorKey", None)
+            if vector_key is not None:
+                valid_vector_key = self.makeValidAttributeName(vector_key)
+                if valid_vector_key not in valid_metric_names:
+                    raise ValueError(
+                        f"Action {name!r} has vectorKey {vector_key!r} which is "
+                        f"not in metrics: {list(self.metrics.keys())}"
+                    )
+                agg_name = name[len(valid_vector_key) + 1 :]
+                if self.aggregationUnits and agg_name in self.aggregationUnits:
+                    units[name] = self.aggregationUnits[agg_name]
+                else:
+                    units[name] = valid_metric_names[valid_vector_key]
+        self.produce.metric.units = units
+        if self.newNames is not None:
+            self.produce.metric.newNames = dict(self.newNames.items())