DM-55280: Add analysis_tools task and tooling to aggregate preliminary_visit_summary table

config/aggregateCalibrateImageMetadata.py

@@ -0,0 +1,82 @@
+# Configuration for lsst.analysis.tools.tasks.AggregatedTaskMetadataAnalysisTask
+# when aggregating calibrateImage task metadata across detectors to produce
+# per-visit aggregate statistics.
+
+from lsst.analysis.tools.actions.scalar import CountAction, MaxAction, MedianAction, MinAction, SigmaMadAction
+from lsst.analysis.tools.atools import AggregatedTaskMetadataMetricTool
+
+config.atools.calibrateImageMetadataMetrics = AggregatedTaskMetadataMetricTool
+config.atools.calibrateImageMetadataMetrics.taskName = "calibrateImage"
+config.atools.calibrateImageMetadataMetrics.metrics = {
+    "initial_psf_positive_footprint_count": "ct",
+    "initial_psf_negative_footprint_count": "ct",
+    "initial_psf_positive_peak_count": "ct",
+    "initial_psf_negative_peak_count": "ct",
+    "simple_psf_positive_footprint_count": "ct",
+    "simple_psf_negative_footprint_count": "ct",
+    "simple_psf_positive_peak_count": "ct",
+    "simple_psf_negative_peak_count": "ct",
+    "bad_mask_fraction": "",
+    "cr_mask_fraction": "",
+    "crosstalk_mask_fraction": "",
+    "detected_mask_fraction": "",
+    "detected_negative_mask_fraction": "",
+    "edge_mask_fraction": "",
+    "intrp_mask_fraction": "",
+    "no_data_mask_fraction": "",
+    "sat_mask_fraction": "",
+    "suspect_mask_fraction": "",
+    "unmaskednan_mask_fraction": "",
+    "numAvailStars": "ct",
+    "numGoodStars": "ct",
+    "sky_footprint_count": "ct",
+    "post_deblend_source_count": "ct",
+    "star_count": "ct",
+    "saturated_source_count": "ct",
+    "bad_source_count": "ct",
+    "cosmic_ray_count": "ct",
+    "matched_psf_star_count": "ct",
+    "final_psf_sigma": "pixel",
+    "astrometry_matches_count": "ct",
+    "photometry_matches_count": "ct",
+    "bg_subtracted_skyPixel_instFlux_median": "adu",
+    "bg_subtracted_skyPixel_instFlux_stdev": "adu",
+    "bg_subtracted_skySource_flux_median": "nJy",
+    "bg_subtracted_skySource_flux_stdev": "nJy",
+    "adaptive_threshold_value": "",
+    "initial_to_final_wcs": "arcsec",
+    "astrom_offset_mean": "arcsec",
+    "astrom_offset_std": "arcsec",
+    "astrom_offset_median": "arcsec",
+    "failed_deblend_source_count": "ct",
+}
+config.atools.calibrateImageMetadataMetrics.subTaskNames = {
+    "numAvailStars": "psf_measure_psf",
+    "numGoodStars": "psf_measure_psf",
+    "sky_footprint_count": "star_sky_sources",
+    "cosmic_ray_count": "psf_repair",
+}
+config.atools.calibrateImageMetadataMetrics.newNames = {
+    "numAvailStars": "psf_available_star_count",
+    "numGoodStars": "psf_good_star_count",
+    "adaptive_threshold_value": "final_adaptive_threshold_value",
+}
+
+aggregations = {
+    "min": MinAction,
+    "max": MaxAction,
+    "median": MedianAction,
+    "mad": SigmaMadAction,
+    "ct": CountAction,
+}
+config.atools.calibrateImageMetadataMetrics.aggregationUnits = {"ct": "ct"}
+
+for metric in config.atools.calibrateImageMetadataMetrics.metrics:
+    for agg_name, agg_cls in aggregations.items():
+        action = agg_cls()
+        action.vectorKey = metric
+        setattr(
+            config.atools.calibrateImageMetadataMetrics.process.calculateActions,
+            f"{metric}_{agg_name}",
+            action,
+        )

pipelines/visitQualityCore.yaml

@@ -38,66 +38,37 @@ tasks:
     config:
       atools.calexpSummaryMetrics: CalexpSummaryMetrics
       python: from lsst.analysis.tools.atools import *
-  analyzeCalibrateImageMetadata:
-    class: lsst.analysis.tools.tasks.TaskMetadataAnalysisTask
+  aggregateCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.AggregatedTaskMetadataAnalysisTask
     config:
       connections.inputName: calibrateImage_metadata
-      connections.outputName: calibrateImage_metadata  # Will be appended with "_metrics"
-      connections.storageClass: TaskMetadata
-      raiseNoWorkFoundOnIncompleteMetadata: true
-      inputDimensions: ["instrument", "visit", "detector"]
-      atools.calibrateImageMetadataMetrics: TaskMetadataMetricTool
-      atools.calibrateImageMetadataMetrics.taskName: calibrateImage
-      atools.calibrateImageMetadataMetrics.metrics:
-        initial_psf_positive_footprint_count: ct
-        initial_psf_negative_footprint_count: ct
-        initial_psf_positive_peak_count: ct
-        initial_psf_negative_peak_count: ct
-        simple_psf_positive_footprint_count: ct
-        simple_psf_negative_footprint_count: ct
-        simple_psf_positive_peak_count: ct
-        simple_psf_negative_peak_count: ct
-        bad_mask_fraction: ""
-        cr_mask_fraction: ""
-        crosstalk_mask_fraction: ""
-        detected_mask_fraction: ""
-        detected_negative_mask_fraction: ""
-        edge_mask_fraction: ""
-        intrp_mask_fraction: ""
-        no_data_mask_fraction: ""
-        sat_mask_fraction: ""
-        suspect_mask_fraction: ""
-        unmaskednan_mask_fraction: ""
-        numAvailStars: ct
-        numGoodStars: ct
-        sky_footprint_count: ct
-        post_deblend_source_count: ct
-        star_count: ct
-        saturated_source_count: ct
-        bad_source_count: ct
-        cosmic_ray_count: ct
-        matched_psf_star_count: ct
-        final_psf_sigma: pixel
-        astrometry_matches_count: ct
-        photometry_matches_count: ct
-        bg_subtracted_skyPixel_instFlux_median: adu
-        bg_subtracted_skyPixel_instFlux_stdev: adu
-        bg_subtracted_skySource_flux_median: nJy
-        bg_subtracted_skySource_flux_stdev: nJy
-        adaptive_threshold_value: ""
-        initial_to_final_wcs: arcsec
-        astrom_offset_mean: arcsec
-        astrom_offset_std: arcsec
-        astrom_offset_median: arcsec
-        failed_deblend_source_count: ct
-      atools.calibrateImageMetadataMetrics.subTaskNames:
-        numAvailStars: psf_measure_psf
-        numGoodStars: psf_measure_psf
-        sky_footprint_count: star_sky_sources
-        cosmic_ray_count: psf_repair
-      atools.calibrateImageMetadataMetrics.newNames:
-        numAvailStars: psf_available_star_count
-        numGoodStars: psf_good_star_count
-        adaptive_threshold_value: final_adaptive_threshold_value
-      python: |
-        from lsst.analysis.tools.atools import TaskMetadataMetricTool
+      connections.outputName: calibrateImage_metadata_aggregate
+      inputDataDimensions: ["instrument", "visit", "detector"]
+      outputDimensions: ["instrument", "visit"]
+      file: $ANALYSIS_TOOLS_DIR/config/aggregateCalibrateImageMetadata.py
+  makeMetricTableAggregateCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.MakeMetricTableTask
+    config:
+      connections.metricBundleName: "calibrateImage_metadata_aggregate_metrics"
+      connections.outputTableName: "calibrateImage_metadata_aggregate_metrics_table"
+      inputDataDimensions: ["instrument", "visit"]
+      outputTableDimensions: ["instrument"]
+      dataIdFieldsToIncludeAsColumns: ["visit", "band"]
+  aggregateVisitSummary:
+    class: lsst.analysis.tools.tasks.ExposureCatalogAnalysisTask
+    config:
+      connections.inputName: visitSummary
+      connections.outputName: aggregateVisitSummary
+      inputTableDimensions: ["visit"]
+      taskDimensions: ["visit"]
+      atools.aggregateCalexpSummaryStats: AggregateCalexpSummaryStats
+      atools.aggregateCornerSeparations: AggregateCornerSeparations
+      python: from lsst.analysis.tools.atools import *
+  makeMetricTableAggregateVisitSummary:
+    class: lsst.analysis.tools.tasks.MakeMetricTableTask
+    config:
+      connections.metricBundleName: aggregateVisitSummary_metrics
+      connections.outputTableName: aggregateVisitSummary_metricsTable
+      inputDataDimensions: ["visit"]
+      outputTableDimensions: ["instrument"]
+      dataIdFieldsToIncludeAsColumns: ["visit", "band"]

pipelines/visitQualityExtended.yaml

@@ -1,8 +1,6 @@
 description: |
   Tier2 plots and metrics to assess visit quality
 tasks:
-  catalogMatchVisit:
-    class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
   refCatSourceVisit:
     class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
   deltaSkyCorrHist:
@@ -13,3 +11,66 @@ tasks:
       atools.deltaSkyCorr: DeltaSkyCorrXYPlot
       python: |
         from lsst.analysis.tools.atools import *
+  analyzeCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.TaskMetadataAnalysisTask
+    config:
+      connections.inputName: calibrateImage_metadata
+      connections.outputName: calibrateImage_metadata  # Will be appended with "_metrics"
+      connections.storageClass: TaskMetadata
+      raiseNoWorkFoundOnIncompleteMetadata: true
+      inputDimensions: ["instrument", "visit", "detector"]
+      atools.calibrateImageMetadataMetrics: TaskMetadataMetricTool
+      atools.calibrateImageMetadataMetrics.taskName: calibrateImage
+      atools.calibrateImageMetadataMetrics.metrics:
+        initial_psf_positive_footprint_count: ct
+        initial_psf_negative_footprint_count: ct
+        initial_psf_positive_peak_count: ct
+        initial_psf_negative_peak_count: ct
+        simple_psf_positive_footprint_count: ct
+        simple_psf_negative_footprint_count: ct
+        simple_psf_positive_peak_count: ct
+        simple_psf_negative_peak_count: ct
+        bad_mask_fraction: ""
+        cr_mask_fraction: ""
+        crosstalk_mask_fraction: ""
+        detected_mask_fraction: ""
+        detected_negative_mask_fraction: ""
+        edge_mask_fraction: ""
+        intrp_mask_fraction: ""
+        no_data_mask_fraction: ""
+        sat_mask_fraction: ""
+        suspect_mask_fraction: ""
+        unmaskednan_mask_fraction: ""
+        numAvailStars: ct
+        numGoodStars: ct
+        sky_footprint_count: ct
+        post_deblend_source_count: ct
+        star_count: ct
+        saturated_source_count: ct
+        bad_source_count: ct
+        cosmic_ray_count: ct
+        matched_psf_star_count: ct
+        final_psf_sigma: pixel
+        astrometry_matches_count: ct
+        photometry_matches_count: ct
+        bg_subtracted_skyPixel_instFlux_median: adu
+        bg_subtracted_skyPixel_instFlux_stdev: adu
+        bg_subtracted_skySource_flux_median: nJy
+        bg_subtracted_skySource_flux_stdev: nJy
+        adaptive_threshold_value: ""
+        initial_to_final_wcs: arcsec
+        astrom_offset_mean: arcsec
+        astrom_offset_std: arcsec
+        astrom_offset_median: arcsec
+        failed_deblend_source_count: ct
+      atools.calibrateImageMetadataMetrics.subTaskNames:
+        numAvailStars: psf_measure_psf
+        numGoodStars: psf_measure_psf
+        sky_footprint_count: star_sky_sources
+        cosmic_ray_count: psf_repair
+      atools.calibrateImageMetadataMetrics.newNames:
+        numAvailStars: psf_available_star_count
+        numGoodStars: psf_good_star_count
+        adaptive_threshold_value: final_adaptive_threshold_value
+      python: |
+        from lsst.analysis.tools.atools import TaskMetadataMetricTool

python/lsst/analysis/tools/actions/plot/matrixPlot.py

@@ -310,7 +310,7 @@ def _validateInput(self, data: KeyedData, **kwargs: Any) -> None:
         if not isinstance(data[self.matrixKey], np.ndarray) and data[self.matrixKey].ndim != 2:
             raise ValueError(f"Input data is not a 2d array: {data[self.matrixKey]}")
         # Check that the keyword arguments are valid.
-        acceptableKwargs = {"plotInfo", "skymap", "band", "metric_tags", "fig"}
+        acceptableKwargs = {"plotInfo", "skymap", "band", "metric_tags", "produce_plots", "fig"}
         if not set(kwargs).issubset(acceptableKwargs):
             raise ValueError(
                 f"Only the following keyword arguments are allowed: {acceptableKwargs}. Got: {kwargs}"

python/lsst/analysis/tools/atools/__init__.py

@@ -1,7 +1,6 @@
 from .actionMagnitudeScatterPlot import *
 from .amplifierCorrelation import *
 from .astrometricRepeatability import *
-from .astrometryMetrics import *
 from .calexpMetrics import *
 from .calibQuantityProfile import *
 from .calibration import *