From 990dc87092c2e5df5aaf268b4a7216ac05852446 Mon Sep 17 00:00:00 2001 From: Alex Broughton Date: Thu, 12 Dec 2024 09:50:33 -0800 Subject: [PATCH] Add merged detector results for calibs --- pipelines/_ingredients/verifyFlat.yaml | 20 ++- pipelines/_ingredients/verifyFlatLSST.yaml | 22 ++- python/lsst/cp/verify/configs/report.yaml | 12 ++ python/lsst/cp/verify/mergeResults.py | 168 +++++++++++++++++++++ 4 files changed, 213 insertions(+), 9 deletions(-) diff --git a/pipelines/_ingredients/verifyFlat.yaml b/pipelines/_ingredients/verifyFlat.yaml index d157984..02cd014 100644 --- a/pipelines/_ingredients/verifyFlat.yaml +++ b/pipelines/_ingredients/verifyFlat.yaml @@ -3,7 +3,7 @@ imports: - location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml include: - analyzeFlatCore - - analyzeFlatDetCore + - analyzeFlatDetMergeCore tasks: verifyFlatIsr: class: lsst.ip.isr.IsrTask @@ -22,6 +22,16 @@ tasks: connections.outputStats: "verifyFlatDetStats" connections.outputResults: "verifyFlatDetResults" useIsrStatistics: true + verifyFlatDetMerge: + class: lsst.cp.verify.CpVerifyDetMergeByFilterTask + config: + connections.inputStats: "verifyFlatDetStats" + connections.inputResults: "verifyFlatDetResults" + connections.outputStats: "verifyFlatDetMergeStats" + connections.outputResults: "verifyFlatDetMergeResults" + hasInputResults: true + hasMatrixCatalog: false + stageName: "flat" verifyFlatExp: class: lsst.cp.verify.CpVerifyFlatExpMergeTask config: @@ -46,14 +56,16 @@ tasks: config: connections.data: "verifyFlatResults" connections.outputName: "verifyFlatAnalysis" - analyzeFlatDetCore: + analyzeFlatDetMergeCore: class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter config: - connections.data: "verifyFlatDetResults" - connections.outputName: "verifyFlatDetAnalysis" + connections.data: "verifyFlatDetMergeResults" + connections.outputName: "verifyFlatDetMergeAnalysis" contracts: - verifyFlatDet.connections.inputExp == verifyFlatIsr.connections.outputExposure - verifyFlatExp.connections.inputStats == verifyFlatDet.connections.outputStats - verifyFlatExp.connections.inputResults == verifyFlatDet.connections.outputResults - verifyFlat.connections.inputStats == verifyFlatExp.connections.outputStats - verifyFlat.connections.inputResults == verifyFlatExp.connections.outputResults + - analyzeFlatCore.connections.data == verifyFlat.connections.outputResults + - analyzeFlatDetMergeCore.connections.data == verifyFlatDetMerge.connections.outputResults diff --git a/pipelines/_ingredients/verifyFlatLSST.yaml b/pipelines/_ingredients/verifyFlatLSST.yaml index e512bf3..1d25c74 100644 --- a/pipelines/_ingredients/verifyFlatLSST.yaml +++ b/pipelines/_ingredients/verifyFlatLSST.yaml @@ -3,7 +3,7 @@ imports: - location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml include: - analyzeFlatCore - - analyzeFlatDetCore + - analyzeFlatDetMergeCore tasks: verifyFlatIsr: class: lsst.ip.isr.IsrTaskLSST @@ -35,6 +35,16 @@ tasks: connections.outputStats: "verifyFlatDetStats" connections.outputResults: "verifyFlatDetResults" useIsrStatistics: true + verifyFlatDetMerge: + class: lsst.cp.verify.CpVerifyDetMergeByFilterTask + config: + connections.inputStats: "verifyFlatDetStats" + connections.inputResults: "verifyFlatDetResults" + connections.outputStats: "verifyFlatDetMergeStats" + connections.outputResults: "verifyFlatDetMergeResults" + hasInputResults: true + hasMatrixCatalog: false + stageName: "flat" verifyFlatExp: class: lsst.cp.verify.CpVerifyFlatExpMergeTask config: @@ -46,7 +56,7 @@ tasks: hasMatrixCatalog: false stageName: "flat" verifyFlat: - class: lsst.cp.verify.CpVerifyRunMergeByFilterTaskByFilter + class: lsst.cp.verify.CpVerifyRunMergeByFilterTask config: connections.inputStats: "verifyFlatExpStats" connections.inputResults: "verifyFlatExpResults" @@ -59,14 +69,16 @@ tasks: config: connections.data: "verifyFlatResults" connections.outputName: "verifyFlatAnalysis" - analyzeFlatDetCore: + analyzeFlatDetMergeCore: class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter config: - connections.data: "verifyFlatDetResults" - connections.outputName: "verifyFlatDetAnalysis" + connections.data: "verifyFlatDetMergeResults" + connections.outputName: "verifyFlatDetMergeAnalysis" contracts: - verifyFlatDet.connections.inputExp == verifyFlatIsr.connections.outputExposure - verifyFlatExp.connections.inputStats == verifyFlatDet.connections.outputStats - verifyFlatExp.connections.inputResults == verifyFlatDet.connections.outputResults - verifyFlat.connections.inputStats == verifyFlatExp.connections.outputStats - verifyFlat.connections.inputResults == verifyFlatExp.connections.outputResults + - analyzeFlatCore.connections.data == verifyFlat.connections.outputResults + - analyzeFlatDetMergeCore.connections.data == verifyFlatDetMerge.connections.outputResults diff --git a/python/lsst/cp/verify/configs/report.yaml b/python/lsst/cp/verify/configs/report.yaml index 1f68026..8ea9020 100644 --- a/python/lsst/cp/verify/configs/report.yaml +++ b/python/lsst/cp/verify/configs/report.yaml @@ -51,6 +51,18 @@ stages: description: "Processing configuration for the combined flat construction." verifyFlatApply_config: description: "Processing configuration for combined flat verification." + cpFlatCore_flatMeanPerAmp_FocalPlaneGeometryPlot: + description: "Focal plane mosaic of the combined flat means per amp." + cpFlatCore_flatNoisePerAmp_FocalPlaneGeometryPlot: + description: "Focal plane mosaic of the combined flat noise per amp." + cpFlatDetMergeCore_flatTestsByDate_GridPlot: + description: "Verfiy flat noise boolean by MJD per amp." + cpFlatDetMergeCore_flatMeansByDate_GridPlot: + description: "Flat mean by MJD per amp." + cpFlatDetMergeCore_flatNoiseByDate_GridPlot: + description: "Flat noise by MJD per amp." + cpFlatDetMergeCore_flatNoiseByMean_GridPlot: + description: "Verify flat noise by mean per amp." defects: defects: description: "The combined defects calibration." diff --git a/python/lsst/cp/verify/mergeResults.py b/python/lsst/cp/verify/mergeResults.py index 574ab8a..889948f 100644 --- a/python/lsst/cp/verify/mergeResults.py +++ b/python/lsst/cp/verify/mergeResults.py @@ -27,6 +27,8 @@ __all__ = ['CpVerifyExpMergeConfig', 'CpVerifyExpMergeTask', 'CpVerifyRunMergeConfig', 'CpVerifyRunMergeTask', + 'CpVerifyDetMergeConfig', 'CpVerifyDetMergeTask', + 'CpVerifyDetMergeByFilterConfig', 'CpVerifyDetMergeByFilterTask', 'CpVerifyExpMergeByFilterConfig', 'CpVerifyExpMergeByFilterTask', 'CpVerifyRunMergeByFilterConfig', 'CpVerifyRunMergeByFilterTask', 'CpVerifyVisitExpMergeConfig', 'CpVerifyVisitExpMergeTask', @@ -477,6 +479,172 @@ class CpVerifyRunMergeTask(CpVerifyExpMergeTask): # End ExpMerge/RunMerge +class CpVerifyDetMergeConnections(pipeBase.PipelineTaskConnections, + dimensions={"instrument", "detector"}, + defaultTemplates={}): + inputStats = cT.Input( + name="detectorStats", + doc="Input statistics to merge.", + storageClass="StructuredDataDict", + dimensions=["instrument", "exposure", "detector"], + multiple=True, + ) + inputResults = cT.Input( + name="detectorResults", + doc="Input results to merge.", + storageClass="ArrowAstropy", + dimensions=["instrument", "exposure", "detector"], + multiple=True, + ) + inputMatrix = cT.Input( + name="detectorMatrix", + doc="Input matrix to merge.", + storageClass="ArrowAstropy", + dimensions=["instrument", "exposure", "detector"], + multiple=True, + ) + camera = cT.PrerequisiteInput( + name="camera", + storageClass="Camera", + doc="Input camera.", + dimensions=["instrument", ], + isCalibration=True, + ) + + outputStats = cT.Output( + name="exposureStats", + doc="Output statistics.", + storageClass="StructuredDataDict", + dimensions=["instrument", "exposure"], + ) + outputResults = cT.Output( + name="exposureResults", + doc="Output results.", + storageClass="ArrowAstropy", + dimensions=["instrument", "exposure"], + ) + outputMatrix = cT.Output( + name="exposureMatrix", + doc="Output matrix.", + storageClass="ArrowAstropy", + dimensions=["instrument", "exposure"], + ) + + def __init__(self, *, config=None): + super().__init__(config=config) + + if not self.config.hasMatrixCatalog: + self.inputs.remove("inputMatrix") + self.outputs.remove("outputMatrix") + if not self.config.hasInputResults: + self.inputs.remove("inputResults") + if self.config.doDropStats: + self.outputs.remove("outputStats") + + +class CpVerifyDetMergeConfig(CpVerifyExpMergeConfig, + pipelineConnections=CpVerifyDetMergeConnections): + """Configuration paramters for exposure stats merging. + """ + mergeDimension = pexConfig.Field( + dtype=str, + doc="Dimension name for this input.", + default="detector", + ) + + +class CpVerifyDetMergeTask(CpVerifyExpMergeTask): + """Merge statistics from detectors together. + """ + ConfigClass = CpVerifyDetMergeConfig + _DefaultName = 'cpVerifyDetMerge' + + pass + + +class CpVerifyDetMergeByFilterConnections(pipeBase.PipelineTaskConnections, + dimensions={"instrument", "detector", "physical_filter"}, + defaultTemplates={}): + inputStats = cT.Input( + name="detectorStats", + doc="Input statistics to merge.", + storageClass="StructuredDataDict", + dimensions=["instrument", "exposure", "detector", "physical_filter"], + multiple=True, + ) + inputResults = cT.Input( + name="detectorResults", + doc="Input results to merge.", + storageClass="ArrowAstropy", + dimensions=["instrument", "exposure", "detector", "physical_filter"], + multiple=True, + ) + inputMatrix = cT.Input( + name="detectorMatrix", + doc="Input matrix to merge.", + storageClass="ArrowAstropy", + dimensions=["instrument", "exposure", "detector", "physical_filter"], + multiple=True, + ) + camera = cT.PrerequisiteInput( + name="camera", + storageClass="Camera", + doc="Input camera.", + dimensions=["instrument", ], + isCalibration=True, + ) + + outputStats = cT.Output( + name="exposureStats", + doc="Output statistics.", + storageClass="StructuredDataDict", + dimensions=["instrument", "detector", "physical_filter"], + ) + outputResults = cT.Output( + name="exposureResults", + doc="Output results.", + storageClass="ArrowAstropy", + dimensions=["instrument", "detector", "physical_filter"], + ) + outputMatrix = cT.Output( + name="exposureMatrix", + doc="Output matrix.", + storageClass="ArrowAstropy", + dimensions=["instrument", "detector", "physical_filter"], + ) + + def __init__(self, *, config=None): + super().__init__(config=config) + + if not self.config.hasMatrixCatalog: + self.inputs.remove("inputMatrix") + self.outputs.remove("outputMatrix") + if not self.config.hasInputResults: + self.inputs.remove("inputResults") + if self.config.doDropStats: + self.outputs.remove("outputStats") + + +class CpVerifyDetMergeByFilterConfig(CpVerifyExpMergeConfig, + pipelineConnections=CpVerifyDetMergeByFilterConnections): + """Configuration paramters for exposure stats merging. + """ + mergeDimension = pexConfig.Field( + dtype=str, + doc="Dimension name for this input.", + default="detector", + ) + + +class CpVerifyDetMergeByFilterTask(CpVerifyExpMergeTask): + """Merge statistics from detectors together. + """ + ConfigClass = CpVerifyDetMergeByFilterConfig + _DefaultName = 'cpVerifyDetMergeByFilter' + + pass + + # Begin ExpMergeByFilter/RunMergeByFilter class CpVerifyExpMergeByFilterConnections(pipeBase.PipelineTaskConnections, dimensions={"instrument", "exposure", "physical_filter"},