Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py: 25%
89 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-25 12:04 +0000
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-25 12:04 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Class for running fgcmcal on a single tract using sourceTable_visit tables.
24"""
25import numpy as np
27import lsst.pipe.base as pipeBase
28from lsst.pipe.base import connectionTypes
29from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
30import lsst.afw.table as afwTable
32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
33from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase,
34 FgcmCalibrateTractBaseTask)
35from .utilities import lookupStaticCalibrations
37__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
40class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
41 dimensions=("instrument",
42 "tract",)):
43 camera = connectionTypes.PrerequisiteInput(
44 doc="Camera instrument",
45 name="camera",
46 storageClass="Camera",
47 dimensions=("instrument",),
48 lookupFunction=lookupStaticCalibrations,
49 isCalibration=True,
50 )
52 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
53 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
54 "chromatic corrections."),
55 name="fgcmLookUpTable",
56 storageClass="Catalog",
57 dimensions=("instrument",),
58 deferLoad=True,
59 )
61 sourceSchema = connectionTypes.InitInput(
62 doc="Schema for source catalogs",
63 name="src_schema",
64 storageClass="SourceCatalog",
65 )
67 refCat = connectionTypes.PrerequisiteInput(
68 doc="Reference catalog to use for photometric calibration",
69 name="cal_ref_cat",
70 storageClass="SimpleCatalog",
71 dimensions=("skypix",),
72 deferLoad=True,
73 multiple=True,
74 )
76 source_catalogs = connectionTypes.Input(
77 doc="Source table in parquet format, per visit",
78 name="sourceTable_visit",
79 storageClass="DataFrame",
80 dimensions=("instrument", "visit"),
81 deferLoad=True,
82 multiple=True,
83 )
85 visitSummary = connectionTypes.Input(
86 doc="Per-visit summary statistics table",
87 name="visitSummary",
88 storageClass="ExposureCatalog",
89 dimensions=("instrument", "visit"),
90 deferLoad=True,
91 multiple=True,
92 )
94 background = connectionTypes.Input(
95 doc="Calexp background model",
96 name="calexpBackground",
97 storageClass="Background",
98 dimensions=("instrument", "visit", "detector"),
99 deferLoad=True,
100 multiple=True,
101 )
103 fgcmPhotoCalib = connectionTypes.Output(
104 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
105 name="fgcmPhotoCalibTractCatalog",
106 storageClass="ExposureCatalog",
107 dimensions=("instrument", "tract", "visit",),
108 multiple=True,
109 )
111 fgcmTransmissionAtmosphere = connectionTypes.Output(
112 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
113 name="transmission_atmosphere_fgcm_tract",
114 storageClass="TransmissionCurve",
115 dimensions=("instrument", "tract", "visit",),
116 multiple=True,
117 )
119 fgcmRepeatability = connectionTypes.Output(
120 doc="Per-band raw repeatability numbers in the fgcm tract calibration",
121 name="fgcmRawRepeatability",
122 storageClass="Catalog",
123 dimensions=("instrument", "tract",),
124 multiple=False,
125 )
127 def __init__(self, *, config=None):
128 super().__init__(config=config)
130 if not config.fgcmBuildStars.doModelErrorsWithBackground:
131 self.inputs.remove("background")
133 if not config.fgcmOutputProducts.doAtmosphereOutput:
134 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
135 if not config.fgcmOutputProducts.doZeropointOutput:
136 self.prerequisiteInputs.remove("fgcmZeropoints")
138 def getSpatialBoundsConnections(self):
139 return ("visitSummary",)
142class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig,
143 pipelineConnections=FgcmCalibrateTractTableConnections):
144 """Config for FgcmCalibrateTractTable task"""
145 def setDefaults(self):
146 super().setDefaults()
148 # For the Table version of CalibrateTract, use the associated
149 # Table version of the BuildStars task.
150 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask)
151 # For tract mode, we set a very high effective density cut.
152 self.fgcmBuildStars.densityCutMaxPerPixel = 10000
155class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask):
156 """
157 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
158 input catalogs.
159 """
160 ConfigClass = FgcmCalibrateTractTableConfig
161 _DefaultName = "fgcmCalibrateTractTable"
163 canMultiprocess = False
165 def __init__(self, initInputs=None, **kwargs):
166 super().__init__(initInputs=initInputs, **kwargs)
167 if initInputs is not None:
168 self.sourceSchema = initInputs["sourceSchema"].schema
170 def runQuantum(self, butlerQC, inputRefs, outputRefs):
171 handleDict = butlerQC.get(inputRefs)
173 self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs'])))
175 # Run the build stars tasks
176 tract = butlerQC.quantum.dataId['tract']
178 handleDict['sourceSchema'] = self.sourceSchema
180 sourceTableHandles = handleDict['source_catalogs']
181 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
182 sourceTableHandle in sourceTableHandles}
184 visitSummaryHandles = handleDict['visitSummary']
185 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
186 visitSummaryHandle in visitSummaryHandles}
188 handleDict['sourceTableHandleDict'] = sourceTableHandleDict
189 handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict
191 # And the outputs
192 if self.config.fgcmOutputProducts.doZeropointOutput:
193 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
194 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
195 handleDict['fgcmPhotoCalibs'] = photoCalibRefDict
197 if self.config.fgcmOutputProducts.doAtmosphereOutput:
198 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
199 atmRef in outputRefs.fgcmTransmissionAtmosphere}
200 handleDict['fgcmTransmissionAtmospheres'] = atmRefDict
202 if self.config.fgcmBuildStars.doReferenceMatches:
203 refConfig = LoadReferenceObjectsConfig()
204 refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap
205 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
206 for ref in inputRefs.refCat],
207 refCats=butlerQC.get(inputRefs.refCat),
208 name=self.config.connections.refCat,
209 config=refConfig,
210 log=self.log)
211 buildStarsRefObjLoader = loader
212 else:
213 buildStarsRefObjLoader = None
215 if self.config.fgcmOutputProducts.doReferenceCalibration:
216 refConfig = self.config.fgcmOutputProducts.refObjLoader
217 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
218 for ref in inputRefs.refCat],
219 refCats=butlerQC.get(inputRefs.refCat),
220 name=self.config.connections.refCat,
221 config=refConfig,
222 log=self.log)
223 self.fgcmOutputProducts.refObjLoader = loader
225 struct = self.run(handleDict, tract,
226 buildStarsRefObjLoader=buildStarsRefObjLoader)
228 if struct.photoCalibCatalogs is not None:
229 self.log.info("Outputting photoCalib catalogs.")
230 for visit, expCatalog in struct.photoCalibCatalogs:
231 butlerQC.put(expCatalog, photoCalibRefDict[visit])
232 self.log.info("Done outputting photoCalib catalogs.")
234 if struct.atmospheres is not None:
235 self.log.info("Outputting atmosphere transmission files.")
236 for visit, atm in struct.atmospheres:
237 butlerQC.put(atm, atmRefDict[visit])
238 self.log.info("Done outputting atmosphere files.")
240 # Turn raw repeatability into simple catalog for persistence
241 schema = afwTable.Schema()
242 schema.addField('rawRepeatability', type=np.float64,
243 doc="Per-band raw repeatability in FGCM calibration.")
244 repeatabilityCat = afwTable.BaseCatalog(schema)
245 repeatabilityCat.resize(len(struct.repeatability))
246 repeatabilityCat['rawRepeatability'][:] = struct.repeatability
248 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
250 return