lsst.fgcmcal g5158719beb+cf3ce15f71
fgcmCalibrateTractTable.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Class for running fgcmcal on a single tract using sourceTable_visit tables.
24"""
25import numpy as np
26
27import lsst.pipe.base as pipeBase
28from lsst.pipe.base import connectionTypes
29from lsst.meas.algorithms import ReferenceObjectLoader
30import lsst.afw.table as afwTable
31
32from .dataIds import TractCheckDataIdContainer
33from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
34from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner,
35 FgcmCalibrateTractBaseTask)
36from .utilities import lookupStaticCalibrations
37
38__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
39
40
41class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument",
43 "tract",)):
44 camera = connectionTypes.PrerequisiteInput(
45 doc="Camera instrument",
46 name="camera",
47 storageClass="Camera",
48 dimensions=("instrument",),
49 lookupFunction=lookupStaticCalibrations,
50 isCalibration=True,
51 )
52
53 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
54 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
55 "chromatic corrections."),
56 name="fgcmLookUpTable",
57 storageClass="Catalog",
58 dimensions=("instrument",),
59 deferLoad=True,
60 )
61
62 sourceSchema = connectionTypes.InitInput(
63 doc="Schema for source catalogs",
64 name="src_schema",
65 storageClass="SourceCatalog",
66 )
67
68 refCat = connectionTypes.PrerequisiteInput(
69 doc="Reference catalog to use for photometric calibration",
70 name="cal_ref_cat",
71 storageClass="SimpleCatalog",
72 dimensions=("skypix",),
73 deferLoad=True,
74 multiple=True,
75 )
76
77 source_catalogs = connectionTypes.Input(
78 doc="Source table in parquet format, per visit",
79 name="sourceTable_visit",
80 storageClass="DataFrame",
81 dimensions=("instrument", "visit"),
82 deferLoad=True,
83 multiple=True,
84 )
85
86 visitSummary = connectionTypes.Input(
87 doc="Per-visit summary statistics table",
88 name="visitSummary",
89 storageClass="ExposureCatalog",
90 dimensions=("instrument", "visit"),
91 deferLoad=True,
92 multiple=True,
93 )
94
95 background = connectionTypes.Input(
96 doc="Calexp background model",
97 name="calexpBackground",
98 storageClass="Background",
99 dimensions=("instrument", "visit", "detector"),
100 deferLoad=True,
101 multiple=True,
102 )
103
104 fgcmPhotoCalib = connectionTypes.Output(
105 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
106 name="fgcmPhotoCalibTractCatalog",
107 storageClass="ExposureCatalog",
108 dimensions=("instrument", "tract", "visit",),
109 multiple=True,
110 )
111
112 fgcmTransmissionAtmosphere = connectionTypes.Output(
113 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
114 name="transmission_atmosphere_fgcm_tract",
115 storageClass="TransmissionCurve",
116 dimensions=("instrument", "tract", "visit",),
117 multiple=True,
118 )
119
120 fgcmRepeatability = connectionTypes.Output(
121 doc="Per-band raw repeatability numbers in the fgcm tract calibration",
122 name="fgcmRawRepeatability",
123 storageClass="Catalog",
124 dimensions=("instrument", "tract",),
125 multiple=False,
126 )
127
128 def __init__(self, *, config=None):
129 super().__init__(config=config)
130
131 # The ref_dataset_name will be deprecated with Gen2
132 loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name
133 if config.connections.refCat != loaderName:
134 raise ValueError("connections.refCat must be the same as "
135 "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name")
136 if config.fgcmOutputProducts.doReferenceCalibration:
137 loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name
138 if config.connections.refCat != loaderName:
139 raise ValueError("connections.refCat must be the same as "
140 "config.fgcmOutputProducts.refObjLoader.ref_dataset_name")
141
142 if not config.fgcmBuildStars.doModelErrorsWithBackground:
143 self.inputs.remove("background")
144
145 if config.fgcmOutputProducts.doRefcatOutput:
146 raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput")
147 if not config.fgcmOutputProducts.doAtmosphereOutput:
148 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
149 if not config.fgcmOutputProducts.doZeropointOutput:
150 self.prerequisiteInputs.remove("fgcmZeropoints")
151
152
154 pipelineConnections=FgcmCalibrateTractTableConnections):
155 """Config for FgcmCalibrateTractTable task"""
156 def setDefaults(self):
157 super().setDefaults()
158
159 # For the Table version of CalibrateTract, use the associated
160 # Table version of the BuildStars task.
161 self.fgcmBuildStarsfgcmBuildStars.retarget(FgcmBuildStarsTableTask)
162 # For tract mode, we set a very high effective density cut.
163 self.fgcmBuildStarsfgcmBuildStars.densityCutMaxPerPixel = 10000
164
165
167 """
168 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
169 input catalogs.
170 """
171 ConfigClass = FgcmCalibrateTractTableConfig
172 RunnerClass = FgcmCalibrateTractRunner
173 _DefaultName = "fgcmCalibrateTractTable"
174
175 canMultiprocess = False
176
177 def __init__(self, initInputs=None, **kwargs):
178 super().__init__(initInputs=initInputs, **kwargs)
179 if initInputs is not None:
180 self.sourceSchemasourceSchema = initInputs["sourceSchema"].schema
181
182 def runQuantum(self, butlerQC, inputRefs, outputRefs):
183 dataRefDict = butlerQC.get(inputRefs)
184
185 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs'])))
186
187 # Run the build stars tasks
188 tract = butlerQC.quantum.dataId['tract']
189
190 dataRefDict['sourceSchema'] = self.sourceSchemasourceSchema
191
192 sourceTableRefs = dataRefDict['source_catalogs']
193 sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for
194 sourceTableRef in sourceTableRefs}
195
196 visitSummaryRefs = dataRefDict['visitSummary']
197 visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for
198 visitSummaryRef in visitSummaryRefs}
199
200 dataRefDict['sourceTableDataRefDict'] = sourceTableDataRefDict
201 dataRefDict['visitSummaryDataRefDict'] = visitSummaryDataRefDict
202
203 # And the outputs
204 if self.config.fgcmOutputProducts.doZeropointOutput:
205 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
206 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
207 dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict
208
209 if self.config.fgcmOutputProducts.doAtmosphereOutput:
210 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
211 atmRef in outputRefs.fgcmTransmissionAtmosphere}
212 dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict
213
214 if self.config.fgcmBuildStars.doReferenceMatches:
215 refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader
216 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
217 for ref in inputRefs.refCat],
218 refCats=butlerQC.get(inputRefs.refCat),
219 config=refConfig,
220 log=self.log)
221 buildStarsRefObjLoader = loader
222 else:
223 buildStarsRefObjLoader = None
224
225 if self.config.fgcmOutputProducts.doReferenceCalibration:
226 refConfig = self.config.fgcmOutputProducts.refObjLoader
227 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
228 for ref in inputRefs.refCat],
229 refCats=butlerQC.get(inputRefs.refCat),
230 config=refConfig,
231 log=self.log)
232 self.fgcmOutputProducts.refObjLoader = loader
233
234 struct = self.runrun(dataRefDict, tract,
235 buildStarsRefObjLoader=buildStarsRefObjLoader)
236
237 if struct.photoCalibCatalogs is not None:
238 self.log.info("Outputting photoCalib catalogs.")
239 for visit, expCatalog in struct.photoCalibCatalogs:
240 butlerQC.put(expCatalog, photoCalibRefDict[visit])
241 self.log.info("Done outputting photoCalib catalogs.")
242
243 if struct.atmospheres is not None:
244 self.log.info("Outputting atmosphere transmission files.")
245 for visit, atm in struct.atmospheres:
246 butlerQC.put(atm, atmRefDict[visit])
247 self.log.info("Done outputting atmosphere files.")
248
249 # Turn raw repeatability into simple catalog for persistence
250 schema = afwTable.Schema()
251 schema.addField('rawRepeatability', type=np.float64,
252 doc="Per-band raw repeatability in FGCM calibration.")
253 repeatabilityCat = afwTable.BaseCatalog(schema)
254 repeatabilityCat.resize(len(struct.repeatability))
255 repeatabilityCat['rawRepeatability'][:] = struct.repeatability
256
257 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
258
259 return
260
261 @classmethod
262 def _makeArgumentParser(cls):
263 parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
264 parser.add_id_argument("--id", "sourceTable_visit",
265 help="Data ID, e.g. --id visit=6789 tract=9617",
266 ContainerClass=TractCheckDataIdContainer)
267
268 return parser
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)