Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py: 23%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Class for running fgcmcal on a single tract using sourceTable_visit tables.
24"""
25import numpy as np
27import lsst.pipe.base as pipeBase
28from lsst.pipe.base import connectionTypes
29from lsst.meas.algorithms import ReferenceObjectLoader
30import lsst.afw.table as afwTable
32from .dataIds import TractCheckDataIdContainer
33from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
34from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner,
35 FgcmCalibrateTractBaseTask)
36from .utilities import lookupStaticCalibrations
38__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
41class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument",
43 "tract",)):
44 camera = connectionTypes.PrerequisiteInput(
45 doc="Camera instrument",
46 name="camera",
47 storageClass="Camera",
48 dimensions=("instrument",),
49 lookupFunction=lookupStaticCalibrations,
50 isCalibration=True,
51 )
53 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
54 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
55 "chromatic corrections."),
56 name="fgcmLookUpTable",
57 storageClass="Catalog",
58 dimensions=("instrument",),
59 deferLoad=True,
60 )
62 sourceSchema = connectionTypes.InitInput(
63 doc="Schema for source catalogs",
64 name="src_schema",
65 storageClass="SourceCatalog",
66 )
68 refCat = connectionTypes.PrerequisiteInput(
69 doc="Reference catalog to use for photometric calibration",
70 name="cal_ref_cat",
71 storageClass="SimpleCatalog",
72 dimensions=("skypix",),
73 deferLoad=True,
74 multiple=True,
75 )
77 source_catalogs = connectionTypes.Input(
78 doc="Source table in parquet format, per visit",
79 name="sourceTable_visit",
80 storageClass="DataFrame",
81 dimensions=("instrument", "visit"),
82 deferLoad=True,
83 multiple=True,
84 )
86 visitSummary = connectionTypes.Input(
87 doc="Per-visit summary statistics table",
88 name="visitSummary",
89 storageClass="ExposureCatalog",
90 dimensions=("instrument", "visit"),
91 deferLoad=True,
92 multiple=True,
93 )
95 background = connectionTypes.Input(
96 doc="Calexp background model",
97 name="calexpBackground",
98 storageClass="Background",
99 dimensions=("instrument", "visit", "detector"),
100 deferLoad=True,
101 multiple=True,
102 )
104 fgcmPhotoCalib = connectionTypes.Output(
105 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
106 name="fgcmPhotoCalibTractCatalog",
107 storageClass="ExposureCatalog",
108 dimensions=("instrument", "tract", "visit",),
109 multiple=True,
110 )
112 fgcmTransmissionAtmosphere = connectionTypes.Output(
113 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
114 name="transmission_atmosphere_fgcm_tract",
115 storageClass="TransmissionCurve",
116 dimensions=("instrument", "tract", "visit",),
117 multiple=True,
118 )
120 fgcmRepeatability = connectionTypes.Output(
121 doc="Per-band raw repeatability numbers in the fgcm tract calibration",
122 name="fgcmRawRepeatability",
123 storageClass="Catalog",
124 dimensions=("instrument", "tract",),
125 multiple=False,
126 )
128 def __init__(self, *, config=None):
129 super().__init__(config=config)
131 # The ref_dataset_name will be deprecated with Gen2
132 loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name
133 if config.connections.refCat != loaderName:
134 raise ValueError("connections.refCat must be the same as "
135 "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name")
136 if config.fgcmOutputProducts.doReferenceCalibration:
137 loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name
138 if config.connections.refCat != loaderName:
139 raise ValueError("connections.refCat must be the same as "
140 "config.fgcmOutputProducts.refObjLoader.ref_dataset_name")
142 if not config.fgcmBuildStars.doModelErrorsWithBackground:
143 self.inputs.remove("background")
145 if config.fgcmOutputProducts.doRefcatOutput:
146 raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput")
147 if not config.fgcmOutputProducts.doAtmosphereOutput:
148 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
149 if not config.fgcmOutputProducts.doZeropointOutput:
150 self.prerequisiteInputs.remove("fgcmZeropoints")
153class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig,
154 pipelineConnections=FgcmCalibrateTractTableConnections):
155 """Config for FgcmCalibrateTractTable task"""
156 def setDefaults(self):
157 super().setDefaults()
159 # For the Table version of CalibrateTract, use the associated
160 # Table version of the BuildStars task.
161 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask)
162 # For tract mode, we set a very high effective density cut.
163 self.fgcmBuildStars.densityCutMaxPerPixel = 10000
166class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask):
167 """
168 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
169 input catalogs.
170 """
171 ConfigClass = FgcmCalibrateTractTableConfig
172 RunnerClass = FgcmCalibrateTractRunner
173 _DefaultName = "fgcmCalibrateTractTable"
175 canMultiprocess = False
177 def __init__(self, initInputs=None, **kwargs):
178 super().__init__(initInputs=initInputs, **kwargs)
179 if initInputs is not None:
180 self.sourceSchema = initInputs["sourceSchema"].schema
182 def runQuantum(self, butlerQC, inputRefs, outputRefs):
183 dataRefDict = butlerQC.get(inputRefs)
185 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs'])))
187 # Run the build stars tasks
188 tract = butlerQC.quantum.dataId['tract']
190 dataRefDict['sourceSchema'] = self.sourceSchema
192 sourceTableRefs = dataRefDict['source_catalogs']
193 sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for
194 sourceTableRef in sourceTableRefs}
196 visitSummaryRefs = dataRefDict['visitSummary']
197 visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for
198 visitSummaryRef in visitSummaryRefs}
200 dataRefDict['sourceTableDataRefDict'] = sourceTableDataRefDict
201 dataRefDict['visitSummaryDataRefDict'] = visitSummaryDataRefDict
203 # And the outputs
204 if self.config.fgcmOutputProducts.doZeropointOutput:
205 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
206 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
207 dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict
209 if self.config.fgcmOutputProducts.doAtmosphereOutput:
210 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
211 atmRef in outputRefs.fgcmTransmissionAtmosphere}
212 dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict
214 if self.config.fgcmBuildStars.doReferenceMatches:
215 refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader
216 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
217 for ref in inputRefs.refCat],
218 refCats=butlerQC.get(inputRefs.refCat),
219 config=refConfig,
220 log=self.log)
221 buildStarsRefObjLoader = loader
222 else:
223 buildStarsRefObjLoader = None
225 if self.config.fgcmOutputProducts.doReferenceCalibration:
226 refConfig = self.config.fgcmOutputProducts.refObjLoader
227 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
228 for ref in inputRefs.refCat],
229 refCats=butlerQC.get(inputRefs.refCat),
230 config=refConfig,
231 log=self.log)
232 self.fgcmOutputProducts.refObjLoader = loader
234 struct = self.run(dataRefDict, tract,
235 buildStarsRefObjLoader=buildStarsRefObjLoader)
237 if struct.photoCalibCatalogs is not None:
238 self.log.info("Outputting photoCalib catalogs.")
239 for visit, expCatalog in struct.photoCalibCatalogs:
240 butlerQC.put(expCatalog, photoCalibRefDict[visit])
241 self.log.info("Done outputting photoCalib catalogs.")
243 if struct.atmospheres is not None:
244 self.log.info("Outputting atmosphere transmission files.")
245 for visit, atm in struct.atmospheres:
246 butlerQC.put(atm, atmRefDict[visit])
247 self.log.info("Done outputting atmosphere files.")
249 # Turn raw repeatability into simple catalog for persistence
250 schema = afwTable.Schema()
251 schema.addField('rawRepeatability', type=np.float64,
252 doc="Per-band raw repeatability in FGCM calibration.")
253 repeatabilityCat = afwTable.BaseCatalog(schema)
254 repeatabilityCat.resize(len(struct.repeatability))
255 repeatabilityCat['rawRepeatability'][:] = struct.repeatability
257 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
259 return
261 @classmethod
262 def _makeArgumentParser(cls):
263 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
264 parser.add_id_argument("--id", "sourceTable_visit",
265 help="Data ID, e.g. --id visit=6789 tract=9617",
266 ContainerClass=TractCheckDataIdContainer)
268 return parser