Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Class for running fgcmcal on a single tract using sourceTable_visit tables.
24"""
25import numpy as np
27import lsst.pipe.base as pipeBase
28from lsst.pipe.base import connectionTypes
29from lsst.meas.algorithms import ReferenceObjectLoader
30import lsst.afw.table as afwTable
32from .dataIds import TractCheckDataIdContainer
33from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
34from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner,
35 FgcmCalibrateTractBaseTask)
36from .utilities import lookupStaticCalibrations
38__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
41class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument",
43 "tract",)):
44 camera = connectionTypes.PrerequisiteInput(
45 doc="Camera instrument",
46 name="camera",
47 storageClass="Camera",
48 dimensions=("instrument",),
49 lookupFunction=lookupStaticCalibrations,
50 isCalibration=True,
51 )
53 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
54 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
55 "chromatic corrections."),
56 name="fgcmLookUpTable",
57 storageClass="Catalog",
58 dimensions=("instrument",),
59 deferLoad=True,
60 )
62 sourceSchema = connectionTypes.PrerequisiteInput(
63 doc="Schema for source catalogs",
64 name="src_schema",
65 storageClass="SourceCatalog",
66 deferLoad=True,
67 )
69 refCat = connectionTypes.PrerequisiteInput(
70 doc="Reference catalog to use for photometric calibration",
71 name="cal_ref_cat",
72 storageClass="SimpleCatalog",
73 dimensions=("skypix",),
74 deferLoad=True,
75 multiple=True,
76 )
78 source_catalogs = connectionTypes.Input(
79 doc="Source table in parquet format, per visit",
80 name="sourceTable_visit",
81 storageClass="DataFrame",
82 dimensions=("instrument", "visit"),
83 deferLoad=True,
84 multiple=True,
85 )
87 calexp = connectionTypes.Input(
88 doc="Calibrated exposures used for psf and metadata",
89 name="calexp",
90 storageClass="ExposureF",
91 dimensions=("instrument", "visit", "detector"),
92 deferLoad=True,
93 multiple=True,
94 )
96 background = connectionTypes.Input(
97 doc="Calexp background model",
98 name="calexpBackground",
99 storageClass="Background",
100 dimensions=("instrument", "visit", "detector"),
101 deferLoad=True,
102 multiple=True,
103 )
105 fgcmPhotoCalib = connectionTypes.Output(
106 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
107 name="fgcmPhotoCalibTractCatalog",
108 storageClass="ExposureCatalog",
109 dimensions=("instrument", "tract", "visit",),
110 multiple=True,
111 )
113 fgcmTransmissionAtmosphere = connectionTypes.Output(
114 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
115 name="transmission_atmosphere_fgcm_tract",
116 storageClass="TransmissionCurve",
117 dimensions=("instrument", "tract", "visit",),
118 multiple=True,
119 )
121 fgcmRepeatability = connectionTypes.Output(
122 doc="Per-band raw repeatability numbers in the fgcm tract calibration",
123 name="fgcmRawRepeatability",
124 storageClass="Catalog",
125 dimensions=("instrument", "tract",),
126 multiple=False,
127 )
129 def __init__(self, *, config=None):
130 super().__init__(config=config)
132 # The ref_dataset_name will be deprecated with Gen2
133 loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name
134 if config.connections.refCat != loaderName:
135 raise ValueError("connections.refCat must be the same as "
136 "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name")
137 if config.fgcmOutputProducts.doReferenceCalibration:
138 loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name
139 if config.connections.refCat != loaderName:
140 raise ValueError("connections.refCat must be the same as "
141 "config.fgcmOutputProducts.refObjLoader.ref_dataset_name")
143 if not config.fgcmBuildStars.doModelErrorsWithBackground:
144 self.inputs.remove("background")
146 if config.fgcmOutputProducts.doRefcatOutput:
147 raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput")
148 if not config.fgcmOutputProducts.doAtmosphereOutput:
149 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
150 if not config.fgcmOutputProducts.doZeropointOutput:
151 self.prerequisiteInputs.remove("fgcmZeropoints")
154class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig,
155 pipelineConnections=FgcmCalibrateTractTableConnections):
156 """Config for FgcmCalibrateTractTable task"""
157 def setDefaults(self):
158 super().setDefaults()
160 # For the Table version of CalibrateTract, use the associated
161 # Table version of the BuildStars task.
162 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask)
163 # For tract mode, we set a very high effective density cut.
164 self.fgcmBuildStars.densityCutMaxPerPixel = 10000
167class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask):
168 """
169 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
170 input catalogs.
171 """
172 ConfigClass = FgcmCalibrateTractTableConfig
173 RunnerClass = FgcmCalibrateTractRunner
174 _DefaultName = "fgcmCalibrateTractTable"
176 canMultiprocess = False
178 def runQuantum(self, butlerQC, inputRefs, outputRefs):
179 dataRefDict = butlerQC.get(inputRefs)
181 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs'])))
183 # Run the build stars tasks
184 tract = butlerQC.quantum.dataId['tract']
186 calexpRefs = dataRefDict['calexp']
187 calexpRefDict = {(calexpRef.dataId.byName()['visit'],
188 calexpRef.dataId.byName()['detector']):
189 calexpRef for calexpRef in calexpRefs}
190 dataRefDict['calexps'] = calexpRefDict
192 # And the outputs
193 if self.config.fgcmOutputProducts.doZeropointOutput:
194 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
195 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
196 dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict
198 if self.config.fgcmOutputProducts.doAtmosphereOutput:
199 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
200 atmRef in outputRefs.fgcmTransmissionAtmosphere}
201 dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict
203 if self.config.fgcmBuildStars.doReferenceMatches:
204 refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader
205 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
206 for ref in inputRefs.refCat],
207 refCats=butlerQC.get(inputRefs.refCat),
208 config=refConfig,
209 log=self.log)
210 buildStarsRefObjLoader = loader
211 else:
212 buildStarsRefObjLoader = None
214 if self.config.fgcmOutputProducts.doReferenceCalibration:
215 refConfig = self.config.fgcmOutputProducts.refObjLoader
216 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
217 for ref in inputRefs.refCat],
218 refCats=butlerQC.get(inputRefs.refCat),
219 config=refConfig,
220 log=self.log)
221 self.fgcmOutputProducts.refObjLoader = loader
223 struct = self.run(dataRefDict, tract,
224 buildStarsRefObjLoader=buildStarsRefObjLoader, butler=butlerQC)
226 if struct.photoCalibCatalogs is not None:
227 self.log.info("Outputting photoCalib catalogs.")
228 for visit, expCatalog in struct.photoCalibCatalogs:
229 butlerQC.put(expCatalog, photoCalibRefDict[visit])
230 self.log.info("Done outputting photoCalib catalogs.")
232 if struct.atmospheres is not None:
233 self.log.info("Outputting atmosphere transmission files.")
234 for visit, atm in struct.atmospheres:
235 butlerQC.put(atm, atmRefDict[visit])
236 self.log.info("Done outputting atmosphere files.")
238 # Turn raw repeatability into simple catalog for persistence
239 schema = afwTable.Schema()
240 schema.addField('rawRepeatability', type=np.float64,
241 doc="Per-band raw repeatability in FGCM calibration.")
242 repeatabilityCat = afwTable.BaseCatalog(schema)
243 repeatabilityCat.resize(len(struct.repeatability))
244 repeatabilityCat['rawRepeatability'][:] = struct.repeatability
246 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
248 return
250 @classmethod
251 def _makeArgumentParser(cls):
252 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
253 parser.add_id_argument("--id", "sourceTable_visit",
254 help="Data ID, e.g. --id visit=6789 tract=9617",
255 ContainerClass=TractCheckDataIdContainer)
257 return parser