Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py: 24%
87 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-05 03:20 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-05 03:20 -0800
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Class for running fgcmcal on a single tract using sourceTable_visit tables.
24"""
25import numpy as np
27import lsst.pipe.base as pipeBase
28from lsst.pipe.base import connectionTypes
29from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
30import lsst.afw.table as afwTable
32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
33from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase,
34 FgcmCalibrateTractBaseTask)
35from .utilities import lookupStaticCalibrations
37__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
40class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
41 dimensions=("instrument",
42 "tract",)):
43 camera = connectionTypes.PrerequisiteInput(
44 doc="Camera instrument",
45 name="camera",
46 storageClass="Camera",
47 dimensions=("instrument",),
48 lookupFunction=lookupStaticCalibrations,
49 isCalibration=True,
50 )
52 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
53 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
54 "chromatic corrections."),
55 name="fgcmLookUpTable",
56 storageClass="Catalog",
57 dimensions=("instrument",),
58 deferLoad=True,
59 )
61 sourceSchema = connectionTypes.InitInput(
62 doc="Schema for source catalogs",
63 name="src_schema",
64 storageClass="SourceCatalog",
65 )
67 refCat = connectionTypes.PrerequisiteInput(
68 doc="Reference catalog to use for photometric calibration",
69 name="cal_ref_cat",
70 storageClass="SimpleCatalog",
71 dimensions=("skypix",),
72 deferLoad=True,
73 multiple=True,
74 )
76 source_catalogs = connectionTypes.Input(
77 doc="Source table in parquet format, per visit",
78 name="sourceTable_visit",
79 storageClass="DataFrame",
80 dimensions=("instrument", "visit"),
81 deferLoad=True,
82 multiple=True,
83 )
85 visitSummary = connectionTypes.Input(
86 doc="Per-visit summary statistics table",
87 name="visitSummary",
88 storageClass="ExposureCatalog",
89 dimensions=("instrument", "visit"),
90 deferLoad=True,
91 multiple=True,
92 )
94 background = connectionTypes.Input(
95 doc="Calexp background model",
96 name="calexpBackground",
97 storageClass="Background",
98 dimensions=("instrument", "visit", "detector"),
99 deferLoad=True,
100 multiple=True,
101 )
103 fgcmPhotoCalib = connectionTypes.Output(
104 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
105 name="fgcmPhotoCalibTractCatalog",
106 storageClass="ExposureCatalog",
107 dimensions=("instrument", "tract", "visit",),
108 multiple=True,
109 )
111 fgcmTransmissionAtmosphere = connectionTypes.Output(
112 doc="Per-visit atmosphere transmission files produced from fgcm calibration",
113 name="transmission_atmosphere_fgcm_tract",
114 storageClass="TransmissionCurve",
115 dimensions=("instrument", "tract", "visit",),
116 multiple=True,
117 )
119 fgcmRepeatability = connectionTypes.Output(
120 doc="Per-band raw repeatability numbers in the fgcm tract calibration",
121 name="fgcmRawRepeatability",
122 storageClass="Catalog",
123 dimensions=("instrument", "tract",),
124 multiple=False,
125 )
127 def __init__(self, *, config=None):
128 super().__init__(config=config)
130 if not config.fgcmBuildStars.doModelErrorsWithBackground:
131 self.inputs.remove("background")
133 if not config.fgcmOutputProducts.doAtmosphereOutput:
134 self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
135 if not config.fgcmOutputProducts.doZeropointOutput:
136 self.prerequisiteInputs.remove("fgcmZeropoints")
139class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig,
140 pipelineConnections=FgcmCalibrateTractTableConnections):
141 """Config for FgcmCalibrateTractTable task"""
142 def setDefaults(self):
143 super().setDefaults()
145 # For the Table version of CalibrateTract, use the associated
146 # Table version of the BuildStars task.
147 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask)
148 # For tract mode, we set a very high effective density cut.
149 self.fgcmBuildStars.densityCutMaxPerPixel = 10000
152class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask):
153 """
154 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
155 input catalogs.
156 """
157 ConfigClass = FgcmCalibrateTractTableConfig
158 _DefaultName = "fgcmCalibrateTractTable"
160 canMultiprocess = False
162 def __init__(self, initInputs=None, **kwargs):
163 super().__init__(initInputs=initInputs, **kwargs)
164 if initInputs is not None:
165 self.sourceSchema = initInputs["sourceSchema"].schema
167 def runQuantum(self, butlerQC, inputRefs, outputRefs):
168 handleDict = butlerQC.get(inputRefs)
170 self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs'])))
172 # Run the build stars tasks
173 tract = butlerQC.quantum.dataId['tract']
175 handleDict['sourceSchema'] = self.sourceSchema
177 sourceTableHandles = handleDict['source_catalogs']
178 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
179 sourceTableHandle in sourceTableHandles}
181 visitSummaryHandles = handleDict['visitSummary']
182 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
183 visitSummaryHandle in visitSummaryHandles}
185 handleDict['sourceTableHandleDict'] = sourceTableHandleDict
186 handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict
188 # And the outputs
189 if self.config.fgcmOutputProducts.doZeropointOutput:
190 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
191 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
192 handleDict['fgcmPhotoCalibs'] = photoCalibRefDict
194 if self.config.fgcmOutputProducts.doAtmosphereOutput:
195 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
196 atmRef in outputRefs.fgcmTransmissionAtmosphere}
197 handleDict['fgcmTransmissionAtmospheres'] = atmRefDict
199 if self.config.fgcmBuildStars.doReferenceMatches:
200 refConfig = LoadReferenceObjectsConfig()
201 refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap
202 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
203 for ref in inputRefs.refCat],
204 refCats=butlerQC.get(inputRefs.refCat),
205 name=self.config.connections.refCat,
206 config=refConfig,
207 log=self.log)
208 buildStarsRefObjLoader = loader
209 else:
210 buildStarsRefObjLoader = None
212 if self.config.fgcmOutputProducts.doReferenceCalibration:
213 refConfig = self.config.fgcmOutputProducts.refObjLoader
214 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
215 for ref in inputRefs.refCat],
216 refCats=butlerQC.get(inputRefs.refCat),
217 name=self.config.connections.refCat,
218 config=refConfig,
219 log=self.log)
220 self.fgcmOutputProducts.refObjLoader = loader
222 struct = self.run(handleDict, tract,
223 buildStarsRefObjLoader=buildStarsRefObjLoader)
225 if struct.photoCalibCatalogs is not None:
226 self.log.info("Outputting photoCalib catalogs.")
227 for visit, expCatalog in struct.photoCalibCatalogs:
228 butlerQC.put(expCatalog, photoCalibRefDict[visit])
229 self.log.info("Done outputting photoCalib catalogs.")
231 if struct.atmospheres is not None:
232 self.log.info("Outputting atmosphere transmission files.")
233 for visit, atm in struct.atmospheres:
234 butlerQC.put(atm, atmRefDict[visit])
235 self.log.info("Done outputting atmosphere files.")
237 # Turn raw repeatability into simple catalog for persistence
238 schema = afwTable.Schema()
239 schema.addField('rawRepeatability', type=np.float64,
240 doc="Per-band raw repeatability in FGCM calibration.")
241 repeatabilityCat = afwTable.BaseCatalog(schema)
242 repeatabilityCat.resize(len(struct.repeatability))
243 repeatabilityCat['rawRepeatability'][:] = struct.repeatability
245 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
247 return