lsst.fgcmcal  21.0.0-6-g5ef7dad+4799c432d5
fgcmCalibrateTractTable.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Class for running fgcmcal on a single tract using sourceTable_visit tables.
24 """
25 import numpy as np
26 
27 import lsst.pipe.base as pipeBase
28 from lsst.pipe.base import connectionTypes
29 from lsst.meas.algorithms import ReferenceObjectLoader
30 import lsst.afw.table as afwTable
31 
32 from .dataIds import TractCheckDataIdContainer
33 from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
34 from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner,
35  FgcmCalibrateTractBaseTask)
36 from .utilities import lookupStaticCalibrations
37 
38 __all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask']
39 
40 
41 class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
42  dimensions=("instrument",
43  "tract",)):
44  camera = connectionTypes.PrerequisiteInput(
45  doc="Camera instrument",
46  name="camera",
47  storageClass="Camera",
48  dimensions=("instrument",),
49  lookupFunction=lookupStaticCalibrations,
50  isCalibration=True,
51  )
52 
53  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
54  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
55  "chromatic corrections."),
56  name="fgcmLookUpTable",
57  storageClass="Catalog",
58  dimensions=("instrument",),
59  deferLoad=True,
60  )
61 
62  sourceSchema = connectionTypes.PrerequisiteInput(
63  doc="Schema for source catalogs",
64  name="src_schema",
65  storageClass="SourceCatalog",
66  deferLoad=True,
67  )
68 
69  refCat = connectionTypes.PrerequisiteInput(
70  doc="Reference catalog to use for photometric calibration",
71  name="cal_ref_cat",
72  storageClass="SimpleCatalog",
73  dimensions=("skypix",),
74  deferLoad=True,
75  multiple=True,
76  )
77 
78  source_catalogs = connectionTypes.Input(
79  doc="Source table in parquet format, per visit",
80  name="sourceTable_visit",
81  storageClass="DataFrame",
82  dimensions=("instrument", "visit"),
83  deferLoad=True,
84  multiple=True,
85  )
86 
87  calexp = connectionTypes.Input(
88  doc="Calibrated exposures used for psf and metadata",
89  name="calexp",
90  storageClass="ExposureF",
91  dimensions=("instrument", "visit", "detector"),
92  deferLoad=True,
93  multiple=True,
94  )
95 
96  background = connectionTypes.Input(
97  doc="Calexp background model",
98  name="calexpBackground",
99  storageClass="Background",
100  dimensions=("instrument", "visit", "detector"),
101  deferLoad=True,
102  multiple=True,
103  )
104 
105  fgcmPhotoCalib = connectionTypes.Output(
106  doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration",
107  name="fgcmPhotoCalibTractCatalog",
108  storageClass="ExposureCatalog",
109  dimensions=("instrument", "tract", "visit",),
110  multiple=True,
111  )
112 
113  fgcmTransmissionAtmosphere = connectionTypes.Output(
114  doc="Per-visit atmosphere transmission files produced from fgcm calibration",
115  name="transmission_atmosphere_fgcm_tract",
116  storageClass="TransmissionCurve",
117  dimensions=("instrument", "tract", "visit",),
118  multiple=True,
119  )
120 
121  fgcmRepeatability = connectionTypes.Output(
122  doc="Per-band raw repeatability numbers in the fgcm tract calibration",
123  name="fgcmRawRepeatability",
124  storageClass="Catalog",
125  dimensions=("instrument", "tract",),
126  multiple=False,
127  )
128 
129  def __init__(self, *, config=None):
130  super().__init__(config=config)
131 
132  # The ref_dataset_name will be deprecated with Gen2
133  loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name
134  if config.connections.refCat != loaderName:
135  raise ValueError("connections.refCat must be the same as "
136  "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name")
137  if config.fgcmOutputProducts.doReferenceCalibration:
138  loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name
139  if config.connections.refCat != loaderName:
140  raise ValueError("connections.refCat must be the same as "
141  "config.fgcmOutputProducts.refObjLoader.ref_dataset_name")
142 
143  if not config.fgcmBuildStars.doModelErrorsWithBackground:
144  self.inputs.remove("background")
145 
146  if config.fgcmOutputProducts.doRefcatOutput:
147  raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput")
148  if not config.fgcmOutputProducts.doAtmosphereOutput:
149  self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
150  if not config.fgcmOutputProducts.doZeropointOutput:
151  self.prerequisiteInputs.remove("fgcmZeropoints")
152 
153 
155  pipelineConnections=FgcmCalibrateTractTableConnections):
156  """Config for FgcmCalibrateTractTable task"""
157  def setDefaults(self):
158  super().setDefaults()
159 
160  # For the Table version of CalibrateTract, use the associated
161  # Table version of the BuildStars task.
162  self.fgcmBuildStarsfgcmBuildStars.retarget(FgcmBuildStarsTableTask)
163  # For tract mode, we set a very high effective density cut.
164  self.fgcmBuildStarsfgcmBuildStars.densityCutMaxPerPixel = 10000
165 
166 
168  """
169  Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet)
170  input catalogs.
171  """
172  ConfigClass = FgcmCalibrateTractTableConfig
173  RunnerClass = FgcmCalibrateTractRunner
174  _DefaultName = "fgcmCalibrateTractTable"
175 
176  canMultiprocess = False
177 
178  def runQuantum(self, butlerQC, inputRefs, outputRefs):
179  dataRefDict = butlerQC.get(inputRefs)
180 
181  self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs'])))
182 
183  # Run the build stars tasks
184  tract = butlerQC.quantum.dataId['tract']
185 
186  calexpRefs = dataRefDict['calexp']
187  calexpRefDict = {(calexpRef.dataId.byName()['visit'],
188  calexpRef.dataId.byName()['detector']):
189  calexpRef for calexpRef in calexpRefs}
190  dataRefDict['calexps'] = calexpRefDict
191 
192  # And the outputs
193  if self.config.fgcmOutputProducts.doZeropointOutput:
194  photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
195  photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
196  dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict
197 
198  if self.config.fgcmOutputProducts.doAtmosphereOutput:
199  atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
200  atmRef in outputRefs.fgcmTransmissionAtmosphere}
201  dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict
202 
203  if self.config.fgcmBuildStars.doReferenceMatches:
204  refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader
205  loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
206  for ref in inputRefs.refCat],
207  refCats=butlerQC.get(inputRefs.refCat),
208  config=refConfig,
209  log=self.log)
210  buildStarsRefObjLoader = loader
211  else:
212  buildStarsRefObjLoader = None
213 
214  if self.config.fgcmOutputProducts.doReferenceCalibration:
215  refConfig = self.config.fgcmOutputProducts.refObjLoader
216  loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
217  for ref in inputRefs.refCat],
218  refCats=butlerQC.get(inputRefs.refCat),
219  config=refConfig,
220  log=self.log)
221  self.fgcmOutputProducts.refObjLoader = loader
222 
223  struct = self.runrun(dataRefDict, tract,
224  buildStarsRefObjLoader=buildStarsRefObjLoader, butler=butlerQC)
225 
226  if struct.photoCalibCatalogs is not None:
227  self.log.info("Outputting photoCalib catalogs.")
228  for visit, expCatalog in struct.photoCalibCatalogs:
229  butlerQC.put(expCatalog, photoCalibRefDict[visit])
230  self.log.info("Done outputting photoCalib catalogs.")
231 
232  if struct.atmospheres is not None:
233  self.log.info("Outputting atmosphere transmission files.")
234  for visit, atm in struct.atmospheres:
235  butlerQC.put(atm, atmRefDict[visit])
236  self.log.info("Done outputting atmosphere files.")
237 
238  # Turn raw repeatability into simple catalog for persistence
239  schema = afwTable.Schema()
240  schema.addField('rawRepeatability', type=np.float64,
241  doc="Per-band raw repeatability in FGCM calibration.")
242  repeatabilityCat = afwTable.BaseCatalog(schema)
243  repeatabilityCat.resize(len(struct.repeatability))
244  repeatabilityCat['rawRepeatability'][:] = struct.repeatability
245 
246  butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
247 
248  return
249 
250  @classmethod
251  def _makeArgumentParser(cls):
252  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
253  parser.add_id_argument("--id", "sourceTable_visit",
254  help="Data ID, e.g. --id visit=6789 tract=9617",
255  ContainerClass=TractCheckDataIdContainer)
256 
257  return parser
def run(self, dataRefDict, tract, buildStarsRefObjLoader=None, returnCatalogs=True, butler=None)