Coverage for python/lsst/fgcmcal/fgcmCalibrateTractBase.py: 17%
153 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 03:56 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 03:56 -0700
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for running fgcmcal on a single tract using src tables
22or sourceTable_visit tables.
23"""
25import abc
27import numpy as np
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask
33from .fgcmFitCycle import FgcmFitCycleConfig
34from .fgcmOutputProducts import FgcmOutputProductsTask
35from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
36from .utilities import computeApertureRadiusFromName, extractReferenceMags
37from .utilities import makeZptSchema, makeZptCat
38from .utilities import makeAtmSchema, makeAtmCat
39from .utilities import makeStdSchema, makeStdCat
40from .focalPlaneProjector import FocalPlaneProjector
42import fgcm
44__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask']
47class FgcmCalibrateTractConfigBase(pexConfig.Config):
48 """Config for FgcmCalibrateTract"""
50 fgcmBuildStars = pexConfig.ConfigurableField(
51 target=FgcmBuildStarsTableTask,
52 doc="Task to load and match stars for fgcm",
53 )
54 fgcmFitCycle = pexConfig.ConfigField(
55 dtype=FgcmFitCycleConfig,
56 doc="Config to run a single fgcm fit cycle",
57 )
58 fgcmOutputProducts = pexConfig.ConfigurableField(
59 target=FgcmOutputProductsTask,
60 doc="Task to output fgcm products",
61 )
62 convergenceTolerance = pexConfig.Field(
63 doc="Tolerance on repeatability convergence (per band)",
64 dtype=float,
65 default=0.005,
66 )
67 maxFitCycles = pexConfig.Field(
68 doc="Maximum number of fit cycles",
69 dtype=int,
70 default=5,
71 )
72 doDebuggingPlots = pexConfig.Field(
73 doc="Make plots for debugging purposes?",
74 dtype=bool,
75 default=False,
76 )
78 def setDefaults(self):
79 pexConfig.Config.setDefaults(self)
81 self.fgcmFitCycle.quietMode = True
82 self.fgcmFitCycle.doPlots = False
83 self.fgcmOutputProducts.doReferenceCalibration = False
84 self.fgcmOutputProducts.cycleNumber = 0
85 self.fgcmOutputProducts.photoCal.applyColorTerms = False
87 def validate(self):
88 super().validate()
90 for band in self.fgcmFitCycle.bands:
91 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
92 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
94 self, msg)
97class FgcmCalibrateTractBaseTask(pipeBase.PipelineTask, abc.ABC):
98 """Base class to calibrate a single tract using fgcmcal
99 """
100 def __init__(self, initInputs=None, **kwargs):
101 super().__init__(**kwargs)
102 self.makeSubtask("fgcmBuildStars", initInputs=initInputs)
103 self.makeSubtask("fgcmOutputProducts")
105 def run(self, handleDict, tract,
106 buildStarsRefObjLoader=None, returnCatalogs=True):
107 """Run the calibrations for a single tract with fgcm.
109 Parameters
110 ----------
111 handleDict : `dict`
112 All handles are `lsst.daf.butler.DeferredDatasetHandle`
113 handle dictionary with the following keys. Note that all
114 keys need not be set based on config parameters.
116 ``"camera"``
117 Camera object (`lsst.afw.cameraGeom.Camera`)
118 ``"source_catalogs"``
119 `list` of handles for input source catalogs.
120 ``"sourceSchema"``
121 Schema for the source catalogs.
122 ``"fgcmLookUpTable"``
123 handle for the FGCM look-up table.
124 ``"calexps"``
125 `list` of handles for the input calexps
126 ``"fgcmPhotoCalibs"``
127 `dict` of output photoCalib handles. Key is
128 (tract, visit, detector).
129 Present if doZeropointOutput is True.
130 ``"fgcmTransmissionAtmospheres"``
131 `dict` of output atmosphere transmission handles.
132 Key is (tract, visit).
133 Present if doAtmosphereOutput is True.
134 tract : `int`
135 Tract number
136 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
137 Reference object loader object for fgcmBuildStars.
138 returnCatalogs : `bool`, optional
139 Return photoCalibs as per-visit exposure catalogs.
141 Returns
142 -------
143 outstruct : `lsst.pipe.base.Struct`
144 Output structure with keys:
146 offsets : `np.ndarray`
147 Final reference offsets, per band.
148 repeatability : `np.ndarray`
149 Raw fgcm repeatability for bright stars, per band.
150 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
151 Generator that returns (visit, transmissionCurve) tuples.
152 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)]
153 Generator that returns (visit, ccd, filtername, photoCalib) tuples.
154 (returned if returnCatalogs is False).
155 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
156 Generator that returns (visit, exposureCatalog) tuples.
157 (returned if returnCatalogs is True).
158 """
159 self.log.info("Running on tract %d", (tract))
161 # Compute the aperture radius if necessary. This is useful to do now before
162 # any heavy lifting has happened (fail early).
163 calibFluxApertureRadius = None
164 if self.config.fgcmBuildStars.doSubtractLocalBackground:
165 try:
166 field = self.config.fgcmBuildStars.instFluxField
167 calibFluxApertureRadius = computeApertureRadiusFromName(field)
168 except RuntimeError:
169 raise RuntimeError("Could not determine aperture radius from %s. "
170 "Cannot use doSubtractLocalBackground." %
171 (field))
173 # Run the build stars tasks
175 # Note that we will need visitCat at the end of the procedure for the outputs
176 groupedHandles = self.fgcmBuildStars._groupHandles(handleDict['sourceTableHandleDict'],
177 handleDict['visitSummaryHandleDict'])
178 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(handleDict['camera'], groupedHandles)
179 rad = calibFluxApertureRadius
180 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedHandles,
181 visitCat,
182 handleDict['sourceSchema'],
183 handleDict['camera'],
184 calibFluxApertureRadius=rad)
186 if self.fgcmBuildStars.config.doReferenceMatches:
187 lutHandle = handleDict['fgcmLookUpTable']
188 self.fgcmBuildStars.makeSubtask("fgcmLoadReferenceCatalog",
189 refObjLoader=buildStarsRefObjLoader,
190 refCatName=self.fgcmBuildStars.config.connections.refCat)
191 else:
192 lutHandle = None
194 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
195 self.fgcmBuildStars.fgcmMatchStars(visitCat,
196 fgcmStarObservationCat,
197 lutHandle=lutHandle)
199 # Load the LUT
200 lutCat = handleDict['fgcmLookUpTable'].get()
201 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
202 dict(self.config.fgcmFitCycle.physicalFilterMap))
203 del lutCat
205 # Translate the visit catalog into fgcm format
206 fgcmExpInfo = translateVisitCatalog(visitCat)
208 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, handleDict['camera'],
209 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
210 True, False, lutIndexVals[0]['FILTERNAMES'],
211 tract=tract)
213 focalPlaneProjector = FocalPlaneProjector(handleDict['camera'],
214 self.config.fgcmFitCycle.defaultCameraOrientation)
216 # Set up the fit cycle task
218 noFitsDict = {'lutIndex': lutIndexVals,
219 'lutStd': lutStd,
220 'expInfo': fgcmExpInfo,
221 'focalPlaneProjector': focalPlaneProjector}
223 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
224 noFitsDict=noFitsDict, noOutput=True)
226 # We determine the conversion from the native units (typically radians) to
227 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
228 # numpy arrays rather than Angles, which would we approximately 600x slower.
229 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra'])
231 # To load the stars, we need an initial parameter object
232 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
233 fgcmLut,
234 fgcmExpInfo)
236 # Match star observations to visits
237 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
238 # actually be transferred into fgcm using the indexing below.
240 obsIndex = fgcmStarIndicesCat['obsIndex']
241 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'],
242 fgcmStarObservationCat['visit'][obsIndex])
244 refMag, refMagErr = extractReferenceMags(fgcmRefCat,
245 self.config.fgcmFitCycle.bands,
246 self.config.fgcmFitCycle.physicalFilterMap)
247 refId = fgcmRefCat['fgcm_id'][:]
249 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
250 fgcmStars.loadStars(fgcmPars,
251 fgcmStarObservationCat['visit'][obsIndex],
252 fgcmStarObservationCat['ccd'][obsIndex],
253 fgcmStarObservationCat['ra'][obsIndex] * conv,
254 fgcmStarObservationCat['dec'][obsIndex] * conv,
255 fgcmStarObservationCat['instMag'][obsIndex],
256 fgcmStarObservationCat['instMagErr'][obsIndex],
257 fgcmExpInfo['FILTERNAME'][visitIndex],
258 fgcmStarIdCat['fgcm_id'][:],
259 fgcmStarIdCat['ra'][:],
260 fgcmStarIdCat['dec'][:],
261 fgcmStarIdCat['obsArrIndex'][:],
262 fgcmStarIdCat['nObs'][:],
263 obsX=fgcmStarObservationCat['x'][obsIndex],
264 obsY=fgcmStarObservationCat['y'][obsIndex],
265 obsDeltaMagBkg=fgcmStarObservationCat['deltaMagBkg'][obsIndex],
266 obsDeltaAper=fgcmStarObservationCat['deltaMagAper'][obsIndex],
267 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex],
268 refID=refId,
269 refMag=refMag,
270 refMagErr=refMagErr,
271 flagID=None,
272 flagFlag=None,
273 computeNobs=True)
275 # Clear out some memory
276 del fgcmStarIdCat
277 del fgcmStarIndicesCat
278 del fgcmRefCat
280 fgcmFitCycle.setLUT(fgcmLut)
281 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
283 converged = False
284 cycleNumber = 0
286 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
287 previousParInfo = None
288 previousParams = None
289 previousSuperStar = None
291 while (not converged and cycleNumber < self.config.maxFitCycles):
293 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
295 if cycleNumber > 0:
296 # Use parameters from previous cycle
297 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
298 fgcmExpInfo,
299 previousParInfo,
300 previousParams,
301 previousSuperStar)
302 # We need to reset the star magnitudes and errors for the next
303 # cycle
304 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
305 fgcmStarObservationCat['instMagErr'][obsIndex])
306 fgcmFitCycle.initialCycle = False
308 fgcmFitCycle.setPars(fgcmPars)
309 fgcmFitCycle.finishSetup()
311 fgcmFitCycle.run()
313 # Grab the parameters for the next cycle
314 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
315 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
317 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber))
318 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
319 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
320 continue
321 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
322 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
324 # Check for convergence
325 if np.alltrue((previousReservedRawRepeatability
326 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
327 < self.config.convergenceTolerance):
328 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
329 converged = True
330 else:
331 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
332 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
333 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False
334 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
335 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
336 self.log.info("Setting exposure gray photometricity cuts to:")
337 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
338 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
339 continue
340 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
341 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut))
343 cycleNumber += 1
345 # Log warning if not converged
346 if not converged:
347 self.log.warning("Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber)
349 # Do final clean-up iteration
350 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False
351 fgcmFitCycle.fgcmConfig.resetParameters = False
352 fgcmFitCycle.fgcmConfig.maxIter = 0
353 fgcmFitCycle.fgcmConfig.outputZeropoints = True
354 fgcmFitCycle.fgcmConfig.outputStandards = True
355 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
356 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
357 fgcmFitCycle.initialCycle = False
359 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
360 fgcmExpInfo,
361 previousParInfo,
362 previousParams,
363 previousSuperStar)
364 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex],
365 fgcmStarObservationCat['instMagErr'][obsIndex])
366 fgcmFitCycle.setPars(fgcmPars)
367 fgcmFitCycle.finishSetup()
369 self.log.info("Running final clean-up fit cycle...")
370 fgcmFitCycle.run()
372 self.log.info("Raw repeatability after clean-up cycle is:")
373 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands):
374 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
375 continue
376 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
377 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep))
379 # Do the outputs. Need to keep track of tract.
381 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
382 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
384 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
385 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
387 atmSchema = makeAtmSchema()
388 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
390 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
391 stdSchema = makeStdSchema(len(goodBands))
392 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
394 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(handleDict,
395 tract,
396 visitCat,
397 zptCat, atmCat, stdCat,
398 self.config.fgcmBuildStars)
400 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
402 fgcmFitCycle.freeSharedMemory()
404 return outStruct