Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 16%
187 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-04 02:45 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-04 02:45 -0800
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
32import time
34import numpy as np
35import collections
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
45from .utilities import lookupStaticCalibrations
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
52 defaultTemplates={}):
53 camera = connectionTypes.PrerequisiteInput(
54 doc="Camera instrument",
55 name="camera",
56 storageClass="Camera",
57 dimensions=("instrument",),
58 lookupFunction=lookupStaticCalibrations,
59 isCalibration=True,
60 )
62 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
64 "chromatic corrections."),
65 name="fgcmLookUpTable",
66 storageClass="Catalog",
67 dimensions=("instrument",),
68 deferLoad=True,
69 )
71 sourceSchema = connectionTypes.InitInput(
72 doc="Schema for source catalogs",
73 name="src_schema",
74 storageClass="SourceCatalog",
75 )
77 refCat = connectionTypes.PrerequisiteInput(
78 doc="Reference catalog to use for photometric calibration",
79 name="cal_ref_cat",
80 storageClass="SimpleCatalog",
81 dimensions=("skypix",),
82 deferLoad=True,
83 multiple=True,
84 )
86 sourceTable_visit = connectionTypes.Input(
87 doc="Source table in parquet format, per visit",
88 name="sourceTable_visit",
89 storageClass="DataFrame",
90 dimensions=("instrument", "visit"),
91 deferLoad=True,
92 multiple=True,
93 )
95 visitSummary = connectionTypes.Input(
96 doc=("Per-visit consolidated exposure metadata. These catalogs use "
97 "detector id for the id and must be sorted for fast lookups of a "
98 "detector."),
99 name="visitSummary",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit"),
102 deferLoad=True,
103 multiple=True,
104 )
106 background = connectionTypes.Input(
107 doc="Calexp background model",
108 name="calexpBackground",
109 storageClass="Background",
110 dimensions=("instrument", "visit", "detector"),
111 deferLoad=True,
112 multiple=True,
113 )
115 fgcmVisitCatalog = connectionTypes.Output(
116 doc="Catalog of visit information for fgcm",
117 name="fgcmVisitCatalog",
118 storageClass="Catalog",
119 dimensions=("instrument",),
120 )
122 fgcmStarObservations = connectionTypes.Output(
123 doc="Catalog of star observations for fgcm",
124 name="fgcmStarObservations",
125 storageClass="Catalog",
126 dimensions=("instrument",),
127 )
129 fgcmStarIds = connectionTypes.Output(
130 doc="Catalog of fgcm calibration star IDs",
131 name="fgcmStarIds",
132 storageClass="Catalog",
133 dimensions=("instrument",),
134 )
136 fgcmStarIndices = connectionTypes.Output(
137 doc="Catalog of fgcm calibration star indices",
138 name="fgcmStarIndices",
139 storageClass="Catalog",
140 dimensions=("instrument",),
141 )
143 fgcmReferenceStars = connectionTypes.Output(
144 doc="Catalog of fgcm-matched reference stars",
145 name="fgcmReferenceStars",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 )
150 def __init__(self, *, config=None):
151 super().__init__(config=config)
153 if not config.doReferenceMatches:
154 self.prerequisiteInputs.remove("refCat")
155 self.prerequisiteInputs.remove("fgcmLookUpTable")
157 if not config.doModelErrorsWithBackground:
158 self.inputs.remove("background")
160 if not config.doReferenceMatches:
161 self.outputs.remove("fgcmReferenceStars")
164class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
165 pipelineConnections=FgcmBuildStarsTableConnections):
166 """Config for FgcmBuildStarsTableTask"""
168 referenceCCD = pexConfig.Field(
169 doc="Reference CCD for checking PSF and background",
170 dtype=int,
171 default=40,
172 )
174 def setDefaults(self):
175 super().setDefaults()
177 # The names here correspond to the post-transformed
178 # sourceTable_visit catalogs, which differ from the raw src
179 # catalogs. Therefore, all field and flag names cannot
180 # be derived from the base config class.
181 self.instFluxField = 'apFlux_12_0_instFlux'
182 self.localBackgroundFluxField = 'localBackground_instFlux'
183 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux'
184 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux'
185 self.psfCandidateName = 'calib_psf_candidate'
187 sourceSelector = self.sourceSelector["science"]
189 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
191 sourceSelector.flags.bad = ['pixelFlags_edge',
192 'pixelFlags_interpolatedCenter',
193 'pixelFlags_saturatedCenter',
194 'pixelFlags_crCenter',
195 'pixelFlags_bad',
196 'pixelFlags_interpolated',
197 'pixelFlags_saturated',
198 'centroid_flag',
199 fluxFlagName]
201 if self.doSubtractLocalBackground:
202 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
203 sourceSelector.flags.bad.append(localBackgroundFlagName)
205 sourceSelector.signalToNoise.fluxField = self.instFluxField
206 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
208 sourceSelector.isolated.parentName = 'parentSourceId'
209 sourceSelector.isolated.nChildName = 'deblend_nChild'
211 sourceSelector.requireFiniteRaDec.raColName = 'ra'
212 sourceSelector.requireFiniteRaDec.decColName = 'decl'
214 sourceSelector.unresolved.name = 'extendedness'
217class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
218 """
219 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
220 """
221 ConfigClass = FgcmBuildStarsTableConfig
222 _DefaultName = "fgcmBuildStarsTable"
224 canMultiprocess = False
226 def __init__(self, initInputs=None, **kwargs):
227 super().__init__(initInputs=initInputs, **kwargs)
228 if initInputs is not None:
229 self.sourceSchema = initInputs["sourceSchema"].schema
231 def runQuantum(self, butlerQC, inputRefs, outputRefs):
232 inputRefDict = butlerQC.get(inputRefs)
234 sourceTableHandles = inputRefDict['sourceTable_visit']
236 self.log.info("Running with %d sourceTable_visit handles",
237 len(sourceTableHandles))
239 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
240 sourceTableHandle in sourceTableHandles}
242 if self.config.doReferenceMatches:
243 # Get the LUT handle
244 lutHandle = inputRefDict['fgcmLookUpTable']
246 # Prepare the reference catalog loader
247 refConfig = LoadReferenceObjectsConfig()
248 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
249 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
250 for ref in inputRefs.refCat],
251 refCats=butlerQC.get(inputRefs.refCat),
252 name=self.config.connections.refCat,
253 log=self.log,
254 config=refConfig)
255 self.makeSubtask('fgcmLoadReferenceCatalog',
256 refObjLoader=refObjLoader,
257 refCatName=self.config.connections.refCat)
258 else:
259 lutHandle = None
261 # Compute aperture radius if necessary. This is useful to do now before
262 # any heave lifting has happened (fail early).
263 calibFluxApertureRadius = None
264 if self.config.doSubtractLocalBackground:
265 try:
266 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
267 except RuntimeError as e:
268 raise RuntimeError("Could not determine aperture radius from %s. "
269 "Cannot use doSubtractLocalBackground." %
270 (self.config.instFluxField)) from e
272 visitSummaryHandles = inputRefDict['visitSummary']
273 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
274 visitSummaryHandle in visitSummaryHandles}
276 camera = inputRefDict['camera']
277 groupedHandles = self._groupHandles(sourceTableHandleDict,
278 visitSummaryHandleDict)
280 if self.config.doModelErrorsWithBackground:
281 bkgHandles = inputRefDict['background']
282 bkgHandleDict = {(bkgHandle.dataId.byName()['visit'],
283 bkgHandle.dataId.byName()['detector']): bkgHandle for
284 bkgHandle in bkgHandles}
285 else:
286 bkgHandleDict = None
288 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles,
289 bkgHandleDict=bkgHandleDict)
291 rad = calibFluxApertureRadius
292 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles,
293 visitCat,
294 self.sourceSchema,
295 camera,
296 calibFluxApertureRadius=rad)
298 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
299 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
301 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
302 fgcmStarObservationCat,
303 lutHandle=lutHandle)
305 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
306 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
307 if fgcmRefCat is not None:
308 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
310 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
311 """Group sourceTable and visitSummary handles.
313 Parameters
314 ----------
315 sourceTableHandleDict : `dict` [`int`, `str`]
316 Dict of source tables, keyed by visit.
317 visitSummaryHandleDict : `dict` [int, `str`]
318 Dict of visit summary catalogs, keyed by visit.
320 Returns
321 -------
322 groupedHandles : `dict` [`int`, `list`]
323 Dictionary with sorted visit keys, and `list`s with
324 `lsst.daf.butler.DeferredDataSetHandle`. The first
325 item in the list will be the visitSummary ref, and
326 the second will be the source table ref.
327 """
328 groupedHandles = collections.defaultdict(list)
329 visits = sorted(sourceTableHandleDict.keys())
331 for visit in visits:
332 groupedHandles[visit] = [visitSummaryHandleDict[visit],
333 sourceTableHandleDict[visit]]
335 return groupedHandles
337 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
338 sourceSchema,
339 camera,
340 calibFluxApertureRadius=None):
341 startTime = time.time()
343 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
344 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
346 # To get the correct output schema, we use the legacy code.
347 # We are not actually using this mapper, except to grab the outputSchema
348 sourceMapper = self._makeSourceMapper(sourceSchema)
349 outputSchema = sourceMapper.getOutputSchema()
351 # Construct mapping from ccd number to index
352 ccdMapping = {}
353 for ccdIndex, detector in enumerate(camera):
354 ccdMapping[detector.getId()] = ccdIndex
356 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
358 fullCatalog = afwTable.BaseCatalog(outputSchema)
360 visitKey = outputSchema['visit'].asKey()
361 ccdKey = outputSchema['ccd'].asKey()
362 instMagKey = outputSchema['instMag'].asKey()
363 instMagErrKey = outputSchema['instMagErr'].asKey()
364 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
366 # Prepare local background if desired
367 if self.config.doSubtractLocalBackground:
368 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
370 columns = None
372 k = 2.5/np.log(10.)
374 for counter, visit in enumerate(visitCat):
375 expTime = visit['exptime']
377 handle = groupedHandles[visit['visit']][-1]
379 if columns is None:
380 inColumns = handle.get(component='columns')
381 columns = self._get_sourceTable_visit_columns(inColumns)
382 df = handle.get(parameters={'columns': columns})
384 goodSrc = self.sourceSelector.selectSources(df)
386 # Need to add a selection based on the local background correction
387 # if necessary
388 if self.config.doSubtractLocalBackground:
389 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
390 use, = np.where((goodSrc.selected)
391 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
392 else:
393 use, = np.where(goodSrc.selected)
395 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
396 tempCat.resize(use.size)
398 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
399 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
400 tempCat['x'][:] = df['x'].values[use]
401 tempCat['y'][:] = df['y'].values[use]
402 # The "visit" name in the parquet table is hard-coded.
403 tempCat[visitKey][:] = df['visit'].values[use]
404 tempCat[ccdKey][:] = df['detector'].values[use]
405 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
407 with np.warnings.catch_warnings():
408 # Ignore warnings, we will filter infinites and nans below
409 np.warnings.simplefilter("ignore")
411 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
412 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
413 / df[self.config.apertureInnerInstFluxField].values[use])
414 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
415 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
416 / df[self.config.apertureOuterInstFluxField].values[use])
417 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
418 # Set bad values to illegal values for fgcm.
419 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
421 if self.config.doSubtractLocalBackground:
422 # At the moment we only adjust the flux and not the flux
423 # error by the background because the error on
424 # base_LocalBackground_instFlux is the rms error in the
425 # background annulus, not the error on the mean in the
426 # background estimate (which is much smaller, by sqrt(n)
427 # pixels used to estimate the background, which we do not
428 # have access to in this task). In the default settings,
429 # the annulus is sufficiently large such that these
430 # additional errors are are negligibly small (much less
431 # than a mmag in quadrature).
433 # This is the difference between the mag with local background correction
434 # and the mag without local background correction.
435 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
436 - localBackground[use]) -
437 -2.5*np.log10(df[self.config.instFluxField].values[use]))
438 else:
439 tempCat['deltaMagBkg'][:] = 0.0
441 # Need to loop over ccds here
442 for detector in camera:
443 ccdId = detector.getId()
444 # used index for all observations with a given ccd
445 use2 = (tempCat[ccdKey] == ccdId)
446 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
447 tempCat['y'][use2])
448 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
449 * visit['scaling'][ccdMapping[ccdId]])
450 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
452 # Compute instMagErr from instFluxErr/instFlux, any scaling
453 # will cancel out.
454 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
455 / df[self.config.instFluxField].values[use])
457 # Apply the jacobian if configured
458 if self.config.doApplyWcsJacobian:
459 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
461 fullCatalog.extend(tempCat)
463 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
464 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
466 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
467 visit['sources_read'] = True
469 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
470 use.size, visit['visit'], visit['deltaAper'])
472 self.log.info("Found all good star observations in %.2f s" %
473 (time.time() - startTime))
475 return fullCatalog
477 def _get_sourceTable_visit_columns(self, inColumns):
478 """
479 Get the sourceTable_visit columns from the config.
481 Parameters
482 ----------
483 inColumns : `list`
484 List of columns available in the sourceTable_visit
486 Returns
487 -------
488 columns : `list`
489 List of columns to read from sourceTable_visit.
490 """
491 # Some names are hard-coded in the parquet table.
492 columns = ['visit', 'detector',
493 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
494 self.config.instFluxField, self.config.instFluxField + 'Err',
495 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
496 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
497 if self.sourceSelector.config.doFlags:
498 columns.extend(self.sourceSelector.config.flags.bad)
499 if self.sourceSelector.config.doUnresolved:
500 columns.append(self.sourceSelector.config.unresolved.name)
501 if self.sourceSelector.config.doIsolated:
502 columns.append(self.sourceSelector.config.isolated.parentName)
503 columns.append(self.sourceSelector.config.isolated.nChildName)
504 if self.config.doSubtractLocalBackground:
505 columns.append(self.config.localBackgroundFluxField)
507 return columns