Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%
181 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-11 12:08 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-11 12:08 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
32import time
33import warnings
35import numpy as np
36import collections
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
46from .utilities import lookupStaticCalibrations
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
53 defaultTemplates={}):
54 camera = connectionTypes.PrerequisiteInput(
55 doc="Camera instrument",
56 name="camera",
57 storageClass="Camera",
58 dimensions=("instrument",),
59 lookupFunction=lookupStaticCalibrations,
60 isCalibration=True,
61 )
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name="fgcmLookUpTable",
67 storageClass="Catalog",
68 dimensions=("instrument",),
69 deferLoad=True,
70 )
72 sourceSchema = connectionTypes.InitInput(
73 doc="Schema for source catalogs",
74 name="src_schema",
75 storageClass="SourceCatalog",
76 )
78 refCat = connectionTypes.PrerequisiteInput(
79 doc="Reference catalog to use for photometric calibration",
80 name="cal_ref_cat",
81 storageClass="SimpleCatalog",
82 dimensions=("skypix",),
83 deferLoad=True,
84 multiple=True,
85 )
87 sourceTable_visit = connectionTypes.Input(
88 doc="Source table in parquet format, per visit",
89 name="sourceTable_visit",
90 storageClass="DataFrame",
91 dimensions=("instrument", "visit"),
92 deferLoad=True,
93 multiple=True,
94 )
96 visitSummary = connectionTypes.Input(
97 doc=("Per-visit consolidated exposure metadata. These catalogs use "
98 "detector id for the id and must be sorted for fast lookups of a "
99 "detector."),
100 name="visitSummary",
101 storageClass="ExposureCatalog",
102 dimensions=("instrument", "visit"),
103 deferLoad=True,
104 multiple=True,
105 )
107 fgcmVisitCatalog = connectionTypes.Output(
108 doc="Catalog of visit information for fgcm",
109 name="fgcmVisitCatalog",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 )
114 fgcmStarObservations = connectionTypes.Output(
115 doc="Catalog of star observations for fgcm",
116 name="fgcmStarObservations",
117 storageClass="Catalog",
118 dimensions=("instrument",),
119 )
121 fgcmStarIds = connectionTypes.Output(
122 doc="Catalog of fgcm calibration star IDs",
123 name="fgcmStarIds",
124 storageClass="Catalog",
125 dimensions=("instrument",),
126 )
128 fgcmStarIndices = connectionTypes.Output(
129 doc="Catalog of fgcm calibration star indices",
130 name="fgcmStarIndices",
131 storageClass="Catalog",
132 dimensions=("instrument",),
133 )
135 fgcmReferenceStars = connectionTypes.Output(
136 doc="Catalog of fgcm-matched reference stars",
137 name="fgcmReferenceStars",
138 storageClass="Catalog",
139 dimensions=("instrument",),
140 )
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
145 if not config.doReferenceMatches:
146 self.prerequisiteInputs.remove("refCat")
147 self.prerequisiteInputs.remove("fgcmLookUpTable")
149 if not config.doReferenceMatches:
150 self.outputs.remove("fgcmReferenceStars")
153class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
154 pipelineConnections=FgcmBuildStarsTableConnections):
155 """Config for FgcmBuildStarsTableTask"""
157 referenceCCD = pexConfig.Field(
158 doc="Reference CCD for checking PSF and background",
159 dtype=int,
160 default=40,
161 )
163 def setDefaults(self):
164 super().setDefaults()
166 # The names here correspond to the post-transformed
167 # sourceTable_visit catalogs, which differ from the raw src
168 # catalogs. Therefore, all field and flag names cannot
169 # be derived from the base config class.
170 self.instFluxField = 'apFlux_12_0_instFlux'
171 self.localBackgroundFluxField = 'localBackground_instFlux'
172 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux'
173 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux'
174 self.psfCandidateName = 'calib_psf_candidate'
176 sourceSelector = self.sourceSelector["science"]
178 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
180 sourceSelector.flags.bad = ['pixelFlags_edge',
181 'pixelFlags_interpolatedCenter',
182 'pixelFlags_saturatedCenter',
183 'pixelFlags_crCenter',
184 'pixelFlags_bad',
185 'pixelFlags_interpolated',
186 'pixelFlags_saturated',
187 'centroid_flag',
188 fluxFlagName]
190 if self.doSubtractLocalBackground:
191 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
192 sourceSelector.flags.bad.append(localBackgroundFlagName)
194 sourceSelector.signalToNoise.fluxField = self.instFluxField
195 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
197 sourceSelector.isolated.parentName = 'parentSourceId'
198 sourceSelector.isolated.nChildName = 'deblend_nChild'
200 sourceSelector.requireFiniteRaDec.raColName = 'ra'
201 sourceSelector.requireFiniteRaDec.decColName = 'decl'
203 sourceSelector.unresolved.name = 'extendedness'
206class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
207 """
208 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
209 """
210 ConfigClass = FgcmBuildStarsTableConfig
211 _DefaultName = "fgcmBuildStarsTable"
213 canMultiprocess = False
215 def __init__(self, initInputs=None, **kwargs):
216 super().__init__(initInputs=initInputs, **kwargs)
217 if initInputs is not None:
218 self.sourceSchema = initInputs["sourceSchema"].schema
220 def runQuantum(self, butlerQC, inputRefs, outputRefs):
221 inputRefDict = butlerQC.get(inputRefs)
223 sourceTableHandles = inputRefDict['sourceTable_visit']
225 self.log.info("Running with %d sourceTable_visit handles",
226 len(sourceTableHandles))
228 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
229 sourceTableHandle in sourceTableHandles}
231 if self.config.doReferenceMatches:
232 # Get the LUT handle
233 lutHandle = inputRefDict['fgcmLookUpTable']
235 # Prepare the reference catalog loader
236 refConfig = LoadReferenceObjectsConfig()
237 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
238 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
239 for ref in inputRefs.refCat],
240 refCats=butlerQC.get(inputRefs.refCat),
241 name=self.config.connections.refCat,
242 log=self.log,
243 config=refConfig)
244 self.makeSubtask('fgcmLoadReferenceCatalog',
245 refObjLoader=refObjLoader,
246 refCatName=self.config.connections.refCat)
247 else:
248 lutHandle = None
250 # Compute aperture radius if necessary. This is useful to do now before
251 # any heave lifting has happened (fail early).
252 calibFluxApertureRadius = None
253 if self.config.doSubtractLocalBackground:
254 try:
255 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
256 except RuntimeError as e:
257 raise RuntimeError("Could not determine aperture radius from %s. "
258 "Cannot use doSubtractLocalBackground." %
259 (self.config.instFluxField)) from e
261 visitSummaryHandles = inputRefDict['visitSummary']
262 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
263 visitSummaryHandle in visitSummaryHandles}
265 camera = inputRefDict['camera']
266 groupedHandles = self._groupHandles(sourceTableHandleDict,
267 visitSummaryHandleDict)
269 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
271 rad = calibFluxApertureRadius
272 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles,
273 visitCat,
274 self.sourceSchema,
275 camera,
276 calibFluxApertureRadius=rad)
278 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
279 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
281 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
282 fgcmStarObservationCat,
283 lutHandle=lutHandle)
285 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
286 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
287 if fgcmRefCat is not None:
288 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
290 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
291 """Group sourceTable and visitSummary handles.
293 Parameters
294 ----------
295 sourceTableHandleDict : `dict` [`int`, `str`]
296 Dict of source tables, keyed by visit.
297 visitSummaryHandleDict : `dict` [int, `str`]
298 Dict of visit summary catalogs, keyed by visit.
300 Returns
301 -------
302 groupedHandles : `dict` [`int`, `list`]
303 Dictionary with sorted visit keys, and `list`s with
304 `lsst.daf.butler.DeferredDataSetHandle`. The first
305 item in the list will be the visitSummary ref, and
306 the second will be the source table ref.
307 """
308 groupedHandles = collections.defaultdict(list)
309 visits = sorted(sourceTableHandleDict.keys())
311 for visit in visits:
312 groupedHandles[visit] = [visitSummaryHandleDict[visit],
313 sourceTableHandleDict[visit]]
315 return groupedHandles
317 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
318 sourceSchema,
319 camera,
320 calibFluxApertureRadius=None):
321 startTime = time.time()
323 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
324 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
326 # To get the correct output schema, we use the legacy code.
327 # We are not actually using this mapper, except to grab the outputSchema
328 sourceMapper = self._makeSourceMapper(sourceSchema)
329 outputSchema = sourceMapper.getOutputSchema()
331 # Construct mapping from ccd number to index
332 ccdMapping = {}
333 for ccdIndex, detector in enumerate(camera):
334 ccdMapping[detector.getId()] = ccdIndex
336 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
338 fullCatalog = afwTable.BaseCatalog(outputSchema)
340 visitKey = outputSchema['visit'].asKey()
341 ccdKey = outputSchema['ccd'].asKey()
342 instMagKey = outputSchema['instMag'].asKey()
343 instMagErrKey = outputSchema['instMagErr'].asKey()
344 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
346 # Prepare local background if desired
347 if self.config.doSubtractLocalBackground:
348 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
350 columns = None
352 k = 2.5/np.log(10.)
354 for counter, visit in enumerate(visitCat):
355 expTime = visit['exptime']
357 handle = groupedHandles[visit['visit']][-1]
359 if columns is None:
360 inColumns = handle.get(component='columns')
361 columns = self._get_sourceTable_visit_columns(inColumns)
362 df = handle.get(parameters={'columns': columns})
364 goodSrc = self.sourceSelector.selectSources(df)
366 # Need to add a selection based on the local background correction
367 # if necessary
368 if self.config.doSubtractLocalBackground:
369 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
370 use, = np.where((goodSrc.selected)
371 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
372 else:
373 use, = np.where(goodSrc.selected)
375 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
376 tempCat.resize(use.size)
378 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
379 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
380 tempCat['x'][:] = df['x'].values[use]
381 tempCat['y'][:] = df['y'].values[use]
382 # The "visit" name in the parquet table is hard-coded.
383 tempCat[visitKey][:] = df['visit'].values[use]
384 tempCat[ccdKey][:] = df['detector'].values[use]
385 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
387 with warnings.catch_warnings():
388 # Ignore warnings, we will filter infinites and nans below
389 warnings.simplefilter("ignore")
391 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
392 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
393 / df[self.config.apertureInnerInstFluxField].values[use])
394 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
395 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
396 / df[self.config.apertureOuterInstFluxField].values[use])
397 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
398 # Set bad values to illegal values for fgcm.
399 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
401 if self.config.doSubtractLocalBackground:
402 # At the moment we only adjust the flux and not the flux
403 # error by the background because the error on
404 # base_LocalBackground_instFlux is the rms error in the
405 # background annulus, not the error on the mean in the
406 # background estimate (which is much smaller, by sqrt(n)
407 # pixels used to estimate the background, which we do not
408 # have access to in this task). In the default settings,
409 # the annulus is sufficiently large such that these
410 # additional errors are are negligibly small (much less
411 # than a mmag in quadrature).
413 # This is the difference between the mag with local background correction
414 # and the mag without local background correction.
415 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
416 - localBackground[use]) -
417 -2.5*np.log10(df[self.config.instFluxField].values[use]))
418 else:
419 tempCat['deltaMagBkg'][:] = 0.0
421 # Need to loop over ccds here
422 for detector in camera:
423 ccdId = detector.getId()
424 # used index for all observations with a given ccd
425 use2 = (tempCat[ccdKey] == ccdId)
426 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
427 tempCat['y'][use2])
428 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
429 * visit['scaling'][ccdMapping[ccdId]])
430 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
432 # Compute instMagErr from instFluxErr/instFlux, any scaling
433 # will cancel out.
434 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
435 / df[self.config.instFluxField].values[use])
437 # Apply the jacobian if configured
438 if self.config.doApplyWcsJacobian:
439 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
441 fullCatalog.extend(tempCat)
443 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
444 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
446 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
447 visit['sources_read'] = True
449 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
450 use.size, visit['visit'], visit['deltaAper'])
452 self.log.info("Found all good star observations in %.2f s" %
453 (time.time() - startTime))
455 return fullCatalog
457 def _get_sourceTable_visit_columns(self, inColumns):
458 """
459 Get the sourceTable_visit columns from the config.
461 Parameters
462 ----------
463 inColumns : `list`
464 List of columns available in the sourceTable_visit
466 Returns
467 -------
468 columns : `list`
469 List of columns to read from sourceTable_visit.
470 """
471 # Some names are hard-coded in the parquet table.
472 columns = ['visit', 'detector',
473 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
474 self.config.instFluxField, self.config.instFluxField + 'Err',
475 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
476 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
477 if self.sourceSelector.config.doFlags:
478 columns.extend(self.sourceSelector.config.flags.bad)
479 if self.sourceSelector.config.doUnresolved:
480 columns.append(self.sourceSelector.config.unresolved.name)
481 if self.sourceSelector.config.doIsolated:
482 columns.append(self.sourceSelector.config.isolated.parentName)
483 columns.append(self.sourceSelector.config.isolated.nChildName)
484 if self.config.doSubtractLocalBackground:
485 columns.append(self.config.localBackgroundFluxField)
487 return columns