Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 16%
185 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-04 12:15 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-04 12:15 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
32import time
33import warnings
35import numpy as np
36import collections
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
52 defaultTemplates={}):
53 camera = connectionTypes.PrerequisiteInput(
54 doc="Camera instrument",
55 name="camera",
56 storageClass="Camera",
57 dimensions=("instrument",),
58 isCalibration=True,
59 )
61 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
62 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
63 "chromatic corrections."),
64 name="fgcmLookUpTable",
65 storageClass="Catalog",
66 dimensions=("instrument",),
67 deferLoad=True,
68 )
70 sourceSchema = connectionTypes.InitInput(
71 doc="Schema for source catalogs",
72 name="src_schema",
73 storageClass="SourceCatalog",
74 )
76 refCat = connectionTypes.PrerequisiteInput(
77 doc="Reference catalog to use for photometric calibration",
78 name="cal_ref_cat",
79 storageClass="SimpleCatalog",
80 dimensions=("skypix",),
81 deferLoad=True,
82 multiple=True,
83 )
85 sourceTable_visit = connectionTypes.Input(
86 doc="Source table in parquet format, per visit",
87 name="sourceTable_visit",
88 storageClass="DataFrame",
89 dimensions=("instrument", "visit"),
90 deferLoad=True,
91 multiple=True,
92 )
94 visitSummary = connectionTypes.Input(
95 doc=("Per-visit consolidated exposure metadata. These catalogs use "
96 "detector id for the id and must be sorted for fast lookups of a "
97 "detector."),
98 name="visitSummary",
99 storageClass="ExposureCatalog",
100 dimensions=("instrument", "visit"),
101 deferLoad=True,
102 multiple=True,
103 )
105 fgcmVisitCatalog = connectionTypes.Output(
106 doc="Catalog of visit information for fgcm",
107 name="fgcmVisitCatalog",
108 storageClass="Catalog",
109 dimensions=("instrument",),
110 )
112 fgcmStarObservations = connectionTypes.Output(
113 doc="Catalog of star observations for fgcm",
114 name="fgcmStarObservations",
115 storageClass="Catalog",
116 dimensions=("instrument",),
117 )
119 fgcmStarIds = connectionTypes.Output(
120 doc="Catalog of fgcm calibration star IDs",
121 name="fgcmStarIds",
122 storageClass="Catalog",
123 dimensions=("instrument",),
124 )
126 fgcmStarIndices = connectionTypes.Output(
127 doc="Catalog of fgcm calibration star indices",
128 name="fgcmStarIndices",
129 storageClass="Catalog",
130 dimensions=("instrument",),
131 )
133 fgcmReferenceStars = connectionTypes.Output(
134 doc="Catalog of fgcm-matched reference stars",
135 name="fgcmReferenceStars",
136 storageClass="Catalog",
137 dimensions=("instrument",),
138 )
140 def __init__(self, *, config=None):
141 super().__init__(config=config)
143 if not config.doReferenceMatches:
144 self.prerequisiteInputs.remove("refCat")
145 self.prerequisiteInputs.remove("fgcmLookUpTable")
147 if not config.doReferenceMatches:
148 self.outputs.remove("fgcmReferenceStars")
150 def getSpatialBoundsConnections(self):
151 return ("visitSummary",)
154class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
155 pipelineConnections=FgcmBuildStarsTableConnections):
156 """Config for FgcmBuildStarsTableTask"""
158 referenceCCD = pexConfig.Field(
159 doc="Reference CCD for checking PSF and background",
160 dtype=int,
161 default=40,
162 )
164 def setDefaults(self):
165 super().setDefaults()
167 # The names here correspond to the post-transformed
168 # sourceTable_visit catalogs, which differ from the raw src
169 # catalogs. Therefore, all field and flag names cannot
170 # be derived from the base config class.
171 self.instFluxField = 'apFlux_12_0_instFlux'
172 self.localBackgroundFluxField = 'localBackground_instFlux'
173 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux'
174 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux'
175 self.psfCandidateName = 'calib_psf_candidate'
177 sourceSelector = self.sourceSelector["science"]
179 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
181 sourceSelector.flags.bad = ['pixelFlags_edge',
182 'pixelFlags_interpolatedCenter',
183 'pixelFlags_saturatedCenter',
184 'pixelFlags_crCenter',
185 'pixelFlags_bad',
186 'pixelFlags_interpolated',
187 'pixelFlags_saturated',
188 'centroid_flag',
189 fluxFlagName]
191 if self.doSubtractLocalBackground:
192 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
193 sourceSelector.flags.bad.append(localBackgroundFlagName)
195 sourceSelector.signalToNoise.fluxField = self.instFluxField
196 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
198 sourceSelector.isolated.parentName = 'parentSourceId'
199 sourceSelector.isolated.nChildName = 'deblend_nChild'
201 sourceSelector.requireFiniteRaDec.raColName = 'ra'
202 sourceSelector.requireFiniteRaDec.decColName = 'dec'
204 sourceSelector.unresolved.name = 'extendedness'
206 sourceSelector.doRequirePrimary = True
209class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
210 """
211 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
212 """
213 ConfigClass = FgcmBuildStarsTableConfig
214 _DefaultName = "fgcmBuildStarsTable"
216 canMultiprocess = False
218 def __init__(self, initInputs=None, **kwargs):
219 super().__init__(initInputs=initInputs, **kwargs)
220 if initInputs is not None:
221 self.sourceSchema = initInputs["sourceSchema"].schema
223 def runQuantum(self, butlerQC, inputRefs, outputRefs):
224 inputRefDict = butlerQC.get(inputRefs)
226 sourceTableHandles = inputRefDict['sourceTable_visit']
228 self.log.info("Running with %d sourceTable_visit handles",
229 len(sourceTableHandles))
231 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
232 sourceTableHandle in sourceTableHandles}
234 if self.config.doReferenceMatches:
235 # Get the LUT handle
236 lutHandle = inputRefDict['fgcmLookUpTable']
238 # Prepare the reference catalog loader
239 refConfig = LoadReferenceObjectsConfig()
240 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
241 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
242 for ref in inputRefs.refCat],
243 refCats=butlerQC.get(inputRefs.refCat),
244 name=self.config.connections.refCat,
245 log=self.log,
246 config=refConfig)
247 self.makeSubtask('fgcmLoadReferenceCatalog',
248 refObjLoader=refObjLoader,
249 refCatName=self.config.connections.refCat)
250 else:
251 lutHandle = None
253 # Compute aperture radius if necessary. This is useful to do now before
254 # any heave lifting has happened (fail early).
255 calibFluxApertureRadius = None
256 if self.config.doSubtractLocalBackground:
257 try:
258 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
259 except RuntimeError as e:
260 raise RuntimeError("Could not determine aperture radius from %s. "
261 "Cannot use doSubtractLocalBackground." %
262 (self.config.instFluxField)) from e
264 visitSummaryHandles = inputRefDict['visitSummary']
265 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
266 visitSummaryHandle in visitSummaryHandles}
268 camera = inputRefDict['camera']
269 groupedHandles = self._groupHandles(sourceTableHandleDict,
270 visitSummaryHandleDict)
272 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
274 rad = calibFluxApertureRadius
275 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles,
276 visitCat,
277 self.sourceSchema,
278 camera,
279 calibFluxApertureRadius=rad)
281 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
282 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
284 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
285 fgcmStarObservationCat,
286 lutHandle=lutHandle)
288 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
289 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
290 if fgcmRefCat is not None:
291 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
293 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
294 """Group sourceTable and visitSummary handles.
296 Parameters
297 ----------
298 sourceTableHandleDict : `dict` [`int`, `str`]
299 Dict of source tables, keyed by visit.
300 visitSummaryHandleDict : `dict` [int, `str`]
301 Dict of visit summary catalogs, keyed by visit.
303 Returns
304 -------
305 groupedHandles : `dict` [`int`, `list`]
306 Dictionary with sorted visit keys, and `list`s with
307 `lsst.daf.butler.DeferredDataSetHandle`. The first
308 item in the list will be the visitSummary ref, and
309 the second will be the source table ref.
310 """
311 groupedHandles = collections.defaultdict(list)
312 visits = sorted(sourceTableHandleDict.keys())
314 for visit in visits:
315 groupedHandles[visit] = [visitSummaryHandleDict[visit],
316 sourceTableHandleDict[visit]]
318 return groupedHandles
320 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
321 sourceSchema,
322 camera,
323 calibFluxApertureRadius=None):
324 startTime = time.time()
326 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
327 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
329 # To get the correct output schema, we use the legacy code.
330 # We are not actually using this mapper, except to grab the outputSchema
331 sourceMapper = self._makeSourceMapper(sourceSchema)
332 outputSchema = sourceMapper.getOutputSchema()
334 # Construct mapping from ccd number to index
335 ccdMapping = {}
336 for ccdIndex, detector in enumerate(camera):
337 ccdMapping[detector.getId()] = ccdIndex
339 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
341 fullCatalog = afwTable.BaseCatalog(outputSchema)
343 visitKey = outputSchema['visit'].asKey()
344 ccdKey = outputSchema['ccd'].asKey()
345 instMagKey = outputSchema['instMag'].asKey()
346 instMagErrKey = outputSchema['instMagErr'].asKey()
347 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
349 # Prepare local background if desired
350 if self.config.doSubtractLocalBackground:
351 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
353 columns = None
355 k = 2.5/np.log(10.)
357 for counter, visit in enumerate(visitCat):
358 expTime = visit['exptime']
360 handle = groupedHandles[visit['visit']][-1]
362 if columns is None:
363 inColumns = handle.get(component='columns')
364 columns = self._get_sourceTable_visit_columns(inColumns)
365 df = handle.get(parameters={'columns': columns})
367 goodSrc = self.sourceSelector.selectSources(df)
369 # Need to add a selection based on the local background correction
370 # if necessary
371 if self.config.doSubtractLocalBackground:
372 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
373 use, = np.where((goodSrc.selected)
374 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
375 else:
376 use, = np.where(goodSrc.selected)
378 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
379 tempCat.resize(use.size)
381 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
382 tempCat['dec'][:] = np.deg2rad(df['dec'].values[use])
383 tempCat['x'][:] = df['x'].values[use]
384 tempCat['y'][:] = df['y'].values[use]
385 # The "visit" name in the parquet table is hard-coded.
386 tempCat[visitKey][:] = df['visit'].values[use]
387 tempCat[ccdKey][:] = df['detector'].values[use]
388 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
390 with warnings.catch_warnings():
391 # Ignore warnings, we will filter infinites and nans below
392 warnings.simplefilter("ignore")
394 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
395 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
396 / df[self.config.apertureInnerInstFluxField].values[use])
397 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
398 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
399 / df[self.config.apertureOuterInstFluxField].values[use])
400 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
401 # Set bad values to illegal values for fgcm.
402 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
404 if self.config.doSubtractLocalBackground:
405 # At the moment we only adjust the flux and not the flux
406 # error by the background because the error on
407 # base_LocalBackground_instFlux is the rms error in the
408 # background annulus, not the error on the mean in the
409 # background estimate (which is much smaller, by sqrt(n)
410 # pixels used to estimate the background, which we do not
411 # have access to in this task). In the default settings,
412 # the annulus is sufficiently large such that these
413 # additional errors are are negligibly small (much less
414 # than a mmag in quadrature).
416 # This is the difference between the mag with local background correction
417 # and the mag without local background correction.
418 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
419 - localBackground[use]) -
420 -2.5*np.log10(df[self.config.instFluxField].values[use]))
421 else:
422 tempCat['deltaMagBkg'][:] = 0.0
424 # Need to loop over ccds here
425 for detector in camera:
426 ccdId = detector.getId()
427 # used index for all observations with a given ccd
428 use2 = (tempCat[ccdKey] == ccdId)
429 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
430 tempCat['y'][use2])
431 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
432 * visit['scaling'][ccdMapping[ccdId]])
433 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
435 # Compute instMagErr from instFluxErr/instFlux, any scaling
436 # will cancel out.
437 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
438 / df[self.config.instFluxField].values[use])
440 # Apply the jacobian if configured
441 if self.config.doApplyWcsJacobian:
442 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
444 fullCatalog.extend(tempCat)
446 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
447 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
449 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
450 visit['sources_read'] = True
452 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
453 use.size, visit['visit'], visit['deltaAper'])
455 self.log.info("Found all good star observations in %.2f s" %
456 (time.time() - startTime))
458 return fullCatalog
460 def _get_sourceTable_visit_columns(self, inColumns):
461 """
462 Get the sourceTable_visit columns from the config.
464 Parameters
465 ----------
466 inColumns : `list`
467 List of columns available in the sourceTable_visit
469 Returns
470 -------
471 columns : `list`
472 List of columns to read from sourceTable_visit.
473 """
474 # Some names are hard-coded in the parquet table.
475 columns = ['visit', 'detector',
476 'ra', 'dec', 'x', 'y', self.config.psfCandidateName,
477 self.config.instFluxField, self.config.instFluxField + 'Err',
478 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
479 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
480 if self.sourceSelector.config.doFlags:
481 columns.extend(self.sourceSelector.config.flags.bad)
482 if self.sourceSelector.config.doUnresolved:
483 columns.append(self.sourceSelector.config.unresolved.name)
484 if self.sourceSelector.config.doIsolated:
485 columns.append(self.sourceSelector.config.isolated.parentName)
486 columns.append(self.sourceSelector.config.isolated.nChildName)
487 if self.sourceSelector.config.doRequirePrimary:
488 columns.append(self.sourceSelector.config.requirePrimary.primaryColName)
489 if self.config.doSubtractLocalBackground:
490 columns.append(self.config.localBackgroundFluxField)
492 return columns