Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%
184 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-01 11:56 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-01 11:56 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
32import time
33import warnings
35import numpy as np
36import collections
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
46from .utilities import lookupStaticCalibrations
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
53 defaultTemplates={}):
54 camera = connectionTypes.PrerequisiteInput(
55 doc="Camera instrument",
56 name="camera",
57 storageClass="Camera",
58 dimensions=("instrument",),
59 lookupFunction=lookupStaticCalibrations,
60 isCalibration=True,
61 )
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name="fgcmLookUpTable",
67 storageClass="Catalog",
68 dimensions=("instrument",),
69 deferLoad=True,
70 )
72 sourceSchema = connectionTypes.InitInput(
73 doc="Schema for source catalogs",
74 name="src_schema",
75 storageClass="SourceCatalog",
76 )
78 refCat = connectionTypes.PrerequisiteInput(
79 doc="Reference catalog to use for photometric calibration",
80 name="cal_ref_cat",
81 storageClass="SimpleCatalog",
82 dimensions=("skypix",),
83 deferLoad=True,
84 multiple=True,
85 )
87 sourceTable_visit = connectionTypes.Input(
88 doc="Source table in parquet format, per visit",
89 name="sourceTable_visit",
90 storageClass="DataFrame",
91 dimensions=("instrument", "visit"),
92 deferLoad=True,
93 multiple=True,
94 )
96 visitSummary = connectionTypes.Input(
97 doc=("Per-visit consolidated exposure metadata. These catalogs use "
98 "detector id for the id and must be sorted for fast lookups of a "
99 "detector."),
100 name="visitSummary",
101 storageClass="ExposureCatalog",
102 dimensions=("instrument", "visit"),
103 deferLoad=True,
104 multiple=True,
105 )
107 fgcmVisitCatalog = connectionTypes.Output(
108 doc="Catalog of visit information for fgcm",
109 name="fgcmVisitCatalog",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 )
114 fgcmStarObservations = connectionTypes.Output(
115 doc="Catalog of star observations for fgcm",
116 name="fgcmStarObservations",
117 storageClass="Catalog",
118 dimensions=("instrument",),
119 )
121 fgcmStarIds = connectionTypes.Output(
122 doc="Catalog of fgcm calibration star IDs",
123 name="fgcmStarIds",
124 storageClass="Catalog",
125 dimensions=("instrument",),
126 )
128 fgcmStarIndices = connectionTypes.Output(
129 doc="Catalog of fgcm calibration star indices",
130 name="fgcmStarIndices",
131 storageClass="Catalog",
132 dimensions=("instrument",),
133 )
135 fgcmReferenceStars = connectionTypes.Output(
136 doc="Catalog of fgcm-matched reference stars",
137 name="fgcmReferenceStars",
138 storageClass="Catalog",
139 dimensions=("instrument",),
140 )
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
145 if not config.doReferenceMatches:
146 self.prerequisiteInputs.remove("refCat")
147 self.prerequisiteInputs.remove("fgcmLookUpTable")
149 if not config.doReferenceMatches:
150 self.outputs.remove("fgcmReferenceStars")
153class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
154 pipelineConnections=FgcmBuildStarsTableConnections):
155 """Config for FgcmBuildStarsTableTask"""
157 referenceCCD = pexConfig.Field(
158 doc="Reference CCD for checking PSF and background",
159 dtype=int,
160 default=40,
161 )
163 def setDefaults(self):
164 super().setDefaults()
166 # The names here correspond to the post-transformed
167 # sourceTable_visit catalogs, which differ from the raw src
168 # catalogs. Therefore, all field and flag names cannot
169 # be derived from the base config class.
170 self.instFluxField = 'apFlux_12_0_instFlux'
171 self.localBackgroundFluxField = 'localBackground_instFlux'
172 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux'
173 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux'
174 self.psfCandidateName = 'calib_psf_candidate'
176 sourceSelector = self.sourceSelector["science"]
178 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
180 sourceSelector.flags.bad = ['pixelFlags_edge',
181 'pixelFlags_interpolatedCenter',
182 'pixelFlags_saturatedCenter',
183 'pixelFlags_crCenter',
184 'pixelFlags_bad',
185 'pixelFlags_interpolated',
186 'pixelFlags_saturated',
187 'centroid_flag',
188 fluxFlagName]
190 if self.doSubtractLocalBackground:
191 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
192 sourceSelector.flags.bad.append(localBackgroundFlagName)
194 sourceSelector.signalToNoise.fluxField = self.instFluxField
195 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
197 sourceSelector.isolated.parentName = 'parentSourceId'
198 sourceSelector.isolated.nChildName = 'deblend_nChild'
200 sourceSelector.requireFiniteRaDec.raColName = 'ra'
201 sourceSelector.requireFiniteRaDec.decColName = 'decl'
203 sourceSelector.unresolved.name = 'extendedness'
205 sourceSelector.doRequirePrimary = True
208class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
209 """
210 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
211 """
212 ConfigClass = FgcmBuildStarsTableConfig
213 _DefaultName = "fgcmBuildStarsTable"
215 canMultiprocess = False
217 def __init__(self, initInputs=None, **kwargs):
218 super().__init__(initInputs=initInputs, **kwargs)
219 if initInputs is not None:
220 self.sourceSchema = initInputs["sourceSchema"].schema
222 def runQuantum(self, butlerQC, inputRefs, outputRefs):
223 inputRefDict = butlerQC.get(inputRefs)
225 sourceTableHandles = inputRefDict['sourceTable_visit']
227 self.log.info("Running with %d sourceTable_visit handles",
228 len(sourceTableHandles))
230 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
231 sourceTableHandle in sourceTableHandles}
233 if self.config.doReferenceMatches:
234 # Get the LUT handle
235 lutHandle = inputRefDict['fgcmLookUpTable']
237 # Prepare the reference catalog loader
238 refConfig = LoadReferenceObjectsConfig()
239 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
240 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
241 for ref in inputRefs.refCat],
242 refCats=butlerQC.get(inputRefs.refCat),
243 name=self.config.connections.refCat,
244 log=self.log,
245 config=refConfig)
246 self.makeSubtask('fgcmLoadReferenceCatalog',
247 refObjLoader=refObjLoader,
248 refCatName=self.config.connections.refCat)
249 else:
250 lutHandle = None
252 # Compute aperture radius if necessary. This is useful to do now before
253 # any heave lifting has happened (fail early).
254 calibFluxApertureRadius = None
255 if self.config.doSubtractLocalBackground:
256 try:
257 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
258 except RuntimeError as e:
259 raise RuntimeError("Could not determine aperture radius from %s. "
260 "Cannot use doSubtractLocalBackground." %
261 (self.config.instFluxField)) from e
263 visitSummaryHandles = inputRefDict['visitSummary']
264 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
265 visitSummaryHandle in visitSummaryHandles}
267 camera = inputRefDict['camera']
268 groupedHandles = self._groupHandles(sourceTableHandleDict,
269 visitSummaryHandleDict)
271 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
273 rad = calibFluxApertureRadius
274 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles,
275 visitCat,
276 self.sourceSchema,
277 camera,
278 calibFluxApertureRadius=rad)
280 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
281 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
283 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
284 fgcmStarObservationCat,
285 lutHandle=lutHandle)
287 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
288 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
289 if fgcmRefCat is not None:
290 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
292 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
293 """Group sourceTable and visitSummary handles.
295 Parameters
296 ----------
297 sourceTableHandleDict : `dict` [`int`, `str`]
298 Dict of source tables, keyed by visit.
299 visitSummaryHandleDict : `dict` [int, `str`]
300 Dict of visit summary catalogs, keyed by visit.
302 Returns
303 -------
304 groupedHandles : `dict` [`int`, `list`]
305 Dictionary with sorted visit keys, and `list`s with
306 `lsst.daf.butler.DeferredDataSetHandle`. The first
307 item in the list will be the visitSummary ref, and
308 the second will be the source table ref.
309 """
310 groupedHandles = collections.defaultdict(list)
311 visits = sorted(sourceTableHandleDict.keys())
313 for visit in visits:
314 groupedHandles[visit] = [visitSummaryHandleDict[visit],
315 sourceTableHandleDict[visit]]
317 return groupedHandles
319 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
320 sourceSchema,
321 camera,
322 calibFluxApertureRadius=None):
323 startTime = time.time()
325 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
326 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
328 # To get the correct output schema, we use the legacy code.
329 # We are not actually using this mapper, except to grab the outputSchema
330 sourceMapper = self._makeSourceMapper(sourceSchema)
331 outputSchema = sourceMapper.getOutputSchema()
333 # Construct mapping from ccd number to index
334 ccdMapping = {}
335 for ccdIndex, detector in enumerate(camera):
336 ccdMapping[detector.getId()] = ccdIndex
338 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
340 fullCatalog = afwTable.BaseCatalog(outputSchema)
342 visitKey = outputSchema['visit'].asKey()
343 ccdKey = outputSchema['ccd'].asKey()
344 instMagKey = outputSchema['instMag'].asKey()
345 instMagErrKey = outputSchema['instMagErr'].asKey()
346 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
348 # Prepare local background if desired
349 if self.config.doSubtractLocalBackground:
350 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
352 columns = None
354 k = 2.5/np.log(10.)
356 for counter, visit in enumerate(visitCat):
357 expTime = visit['exptime']
359 handle = groupedHandles[visit['visit']][-1]
361 if columns is None:
362 inColumns = handle.get(component='columns')
363 columns = self._get_sourceTable_visit_columns(inColumns)
364 df = handle.get(parameters={'columns': columns})
366 goodSrc = self.sourceSelector.selectSources(df)
368 # Need to add a selection based on the local background correction
369 # if necessary
370 if self.config.doSubtractLocalBackground:
371 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
372 use, = np.where((goodSrc.selected)
373 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
374 else:
375 use, = np.where(goodSrc.selected)
377 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
378 tempCat.resize(use.size)
380 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
381 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
382 tempCat['x'][:] = df['x'].values[use]
383 tempCat['y'][:] = df['y'].values[use]
384 # The "visit" name in the parquet table is hard-coded.
385 tempCat[visitKey][:] = df['visit'].values[use]
386 tempCat[ccdKey][:] = df['detector'].values[use]
387 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
389 with warnings.catch_warnings():
390 # Ignore warnings, we will filter infinites and nans below
391 warnings.simplefilter("ignore")
393 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
394 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
395 / df[self.config.apertureInnerInstFluxField].values[use])
396 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
397 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
398 / df[self.config.apertureOuterInstFluxField].values[use])
399 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
400 # Set bad values to illegal values for fgcm.
401 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
403 if self.config.doSubtractLocalBackground:
404 # At the moment we only adjust the flux and not the flux
405 # error by the background because the error on
406 # base_LocalBackground_instFlux is the rms error in the
407 # background annulus, not the error on the mean in the
408 # background estimate (which is much smaller, by sqrt(n)
409 # pixels used to estimate the background, which we do not
410 # have access to in this task). In the default settings,
411 # the annulus is sufficiently large such that these
412 # additional errors are are negligibly small (much less
413 # than a mmag in quadrature).
415 # This is the difference between the mag with local background correction
416 # and the mag without local background correction.
417 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
418 - localBackground[use]) -
419 -2.5*np.log10(df[self.config.instFluxField].values[use]))
420 else:
421 tempCat['deltaMagBkg'][:] = 0.0
423 # Need to loop over ccds here
424 for detector in camera:
425 ccdId = detector.getId()
426 # used index for all observations with a given ccd
427 use2 = (tempCat[ccdKey] == ccdId)
428 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
429 tempCat['y'][use2])
430 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
431 * visit['scaling'][ccdMapping[ccdId]])
432 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
434 # Compute instMagErr from instFluxErr/instFlux, any scaling
435 # will cancel out.
436 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
437 / df[self.config.instFluxField].values[use])
439 # Apply the jacobian if configured
440 if self.config.doApplyWcsJacobian:
441 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
443 fullCatalog.extend(tempCat)
445 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
446 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
448 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
449 visit['sources_read'] = True
451 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
452 use.size, visit['visit'], visit['deltaAper'])
454 self.log.info("Found all good star observations in %.2f s" %
455 (time.time() - startTime))
457 return fullCatalog
459 def _get_sourceTable_visit_columns(self, inColumns):
460 """
461 Get the sourceTable_visit columns from the config.
463 Parameters
464 ----------
465 inColumns : `list`
466 List of columns available in the sourceTable_visit
468 Returns
469 -------
470 columns : `list`
471 List of columns to read from sourceTable_visit.
472 """
473 # Some names are hard-coded in the parquet table.
474 columns = ['visit', 'detector',
475 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
476 self.config.instFluxField, self.config.instFluxField + 'Err',
477 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
478 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
479 if self.sourceSelector.config.doFlags:
480 columns.extend(self.sourceSelector.config.flags.bad)
481 if self.sourceSelector.config.doUnresolved:
482 columns.append(self.sourceSelector.config.unresolved.name)
483 if self.sourceSelector.config.doIsolated:
484 columns.append(self.sourceSelector.config.isolated.parentName)
485 columns.append(self.sourceSelector.config.isolated.nChildName)
486 if self.sourceSelector.config.doRequirePrimary:
487 columns.append(self.sourceSelector.config.requirePrimary.primaryColName)
488 if self.config.doSubtractLocalBackground:
489 columns.append(self.config.localBackgroundFluxField)
491 return columns