Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%
180 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-30 04:02 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-30 04:02 -0700
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
32import time
34import numpy as np
35import collections
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
45from .utilities import lookupStaticCalibrations
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
52 defaultTemplates={}):
53 camera = connectionTypes.PrerequisiteInput(
54 doc="Camera instrument",
55 name="camera",
56 storageClass="Camera",
57 dimensions=("instrument",),
58 lookupFunction=lookupStaticCalibrations,
59 isCalibration=True,
60 )
62 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
64 "chromatic corrections."),
65 name="fgcmLookUpTable",
66 storageClass="Catalog",
67 dimensions=("instrument",),
68 deferLoad=True,
69 )
71 sourceSchema = connectionTypes.InitInput(
72 doc="Schema for source catalogs",
73 name="src_schema",
74 storageClass="SourceCatalog",
75 )
77 refCat = connectionTypes.PrerequisiteInput(
78 doc="Reference catalog to use for photometric calibration",
79 name="cal_ref_cat",
80 storageClass="SimpleCatalog",
81 dimensions=("skypix",),
82 deferLoad=True,
83 multiple=True,
84 )
86 sourceTable_visit = connectionTypes.Input(
87 doc="Source table in parquet format, per visit",
88 name="sourceTable_visit",
89 storageClass="DataFrame",
90 dimensions=("instrument", "visit"),
91 deferLoad=True,
92 multiple=True,
93 )
95 visitSummary = connectionTypes.Input(
96 doc=("Per-visit consolidated exposure metadata. These catalogs use "
97 "detector id for the id and must be sorted for fast lookups of a "
98 "detector."),
99 name="visitSummary",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit"),
102 deferLoad=True,
103 multiple=True,
104 )
106 fgcmVisitCatalog = connectionTypes.Output(
107 doc="Catalog of visit information for fgcm",
108 name="fgcmVisitCatalog",
109 storageClass="Catalog",
110 dimensions=("instrument",),
111 )
113 fgcmStarObservations = connectionTypes.Output(
114 doc="Catalog of star observations for fgcm",
115 name="fgcmStarObservations",
116 storageClass="Catalog",
117 dimensions=("instrument",),
118 )
120 fgcmStarIds = connectionTypes.Output(
121 doc="Catalog of fgcm calibration star IDs",
122 name="fgcmStarIds",
123 storageClass="Catalog",
124 dimensions=("instrument",),
125 )
127 fgcmStarIndices = connectionTypes.Output(
128 doc="Catalog of fgcm calibration star indices",
129 name="fgcmStarIndices",
130 storageClass="Catalog",
131 dimensions=("instrument",),
132 )
134 fgcmReferenceStars = connectionTypes.Output(
135 doc="Catalog of fgcm-matched reference stars",
136 name="fgcmReferenceStars",
137 storageClass="Catalog",
138 dimensions=("instrument",),
139 )
141 def __init__(self, *, config=None):
142 super().__init__(config=config)
144 if not config.doReferenceMatches:
145 self.prerequisiteInputs.remove("refCat")
146 self.prerequisiteInputs.remove("fgcmLookUpTable")
148 if not config.doReferenceMatches:
149 self.outputs.remove("fgcmReferenceStars")
152class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
153 pipelineConnections=FgcmBuildStarsTableConnections):
154 """Config for FgcmBuildStarsTableTask"""
156 referenceCCD = pexConfig.Field(
157 doc="Reference CCD for checking PSF and background",
158 dtype=int,
159 default=40,
160 )
162 def setDefaults(self):
163 super().setDefaults()
165 # The names here correspond to the post-transformed
166 # sourceTable_visit catalogs, which differ from the raw src
167 # catalogs. Therefore, all field and flag names cannot
168 # be derived from the base config class.
169 self.instFluxField = 'apFlux_12_0_instFlux'
170 self.localBackgroundFluxField = 'localBackground_instFlux'
171 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux'
172 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux'
173 self.psfCandidateName = 'calib_psf_candidate'
175 sourceSelector = self.sourceSelector["science"]
177 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
179 sourceSelector.flags.bad = ['pixelFlags_edge',
180 'pixelFlags_interpolatedCenter',
181 'pixelFlags_saturatedCenter',
182 'pixelFlags_crCenter',
183 'pixelFlags_bad',
184 'pixelFlags_interpolated',
185 'pixelFlags_saturated',
186 'centroid_flag',
187 fluxFlagName]
189 if self.doSubtractLocalBackground:
190 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
191 sourceSelector.flags.bad.append(localBackgroundFlagName)
193 sourceSelector.signalToNoise.fluxField = self.instFluxField
194 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
196 sourceSelector.isolated.parentName = 'parentSourceId'
197 sourceSelector.isolated.nChildName = 'deblend_nChild'
199 sourceSelector.requireFiniteRaDec.raColName = 'ra'
200 sourceSelector.requireFiniteRaDec.decColName = 'decl'
202 sourceSelector.unresolved.name = 'extendedness'
205class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
206 """
207 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
208 """
209 ConfigClass = FgcmBuildStarsTableConfig
210 _DefaultName = "fgcmBuildStarsTable"
212 canMultiprocess = False
214 def __init__(self, initInputs=None, **kwargs):
215 super().__init__(initInputs=initInputs, **kwargs)
216 if initInputs is not None:
217 self.sourceSchema = initInputs["sourceSchema"].schema
219 def runQuantum(self, butlerQC, inputRefs, outputRefs):
220 inputRefDict = butlerQC.get(inputRefs)
222 sourceTableHandles = inputRefDict['sourceTable_visit']
224 self.log.info("Running with %d sourceTable_visit handles",
225 len(sourceTableHandles))
227 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
228 sourceTableHandle in sourceTableHandles}
230 if self.config.doReferenceMatches:
231 # Get the LUT handle
232 lutHandle = inputRefDict['fgcmLookUpTable']
234 # Prepare the reference catalog loader
235 refConfig = LoadReferenceObjectsConfig()
236 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
238 for ref in inputRefs.refCat],
239 refCats=butlerQC.get(inputRefs.refCat),
240 name=self.config.connections.refCat,
241 log=self.log,
242 config=refConfig)
243 self.makeSubtask('fgcmLoadReferenceCatalog',
244 refObjLoader=refObjLoader,
245 refCatName=self.config.connections.refCat)
246 else:
247 lutHandle = None
249 # Compute aperture radius if necessary. This is useful to do now before
250 # any heave lifting has happened (fail early).
251 calibFluxApertureRadius = None
252 if self.config.doSubtractLocalBackground:
253 try:
254 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
255 except RuntimeError as e:
256 raise RuntimeError("Could not determine aperture radius from %s. "
257 "Cannot use doSubtractLocalBackground." %
258 (self.config.instFluxField)) from e
260 visitSummaryHandles = inputRefDict['visitSummary']
261 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
262 visitSummaryHandle in visitSummaryHandles}
264 camera = inputRefDict['camera']
265 groupedHandles = self._groupHandles(sourceTableHandleDict,
266 visitSummaryHandleDict)
268 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
270 rad = calibFluxApertureRadius
271 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles,
272 visitCat,
273 self.sourceSchema,
274 camera,
275 calibFluxApertureRadius=rad)
277 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
278 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
280 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
281 fgcmStarObservationCat,
282 lutHandle=lutHandle)
284 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
285 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
286 if fgcmRefCat is not None:
287 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
289 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
290 """Group sourceTable and visitSummary handles.
292 Parameters
293 ----------
294 sourceTableHandleDict : `dict` [`int`, `str`]
295 Dict of source tables, keyed by visit.
296 visitSummaryHandleDict : `dict` [int, `str`]
297 Dict of visit summary catalogs, keyed by visit.
299 Returns
300 -------
301 groupedHandles : `dict` [`int`, `list`]
302 Dictionary with sorted visit keys, and `list`s with
303 `lsst.daf.butler.DeferredDataSetHandle`. The first
304 item in the list will be the visitSummary ref, and
305 the second will be the source table ref.
306 """
307 groupedHandles = collections.defaultdict(list)
308 visits = sorted(sourceTableHandleDict.keys())
310 for visit in visits:
311 groupedHandles[visit] = [visitSummaryHandleDict[visit],
312 sourceTableHandleDict[visit]]
314 return groupedHandles
316 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
317 sourceSchema,
318 camera,
319 calibFluxApertureRadius=None):
320 startTime = time.time()
322 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
323 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
325 # To get the correct output schema, we use the legacy code.
326 # We are not actually using this mapper, except to grab the outputSchema
327 sourceMapper = self._makeSourceMapper(sourceSchema)
328 outputSchema = sourceMapper.getOutputSchema()
330 # Construct mapping from ccd number to index
331 ccdMapping = {}
332 for ccdIndex, detector in enumerate(camera):
333 ccdMapping[detector.getId()] = ccdIndex
335 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
337 fullCatalog = afwTable.BaseCatalog(outputSchema)
339 visitKey = outputSchema['visit'].asKey()
340 ccdKey = outputSchema['ccd'].asKey()
341 instMagKey = outputSchema['instMag'].asKey()
342 instMagErrKey = outputSchema['instMagErr'].asKey()
343 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
345 # Prepare local background if desired
346 if self.config.doSubtractLocalBackground:
347 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
349 columns = None
351 k = 2.5/np.log(10.)
353 for counter, visit in enumerate(visitCat):
354 expTime = visit['exptime']
356 handle = groupedHandles[visit['visit']][-1]
358 if columns is None:
359 inColumns = handle.get(component='columns')
360 columns = self._get_sourceTable_visit_columns(inColumns)
361 df = handle.get(parameters={'columns': columns})
363 goodSrc = self.sourceSelector.selectSources(df)
365 # Need to add a selection based on the local background correction
366 # if necessary
367 if self.config.doSubtractLocalBackground:
368 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
369 use, = np.where((goodSrc.selected)
370 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
371 else:
372 use, = np.where(goodSrc.selected)
374 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
375 tempCat.resize(use.size)
377 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
378 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
379 tempCat['x'][:] = df['x'].values[use]
380 tempCat['y'][:] = df['y'].values[use]
381 # The "visit" name in the parquet table is hard-coded.
382 tempCat[visitKey][:] = df['visit'].values[use]
383 tempCat[ccdKey][:] = df['detector'].values[use]
384 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
386 with np.warnings.catch_warnings():
387 # Ignore warnings, we will filter infinites and nans below
388 np.warnings.simplefilter("ignore")
390 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
391 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
392 / df[self.config.apertureInnerInstFluxField].values[use])
393 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
394 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
395 / df[self.config.apertureOuterInstFluxField].values[use])
396 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
397 # Set bad values to illegal values for fgcm.
398 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
400 if self.config.doSubtractLocalBackground:
401 # At the moment we only adjust the flux and not the flux
402 # error by the background because the error on
403 # base_LocalBackground_instFlux is the rms error in the
404 # background annulus, not the error on the mean in the
405 # background estimate (which is much smaller, by sqrt(n)
406 # pixels used to estimate the background, which we do not
407 # have access to in this task). In the default settings,
408 # the annulus is sufficiently large such that these
409 # additional errors are are negligibly small (much less
410 # than a mmag in quadrature).
412 # This is the difference between the mag with local background correction
413 # and the mag without local background correction.
414 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
415 - localBackground[use]) -
416 -2.5*np.log10(df[self.config.instFluxField].values[use]))
417 else:
418 tempCat['deltaMagBkg'][:] = 0.0
420 # Need to loop over ccds here
421 for detector in camera:
422 ccdId = detector.getId()
423 # used index for all observations with a given ccd
424 use2 = (tempCat[ccdKey] == ccdId)
425 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
426 tempCat['y'][use2])
427 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
428 * visit['scaling'][ccdMapping[ccdId]])
429 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
431 # Compute instMagErr from instFluxErr/instFlux, any scaling
432 # will cancel out.
433 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
434 / df[self.config.instFluxField].values[use])
436 # Apply the jacobian if configured
437 if self.config.doApplyWcsJacobian:
438 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
440 fullCatalog.extend(tempCat)
442 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
443 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
445 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
446 visit['sources_read'] = True
448 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
449 use.size, visit['visit'], visit['deltaAper'])
451 self.log.info("Found all good star observations in %.2f s" %
452 (time.time() - startTime))
454 return fullCatalog
456 def _get_sourceTable_visit_columns(self, inColumns):
457 """
458 Get the sourceTable_visit columns from the config.
460 Parameters
461 ----------
462 inColumns : `list`
463 List of columns available in the sourceTable_visit
465 Returns
466 -------
467 columns : `list`
468 List of columns to read from sourceTable_visit.
469 """
470 # Some names are hard-coded in the parquet table.
471 columns = ['visit', 'detector',
472 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
473 self.config.instFluxField, self.config.instFluxField + 'Err',
474 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
475 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
476 if self.sourceSelector.config.doFlags:
477 columns.extend(self.sourceSelector.config.flags.bad)
478 if self.sourceSelector.config.doUnresolved:
479 columns.append(self.sourceSelector.config.unresolved.name)
480 if self.sourceSelector.config.doIsolated:
481 columns.append(self.sourceSelector.config.isolated.parentName)
482 columns.append(self.sourceSelector.config.isolated.nChildName)
483 if self.config.doSubtractLocalBackground:
484 columns.append(self.config.localBackgroundFluxField)
486 return columns