lsst.fgcmcal g85d881bf73+29b09e408b
fgcmBuildStarsTable.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
24
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
31
32import time
33
34import numpy as np
35import collections
36
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
42
43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
45from .utilities import lookupStaticCalibrations
46
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
48
49
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
52 defaultTemplates={}):
53 camera = connectionTypes.PrerequisiteInput(
54 doc="Camera instrument",
55 name="camera",
56 storageClass="Camera",
57 dimensions=("instrument",),
58 lookupFunction=lookupStaticCalibrations,
59 isCalibration=True,
60 )
61
62 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
64 "chromatic corrections."),
65 name="fgcmLookUpTable",
66 storageClass="Catalog",
67 dimensions=("instrument",),
68 deferLoad=True,
69 )
70
71 sourceSchema = connectionTypes.InitInput(
72 doc="Schema for source catalogs",
73 name="src_schema",
74 storageClass="SourceCatalog",
75 )
76
77 refCat = connectionTypes.PrerequisiteInput(
78 doc="Reference catalog to use for photometric calibration",
79 name="cal_ref_cat",
80 storageClass="SimpleCatalog",
81 dimensions=("skypix",),
82 deferLoad=True,
83 multiple=True,
84 )
85
86 sourceTable_visit = connectionTypes.Input(
87 doc="Source table in parquet format, per visit",
88 name="sourceTable_visit",
89 storageClass="DataFrame",
90 dimensions=("instrument", "visit"),
91 deferLoad=True,
92 multiple=True,
93 )
94
95 visitSummary = connectionTypes.Input(
96 doc=("Per-visit consolidated exposure metadata. These catalogs use "
97 "detector id for the id and must be sorted for fast lookups of a "
98 "detector."),
99 name="visitSummary",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit"),
102 deferLoad=True,
103 multiple=True,
104 )
105
106 background = connectionTypes.Input(
107 doc="Calexp background model",
108 name="calexpBackground",
109 storageClass="Background",
110 dimensions=("instrument", "visit", "detector"),
111 deferLoad=True,
112 multiple=True,
113 )
114
115 fgcmVisitCatalog = connectionTypes.Output(
116 doc="Catalog of visit information for fgcm",
117 name="fgcmVisitCatalog",
118 storageClass="Catalog",
119 dimensions=("instrument",),
120 )
121
122 fgcmStarObservations = connectionTypes.Output(
123 doc="Catalog of star observations for fgcm",
124 name="fgcmStarObservations",
125 storageClass="Catalog",
126 dimensions=("instrument",),
127 )
128
129 fgcmStarIds = connectionTypes.Output(
130 doc="Catalog of fgcm calibration star IDs",
131 name="fgcmStarIds",
132 storageClass="Catalog",
133 dimensions=("instrument",),
134 )
135
136 fgcmStarIndices = connectionTypes.Output(
137 doc="Catalog of fgcm calibration star indices",
138 name="fgcmStarIndices",
139 storageClass="Catalog",
140 dimensions=("instrument",),
141 )
142
143 fgcmReferenceStars = connectionTypes.Output(
144 doc="Catalog of fgcm-matched reference stars",
145 name="fgcmReferenceStars",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 )
149
150 def __init__(self, *, config=None):
151 super().__init__(config=config)
152
153 if not config.doReferenceMatches:
154 self.prerequisiteInputs.remove("refCat")
155 self.prerequisiteInputs.remove("fgcmLookUpTable")
156
157 if not config.doModelErrorsWithBackground:
158 self.inputs.remove("background")
159
160 if not config.doReferenceMatches:
161 self.outputs.remove("fgcmReferenceStars")
162
163
164class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
165 pipelineConnections=FgcmBuildStarsTableConnections):
166 """Config for FgcmBuildStarsTableTask"""
167
168 referenceCCD = pexConfig.Field(
169 doc="Reference CCD for checking PSF and background",
170 dtype=int,
171 default=40,
172 )
173
174 def setDefaults(self):
175 super().setDefaults()
176
177 # The names here correspond to the post-transformed
178 # sourceTable_visit catalogs, which differ from the raw src
179 # catalogs. Therefore, all field and flag names cannot
180 # be derived from the base config class.
181 self.instFluxFieldinstFluxField = 'apFlux_12_0_instFlux'
182 self.localBackgroundFluxFieldlocalBackgroundFluxField = 'localBackground_instFlux'
185 self.psfCandidateNamepsfCandidateName = 'calib_psf_candidate'
186
187 sourceSelector = self.sourceSelector["science"]
188
189 fluxFlagName = self.instFluxFieldinstFluxField[0: -len('instFlux')] + 'flag'
190
191 sourceSelector.flags.bad = ['pixelFlags_edge',
192 'pixelFlags_interpolatedCenter',
193 'pixelFlags_saturatedCenter',
194 'pixelFlags_crCenter',
195 'pixelFlags_bad',
196 'pixelFlags_interpolated',
197 'pixelFlags_saturated',
198 'centroid_flag',
199 fluxFlagName]
200
202 localBackgroundFlagName = self.localBackgroundFluxFieldlocalBackgroundFluxField[0: -len('instFlux')] + 'flag'
203 sourceSelector.flags.bad.append(localBackgroundFlagName)
204
205 sourceSelector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
206 sourceSelector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
207
208 sourceSelector.isolated.parentName = 'parentSourceId'
209 sourceSelector.isolated.nChildName = 'deblend_nChild'
210
211 sourceSelector.unresolved.name = 'extendedness'
212
213
215 """
216 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
217 """
218 ConfigClass = FgcmBuildStarsTableConfig
219 _DefaultName = "fgcmBuildStarsTable"
220
221 canMultiprocess = False
222
223 def __init__(self, initInputs=None, **kwargs):
224 super().__init__(initInputs=initInputs, **kwargs)
225 if initInputs is not None:
226 self.sourceSchema = initInputs["sourceSchema"].schema
227
228 def runQuantum(self, butlerQC, inputRefs, outputRefs):
229 inputRefDict = butlerQC.get(inputRefs)
230
231 sourceTableHandles = inputRefDict['sourceTable_visit']
232
233 self.log.info("Running with %d sourceTable_visit handles",
234 len(sourceTableHandles))
235
236 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
237 sourceTableHandle in sourceTableHandles}
238
239 if self.config.doReferenceMatches:
240 # Get the LUT handle
241 lutHandle = inputRefDict['fgcmLookUpTable']
242
243 # Prepare the reference catalog loader
244 refConfig = LoadReferenceObjectsConfig()
245 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
246 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
247 for ref in inputRefs.refCat],
248 refCats=butlerQC.get(inputRefs.refCat),
249 name=self.config.connections.refCat,
250 log=self.log,
251 config=refConfig)
252 self.makeSubtask('fgcmLoadReferenceCatalog',
253 refObjLoader=refObjLoader,
254 refCatName=self.config.connections.refCat)
255 else:
256 lutHandle = None
257
258 # Compute aperture radius if necessary. This is useful to do now before
259 # any heave lifting has happened (fail early).
260 calibFluxApertureRadius = None
261 if self.config.doSubtractLocalBackground:
262 try:
263 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
264 except RuntimeError as e:
265 raise RuntimeError("Could not determine aperture radius from %s. "
266 "Cannot use doSubtractLocalBackground." %
267 (self.config.instFluxField)) from e
268
269 visitSummaryHandles = inputRefDict['visitSummary']
270 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
271 visitSummaryHandle in visitSummaryHandles}
272
273 camera = inputRefDict['camera']
274 groupedHandles = self._groupHandles(sourceTableHandleDict,
275 visitSummaryHandleDict)
276
277 if self.config.doModelErrorsWithBackground:
278 bkgHandles = inputRefDict['background']
279 bkgHandleDict = {(bkgHandle.dataId.byName()['visit'],
280 bkgHandle.dataId.byName()['detector']): bkgHandle for
281 bkgHandle in bkgHandles}
282 else:
283 bkgHandleDict = None
284
285 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles,
286 bkgHandleDict=bkgHandleDict)
287
288 rad = calibFluxApertureRadius
289 fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedHandles,
290 visitCat,
291 self.sourceSchema,
292 camera,
293 calibFluxApertureRadius=rad)
294
295 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
296 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
297
298 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
299 fgcmStarObservationCat,
300 lutHandle=lutHandle)
301
302 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
303 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
304 if fgcmRefCat is not None:
305 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
306
307 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
308 """Group sourceTable and visitSummary handles.
309
310 Parameters
311 ----------
312 sourceTableHandleDict : `dict` [`int`, `str`]
313 Dict of source tables, keyed by visit.
314 visitSummaryHandleDict : `dict` [int, `str`]
315 Dict of visit summary catalogs, keyed by visit.
316
317 Returns
318 -------
319 groupedHandles : `dict` [`int`, `list`]
320 Dictionary with sorted visit keys, and `list`s with
321 `lsst.daf.butler.DeferredDataSetHandle`. The first
322 item in the list will be the visitSummary ref, and
323 the second will be the source table ref.
324 """
325 groupedHandles = collections.defaultdict(list)
326 visits = sorted(sourceTableHandleDict.keys())
327
328 for visit in visits:
329 groupedHandles[visit] = [visitSummaryHandleDict[visit],
330 sourceTableHandleDict[visit]]
331
332 return groupedHandles
333
334 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
335 sourceSchema,
336 camera,
337 calibFluxApertureRadius=None):
338 startTime = time.time()
339
340 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
341 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
342
343 # To get the correct output schema, we use the legacy code.
344 # We are not actually using this mapper, except to grab the outputSchema
345 sourceMapper = self._makeSourceMapper(sourceSchema)
346 outputSchema = sourceMapper.getOutputSchema()
347
348 # Construct mapping from ccd number to index
349 ccdMapping = {}
350 for ccdIndex, detector in enumerate(camera):
351 ccdMapping[detector.getId()] = ccdIndex
352
353 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
354
355 fullCatalog = afwTable.BaseCatalog(outputSchema)
356
357 visitKey = outputSchema['visit'].asKey()
358 ccdKey = outputSchema['ccd'].asKey()
359 instMagKey = outputSchema['instMag'].asKey()
360 instMagErrKey = outputSchema['instMagErr'].asKey()
361 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
362
363 # Prepare local background if desired
364 if self.config.doSubtractLocalBackground:
365 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
366
367 columns = None
368
369 k = 2.5/np.log(10.)
370
371 for counter, visit in enumerate(visitCat):
372 expTime = visit['exptime']
373
374 handle = groupedHandles[visit['visit']][-1]
375
376 if columns is None:
377 inColumns = handle.get(component='columns')
378 columns, detColumn = self._get_sourceTable_visit_columns(inColumns)
379 df = handle.get(parameters={'columns': columns})
380
381 goodSrc = self.sourceSelector.selectSources(df)
382
383 # Need to add a selection based on the local background correction
384 # if necessary
385 if self.config.doSubtractLocalBackground:
386 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
387 use, = np.where((goodSrc.selected)
388 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
389 else:
390 use, = np.where(goodSrc.selected)
391
392 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
393 tempCat.resize(use.size)
394
395 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
396 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
397 tempCat['x'][:] = df['x'].values[use]
398 tempCat['y'][:] = df['y'].values[use]
399 # The "visit" name in the parquet table is hard-coded.
400 tempCat[visitKey][:] = df['visit'].values[use]
401 tempCat[ccdKey][:] = df[detColumn].values[use]
402 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
403
404 with np.warnings.catch_warnings():
405 # Ignore warnings, we will filter infinites and nans below
406 np.warnings.simplefilter("ignore")
407
408 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
409 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
410 / df[self.config.apertureInnerInstFluxField].values[use])
411 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
412 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
413 / df[self.config.apertureOuterInstFluxField].values[use])
414 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
415 # Set bad values to illegal values for fgcm.
416 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
417
418 if self.config.doSubtractLocalBackground:
419 # At the moment we only adjust the flux and not the flux
420 # error by the background because the error on
421 # base_LocalBackground_instFlux is the rms error in the
422 # background annulus, not the error on the mean in the
423 # background estimate (which is much smaller, by sqrt(n)
424 # pixels used to estimate the background, which we do not
425 # have access to in this task). In the default settings,
426 # the annulus is sufficiently large such that these
427 # additional errors are are negligibly small (much less
428 # than a mmag in quadrature).
429
430 # This is the difference between the mag with local background correction
431 # and the mag without local background correction.
432 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
433 - localBackground[use]) -
434 -2.5*np.log10(df[self.config.instFluxField].values[use]))
435 else:
436 tempCat['deltaMagBkg'][:] = 0.0
437
438 # Need to loop over ccds here
439 for detector in camera:
440 ccdId = detector.getId()
441 # used index for all observations with a given ccd
442 use2 = (tempCat[ccdKey] == ccdId)
443 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
444 tempCat['y'][use2])
445 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
446 * visit['scaling'][ccdMapping[ccdId]])
447 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
448
449 # Compute instMagErr from instFluxErr/instFlux, any scaling
450 # will cancel out.
451 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
452 / df[self.config.instFluxField].values[use])
453
454 # Apply the jacobian if configured
455 if self.config.doApplyWcsJacobian:
456 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
457
458 fullCatalog.extend(tempCat)
459
460 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
461 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
462
463 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
464 visit['sources_read'] = True
465
466 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
467 use.size, visit['visit'], visit['deltaAper'])
468
469 self.log.info("Found all good star observations in %.2f s" %
470 (time.time() - startTime))
471
472 return fullCatalog
473
474 def _get_sourceTable_visit_columns(self, inColumns):
475 """
476 Get the sourceTable_visit columns from the config.
477
478 Parameters
479 ----------
480 inColumns : `list`
481 List of columns available in the sourceTable_visit
482
483 Returns
484 -------
485 columns : `list`
486 List of columns to read from sourceTable_visit.
487 detectorColumn : `str`
488 Name of the detector column.
489 """
490 if 'detector' in inColumns:
491 # Default name for Gen3.
492 detectorColumn = 'detector'
493 else:
494 # Default name for Gen2 conversions (including test data).
495 detectorColumn = 'ccd'
496 # Some names are hard-coded in the parquet table.
497 columns = ['visit', detectorColumn,
498 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
499 self.config.instFluxField, self.config.instFluxField + 'Err',
500 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
501 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
502 if self.sourceSelector.config.doFlags:
503 columns.extend(self.sourceSelector.config.flags.bad)
504 if self.sourceSelector.config.doUnresolved:
505 columns.append(self.sourceSelector.config.unresolved.name)
506 if self.sourceSelector.config.doIsolated:
507 columns.append(self.sourceSelector.config.isolated.parentName)
508 columns.append(self.sourceSelector.config.isolated.nChildName)
509 if self.config.doSubtractLocalBackground:
510 columns.append(self.config.localBackgroundFluxField)
511
512 return columns, detectorColumn
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def fgcmMakeVisitCatalog(self, camera, groupedHandles, bkgHandleDict=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)
def computeApertureRadiusFromName(fluxField)
Definition: utilities.py:839
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:522