lsst.fgcmcal gbf1a427e67+7b9a4f1f37
Loading...
Searching...
No Matches
fgcmBuildStarsTable.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
24
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
31
32import time
33import warnings
34
35import numpy as np
36import collections
37
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
46from .utilities import lookupStaticCalibrations
47
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
49
50
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
53 defaultTemplates={}):
54 camera = connectionTypes.PrerequisiteInput(
55 doc="Camera instrument",
56 name="camera",
57 storageClass="Camera",
58 dimensions=("instrument",),
59 lookupFunction=lookupStaticCalibrations,
60 isCalibration=True,
61 )
62
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name="fgcmLookUpTable",
67 storageClass="Catalog",
68 dimensions=("instrument",),
69 deferLoad=True,
70 )
71
72 sourceSchema = connectionTypes.InitInput(
73 doc="Schema for source catalogs",
74 name="src_schema",
75 storageClass="SourceCatalog",
76 )
77
78 refCat = connectionTypes.PrerequisiteInput(
79 doc="Reference catalog to use for photometric calibration",
80 name="cal_ref_cat",
81 storageClass="SimpleCatalog",
82 dimensions=("skypix",),
83 deferLoad=True,
84 multiple=True,
85 )
86
87 sourceTable_visit = connectionTypes.Input(
88 doc="Source table in parquet format, per visit",
89 name="sourceTable_visit",
90 storageClass="DataFrame",
91 dimensions=("instrument", "visit"),
92 deferLoad=True,
93 multiple=True,
94 )
95
96 visitSummary = connectionTypes.Input(
97 doc=("Per-visit consolidated exposure metadata. These catalogs use "
98 "detector id for the id and must be sorted for fast lookups of a "
99 "detector."),
100 name="visitSummary",
101 storageClass="ExposureCatalog",
102 dimensions=("instrument", "visit"),
103 deferLoad=True,
104 multiple=True,
105 )
106
107 fgcmVisitCatalog = connectionTypes.Output(
108 doc="Catalog of visit information for fgcm",
109 name="fgcmVisitCatalog",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 )
113
114 fgcmStarObservations = connectionTypes.Output(
115 doc="Catalog of star observations for fgcm",
116 name="fgcmStarObservations",
117 storageClass="Catalog",
118 dimensions=("instrument",),
119 )
120
121 fgcmStarIds = connectionTypes.Output(
122 doc="Catalog of fgcm calibration star IDs",
123 name="fgcmStarIds",
124 storageClass="Catalog",
125 dimensions=("instrument",),
126 )
127
128 fgcmStarIndices = connectionTypes.Output(
129 doc="Catalog of fgcm calibration star indices",
130 name="fgcmStarIndices",
131 storageClass="Catalog",
132 dimensions=("instrument",),
133 )
134
135 fgcmReferenceStars = connectionTypes.Output(
136 doc="Catalog of fgcm-matched reference stars",
137 name="fgcmReferenceStars",
138 storageClass="Catalog",
139 dimensions=("instrument",),
140 )
141
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
144
145 if not config.doReferenceMatches:
146 self.prerequisiteInputs.remove("refCat")
147 self.prerequisiteInputs.remove("fgcmLookUpTable")
148
149 if not config.doReferenceMatches:
150 self.outputs.remove("fgcmReferenceStars")
151
152
153class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
154 pipelineConnections=FgcmBuildStarsTableConnections):
155 """Config for FgcmBuildStarsTableTask"""
156
157 referenceCCD = pexConfig.Field(
158 doc="Reference CCD for checking PSF and background",
159 dtype=int,
160 default=40,
161 )
162
163 def setDefaults(self):
164 super().setDefaults()
165
166 # The names here correspond to the post-transformed
167 # sourceTable_visit catalogs, which differ from the raw src
168 # catalogs. Therefore, all field and flag names cannot
169 # be derived from the base config class.
170 self.instFluxFieldinstFluxField = 'apFlux_12_0_instFlux'
171 self.localBackgroundFluxFieldlocalBackgroundFluxField = 'localBackground_instFlux'
174 self.psfCandidateNamepsfCandidateName = 'calib_psf_candidate'
175
176 sourceSelector = self.sourceSelector["science"]
177
178 fluxFlagName = self.instFluxFieldinstFluxField[0: -len('instFlux')] + 'flag'
179
180 sourceSelector.flags.bad = ['pixelFlags_edge',
181 'pixelFlags_interpolatedCenter',
182 'pixelFlags_saturatedCenter',
183 'pixelFlags_crCenter',
184 'pixelFlags_bad',
185 'pixelFlags_interpolated',
186 'pixelFlags_saturated',
187 'centroid_flag',
188 fluxFlagName]
189
191 localBackgroundFlagName = self.localBackgroundFluxFieldlocalBackgroundFluxField[0: -len('instFlux')] + 'flag'
192 sourceSelector.flags.bad.append(localBackgroundFlagName)
193
194 sourceSelector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
195 sourceSelector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
196
197 sourceSelector.isolated.parentName = 'parentSourceId'
198 sourceSelector.isolated.nChildName = 'deblend_nChild'
199
200 sourceSelector.requireFiniteRaDec.raColName = 'ra'
201 sourceSelector.requireFiniteRaDec.decColName = 'decl'
202
203 sourceSelector.unresolved.name = 'extendedness'
204
205
207 """
208 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
209 """
210 ConfigClass = FgcmBuildStarsTableConfig
211 _DefaultName = "fgcmBuildStarsTable"
212
213 canMultiprocess = False
214
215 def __init__(self, initInputs=None, **kwargs):
216 super().__init__(initInputs=initInputs, **kwargs)
217 if initInputs is not None:
218 self.sourceSchema = initInputs["sourceSchema"].schema
219
220 def runQuantum(self, butlerQC, inputRefs, outputRefs):
221 inputRefDict = butlerQC.get(inputRefs)
222
223 sourceTableHandles = inputRefDict['sourceTable_visit']
224
225 self.log.info("Running with %d sourceTable_visit handles",
226 len(sourceTableHandles))
227
228 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
229 sourceTableHandle in sourceTableHandles}
230
231 if self.config.doReferenceMatches:
232 # Get the LUT handle
233 lutHandle = inputRefDict['fgcmLookUpTable']
234
235 # Prepare the reference catalog loader
236 refConfig = LoadReferenceObjectsConfig()
237 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
238 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
239 for ref in inputRefs.refCat],
240 refCats=butlerQC.get(inputRefs.refCat),
241 name=self.config.connections.refCat,
242 log=self.log,
243 config=refConfig)
244 self.makeSubtask('fgcmLoadReferenceCatalog',
245 refObjLoader=refObjLoader,
246 refCatName=self.config.connections.refCat)
247 else:
248 lutHandle = None
249
250 # Compute aperture radius if necessary. This is useful to do now before
251 # any heave lifting has happened (fail early).
252 calibFluxApertureRadius = None
253 if self.config.doSubtractLocalBackground:
254 try:
255 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
256 except RuntimeError as e:
257 raise RuntimeError("Could not determine aperture radius from %s. "
258 "Cannot use doSubtractLocalBackground." %
259 (self.config.instFluxField)) from e
260
261 visitSummaryHandles = inputRefDict['visitSummary']
262 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
263 visitSummaryHandle in visitSummaryHandles}
264
265 camera = inputRefDict['camera']
266 groupedHandles = self._groupHandles(sourceTableHandleDict,
267 visitSummaryHandleDict)
268
269 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
270
271 rad = calibFluxApertureRadius
272 fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedHandles,
273 visitCat,
274 self.sourceSchema,
275 camera,
276 calibFluxApertureRadius=rad)
277
278 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
279 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
280
281 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
282 fgcmStarObservationCat,
283 lutHandle=lutHandle)
284
285 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
286 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
287 if fgcmRefCat is not None:
288 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
289
290 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
291 """Group sourceTable and visitSummary handles.
292
293 Parameters
294 ----------
295 sourceTableHandleDict : `dict` [`int`, `str`]
296 Dict of source tables, keyed by visit.
297 visitSummaryHandleDict : `dict` [int, `str`]
298 Dict of visit summary catalogs, keyed by visit.
299
300 Returns
301 -------
302 groupedHandles : `dict` [`int`, `list`]
303 Dictionary with sorted visit keys, and `list`s with
304 `lsst.daf.butler.DeferredDataSetHandle`. The first
305 item in the list will be the visitSummary ref, and
306 the second will be the source table ref.
307 """
308 groupedHandles = collections.defaultdict(list)
309 visits = sorted(sourceTableHandleDict.keys())
310
311 for visit in visits:
312 groupedHandles[visit] = [visitSummaryHandleDict[visit],
313 sourceTableHandleDict[visit]]
314
315 return groupedHandles
316
317 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
318 sourceSchema,
319 camera,
320 calibFluxApertureRadius=None):
321 startTime = time.time()
322
323 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
324 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
325
326 # To get the correct output schema, we use the legacy code.
327 # We are not actually using this mapper, except to grab the outputSchema
328 sourceMapper = self._makeSourceMapper(sourceSchema)
329 outputSchema = sourceMapper.getOutputSchema()
330
331 # Construct mapping from ccd number to index
332 ccdMapping = {}
333 for ccdIndex, detector in enumerate(camera):
334 ccdMapping[detector.getId()] = ccdIndex
335
336 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
337
338 fullCatalog = afwTable.BaseCatalog(outputSchema)
339
340 visitKey = outputSchema['visit'].asKey()
341 ccdKey = outputSchema['ccd'].asKey()
342 instMagKey = outputSchema['instMag'].asKey()
343 instMagErrKey = outputSchema['instMagErr'].asKey()
344 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
345
346 # Prepare local background if desired
347 if self.config.doSubtractLocalBackground:
348 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
349
350 columns = None
351
352 k = 2.5/np.log(10.)
353
354 for counter, visit in enumerate(visitCat):
355 expTime = visit['exptime']
356
357 handle = groupedHandles[visit['visit']][-1]
358
359 if columns is None:
360 inColumns = handle.get(component='columns')
361 columns = self._get_sourceTable_visit_columns(inColumns)
362 df = handle.get(parameters={'columns': columns})
363
364 goodSrc = self.sourceSelector.selectSources(df)
365
366 # Need to add a selection based on the local background correction
367 # if necessary
368 if self.config.doSubtractLocalBackground:
369 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
370 use, = np.where((goodSrc.selected)
371 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
372 else:
373 use, = np.where(goodSrc.selected)
374
375 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
376 tempCat.resize(use.size)
377
378 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
379 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
380 tempCat['x'][:] = df['x'].values[use]
381 tempCat['y'][:] = df['y'].values[use]
382 # The "visit" name in the parquet table is hard-coded.
383 tempCat[visitKey][:] = df['visit'].values[use]
384 tempCat[ccdKey][:] = df['detector'].values[use]
385 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
386
387 with warnings.catch_warnings():
388 # Ignore warnings, we will filter infinites and nans below
389 warnings.simplefilter("ignore")
390
391 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
392 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
393 / df[self.config.apertureInnerInstFluxField].values[use])
394 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
395 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
396 / df[self.config.apertureOuterInstFluxField].values[use])
397 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
398 # Set bad values to illegal values for fgcm.
399 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
400
401 if self.config.doSubtractLocalBackground:
402 # At the moment we only adjust the flux and not the flux
403 # error by the background because the error on
404 # base_LocalBackground_instFlux is the rms error in the
405 # background annulus, not the error on the mean in the
406 # background estimate (which is much smaller, by sqrt(n)
407 # pixels used to estimate the background, which we do not
408 # have access to in this task). In the default settings,
409 # the annulus is sufficiently large such that these
410 # additional errors are are negligibly small (much less
411 # than a mmag in quadrature).
412
413 # This is the difference between the mag with local background correction
414 # and the mag without local background correction.
415 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
416 - localBackground[use]) -
417 -2.5*np.log10(df[self.config.instFluxField].values[use]))
418 else:
419 tempCat['deltaMagBkg'][:] = 0.0
420
421 # Need to loop over ccds here
422 for detector in camera:
423 ccdId = detector.getId()
424 # used index for all observations with a given ccd
425 use2 = (tempCat[ccdKey] == ccdId)
426 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
427 tempCat['y'][use2])
428 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
429 * visit['scaling'][ccdMapping[ccdId]])
430 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
431
432 # Compute instMagErr from instFluxErr/instFlux, any scaling
433 # will cancel out.
434 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
435 / df[self.config.instFluxField].values[use])
436
437 # Apply the jacobian if configured
438 if self.config.doApplyWcsJacobian:
439 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
440
441 fullCatalog.extend(tempCat)
442
443 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
444 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
445
446 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
447 visit['sources_read'] = True
448
449 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
450 use.size, visit['visit'], visit['deltaAper'])
451
452 self.log.info("Found all good star observations in %.2f s" %
453 (time.time() - startTime))
454
455 return fullCatalog
456
457 def _get_sourceTable_visit_columns(self, inColumns):
458 """
459 Get the sourceTable_visit columns from the config.
460
461 Parameters
462 ----------
463 inColumns : `list`
464 List of columns available in the sourceTable_visit
465
466 Returns
467 -------
468 columns : `list`
469 List of columns to read from sourceTable_visit.
470 """
471 # Some names are hard-coded in the parquet table.
472 columns = ['visit', 'detector',
473 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
474 self.config.instFluxField, self.config.instFluxField + 'Err',
475 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
476 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
477 if self.sourceSelector.config.doFlags:
478 columns.extend(self.sourceSelector.config.flags.bad)
479 if self.sourceSelector.config.doUnresolved:
480 columns.append(self.sourceSelector.config.unresolved.name)
481 if self.sourceSelector.config.doIsolated:
482 columns.append(self.sourceSelector.config.isolated.parentName)
483 columns.append(self.sourceSelector.config.isolated.nChildName)
484 if self.config.doSubtractLocalBackground:
485 columns.append(self.config.localBackgroundFluxField)
486
487 return columns
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)