lsst.fgcmcal g01934ee372+6d139e3a4f
Loading...
Searching...
No Matches
fgcmBuildStarsTable.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
24
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
31
32import time
33
34import numpy as np
35import collections
36
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
42
43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
45from .utilities import lookupStaticCalibrations
46
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
48
49
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
52 defaultTemplates={}):
53 camera = connectionTypes.PrerequisiteInput(
54 doc="Camera instrument",
55 name="camera",
56 storageClass="Camera",
57 dimensions=("instrument",),
58 lookupFunction=lookupStaticCalibrations,
59 isCalibration=True,
60 )
61
62 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
64 "chromatic corrections."),
65 name="fgcmLookUpTable",
66 storageClass="Catalog",
67 dimensions=("instrument",),
68 deferLoad=True,
69 )
70
71 sourceSchema = connectionTypes.InitInput(
72 doc="Schema for source catalogs",
73 name="src_schema",
74 storageClass="SourceCatalog",
75 )
76
77 refCat = connectionTypes.PrerequisiteInput(
78 doc="Reference catalog to use for photometric calibration",
79 name="cal_ref_cat",
80 storageClass="SimpleCatalog",
81 dimensions=("skypix",),
82 deferLoad=True,
83 multiple=True,
84 )
85
86 sourceTable_visit = connectionTypes.Input(
87 doc="Source table in parquet format, per visit",
88 name="sourceTable_visit",
89 storageClass="DataFrame",
90 dimensions=("instrument", "visit"),
91 deferLoad=True,
92 multiple=True,
93 )
94
95 visitSummary = connectionTypes.Input(
96 doc=("Per-visit consolidated exposure metadata. These catalogs use "
97 "detector id for the id and must be sorted for fast lookups of a "
98 "detector."),
99 name="visitSummary",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit"),
102 deferLoad=True,
103 multiple=True,
104 )
105
106 fgcmVisitCatalog = connectionTypes.Output(
107 doc="Catalog of visit information for fgcm",
108 name="fgcmVisitCatalog",
109 storageClass="Catalog",
110 dimensions=("instrument",),
111 )
112
113 fgcmStarObservations = connectionTypes.Output(
114 doc="Catalog of star observations for fgcm",
115 name="fgcmStarObservations",
116 storageClass="Catalog",
117 dimensions=("instrument",),
118 )
119
120 fgcmStarIds = connectionTypes.Output(
121 doc="Catalog of fgcm calibration star IDs",
122 name="fgcmStarIds",
123 storageClass="Catalog",
124 dimensions=("instrument",),
125 )
126
127 fgcmStarIndices = connectionTypes.Output(
128 doc="Catalog of fgcm calibration star indices",
129 name="fgcmStarIndices",
130 storageClass="Catalog",
131 dimensions=("instrument",),
132 )
133
134 fgcmReferenceStars = connectionTypes.Output(
135 doc="Catalog of fgcm-matched reference stars",
136 name="fgcmReferenceStars",
137 storageClass="Catalog",
138 dimensions=("instrument",),
139 )
140
141 def __init__(self, *, config=None):
142 super().__init__(config=config)
143
144 if not config.doReferenceMatches:
145 self.prerequisiteInputs.remove("refCat")
146 self.prerequisiteInputs.remove("fgcmLookUpTable")
147
148 if not config.doReferenceMatches:
149 self.outputs.remove("fgcmReferenceStars")
150
151
152class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
153 pipelineConnections=FgcmBuildStarsTableConnections):
154 """Config for FgcmBuildStarsTableTask"""
155
156 referenceCCD = pexConfig.Field(
157 doc="Reference CCD for checking PSF and background",
158 dtype=int,
159 default=40,
160 )
161
162 def setDefaults(self):
163 super().setDefaults()
164
165 # The names here correspond to the post-transformed
166 # sourceTable_visit catalogs, which differ from the raw src
167 # catalogs. Therefore, all field and flag names cannot
168 # be derived from the base config class.
169 self.instFluxFieldinstFluxField = 'apFlux_12_0_instFlux'
170 self.localBackgroundFluxFieldlocalBackgroundFluxField = 'localBackground_instFlux'
173 self.psfCandidateNamepsfCandidateName = 'calib_psf_candidate'
174
175 sourceSelector = self.sourceSelector["science"]
176
177 fluxFlagName = self.instFluxFieldinstFluxField[0: -len('instFlux')] + 'flag'
178
179 sourceSelector.flags.bad = ['pixelFlags_edge',
180 'pixelFlags_interpolatedCenter',
181 'pixelFlags_saturatedCenter',
182 'pixelFlags_crCenter',
183 'pixelFlags_bad',
184 'pixelFlags_interpolated',
185 'pixelFlags_saturated',
186 'centroid_flag',
187 fluxFlagName]
188
190 localBackgroundFlagName = self.localBackgroundFluxFieldlocalBackgroundFluxField[0: -len('instFlux')] + 'flag'
191 sourceSelector.flags.bad.append(localBackgroundFlagName)
192
193 sourceSelector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
194 sourceSelector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
195
196 sourceSelector.isolated.parentName = 'parentSourceId'
197 sourceSelector.isolated.nChildName = 'deblend_nChild'
198
199 sourceSelector.requireFiniteRaDec.raColName = 'ra'
200 sourceSelector.requireFiniteRaDec.decColName = 'decl'
201
202 sourceSelector.unresolved.name = 'extendedness'
203
204
206 """
207 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
208 """
209 ConfigClass = FgcmBuildStarsTableConfig
210 _DefaultName = "fgcmBuildStarsTable"
211
212 canMultiprocess = False
213
214 def __init__(self, initInputs=None, **kwargs):
215 super().__init__(initInputs=initInputs, **kwargs)
216 if initInputs is not None:
217 self.sourceSchema = initInputs["sourceSchema"].schema
218
219 def runQuantum(self, butlerQC, inputRefs, outputRefs):
220 inputRefDict = butlerQC.get(inputRefs)
221
222 sourceTableHandles = inputRefDict['sourceTable_visit']
223
224 self.log.info("Running with %d sourceTable_visit handles",
225 len(sourceTableHandles))
226
227 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
228 sourceTableHandle in sourceTableHandles}
229
230 if self.config.doReferenceMatches:
231 # Get the LUT handle
232 lutHandle = inputRefDict['fgcmLookUpTable']
233
234 # Prepare the reference catalog loader
235 refConfig = LoadReferenceObjectsConfig()
236 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
238 for ref in inputRefs.refCat],
239 refCats=butlerQC.get(inputRefs.refCat),
240 name=self.config.connections.refCat,
241 log=self.log,
242 config=refConfig)
243 self.makeSubtask('fgcmLoadReferenceCatalog',
244 refObjLoader=refObjLoader,
245 refCatName=self.config.connections.refCat)
246 else:
247 lutHandle = None
248
249 # Compute aperture radius if necessary. This is useful to do now before
250 # any heave lifting has happened (fail early).
251 calibFluxApertureRadius = None
252 if self.config.doSubtractLocalBackground:
253 try:
254 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
255 except RuntimeError as e:
256 raise RuntimeError("Could not determine aperture radius from %s. "
257 "Cannot use doSubtractLocalBackground." %
258 (self.config.instFluxField)) from e
259
260 visitSummaryHandles = inputRefDict['visitSummary']
261 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
262 visitSummaryHandle in visitSummaryHandles}
263
264 camera = inputRefDict['camera']
265 groupedHandles = self._groupHandles(sourceTableHandleDict,
266 visitSummaryHandleDict)
267
268 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
269
270 rad = calibFluxApertureRadius
271 fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedHandles,
272 visitCat,
273 self.sourceSchema,
274 camera,
275 calibFluxApertureRadius=rad)
276
277 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
278 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
279
280 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
281 fgcmStarObservationCat,
282 lutHandle=lutHandle)
283
284 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
285 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
286 if fgcmRefCat is not None:
287 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
288
289 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
290 """Group sourceTable and visitSummary handles.
291
292 Parameters
293 ----------
294 sourceTableHandleDict : `dict` [`int`, `str`]
295 Dict of source tables, keyed by visit.
296 visitSummaryHandleDict : `dict` [int, `str`]
297 Dict of visit summary catalogs, keyed by visit.
298
299 Returns
300 -------
301 groupedHandles : `dict` [`int`, `list`]
302 Dictionary with sorted visit keys, and `list`s with
303 `lsst.daf.butler.DeferredDataSetHandle`. The first
304 item in the list will be the visitSummary ref, and
305 the second will be the source table ref.
306 """
307 groupedHandles = collections.defaultdict(list)
308 visits = sorted(sourceTableHandleDict.keys())
309
310 for visit in visits:
311 groupedHandles[visit] = [visitSummaryHandleDict[visit],
312 sourceTableHandleDict[visit]]
313
314 return groupedHandles
315
316 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
317 sourceSchema,
318 camera,
319 calibFluxApertureRadius=None):
320 startTime = time.time()
321
322 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
323 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
324
325 # To get the correct output schema, we use the legacy code.
326 # We are not actually using this mapper, except to grab the outputSchema
327 sourceMapper = self._makeSourceMapper(sourceSchema)
328 outputSchema = sourceMapper.getOutputSchema()
329
330 # Construct mapping from ccd number to index
331 ccdMapping = {}
332 for ccdIndex, detector in enumerate(camera):
333 ccdMapping[detector.getId()] = ccdIndex
334
335 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
336
337 fullCatalog = afwTable.BaseCatalog(outputSchema)
338
339 visitKey = outputSchema['visit'].asKey()
340 ccdKey = outputSchema['ccd'].asKey()
341 instMagKey = outputSchema['instMag'].asKey()
342 instMagErrKey = outputSchema['instMagErr'].asKey()
343 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
344
345 # Prepare local background if desired
346 if self.config.doSubtractLocalBackground:
347 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
348
349 columns = None
350
351 k = 2.5/np.log(10.)
352
353 for counter, visit in enumerate(visitCat):
354 expTime = visit['exptime']
355
356 handle = groupedHandles[visit['visit']][-1]
357
358 if columns is None:
359 inColumns = handle.get(component='columns')
360 columns = self._get_sourceTable_visit_columns(inColumns)
361 df = handle.get(parameters={'columns': columns})
362
363 goodSrc = self.sourceSelector.selectSources(df)
364
365 # Need to add a selection based on the local background correction
366 # if necessary
367 if self.config.doSubtractLocalBackground:
368 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
369 use, = np.where((goodSrc.selected)
370 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
371 else:
372 use, = np.where(goodSrc.selected)
373
374 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
375 tempCat.resize(use.size)
376
377 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
378 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
379 tempCat['x'][:] = df['x'].values[use]
380 tempCat['y'][:] = df['y'].values[use]
381 # The "visit" name in the parquet table is hard-coded.
382 tempCat[visitKey][:] = df['visit'].values[use]
383 tempCat[ccdKey][:] = df['detector'].values[use]
384 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
385
386 with np.warnings.catch_warnings():
387 # Ignore warnings, we will filter infinites and nans below
388 np.warnings.simplefilter("ignore")
389
390 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
391 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
392 / df[self.config.apertureInnerInstFluxField].values[use])
393 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
394 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
395 / df[self.config.apertureOuterInstFluxField].values[use])
396 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
397 # Set bad values to illegal values for fgcm.
398 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
399
400 if self.config.doSubtractLocalBackground:
401 # At the moment we only adjust the flux and not the flux
402 # error by the background because the error on
403 # base_LocalBackground_instFlux is the rms error in the
404 # background annulus, not the error on the mean in the
405 # background estimate (which is much smaller, by sqrt(n)
406 # pixels used to estimate the background, which we do not
407 # have access to in this task). In the default settings,
408 # the annulus is sufficiently large such that these
409 # additional errors are are negligibly small (much less
410 # than a mmag in quadrature).
411
412 # This is the difference between the mag with local background correction
413 # and the mag without local background correction.
414 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
415 - localBackground[use]) -
416 -2.5*np.log10(df[self.config.instFluxField].values[use]))
417 else:
418 tempCat['deltaMagBkg'][:] = 0.0
419
420 # Need to loop over ccds here
421 for detector in camera:
422 ccdId = detector.getId()
423 # used index for all observations with a given ccd
424 use2 = (tempCat[ccdKey] == ccdId)
425 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
426 tempCat['y'][use2])
427 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
428 * visit['scaling'][ccdMapping[ccdId]])
429 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
430
431 # Compute instMagErr from instFluxErr/instFlux, any scaling
432 # will cancel out.
433 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
434 / df[self.config.instFluxField].values[use])
435
436 # Apply the jacobian if configured
437 if self.config.doApplyWcsJacobian:
438 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
439
440 fullCatalog.extend(tempCat)
441
442 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
443 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
444
445 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
446 visit['sources_read'] = True
447
448 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
449 use.size, visit['visit'], visit['deltaAper'])
450
451 self.log.info("Found all good star observations in %.2f s" %
452 (time.time() - startTime))
453
454 return fullCatalog
455
456 def _get_sourceTable_visit_columns(self, inColumns):
457 """
458 Get the sourceTable_visit columns from the config.
459
460 Parameters
461 ----------
462 inColumns : `list`
463 List of columns available in the sourceTable_visit
464
465 Returns
466 -------
467 columns : `list`
468 List of columns to read from sourceTable_visit.
469 """
470 # Some names are hard-coded in the parquet table.
471 columns = ['visit', 'detector',
472 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
473 self.config.instFluxField, self.config.instFluxField + 'Err',
474 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
475 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
476 if self.sourceSelector.config.doFlags:
477 columns.extend(self.sourceSelector.config.flags.bad)
478 if self.sourceSelector.config.doUnresolved:
479 columns.append(self.sourceSelector.config.unresolved.name)
480 if self.sourceSelector.config.doIsolated:
481 columns.append(self.sourceSelector.config.isolated.parentName)
482 columns.append(self.sourceSelector.config.isolated.nChildName)
483 if self.config.doSubtractLocalBackground:
484 columns.append(self.config.localBackgroundFluxField)
485
486 return columns
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)