lsst.fgcmcal g3eccfa81ec+e3875f2715
Loading...
Searching...
No Matches
fgcmBuildStarsTable.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
24
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
31
32import time
33import warnings
34
35import numpy as np
36import collections
37
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
46from .utilities import lookupStaticCalibrations
47
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
49
50
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
53 defaultTemplates={}):
54 camera = connectionTypes.PrerequisiteInput(
55 doc="Camera instrument",
56 name="camera",
57 storageClass="Camera",
58 dimensions=("instrument",),
59 lookupFunction=lookupStaticCalibrations,
60 isCalibration=True,
61 )
62
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name="fgcmLookUpTable",
67 storageClass="Catalog",
68 dimensions=("instrument",),
69 deferLoad=True,
70 )
71
72 sourceSchema = connectionTypes.InitInput(
73 doc="Schema for source catalogs",
74 name="src_schema",
75 storageClass="SourceCatalog",
76 )
77
78 refCat = connectionTypes.PrerequisiteInput(
79 doc="Reference catalog to use for photometric calibration",
80 name="cal_ref_cat",
81 storageClass="SimpleCatalog",
82 dimensions=("skypix",),
83 deferLoad=True,
84 multiple=True,
85 )
86
87 sourceTable_visit = connectionTypes.Input(
88 doc="Source table in parquet format, per visit",
89 name="sourceTable_visit",
90 storageClass="DataFrame",
91 dimensions=("instrument", "visit"),
92 deferLoad=True,
93 multiple=True,
94 )
95
96 visitSummary = connectionTypes.Input(
97 doc=("Per-visit consolidated exposure metadata. These catalogs use "
98 "detector id for the id and must be sorted for fast lookups of a "
99 "detector."),
100 name="visitSummary",
101 storageClass="ExposureCatalog",
102 dimensions=("instrument", "visit"),
103 deferLoad=True,
104 multiple=True,
105 )
106
107 fgcmVisitCatalog = connectionTypes.Output(
108 doc="Catalog of visit information for fgcm",
109 name="fgcmVisitCatalog",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 )
113
114 fgcmStarObservations = connectionTypes.Output(
115 doc="Catalog of star observations for fgcm",
116 name="fgcmStarObservations",
117 storageClass="Catalog",
118 dimensions=("instrument",),
119 )
120
121 fgcmStarIds = connectionTypes.Output(
122 doc="Catalog of fgcm calibration star IDs",
123 name="fgcmStarIds",
124 storageClass="Catalog",
125 dimensions=("instrument",),
126 )
127
128 fgcmStarIndices = connectionTypes.Output(
129 doc="Catalog of fgcm calibration star indices",
130 name="fgcmStarIndices",
131 storageClass="Catalog",
132 dimensions=("instrument",),
133 )
134
135 fgcmReferenceStars = connectionTypes.Output(
136 doc="Catalog of fgcm-matched reference stars",
137 name="fgcmReferenceStars",
138 storageClass="Catalog",
139 dimensions=("instrument",),
140 )
141
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
144
145 if not config.doReferenceMatches:
146 self.prerequisiteInputs.remove("refCat")
147 self.prerequisiteInputs.remove("fgcmLookUpTable")
148
149 if not config.doReferenceMatches:
150 self.outputs.remove("fgcmReferenceStars")
151
152
153class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
154 pipelineConnections=FgcmBuildStarsTableConnections):
155 """Config for FgcmBuildStarsTableTask"""
156
157 referenceCCD = pexConfig.Field(
158 doc="Reference CCD for checking PSF and background",
159 dtype=int,
160 default=40,
161 )
162
163 def setDefaults(self):
164 super().setDefaults()
165
166 # The names here correspond to the post-transformed
167 # sourceTable_visit catalogs, which differ from the raw src
168 # catalogs. Therefore, all field and flag names cannot
169 # be derived from the base config class.
170 self.instFluxFieldinstFluxField = 'apFlux_12_0_instFlux'
171 self.localBackgroundFluxFieldlocalBackgroundFluxField = 'localBackground_instFlux'
174 self.psfCandidateNamepsfCandidateName = 'calib_psf_candidate'
175
176 sourceSelector = self.sourceSelector["science"]
177
178 fluxFlagName = self.instFluxFieldinstFluxField[0: -len('instFlux')] + 'flag'
179
180 sourceSelector.flags.bad = ['pixelFlags_edge',
181 'pixelFlags_interpolatedCenter',
182 'pixelFlags_saturatedCenter',
183 'pixelFlags_crCenter',
184 'pixelFlags_bad',
185 'pixelFlags_interpolated',
186 'pixelFlags_saturated',
187 'centroid_flag',
188 fluxFlagName]
189
191 localBackgroundFlagName = self.localBackgroundFluxFieldlocalBackgroundFluxField[0: -len('instFlux')] + 'flag'
192 sourceSelector.flags.bad.append(localBackgroundFlagName)
193
194 sourceSelector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
195 sourceSelector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
196
197 sourceSelector.isolated.parentName = 'parentSourceId'
198 sourceSelector.isolated.nChildName = 'deblend_nChild'
199
200 sourceSelector.requireFiniteRaDec.raColName = 'ra'
201 sourceSelector.requireFiniteRaDec.decColName = 'dec'
202
203 sourceSelector.unresolved.name = 'extendedness'
204
205 sourceSelector.doRequirePrimary = True
206
207
209 """
210 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
211 """
212 ConfigClass = FgcmBuildStarsTableConfig
213 _DefaultName = "fgcmBuildStarsTable"
214
215 canMultiprocess = False
216
217 def __init__(self, initInputs=None, **kwargs):
218 super().__init__(initInputs=initInputs, **kwargs)
219 if initInputs is not None:
220 self.sourceSchema = initInputs["sourceSchema"].schema
221
222 def runQuantum(self, butlerQC, inputRefs, outputRefs):
223 inputRefDict = butlerQC.get(inputRefs)
224
225 sourceTableHandles = inputRefDict['sourceTable_visit']
226
227 self.log.info("Running with %d sourceTable_visit handles",
228 len(sourceTableHandles))
229
230 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for
231 sourceTableHandle in sourceTableHandles}
232
233 if self.config.doReferenceMatches:
234 # Get the LUT handle
235 lutHandle = inputRefDict['fgcmLookUpTable']
236
237 # Prepare the reference catalog loader
238 refConfig = LoadReferenceObjectsConfig()
239 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
240 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
241 for ref in inputRefs.refCat],
242 refCats=butlerQC.get(inputRefs.refCat),
243 name=self.config.connections.refCat,
244 log=self.log,
245 config=refConfig)
246 self.makeSubtask('fgcmLoadReferenceCatalog',
247 refObjLoader=refObjLoader,
248 refCatName=self.config.connections.refCat)
249 else:
250 lutHandle = None
251
252 # Compute aperture radius if necessary. This is useful to do now before
253 # any heave lifting has happened (fail early).
254 calibFluxApertureRadius = None
255 if self.config.doSubtractLocalBackground:
256 try:
257 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
258 except RuntimeError as e:
259 raise RuntimeError("Could not determine aperture radius from %s. "
260 "Cannot use doSubtractLocalBackground." %
261 (self.config.instFluxField)) from e
262
263 visitSummaryHandles = inputRefDict['visitSummary']
264 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for
265 visitSummaryHandle in visitSummaryHandles}
266
267 camera = inputRefDict['camera']
268 groupedHandles = self._groupHandles(sourceTableHandleDict,
269 visitSummaryHandleDict)
270
271 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles)
272
273 rad = calibFluxApertureRadius
274 fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedHandles,
275 visitCat,
276 self.sourceSchema,
277 camera,
278 calibFluxApertureRadius=rad)
279
280 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
281 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
282
283 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
284 fgcmStarObservationCat,
285 lutHandle=lutHandle)
286
287 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
288 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
289 if fgcmRefCat is not None:
290 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
291
292 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
293 """Group sourceTable and visitSummary handles.
294
295 Parameters
296 ----------
297 sourceTableHandleDict : `dict` [`int`, `str`]
298 Dict of source tables, keyed by visit.
299 visitSummaryHandleDict : `dict` [int, `str`]
300 Dict of visit summary catalogs, keyed by visit.
301
302 Returns
303 -------
304 groupedHandles : `dict` [`int`, `list`]
305 Dictionary with sorted visit keys, and `list`s with
306 `lsst.daf.butler.DeferredDataSetHandle`. The first
307 item in the list will be the visitSummary ref, and
308 the second will be the source table ref.
309 """
310 groupedHandles = collections.defaultdict(list)
311 visits = sorted(sourceTableHandleDict.keys())
312
313 for visit in visits:
314 groupedHandles[visit] = [visitSummaryHandleDict[visit],
315 sourceTableHandleDict[visit]]
316
317 return groupedHandles
318
319 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat,
320 sourceSchema,
321 camera,
322 calibFluxApertureRadius=None):
323 startTime = time.time()
324
325 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
326 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
327
328 # To get the correct output schema, we use the legacy code.
329 # We are not actually using this mapper, except to grab the outputSchema
330 sourceMapper = self._makeSourceMapper(sourceSchema)
331 outputSchema = sourceMapper.getOutputSchema()
332
333 # Construct mapping from ccd number to index
334 ccdMapping = {}
335 for ccdIndex, detector in enumerate(camera):
336 ccdMapping[detector.getId()] = ccdIndex
337
338 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
339
340 fullCatalog = afwTable.BaseCatalog(outputSchema)
341
342 visitKey = outputSchema['visit'].asKey()
343 ccdKey = outputSchema['ccd'].asKey()
344 instMagKey = outputSchema['instMag'].asKey()
345 instMagErrKey = outputSchema['instMagErr'].asKey()
346 deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
347
348 # Prepare local background if desired
349 if self.config.doSubtractLocalBackground:
350 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
351
352 columns = None
353
354 k = 2.5/np.log(10.)
355
356 for counter, visit in enumerate(visitCat):
357 expTime = visit['exptime']
358
359 handle = groupedHandles[visit['visit']][-1]
360
361 if columns is None:
362 inColumns = handle.get(component='columns')
363 columns = self._get_sourceTable_visit_columns(inColumns)
364 df = handle.get(parameters={'columns': columns})
365
366 goodSrc = self.sourceSelector.selectSources(df)
367
368 # Need to add a selection based on the local background correction
369 # if necessary
370 if self.config.doSubtractLocalBackground:
371 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
372 use, = np.where((goodSrc.selected)
373 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
374 else:
375 use, = np.where(goodSrc.selected)
376
377 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
378 tempCat.resize(use.size)
379
380 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
381 tempCat['dec'][:] = np.deg2rad(df['dec'].values[use])
382 tempCat['x'][:] = df['x'].values[use]
383 tempCat['y'][:] = df['y'].values[use]
384 # The "visit" name in the parquet table is hard-coded.
385 tempCat[visitKey][:] = df['visit'].values[use]
386 tempCat[ccdKey][:] = df['detector'].values[use]
387 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use]
388
389 with warnings.catch_warnings():
390 # Ignore warnings, we will filter infinites and nans below
391 warnings.simplefilter("ignore")
392
393 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
394 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
395 / df[self.config.apertureInnerInstFluxField].values[use])
396 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
397 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
398 / df[self.config.apertureOuterInstFluxField].values[use])
399 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
400 # Set bad values to illegal values for fgcm.
401 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
402
403 if self.config.doSubtractLocalBackground:
404 # At the moment we only adjust the flux and not the flux
405 # error by the background because the error on
406 # base_LocalBackground_instFlux is the rms error in the
407 # background annulus, not the error on the mean in the
408 # background estimate (which is much smaller, by sqrt(n)
409 # pixels used to estimate the background, which we do not
410 # have access to in this task). In the default settings,
411 # the annulus is sufficiently large such that these
412 # additional errors are are negligibly small (much less
413 # than a mmag in quadrature).
414
415 # This is the difference between the mag with local background correction
416 # and the mag without local background correction.
417 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
418 - localBackground[use]) -
419 -2.5*np.log10(df[self.config.instFluxField].values[use]))
420 else:
421 tempCat['deltaMagBkg'][:] = 0.0
422
423 # Need to loop over ccds here
424 for detector in camera:
425 ccdId = detector.getId()
426 # used index for all observations with a given ccd
427 use2 = (tempCat[ccdKey] == ccdId)
428 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
429 tempCat['y'][use2])
430 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
431 * visit['scaling'][ccdMapping[ccdId]])
432 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
433
434 # Compute instMagErr from instFluxErr/instFlux, any scaling
435 # will cancel out.
436 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
437 / df[self.config.instFluxField].values[use])
438
439 # Apply the jacobian if configured
440 if self.config.doApplyWcsJacobian:
441 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
442
443 fullCatalog.extend(tempCat)
444
445 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
446 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
447
448 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
449 visit['sources_read'] = True
450
451 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
452 use.size, visit['visit'], visit['deltaAper'])
453
454 self.log.info("Found all good star observations in %.2f s" %
455 (time.time() - startTime))
456
457 return fullCatalog
458
459 def _get_sourceTable_visit_columns(self, inColumns):
460 """
461 Get the sourceTable_visit columns from the config.
462
463 Parameters
464 ----------
465 inColumns : `list`
466 List of columns available in the sourceTable_visit
467
468 Returns
469 -------
470 columns : `list`
471 List of columns to read from sourceTable_visit.
472 """
473 # Some names are hard-coded in the parquet table.
474 columns = ['visit', 'detector',
475 'ra', 'dec', 'x', 'y', self.config.psfCandidateName,
476 self.config.instFluxField, self.config.instFluxField + 'Err',
477 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
478 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
479 if self.sourceSelector.config.doFlags:
480 columns.extend(self.sourceSelector.config.flags.bad)
481 if self.sourceSelector.config.doUnresolved:
482 columns.append(self.sourceSelector.config.unresolved.name)
483 if self.sourceSelector.config.doIsolated:
484 columns.append(self.sourceSelector.config.isolated.parentName)
485 columns.append(self.sourceSelector.config.isolated.nChildName)
486 if self.sourceSelector.config.doRequirePrimary:
487 columns.append(self.sourceSelector.config.requirePrimary.primaryColName)
488 if self.config.doSubtractLocalBackground:
489 columns.append(self.config.localBackgroundFluxField)
490
491 return columns
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)