Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
32import time
34import numpy as np
35import collections
37import lsst.daf.persistence as dafPersist
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromDataRef
46from .utilities import lookupStaticCalibrations
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
53 defaultTemplates={}):
54 camera = connectionTypes.PrerequisiteInput(
55 doc="Camera instrument",
56 name="camera",
57 storageClass="Camera",
58 dimensions=("instrument",),
59 lookupFunction=lookupStaticCalibrations,
60 isCalibration=True,
61 )
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name="fgcmLookUpTable",
67 storageClass="Catalog",
68 dimensions=("instrument",),
69 deferLoad=True,
70 )
72 sourceSchema = connectionTypes.PrerequisiteInput(
73 doc="Schema for source catalogs",
74 name="src_schema",
75 storageClass="SourceCatalog",
76 deferLoad=True,
77 )
79 refCat = connectionTypes.PrerequisiteInput(
80 doc="Reference catalog to use for photometric calibration",
81 name="cal_ref_cat",
82 storageClass="SimpleCatalog",
83 dimensions=("skypix",),
84 deferLoad=True,
85 multiple=True,
86 )
88 sourceTable_visit = connectionTypes.Input(
89 doc="Source table in parquet format, per visit",
90 name="sourceTable_visit",
91 storageClass="DataFrame",
92 dimensions=("instrument", "visit"),
93 deferLoad=True,
94 multiple=True,
95 )
97 calexp = connectionTypes.Input(
98 doc="Calibrated exposures used for psf and metadata",
99 name="calexp",
100 storageClass="ExposureF",
101 dimensions=("instrument", "visit", "detector"),
102 deferLoad=True,
103 multiple=True,
104 )
106 background = connectionTypes.Input(
107 doc="Calexp background model",
108 name="calexpBackground",
109 storageClass="Background",
110 dimensions=("instrument", "visit", "detector"),
111 deferLoad=True,
112 multiple=True,
113 )
115 fgcmVisitCatalog = connectionTypes.Output(
116 doc="Catalog of visit information for fgcm",
117 name="fgcmVisitCatalog",
118 storageClass="Catalog",
119 dimensions=("instrument",),
120 )
122 fgcmStarObservations = connectionTypes.Output(
123 doc="Catalog of star observations for fgcm",
124 name="fgcmStarObservations",
125 storageClass="Catalog",
126 dimensions=("instrument",),
127 )
129 fgcmStarIds = connectionTypes.Output(
130 doc="Catalog of fgcm calibration star IDs",
131 name="fgcmStarIds",
132 storageClass="Catalog",
133 dimensions=("instrument",),
134 )
136 fgcmStarIndices = connectionTypes.Output(
137 doc="Catalog of fgcm calibration star indices",
138 name="fgcmStarIndices",
139 storageClass="Catalog",
140 dimensions=("instrument",),
141 )
143 fgcmReferenceStars = connectionTypes.Output(
144 doc="Catalog of fgcm-matched reference stars",
145 name="fgcmReferenceStars",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 )
150 def __init__(self, *, config=None):
151 super().__init__(config=config)
153 if not config.doReferenceMatches:
154 self.prerequisiteInputs.remove("refCat")
155 self.prerequisiteInputs.remove("fgcmLookUpTable")
157 if not config.doModelErrorsWithBackground:
158 self.inputs.remove("background")
160 if not config.doReferenceMatches:
161 self.outputs.remove("fgcmReferenceStars")
164class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig,
165 pipelineConnections=FgcmBuildStarsTableConnections):
166 """Config for FgcmBuildStarsTableTask"""
168 referenceCCD = pexConfig.Field(
169 doc="Reference CCD for checking PSF and background",
170 dtype=int,
171 default=40,
172 )
174 def setDefaults(self):
175 super().setDefaults()
177 # The names here correspond to the post-transformed
178 # sourceTable_visit catalogs, which differ from the raw src
179 # catalogs. Therefore, all field and flag names cannot
180 # be derived from the base config class.
181 self.instFluxField = 'ApFlux_12_0_instFlux'
182 self.localBackgroundFluxField = 'LocalBackground_instFlux'
183 self.apertureInnerInstFluxField = 'ApFlux_12_0_instFlux'
184 self.apertureOuterInstFluxField = 'ApFlux_17_0_instFlux'
185 self.psfCandidateName = 'Calib_psf_candidate'
187 sourceSelector = self.sourceSelector["science"]
189 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
191 sourceSelector.flags.bad = ['PixelFlags_edge',
192 'PixelFlags_interpolatedCenter',
193 'PixelFlags_saturatedCenter',
194 'PixelFlags_crCenter',
195 'PixelFlags_bad',
196 'PixelFlags_interpolated',
197 'PixelFlags_saturated',
198 'Centroid_flag',
199 fluxFlagName]
201 if self.doSubtractLocalBackground:
202 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
203 sourceSelector.flags.bad.append(localBackgroundFlagName)
205 sourceSelector.signalToNoise.fluxField = self.instFluxField
206 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
208 sourceSelector.isolated.parentName = 'parentSourceId'
209 sourceSelector.isolated.nChildName = 'Deblend_nChild'
211 sourceSelector.unresolved.name = 'extendedness'
214class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
215 """
216 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
217 """
218 ConfigClass = FgcmBuildStarsTableConfig
219 RunnerClass = FgcmBuildStarsRunner
220 _DefaultName = "fgcmBuildStarsTable"
222 canMultiprocess = False
224 def runQuantum(self, butlerQC, inputRefs, outputRefs):
225 inputRefDict = butlerQC.get(inputRefs)
227 dataRefs = inputRefDict['sourceTable_visit']
229 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefs)))
231 if self.config.doReferenceMatches:
232 # Get the LUT dataRef
233 lutDataRef = inputRefDict['fgcmLookUpTable']
235 # Prepare the refCat loader
236 refConfig = self.config.fgcmLoadReferenceCatalog.refObjLoader
237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
238 for ref in inputRefs.refCat],
239 refCats=butlerQC.get(inputRefs.refCat),
240 config=refConfig,
241 log=self.log)
242 self.makeSubtask('fgcmLoadReferenceCatalog', refObjLoader=refObjLoader)
243 else:
244 lutDataRef = None
246 # Compute aperture radius if necessary. This is useful to do now before
247 # any heave lifting has happened (fail early).
248 calibFluxApertureRadius = None
249 if self.config.doSubtractLocalBackground:
250 try:
251 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
252 self.config.instFluxField)
253 except RuntimeError as e:
254 raise RuntimeError("Could not determine aperture radius from %s. "
255 "Cannot use doSubtractLocalBackground." %
256 (self.config.instFluxField)) from e
258 calexpRefs = inputRefDict['calexp']
259 calexpDataRefDict = {(calexpRef.dataId.byName()['visit'],
260 calexpRef.dataId.byName()['detector']): calexpRef for
261 calexpRef in calexpRefs}
263 camera = inputRefDict['camera']
264 groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs,
265 calexpDataRefDict=calexpDataRefDict)
267 if self.config.doModelErrorsWithBackground:
268 bkgRefs = inputRefDict['background']
269 bkgDataRefDict = {(bkgRef.dataId.byName()['visit'],
270 bkgRef.dataId.byName()['detector']): bkgRef for
271 bkgRef in bkgRefs}
272 else:
273 bkgDataRefDict = None
275 # Gen3 does not currently allow "checkpoint" saving of datasets,
276 # so we need to have this all in one go.
277 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
278 bkgDataRefDict=bkgDataRefDict,
279 visitCatDataRef=None,
280 inVisitCat=None)
282 rad = calibFluxApertureRadius
283 sourceSchemaDataRef = inputRefDict['sourceSchema']
284 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
285 visitCat,
286 sourceSchemaDataRef,
287 camera,
288 calibFluxApertureRadius=rad,
289 starObsDataRef=None,
290 visitCatDataRef=None,
291 inStarObsCat=None)
293 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
294 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
296 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
297 fgcmStarObservationCat,
298 lutDataRef=lutDataRef)
300 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
301 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
302 if fgcmRefCat is not None:
303 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
305 @classmethod
306 def _makeArgumentParser(cls):
307 """Create an argument parser"""
308 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
309 parser.add_id_argument("--id", "sourceTable_visit", help="Data ID, e.g. --id visit=6789")
311 return parser
313 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
314 if (butler is None and calexpDataRefDict is None) or \
315 (butler is not None and calexpDataRefDict is not None):
316 raise RuntimeError("Must either set butler (Gen2) or dataRefDict (Gen3)")
318 self.log.info("Grouping dataRefs by %s", (self.config.visitDataRefName))
320 ccdIds = []
321 for detector in camera:
322 ccdIds.append(detector.getId())
323 # Insert our preferred referenceCCD first:
324 # It is fine that this is listed twice, because we only need
325 # the first calexp that is found.
326 ccdIds.insert(0, self.config.referenceCCD)
328 # The visitTable building code expects a dictionary of groupedDataRefs
329 # keyed by visit, the first element as the "primary" calexp dataRef.
330 # We then append the sourceTable_visit dataRef at the end for the
331 # code which does the data reading (fgcmMakeAllStarObservations).
333 groupedDataRefs = collections.defaultdict(list)
334 for dataRef in dataRefs:
335 visit = dataRef.dataId[self.config.visitDataRefName]
337 # Find an existing calexp (we need for psf and metadata)
338 # and make the relevant dataRef
339 for ccdId in ccdIds:
340 if butler is not None:
341 # Gen2 Mode
342 try:
343 calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit,
344 self.config.ccdDataRefName: ccdId})
345 except RuntimeError:
346 # Not found
347 continue
348 else:
349 # Gen3 mode
350 calexpRef = calexpDataRefDict.get((visit, ccdId))
351 if calexpRef is None:
352 continue
354 # It was found. Add and quit out, since we only
355 # need one calexp per visit.
356 groupedDataRefs[visit].append(calexpRef)
357 break
359 # And append this dataRef
360 groupedDataRefs[visit].append(dataRef)
362 # This should be sorted by visit (the key)
363 return dict(sorted(groupedDataRefs.items()))
365 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
366 sourceSchemaDataRef,
367 camera,
368 calibFluxApertureRadius=None,
369 visitCatDataRef=None,
370 starObsDataRef=None,
371 inStarObsCat=None):
372 startTime = time.time()
374 # If both dataRefs are None, then we assume the caller does not
375 # want to store checkpoint files. If both are set, we will
376 # do checkpoint files. And if only one is set, this is potentially
377 # unintentional and we will warn.
378 if (visitCatDataRef is not None and starObsDataRef is None
379 or visitCatDataRef is None and starObsDataRef is not None):
380 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
381 "no checkpoint files will be persisted.")
383 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
384 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
386 # To get the correct output schema, we use similar code as fgcmBuildStarsTask
387 # We are not actually using this mapper, except to grab the outputSchema
388 sourceSchema = sourceSchemaDataRef.get().schema
389 sourceMapper = self._makeSourceMapper(sourceSchema)
390 outputSchema = sourceMapper.getOutputSchema()
392 # Construct mapping from ccd number to index
393 ccdMapping = {}
394 for ccdIndex, detector in enumerate(camera):
395 ccdMapping[detector.getId()] = ccdIndex
397 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
399 if inStarObsCat is not None:
400 fullCatalog = inStarObsCat
401 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
402 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
403 if not comp1 or not comp2:
404 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
405 else:
406 fullCatalog = afwTable.BaseCatalog(outputSchema)
408 visitKey = outputSchema['visit'].asKey()
409 ccdKey = outputSchema['ccd'].asKey()
410 instMagKey = outputSchema['instMag'].asKey()
411 instMagErrKey = outputSchema['instMagErr'].asKey()
413 # Prepare local background if desired
414 if self.config.doSubtractLocalBackground:
415 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
417 # Determine which columns we need from the sourceTable_visit catalogs
418 columns = self._get_sourceTable_visit_columns()
420 k = 2.5/np.log(10.)
422 for counter, visit in enumerate(visitCat):
423 # Check if these sources have already been read and stored in the checkpoint file
424 if visit['sources_read']:
425 continue
427 expTime = visit['exptime']
429 dataRef = groupedDataRefs[visit['visit']][-1]
431 if isinstance(dataRef, dafPersist.ButlerDataRef):
432 srcTable = dataRef.get()
433 df = srcTable.toDataFrame(columns)
434 else:
435 df = dataRef.get(parameters={'columns': columns})
437 goodSrc = self.sourceSelector.selectSources(df)
439 # Need to add a selection based on the local background correction
440 # if necessary
441 if self.config.doSubtractLocalBackground:
442 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
443 use, = np.where((goodSrc.selected)
444 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
445 else:
446 use, = np.where(goodSrc.selected)
448 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
449 tempCat.resize(use.size)
451 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
452 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
453 tempCat['x'][:] = df['x'].values[use]
454 tempCat['y'][:] = df['y'].values[use]
455 # These "visit" and "ccd" names in the parquet tables are
456 # hard-coded.
457 tempCat[visitKey][:] = df['visit'].values[use]
458 tempCat[ccdKey][:] = df['ccd'].values[use]
459 tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use]
461 if self.config.doSubtractLocalBackground:
462 # At the moment we only adjust the flux and not the flux
463 # error by the background because the error on
464 # base_LocalBackground_instFlux is the rms error in the
465 # background annulus, not the error on the mean in the
466 # background estimate (which is much smaller, by sqrt(n)
467 # pixels used to estimate the background, which we do not
468 # have access to in this task). In the default settings,
469 # the annulus is sufficiently large such that these
470 # additional errors are are negligibly small (much less
471 # than a mmag in quadrature).
473 # This is the difference between the mag with local background correction
474 # and the mag without local background correction.
475 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
476 - localBackground[use]) -
477 -2.5*np.log10(df[self.config.instFluxField].values[use]))
478 else:
479 tempCat['deltaMagBkg'][:] = 0.0
481 # Need to loop over ccds here
482 for detector in camera:
483 ccdId = detector.getId()
484 # used index for all observations with a given ccd
485 use2 = (tempCat[ccdKey] == ccdId)
486 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
487 tempCat['y'][use2])
488 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
489 * visit['scaling'][ccdMapping[ccdId]])
490 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
492 # Compute instMagErr from instFluxErr/instFlux, any scaling
493 # will cancel out.
494 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use]
495 / df[self.config.instFluxField].values[use])
497 # Apply the jacobian if configured
498 if self.config.doApplyWcsJacobian:
499 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
501 fullCatalog.extend(tempCat)
503 # Now do the aperture information
504 with np.warnings.catch_warnings():
505 # Ignore warnings, we will filter infinites and nans below
506 np.warnings.simplefilter("ignore")
508 instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
509 instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use]
510 / df[self.config.apertureInnerInstFluxField].values[use])
511 instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
512 instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use]
513 / df[self.config.apertureOuterInstFluxField].values[use])
515 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
516 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
518 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
519 visit['sources_read'] = True
521 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
522 use.size, visit['visit'], visit['deltaAper'])
524 if ((counter % self.config.nVisitsPerCheckpoint) == 0
525 and starObsDataRef is not None and visitCatDataRef is not None):
526 # We need to persist both the stars and the visit catalog which gets
527 # additional metadata from each visit.
528 starObsDataRef.put(fullCatalog)
529 visitCatDataRef.put(visitCat)
531 self.log.info("Found all good star observations in %.2f s" %
532 (time.time() - startTime))
534 return fullCatalog
536 def _get_sourceTable_visit_columns(self):
537 """
538 Get the sourceTable_visit columns from the config.
540 Returns
541 -------
542 columns : `list`
543 List of columns to read from sourceTable_visit
544 """
545 # These "visit" and "ccd" names in the parquet tables are hard-coded.
546 columns = ['visit', 'ccd',
547 'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
548 self.config.instFluxField, self.config.instFluxField + 'Err',
549 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
550 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
551 if self.sourceSelector.config.doFlags:
552 columns.extend(self.sourceSelector.config.flags.bad)
553 if self.sourceSelector.config.doUnresolved:
554 columns.append(self.sourceSelector.config.unresolved.name)
555 if self.sourceSelector.config.doIsolated:
556 columns.append(self.sourceSelector.config.isolated.parentName)
557 columns.append(self.sourceSelector.config.isolated.nChildName)
558 if self.config.doSubtractLocalBackground:
559 columns.append(self.config.localBackgroundFluxField)
561 return columns