Coverage for python/lsst/fgcmcal/fgcmBuildStars.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM.
25This task finds all the visits and calexps in a repository (or a subset
26based on command line parameters) and extract all the potential calibration
27stars for input into fgcm. This task additionally uses fgcm to match
28star observations into unique stars, and performs as much cleaning of
29the input catalog as possible.
30"""
32import time
34import numpy as np
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38import lsst.afw.table as afwTable
40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41from .utilities import computeApproxPixelAreaFields
43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase):
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc="Reference CCD for scanning visits",
51 dtype=int,
52 default=13,
53 )
54 checkAllCcds = pexConfig.Field(
55 doc=("Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
59 dtype=bool,
60 default=True,
61 )
63 def setDefaults(self):
64 super().setDefaults()
66 sourceSelector = self.sourceSelector["science"]
68 # The names here correspond to raw src catalogs, which differ
69 # from the post-transformed sourceTable_visit catalogs.
70 # Therefore, field and flag names cannot be easily
71 # derived from the base config class.
72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
80 'slot_Centroid_flag',
81 fluxFlagName]
83 if self.doSubtractLocalBackground:
84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
87 sourceSelector.signalToNoise.fluxField = self.instFluxField
88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask):
92 """
93 Build stars for the FGCM global calibration, using src catalogs.
94 """
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName = "fgcmBuildStars"
99 @classmethod
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
102 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
105 return parser
107 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
108 if butler is None:
109 raise RuntimeError("Gen2 _findAndGroupDataRefs must be called with a butler.")
110 if calexpDataRefDict is not None:
111 self.log.warn("Ignoring calexpDataRefDict in gen2 _findAndGroupDataRefs")
113 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
115 ccdIds = []
116 for detector in camera:
117 ccdIds.append(detector.getId())
119 # TODO: related to DM-13730, this dance of looking for source visits
120 # will be unnecessary with Gen3 Butler. This should be part of
121 # DM-13730.
123 nVisits = 0
125 groupedDataRefs = {}
126 for dataRef in dataRefs:
127 visit = dataRef.dataId[self.config.visitDataRefName]
128 # If we don't have the dataset, just continue
129 if not dataRef.datasetExists(datasetType='src'):
130 continue
131 # If we need to check all ccds, do it here
132 if self.config.checkAllCcds:
133 if visit in groupedDataRefs:
134 # We already have found this visit
135 continue
136 dataId = dataRef.dataId.copy()
137 # For each ccd we must check that a valid source catalog exists.
138 for ccdId in ccdIds:
139 dataId[self.config.ccdDataRefName] = ccdId
140 if butler.datasetExists('src', dataId=dataId):
141 goodDataRef = butler.dataRef('src', dataId=dataId)
142 if visit in groupedDataRefs:
143 if (goodDataRef.dataId[self.config.ccdDataRefName] not in
144 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
145 groupedDataRefs[visit].append(goodDataRef)
146 else:
147 # This is a new visit
148 nVisits += 1
149 groupedDataRefs[visit] = [goodDataRef]
150 else:
151 # We have already confirmed that the dataset exists, so no need
152 # to check here.
153 if visit in groupedDataRefs:
154 if (dataRef.dataId[self.config.ccdDataRefName] not in
155 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
156 groupedDataRefs[visit].append(dataRef)
157 else:
158 # This is a new visit
159 nVisits += 1
160 groupedDataRefs[visit] = [dataRef]
162 if (nVisits % 100) == 0 and nVisits > 0:
163 self.log.info("Found %d unique %ss..." % (nVisits,
164 self.config.visitDataRefName))
166 self.log.info("Found %d unique %ss total." % (nVisits,
167 self.config.visitDataRefName))
169 # Put them in ccd order, with the reference ccd first (if available)
170 def ccdSorter(dataRef):
171 ccdId = dataRef.dataId[self.config.ccdDataRefName]
172 if ccdId == self.config.referenceCCD:
173 return -100
174 else:
175 return ccdId
177 # If we did not check all ccds, put them in ccd order
178 if not self.config.checkAllCcds:
179 for visit in groupedDataRefs:
180 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
182 # This should be sorted by visit (the key)
183 return dict(sorted(groupedDataRefs.items()))
185 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
186 srcSchemaDataRef,
187 camera,
188 calibFluxApertureRadius=None,
189 visitCatDataRef=None,
190 starObsDataRef=None,
191 inStarObsCat=None):
192 startTime = time.time()
194 # If both dataRefs are None, then we assume the caller does not
195 # want to store checkpoint files. If both are set, we will
196 # do checkpoint files. And if only one is set, this is potentially
197 # unintentional and we will warn.
198 if (visitCatDataRef is not None and starObsDataRef is None
199 or visitCatDataRef is None and starObsDataRef is not None):
200 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
201 "no checkpoint files will be persisted.")
203 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
204 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
206 # create our source schema. Use the first valid dataRef
207 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
208 sourceSchema = dataRef.get('src_schema', immediate=True).schema
210 # Construct a mapping from ccd number to index
211 ccdMapping = {}
212 for ccdIndex, detector in enumerate(camera):
213 ccdMapping[detector.getId()] = ccdIndex
215 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
217 sourceMapper = self._makeSourceMapper(sourceSchema)
219 # We also have a temporary catalog that will accumulate aperture measurements
220 aperMapper = self._makeAperMapper(sourceSchema)
222 outputSchema = sourceMapper.getOutputSchema()
224 if inStarObsCat is not None:
225 fullCatalog = inStarObsCat
226 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
227 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
228 if not comp1 or not comp2:
229 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
230 else:
231 fullCatalog = afwTable.BaseCatalog(outputSchema)
233 # FGCM will provide relative calibration for the flux in config.instFluxField
235 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
236 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
237 visitKey = outputSchema['visit'].asKey()
238 ccdKey = outputSchema['ccd'].asKey()
239 instMagKey = outputSchema['instMag'].asKey()
240 instMagErrKey = outputSchema['instMagErr'].asKey()
241 deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
243 # Prepare local background if desired
244 if self.config.doSubtractLocalBackground:
245 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
246 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
248 aperOutputSchema = aperMapper.getOutputSchema()
250 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
251 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
252 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
253 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
254 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
255 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
256 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
257 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
259 k = 2.5/np.log(10.)
261 # loop over visits
262 for ctr, visit in enumerate(visitCat):
263 if visit['sources_read']:
264 continue
266 expTime = visit['exptime']
268 nStarInVisit = 0
270 # Reset the aperture catalog (per visit)
271 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
273 for dataRef in groupedDataRefs[visit['visit']]:
275 ccdId = dataRef.dataId[self.config.ccdDataRefName]
277 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
278 goodSrc = self.sourceSelector.selectSources(sources)
280 # Need to add a selection based on the local background correction
281 # if necessary
282 if self.config.doSubtractLocalBackground:
283 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
285 bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0)
286 goodSrc.selected[bad] = False
288 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
289 tempCat.reserve(goodSrc.selected.sum())
290 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
291 tempCat[visitKey][:] = visit['visit']
292 tempCat[ccdKey][:] = ccdId
294 # Compute "instrumental magnitude" by scaling flux with exposure time.
295 scaledInstFlux = (sources[instFluxKey][goodSrc.selected]
296 * visit['scaling'][ccdMapping[ccdId]])
297 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
299 # Compute the change in magnitude from the background offset
300 if self.config.doSubtractLocalBackground:
301 # At the moment we only adjust the flux and not the flux
302 # error by the background because the error on
303 # base_LocalBackground_instFlux is the rms error in the
304 # background annulus, not the error on the mean in the
305 # background estimate (which is much smaller, by sqrt(n)
306 # pixels used to estimate the background, which we do not
307 # have access to in this task). In the default settings,
308 # the annulus is sufficiently large such that these
309 # additional errors are are negligibly small (much less
310 # than a mmag in quadrature).
312 # This is the difference between the mag with background correction
313 # and the mag without background correction.
314 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected]
315 - localBackground[goodSrc.selected]) -
316 -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
317 else:
318 tempCat[deltaMagBkgKey][:] = 0.0
320 # Compute instMagErr from instFluxErr/instFlux, any scaling
321 # will cancel out.
323 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected]
324 / sources[instFluxKey][goodSrc.selected])
326 # Compute the jacobian from an approximate PixelAreaBoundedField
327 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
328 tempCat['y'])
330 # Apply the jacobian if configured
331 if self.config.doApplyWcsJacobian:
332 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
334 fullCatalog.extend(tempCat)
336 # And the aperture information
337 # This does not need the jacobian because it is all locally relative
338 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
339 tempAperCat.reserve(goodSrc.selected.sum())
340 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
342 with np.warnings.catch_warnings():
343 # Ignore warnings, we will filter infinities and
344 # nans below.
345 np.warnings.simplefilter("ignore")
347 tempAperCat[instMagInKey][:] = -2.5*np.log10(
348 sources[instFluxAperInKey][goodSrc.selected])
349 tempAperCat[instMagErrInKey][:] = k*(
350 sources[instFluxErrAperInKey][goodSrc.selected]
351 / sources[instFluxAperInKey][goodSrc.selected])
352 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
353 sources[instFluxAperOutKey][goodSrc.selected])
354 tempAperCat[instMagErrOutKey][:] = k*(
355 sources[instFluxErrAperOutKey][goodSrc.selected]
356 / sources[instFluxAperOutKey][goodSrc.selected])
358 aperVisitCatalog.extend(tempAperCat)
360 nStarInVisit += len(tempCat)
362 # Compute the median delta-aper
363 if not aperVisitCatalog.isContiguous():
364 aperVisitCatalog = aperVisitCatalog.copy(deep=True)
366 instMagIn = aperVisitCatalog[instMagInKey]
367 instMagErrIn = aperVisitCatalog[instMagErrInKey]
368 instMagOut = aperVisitCatalog[instMagOutKey]
369 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
371 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
372 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
374 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
375 visit['sources_read'] = True
377 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
378 (nStarInVisit, visit['visit'], visit['deltaAper']))
380 if ((ctr % self.config.nVisitsPerCheckpoint) == 0
381 and starObsDataRef is not None and visitCatDataRef is not None):
382 # We need to persist both the stars and the visit catalog which gets
383 # additional metadata from each visit.
384 starObsDataRef.put(fullCatalog)
385 visitCatDataRef.put(visitCat)
387 self.log.info("Found all good star observations in %.2f s" %
388 (time.time() - startTime))
390 return fullCatalog
392 def _makeAperMapper(self, sourceSchema):
393 """
394 Make a schema mapper for fgcm aperture measurements
396 Parameters
397 ----------
398 sourceSchema: `afwTable.Schema`
399 Default source schema from the butler
401 Returns
402 -------
403 aperMapper: `afwTable.schemaMapper`
404 Mapper to the FGCM aperture schema
405 """
407 aperMapper = afwTable.SchemaMapper(sourceSchema)
408 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
409 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
410 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
411 doc="Magnitude at inner aperture")
412 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
413 doc="Magnitude error at inner aperture")
414 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
415 doc="Magnitude at outer aperture")
416 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
417 doc="Magnitude error at outer aperture")
419 return aperMapper