Coverage for python/lsst/fgcmcal/fgcmBuildStars.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM.
25This task finds all the visits and calexps in a repository (or a subset
26based on command line parameters) and extract all the potential calibration
27stars for input into fgcm. This task additionally uses fgcm to match
28star observations into unique stars, and performs as much cleaning of
29the input catalog as possible.
30"""
32import time
34import numpy as np
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38import lsst.afw.table as afwTable
40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41from .utilities import computeApproxPixelAreaFields
43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase):
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc="Reference CCD for scanning visits",
51 dtype=int,
52 default=13,
53 )
54 checkAllCcds = pexConfig.Field(
55 doc=("Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
59 dtype=bool,
60 default=True,
61 )
63 def setDefaults(self):
64 super().setDefaults()
66 sourceSelector = self.sourceSelector["science"]
68 # The names here correspond to raw src catalogs, which differ
69 # from the post-transformed sourceTable_visit catalogs.
70 # Therefore, field and flag names cannot be easily
71 # derived from the base config class.
72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
80 'slot_Centroid_flag',
81 fluxFlagName]
83 if self.doSubtractLocalBackground:
84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
87 sourceSelector.signalToNoise.fluxField = self.instFluxField
88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask):
92 """
93 Build stars for the FGCM global calibration, using src catalogs.
94 """
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName = "fgcmBuildStars"
99 @classmethod
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
102 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
105 return parser
107 def findAndGroupDataRefs(self, butler, dataRefs):
108 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
110 camera = butler.get('camera')
112 ccdIds = []
113 for detector in camera:
114 ccdIds.append(detector.getId())
116 # TODO: related to DM-13730, this dance of looking for source visits
117 # will be unnecessary with Gen3 Butler. This should be part of
118 # DM-13730.
120 nVisits = 0
122 groupedDataRefs = {}
123 for dataRef in dataRefs:
124 visit = dataRef.dataId[self.config.visitDataRefName]
125 # If we don't have the dataset, just continue
126 if not dataRef.datasetExists(datasetType='src'):
127 continue
128 # If we need to check all ccds, do it here
129 if self.config.checkAllCcds:
130 if visit in groupedDataRefs:
131 # We already have found this visit
132 continue
133 dataId = dataRef.dataId.copy()
134 # For each ccd we must check that a valid source catalog exists.
135 for ccdId in ccdIds:
136 dataId[self.config.ccdDataRefName] = ccdId
137 if butler.datasetExists('src', dataId=dataId):
138 goodDataRef = butler.dataRef('src', dataId=dataId)
139 if visit in groupedDataRefs:
140 if (goodDataRef.dataId[self.config.ccdDataRefName] not in
141 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
142 groupedDataRefs[visit].append(goodDataRef)
143 else:
144 # This is a new visit
145 nVisits += 1
146 groupedDataRefs[visit] = [goodDataRef]
147 else:
148 # We have already confirmed that the dataset exists, so no need
149 # to check here.
150 if visit in groupedDataRefs:
151 if (dataRef.dataId[self.config.ccdDataRefName] not in
152 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
153 groupedDataRefs[visit].append(dataRef)
154 else:
155 # This is a new visit
156 nVisits += 1
157 groupedDataRefs[visit] = [dataRef]
159 if (nVisits % 100) == 0 and nVisits > 0:
160 self.log.info("Found %d unique %ss..." % (nVisits,
161 self.config.visitDataRefName))
163 self.log.info("Found %d unique %ss total." % (nVisits,
164 self.config.visitDataRefName))
166 # Put them in ccd order, with the reference ccd first (if available)
167 def ccdSorter(dataRef):
168 ccdId = dataRef.dataId[self.config.ccdDataRefName]
169 if ccdId == self.config.referenceCCD:
170 return -100
171 else:
172 return ccdId
174 # If we did not check all ccds, put them in ccd order
175 if not self.config.checkAllCcds:
176 for visit in groupedDataRefs:
177 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
179 return groupedDataRefs
181 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
182 calibFluxApertureRadius=None,
183 visitCatDataRef=None,
184 starObsDataRef=None,
185 inStarObsCat=None):
186 startTime = time.time()
188 # If both dataRefs are None, then we assume the caller does not
189 # want to store checkpoint files. If both are set, we will
190 # do checkpoint files. And if only one is set, this is potentially
191 # unintentional and we will warn.
192 if (visitCatDataRef is not None and starObsDataRef is None or
193 visitCatDataRef is None and starObsDataRef is not None):
194 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
195 "no checkpoint files will be persisted.")
197 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
198 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
200 # create our source schema. Use the first valid dataRef
201 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202 sourceSchema = dataRef.get('src_schema', immediate=True).schema
204 # Construct a mapping from ccd number to index
205 camera = dataRef.get('camera')
206 ccdMapping = {}
207 for ccdIndex, detector in enumerate(camera):
208 ccdMapping[detector.getId()] = ccdIndex
210 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
212 sourceMapper = self._makeSourceMapper(sourceSchema)
214 # We also have a temporary catalog that will accumulate aperture measurements
215 aperMapper = self._makeAperMapper(sourceSchema)
217 outputSchema = sourceMapper.getOutputSchema()
219 if inStarObsCat is not None:
220 fullCatalog = inStarObsCat
221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223 if not comp1 or not comp2:
224 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
225 else:
226 fullCatalog = afwTable.BaseCatalog(outputSchema)
228 # FGCM will provide relative calibration for the flux in config.instFluxField
230 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
232 visitKey = outputSchema['visit'].asKey()
233 ccdKey = outputSchema['ccd'].asKey()
234 instMagKey = outputSchema['instMag'].asKey()
235 instMagErrKey = outputSchema['instMagErr'].asKey()
236 deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
238 # Prepare local background if desired
239 if self.config.doSubtractLocalBackground:
240 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
241 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
243 aperOutputSchema = aperMapper.getOutputSchema()
245 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
246 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
247 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
248 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
249 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
250 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
251 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
252 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
254 k = 2.5/np.log(10.)
256 # loop over visits
257 for ctr, visit in enumerate(visitCat):
258 if visit['sources_read']:
259 continue
261 expTime = visit['exptime']
263 nStarInVisit = 0
265 # Reset the aperture catalog (per visit)
266 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
268 for dataRef in groupedDataRefs[visit['visit']]:
270 ccdId = dataRef.dataId[self.config.ccdDataRefName]
272 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
273 goodSrc = self.sourceSelector.selectSources(sources)
275 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
276 tempCat.reserve(goodSrc.selected.sum())
277 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
278 tempCat[visitKey][:] = visit['visit']
279 tempCat[ccdKey][:] = ccdId
281 # Compute "instrumental magnitude" by scaling flux with exposure time.
282 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
283 visit['scaling'][ccdMapping[ccdId]])
284 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
286 # Compute the change in magnitude from the background offset
287 if self.config.doSubtractLocalBackground:
288 # At the moment we only adjust the flux and not the flux
289 # error by the background because the error on
290 # base_LocalBackground_instFlux is the rms error in the
291 # background annulus, not the error on the mean in the
292 # background estimate (which is much smaller, by sqrt(n)
293 # pixels used to estimate the background, which we do not
294 # have access to in this task). In the default settings,
295 # the annulus is sufficiently large such that these
296 # additional errors are are negligibly small (much less
297 # than a mmag in quadrature).
299 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
301 # This is the difference between the mag with background correction
302 # and the mag without background correction.
303 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected] -
304 localBackground[goodSrc.selected]) -
305 -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
306 else:
307 tempCat[deltaMagBkgKey][:] = 0.0
309 # Compute instMagErr from instFluxErr/instFlux, any scaling
310 # will cancel out.
312 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
313 sources[instFluxKey][goodSrc.selected])
315 # Compute the jacobian from an approximate PixelAreaBoundedField
316 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
317 tempCat['y'])
319 # Apply the jacobian if configured
320 if self.config.doApplyWcsJacobian:
321 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
323 fullCatalog.extend(tempCat)
325 # And the aperture information
326 # This does not need the jacobian because it is all locally relative
327 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
328 tempAperCat.reserve(goodSrc.selected.sum())
329 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
331 with np.warnings.catch_warnings():
332 # Ignore warnings, we will filter infinities and
333 # nans below.
334 np.warnings.simplefilter("ignore")
336 tempAperCat[instMagInKey][:] = -2.5*np.log10(
337 sources[instFluxAperInKey][goodSrc.selected])
338 tempAperCat[instMagErrInKey][:] = k*(
339 sources[instFluxErrAperInKey][goodSrc.selected] /
340 sources[instFluxAperInKey][goodSrc.selected])
341 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
342 sources[instFluxAperOutKey][goodSrc.selected])
343 tempAperCat[instMagErrOutKey][:] = k*(
344 sources[instFluxErrAperOutKey][goodSrc.selected] /
345 sources[instFluxAperOutKey][goodSrc.selected])
347 aperVisitCatalog.extend(tempAperCat)
349 nStarInVisit += len(tempCat)
351 # Compute the median delta-aper
352 if not aperVisitCatalog.isContiguous():
353 aperVisitCatalog = aperVisitCatalog.copy(deep=True)
355 instMagIn = aperVisitCatalog[instMagInKey]
356 instMagErrIn = aperVisitCatalog[instMagErrInKey]
357 instMagOut = aperVisitCatalog[instMagOutKey]
358 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
360 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
361 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
363 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
364 visit['sources_read'] = True
366 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
367 (nStarInVisit, visit['visit'], visit['deltaAper']))
369 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and
370 starObsDataRef is not None and visitCatDataRef is not None):
371 # We need to persist both the stars and the visit catalog which gets
372 # additional metadata from each visit.
373 starObsDataRef.put(fullCatalog)
374 visitCatDataRef.put(visitCat)
376 self.log.info("Found all good star observations in %.2f s" %
377 (time.time() - startTime))
379 return fullCatalog
381 def _makeAperMapper(self, sourceSchema):
382 """
383 Make a schema mapper for fgcm aperture measurements
385 Parameters
386 ----------
387 sourceSchema: `afwTable.Schema`
388 Default source schema from the butler
390 Returns
391 -------
392 aperMapper: `afwTable.schemaMapper`
393 Mapper to the FGCM aperture schema
394 """
396 aperMapper = afwTable.SchemaMapper(sourceSchema)
397 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
398 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
399 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
400 doc="Magnitude at inner aperture")
401 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
402 doc="Magnitude error at inner aperture")
403 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
404 doc="Magnitude at outer aperture")
405 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
406 doc="Magnitude error at outer aperture")
408 return aperMapper