Coverage for python/lsst/fgcmcal/fgcmBuildStars.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM.
25This task finds all the visits and calexps in a repository (or a subset
26based on command line parameters) and extract all the potential calibration
27stars for input into fgcm. This task additionally uses fgcm to match
28star observations into unique stars, and performs as much cleaning of
29the input catalog as possible.
30"""
32import time
34import numpy as np
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38import lsst.afw.table as afwTable
40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41from .utilities import computeApproxPixelAreaFields
43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase):
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc="Reference CCD for scanning visits",
51 dtype=int,
52 default=13,
53 )
54 checkAllCcds = pexConfig.Field(
55 doc=("Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
59 dtype=bool,
60 default=True,
61 )
63 def setDefaults(self):
64 super().setDefaults()
66 sourceSelector = self.sourceSelector["science"]
68 # The names here correspond to raw src catalogs, which differ
69 # from the post-transformed sourceTable_visit catalogs.
70 # Therefore, field and flag names cannot be easily
71 # derived from the base config class.
72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
80 'slot_Centroid_flag',
81 fluxFlagName]
83 if self.doSubtractLocalBackground:
84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
87 sourceSelector.signalToNoise.fluxField = self.instFluxField
88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask):
92 """
93 Build stars for the FGCM global calibration, using src catalogs.
94 """
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName = "fgcmBuildStars"
99 @classmethod
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
102 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
105 return parser
107 def findAndGroupDataRefs(self, butler, dataRefs):
108 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
110 camera = butler.get('camera')
112 ccdIds = []
113 for detector in camera:
114 ccdIds.append(detector.getId())
116 # TODO: related to DM-13730, this dance of looking for source visits
117 # will be unnecessary with Gen3 Butler. This should be part of
118 # DM-13730.
120 nVisits = 0
122 groupedDataRefs = {}
123 for dataRef in dataRefs:
124 visit = dataRef.dataId[self.config.visitDataRefName]
125 # If we don't have the dataset, just continue
126 if not dataRef.datasetExists(datasetType='src'):
127 continue
128 # If we need to check all ccds, do it here
129 if self.config.checkAllCcds:
130 if visit in groupedDataRefs:
131 # We already have found this visit
132 continue
133 dataId = dataRef.dataId.copy()
134 # For each ccd we must check that a valid source catalog exists.
135 for ccdId in ccdIds:
136 dataId[self.config.ccdDataRefName] = ccdId
137 if butler.datasetExists('src', dataId=dataId):
138 goodDataRef = butler.dataRef('src', dataId=dataId)
139 if visit in groupedDataRefs:
140 if (goodDataRef.dataId[self.config.ccdDataRefName] not in
141 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
142 groupedDataRefs[visit].append(goodDataRef)
143 else:
144 # This is a new visit
145 nVisits += 1
146 groupedDataRefs[visit] = [goodDataRef]
147 else:
148 # We have already confirmed that the dataset exists, so no need
149 # to check here.
150 if visit in groupedDataRefs:
151 if (dataRef.dataId[self.config.ccdDataRefName] not in
152 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
153 groupedDataRefs[visit].append(dataRef)
154 else:
155 # This is a new visit
156 nVisits += 1
157 groupedDataRefs[visit] = [dataRef]
159 if (nVisits % 100) == 0 and nVisits > 0:
160 self.log.info("Found %d unique %ss..." % (nVisits,
161 self.config.visitDataRefName))
163 self.log.info("Found %d unique %ss total." % (nVisits,
164 self.config.visitDataRefName))
166 # Put them in ccd order, with the reference ccd first (if available)
167 def ccdSorter(dataRef):
168 ccdId = dataRef.dataId[self.config.ccdDataRefName]
169 if ccdId == self.config.referenceCCD:
170 return -100
171 else:
172 return ccdId
174 # If we did not check all ccds, put them in ccd order
175 if not self.config.checkAllCcds:
176 for visit in groupedDataRefs:
177 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
179 return groupedDataRefs
181 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
182 calibFluxApertureRadius=None,
183 visitCatDataRef=None,
184 starObsDataRef=None,
185 inStarObsCat=None):
186 startTime = time.time()
188 # If both dataRefs are None, then we assume the caller does not
189 # want to store checkpoint files. If both are set, we will
190 # do checkpoint files. And if only one is set, this is potentially
191 # unintentional and we will warn.
192 if (visitCatDataRef is not None and starObsDataRef is None or
193 visitCatDataRef is None and starObsDataRef is not None):
194 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
195 "no checkpoint files will be persisted.")
197 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
198 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
200 # create our source schema. Use the first valid dataRef
201 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202 sourceSchema = dataRef.get('src_schema', immediate=True).schema
204 # Construct a mapping from ccd number to index
205 camera = dataRef.get('camera')
206 ccdMapping = {}
207 for ccdIndex, detector in enumerate(camera):
208 ccdMapping[detector.getId()] = ccdIndex
210 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
212 sourceMapper = self._makeSourceMapper(sourceSchema)
214 # We also have a temporary catalog that will accumulate aperture measurements
215 aperMapper = self._makeAperMapper(sourceSchema)
217 outputSchema = sourceMapper.getOutputSchema()
219 if inStarObsCat is not None:
220 fullCatalog = inStarObsCat
221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223 if not comp1 or not comp2:
224 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
225 else:
226 fullCatalog = afwTable.BaseCatalog(outputSchema)
228 # FGCM will provide relative calibration for the flux in config.instFluxField
230 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
232 visitKey = outputSchema['visit'].asKey()
233 ccdKey = outputSchema['ccd'].asKey()
234 instMagKey = outputSchema['instMag'].asKey()
235 instMagErrKey = outputSchema['instMagErr'].asKey()
236 deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
238 # Prepare local background if desired
239 if self.config.doSubtractLocalBackground:
240 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
241 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
243 aperOutputSchema = aperMapper.getOutputSchema()
245 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
246 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
247 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
248 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
249 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
250 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
251 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
252 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
254 k = 2.5/np.log(10.)
256 # loop over visits
257 for ctr, visit in enumerate(visitCat):
258 if visit['sources_read']:
259 continue
261 expTime = visit['exptime']
263 nStarInVisit = 0
265 # Reset the aperture catalog (per visit)
266 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
268 for dataRef in groupedDataRefs[visit['visit']]:
270 ccdId = dataRef.dataId[self.config.ccdDataRefName]
272 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
273 goodSrc = self.sourceSelector.selectSources(sources)
275 # Need to add a selection based on the local background correction
276 # if necessary
277 if self.config.doSubtractLocalBackground:
278 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
280 bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0)
281 goodSrc.selected[bad] = False
283 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
284 tempCat.reserve(goodSrc.selected.sum())
285 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
286 tempCat[visitKey][:] = visit['visit']
287 tempCat[ccdKey][:] = ccdId
289 # Compute "instrumental magnitude" by scaling flux with exposure time.
290 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
291 visit['scaling'][ccdMapping[ccdId]])
292 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
294 # Compute the change in magnitude from the background offset
295 if self.config.doSubtractLocalBackground:
296 # At the moment we only adjust the flux and not the flux
297 # error by the background because the error on
298 # base_LocalBackground_instFlux is the rms error in the
299 # background annulus, not the error on the mean in the
300 # background estimate (which is much smaller, by sqrt(n)
301 # pixels used to estimate the background, which we do not
302 # have access to in this task). In the default settings,
303 # the annulus is sufficiently large such that these
304 # additional errors are are negligibly small (much less
305 # than a mmag in quadrature).
307 # This is the difference between the mag with background correction
308 # and the mag without background correction.
309 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected] -
310 localBackground[goodSrc.selected]) -
311 -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
312 else:
313 tempCat[deltaMagBkgKey][:] = 0.0
315 # Compute instMagErr from instFluxErr/instFlux, any scaling
316 # will cancel out.
318 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
319 sources[instFluxKey][goodSrc.selected])
321 # Compute the jacobian from an approximate PixelAreaBoundedField
322 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
323 tempCat['y'])
325 # Apply the jacobian if configured
326 if self.config.doApplyWcsJacobian:
327 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
329 fullCatalog.extend(tempCat)
331 # And the aperture information
332 # This does not need the jacobian because it is all locally relative
333 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
334 tempAperCat.reserve(goodSrc.selected.sum())
335 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
337 with np.warnings.catch_warnings():
338 # Ignore warnings, we will filter infinities and
339 # nans below.
340 np.warnings.simplefilter("ignore")
342 tempAperCat[instMagInKey][:] = -2.5*np.log10(
343 sources[instFluxAperInKey][goodSrc.selected])
344 tempAperCat[instMagErrInKey][:] = k*(
345 sources[instFluxErrAperInKey][goodSrc.selected] /
346 sources[instFluxAperInKey][goodSrc.selected])
347 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
348 sources[instFluxAperOutKey][goodSrc.selected])
349 tempAperCat[instMagErrOutKey][:] = k*(
350 sources[instFluxErrAperOutKey][goodSrc.selected] /
351 sources[instFluxAperOutKey][goodSrc.selected])
353 aperVisitCatalog.extend(tempAperCat)
355 nStarInVisit += len(tempCat)
357 # Compute the median delta-aper
358 if not aperVisitCatalog.isContiguous():
359 aperVisitCatalog = aperVisitCatalog.copy(deep=True)
361 instMagIn = aperVisitCatalog[instMagInKey]
362 instMagErrIn = aperVisitCatalog[instMagErrInKey]
363 instMagOut = aperVisitCatalog[instMagOutKey]
364 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
366 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
367 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
369 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
370 visit['sources_read'] = True
372 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
373 (nStarInVisit, visit['visit'], visit['deltaAper']))
375 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and
376 starObsDataRef is not None and visitCatDataRef is not None):
377 # We need to persist both the stars and the visit catalog which gets
378 # additional metadata from each visit.
379 starObsDataRef.put(fullCatalog)
380 visitCatDataRef.put(visitCat)
382 self.log.info("Found all good star observations in %.2f s" %
383 (time.time() - startTime))
385 return fullCatalog
387 def _makeAperMapper(self, sourceSchema):
388 """
389 Make a schema mapper for fgcm aperture measurements
391 Parameters
392 ----------
393 sourceSchema: `afwTable.Schema`
394 Default source schema from the butler
396 Returns
397 -------
398 aperMapper: `afwTable.schemaMapper`
399 Mapper to the FGCM aperture schema
400 """
402 aperMapper = afwTable.SchemaMapper(sourceSchema)
403 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
404 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
405 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
406 doc="Magnitude at inner aperture")
407 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
408 doc="Magnitude error at inner aperture")
409 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
410 doc="Magnitude at outer aperture")
411 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
412 doc="Magnitude error at outer aperture")
414 return aperMapper