Coverage for python/lsst/fgcmcal/fgcmBuildStars.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM.
25This task finds all the visits and calexps in a repository (or a subset
26based on command line parameters) and extract all the potential calibration
27stars for input into fgcm. This task additionally uses fgcm to match
28star observations into unique stars, and performs as much cleaning of
29the input catalog as possible.
30"""
32import time
34import numpy as np
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38import lsst.afw.table as afwTable
40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41from .utilities import computeApproxPixelAreaFields
43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase):
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc="Reference CCD for scanning visits",
51 dtype=int,
52 default=13,
53 )
54 checkAllCcds = pexConfig.Field(
55 doc=("Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
59 dtype=bool,
60 default=True,
61 )
63 def setDefaults(self):
64 super().setDefaults()
66 sourceSelector = self.sourceSelector["science"]
68 # The names here correspond to raw src catalogs, which differ
69 # from the post-transformed sourceTable_visit catalogs.
70 # Therefore, field and flag names cannot be easily
71 # derived from the base config class.
72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
80 'slot_Centroid_flag',
81 fluxFlagName]
83 if self.doSubtractLocalBackground:
84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
87 sourceSelector.signalToNoise.fluxField = self.instFluxField
88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask):
92 """
93 Build stars for the FGCM global calibration, using src catalogs.
94 """
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName = "fgcmBuildStars"
99 @classmethod
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
102 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
105 return parser
107 def findAndGroupDataRefs(self, butler, dataRefs):
108 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
110 camera = butler.get('camera')
112 ccdIds = []
113 for detector in camera:
114 ccdIds.append(detector.getId())
116 # TODO: related to DM-13730, this dance of looking for source visits
117 # will be unnecessary with Gen3 Butler. This should be part of
118 # DM-13730.
120 nVisits = 0
122 groupedDataRefs = {}
123 for dataRef in dataRefs:
124 visit = dataRef.dataId[self.config.visitDataRefName]
125 # If we don't have the dataset, just continue
126 if not dataRef.datasetExists(datasetType='src'):
127 continue
128 # If we need to check all ccds, do it here
129 if self.config.checkAllCcds:
130 if visit in groupedDataRefs:
131 # We already have found this visit
132 continue
133 dataId = dataRef.dataId.copy()
134 # For each ccd we must check that a valid source catalog exists.
135 for ccdId in ccdIds:
136 dataId[self.config.ccdDataRefName] = ccdId
137 if butler.datasetExists('src', dataId=dataId):
138 goodDataRef = butler.dataRef('src', dataId=dataId)
139 if visit in groupedDataRefs:
140 if (goodDataRef.dataId[self.config.ccdDataRefName] not in
141 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
142 groupedDataRefs[visit].append(goodDataRef)
143 else:
144 # This is a new visit
145 nVisits += 1
146 groupedDataRefs[visit] = [goodDataRef]
147 else:
148 # We have already confirmed that the dataset exists, so no need
149 # to check here.
150 if visit in groupedDataRefs:
151 if (dataRef.dataId[self.config.ccdDataRefName] not in
152 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
153 groupedDataRefs[visit].append(dataRef)
154 else:
155 # This is a new visit
156 nVisits += 1
157 groupedDataRefs[visit] = [dataRef]
159 if (nVisits % 100) == 0 and nVisits > 0:
160 self.log.info("Found %d unique %ss..." % (nVisits,
161 self.config.visitDataRefName))
163 self.log.info("Found %d unique %ss total." % (nVisits,
164 self.config.visitDataRefName))
166 # Put them in ccd order, with the reference ccd first (if available)
167 def ccdSorter(dataRef):
168 ccdId = dataRef.dataId[self.config.ccdDataRefName]
169 if ccdId == self.config.referenceCCD:
170 return -100
171 else:
172 return ccdId
174 # If we did not check all ccds, put them in ccd order
175 if not self.config.checkAllCcds:
176 for visit in groupedDataRefs:
177 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
179 return groupedDataRefs
181 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
182 calibFluxApertureRadius=None,
183 visitCatDataRef=None,
184 starObsDataRef=None,
185 inStarObsCat=None):
186 startTime = time.time()
188 # If both dataRefs are None, then we assume the caller does not
189 # want to store checkpoint files. If both are set, we will
190 # do checkpoint files. And if only one is set, this is potentially
191 # unintentional and we will warn.
192 if (visitCatDataRef is not None and starObsDataRef is None or
193 visitCatDataRef is None and starObsDataRef is not None):
194 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
195 "no checkpoint files will be persisted.")
197 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
198 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
200 # create our source schema. Use the first valid dataRef
201 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202 sourceSchema = dataRef.get('src_schema', immediate=True).schema
204 # Construct a mapping from ccd number to index
205 camera = dataRef.get('camera')
206 ccdMapping = {}
207 for ccdIndex, detector in enumerate(camera):
208 ccdMapping[detector.getId()] = ccdIndex
210 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
212 sourceMapper = self._makeSourceMapper(sourceSchema)
214 # We also have a temporary catalog that will accumulate aperture measurements
215 aperMapper = self._makeAperMapper(sourceSchema)
217 outputSchema = sourceMapper.getOutputSchema()
219 if inStarObsCat is not None:
220 fullCatalog = inStarObsCat
221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223 if not comp1 or not comp2:
224 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
225 else:
226 fullCatalog = afwTable.BaseCatalog(outputSchema)
228 # FGCM will provide relative calibration for the flux in config.instFluxField
230 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
232 visitKey = outputSchema['visit'].asKey()
233 ccdKey = outputSchema['ccd'].asKey()
234 instMagKey = outputSchema['instMag'].asKey()
235 instMagErrKey = outputSchema['instMagErr'].asKey()
237 # Prepare local background if desired
238 if self.config.doSubtractLocalBackground:
239 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
240 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
241 else:
242 localBackground = 0.0
244 aperOutputSchema = aperMapper.getOutputSchema()
246 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
247 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
248 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
249 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
250 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
251 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
252 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
253 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
255 k = 2.5/np.log(10.)
257 # loop over visits
258 for ctr, visit in enumerate(visitCat):
259 if visit['sources_read']:
260 continue
262 expTime = visit['exptime']
264 nStarInVisit = 0
266 # Reset the aperture catalog (per visit)
267 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
269 for dataRef in groupedDataRefs[visit['visit']]:
271 ccdId = dataRef.dataId[self.config.ccdDataRefName]
273 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
275 # If we are subtracting the local background, then correct here
276 # before we do the s/n selection. This ensures we do not have
277 # bad stars after local background subtraction.
279 if self.config.doSubtractLocalBackground:
280 # At the moment we only adjust the flux and not the flux
281 # error by the background because the error on
282 # base_LocalBackground_instFlux is the rms error in the
283 # background annulus, not the error on the mean in the
284 # background estimate (which is much smaller, by sqrt(n)
285 # pixels used to estimate the background, which we do not
286 # have access to in this task). In the default settings,
287 # the annulus is sufficiently large such that these
288 # additional errors are are negligibly small (much less
289 # than a mmag in quadrature).
291 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
292 sources[instFluxKey] -= localBackground
294 goodSrc = self.sourceSelector.selectSources(sources)
296 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
297 tempCat.reserve(goodSrc.selected.sum())
298 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
299 tempCat[visitKey][:] = visit['visit']
300 tempCat[ccdKey][:] = ccdId
302 # Compute "instrumental magnitude" by scaling flux with exposure time.
303 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
304 visit['scaling'][ccdMapping[ccdId]])
305 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
307 # Compute instMagErr from instFluxErr/instFlux, any scaling
308 # will cancel out.
310 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
311 sources[instFluxKey][goodSrc.selected])
313 # Compute the jacobian from an approximate PixelAreaBoundedField
314 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
315 tempCat['y'])
317 # Apply the jacobian if configured
318 if self.config.doApplyWcsJacobian:
319 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
321 fullCatalog.extend(tempCat)
323 # And the aperture information
324 # This does not need the jacobian because it is all locally relative
325 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
326 tempAperCat.reserve(goodSrc.selected.sum())
327 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
329 with np.warnings.catch_warnings():
330 # Ignore warnings, we will filter infinities and
331 # nans below.
332 np.warnings.simplefilter("ignore")
334 tempAperCat[instMagInKey][:] = -2.5*np.log10(
335 sources[instFluxAperInKey][goodSrc.selected])
336 tempAperCat[instMagErrInKey][:] = k*(
337 sources[instFluxErrAperInKey][goodSrc.selected] /
338 sources[instFluxAperInKey][goodSrc.selected])
339 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
340 sources[instFluxAperOutKey][goodSrc.selected])
341 tempAperCat[instMagErrOutKey][:] = k*(
342 sources[instFluxErrAperOutKey][goodSrc.selected] /
343 sources[instFluxAperOutKey][goodSrc.selected])
345 aperVisitCatalog.extend(tempAperCat)
347 nStarInVisit += len(tempCat)
349 # Compute the median delta-aper
350 if not aperVisitCatalog.isContiguous():
351 aperVisitCatalog = aperVisitCatalog.copy(deep=True)
353 instMagIn = aperVisitCatalog[instMagInKey]
354 instMagErrIn = aperVisitCatalog[instMagErrInKey]
355 instMagOut = aperVisitCatalog[instMagOutKey]
356 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
358 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
359 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
361 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
362 visit['sources_read'] = True
364 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
365 (nStarInVisit, visit['visit'], visit['deltaAper']))
367 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and
368 starObsDataRef is not None and visitCatDataRef is not None):
369 # We need to persist both the stars and the visit catalog which gets
370 # additional metadata from each visit.
371 starObsDataRef.put(fullCatalog)
372 visitCatDataRef.put(visitCat)
374 self.log.info("Found all good star observations in %.2f s" %
375 (time.time() - startTime))
377 return fullCatalog
379 def _makeAperMapper(self, sourceSchema):
380 """
381 Make a schema mapper for fgcm aperture measurements
383 Parameters
384 ----------
385 sourceSchema: `afwTable.Schema`
386 Default source schema from the butler
388 Returns
389 -------
390 aperMapper: `afwTable.schemaMapper`
391 Mapper to the FGCM aperture schema
392 """
394 aperMapper = afwTable.SchemaMapper(sourceSchema)
395 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
396 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
397 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
398 doc="Magnitude at inner aperture")
399 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
400 doc="Magnitude error at inner aperture")
401 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
402 doc="Magnitude at outer aperture")
403 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
404 doc="Magnitude error at outer aperture")
406 return aperMapper