Coverage for python/lsst/fgcmcal/fgcmBuildStars.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM.
25This task finds all the visits and calexps in a repository (or a subset
26based on command line parameters) and extract all the potential calibration
27stars for input into fgcm. This task additionally uses fgcm to match
28star observations into unique stars, and performs as much cleaning of
29the input catalog as possible.
30"""
32import time
34import numpy as np
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38import lsst.afw.table as afwTable
40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41from .utilities import computeApproxPixelAreaFields
43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask']
46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase):
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc="Reference CCD for scanning visits",
51 dtype=int,
52 default=13,
53 )
54 checkAllCcds = pexConfig.Field(
55 doc=("Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
59 dtype=bool,
60 default=True,
61 )
63 def setDefaults(self):
64 super().setDefaults()
66 sourceSelector = self.sourceSelector["science"]
68 # The names here correspond to raw src catalogs, which differ
69 # from the post-transformed sourceTable_visit catalogs.
70 # Therefore, field and flag names cannot be easily
71 # derived from the base config class.
72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
80 'slot_Centroid_flag',
81 fluxFlagName]
83 if self.doSubtractLocalBackground:
84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
87 sourceSelector.signalToNoise.fluxField = self.instFluxField
88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask):
92 """
93 Build stars for the FGCM global calibration, using src catalogs.
94 """
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName = "fgcmBuildStars"
99 @classmethod
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
102 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789")
105 return parser
107 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
108 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
110 ccdIds = []
111 for detector in camera:
112 ccdIds.append(detector.getId())
114 # TODO: related to DM-13730, this dance of looking for source visits
115 # will be unnecessary with Gen3 Butler. This should be part of
116 # DM-13730.
118 nVisits = 0
120 groupedDataRefs = {}
121 for dataRef in dataRefs:
122 visit = dataRef.dataId[self.config.visitDataRefName]
123 # If we don't have the dataset, just continue
124 if not dataRef.datasetExists(datasetType='src'):
125 continue
126 # If we need to check all ccds, do it here
127 if self.config.checkAllCcds:
128 if visit in groupedDataRefs:
129 # We already have found this visit
130 continue
131 dataId = dataRef.dataId.copy()
132 # For each ccd we must check that a valid source catalog exists.
133 for ccdId in ccdIds:
134 dataId[self.config.ccdDataRefName] = ccdId
135 if butler.datasetExists('src', dataId=dataId):
136 goodDataRef = butler.dataRef('src', dataId=dataId)
137 if visit in groupedDataRefs:
138 if (goodDataRef.dataId[self.config.ccdDataRefName] not in
139 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
140 groupedDataRefs[visit].append(goodDataRef)
141 else:
142 # This is a new visit
143 nVisits += 1
144 groupedDataRefs[visit] = [goodDataRef]
145 else:
146 # We have already confirmed that the dataset exists, so no need
147 # to check here.
148 if visit in groupedDataRefs:
149 if (dataRef.dataId[self.config.ccdDataRefName] not in
150 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
151 groupedDataRefs[visit].append(dataRef)
152 else:
153 # This is a new visit
154 nVisits += 1
155 groupedDataRefs[visit] = [dataRef]
157 if (nVisits % 100) == 0 and nVisits > 0:
158 self.log.info("Found %d unique %ss..." % (nVisits,
159 self.config.visitDataRefName))
161 self.log.info("Found %d unique %ss total." % (nVisits,
162 self.config.visitDataRefName))
164 # Put them in ccd order, with the reference ccd first (if available)
165 def ccdSorter(dataRef):
166 ccdId = dataRef.dataId[self.config.ccdDataRefName]
167 if ccdId == self.config.referenceCCD:
168 return -100
169 else:
170 return ccdId
172 # If we did not check all ccds, put them in ccd order
173 if not self.config.checkAllCcds:
174 for visit in groupedDataRefs:
175 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
177 # This should be sorted by visit (the key)
178 return dict(sorted(groupedDataRefs.items()))
180 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
181 srcSchemaDataRef,
182 camera,
183 calibFluxApertureRadius=None,
184 visitCatDataRef=None,
185 starObsDataRef=None,
186 inStarObsCat=None):
187 startTime = time.time()
189 # If both dataRefs are None, then we assume the caller does not
190 # want to store checkpoint files. If both are set, we will
191 # do checkpoint files. And if only one is set, this is potentially
192 # unintentional and we will warn.
193 if (visitCatDataRef is not None and starObsDataRef is None
194 or visitCatDataRef is None and starObsDataRef is not None):
195 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
196 "no checkpoint files will be persisted.")
198 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
199 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
201 # create our source schema. Use the first valid dataRef
202 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
203 sourceSchema = dataRef.get('src_schema', immediate=True).schema
205 # Construct a mapping from ccd number to index
206 ccdMapping = {}
207 for ccdIndex, detector in enumerate(camera):
208 ccdMapping[detector.getId()] = ccdIndex
210 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
212 sourceMapper = self._makeSourceMapper(sourceSchema)
214 # We also have a temporary catalog that will accumulate aperture measurements
215 aperMapper = self._makeAperMapper(sourceSchema)
217 outputSchema = sourceMapper.getOutputSchema()
219 if inStarObsCat is not None:
220 fullCatalog = inStarObsCat
221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223 if not comp1 or not comp2:
224 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
225 else:
226 fullCatalog = afwTable.BaseCatalog(outputSchema)
228 # FGCM will provide relative calibration for the flux in config.instFluxField
230 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
232 visitKey = outputSchema['visit'].asKey()
233 ccdKey = outputSchema['ccd'].asKey()
234 instMagKey = outputSchema['instMag'].asKey()
235 instMagErrKey = outputSchema['instMagErr'].asKey()
236 deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
238 # Prepare local background if desired
239 if self.config.doSubtractLocalBackground:
240 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
241 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
243 aperOutputSchema = aperMapper.getOutputSchema()
245 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
246 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
247 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
248 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
249 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
250 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
251 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
252 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
254 k = 2.5/np.log(10.)
256 # loop over visits
257 for ctr, visit in enumerate(visitCat):
258 if visit['sources_read']:
259 continue
261 expTime = visit['exptime']
263 nStarInVisit = 0
265 # Reset the aperture catalog (per visit)
266 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
268 for dataRef in groupedDataRefs[visit['visit']]:
270 ccdId = dataRef.dataId[self.config.ccdDataRefName]
272 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
273 goodSrc = self.sourceSelector.selectSources(sources)
275 # Need to add a selection based on the local background correction
276 # if necessary
277 if self.config.doSubtractLocalBackground:
278 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
280 bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0)
281 goodSrc.selected[bad] = False
283 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
284 tempCat.reserve(goodSrc.selected.sum())
285 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
286 tempCat[visitKey][:] = visit['visit']
287 tempCat[ccdKey][:] = ccdId
289 # Compute "instrumental magnitude" by scaling flux with exposure time.
290 scaledInstFlux = (sources[instFluxKey][goodSrc.selected]
291 * visit['scaling'][ccdMapping[ccdId]])
292 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
294 # Compute the change in magnitude from the background offset
295 if self.config.doSubtractLocalBackground:
296 # At the moment we only adjust the flux and not the flux
297 # error by the background because the error on
298 # base_LocalBackground_instFlux is the rms error in the
299 # background annulus, not the error on the mean in the
300 # background estimate (which is much smaller, by sqrt(n)
301 # pixels used to estimate the background, which we do not
302 # have access to in this task). In the default settings,
303 # the annulus is sufficiently large such that these
304 # additional errors are are negligibly small (much less
305 # than a mmag in quadrature).
307 # This is the difference between the mag with background correction
308 # and the mag without background correction.
309 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected]
310 - localBackground[goodSrc.selected]) -
311 -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
312 else:
313 tempCat[deltaMagBkgKey][:] = 0.0
315 # Compute instMagErr from instFluxErr/instFlux, any scaling
316 # will cancel out.
318 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected]
319 / sources[instFluxKey][goodSrc.selected])
321 # Compute the jacobian from an approximate PixelAreaBoundedField
322 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
323 tempCat['y'])
325 # Apply the jacobian if configured
326 if self.config.doApplyWcsJacobian:
327 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
329 fullCatalog.extend(tempCat)
331 # And the aperture information
332 # This does not need the jacobian because it is all locally relative
333 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
334 tempAperCat.reserve(goodSrc.selected.sum())
335 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
337 with np.warnings.catch_warnings():
338 # Ignore warnings, we will filter infinities and
339 # nans below.
340 np.warnings.simplefilter("ignore")
342 tempAperCat[instMagInKey][:] = -2.5*np.log10(
343 sources[instFluxAperInKey][goodSrc.selected])
344 tempAperCat[instMagErrInKey][:] = k*(
345 sources[instFluxErrAperInKey][goodSrc.selected]
346 / sources[instFluxAperInKey][goodSrc.selected])
347 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
348 sources[instFluxAperOutKey][goodSrc.selected])
349 tempAperCat[instMagErrOutKey][:] = k*(
350 sources[instFluxErrAperOutKey][goodSrc.selected]
351 / sources[instFluxAperOutKey][goodSrc.selected])
353 aperVisitCatalog.extend(tempAperCat)
355 nStarInVisit += len(tempCat)
357 # Compute the median delta-aper
358 if not aperVisitCatalog.isContiguous():
359 aperVisitCatalog = aperVisitCatalog.copy(deep=True)
361 instMagIn = aperVisitCatalog[instMagInKey]
362 instMagErrIn = aperVisitCatalog[instMagErrInKey]
363 instMagOut = aperVisitCatalog[instMagOutKey]
364 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
366 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
367 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
369 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
370 visit['sources_read'] = True
372 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
373 (nStarInVisit, visit['visit'], visit['deltaAper']))
375 if ((ctr % self.config.nVisitsPerCheckpoint) == 0
376 and starObsDataRef is not None and visitCatDataRef is not None):
377 # We need to persist both the stars and the visit catalog which gets
378 # additional metadata from each visit.
379 starObsDataRef.put(fullCatalog)
380 visitCatDataRef.put(visitCat)
382 self.log.info("Found all good star observations in %.2f s" %
383 (time.time() - startTime))
385 return fullCatalog
387 def _makeAperMapper(self, sourceSchema):
388 """
389 Make a schema mapper for fgcm aperture measurements
391 Parameters
392 ----------
393 sourceSchema: `afwTable.Schema`
394 Default source schema from the butler
396 Returns
397 -------
398 aperMapper: `afwTable.schemaMapper`
399 Mapper to the FGCM aperture schema
400 """
402 aperMapper = afwTable.SchemaMapper(sourceSchema)
403 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
404 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
405 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
406 doc="Magnitude at inner aperture")
407 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
408 doc="Magnitude error at inner aperture")
409 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
410 doc="Magnitude at outer aperture")
411 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
412 doc="Magnitude error at outer aperture")
414 return aperMapper