Coverage for python/lsst/pipe/tasks/propagateVisitFlags.py: 28%

58 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 21:10 +0000

1#!/usr/bin/env python 

2# 

3# LSST Data Management System 

4# Copyright 2014-2015 LSST/AURA 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <http://www.lsstcorp.org/LegalNotices/>. 

22# 

23import numpy 

24from lsst.pex.config import Config, Field, DictField 

25from lsst.pipe.base import Task 

26import lsst.geom as geom 

27import lsst.afw.table as afwTable 

28import lsst.pex.exceptions as pexExceptions 

29 

30 

31class PropagateVisitFlagsConfig(Config): 

32 """!Configuration for propagating flags to coadd""" 

33 flags = DictField(keytype=str, itemtype=float, 

34 default={"calib_psf_candidate": 0.2, "calib_psf_used": 0.2, "calib_psf_reserved": 0.2, 

35 "calib_astrometry_used": 0.2, "calib_photometry_used": 0.2, 

36 "calib_photometry_reserved": 0.2, }, 

37 doc=("Source catalog flags to propagate, with the threshold of relative occurrence " 

38 "(valid range: [0-1], default is 0.2). Coadd object will have flag set if the " 

39 "fraction of input visits in which it is flagged is greater than the threshold.")) 

40 matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)") 

41 ccdName = Field(dtype=str, default='ccd', doc="Name of ccd to give to butler") 

42 

43 

44## \addtogroup LSST_task_documentation 

45## \{ 

46## \page PropagateVisitFlagsTask 

47## \ref PropagateVisitFlagsTask_ "PropagateVisitFlagsTask" 

48## \copybrief PropagateVisitFlagsTask 

49## \} 

50 

51class PropagateVisitFlagsTask(Task): 

52 r"""!Task to propagate flags from single-frame measurements to coadd measurements 

53 

54\anchor PropagateVisitFlagsTask_ 

55 

56\brief Propagate flags from individual visit measurements to coadd measurements 

57 

58\section pipe_tasks_propagateVisitFlagsTask_Contents Contents 

59 

60 - \ref pipe_tasks_propagateVisitFlagsTask_Description 

61 - \ref pipe_tasks_propagateVisitFlagsTask_Initialization 

62 - \ref pipe_tasks_propagateVisitFlagsTask_Config 

63 - \ref pipe_tasks_propagateVisitFlagsTask_Use 

64 - \ref pipe_tasks_propagateVisitFlagsTask_Example 

65 

66\section pipe_tasks_propagateVisitFlagsTask_Description Description 

67 

68\copybrief PropagateVisitFlagsTask 

69 

70We want to be able to set a flag for sources on the coadds using flags 

71that were determined from the individual visits. A common example is sources 

72that were used for PSF determination, since we do not do any PSF determination 

73on the coadd but use the individual visits. This requires matching the coadd 

74source catalog to each of the catalogs from the inputs (see 

75PropagateVisitFlagsConfig.matchRadius), and thresholding on the number of 

76times a source is flagged on the input catalog. 

77 

78An important consideration in this is that the flagging of sources in the 

79individual visits can be somewhat stochastic, e.g., the same stars may not 

80always be used for PSF determination because the field of view moves slightly 

81between visits, or the seeing changed. We there threshold on the relative 

82occurrence of the flag in the visits (see PropagateVisitFlagsConfig.flags). 

83Flagging a source that is always flagged in inputs corresponds to a threshold 

84of 1, while flagging a source that is flagged in any of the input corresponds 

85to a threshold of 0. But neither of these extrema are really useful in 

86practise. 

87 

88Setting the threshold too high means that sources that are not consistently 

89flagged (e.g., due to chip gaps) will not have the flag propagated. Setting 

90that threshold too low means that random sources which are falsely flagged in 

91the inputs will start to dominate. If in doubt, we suggest making this 

92threshold relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more 

93confidence in the quality of the flagging, the lower the threshold can be. 

94 

95The relative occurrence accounts for the edge of the field-of-view of the 

96camera, but does not include chip gaps, bad or saturated pixels, etc. 

97 

98\section pipe_tasks_propagateVisitFlagsTask_Initialization Initialization 

99 

100Beyond the usual Task initialization, PropagateVisitFlagsTask also requires 

101a schema for the catalog that is being constructed. 

102 

103\section pipe_tasks_propagateVisitFlagsTask_Config Configuration parameters 

104 

105See \ref PropagateVisitFlagsConfig 

106 

107\section pipe_tasks_propagateVisitFlagsTask_Use Use 

108 

109The 'run' method (described below) is the entry-point for operations. The 

110'getCcdInputs' staticmethod is provided as a convenience for retrieving the 

111'ccdInputs' (CCD inputs table) from an Exposure. 

112 

113\copydoc run 

114 

115\section pipe_tasks_propagateVisitFlagsTask_Example Example 

116 

117\code{.py} 

118# Requires: 

119# * butler: data butler, for retrieving the CCD catalogs 

120# * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog) 

121# * coaddExposure: coadd (lsst.afw.image.Exposure) 

122from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig 

123config = PropagateVisitFlagsConfig() 

124config.flags["calib_psf_used"] = 0.3 # Relative threshold for this flag 

125config.matchRadius = 0.5 # Matching radius in arcsec 

126task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config) 

127ccdInputs = task.getCcdInputs(coaddExposure) 

128task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs()) 

129\endcode 

130""" 

131 ConfigClass = PropagateVisitFlagsConfig 

132 

133 def __init__(self, schema, **kwargs): 

134 Task.__init__(self, **kwargs) 

135 self.schema = schema 

136 self._keys = dict((f, self.schema.addField(f, type="Flag", doc="Propagated from visits")) for 

137 f in self.config.flags) 

138 

139 @staticmethod 

140 def getCcdInputs(coaddExposure): 

141 """!Convenience method to retrieve the CCD inputs table from a coadd exposure""" 

142 return coaddExposure.getInfo().getCoaddInputs().ccds 

143 

144 def run(self, butler, coaddSources, ccdInputs, coaddWcs, visitCatalogs=None, wcsUpdates=None): 

145 """!Propagate flags from individual visit measurements to coadd 

146 

147 This requires matching the coadd source catalog to each of the catalogs 

148 from the inputs, and thresholding on the number of times a source is 

149 flagged on the input catalog. The threshold is made on the relative 

150 occurrence of the flag in each source. Flagging a source that is always 

151 flagged in inputs corresponds to a threshold of 1, while flagging a 

152 source that is flagged in any of the input corresponds to a threshold of 

153 0. But neither of these extrema are really useful in practise. 

154 

155 Setting the threshold too high means that sources that are not consistently 

156 flagged (e.g., due to chip gaps) will not have the flag propagated. Setting 

157 that threshold too low means that random sources which are falsely flagged in 

158 the inputs will start to dominate. If in doubt, we suggest making this threshold 

159 relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more confidence in 

160 the quality of the flagging, the lower the threshold can be. 

161 

162 The relative occurrence accounts for the edge of the field-of-view of 

163 the camera, but does not include chip gaps, bad or saturated pixels, etc. 

164 

165 @param[in] butler Data butler, for retrieving the input source catalogs 

166 @param[in,out] coaddSources Source catalog from the coadd 

167 @param[in] ccdInputs Table of CCDs that contribute to the coadd 

168 @param[in] coaddWcs Wcs for coadd 

169 @param[in] visitCatalogs List of loaded source catalogs for each input ccd in 

170 the coadd. If provided this is used instead of this 

171 method loading in the catalogs itself 

172 @param[in] wcsUpdates optional, If visitCatalogs is a list of ccd catalogs, this 

173 should be a list of updated wcs to apply 

174 """ 

175 if len(self.config.flags) == 0: 175 ↛ 178line 175 didn't jump to line 178, because the condition on line 175 was never false

176 return 

177 

178 flags = self._keys.keys() 

179 counts = dict((f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags) 

180 indices = numpy.array([s.getId() for s in coaddSources]) # Allowing for non-contiguous data 

181 radius = self.config.matchRadius*geom.arcseconds 

182 

183 def processCcd(ccdSources, wcsUpdate): 

184 for sourceRecord in ccdSources: 

185 sourceRecord.updateCoord(wcsUpdate) 

186 for flag in flags: 

187 # We assume that the flags will be relatively rare, so it is more efficient to match 

188 # against a subset of the input catalog for each flag than it is to match once against 

189 # the entire catalog. It would be best to have built a kd-tree on coaddSources and 

190 # keep reusing that for the matching, but we don't have a suitable implementation. 

191 mc = afwTable.MatchControl() 

192 mc.findOnlyClosest = False 

193 matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc) 

194 for m in matches: 

195 index = (numpy.where(indices == m.first.getId()))[0][0] 

196 counts[flag][index] += 1 

197 

198 if visitCatalogs is not None: 

199 if wcsUpdates is None: 

200 raise pexExceptions.ValueError("If ccdInputs is a list of src catalogs, a list of wcs" 

201 " updates for each catalog must be supplied in the " 

202 "wcsUpdates parameter") 

203 for i, ccdSource in enumerate(visitCatalogs): 

204 processCcd(ccdSource, wcsUpdates[i]) 

205 else: 

206 if ccdInputs is None: 

207 raise pexExceptions.ValueError("The visitCatalogs and ccdInput parameters can't both be None") 

208 visitKey = ccdInputs.schema.find("visit").key 

209 ccdKey = ccdInputs.schema.find("ccd").key 

210 

211 self.log.info("Propagating flags %s from inputs", flags) 

212 

213 # Accumulate counts of flags being set 

214 for ccdRecord in ccdInputs: 

215 v = ccdRecord.get(visitKey) 

216 c = ccdRecord.get(ccdKey) 

217 dataId = {"visit": int(v), self.config.ccdName: int(c)} 

218 ccdSources = butler.get("src", dataId=dataId, immediate=True) 

219 processCcd(ccdSources, ccdRecord.getWcs()) 

220 

221 # Apply threshold 

222 for f in flags: 

223 key = self._keys[f] 

224 for s, num in zip(coaddSources, counts[f]): 

225 numOverlaps = len(ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True)) 

226 s.setFlag(key, bool(num > numOverlaps*self.config.flags[f])) 

227 self.log.info("Propagated %d sources with flag %s", sum(s.get(key) for s in coaddSources), f)