Coverage for python/lsst/ap/association/filterDiaSourceCatalog.py: 37%

59 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 10:38 +0000

1# This file is part of ap_association 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ( 

23 "FilterDiaSourceCatalogConfig", 

24 "FilterDiaSourceCatalogTask", 

25) 

26 

27import numpy as np 

28 

29from lsst.afw.table import SourceCatalog 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as connTypes 

33from lsst.utils.timer import timeMethod 

34 

35 

36class FilterDiaSourceCatalogConnections( 

37 pipeBase.PipelineTaskConnections, 

38 dimensions=("instrument", "visit", "detector"), 

39 defaultTemplates={"coaddName": "deep", "fakesType": ""}, 

40): 

41 """Connections class for FilterDiaSourceCatalogTask.""" 

42 

43 diaSourceCat = connTypes.Input( 

44 doc="Catalog of DiaSources produced during image differencing.", 

45 name="{fakesType}{coaddName}Diff_diaSrc", 

46 storageClass="SourceCatalog", 

47 dimensions=("instrument", "visit", "detector"), 

48 ) 

49 

50 filteredDiaSourceCat = connTypes.Output( 

51 doc="Output catalog of DiaSources after filtering.", 

52 name="{fakesType}{coaddName}Diff_candidateDiaSrc", 

53 storageClass="SourceCatalog", 

54 dimensions=("instrument", "visit", "detector"), 

55 ) 

56 

57 rejectedDiaSources = connTypes.Output( 

58 doc="Optional output storing all the rejected DiaSources.", 

59 name="{fakesType}{coaddName}Diff_rejectedDiaSrc", 

60 storageClass="SourceCatalog", 

61 dimensions={"instrument", "visit", "detector"}, 

62 ) 

63 

64 diffImVisitInfo = connTypes.Input( 

65 doc="VisitInfo of diffIm.", 

66 name="{fakesType}{coaddName}Diff_differenceExp.visitInfo", 

67 storageClass="VisitInfo", 

68 dimensions=("instrument", "visit", "detector"), 

69 ) 

70 

71 longTrailedSources = connTypes.Output( 

72 doc="Optional output temporarily storing long trailed diaSources.", 

73 dimensions=("instrument", "visit", "detector"), 

74 storageClass="ArrowAstropy", 

75 name="{fakesType}{coaddName}Diff_longTrailedSrc", 

76 ) 

77 

78 def __init__(self, *, config=None): 

79 super().__init__(config=config) 

80 if not self.config.doWriteRejectedSkySources: 

81 self.outputs.remove("rejectedDiaSources") 

82 if not self.config.doTrailedSourceFilter: 

83 self.outputs.remove("longTrailedSources") 

84 if not self.config.doWriteTrailedSources: 

85 self.outputs.remove("longTrailedSources") 

86 

87 

88class FilterDiaSourceCatalogConfig( 

89 pipeBase.PipelineTaskConfig, pipelineConnections=FilterDiaSourceCatalogConnections 

90): 

91 """Config class for FilterDiaSourceCatalogTask.""" 

92 

93 doRemoveSkySources = pexConfig.Field( 

94 dtype=bool, 

95 default=False, 

96 doc="Input DiaSource catalog contains SkySources that should be " 

97 "removed before storing the output DiaSource catalog.", 

98 ) 

99 

100 doWriteRejectedSkySources = pexConfig.Field( 

101 dtype=bool, 

102 default=True, 

103 doc="Store the output DiaSource catalog containing all the rejected " 

104 "sky sources." 

105 ) 

106 

107 doTrailedSourceFilter = pexConfig.Field( 

108 doc="Run trailedSourceFilter to remove long trailed sources from the" 

109 "diaSource output catalog.", 

110 dtype=bool, 

111 default=True, 

112 ) 

113 

114 doWriteTrailedSources = pexConfig.Field( 

115 doc="Write trailed diaSources sources to a table.", 

116 dtype=bool, 

117 default=True, 

118 deprecated="Trailed sources will not be written out during production." 

119 ) 

120 

121 max_trail_length = pexConfig.Field( 

122 dtype=float, 

123 doc="Length of long trailed sources to remove from the input catalog, " 

124 "in arcseconds per second. Default comes from DMTN-199, which " 

125 "requires removal of sources with trails longer than 10 " 

126 "degrees/day, which is 36000/3600/24 arcsec/second, or roughly" 

127 "0.416 arcseconds per second.", 

128 default=36000/3600.0/24.0, 

129 ) 

130 

131 

132class FilterDiaSourceCatalogTask(pipeBase.PipelineTask): 

133 """Filter out sky sources from a DiaSource catalog.""" 

134 

135 ConfigClass = FilterDiaSourceCatalogConfig 

136 _DefaultName = "filterDiaSourceCatalog" 

137 

138 @timeMethod 

139 def run(self, diaSourceCat, diffImVisitInfo): 

140 """Filter sky sources from the supplied DiaSource catalog. 

141 

142 Parameters 

143 ---------- 

144 diaSourceCat : `lsst.afw.table.SourceCatalog` 

145 Catalog of sources measured on the difference image. 

146 diffImVisitInfo: `lsst.afw.image.VisitInfo` 

147 VisitInfo for the difference image corresponding to diaSourceCat. 

148 

149 Returns 

150 ------- 

151 filterResults : `lsst.pipe.base.Struct` 

152 

153 ``filteredDiaSourceCat`` : `lsst.afw.table.SourceCatalog` 

154 The catalog of filtered sources. 

155 ``rejectedDiaSources`` : `lsst.afw.table.SourceCatalog` 

156 The catalog of rejected sky sources. 

157 ``longTrailedDiaSources`` : `astropy.table.Table` 

158 DiaSources which have trail lengths greater than 

159 max_trail_length*exposure_time. 

160 """ 

161 rejectedSkySources = None 

162 exposure_time = diffImVisitInfo.exposureTime 

163 if self.config.doRemoveSkySources: 

164 sky_source_column = diaSourceCat["sky_source"] 

165 num_sky_sources = np.sum(sky_source_column) 

166 rejectedSkySources = diaSourceCat[sky_source_column].copy(deep=True) 

167 diaSourceCat = diaSourceCat[~sky_source_column].copy(deep=True) 

168 self.log.info(f"Filtered {num_sky_sources} sky sources.") 

169 if not rejectedSkySources: 

170 rejectedSkySources = SourceCatalog(diaSourceCat.getSchema()) 

171 

172 if self.config.doTrailedSourceFilter: 

173 trail_mask = self._check_dia_source_trail(diaSourceCat, exposure_time) 

174 longTrailedDiaSources = diaSourceCat[trail_mask].copy(deep=True) 

175 diaSourceCat = diaSourceCat[~trail_mask] 

176 

177 self.log.info("%i DiaSources exceed max_trail_length %f arcseconds per second, " 

178 "dropping from source catalog." 

179 % (self.config.max_trail_length, len(diaSourceCat))) 

180 self.metadata.add("num_filtered", len(longTrailedDiaSources)) 

181 

182 if self.config.doWriteTrailedSources: 

183 filterResults = pipeBase.Struct(filteredDiaSourceCat=diaSourceCat, 

184 rejectedDiaSources=rejectedSkySources, 

185 longTrailedSources=longTrailedDiaSources.asAstropy()) 

186 else: 

187 filterResults = pipeBase.Struct(filteredDiaSourceCat=diaSourceCat, 

188 rejectedDiaSources=rejectedSkySources) 

189 

190 else: 

191 filterResults = pipeBase.Struct(filteredDiaSourceCat=diaSourceCat, 

192 rejectedDiaSources=rejectedSkySources) 

193 

194 return filterResults 

195 

196 def _check_dia_source_trail(self, dia_sources, exposure_time): 

197 """Find DiaSources that have long trails or trails with indeterminant 

198 end points. 

199 

200 Return a mask of sources with lengths greater than 

201 (``config.max_trail_length`` multiplied by the exposure time) 

202 arcseconds. 

203 Additionally, set mask if 

204 ``ext_trailedSources_Naive_flag_off_image`` is set or if 

205 ``ext_trailedSources_Naive_flag_suspect_long_trail`` and 

206 ``ext_trailedSources_Naive_flag_edge`` are both set. 

207 

208 Parameters 

209 ---------- 

210 dia_sources : `pandas.DataFrame` 

211 Input diaSources to check for trail lengths. 

212 exposure_time : `float` 

213 Exposure time from difference image. 

214 

215 Returns 

216 ------- 

217 trail_mask : `pandas.DataFrame` 

218 Boolean mask for diaSources which are greater than the 

219 Boolean mask for diaSources which are greater than the 

220 cutoff length or have trails which extend beyond the edge of the 

221 detector (off_image set). Also checks if both 

222 suspect_long_trail and edge are set and masks those sources out. 

223 """ 

224 print(dia_sources.getSchema()) 

225 trail_mask = (dia_sources["ext_trailedSources_Naive_length"] 

226 >= (self.config.max_trail_length*exposure_time)) 

227 trail_mask |= dia_sources['ext_trailedSources_Naive_flag_off_image'] 

228 trail_mask |= (dia_sources['ext_trailedSources_Naive_flag_suspect_long_trail'] 

229 & dia_sources['ext_trailedSources_Naive_flag_edge']) 

230 

231 return trail_mask