Coverage for python/lsst/pipe/tasks/matchFakes.py: 47%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

88 statements  

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import astropy.units as u 

23import numpy as np 

24import pandas as pd 

25from scipy.spatial import cKDTree 

26 

27from lsst.geom import Box2D, radians, SpherePoint 

28import lsst.pex.config as pexConfig 

29from lsst.pipe.base import PipelineTask, PipelineTaskConnections, Struct 

30import lsst.pipe.base.connectionTypes as connTypes 

31from lsst.skymap import BaseSkyMap 

32 

33from lsst.pipe.tasks.insertFakes import InsertFakesConfig 

34 

35__all__ = ["MatchFakesTask", 

36 "MatchFakesConfig", 

37 "MatchVariableFakesConfig", 

38 "MatchVariableFakesTask"] 

39 

40 

41class MatchFakesConnections(PipelineTaskConnections, 

42 defaultTemplates={"coaddName": "deep", 

43 "fakesType": "fakes_"}, 

44 dimensions=("instrument", 

45 "visit", 

46 "detector")): 

47 skyMap = connTypes.Input( 

48 doc="Input definition of geometry/bbox and projection/wcs for " 

49 "template exposures. Needed to test which tract to generate ", 

50 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

51 dimensions=("skymap",), 

52 storageClass="SkyMap", 

53 ) 

54 fakeCats = connTypes.Input( 

55 doc="Catalog of fake sources inserted into an image.", 

56 name="{fakesType}fakeSourceCat", 

57 storageClass="DataFrame", 

58 dimensions=("tract", "skymap"), 

59 deferLoad=True, 

60 multiple=True 

61 ) 

62 diffIm = connTypes.Input( 

63 doc="Difference image on which the DiaSources were detected.", 

64 name="{fakesType}{coaddName}Diff_differenceExp", 

65 storageClass="ExposureF", 

66 dimensions=("instrument", "visit", "detector"), 

67 ) 

68 associatedDiaSources = connTypes.Input( 

69 doc="A DiaSource catalog to match against fakeCat. Assumed " 

70 "to be SDMified.", 

71 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

72 storageClass="DataFrame", 

73 dimensions=("instrument", "visit", "detector"), 

74 ) 

75 matchedDiaSources = connTypes.Output( 

76 doc="A catalog of those fakeCat sources that have a match in " 

77 "associatedDiaSources. The schema is the union of the schemas for " 

78 "``fakeCat`` and ``associatedDiaSources``.", 

79 name="{fakesType}{coaddName}Diff_matchDiaSrc", 

80 storageClass="DataFrame", 

81 dimensions=("instrument", "visit", "detector"), 

82 ) 

83 

84 

85class MatchFakesConfig( 

86 InsertFakesConfig, 

87 pipelineConnections=MatchFakesConnections): 

88 """Config for MatchFakesTask. 

89 """ 

90 matchDistanceArcseconds = pexConfig.RangeField( 

91 doc="Distance in arcseconds to match within.", 

92 dtype=float, 

93 default=0.5, 

94 min=0, 

95 max=10, 

96 ) 

97 

98 

99class MatchFakesTask(PipelineTask): 

100 """Match a pre-existing catalog of fakes to a catalog of detections on 

101 a difference image. 

102 

103 This task is generally for injected sources that cannot be easily 

104 identified by their footprints such as in the case of detector sources 

105 post image differencing. 

106 """ 

107 

108 _DefaultName = "matchFakes" 

109 ConfigClass = MatchFakesConfig 

110 

111 def run(self, fakeCats, skyMap, diffIm, associatedDiaSources): 

112 """Compose fakes into a single catalog and match fakes to detected 

113 diaSources within a difference image bound. 

114 

115 Parameters 

116 ---------- 

117 fakeCats : `pandas.DataFrame` 

118 List of catalog of fakes to match to detected diaSources. 

119 skyMap : `lsst.skymap.SkyMap` 

120 SkyMap defining the tracts and patches the fakes are stored over. 

121 diffIm : `lsst.afw.image.Exposure` 

122 Difference image where ``associatedDiaSources`` were detected. 

123 associatedDiaSources : `pandas.DataFrame` 

124 Catalog of difference image sources detected in ``diffIm``. 

125 

126 Returns 

127 ------- 

128 result : `lsst.pipe.base.Struct` 

129 Results struct with components. 

130 

131 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has 

132 length of ``fakeCat``. (`pandas.DataFrame`) 

133 """ 

134 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

135 return self._processFakes(fakeCat, diffIm, associatedDiaSources) 

136 

137 def _processFakes(self, fakeCat, diffIm, associatedDiaSources): 

138 """Match fakes to detected diaSources within a difference image bound. 

139 

140 Parameters 

141 ---------- 

142 fakeCat : `pandas.DataFrame` 

143 Catalog of fakes to match to detected diaSources. 

144 diffIm : `lsst.afw.image.Exposure` 

145 Difference image where ``associatedDiaSources`` were detected. 

146 associatedDiaSources : `pandas.DataFrame` 

147 Catalog of difference image sources detected in ``diffIm``. 

148 

149 Returns 

150 ------- 

151 result : `lsst.pipe.base.Struct` 

152 Results struct with components. 

153 

154 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has 

155 length of ``fakeCat``. (`pandas.DataFrame`) 

156 """ 

157 trimmedFakes = self._trimFakeCat(fakeCat, diffIm) 

158 nPossibleFakes = len(trimmedFakes) 

159 

160 fakeVects = self._getVectors(trimmedFakes[self.config.ra_col], 

161 trimmedFakes[self.config.dec_col]) 

162 diaSrcVects = self._getVectors( 

163 np.radians(associatedDiaSources.loc[:, "ra"]), 

164 np.radians(associatedDiaSources.loc[:, "decl"])) 

165 

166 diaSrcTree = cKDTree(diaSrcVects) 

167 dist, idxs = diaSrcTree.query( 

168 fakeVects, 

169 distance_upper_bound=np.radians(self.config.matchDistanceArcseconds / 3600)) 

170 nFakesFound = np.isfinite(dist).sum() 

171 

172 self.log.info("Found %d out of %d possible.", nFakesFound, nPossibleFakes) 

173 diaSrcIds = associatedDiaSources.iloc[np.where(np.isfinite(dist), idxs, 0)]["diaSourceId"].to_numpy() 

174 matchedFakes = trimmedFakes.assign(diaSourceId=np.where(np.isfinite(dist), diaSrcIds, 0)) 

175 

176 return Struct( 

177 matchedDiaSources=matchedFakes.merge( 

178 associatedDiaSources.reset_index(drop=True), on="diaSourceId", how="left") 

179 ) 

180 

181 def composeFakeCat(self, fakeCats, skyMap): 

182 """Concatenate the fakeCats from tracts that may cover the exposure. 

183 

184 Parameters 

185 ---------- 

186 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle` 

187 Set of fake cats to concatenate. 

188 skyMap : `lsst.skymap.SkyMap` 

189 SkyMap defining the geometry of the tracts and patches. 

190 

191 Returns 

192 ------- 

193 combinedFakeCat : `pandas.DataFrame` 

194 All fakes that cover the inner polygon of the tracts in this 

195 quantum. 

196 """ 

197 if len(fakeCats) == 1: 

198 return fakeCats[0].get( 

199 datasetType=self.config.connections.fakeCats) 

200 outputCat = [] 

201 for fakeCatRef in fakeCats: 

202 cat = fakeCatRef.get( 

203 datasetType=self.config.connections.fakeCats) 

204 tractId = fakeCatRef.dataId["tract"] 

205 # Make sure all data is within the inner part of the tract. 

206 outputCat.append(cat[ 

207 skyMap.findTractIdArray(cat[self.config.ra_col], 

208 cat[self.config.dec_col], 

209 degrees=False) 

210 == tractId]) 

211 

212 return pd.concat(outputCat) 

213 

214 def _trimFakeCat(self, fakeCat, image): 

215 """Trim the fake cat to about the size of the input image. 

216 

217 Parameters 

218 ---------- 

219 fakeCat : `pandas.core.frame.DataFrame` 

220 The catalog of fake sources to be input 

221 image : `lsst.afw.image.exposure.exposure.ExposureF` 

222 The image into which the fake sources should be added 

223 skyMap : `lsst.skymap.SkyMap` 

224 SkyMap defining the tracts and patches the fakes are stored over. 

225 

226 Returns 

227 ------- 

228 fakeCats : `pandas.core.frame.DataFrame` 

229 The original fakeCat trimmed to the area of the image 

230 """ 

231 wcs = image.getWcs() 

232 

233 bbox = Box2D(image.getBBox()) 

234 

235 def trim(row): 

236 coord = SpherePoint(row[self.config.ra_col], 

237 row[self.config.dec_col], 

238 radians) 

239 cent = wcs.skyToPixel(coord) 

240 return bbox.contains(cent) 

241 

242 return fakeCat[fakeCat.apply(trim, axis=1)] 

243 

244 def _getVectors(self, ras, decs): 

245 """Convert ra dec to unit vectors on the sphere. 

246 

247 Parameters 

248 ---------- 

249 ras : `numpy.ndarray`, (N,) 

250 RA coordinates in radians. 

251 decs : `numpy.ndarray`, (N,) 

252 Dec coordinates in radians. 

253 

254 Returns 

255 ------- 

256 vectors : `numpy.ndarray`, (N, 3) 

257 Vectors on the unit sphere for the given RA/DEC values. 

258 """ 

259 vectors = np.empty((len(ras), 3)) 

260 

261 vectors[:, 2] = np.sin(decs) 

262 vectors[:, 0] = np.cos(decs) * np.cos(ras) 

263 vectors[:, 1] = np.cos(decs) * np.sin(ras) 

264 

265 return vectors 

266 

267 

268class MatchVariableFakesConnections(MatchFakesConnections): 

269 ccdVisitFakeMagnitudes = connTypes.Input( 

270 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

271 name="{fakesType}ccdVisitFakeMagnitudes", 

272 storageClass="DataFrame", 

273 dimensions=("instrument", "visit", "detector"), 

274 ) 

275 

276 

277class MatchVariableFakesConfig(MatchFakesConfig, 

278 pipelineConnections=MatchVariableFakesConnections): 

279 """Config for MatchFakesTask. 

280 """ 

281 pass 

282 

283 

284class MatchVariableFakesTask(MatchFakesTask): 

285 """Match injected fakes to their detected sources in the catalog and 

286 compute their expected brightness in a difference image assuming perfect 

287 subtraction. 

288 

289 This task is generally for injected sources that cannot be easily 

290 identified by their footprints such as in the case of detector sources 

291 post image differencing. 

292 """ 

293 _DefaultName = "matchVariableFakes" 

294 ConfigClass = MatchVariableFakesConfig 

295 

296 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

297 inputs = butlerQC.get(inputRefs) 

298 inputs["band"] = butlerQC.quantum.dataId["band"] 

299 

300 outputs = self.run(**inputs) 

301 butlerQC.put(outputs, outputRefs) 

302 

303 def run(self, fakeCats, ccdVisitFakeMagnitudes, skyMap, diffIm, associatedDiaSources, band): 

304 """Match fakes to detected diaSources within a difference image bound. 

305 

306 Parameters 

307 ---------- 

308 fakeCat : `pandas.DataFrame` 

309 Catalog of fakes to match to detected diaSources. 

310 diffIm : `lsst.afw.image.Exposure` 

311 Difference image where ``associatedDiaSources`` were detected in. 

312 associatedDiaSources : `pandas.DataFrame` 

313 Catalog of difference image sources detected in ``diffIm``. 

314 

315 Returns 

316 ------- 

317 result : `lsst.pipe.base.Struct` 

318 Results struct with components. 

319 

320 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has 

321 length of ``fakeCat``. (`pandas.DataFrame`) 

322 """ 

323 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

324 self.computeExpectedDiffMag(fakeCat, ccdVisitFakeMagnitudes, band) 

325 return self._processFakes(fakeCat, diffIm, associatedDiaSources) 

326 

327 def computeExpectedDiffMag(self, fakeCat, ccdVisitFakeMagnitudes, band): 

328 """Compute the magnitude expected in the difference image for this 

329 detector/visit. Modify fakeCat in place. 

330 

331 Negative magnitudes indicate that the source should be detected as 

332 a negative source. 

333 

334 Parameters 

335 ---------- 

336 fakeCat : `pandas.DataFrame` 

337 Catalog of fake sources. 

338 ccdVisitFakeMagnitudes : `pandas.DataFrame` 

339 Magnitudes for variable sources in this specific ccdVisit. 

340 band : `str` 

341 Band that this ccdVisit was observed in. 

342 """ 

343 magName = self.config.mag_col % band 

344 magnitudes = fakeCat[magName].to_numpy() 

345 visitMags = ccdVisitFakeMagnitudes["variableMag"].to_numpy() 

346 diffFlux = (visitMags * u.ABmag).to_value(u.nJy) - (magnitudes * u.ABmag).to_value(u.nJy) 

347 diffMag = np.where(diffFlux > 0, 

348 (diffFlux * u.nJy).to_value(u.ABmag), 

349 -(-diffFlux * u.nJy).to_value(u.ABmag)) 

350 

351 noVisit = ~fakeCat["isVisitSource"] 

352 noTemplate = ~fakeCat["isTemplateSource"] 

353 both = np.logical_and(fakeCat["isVisitSource"], 

354 fakeCat["isTemplateSource"]) 

355 

356 fakeCat.loc[noVisit, magName] = -magnitudes[noVisit] 

357 fakeCat.loc[noTemplate, magName] = visitMags[noTemplate] 

358 fakeCat.loc[both, magName] = diffMag[both]