Coverage for python/lsst/pipe/tasks/matchFakes.py: 46%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

94 statements  

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import astropy.units as u 

23import numpy as np 

24import pandas as pd 

25from scipy.spatial import cKDTree 

26 

27from lsst.geom import Box2D, radians, SpherePoint 

28import lsst.pex.config as pexConfig 

29from lsst.pipe.base import PipelineTask, PipelineTaskConnections, Struct 

30import lsst.pipe.base.connectionTypes as connTypes 

31from lsst.skymap import BaseSkyMap 

32 

33from lsst.pipe.tasks.insertFakes import InsertFakesConfig 

34 

35__all__ = ["MatchFakesTask", 

36 "MatchFakesConfig", 

37 "MatchVariableFakesConfig", 

38 "MatchVariableFakesTask"] 

39 

40 

41class MatchFakesConnections(PipelineTaskConnections, 

42 defaultTemplates={"coaddName": "deep", 

43 "fakesType": "fakes_"}, 

44 dimensions=("instrument", 

45 "visit", 

46 "detector")): 

47 skyMap = connTypes.Input( 

48 doc="Input definition of geometry/bbox and projection/wcs for " 

49 "template exposures. Needed to test which tract to generate ", 

50 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

51 dimensions=("skymap",), 

52 storageClass="SkyMap", 

53 ) 

54 fakeCats = connTypes.Input( 

55 doc="Catalog of fake sources inserted into an image.", 

56 name="{fakesType}fakeSourceCat", 

57 storageClass="DataFrame", 

58 dimensions=("tract", "skymap"), 

59 deferLoad=True, 

60 multiple=True 

61 ) 

62 diffIm = connTypes.Input( 

63 doc="Difference image on which the DiaSources were detected.", 

64 name="{fakesType}{coaddName}Diff_differenceExp", 

65 storageClass="ExposureF", 

66 dimensions=("instrument", "visit", "detector"), 

67 ) 

68 associatedDiaSources = connTypes.Input( 

69 doc="A DiaSource catalog to match against fakeCat. Assumed " 

70 "to be SDMified.", 

71 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

72 storageClass="DataFrame", 

73 dimensions=("instrument", "visit", "detector"), 

74 ) 

75 matchedDiaSources = connTypes.Output( 

76 doc="A catalog of those fakeCat sources that have a match in " 

77 "associatedDiaSources. The schema is the union of the schemas for " 

78 "``fakeCat`` and ``associatedDiaSources``.", 

79 name="{fakesType}{coaddName}Diff_matchDiaSrc", 

80 storageClass="DataFrame", 

81 dimensions=("instrument", "visit", "detector"), 

82 ) 

83 

84 

85class MatchFakesConfig( 

86 InsertFakesConfig, 

87 pipelineConnections=MatchFakesConnections): 

88 """Config for MatchFakesTask. 

89 """ 

90 matchDistanceArcseconds = pexConfig.RangeField( 

91 doc="Distance in arcseconds to match within.", 

92 dtype=float, 

93 default=0.5, 

94 min=0, 

95 max=10, 

96 ) 

97 

98 doMatchVisit = pexConfig.Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Match visit to trim the fakeCat" 

102 ) 

103 

104 

105class MatchFakesTask(PipelineTask): 

106 """Match a pre-existing catalog of fakes to a catalog of detections on 

107 a difference image. 

108 

109 This task is generally for injected sources that cannot be easily 

110 identified by their footprints such as in the case of detector sources 

111 post image differencing. 

112 """ 

113 

114 _DefaultName = "matchFakes" 

115 ConfigClass = MatchFakesConfig 

116 

117 def run(self, fakeCats, skyMap, diffIm, associatedDiaSources): 

118 """Compose fakes into a single catalog and match fakes to detected 

119 diaSources within a difference image bound. 

120 

121 Parameters 

122 ---------- 

123 fakeCats : `pandas.DataFrame` 

124 List of catalog of fakes to match to detected diaSources. 

125 skyMap : `lsst.skymap.SkyMap` 

126 SkyMap defining the tracts and patches the fakes are stored over. 

127 diffIm : `lsst.afw.image.Exposure` 

128 Difference image where ``associatedDiaSources`` were detected. 

129 associatedDiaSources : `pandas.DataFrame` 

130 Catalog of difference image sources detected in ``diffIm``. 

131 

132 Returns 

133 ------- 

134 result : `lsst.pipe.base.Struct` 

135 Results struct with components. 

136 

137 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has 

138 length of ``fakeCat``. (`pandas.DataFrame`) 

139 """ 

140 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

141 

142 if self.config.doMatchVisit: 

143 fakeCat = self.getVisitMatchedFakeCat(fakeCat, diffIm) 

144 

145 return self._processFakes(fakeCat, diffIm, associatedDiaSources) 

146 

147 def _processFakes(self, fakeCat, diffIm, associatedDiaSources): 

148 """Match fakes to detected diaSources within a difference image bound. 

149 

150 Parameters 

151 ---------- 

152 fakeCat : `pandas.DataFrame` 

153 Catalog of fakes to match to detected diaSources. 

154 diffIm : `lsst.afw.image.Exposure` 

155 Difference image where ``associatedDiaSources`` were detected. 

156 associatedDiaSources : `pandas.DataFrame` 

157 Catalog of difference image sources detected in ``diffIm``. 

158 

159 Returns 

160 ------- 

161 result : `lsst.pipe.base.Struct` 

162 Results struct with components. 

163 

164 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has 

165 length of ``fakeCat``. (`pandas.DataFrame`) 

166 """ 

167 trimmedFakes = self._trimFakeCat(fakeCat, diffIm) 

168 nPossibleFakes = len(trimmedFakes) 

169 

170 fakeVects = self._getVectors(trimmedFakes[self.config.ra_col], 

171 trimmedFakes[self.config.dec_col]) 

172 diaSrcVects = self._getVectors( 

173 np.radians(associatedDiaSources.loc[:, "ra"]), 

174 np.radians(associatedDiaSources.loc[:, "decl"])) 

175 

176 diaSrcTree = cKDTree(diaSrcVects) 

177 dist, idxs = diaSrcTree.query( 

178 fakeVects, 

179 distance_upper_bound=np.radians(self.config.matchDistanceArcseconds / 3600)) 

180 nFakesFound = np.isfinite(dist).sum() 

181 

182 self.log.info("Found %d out of %d possible.", nFakesFound, nPossibleFakes) 

183 diaSrcIds = associatedDiaSources.iloc[np.where(np.isfinite(dist), idxs, 0)]["diaSourceId"].to_numpy() 

184 matchedFakes = trimmedFakes.assign(diaSourceId=np.where(np.isfinite(dist), diaSrcIds, 0)) 

185 

186 return Struct( 

187 matchedDiaSources=matchedFakes.merge( 

188 associatedDiaSources.reset_index(drop=True), on="diaSourceId", how="left") 

189 ) 

190 

191 def composeFakeCat(self, fakeCats, skyMap): 

192 """Concatenate the fakeCats from tracts that may cover the exposure. 

193 

194 Parameters 

195 ---------- 

196 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle` 

197 Set of fake cats to concatenate. 

198 skyMap : `lsst.skymap.SkyMap` 

199 SkyMap defining the geometry of the tracts and patches. 

200 

201 Returns 

202 ------- 

203 combinedFakeCat : `pandas.DataFrame` 

204 All fakes that cover the inner polygon of the tracts in this 

205 quantum. 

206 """ 

207 if len(fakeCats) == 1: 

208 return fakeCats[0].get( 

209 datasetType=self.config.connections.fakeCats) 

210 outputCat = [] 

211 for fakeCatRef in fakeCats: 

212 cat = fakeCatRef.get( 

213 datasetType=self.config.connections.fakeCats) 

214 tractId = fakeCatRef.dataId["tract"] 

215 # Make sure all data is within the inner part of the tract. 

216 outputCat.append(cat[ 

217 skyMap.findTractIdArray(cat[self.config.ra_col], 

218 cat[self.config.dec_col], 

219 degrees=False) 

220 == tractId]) 

221 

222 return pd.concat(outputCat) 

223 

224 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

225 """Trim the fakeCat to select particular visit 

226 

227 Parameters 

228 ---------- 

229 fakeCat : `pandas.core.frame.DataFrame` 

230 The catalog of fake sources to add to the exposure 

231 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

232 The exposure to add the fake sources to 

233 

234 Returns 

235 ------- 

236 movingFakeCat : `pandas.DataFrame` 

237 All fakes that belong to the visit 

238 """ 

239 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

240 

241 return fakeCat[selected] 

242 

243 def _trimFakeCat(self, fakeCat, image): 

244 """Trim the fake cat to about the size of the input image. 

245 

246 Parameters 

247 ---------- 

248 fakeCat : `pandas.core.frame.DataFrame` 

249 The catalog of fake sources to be input 

250 image : `lsst.afw.image.exposure.exposure.ExposureF` 

251 The image into which the fake sources should be added 

252 skyMap : `lsst.skymap.SkyMap` 

253 SkyMap defining the tracts and patches the fakes are stored over. 

254 

255 Returns 

256 ------- 

257 fakeCats : `pandas.core.frame.DataFrame` 

258 The original fakeCat trimmed to the area of the image 

259 """ 

260 wcs = image.getWcs() 

261 

262 bbox = Box2D(image.getBBox()) 

263 

264 def trim(row): 

265 coord = SpherePoint(row[self.config.ra_col], 

266 row[self.config.dec_col], 

267 radians) 

268 cent = wcs.skyToPixel(coord) 

269 return bbox.contains(cent) 

270 

271 return fakeCat[fakeCat.apply(trim, axis=1)] 

272 

273 def _getVectors(self, ras, decs): 

274 """Convert ra dec to unit vectors on the sphere. 

275 

276 Parameters 

277 ---------- 

278 ras : `numpy.ndarray`, (N,) 

279 RA coordinates in radians. 

280 decs : `numpy.ndarray`, (N,) 

281 Dec coordinates in radians. 

282 

283 Returns 

284 ------- 

285 vectors : `numpy.ndarray`, (N, 3) 

286 Vectors on the unit sphere for the given RA/DEC values. 

287 """ 

288 vectors = np.empty((len(ras), 3)) 

289 

290 vectors[:, 2] = np.sin(decs) 

291 vectors[:, 0] = np.cos(decs) * np.cos(ras) 

292 vectors[:, 1] = np.cos(decs) * np.sin(ras) 

293 

294 return vectors 

295 

296 

297class MatchVariableFakesConnections(MatchFakesConnections): 

298 ccdVisitFakeMagnitudes = connTypes.Input( 

299 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

300 name="{fakesType}ccdVisitFakeMagnitudes", 

301 storageClass="DataFrame", 

302 dimensions=("instrument", "visit", "detector"), 

303 ) 

304 

305 

306class MatchVariableFakesConfig(MatchFakesConfig, 

307 pipelineConnections=MatchVariableFakesConnections): 

308 """Config for MatchFakesTask. 

309 """ 

310 pass 

311 

312 

313class MatchVariableFakesTask(MatchFakesTask): 

314 """Match injected fakes to their detected sources in the catalog and 

315 compute their expected brightness in a difference image assuming perfect 

316 subtraction. 

317 

318 This task is generally for injected sources that cannot be easily 

319 identified by their footprints such as in the case of detector sources 

320 post image differencing. 

321 """ 

322 _DefaultName = "matchVariableFakes" 

323 ConfigClass = MatchVariableFakesConfig 

324 

325 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

326 inputs = butlerQC.get(inputRefs) 

327 inputs["band"] = butlerQC.quantum.dataId["band"] 

328 

329 outputs = self.run(**inputs) 

330 butlerQC.put(outputs, outputRefs) 

331 

332 def run(self, fakeCats, ccdVisitFakeMagnitudes, skyMap, diffIm, associatedDiaSources, band): 

333 """Match fakes to detected diaSources within a difference image bound. 

334 

335 Parameters 

336 ---------- 

337 fakeCat : `pandas.DataFrame` 

338 Catalog of fakes to match to detected diaSources. 

339 diffIm : `lsst.afw.image.Exposure` 

340 Difference image where ``associatedDiaSources`` were detected in. 

341 associatedDiaSources : `pandas.DataFrame` 

342 Catalog of difference image sources detected in ``diffIm``. 

343 

344 Returns 

345 ------- 

346 result : `lsst.pipe.base.Struct` 

347 Results struct with components. 

348 

349 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has 

350 length of ``fakeCat``. (`pandas.DataFrame`) 

351 """ 

352 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

353 self.computeExpectedDiffMag(fakeCat, ccdVisitFakeMagnitudes, band) 

354 return self._processFakes(fakeCat, diffIm, associatedDiaSources) 

355 

356 def computeExpectedDiffMag(self, fakeCat, ccdVisitFakeMagnitudes, band): 

357 """Compute the magnitude expected in the difference image for this 

358 detector/visit. Modify fakeCat in place. 

359 

360 Negative magnitudes indicate that the source should be detected as 

361 a negative source. 

362 

363 Parameters 

364 ---------- 

365 fakeCat : `pandas.DataFrame` 

366 Catalog of fake sources. 

367 ccdVisitFakeMagnitudes : `pandas.DataFrame` 

368 Magnitudes for variable sources in this specific ccdVisit. 

369 band : `str` 

370 Band that this ccdVisit was observed in. 

371 """ 

372 magName = self.config.mag_col % band 

373 magnitudes = fakeCat[magName].to_numpy() 

374 visitMags = ccdVisitFakeMagnitudes["variableMag"].to_numpy() 

375 diffFlux = (visitMags * u.ABmag).to_value(u.nJy) - (magnitudes * u.ABmag).to_value(u.nJy) 

376 diffMag = np.where(diffFlux > 0, 

377 (diffFlux * u.nJy).to_value(u.ABmag), 

378 -(-diffFlux * u.nJy).to_value(u.ABmag)) 

379 

380 noVisit = ~fakeCat["isVisitSource"] 

381 noTemplate = ~fakeCat["isTemplateSource"] 

382 both = np.logical_and(fakeCat["isVisitSource"], 

383 fakeCat["isTemplateSource"]) 

384 

385 fakeCat.loc[noVisit, magName] = -magnitudes[noVisit] 

386 fakeCat.loc[noTemplate, magName] = visitMags[noTemplate] 

387 fakeCat.loc[both, magName] = diffMag[both]