lsst.pipe.tasks gb4d8b8e895+4b50854106
matchFakes.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import astropy.units as u
23import numpy as np
24import pandas as pd
25from scipy.spatial import cKDTree
26
27from lsst.geom import Box2D, radians, SpherePoint
28import lsst.pex.config as pexConfig
29from lsst.pipe.base import PipelineTask, PipelineTaskConnections, Struct
30import lsst.pipe.base.connectionTypes as connTypes
31from lsst.skymap import BaseSkyMap
32
33from lsst.pipe.tasks.insertFakes import InsertFakesConfig
34
35__all__ = ["MatchFakesTask",
36 "MatchFakesConfig",
37 "MatchVariableFakesConfig",
38 "MatchVariableFakesTask"]
39
40
41class MatchFakesConnections(PipelineTaskConnections,
42 defaultTemplates={"coaddName": "deep",
43 "fakesType": "fakes_"},
44 dimensions=("instrument",
45 "visit",
46 "detector")):
47 skyMap = connTypes.Input(
48 doc="Input definition of geometry/bbox and projection/wcs for "
49 "template exposures. Needed to test which tract to generate ",
50 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
51 dimensions=("skymap",),
52 storageClass="SkyMap",
53 )
54 fakeCats = connTypes.Input(
55 doc="Catalog of fake sources inserted into an image.",
56 name="{fakesType}fakeSourceCat",
57 storageClass="DataFrame",
58 dimensions=("tract", "skymap"),
59 deferLoad=True,
60 multiple=True
61 )
62 diffIm = connTypes.Input(
63 doc="Difference image on which the DiaSources were detected.",
64 name="{fakesType}{coaddName}Diff_differenceExp",
65 storageClass="ExposureF",
66 dimensions=("instrument", "visit", "detector"),
67 )
68 associatedDiaSources = connTypes.Input(
69 doc="A DiaSource catalog to match against fakeCat. Assumed "
70 "to be SDMified.",
71 name="{fakesType}{coaddName}Diff_assocDiaSrc",
72 storageClass="DataFrame",
73 dimensions=("instrument", "visit", "detector"),
74 )
75 matchedDiaSources = connTypes.Output(
76 doc="A catalog of those fakeCat sources that have a match in "
77 "associatedDiaSources. The schema is the union of the schemas for "
78 "``fakeCat`` and ``associatedDiaSources``.",
79 name="{fakesType}{coaddName}Diff_matchDiaSrc",
80 storageClass="DataFrame",
81 dimensions=("instrument", "visit", "detector"),
82 )
83
84
85class MatchFakesConfig(
86 InsertFakesConfig,
87 pipelineConnections=MatchFakesConnections):
88 """Config for MatchFakesTask.
89 """
90 matchDistanceArcseconds = pexConfig.RangeField(
91 doc="Distance in arcseconds to match within.",
92 dtype=float,
93 default=0.5,
94 min=0,
95 max=10,
96 )
97
98
99class MatchFakesTask(PipelineTask):
100 """Match a pre-existing catalog of fakes to a catalog of detections on
101 a difference image.
102
103 This task is generally for injected sources that cannot be easily
104 identified by their footprints such as in the case of detector sources
105 post image differencing.
106 """
107
108 _DefaultName = "matchFakes"
109 ConfigClass = MatchFakesConfig
110
111 def run(self, fakeCats, skyMap, diffIm, associatedDiaSources):
112 """Compose fakes into a single catalog and match fakes to detected
113 diaSources within a difference image bound.
114
115 Parameters
116 ----------
117 fakeCats : `pandas.DataFrame`
118 List of catalog of fakes to match to detected diaSources.
119 skyMap : `lsst.skymap.SkyMap`
120 SkyMap defining the tracts and patches the fakes are stored over.
121 diffIm : `lsst.afw.image.Exposure`
122 Difference image where ``associatedDiaSources`` were detected.
123 associatedDiaSources : `pandas.DataFrame`
124 Catalog of difference image sources detected in ``diffIm``.
125
126 Returns
127 -------
128 result : `lsst.pipe.base.Struct`
129 Results struct with components.
130
131 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
132 length of ``fakeCat``. (`pandas.DataFrame`)
133 """
134 fakeCat = self.composeFakeCat(fakeCats, skyMap)
135 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
136
137 def _processFakes(self, fakeCat, diffIm, associatedDiaSources):
138 """Match fakes to detected diaSources within a difference image bound.
139
140 Parameters
141 ----------
142 fakeCat : `pandas.DataFrame`
143 Catalog of fakes to match to detected diaSources.
144 diffIm : `lsst.afw.image.Exposure`
145 Difference image where ``associatedDiaSources`` were detected.
146 associatedDiaSources : `pandas.DataFrame`
147 Catalog of difference image sources detected in ``diffIm``.
148
149 Returns
150 -------
151 result : `lsst.pipe.base.Struct`
152 Results struct with components.
153
154 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
155 length of ``fakeCat``. (`pandas.DataFrame`)
156 """
157 trimmedFakes = self._trimFakeCat(fakeCat, diffIm)
158 nPossibleFakes = len(trimmedFakes)
159
160 fakeVects = self._getVectors(trimmedFakes[self.config.ra_col],
161 trimmedFakes[self.config.dec_col])
162 diaSrcVects = self._getVectors(
163 np.radians(associatedDiaSources.loc[:, "ra"]),
164 np.radians(associatedDiaSources.loc[:, "decl"]))
165
166 diaSrcTree = cKDTree(diaSrcVects)
167 dist, idxs = diaSrcTree.query(
168 fakeVects,
169 distance_upper_bound=np.radians(self.config.matchDistanceArcseconds / 3600))
170 nFakesFound = np.isfinite(dist).sum()
171
172 self.log.info("Found %d out of %d possible.", nFakesFound, nPossibleFakes)
173 diaSrcIds = associatedDiaSources.iloc[np.where(np.isfinite(dist), idxs, 0)]["diaSourceId"].to_numpy()
174 matchedFakes = trimmedFakes.assign(diaSourceId=np.where(np.isfinite(dist), diaSrcIds, 0))
175
176 return Struct(
177 matchedDiaSources=matchedFakes.merge(
178 associatedDiaSources.reset_index(drop=True), on="diaSourceId", how="left")
179 )
180
181 def composeFakeCat(self, fakeCats, skyMap):
182 """Concatenate the fakeCats from tracts that may cover the exposure.
183
184 Parameters
185 ----------
186 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
187 Set of fake cats to concatenate.
188 skyMap : `lsst.skymap.SkyMap`
189 SkyMap defining the geometry of the tracts and patches.
190
191 Returns
192 -------
193 combinedFakeCat : `pandas.DataFrame`
194 All fakes that cover the inner polygon of the tracts in this
195 quantum.
196 """
197 if len(fakeCats) == 1:
198 return fakeCats[0].get(
199 datasetType=self.config.connections.fakeCats)
200 outputCat = []
201 for fakeCatRef in fakeCats:
202 cat = fakeCatRef.get(
203 datasetType=self.config.connections.fakeCats)
204 tractId = fakeCatRef.dataId["tract"]
205 # Make sure all data is within the inner part of the tract.
206 outputCat.append(cat[
207 skyMap.findTractIdArray(cat[self.config.ra_col],
208 cat[self.config.dec_col],
209 degrees=False)
210 == tractId])
211
212 return pd.concat(outputCat)
213
214 def _trimFakeCat(self, fakeCat, image):
215 """Trim the fake cat to about the size of the input image.
216
217 Parameters
218 ----------
219 fakeCat : `pandas.core.frame.DataFrame`
220 The catalog of fake sources to be input
221 image : `lsst.afw.image.exposure.exposure.ExposureF`
222 The image into which the fake sources should be added
223 skyMap : `lsst.skymap.SkyMap`
224 SkyMap defining the tracts and patches the fakes are stored over.
225
226 Returns
227 -------
228 fakeCats : `pandas.core.frame.DataFrame`
229 The original fakeCat trimmed to the area of the image
230 """
231 wcs = image.getWcs()
232
233 bbox = Box2D(image.getBBox())
234
235 def trim(row):
236 coord = SpherePoint(row[self.config.ra_col],
237 row[self.config.dec_col],
238 radians)
239 cent = wcs.skyToPixel(coord)
240 return bbox.contains(cent)
241
242 return fakeCat[fakeCat.apply(trim, axis=1)]
243
244 def _getVectors(self, ras, decs):
245 """Convert ra dec to unit vectors on the sphere.
246
247 Parameters
248 ----------
249 ras : `numpy.ndarray`, (N,)
250 RA coordinates in radians.
251 decs : `numpy.ndarray`, (N,)
252 Dec coordinates in radians.
253
254 Returns
255 -------
256 vectors : `numpy.ndarray`, (N, 3)
257 Vectors on the unit sphere for the given RA/DEC values.
258 """
259 vectors = np.empty((len(ras), 3))
260
261 vectors[:, 2] = np.sin(decs)
262 vectors[:, 0] = np.cos(decs) * np.cos(ras)
263 vectors[:, 1] = np.cos(decs) * np.sin(ras)
264
265 return vectors
266
267
268class MatchVariableFakesConnections(MatchFakesConnections):
269 ccdVisitFakeMagnitudes = connTypes.Input(
270 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
271 name="{fakesType}ccdVisitFakeMagnitudes",
272 storageClass="DataFrame",
273 dimensions=("instrument", "visit", "detector"),
274 )
275
276
277class MatchVariableFakesConfig(MatchFakesConfig,
278 pipelineConnections=MatchVariableFakesConnections):
279 """Config for MatchFakesTask.
280 """
281 pass
282
283
284class MatchVariableFakesTask(MatchFakesTask):
285 """Match injected fakes to their detected sources in the catalog and
286 compute their expected brightness in a difference image assuming perfect
287 subtraction.
288
289 This task is generally for injected sources that cannot be easily
290 identified by their footprints such as in the case of detector sources
291 post image differencing.
292 """
293 _DefaultName = "matchVariableFakes"
294 ConfigClass = MatchVariableFakesConfig
295
296 def runQuantum(self, butlerQC, inputRefs, outputRefs):
297 inputs = butlerQC.get(inputRefs)
298 inputs["band"] = butlerQC.quantum.dataId["band"]
299
300 outputs = self.run(**inputs)
301 butlerQC.put(outputs, outputRefs)
302
303 def run(self, fakeCats, ccdVisitFakeMagnitudes, skyMap, diffIm, associatedDiaSources, band):
304 """Match fakes to detected diaSources within a difference image bound.
305
306 Parameters
307 ----------
308 fakeCat : `pandas.DataFrame`
309 Catalog of fakes to match to detected diaSources.
310 diffIm : `lsst.afw.image.Exposure`
311 Difference image where ``associatedDiaSources`` were detected in.
312 associatedDiaSources : `pandas.DataFrame`
313 Catalog of difference image sources detected in ``diffIm``.
314
315 Returns
316 -------
317 result : `lsst.pipe.base.Struct`
318 Results struct with components.
319
320 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
321 length of ``fakeCat``. (`pandas.DataFrame`)
322 """
323 fakeCat = self.composeFakeCat(fakeCats, skyMap)
324 self.computeExpectedDiffMag(fakeCat, ccdVisitFakeMagnitudes, band)
325 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
326
327 def computeExpectedDiffMag(self, fakeCat, ccdVisitFakeMagnitudes, band):
328 """Compute the magnitude expected in the difference image for this
329 detector/visit. Modify fakeCat in place.
330
331 Negative magnitudes indicate that the source should be detected as
332 a negative source.
333
334 Parameters
335 ----------
336 fakeCat : `pandas.DataFrame`
337 Catalog of fake sources.
338 ccdVisitFakeMagnitudes : `pandas.DataFrame`
339 Magnitudes for variable sources in this specific ccdVisit.
340 band : `str`
341 Band that this ccdVisit was observed in.
342 """
343 magName = self.config.mag_col % band
344 magnitudes = fakeCat[magName].to_numpy()
345 visitMags = ccdVisitFakeMagnitudes["variableMag"].to_numpy()
346 diffFlux = (visitMags * u.ABmag).to_value(u.nJy) - (magnitudes * u.ABmag).to_value(u.nJy)
347 diffMag = np.where(diffFlux > 0,
348 (diffFlux * u.nJy).to_value(u.ABmag),
349 -(-diffFlux * u.nJy).to_value(u.ABmag))
350
351 noVisit = ~fakeCat["isVisitSource"]
352 noTemplate = ~fakeCat["isTemplateSource"]
353 both = np.logical_and(fakeCat["isVisitSource"],
354 fakeCat["isTemplateSource"])
355
356 fakeCat.loc[noVisit, magName] = -magnitudes[noVisit]
357 fakeCat.loc[noTemplate, magName] = visitMags[noTemplate]
358 fakeCat.loc[both, magName] = diffMag[both]