lsst.pipe.tasks g84bf843041+72efc5b9b3
matchFakes.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import astropy.units as u
23import numpy as np
24import pandas as pd
25from scipy.spatial import cKDTree
26
27from lsst.geom import Box2D, radians, SpherePoint
28import lsst.pex.config as pexConfig
29from lsst.pipe.base import PipelineTask, PipelineTaskConnections, Struct
30import lsst.pipe.base.connectionTypes as connTypes
31from lsst.skymap import BaseSkyMap
32
33from lsst.pipe.tasks.insertFakes import InsertFakesConfig
34
35__all__ = ["MatchFakesTask",
36 "MatchFakesConfig",
37 "MatchVariableFakesConfig",
38 "MatchVariableFakesTask"]
39
40
41class MatchFakesConnections(PipelineTaskConnections,
42 defaultTemplates={"coaddName": "deep",
43 "fakesType": "fakes_"},
44 dimensions=("instrument",
45 "visit",
46 "detector")):
47 skyMap = connTypes.Input(
48 doc="Input definition of geometry/bbox and projection/wcs for "
49 "template exposures. Needed to test which tract to generate ",
50 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
51 dimensions=("skymap",),
52 storageClass="SkyMap",
53 )
54 fakeCats = connTypes.Input(
55 doc="Catalog of fake sources inserted into an image.",
56 name="{fakesType}fakeSourceCat",
57 storageClass="DataFrame",
58 dimensions=("tract", "skymap"),
59 deferLoad=True,
60 multiple=True
61 )
62 diffIm = connTypes.Input(
63 doc="Difference image on which the DiaSources were detected.",
64 name="{fakesType}{coaddName}Diff_differenceExp",
65 storageClass="ExposureF",
66 dimensions=("instrument", "visit", "detector"),
67 )
68 associatedDiaSources = connTypes.Input(
69 doc="A DiaSource catalog to match against fakeCat. Assumed "
70 "to be SDMified.",
71 name="{fakesType}{coaddName}Diff_assocDiaSrc",
72 storageClass="DataFrame",
73 dimensions=("instrument", "visit", "detector"),
74 )
75 matchedDiaSources = connTypes.Output(
76 doc="A catalog of those fakeCat sources that have a match in "
77 "associatedDiaSources. The schema is the union of the schemas for "
78 "``fakeCat`` and ``associatedDiaSources``.",
79 name="{fakesType}{coaddName}Diff_matchDiaSrc",
80 storageClass="DataFrame",
81 dimensions=("instrument", "visit", "detector"),
82 )
83
84
85class MatchFakesConfig(
86 InsertFakesConfig,
87 pipelineConnections=MatchFakesConnections):
88 """Config for MatchFakesTask.
89 """
90 matchDistanceArcseconds = pexConfig.RangeField(
91 doc="Distance in arcseconds to match within.",
92 dtype=float,
93 default=0.5,
94 min=0,
95 max=10,
96 )
97
98 doMatchVisit = pexConfig.Field(
99 dtype=bool,
100 default=False,
101 doc="Match visit to trim the fakeCat"
102 )
103
104
105class MatchFakesTask(PipelineTask):
106 """Match a pre-existing catalog of fakes to a catalog of detections on
107 a difference image.
108
109 This task is generally for injected sources that cannot be easily
110 identified by their footprints such as in the case of detector sources
111 post image differencing.
112 """
113
114 _DefaultName = "matchFakes"
115 ConfigClass = MatchFakesConfig
116
117 def run(self, fakeCats, skyMap, diffIm, associatedDiaSources):
118 """Compose fakes into a single catalog and match fakes to detected
119 diaSources within a difference image bound.
120
121 Parameters
122 ----------
123 fakeCats : `pandas.DataFrame`
124 List of catalog of fakes to match to detected diaSources.
125 skyMap : `lsst.skymap.SkyMap`
126 SkyMap defining the tracts and patches the fakes are stored over.
127 diffIm : `lsst.afw.image.Exposure`
128 Difference image where ``associatedDiaSources`` were detected.
129 associatedDiaSources : `pandas.DataFrame`
130 Catalog of difference image sources detected in ``diffIm``.
131
132 Returns
133 -------
134 result : `lsst.pipe.base.Struct`
135 Results struct with components.
136
137 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
138 length of ``fakeCat``. (`pandas.DataFrame`)
139 """
140 fakeCat = self.composeFakeCat(fakeCats, skyMap)
141
142 if self.config.doMatchVisit:
143 fakeCat = self.getVisitMatchedFakeCat(fakeCat, diffIm)
144
145 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
146
147 def _processFakes(self, fakeCat, diffIm, associatedDiaSources):
148 """Match fakes to detected diaSources within a difference image bound.
149
150 Parameters
151 ----------
152 fakeCat : `pandas.DataFrame`
153 Catalog of fakes to match to detected diaSources.
154 diffIm : `lsst.afw.image.Exposure`
155 Difference image where ``associatedDiaSources`` were detected.
156 associatedDiaSources : `pandas.DataFrame`
157 Catalog of difference image sources detected in ``diffIm``.
158
159 Returns
160 -------
161 result : `lsst.pipe.base.Struct`
162 Results struct with components.
163
164 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
165 length of ``fakeCat``. (`pandas.DataFrame`)
166 """
167 trimmedFakes = self._trimFakeCat(fakeCat, diffIm)
168 nPossibleFakes = len(trimmedFakes)
169
170 fakeVects = self._getVectors(trimmedFakes[self.config.ra_col],
171 trimmedFakes[self.config.dec_col])
172 diaSrcVects = self._getVectors(
173 np.radians(associatedDiaSources.loc[:, "ra"]),
174 np.radians(associatedDiaSources.loc[:, "decl"]))
175
176 diaSrcTree = cKDTree(diaSrcVects)
177 dist, idxs = diaSrcTree.query(
178 fakeVects,
179 distance_upper_bound=np.radians(self.config.matchDistanceArcseconds / 3600))
180 nFakesFound = np.isfinite(dist).sum()
181
182 self.log.info("Found %d out of %d possible.", nFakesFound, nPossibleFakes)
183 diaSrcIds = associatedDiaSources.iloc[np.where(np.isfinite(dist), idxs, 0)]["diaSourceId"].to_numpy()
184 matchedFakes = trimmedFakes.assign(diaSourceId=np.where(np.isfinite(dist), diaSrcIds, 0))
185
186 return Struct(
187 matchedDiaSources=matchedFakes.merge(
188 associatedDiaSources.reset_index(drop=True), on="diaSourceId", how="left")
189 )
190
191 def composeFakeCat(self, fakeCats, skyMap):
192 """Concatenate the fakeCats from tracts that may cover the exposure.
193
194 Parameters
195 ----------
196 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
197 Set of fake cats to concatenate.
198 skyMap : `lsst.skymap.SkyMap`
199 SkyMap defining the geometry of the tracts and patches.
200
201 Returns
202 -------
203 combinedFakeCat : `pandas.DataFrame`
204 All fakes that cover the inner polygon of the tracts in this
205 quantum.
206 """
207 if len(fakeCats) == 1:
208 return fakeCats[0].get(
209 datasetType=self.config.connections.fakeCats)
210 outputCat = []
211 for fakeCatRef in fakeCats:
212 cat = fakeCatRef.get(
213 datasetType=self.config.connections.fakeCats)
214 tractId = fakeCatRef.dataId["tract"]
215 # Make sure all data is within the inner part of the tract.
216 outputCat.append(cat[
217 skyMap.findTractIdArray(cat[self.config.ra_col],
218 cat[self.config.dec_col],
219 degrees=False)
220 == tractId])
221
222 return pd.concat(outputCat)
223
224 def getVisitMatchedFakeCat(self, fakeCat, exposure):
225 """Trim the fakeCat to select particular visit
226
227 Parameters
228 ----------
229 fakeCat : `pandas.core.frame.DataFrame`
230 The catalog of fake sources to add to the exposure
231 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
232 The exposure to add the fake sources to
233
234 Returns
235 -------
236 movingFakeCat : `pandas.DataFrame`
237 All fakes that belong to the visit
238 """
239 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
240
241 return fakeCat[selected]
242
243 def _trimFakeCat(self, fakeCat, image):
244 """Trim the fake cat to about the size of the input image.
245
246 Parameters
247 ----------
248 fakeCat : `pandas.core.frame.DataFrame`
249 The catalog of fake sources to be input
250 image : `lsst.afw.image.exposure.exposure.ExposureF`
251 The image into which the fake sources should be added
252 skyMap : `lsst.skymap.SkyMap`
253 SkyMap defining the tracts and patches the fakes are stored over.
254
255 Returns
256 -------
257 fakeCats : `pandas.core.frame.DataFrame`
258 The original fakeCat trimmed to the area of the image
259 """
260 wcs = image.getWcs()
261
262 bbox = Box2D(image.getBBox())
263
264 def trim(row):
265 coord = SpherePoint(row[self.config.ra_col],
266 row[self.config.dec_col],
267 radians)
268 cent = wcs.skyToPixel(coord)
269 return bbox.contains(cent)
270
271 return fakeCat[fakeCat.apply(trim, axis=1)]
272
273 def _getVectors(self, ras, decs):
274 """Convert ra dec to unit vectors on the sphere.
275
276 Parameters
277 ----------
278 ras : `numpy.ndarray`, (N,)
279 RA coordinates in radians.
280 decs : `numpy.ndarray`, (N,)
281 Dec coordinates in radians.
282
283 Returns
284 -------
285 vectors : `numpy.ndarray`, (N, 3)
286 Vectors on the unit sphere for the given RA/DEC values.
287 """
288 vectors = np.empty((len(ras), 3))
289
290 vectors[:, 2] = np.sin(decs)
291 vectors[:, 0] = np.cos(decs) * np.cos(ras)
292 vectors[:, 1] = np.cos(decs) * np.sin(ras)
293
294 return vectors
295
296
297class MatchVariableFakesConnections(MatchFakesConnections):
298 ccdVisitFakeMagnitudes = connTypes.Input(
299 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
300 name="{fakesType}ccdVisitFakeMagnitudes",
301 storageClass="DataFrame",
302 dimensions=("instrument", "visit", "detector"),
303 )
304
305
306class MatchVariableFakesConfig(MatchFakesConfig,
307 pipelineConnections=MatchVariableFakesConnections):
308 """Config for MatchFakesTask.
309 """
310 pass
311
312
313class MatchVariableFakesTask(MatchFakesTask):
314 """Match injected fakes to their detected sources in the catalog and
315 compute their expected brightness in a difference image assuming perfect
316 subtraction.
317
318 This task is generally for injected sources that cannot be easily
319 identified by their footprints such as in the case of detector sources
320 post image differencing.
321 """
322 _DefaultName = "matchVariableFakes"
323 ConfigClass = MatchVariableFakesConfig
324
325 def runQuantum(self, butlerQC, inputRefs, outputRefs):
326 inputs = butlerQC.get(inputRefs)
327 inputs["band"] = butlerQC.quantum.dataId["band"]
328
329 outputs = self.run(**inputs)
330 butlerQC.put(outputs, outputRefs)
331
332 def run(self, fakeCats, ccdVisitFakeMagnitudes, skyMap, diffIm, associatedDiaSources, band):
333 """Match fakes to detected diaSources within a difference image bound.
334
335 Parameters
336 ----------
337 fakeCat : `pandas.DataFrame`
338 Catalog of fakes to match to detected diaSources.
339 diffIm : `lsst.afw.image.Exposure`
340 Difference image where ``associatedDiaSources`` were detected in.
341 associatedDiaSources : `pandas.DataFrame`
342 Catalog of difference image sources detected in ``diffIm``.
343
344 Returns
345 -------
346 result : `lsst.pipe.base.Struct`
347 Results struct with components.
348
349 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
350 length of ``fakeCat``. (`pandas.DataFrame`)
351 """
352 fakeCat = self.composeFakeCat(fakeCats, skyMap)
353 self.computeExpectedDiffMag(fakeCat, ccdVisitFakeMagnitudes, band)
354 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
355
356 def computeExpectedDiffMag(self, fakeCat, ccdVisitFakeMagnitudes, band):
357 """Compute the magnitude expected in the difference image for this
358 detector/visit. Modify fakeCat in place.
359
360 Negative magnitudes indicate that the source should be detected as
361 a negative source.
362
363 Parameters
364 ----------
365 fakeCat : `pandas.DataFrame`
366 Catalog of fake sources.
367 ccdVisitFakeMagnitudes : `pandas.DataFrame`
368 Magnitudes for variable sources in this specific ccdVisit.
369 band : `str`
370 Band that this ccdVisit was observed in.
371 """
372 magName = self.config.mag_col % band
373 magnitudes = fakeCat[magName].to_numpy()
374 visitMags = ccdVisitFakeMagnitudes["variableMag"].to_numpy()
375 diffFlux = (visitMags * u.ABmag).to_value(u.nJy) - (magnitudes * u.ABmag).to_value(u.nJy)
376 diffMag = np.where(diffFlux > 0,
377 (diffFlux * u.nJy).to_value(u.ABmag),
378 -(-diffFlux * u.nJy).to_value(u.ABmag))
379
380 noVisit = ~fakeCat["isVisitSource"]
381 noTemplate = ~fakeCat["isTemplateSource"]
382 both = np.logical_and(fakeCat["isVisitSource"],
383 fakeCat["isTemplateSource"])
384
385 fakeCat.loc[noVisit, magName] = -magnitudes[noVisit]
386 fakeCat.loc[noTemplate, magName] = visitMags[noTemplate]
387 fakeCat.loc[both, magName] = diffMag[both]