22import astropy.units
as u
25from scipy.spatial
import cKDTree
27from lsst.geom import Box2D, radians, SpherePoint
29from lsst.pipe.base import PipelineTask, PipelineTaskConnections, Struct
30import lsst.pipe.base.connectionTypes
as connTypes
35__all__ = [
"MatchFakesTask",
37 "MatchVariableFakesConfig",
38 "MatchVariableFakesTask"]
42 defaultTemplates={
"coaddName":
"deep",
43 "fakesType":
"fakes_"},
44 dimensions=(
"instrument",
47 skyMap = connTypes.Input(
48 doc=
"Input definition of geometry/bbox and projection/wcs for "
49 "template exposures. Needed to test which tract to generate ",
50 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
51 dimensions=(
"skymap",),
52 storageClass=
"SkyMap",
54 fakeCats = connTypes.Input(
55 doc=
"Catalog of fake sources inserted into an image.",
56 name=
"{fakesType}fakeSourceCat",
57 storageClass=
"DataFrame",
58 dimensions=(
"tract",
"skymap"),
62 diffIm = connTypes.Input(
63 doc=
"Difference image on which the DiaSources were detected.",
64 name=
"{fakesType}{coaddName}Diff_differenceExp",
65 storageClass=
"ExposureF",
66 dimensions=(
"instrument",
"visit",
"detector"),
68 associatedDiaSources = connTypes.Input(
69 doc=
"A DiaSource catalog to match against fakeCat. Assumed "
71 name=
"{fakesType}{coaddName}Diff_assocDiaSrc",
72 storageClass=
"DataFrame",
73 dimensions=(
"instrument",
"visit",
"detector"),
75 matchedDiaSources = connTypes.Output(
76 doc=
"A catalog of those fakeCat sources that have a match in "
77 "associatedDiaSources. The schema is the union of the schemas for "
78 "``fakeCat`` and ``associatedDiaSources``.",
79 name=
"{fakesType}{coaddName}Diff_matchDiaSrc",
80 storageClass=
"DataFrame",
81 dimensions=(
"instrument",
"visit",
"detector"),
85class MatchFakesConfig(
87 pipelineConnections=MatchFakesConnections):
88 """Config for MatchFakesTask.
90 matchDistanceArcseconds = pexConfig.RangeField(
91 doc="Distance in arcseconds to match within.",
99class MatchFakesTask(PipelineTask):
100 """Match a pre-existing catalog of fakes to a catalog of detections on
103 This task is generally
for injected sources that cannot be easily
104 identified by their footprints such
as in the case of detector sources
105 post image differencing.
108 _DefaultName = "matchFakes"
109 ConfigClass = MatchFakesConfig
111 def run(self, fakeCats, skyMap, diffIm, associatedDiaSources):
112 """Compose fakes into a single catalog and match fakes to detected
113 diaSources within a difference image bound.
117 fakeCats : `pandas.DataFrame`
118 List of catalog of fakes to match to detected diaSources.
119 skyMap : `lsst.skymap.SkyMap`
120 SkyMap defining the tracts and patches the fakes are stored over.
122 Difference image where ``associatedDiaSources`` were detected.
123 associatedDiaSources : `pandas.DataFrame`
124 Catalog of difference image sources detected
in ``diffIm``.
128 result : `lsst.pipe.base.Struct`
129 Results struct
with components.
131 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
132 length of ``fakeCat``. (`pandas.DataFrame`)
134 fakeCat = self.composeFakeCat(fakeCats, skyMap)
135 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
137 def _processFakes(self, fakeCat, diffIm, associatedDiaSources):
138 """Match fakes to detected diaSources within a difference image bound.
142 fakeCat : `pandas.DataFrame`
143 Catalog of fakes to match to detected diaSources.
145 Difference image where ``associatedDiaSources`` were detected.
146 associatedDiaSources : `pandas.DataFrame`
147 Catalog of difference image sources detected in ``diffIm``.
151 result : `lsst.pipe.base.Struct`
152 Results struct
with components.
154 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
155 length of ``fakeCat``. (`pandas.DataFrame`)
157 trimmedFakes = self._trimFakeCat(fakeCat, diffIm)
158 nPossibleFakes = len(trimmedFakes)
160 fakeVects = self._getVectors(trimmedFakes[self.config.ra_col],
161 trimmedFakes[self.config.dec_col])
162 diaSrcVects = self._getVectors(
163 np.radians(associatedDiaSources.loc[:, "ra"]),
164 np.radians(associatedDiaSources.loc[:,
"decl"]))
166 diaSrcTree = cKDTree(diaSrcVects)
167 dist, idxs = diaSrcTree.query(
169 distance_upper_bound=np.radians(self.config.matchDistanceArcseconds / 3600))
170 nFakesFound = np.isfinite(dist).sum()
172 self.log.info(
"Found %d out of %d possible.", nFakesFound, nPossibleFakes)
173 diaSrcIds = associatedDiaSources.iloc[np.where(np.isfinite(dist), idxs, 0)][
"diaSourceId"].to_numpy()
174 matchedFakes = trimmedFakes.assign(diaSourceId=np.where(np.isfinite(dist), diaSrcIds, 0))
177 matchedDiaSources=matchedFakes.merge(
178 associatedDiaSources.reset_index(drop=
True), on=
"diaSourceId", how=
"left")
181 def composeFakeCat(self, fakeCats, skyMap):
182 """Concatenate the fakeCats from tracts that may cover the exposure.
186 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
187 Set of fake cats to concatenate.
188 skyMap : `lsst.skymap.SkyMap`
189 SkyMap defining the geometry of the tracts and patches.
193 combinedFakeCat : `pandas.DataFrame`
194 All fakes that cover the inner polygon of the tracts
in this
197 if len(fakeCats) == 1:
198 return fakeCats[0].get(
199 datasetType=self.config.connections.fakeCats)
201 for fakeCatRef
in fakeCats:
202 cat = fakeCatRef.get(
203 datasetType=self.config.connections.fakeCats)
204 tractId = fakeCatRef.dataId[
"tract"]
206 outputCat.append(cat[
207 skyMap.findTractIdArray(cat[self.config.ra_col],
208 cat[self.config.dec_col],
212 return pd.concat(outputCat)
214 def _trimFakeCat(self, fakeCat, image):
215 """Trim the fake cat to about the size of the input image.
219 fakeCat : `pandas.core.frame.DataFrame`
220 The catalog of fake sources to be input
221 image : `lsst.afw.image.exposure.exposure.ExposureF`
222 The image into which the fake sources should be added
223 skyMap : `lsst.skymap.SkyMap`
224 SkyMap defining the tracts and patches the fakes are stored over.
228 fakeCats : `pandas.core.frame.DataFrame`
229 The original fakeCat trimmed to the area of the image
233 bbox = Box2D(image.getBBox())
237 row[self.config.dec_col],
239 cent = wcs.skyToPixel(coord)
240 return bbox.contains(cent)
242 return fakeCat[fakeCat.apply(trim, axis=1)]
244 def _getVectors(self, ras, decs):
245 """Convert ra dec to unit vectors on the sphere.
249 ras : `numpy.ndarray`, (N,)
250 RA coordinates in radians.
251 decs : `numpy.ndarray`, (N,)
252 Dec coordinates
in radians.
256 vectors : `numpy.ndarray`, (N, 3)
257 Vectors on the unit sphere
for the given RA/DEC values.
259 vectors = np.empty((len(ras), 3))
261 vectors[:, 2] = np.sin(decs)
262 vectors[:, 0] = np.cos(decs) * np.cos(ras)
263 vectors[:, 1] = np.cos(decs) * np.sin(ras)
269 ccdVisitFakeMagnitudes = connTypes.Input(
270 doc=
"Catalog of fakes with magnitudes scattered for this ccdVisit.",
271 name=
"{fakesType}ccdVisitFakeMagnitudes",
272 storageClass=
"DataFrame",
273 dimensions=(
"instrument",
"visit",
"detector"),
277class MatchVariableFakesConfig(MatchFakesConfig,
278 pipelineConnections=MatchVariableFakesConnections):
279 """Config for MatchFakesTask.
284class MatchVariableFakesTask(MatchFakesTask):
285 """Match injected fakes to their detected sources in the catalog and
286 compute their expected brightness in a difference image assuming perfect
289 This task
is generally
for injected sources that cannot be easily
290 identified by their footprints such
as in the case of detector sources
291 post image differencing.
293 _DefaultName = "matchVariableFakes"
294 ConfigClass = MatchVariableFakesConfig
296 def runQuantum(self, butlerQC, inputRefs, outputRefs):
297 inputs = butlerQC.get(inputRefs)
298 inputs[
"band"] = butlerQC.quantum.dataId[
"band"]
300 outputs = self.run(**inputs)
301 butlerQC.put(outputs, outputRefs)
303 def run(self, fakeCats, ccdVisitFakeMagnitudes, skyMap, diffIm, associatedDiaSources, band):
304 """Match fakes to detected diaSources within a difference image bound.
308 fakeCat : `pandas.DataFrame`
309 Catalog of fakes to match to detected diaSources.
311 Difference image where ``associatedDiaSources`` were detected in.
312 associatedDiaSources : `pandas.DataFrame`
313 Catalog of difference image sources detected
in ``diffIm``.
317 result : `lsst.pipe.base.Struct`
318 Results struct
with components.
320 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
321 length of ``fakeCat``. (`pandas.DataFrame`)
323 fakeCat = self.composeFakeCat(fakeCats, skyMap)
324 self.computeExpectedDiffMag(fakeCat, ccdVisitFakeMagnitudes, band)
325 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
327 def computeExpectedDiffMag(self, fakeCat, ccdVisitFakeMagnitudes, band):
328 """Compute the magnitude expected in the difference image for this
329 detector/visit. Modify fakeCat in place.
331 Negative magnitudes indicate that the source should be detected
as
336 fakeCat : `pandas.DataFrame`
337 Catalog of fake sources.
338 ccdVisitFakeMagnitudes : `pandas.DataFrame`
339 Magnitudes
for variable sources
in this specific ccdVisit.
341 Band that this ccdVisit was observed
in.
343 magName = self.config.mag_col % band
344 magnitudes = fakeCat[magName].to_numpy()
345 visitMags = ccdVisitFakeMagnitudes["variableMag"].to_numpy()
346 diffFlux = (visitMags * u.ABmag).to_value(u.nJy) - (magnitudes * u.ABmag).to_value(u.nJy)
347 diffMag = np.where(diffFlux > 0,
348 (diffFlux * u.nJy).to_value(u.ABmag),
349 -(-diffFlux * u.nJy).to_value(u.ABmag))
351 noVisit = ~fakeCat[
"isVisitSource"]
352 noTemplate = ~fakeCat[
"isTemplateSource"]
353 both = np.logical_and(fakeCat[
"isVisitSource"],
354 fakeCat[
"isTemplateSource"])
356 fakeCat.loc[noVisit, magName] = -magnitudes[noVisit]
357 fakeCat.loc[noTemplate, magName] = visitMags[noTemplate]
358 fakeCat.loc[both, magName] = diffMag[both]