22import astropy.units
as u
25from scipy.spatial
import cKDTree
27from lsst.geom import Box2D, radians, SpherePoint
29from lsst.pipe.base import PipelineTask, PipelineTaskConnections, Struct
30import lsst.pipe.base.connectionTypes
as connTypes
35__all__ = [
"MatchFakesTask",
37 "MatchVariableFakesConfig",
38 "MatchVariableFakesTask"]
42 defaultTemplates={
"coaddName":
"deep",
43 "fakesType":
"fakes_"},
44 dimensions=(
"instrument",
47 skyMap = connTypes.Input(
48 doc=
"Input definition of geometry/bbox and projection/wcs for "
49 "template exposures. Needed to test which tract to generate ",
50 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
51 dimensions=(
"skymap",),
52 storageClass=
"SkyMap",
54 fakeCats = connTypes.Input(
55 doc=
"Catalog of fake sources inserted into an image.",
56 name=
"{fakesType}fakeSourceCat",
57 storageClass=
"DataFrame",
58 dimensions=(
"tract",
"skymap"),
62 diffIm = connTypes.Input(
63 doc=
"Difference image on which the DiaSources were detected.",
64 name=
"{fakesType}{coaddName}Diff_differenceExp",
65 storageClass=
"ExposureF",
66 dimensions=(
"instrument",
"visit",
"detector"),
68 associatedDiaSources = connTypes.Input(
69 doc=
"A DiaSource catalog to match against fakeCat. Assumed "
71 name=
"{fakesType}{coaddName}Diff_assocDiaSrc",
72 storageClass=
"DataFrame",
73 dimensions=(
"instrument",
"visit",
"detector"),
75 matchedDiaSources = connTypes.Output(
76 doc=
"A catalog of those fakeCat sources that have a match in "
77 "associatedDiaSources. The schema is the union of the schemas for "
78 "``fakeCat`` and ``associatedDiaSources``.",
79 name=
"{fakesType}{coaddName}Diff_matchDiaSrc",
80 storageClass=
"DataFrame",
81 dimensions=(
"instrument",
"visit",
"detector"),
85class MatchFakesConfig(
87 pipelineConnections=MatchFakesConnections):
88 """Config for MatchFakesTask.
90 matchDistanceArcseconds = pexConfig.RangeField(
91 doc="Distance in arcseconds to match within.",
98 doMatchVisit = pexConfig.Field(
101 doc=
"Match visit to trim the fakeCat"
105class MatchFakesTask(PipelineTask):
106 """Match a pre-existing catalog of fakes to a catalog of detections on
109 This task is generally
for injected sources that cannot be easily
110 identified by their footprints such
as in the case of detector sources
111 post image differencing.
114 _DefaultName = "matchFakes"
115 ConfigClass = MatchFakesConfig
117 def run(self, fakeCats, skyMap, diffIm, associatedDiaSources):
118 """Compose fakes into a single catalog and match fakes to detected
119 diaSources within a difference image bound.
123 fakeCats : `pandas.DataFrame`
124 List of catalog of fakes to match to detected diaSources.
125 skyMap : `lsst.skymap.SkyMap`
126 SkyMap defining the tracts and patches the fakes are stored over.
128 Difference image where ``associatedDiaSources`` were detected.
129 associatedDiaSources : `pandas.DataFrame`
130 Catalog of difference image sources detected
in ``diffIm``.
134 result : `lsst.pipe.base.Struct`
135 Results struct
with components.
137 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
138 length of ``fakeCat``. (`pandas.DataFrame`)
140 fakeCat = self.composeFakeCat(fakeCats, skyMap)
142 if self.config.doMatchVisit:
143 fakeCat = self.getVisitMatchedFakeCat(fakeCat, diffIm)
145 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
147 def _processFakes(self, fakeCat, diffIm, associatedDiaSources):
148 """Match fakes to detected diaSources within a difference image bound.
152 fakeCat : `pandas.DataFrame`
153 Catalog of fakes to match to detected diaSources.
155 Difference image where ``associatedDiaSources`` were detected.
156 associatedDiaSources : `pandas.DataFrame`
157 Catalog of difference image sources detected in ``diffIm``.
161 result : `lsst.pipe.base.Struct`
162 Results struct
with components.
164 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
165 length of ``fakeCat``. (`pandas.DataFrame`)
167 trimmedFakes = self._trimFakeCat(fakeCat, diffIm)
168 nPossibleFakes = len(trimmedFakes)
170 fakeVects = self._getVectors(trimmedFakes[self.config.ra_col],
171 trimmedFakes[self.config.dec_col])
172 diaSrcVects = self._getVectors(
173 np.radians(associatedDiaSources.loc[:, "ra"]),
174 np.radians(associatedDiaSources.loc[:,
"decl"]))
176 diaSrcTree = cKDTree(diaSrcVects)
177 dist, idxs = diaSrcTree.query(
179 distance_upper_bound=np.radians(self.config.matchDistanceArcseconds / 3600))
180 nFakesFound = np.isfinite(dist).sum()
182 self.log.info(
"Found %d out of %d possible.", nFakesFound, nPossibleFakes)
183 diaSrcIds = associatedDiaSources.iloc[np.where(np.isfinite(dist), idxs, 0)][
"diaSourceId"].to_numpy()
184 matchedFakes = trimmedFakes.assign(diaSourceId=np.where(np.isfinite(dist), diaSrcIds, 0))
187 matchedDiaSources=matchedFakes.merge(
188 associatedDiaSources.reset_index(drop=
True), on=
"diaSourceId", how=
"left")
191 def composeFakeCat(self, fakeCats, skyMap):
192 """Concatenate the fakeCats from tracts that may cover the exposure.
196 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
197 Set of fake cats to concatenate.
198 skyMap : `lsst.skymap.SkyMap`
199 SkyMap defining the geometry of the tracts and patches.
203 combinedFakeCat : `pandas.DataFrame`
204 All fakes that cover the inner polygon of the tracts
in this
207 if len(fakeCats) == 1:
208 return fakeCats[0].get(
209 datasetType=self.config.connections.fakeCats)
211 for fakeCatRef
in fakeCats:
212 cat = fakeCatRef.get(
213 datasetType=self.config.connections.fakeCats)
214 tractId = fakeCatRef.dataId[
"tract"]
216 outputCat.append(cat[
217 skyMap.findTractIdArray(cat[self.config.ra_col],
218 cat[self.config.dec_col],
222 return pd.concat(outputCat)
224 def getVisitMatchedFakeCat(self, fakeCat, exposure):
225 """Trim the fakeCat to select particular visit
229 fakeCat : `pandas.core.frame.DataFrame`
230 The catalog of fake sources to add to the exposure
231 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
232 The exposure to add the fake sources to
236 movingFakeCat : `pandas.DataFrame`
237 All fakes that belong to the visit
239 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
241 return fakeCat[selected]
243 def _trimFakeCat(self, fakeCat, image):
244 """Trim the fake cat to about the size of the input image.
248 fakeCat : `pandas.core.frame.DataFrame`
249 The catalog of fake sources to be input
250 image : `lsst.afw.image.exposure.exposure.ExposureF`
251 The image into which the fake sources should be added
252 skyMap : `lsst.skymap.SkyMap`
253 SkyMap defining the tracts and patches the fakes are stored over.
257 fakeCats : `pandas.core.frame.DataFrame`
258 The original fakeCat trimmed to the area of the image
262 bbox = Box2D(image.getBBox())
266 row[self.config.dec_col],
268 cent = wcs.skyToPixel(coord)
269 return bbox.contains(cent)
271 return fakeCat[fakeCat.apply(trim, axis=1)]
273 def _getVectors(self, ras, decs):
274 """Convert ra dec to unit vectors on the sphere.
278 ras : `numpy.ndarray`, (N,)
279 RA coordinates in radians.
280 decs : `numpy.ndarray`, (N,)
281 Dec coordinates
in radians.
285 vectors : `numpy.ndarray`, (N, 3)
286 Vectors on the unit sphere
for the given RA/DEC values.
288 vectors = np.empty((len(ras), 3))
290 vectors[:, 2] = np.sin(decs)
291 vectors[:, 0] = np.cos(decs) * np.cos(ras)
292 vectors[:, 1] = np.cos(decs) * np.sin(ras)
298 ccdVisitFakeMagnitudes = connTypes.Input(
299 doc=
"Catalog of fakes with magnitudes scattered for this ccdVisit.",
300 name=
"{fakesType}ccdVisitFakeMagnitudes",
301 storageClass=
"DataFrame",
302 dimensions=(
"instrument",
"visit",
"detector"),
306class MatchVariableFakesConfig(MatchFakesConfig,
307 pipelineConnections=MatchVariableFakesConnections):
308 """Config for MatchFakesTask.
313class MatchVariableFakesTask(MatchFakesTask):
314 """Match injected fakes to their detected sources in the catalog and
315 compute their expected brightness in a difference image assuming perfect
318 This task
is generally
for injected sources that cannot be easily
319 identified by their footprints such
as in the case of detector sources
320 post image differencing.
322 _DefaultName = "matchVariableFakes"
323 ConfigClass = MatchVariableFakesConfig
325 def runQuantum(self, butlerQC, inputRefs, outputRefs):
326 inputs = butlerQC.get(inputRefs)
327 inputs[
"band"] = butlerQC.quantum.dataId[
"band"]
329 outputs = self.run(**inputs)
330 butlerQC.put(outputs, outputRefs)
332 def run(self, fakeCats, ccdVisitFakeMagnitudes, skyMap, diffIm, associatedDiaSources, band):
333 """Match fakes to detected diaSources within a difference image bound.
337 fakeCat : `pandas.DataFrame`
338 Catalog of fakes to match to detected diaSources.
340 Difference image where ``associatedDiaSources`` were detected in.
341 associatedDiaSources : `pandas.DataFrame`
342 Catalog of difference image sources detected
in ``diffIm``.
346 result : `lsst.pipe.base.Struct`
347 Results struct
with components.
349 - ``matchedDiaSources`` : Fakes matched to input diaSources. Has
350 length of ``fakeCat``. (`pandas.DataFrame`)
352 fakeCat = self.composeFakeCat(fakeCats, skyMap)
353 self.computeExpectedDiffMag(fakeCat, ccdVisitFakeMagnitudes, band)
354 return self._processFakes(fakeCat, diffIm, associatedDiaSources)
356 def computeExpectedDiffMag(self, fakeCat, ccdVisitFakeMagnitudes, band):
357 """Compute the magnitude expected in the difference image for this
358 detector/visit. Modify fakeCat in place.
360 Negative magnitudes indicate that the source should be detected
as
365 fakeCat : `pandas.DataFrame`
366 Catalog of fake sources.
367 ccdVisitFakeMagnitudes : `pandas.DataFrame`
368 Magnitudes
for variable sources
in this specific ccdVisit.
370 Band that this ccdVisit was observed
in.
372 magName = self.config.mag_col % band
373 magnitudes = fakeCat[magName].to_numpy()
374 visitMags = ccdVisitFakeMagnitudes["variableMag"].to_numpy()
375 diffFlux = (visitMags * u.ABmag).to_value(u.nJy) - (magnitudes * u.ABmag).to_value(u.nJy)
376 diffMag = np.where(diffFlux > 0,
377 (diffFlux * u.nJy).to_value(u.ABmag),
378 -(-diffFlux * u.nJy).to_value(u.ABmag))
380 noVisit = ~fakeCat[
"isVisitSource"]
381 noTemplate = ~fakeCat[
"isTemplateSource"]
382 both = np.logical_and(fakeCat[
"isVisitSource"],
383 fakeCat[
"isTemplateSource"])
385 fakeCat.loc[noVisit, magName] = -magnitudes[noVisit]
386 fakeCat.loc[noTemplate, magName] = visitMags[noTemplate]
387 fakeCat.loc[both, magName] = diffMag[both]