lsst.meas.base g61ffd41259+4f4d0d1589
Loading...
Searching...
No Matches
tests.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import warnings
23
24import numpy as np
25
26import lsst.geom
27import lsst.afw.table
28import lsst.afw.image
30import lsst.afw.geom
31import lsst.afw.coord
32import lsst.daf.base
34
35from .sfm import SingleFrameMeasurementTask
36from .forcedMeasurement import ForcedMeasurementTask
37from ._measBaseLib import CentroidResultKey
38
39__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
40 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
41 "FluxTransformTestCase", "CentroidTransformTestCase")
42
43
45 """Context manager which adds multiple overlapping sources and a parent.
46
47 Notes
48 -----
49 This is used as the return value for `TestDataset.addBlend`, and this is
50 the only way it should be used.
51 """
52
53 def __init__(self, owner):
54 self.owner = owner
55 self.parentRecord = self.owner.catalog.addNew()
56 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
57 self.children = []
58
59 def __enter__(self):
60 # BlendContext is its own context manager, so we just return self.
61 return self
62
63 def addChild(self, instFlux, centroid, shape=None):
64 """Add a child to the blend; return corresponding truth catalog record.
65
66 instFlux : `float`
67 Total instFlux of the source to be added.
68 centroid : `lsst.geom.Point2D`
69 Position of the source to be added.
70 shape : `lsst.afw.geom.Quadrupole`
71 Second moments of the source before PSF convolution. Note that
72 the truth catalog records post-convolution moments)
73 """
74 record, image = self.owner.addSource(instFlux, centroid, shape)
75 record.set(self.owner.keys["parent"], self.parentRecord.getId())
76 self.parentImage += image
77 self.children.append((record, image))
78 return record
79
80 def __exit__(self, type_, value, tb):
81 # We're not using the context manager for any kind of exception safety
82 # or guarantees; we just want the nice "with" statement syntax.
83
84 if type_ is not None:
85 # exception was raised; just skip all this and let it propagate
86 return
87
88 # On exit, compute and set the truth values for the parent object.
89 self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
90 # Compute instFlux from sum of component fluxes
91 instFlux = 0.0
92 for record, image in self.children:
93 instFlux += record.get(self.owner.keys["instFlux"])
94 self.parentRecord.set(self.owner.keys["instFlux"], instFlux)
95 # Compute centroid from instFlux-weighted mean of component centroids
96 x = 0.0
97 y = 0.0
98 for record, image in self.children:
99 w = record.get(self.owner.keys["instFlux"])/instFlux
100 x += record.get(self.owner.keys["centroid"].getX())*w
101 y += record.get(self.owner.keys["centroid"].getY())*w
102 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
103 # Compute shape from instFlux-weighted mean of offset component shapes
104 xx = 0.0
105 yy = 0.0
106 xy = 0.0
107 for record, image in self.children:
108 w = record.get(self.owner.keys["instFlux"])/instFlux
109 dx = record.get(self.owner.keys["centroid"].getX()) - x
110 dy = record.get(self.owner.keys["centroid"].getY()) - y
111 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
112 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
113 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
114 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
115 # Run detection on the parent image to get the parent Footprint.
116 self.owner._installFootprint(self.parentRecord, self.parentImage)
117 # Create perfect HeavyFootprints for all children; these will need to
118 # be modified later to account for the noise we'll add to the image.
119 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.maskedImage, True)
120 for record, image in self.children:
121 deblend.image.array[:, :] = image.array
122 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
123 record.setFootprint(heavyFootprint)
124
125
127 """A simulated dataset consisuting of test image and truth catalog.
128
129 TestDataset creates an idealized image made of pure Gaussians (including a
130 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints
131 that simulated the outputs of detection and deblending. Multiple noise
132 realizations can be created from the same underlying sources, allowing
133 uncertainty estimates to be verified via Monte Carlo.
134
135 Parameters
136 ----------
137 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
138 Bounding box of the test image.
139 threshold : `float`
140 Threshold absolute value used to determine footprints for
141 simulated sources. This thresholding will be applied before noise is
142 actually added to images (or before the noise level is even known), so
143 this will necessarily produce somewhat artificial footprints.
144 exposure : `lsst.afw.image.ExposureF`
145 The image to which test sources should be added. Ownership should
146 be considered transferred from the caller to the TestDataset.
147 Must have a Gaussian PSF for truth catalog shapes to be exact.
148 **kwds
149 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`.
150
151 Notes
152 -----
153 Typical usage:
154
155 .. code-block: py
156
157 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100,
158 100))
159 dataset = TestDataset(bbox)
160 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26))
161 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24),
162 shape=lsst.afw.geom.Quadrupole(8, 7, 2))
163 with dataset.addBlend() as family:
164 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72))
165 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
166 exposure, catalog = dataset.realize(noise=100.0,
167 schema=TestDataset.makeMinimalSchema())
168 """
169
170 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
171 if exposure is None:
172 exposure = self.makeEmptyExposure(bbox, **kwds)
173 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
174 self.exposure = exposure
175 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter())
178
179 @classmethod
181 """Return the minimal schema needed to hold truth catalog fields.
182
183 Notes
184 -----
185 When `TestDataset.realize` is called, the schema must include at least
186 these fields. Usually it will include additional fields for
187 measurement algorithm outputs, allowing the same catalog to be used
188 for both truth values (the fields from the minimal schema) and the
189 measurements.
190 """
191 if not hasattr(cls, "_schema"):
193 cls.keys = {}
194 cls.keys["coordErr"] = lsst.afw.table.CoordKey.addErrorFields(schema)
195 cls.keys["parent"] = schema.find("parent").key
196 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
197 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64,
198 doc="true instFlux", units="count")
199 cls.keys["instFluxErr"] = schema.addField("truth_instFluxErr", type=np.float64,
200 doc="true instFluxErr", units="count")
201 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
202 schema, "truth", "true simulated centroid", "pixel"
203 )
204 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
205 schema, "truth", ['x', 'y'], "pixel"
206 )
207 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
208 doc="set if the object is a star")
210 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
211 )
212 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
213 doc="set if the object is a star")
214 schema.getAliasMap().set("slot_Shape", "truth")
215 schema.getAliasMap().set("slot_Centroid", "truth")
216 schema.getAliasMap().set("slot_ModelFlux", "truth")
217 cls._schema = schema
218 schema = lsst.afw.table.Schema(cls._schema)
219 schema.disconnectAliases()
220 return schema
221
222 @staticmethod
223 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
224 minRotation=None, maxRotation=None,
225 minRefShift=None, maxRefShift=None,
226 minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
227 """Return a perturbed version of the input WCS.
228
229 Create a new undistorted TAN WCS that is similar but not identical to
230 another, with random scaling, rotation, and offset (in both pixel
231 position and reference position).
232
233 Parameters
234 ----------
235 oldWcs : `lsst.afw.geom.SkyWcs`
236 The input WCS.
237 minScaleFactor : `float`
238 Minimum scale factor to apply to the input WCS.
239 maxScaleFactor : `float`
240 Maximum scale factor to apply to the input WCS.
241 minRotation : `lsst.geom.Angle` or `None`
242 Minimum rotation to apply to the input WCS. If `None`, defaults to
243 30 degrees.
244 maxRotation : `lsst.geom.Angle` or `None`
245 Minimum rotation to apply to the input WCS. If `None`, defaults to
246 60 degrees.
247 minRefShift : `lsst.geom.Angle` or `None`
248 Miniumum shift to apply to the input WCS reference value. If
249 `None`, defaults to 0.5 arcsec.
250 maxRefShift : `lsst.geom.Angle` or `None`
251 Miniumum shift to apply to the input WCS reference value. If
252 `None`, defaults to 1.0 arcsec.
253 minPixShift : `float`
254 Minimum shift to apply to the input WCS reference pixel.
255 maxPixShift : `float`
256 Maximum shift to apply to the input WCS reference pixel.
257 randomSeed : `int`
258 Random seed.
259
260 Returns
261 -------
262 newWcs : `lsst.afw.geom.SkyWcs`
263 A perturbed version of the input WCS.
264
265 Notes
266 -----
267 The maximum and minimum arguments are interpreted as absolute values
268 for a split range that covers both positive and negative values (as
269 this method is used in testing, it is typically most important to
270 avoid perturbations near zero). Scale factors are treated somewhat
271 differently: the actual scale factor is chosen between
272 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``)
273 and (``1/minScaleFactor``).
274
275 The default range for rotation is 30-60 degrees, and the default range
276 for reference shift is 0.5-1.0 arcseconds (these cannot be safely
277 included directly as default values because Angle objects are
278 mutable).
279
280 The random number generator is primed with the seed given. If
281 `None`, a seed is automatically chosen.
282 """
283 random_state = np.random.RandomState(randomSeed)
284 if minRotation is None:
285 minRotation = 30.0*lsst.geom.degrees
286 if maxRotation is None:
287 maxRotation = 60.0*lsst.geom.degrees
288 if minRefShift is None:
289 minRefShift = 0.5*lsst.geom.arcseconds
290 if maxRefShift is None:
291 maxRefShift = 1.0*lsst.geom.arcseconds
292
293 def splitRandom(min1, max1, min2=None, max2=None):
294 if min2 is None:
295 min2 = -max1
296 if max2 is None:
297 max2 = -min1
298 if random_state.uniform() > 0.5:
299 return float(random_state.uniform(min1, max1))
300 else:
301 return float(random_state.uniform(min2, max2))
302 # Generate random perturbations
303 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
304 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
305 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
306 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
307 pixShiftX = splitRandom(minPixShift, maxPixShift)
308 pixShiftY = splitRandom(minPixShift, maxPixShift)
309 # Compute new CD matrix
310 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
311 rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
312 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
313 newTransform = oldTransform*rTransform*sTransform
314 matrix = newTransform.getMatrix()
315 # Compute new coordinate reference pixel (CRVAL)
316 oldSkyOrigin = oldWcs.getSkyOrigin()
317 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
318 oldSkyOrigin.getDec() + refShiftDec)
319 # Compute new pixel reference pixel (CRPIX)
320 oldPixOrigin = oldWcs.getPixelOrigin()
321 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
322 oldPixOrigin.getY() + pixShiftY)
323 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
324
325 @staticmethod
326 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4):
327 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values.
328
329 Parameters
330 ----------
331 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
332 Bounding box of the image in image coordinates.
333 wcs : `lsst.afw.geom.SkyWcs`, optional
334 New WCS for the exposure (created from CRVAL and CDELT if `None`).
335 crval : `lsst.afw.geom.SpherePoint`, optional
336 ICRS center of the TAN WCS attached to the image. If `None`, (45
337 degrees, 45 degrees) is assumed.
338 cdelt : `lsst.geom.Angle`, optional
339 Pixel scale of the image. If `None`, 0.2 arcsec is assumed.
340 psfSigma : `float`, optional
341 Radius (sigma) of the Gaussian PSF attached to the image
342 psfDim : `int`, optional
343 Width and height of the image's Gaussian PSF attached to the image
344 calibration : `float`, optional
345 The spatially-constant calibration (in nJy/count) to set the
346 PhotoCalib of the exposure.
347
348 Returns
349 -------
350 exposure : `lsst.age.image.ExposureF`
351 An empty image.
352 """
353 if wcs is None:
354 if crval is None:
355 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
356 if cdelt is None:
357 cdelt = 0.2*lsst.geom.arcseconds
358 crpix = lsst.geom.Box2D(bbox).getCenter()
359 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
360 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
361 exposure = lsst.afw.image.ExposureF(bbox)
362 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
363 photoCalib = lsst.afw.image.PhotoCalib(calibration)
364 visitInfo = lsst.afw.image.VisitInfo(id=1234,
365 exposureTime=30.0,
366 date=lsst.daf.base.DateTime(60000.0),
367 observatory=lsst.afw.coord.Observatory(11.1*lsst.geom.degrees,
368 22.2*lsst.geom.degrees,
369 0.333))
370 exposure.setWcs(wcs)
371 exposure.setPsf(psf)
372 exposure.setPhotoCalib(photoCalib)
373 exposure.info.setVisitInfo(visitInfo)
374 return exposure
375
376 @staticmethod
377 def drawGaussian(bbox, instFlux, ellipse):
378 """Create an image of an elliptical Gaussian.
379
380 Parameters
381 ----------
382 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
383 Bounding box of image to create.
384 instFlux : `float`
385 Total instrumental flux of the Gaussian (normalized analytically,
386 not using pixel values).
387 ellipse : `lsst.afw.geom.Ellipse`
388 Defines the centroid and shape.
389
390 Returns
391 -------
392 image : `lsst.afw.image.ImageF`
393 An image of the Gaussian.
394 """
395 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
396 np.arange(bbox.getBeginY(), bbox.getEndY()))
397 t = ellipse.getGridTransform()
398 xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
399 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
400 image = lsst.afw.image.ImageF(bbox)
401 image.array[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea())
402 return image
403
404 def _installFootprint(self, record, image, setPeakSignificance=True):
405 """Create simulated Footprint and add it to a truth catalog record.
406 """
408 if setPeakSignificance:
409 schema.addField("significance", type=float,
410 doc="Ratio of peak value to configured standard deviation.")
411 # Run detection on the single-source image
412 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema)
413 # the call below to the FootprintSet ctor is actually a grow operation
414 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
415 if setPeakSignificance:
416 # This isn't a traditional significance, since we're using the VALUE
417 # threshold type, but it's the best we can do in that case.
418 for footprint in fpSet.getFootprints():
419 footprint.updatePeakSignificance(self.threshold.getValue())
420 # Update the full exposure's mask plane to indicate the detection
421 fpSet.setMask(self.exposure.mask, "DETECTED")
422 # Attach the new footprint to the exposure
423 if len(fpSet.getFootprints()) > 1:
424 raise RuntimeError("Threshold value results in multiple Footprints for a single object")
425 if len(fpSet.getFootprints()) == 0:
426 raise RuntimeError("Threshold value results in zero Footprints for object")
427 record.setFootprint(fpSet.getFootprints()[0])
428
429 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True):
430 """Add a source to the simulation.
431
432 To insert a point source with a given signal-to-noise (sn), the total
433 ``instFlux`` should be: ``sn*noise*psf_scale``, where ``noise`` is the
434 noise you will pass to ``realize()``, and
435 ``psf_scale=sqrt(4*pi*r^2)``, where ``r`` is the width of the PSF.
436
437 Parameters
438 ----------
439 instFlux : `float`
440 Total instFlux of the source to be added.
441 centroid : `lsst.geom.Point2D`
442 Position of the source to be added.
443 shape : `lsst.afw.geom.Quadrupole`
444 Second moments of the source before PSF convolution. Note that the
445 truth catalog records post-convolution moments. If `None`, a point
446 source will be added.
447 setPeakSignificance : `bool`
448 Set the ``significance`` field for peaks in the footprints?
449 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance``
450 for how this field is computed for real datasets.
451
452 Returns
453 -------
454 record : `lsst.afw.table.SourceRecord`
455 A truth catalog record.
456 image : `lsst.afw.image.ImageF`
457 Single-source image corresponding to the new source.
458 """
459 # Create and set the truth catalog fields
460 record = self.catalog.addNew()
461 record.set(self.keys["instFlux"], instFlux)
462 record.set(self.keys["instFluxErr"], 0)
463 record.set(self.keys["centroid"], centroid)
464 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
465 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
466 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
467 if shape is None:
468 record.set(self.keys["isStar"], True)
469 fullShape = self.psfShape
470 else:
471 record.set(self.keys["isStar"], False)
472 fullShape = shape.convolve(self.psfShape)
473 record.set(self.keys["shape"], fullShape)
474 # Create an image containing just this source
475 image = self.drawGaussian(self.exposure.getBBox(), instFlux,
476 lsst.afw.geom.Ellipse(fullShape, centroid))
477 # Generate a footprint for this source
478 self._installFootprint(record, image, setPeakSignificance)
479 # Actually add the source to the full exposure
480 self.exposure.image.array[:, :] += image.array
481 return record, image
482
483 def addBlend(self):
484 """Return a context manager which can add a blend of multiple sources.
485
486 Notes
487 -----
488 Note that nothing stops you from creating overlapping sources just using the addSource() method,
489 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
490 produced by the detection and deblending pipelines.
491
492 Examples
493 --------
494 .. code-block: py
495 d = TestDataset(...)
496 with d.addBlend() as b:
497 b.addChild(flux1, centroid1)
498 b.addChild(flux2, centroid2, shape2)
499 """
500 return BlendContext(self)
501
502 def transform(self, wcs, **kwds):
503 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib.
504
505 Parameters
506 ----------
507 wcs : `lsst.afw.geom.SkyWcs`
508 WCS for the new dataset.
509 **kwds
510 Additional keyword arguments passed on to
511 `TestDataset.makeEmptyExposure`. If not specified, these revert
512 to the defaults for `~TestDataset.makeEmptyExposure`, not the
513 values in the current dataset.
514
515 Returns
516 -------
517 newDataset : `TestDataset`
518 Transformed copy of this dataset.
519 """
520 bboxD = lsst.geom.Box2D()
521 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
522 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
523 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
524 bboxI = lsst.geom.Box2I(bboxD)
525 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
526 oldPhotoCalib = self.exposure.getPhotoCalib()
527 newPhotoCalib = result.exposure.getPhotoCalib()
528 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter())
529 for record in self.catalog:
530 if record.get(self.keys["nChild"]):
531 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
532 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"]))
533 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude)
534 oldCentroid = record.get(self.keys["centroid"])
535 newCentroid = xyt.applyForward(oldCentroid)
536 if record.get(self.keys["isStar"]):
537 newDeconvolvedShape = None
538 else:
539 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
540 oldFullShape = record.get(self.keys["shape"])
541 oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
542 oldFullShape.getIxx() - oldPsfShape.getIxx(),
543 oldFullShape.getIyy() - oldPsfShape.getIyy(),
544 oldFullShape.getIxy() - oldPsfShape.getIxy(),
545 False
546 )
547 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
548 result.addSource(newFlux, newCentroid, newDeconvolvedShape)
549 return result
550
551 def realize(self, noise, schema, randomSeed=1):
552 r"""Simulate an exposure and detection catalog for this dataset.
553
554 The simulation includes noise, and the detection catalog includes
555 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s.
556
557 Parameters
558 ----------
559 noise : `float`
560 Standard deviation of noise to be added to the exposure. The
561 noise will be Gaussian and constant, appropriate for the
562 sky-limited regime.
563 schema : `lsst.afw.table.Schema`
564 Schema of the new catalog to be created. Must start with
565 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be
566 `True`), but typically contains fields for already-configured
567 measurement algorithms as well.
568 randomSeed : `int`, optional
569 Seed for the random number generator.
570 If `None`, a seed is chosen automatically.
571
572 Returns
573 -------
574 `exposure` : `lsst.afw.image.ExposureF`
575 Simulated image.
576 `catalog` : `lsst.afw.table.SourceCatalog`
577 Simulated detection catalog.
578 """
579 random_state = np.random.RandomState(randomSeed)
580 assert schema.contains(self.schema)
582 mapper.addMinimalSchema(self.schema, True)
583 exposure = self.exposure.clone()
584 exposure.variance.array[:, :] = noise**2
585 exposure.image.array[:, :] += random_state.randn(exposure.height, exposure.width)*noise
586 catalog = lsst.afw.table.SourceCatalog(schema)
587 catalog.extend(self.catalog, mapper=mapper)
588 # Loop over sources and generate new HeavyFootprints that divide up
589 # the noisy pixels, not the ideal no-noise pixels.
590 for record in catalog:
591 # parent objects have non-Heavy Footprints, which don't need to be
592 # updated after adding noise.
593 if record.getParent() == 0:
594 continue
595 # get flattened arrays that correspond to the no-noise and noisy
596 # parent images
597 parent = catalog.find(record.getParent())
598 footprint = parent.getFootprint()
599 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
600 footprint.spans.flatten(parentFluxArrayNoNoise, self.exposure.image.array, self.exposure.getXY0())
601 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
602 footprint.spans.flatten(parentFluxArrayNoisy, exposure.image.array, exposure.getXY0())
603 oldHeavy = record.getFootprint()
604 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
605 # N.B. this isn't a copy ctor - it's a copy from a vanilla
606 # Footprint, so it doesn't copy the arrays we don't want to
607 # change, and hence we have to do that ourselves below.
608 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
609 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
610 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
611 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
612 record.setFootprint(newHeavy)
613 lsst.afw.table.updateSourceCoords(exposure.wcs, catalog)
614 return exposure, catalog
615
616
618 """Base class for tests of measurement tasks.
619 """
620 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
621 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`.
622
623 Only the specified plugin and its dependencies will be run; the
624 Centroid, Shape, and ModelFlux slots will be set to the truth fields
625 generated by the `TestDataset` class.
626
627 Parameters
628 ----------
629 plugin : `str`
630 Name of measurement plugin to enable.
631 dependencies : iterable of `str`, optional
632 Names of dependencies of the measurement plugin.
633
634 Returns
635 -------
636 config : `SingleFrameMeasurementTask.ConfigClass`
637 The resulting task configuration.
638 """
639 config = SingleFrameMeasurementTask.ConfigClass()
640 with warnings.catch_warnings():
641 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning)
642 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True)
643 config.slots.centroid = "truth"
644 config.slots.shape = "truth"
645 config.slots.modelFlux = None
646 config.slots.apFlux = None
647 config.slots.psfFlux = None
648 config.slots.gaussianFlux = None
649 config.slots.calibFlux = None
650 config.plugins.names = (plugin,) + tuple(dependencies)
651 return config
652
653 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
654 algMetadata=None):
655 """Create a configured instance of `SingleFrameMeasurementTask`.
656
657 Parameters
658 ----------
659 plugin : `str`, optional
660 Name of measurement plugin to enable. If `None`, a configuration
661 must be supplied as the ``config`` parameter. If both are
662 specified, ``config`` takes precedence.
663 dependencies : iterable of `str`, optional
664 Names of dependencies of the specified measurement plugin.
665 config : `SingleFrameMeasurementTask.ConfigClass`, optional
666 Configuration for the task. If `None`, a measurement plugin must
667 be supplied as the ``plugin`` paramter. If both are specified,
668 ``config`` takes precedence.
669 schema : `lsst.afw.table.Schema`, optional
670 Measurement table schema. If `None`, a default schema is
671 generated.
672 algMetadata : `lsst.daf.base.PropertyList`, optional
673 Measurement algorithm metadata. If `None`, a default container
674 will be generated.
675
676 Returns
677 -------
678 task : `SingleFrameMeasurementTask`
679 A configured instance of the measurement task.
680 """
681 if config is None:
682 if plugin is None:
683 raise ValueError("Either plugin or config argument must not be None")
684 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
685 if schema is None:
686 schema = TestDataset.makeMinimalSchema()
687 # Clear all aliases so only those defined by config are set.
688 schema.setAliasMap(None)
689 if algMetadata is None:
690 algMetadata = lsst.daf.base.PropertyList()
691 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
692
693 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
694 """Create an instance of `ForcedMeasurementTask.ConfigClass`.
695
696 In addition to the plugins specified in the plugin and dependencies
697 arguments, the `TransformedCentroid` and `TransformedShape` plugins
698 will be run and used as the centroid and shape slots; these simply
699 transform the reference catalog centroid and shape to the measurement
700 coordinate system.
701
702 Parameters
703 ----------
704 plugin : `str`
705 Name of measurement plugin to enable.
706 dependencies : iterable of `str`, optional
707 Names of dependencies of the measurement plugin.
708
709 Returns
710 -------
711 config : `ForcedMeasurementTask.ConfigClass`
712 The resulting task configuration.
713 """
714
715 config = ForcedMeasurementTask.ConfigClass()
716 config.slots.centroid = "base_TransformedCentroid"
717 config.slots.shape = "base_TransformedShape"
718 config.slots.modelFlux = None
719 config.slots.apFlux = None
720 config.slots.psfFlux = None
721 config.slots.gaussianFlux = None
722 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
723 "base_TransformedShape")
724 return config
725
726 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
727 algMetadata=None):
728 """Create a configured instance of `ForcedMeasurementTask`.
729
730 Parameters
731 ----------
732 plugin : `str`, optional
733 Name of measurement plugin to enable. If `None`, a configuration
734 must be supplied as the ``config`` parameter. If both are
735 specified, ``config`` takes precedence.
736 dependencies : iterable of `str`, optional
737 Names of dependencies of the specified measurement plugin.
738 config : `SingleFrameMeasurementTask.ConfigClass`, optional
739 Configuration for the task. If `None`, a measurement plugin must
740 be supplied as the ``plugin`` paramter. If both are specified,
741 ``config`` takes precedence.
742 refSchema : `lsst.afw.table.Schema`, optional
743 Reference table schema. If `None`, a default schema is
744 generated.
745 algMetadata : `lsst.daf.base.PropertyList`, optional
746 Measurement algorithm metadata. If `None`, a default container
747 will be generated.
748
749 Returns
750 -------
751 task : `ForcedMeasurementTask`
752 A configured instance of the measurement task.
753 """
754 if config is None:
755 if plugin is None:
756 raise ValueError("Either plugin or config argument must not be None")
757 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
758 if refSchema is None:
759 refSchema = TestDataset.makeMinimalSchema()
760 if algMetadata is None:
761 algMetadata = lsst.daf.base.PropertyList()
762 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
763
764
766 """Base class for testing measurement transformations.
767
768 Notes
769 -----
770 We test both that the transform itself operates successfully (fluxes are
771 converted to magnitudes, flags are propagated properly) and that the
772 transform is registered as the default for the appropriate measurement
773 algorithms.
774
775 In the simple case of one-measurement-per-transformation, the developer
776 need not directly write any tests themselves: simply customizing the class
777 variables is all that is required. More complex measurements (e.g.
778 multiple aperture fluxes) require extra effort.
779 """
780 name = "MeasurementTransformTest"
781 """The name used for the measurement algorithm (str).
782
783 Notes
784 -----
785 This determines the names of the fields in the resulting catalog. This
786 default should generally be fine, but subclasses can override if
787 required.
788 """
789
790 # These should be customized by subclassing.
791 controlClass = None
792 algorithmClass = None
793 transformClass = None
794
795 flagNames = ("flag",)
796 """Flags which may be set by the algorithm being tested (iterable of `str`).
797 """
798
799 # The plugin being tested should be registered under these names for
800 # single frame and forced measurement. Should be customized by
801 # subclassing.
802 singleFramePlugins = ()
803 forcedPlugins = ()
804
805 def setUp(self):
807 self.calexp = TestDataset.makeEmptyExposure(bbox)
808 self._setupTransform()
809
810 def tearDown(self):
811 del self.calexp
812 del self.inputCat
813 del self.mapper
814 del self.transform
815 del self.outputCat
816
817 def _populateCatalog(self, baseNames):
818 records = []
819 for flagValue in (True, False):
820 records.append(self.inputCat.addNew())
821 for baseName in baseNames:
822 for flagName in self.flagNames:
823 if records[-1].schema.join(baseName, flagName) in records[-1].schema:
824 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
825 self._setFieldsInRecords(records, baseName)
826
827 def _checkOutput(self, baseNames):
828 for inSrc, outSrc in zip(self.inputCat, self.outputCat):
829 for baseName in baseNames:
830 self._compareFieldsInRecords(inSrc, outSrc, baseName)
831 for flagName in self.flagNames:
832 keyName = outSrc.schema.join(baseName, flagName)
833 if keyName in inSrc.schema:
834 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
835 else:
836 self.assertFalse(keyName in outSrc.schema)
837
838 def _runTransform(self, doExtend=True):
839 if doExtend:
840 self.outputCat.extend(self.inputCat, mapper=self.mapper)
841 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib())
842
843 def testTransform(self, baseNames=None):
844 """Test the transformation on a catalog containing random data.
845
846 Parameters
847 ----------
848 baseNames : iterable of `str`
849 Iterable of the initial parts of measurement field names.
850
851 Notes
852 -----
853 We check that:
854
855 - An appropriate exception is raised on an attempt to transform
856 between catalogs with different numbers of rows;
857 - Otherwise, all appropriate conversions are properly appled and that
858 flags have been propagated.
859
860 The ``baseNames`` argument requires some explanation. This should be
861 an iterable of the leading parts of the field names for each
862 measurement; that is, everything that appears before ``_instFlux``,
863 ``_flag``, etc. In the simple case of a single measurement per plugin,
864 this is simply equal to ``self.name`` (thus measurements are stored as
865 ``self.name + "_instFlux"``, etc). More generally, the developer may
866 specify whatever iterable they require. For example, to handle
867 multiple apertures, we could have ``(self.name + "_0", self.name +
868 "_1", ...)``.
869 """
870 baseNames = baseNames or [self.name]
871 self._populateCatalog(baseNames)
874 self._checkOutput(baseNames)
875
876 def _checkRegisteredTransform(self, registry, name):
877 # If this is a Python-based transform, we can compare directly; if
878 # it's wrapped C++, we need to compare the wrapped class.
879 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
880
882 """Test that the transformation is appropriately registered.
883 """
884 for pluginName in self.singleFramePlugins:
885 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
886 for pluginName in self.forcedPlugins:
887 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
888
889
891
893 self.control = self.controlClass()
895 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
896 # it doesn't matter for this test since we won't actually use the plugins for anything besides
897 # defining the schema.
898 inputSchema.getAliasMap().set("slot_Centroid", "dummy")
899 inputSchema.getAliasMap().set("slot_Shape", "dummy")
900 self.algorithmClass(self.control, self.name, inputSchema)
901 inputSchema.getAliasMap().erase("slot_Centroid")
902 inputSchema.getAliasMap().erase("slot_Shape")
905 self.transform = self.transformClass(self.control, self.name, self.mapper)
906 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
907
908
910
912 self.control = self.controlClass()
915 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
916 # it doesn't matter for this test since we won't actually use the plugins for anything besides
917 # defining the schema.
918 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
919 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
920 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
921 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
922 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
923 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
924 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
925 self.transform = self.transformClass(self.control, self.name, self.mapper)
926 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
927
928
930
931 def _setFieldsInRecords(self, records, name):
932 for record in records:
933 record[record.schema.join(name, 'instFlux')] = np.random.random()
934 record[record.schema.join(name, 'instFluxErr')] = np.random.random()
935
936 # Negative instFluxes should be converted to NaNs.
937 assert len(records) > 1
938 records[0][record.schema.join(name, 'instFlux')] = -1
939
940 def _compareFieldsInRecords(self, inSrc, outSrc, name):
941 instFluxName = inSrc.schema.join(name, 'instFlux')
942 instFluxErrName = inSrc.schema.join(name, 'instFluxErr')
943 if inSrc[instFluxName] > 0:
944 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
945 inSrc[instFluxErrName])
946 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value)
947 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
948 else:
949 # negative instFlux results in NaN magnitude, but can still have finite error
950 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
951 if np.isnan(inSrc[instFluxErrName]):
952 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
953 else:
954 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
955 inSrc[instFluxErrName])
956 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
957
958
960
961 def _setFieldsInRecords(self, records, name):
962 for record in records:
963 record[record.schema.join(name, 'x')] = np.random.random()
964 record[record.schema.join(name, 'y')] = np.random.random()
965 # Some algorithms set no errors; some set only sigma on x & y; some provide
966 # a full covariance matrix. Set only those which exist in the schema.
967 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'):
968 fieldName = record.schema.join(name, fieldSuffix)
969 if fieldName in record.schema:
970 record[fieldName] = np.random.random()
971
972 def _compareFieldsInRecords(self, inSrc, outSrc, name):
973 centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
974 centroidResult = centroidResultKey.get(inSrc)
975
976 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
977 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
978 self.assertEqual(coordTruth, coord)
979
980 # If the centroid has an associated uncertainty matrix, the coordinate
981 # must have one too, and vice versa.
982 try:
983 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
984 ["ra", "dec"]).get(outSrc)
986 self.assertFalse(centroidResultKey.getCentroidErr().isValid())
987 else:
988 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
989 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
990 centroidResult.getCentroidErr()),
991 transform.getLinear().getMatrix().transpose())
992 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
static afw::table::Schema makeMinimalSchema()
static ErrorKey addErrorFields(Schema &schema)
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
static Schema makeMinimalSchema()
static LinearTransform makeRotation(Angle t) noexcept
static LinearTransform makeScaling(double s) noexcept
makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition tests.py:620
makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition tests.py:693
makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition tests.py:727
makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition tests.py:654
addChild(self, instFlux, centroid, shape=None)
Definition tests.py:63
__exit__(self, type_, value, tb)
Definition tests.py:80
_compareFieldsInRecords(self, inSrc, outSrc, name)
Definition tests.py:972
_compareFieldsInRecords(self, inSrc, outSrc, name)
Definition tests.py:940
_setFieldsInRecords(self, records, name)
Definition tests.py:931
makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=1)
Definition tests.py:226
realize(self, noise, schema, randomSeed=1)
Definition tests.py:551
addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True)
Definition tests.py:429
_installFootprint(self, record, image, setPeakSignificance=True)
Definition tests.py:404
drawGaussian(bbox, instFlux, ellipse)
Definition tests.py:377
makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4)
Definition tests.py:326
__init__(self, bbox, threshold=10.0, exposure=None, **kwds)
Definition tests.py:170
testTransform(self, baseNames=None)
Definition tests.py:843
_runTransform(self, doExtend=True)
Definition tests.py:838
_checkRegisteredTransform(self, registry, name)
Definition tests.py:876
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
Eigen::Matrix2d makeCdMatrix(lsst::geom::Angle const &scale, lsst::geom::Angle const &orientation=0 *lsst::geom::degrees, bool flipX=false)
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
lsst::geom::AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, lsst::geom::Point2D const &inPoint)
void updateSourceCoords(geom::SkyWcs const &wcs, SourceCollection &sourceList, bool include_covariance=true)