Coverage for python/lsst/meas/base/tests.py: 14%
407 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-05 10:24 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-05 10:24 +0000
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import warnings
24import numpy as np
26import lsst.geom
27import lsst.afw.table
28import lsst.afw.image
29import lsst.afw.detection
30import lsst.afw.geom
31import lsst.afw.coord
32import lsst.daf.base
33import lsst.pex.exceptions
35from .sfm import SingleFrameMeasurementTask
36from .forcedMeasurement import ForcedMeasurementTask
37from ._measBaseLib import CentroidResultKey
39__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
40 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
41 "FluxTransformTestCase", "CentroidTransformTestCase")
44class BlendContext:
45 """Context manager which adds multiple overlapping sources and a parent.
47 Notes
48 -----
49 This is used as the return value for `TestDataset.addBlend`, and this is
50 the only way it should be used.
51 """
53 def __init__(self, owner):
54 self.owner = owner
55 self.parentRecord = self.owner.catalog.addNew()
56 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
57 self.children = []
59 def __enter__(self):
60 # BlendContext is its own context manager, so we just return self.
61 return self
63 def addChild(self, instFlux, centroid, shape=None):
64 """Add a child to the blend; return corresponding truth catalog record.
66 instFlux : `float`
67 Total instFlux of the source to be added.
68 centroid : `lsst.geom.Point2D`
69 Position of the source to be added.
70 shape : `lsst.afw.geom.Quadrupole`
71 Second moments of the source before PSF convolution. Note that
72 the truth catalog records post-convolution moments)
73 """
74 record, image = self.owner.addSource(instFlux, centroid, shape)
75 record.set(self.owner.keys["parent"], self.parentRecord.getId())
76 self.parentImage += image
77 self.children.append((record, image))
78 return record
80 def __exit__(self, type_, value, tb):
81 # We're not using the context manager for any kind of exception safety
82 # or guarantees; we just want the nice "with" statement syntax.
84 if type_ is not None:
85 # exception was raised; just skip all this and let it propagate
86 return
88 # On exit, compute and set the truth values for the parent object.
89 self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
90 # Compute instFlux from sum of component fluxes
91 instFlux = 0.0
92 for record, image in self.children:
93 instFlux += record.get(self.owner.keys["instFlux"])
94 self.parentRecord.set(self.owner.keys["instFlux"], instFlux)
95 # Compute centroid from instFlux-weighted mean of component centroids
96 x = 0.0
97 y = 0.0
98 for record, image in self.children:
99 w = record.get(self.owner.keys["instFlux"])/instFlux
100 x += record.get(self.owner.keys["centroid"].getX())*w
101 y += record.get(self.owner.keys["centroid"].getY())*w
102 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
103 # Compute shape from instFlux-weighted mean of offset component shapes
104 xx = 0.0
105 yy = 0.0
106 xy = 0.0
107 for record, image in self.children:
108 w = record.get(self.owner.keys["instFlux"])/instFlux
109 dx = record.get(self.owner.keys["centroid"].getX()) - x
110 dy = record.get(self.owner.keys["centroid"].getY()) - y
111 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
112 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
113 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
114 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
115 # Run detection on the parent image to get the parent Footprint.
116 self.owner._installFootprint(self.parentRecord, self.parentImage)
117 # Create perfect HeavyFootprints for all children; these will need to
118 # be modified later to account for the noise we'll add to the image.
119 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.maskedImage, True)
120 for record, image in self.children:
121 deblend.image.array[:, :] = image.array
122 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
123 record.setFootprint(heavyFootprint)
126class TestDataset:
127 """A simulated dataset consisuting of test image and truth catalog.
129 TestDataset creates an idealized image made of pure Gaussians (including a
130 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints
131 that simulated the outputs of detection and deblending. Multiple noise
132 realizations can be created from the same underlying sources, allowing
133 uncertainty estimates to be verified via Monte Carlo.
135 Parameters
136 ----------
137 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
138 Bounding box of the test image.
139 threshold : `float`
140 Threshold absolute value used to determine footprints for
141 simulated sources. This thresholding will be applied before noise is
142 actually added to images (or before the noise level is even known), so
143 this will necessarily produce somewhat artificial footprints.
144 exposure : `lsst.afw.image.ExposureF`
145 The image to which test sources should be added. Ownership should
146 be considered transferred from the caller to the TestDataset.
147 Must have a Gaussian PSF for truth catalog shapes to be exact.
148 **kwds
149 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`.
151 Notes
152 -----
153 Typical usage:
155 .. code-block: py
157 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100,
158 100))
159 dataset = TestDataset(bbox)
160 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26))
161 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24),
162 shape=lsst.afw.geom.Quadrupole(8, 7, 2))
163 with dataset.addBlend() as family:
164 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72))
165 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
166 exposure, catalog = dataset.realize(noise=100.0,
167 schema=TestDataset.makeMinimalSchema())
168 """
170 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
171 if exposure is None:
172 exposure = self.makeEmptyExposure(bbox, **kwds)
173 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
174 self.exposure = exposure
175 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter())
176 self.schema = self.makeMinimalSchema()
177 self.catalog = lsst.afw.table.SourceCatalog(self.schema)
179 @classmethod
180 def makeMinimalSchema(cls):
181 """Return the minimal schema needed to hold truth catalog fields.
183 Notes
184 -----
185 When `TestDataset.realize` is called, the schema must include at least
186 these fields. Usually it will include additional fields for
187 measurement algorithm outputs, allowing the same catalog to be used
188 for both truth values (the fields from the minimal schema) and the
189 measurements.
190 """
191 if not hasattr(cls, "_schema"):
192 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
193 cls.keys = {}
194 cls.keys["parent"] = schema.find("parent").key
195 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
196 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64,
197 doc="true instFlux", units="count")
198 cls.keys["instFluxErr"] = schema.addField("truth_instFluxErr", type=np.float64,
199 doc="true instFluxErr", units="count")
200 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
201 schema, "truth", "true simulated centroid", "pixel"
202 )
203 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
204 schema, "truth", ['x', 'y'], "pixel"
205 )
206 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
207 doc="set if the object is a star")
208 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields(
209 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
210 )
211 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
212 doc="set if the object is a star")
213 schema.getAliasMap().set("slot_Shape", "truth")
214 schema.getAliasMap().set("slot_Centroid", "truth")
215 schema.getAliasMap().set("slot_ModelFlux", "truth")
216 cls._schema = schema
217 schema = lsst.afw.table.Schema(cls._schema)
218 schema.disconnectAliases()
219 return schema
221 @staticmethod
222 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
223 minRotation=None, maxRotation=None,
224 minRefShift=None, maxRefShift=None,
225 minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
226 """Return a perturbed version of the input WCS.
228 Create a new undistorted TAN WCS that is similar but not identical to
229 another, with random scaling, rotation, and offset (in both pixel
230 position and reference position).
232 Parameters
233 ----------
234 oldWcs : `lsst.afw.geom.SkyWcs`
235 The input WCS.
236 minScaleFactor : `float`
237 Minimum scale factor to apply to the input WCS.
238 maxScaleFactor : `float`
239 Maximum scale factor to apply to the input WCS.
240 minRotation : `lsst.geom.Angle` or `None`
241 Minimum rotation to apply to the input WCS. If `None`, defaults to
242 30 degrees.
243 maxRotation : `lsst.geom.Angle` or `None`
244 Minimum rotation to apply to the input WCS. If `None`, defaults to
245 60 degrees.
246 minRefShift : `lsst.geom.Angle` or `None`
247 Miniumum shift to apply to the input WCS reference value. If
248 `None`, defaults to 0.5 arcsec.
249 maxRefShift : `lsst.geom.Angle` or `None`
250 Miniumum shift to apply to the input WCS reference value. If
251 `None`, defaults to 1.0 arcsec.
252 minPixShift : `float`
253 Minimum shift to apply to the input WCS reference pixel.
254 maxPixShift : `float`
255 Maximum shift to apply to the input WCS reference pixel.
256 randomSeed : `int`
257 Random seed.
259 Returns
260 -------
261 newWcs : `lsst.afw.geom.SkyWcs`
262 A perturbed version of the input WCS.
264 Notes
265 -----
266 The maximum and minimum arguments are interpreted as absolute values
267 for a split range that covers both positive and negative values (as
268 this method is used in testing, it is typically most important to
269 avoid perturbations near zero). Scale factors are treated somewhat
270 differently: the actual scale factor is chosen between
271 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``)
272 and (``1/minScaleFactor``).
274 The default range for rotation is 30-60 degrees, and the default range
275 for reference shift is 0.5-1.0 arcseconds (these cannot be safely
276 included directly as default values because Angle objects are
277 mutable).
279 The random number generator is primed with the seed given. If
280 `None`, a seed is automatically chosen.
281 """
282 random_state = np.random.RandomState(randomSeed)
283 if minRotation is None:
284 minRotation = 30.0*lsst.geom.degrees
285 if maxRotation is None:
286 maxRotation = 60.0*lsst.geom.degrees
287 if minRefShift is None:
288 minRefShift = 0.5*lsst.geom.arcseconds
289 if maxRefShift is None:
290 maxRefShift = 1.0*lsst.geom.arcseconds
292 def splitRandom(min1, max1, min2=None, max2=None):
293 if min2 is None:
294 min2 = -max1
295 if max2 is None:
296 max2 = -min1
297 if random_state.uniform() > 0.5:
298 return float(random_state.uniform(min1, max1))
299 else:
300 return float(random_state.uniform(min2, max2))
301 # Generate random perturbations
302 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
303 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
304 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
305 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
306 pixShiftX = splitRandom(minPixShift, maxPixShift)
307 pixShiftY = splitRandom(minPixShift, maxPixShift)
308 # Compute new CD matrix
309 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
310 rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
311 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
312 newTransform = oldTransform*rTransform*sTransform
313 matrix = newTransform.getMatrix()
314 # Compute new coordinate reference pixel (CRVAL)
315 oldSkyOrigin = oldWcs.getSkyOrigin()
316 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
317 oldSkyOrigin.getDec() + refShiftDec)
318 # Compute new pixel reference pixel (CRPIX)
319 oldPixOrigin = oldWcs.getPixelOrigin()
320 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
321 oldPixOrigin.getY() + pixShiftY)
322 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
324 @staticmethod
325 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4):
326 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values.
328 Parameters
329 ----------
330 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
331 Bounding box of the image in image coordinates.
332 wcs : `lsst.afw.geom.SkyWcs`, optional
333 New WCS for the exposure (created from CRVAL and CDELT if `None`).
334 crval : `lsst.afw.geom.SpherePoint`, optional
335 ICRS center of the TAN WCS attached to the image. If `None`, (45
336 degrees, 45 degrees) is assumed.
337 cdelt : `lsst.geom.Angle`, optional
338 Pixel scale of the image. If `None`, 0.2 arcsec is assumed.
339 psfSigma : `float`, optional
340 Radius (sigma) of the Gaussian PSF attached to the image
341 psfDim : `int`, optional
342 Width and height of the image's Gaussian PSF attached to the image
343 calibration : `float`, optional
344 The spatially-constant calibration (in nJy/count) to set the
345 PhotoCalib of the exposure.
347 Returns
348 -------
349 exposure : `lsst.age.image.ExposureF`
350 An empty image.
351 """
352 if wcs is None:
353 if crval is None:
354 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
355 if cdelt is None:
356 cdelt = 0.2*lsst.geom.arcseconds
357 crpix = lsst.geom.Box2D(bbox).getCenter()
358 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
359 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
360 exposure = lsst.afw.image.ExposureF(bbox)
361 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
362 photoCalib = lsst.afw.image.PhotoCalib(calibration)
363 visitInfo = lsst.afw.image.VisitInfo(id=1234,
364 exposureTime=30.0,
365 date=lsst.daf.base.DateTime(60000.0),
366 observatory=lsst.afw.coord.Observatory(11.1*lsst.geom.degrees,
367 22.2*lsst.geom.degrees,
368 0.333))
369 exposure.setWcs(wcs)
370 exposure.setPsf(psf)
371 exposure.setPhotoCalib(photoCalib)
372 exposure.info.setVisitInfo(visitInfo)
373 return exposure
375 @staticmethod
376 def drawGaussian(bbox, instFlux, ellipse):
377 """Create an image of an elliptical Gaussian.
379 Parameters
380 ----------
381 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
382 Bounding box of image to create.
383 instFlux : `float`
384 Total instrumental flux of the Gaussian (normalized analytically,
385 not using pixel values).
386 ellipse : `lsst.afw.geom.Ellipse`
387 Defines the centroid and shape.
389 Returns
390 -------
391 image : `lsst.afw.image.ImageF`
392 An image of the Gaussian.
393 """
394 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
395 np.arange(bbox.getBeginY(), bbox.getEndY()))
396 t = ellipse.getGridTransform()
397 xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
398 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
399 image = lsst.afw.image.ImageF(bbox)
400 image.array[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea())
401 return image
403 def _installFootprint(self, record, image, setPeakSignificance=True):
404 """Create simulated Footprint and add it to a truth catalog record.
405 """
406 schema = lsst.afw.detection.PeakTable.makeMinimalSchema()
407 if setPeakSignificance:
408 schema.addField("significance", type=float,
409 doc="Ratio of peak value to configured standard deviation.")
410 # Run detection on the single-source image
411 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema)
412 # the call below to the FootprintSet ctor is actually a grow operation
413 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
414 if setPeakSignificance:
415 # This isn't a traditional significance, since we're using the VALUE
416 # threshold type, but it's the best we can do in that case.
417 for footprint in fpSet.getFootprints():
418 footprint.updatePeakSignificance(self.threshold.getValue())
419 # Update the full exposure's mask plane to indicate the detection
420 fpSet.setMask(self.exposure.mask, "DETECTED")
421 # Attach the new footprint to the exposure
422 if len(fpSet.getFootprints()) > 1:
423 raise RuntimeError("Threshold value results in multiple Footprints for a single object")
424 if len(fpSet.getFootprints()) == 0:
425 raise RuntimeError("Threshold value results in zero Footprints for object")
426 record.setFootprint(fpSet.getFootprints()[0])
428 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True):
429 """Add a source to the simulation.
431 To insert a point source with a given signal-to-noise (sn), the total
432 ``instFlux`` should be: ``sn*noise*psf_scale``, where ``noise`` is the
433 noise you will pass to ``realize()``, and
434 ``psf_scale=sqrt(4*pi*r^2)``, where ``r`` is the width of the PSF.
436 Parameters
437 ----------
438 instFlux : `float`
439 Total instFlux of the source to be added.
440 centroid : `lsst.geom.Point2D`
441 Position of the source to be added.
442 shape : `lsst.afw.geom.Quadrupole`
443 Second moments of the source before PSF convolution. Note that the
444 truth catalog records post-convolution moments. If `None`, a point
445 source will be added.
446 setPeakSignificance : `bool`
447 Set the ``significance`` field for peaks in the footprints?
448 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance``
449 for how this field is computed for real datasets.
451 Returns
452 -------
453 record : `lsst.afw.table.SourceRecord`
454 A truth catalog record.
455 image : `lsst.afw.image.ImageF`
456 Single-source image corresponding to the new source.
457 """
458 # Create and set the truth catalog fields
459 record = self.catalog.addNew()
460 record.set(self.keys["instFlux"], instFlux)
461 record.set(self.keys["instFluxErr"], 0)
462 record.set(self.keys["centroid"], centroid)
463 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
464 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
465 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
466 if shape is None:
467 record.set(self.keys["isStar"], True)
468 fullShape = self.psfShape
469 else:
470 record.set(self.keys["isStar"], False)
471 fullShape = shape.convolve(self.psfShape)
472 record.set(self.keys["shape"], fullShape)
473 # Create an image containing just this source
474 image = self.drawGaussian(self.exposure.getBBox(), instFlux,
475 lsst.afw.geom.Ellipse(fullShape, centroid))
476 # Generate a footprint for this source
477 self._installFootprint(record, image, setPeakSignificance)
478 # Actually add the source to the full exposure
479 self.exposure.image.array[:, :] += image.array
480 return record, image
482 def addBlend(self):
483 """Return a context manager which can add a blend of multiple sources.
485 Notes
486 -----
487 Note that nothing stops you from creating overlapping sources just using the addSource() method,
488 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
489 produced by the detection and deblending pipelines.
491 Examples
492 --------
493 .. code-block: py
494 d = TestDataset(...)
495 with d.addBlend() as b:
496 b.addChild(flux1, centroid1)
497 b.addChild(flux2, centroid2, shape2)
498 """
499 return BlendContext(self)
501 def transform(self, wcs, **kwds):
502 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib.
504 Parameters
505 ----------
506 wcs : `lsst.afw.geom.SkyWcs`
507 WCS for the new dataset.
508 **kwds
509 Additional keyword arguments passed on to
510 `TestDataset.makeEmptyExposure`. If not specified, these revert
511 to the defaults for `~TestDataset.makeEmptyExposure`, not the
512 values in the current dataset.
514 Returns
515 -------
516 newDataset : `TestDataset`
517 Transformed copy of this dataset.
518 """
519 bboxD = lsst.geom.Box2D()
520 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
521 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
522 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
523 bboxI = lsst.geom.Box2I(bboxD)
524 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
525 oldPhotoCalib = self.exposure.getPhotoCalib()
526 newPhotoCalib = result.exposure.getPhotoCalib()
527 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter())
528 for record in self.catalog:
529 if record.get(self.keys["nChild"]):
530 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
531 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"]))
532 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude)
533 oldCentroid = record.get(self.keys["centroid"])
534 newCentroid = xyt.applyForward(oldCentroid)
535 if record.get(self.keys["isStar"]):
536 newDeconvolvedShape = None
537 else:
538 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
539 oldFullShape = record.get(self.keys["shape"])
540 oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
541 oldFullShape.getIxx() - oldPsfShape.getIxx(),
542 oldFullShape.getIyy() - oldPsfShape.getIyy(),
543 oldFullShape.getIxy() - oldPsfShape.getIxy(),
544 False
545 )
546 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
547 result.addSource(newFlux, newCentroid, newDeconvolvedShape)
548 return result
550 def realize(self, noise, schema, randomSeed=1):
551 r"""Simulate an exposure and detection catalog for this dataset.
553 The simulation includes noise, and the detection catalog includes
554 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s.
556 Parameters
557 ----------
558 noise : `float`
559 Standard deviation of noise to be added to the exposure. The
560 noise will be Gaussian and constant, appropriate for the
561 sky-limited regime.
562 schema : `lsst.afw.table.Schema`
563 Schema of the new catalog to be created. Must start with
564 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be
565 `True`), but typically contains fields for already-configured
566 measurement algorithms as well.
567 randomSeed : `int`, optional
568 Seed for the random number generator.
569 If `None`, a seed is chosen automatically.
571 Returns
572 -------
573 `exposure` : `lsst.afw.image.ExposureF`
574 Simulated image.
575 `catalog` : `lsst.afw.table.SourceCatalog`
576 Simulated detection catalog.
577 """
578 random_state = np.random.RandomState(randomSeed)
579 assert schema.contains(self.schema)
580 mapper = lsst.afw.table.SchemaMapper(self.schema)
581 mapper.addMinimalSchema(self.schema, True)
582 exposure = self.exposure.clone()
583 exposure.variance.array[:, :] = noise**2
584 exposure.image.array[:, :] += random_state.randn(exposure.height, exposure.width)*noise
585 catalog = lsst.afw.table.SourceCatalog(schema)
586 catalog.extend(self.catalog, mapper=mapper)
587 # Loop over sources and generate new HeavyFootprints that divide up
588 # the noisy pixels, not the ideal no-noise pixels.
589 for record in catalog:
590 # parent objects have non-Heavy Footprints, which don't need to be
591 # updated after adding noise.
592 if record.getParent() == 0:
593 continue
594 # get flattened arrays that correspond to the no-noise and noisy
595 # parent images
596 parent = catalog.find(record.getParent())
597 footprint = parent.getFootprint()
598 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
599 footprint.spans.flatten(parentFluxArrayNoNoise, self.exposure.image.array, self.exposure.getXY0())
600 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
601 footprint.spans.flatten(parentFluxArrayNoisy, exposure.image.array, exposure.getXY0())
602 oldHeavy = record.getFootprint()
603 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
604 # N.B. this isn't a copy ctor - it's a copy from a vanilla
605 # Footprint, so it doesn't copy the arrays we don't want to
606 # change, and hence we have to do that ourselves below.
607 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
608 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
609 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
610 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
611 record.setFootprint(newHeavy)
612 return exposure, catalog
615class AlgorithmTestCase:
616 """Base class for tests of measurement tasks.
617 """
618 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
619 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`.
621 Only the specified plugin and its dependencies will be run; the
622 Centroid, Shape, and ModelFlux slots will be set to the truth fields
623 generated by the `TestDataset` class.
625 Parameters
626 ----------
627 plugin : `str`
628 Name of measurement plugin to enable.
629 dependencies : iterable of `str`, optional
630 Names of dependencies of the measurement plugin.
632 Returns
633 -------
634 config : `SingleFrameMeasurementTask.ConfigClass`
635 The resulting task configuration.
636 """
637 config = SingleFrameMeasurementTask.ConfigClass()
638 with warnings.catch_warnings():
639 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning)
640 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True)
641 config.slots.centroid = "truth"
642 config.slots.shape = "truth"
643 config.slots.modelFlux = None
644 config.slots.apFlux = None
645 config.slots.psfFlux = None
646 config.slots.gaussianFlux = None
647 config.slots.calibFlux = None
648 config.plugins.names = (plugin,) + tuple(dependencies)
649 return config
651 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
652 algMetadata=None):
653 """Create a configured instance of `SingleFrameMeasurementTask`.
655 Parameters
656 ----------
657 plugin : `str`, optional
658 Name of measurement plugin to enable. If `None`, a configuration
659 must be supplied as the ``config`` parameter. If both are
660 specified, ``config`` takes precedence.
661 dependencies : iterable of `str`, optional
662 Names of dependencies of the specified measurement plugin.
663 config : `SingleFrameMeasurementTask.ConfigClass`, optional
664 Configuration for the task. If `None`, a measurement plugin must
665 be supplied as the ``plugin`` paramter. If both are specified,
666 ``config`` takes precedence.
667 schema : `lsst.afw.table.Schema`, optional
668 Measurement table schema. If `None`, a default schema is
669 generated.
670 algMetadata : `lsst.daf.base.PropertyList`, optional
671 Measurement algorithm metadata. If `None`, a default container
672 will be generated.
674 Returns
675 -------
676 task : `SingleFrameMeasurementTask`
677 A configured instance of the measurement task.
678 """
679 if config is None:
680 if plugin is None:
681 raise ValueError("Either plugin or config argument must not be None")
682 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
683 if schema is None:
684 schema = TestDataset.makeMinimalSchema()
685 # Clear all aliases so only those defined by config are set.
686 schema.setAliasMap(None)
687 if algMetadata is None:
688 algMetadata = lsst.daf.base.PropertyList()
689 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
691 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
692 """Create an instance of `ForcedMeasurementTask.ConfigClass`.
694 In addition to the plugins specified in the plugin and dependencies
695 arguments, the `TransformedCentroid` and `TransformedShape` plugins
696 will be run and used as the centroid and shape slots; these simply
697 transform the reference catalog centroid and shape to the measurement
698 coordinate system.
700 Parameters
701 ----------
702 plugin : `str`
703 Name of measurement plugin to enable.
704 dependencies : iterable of `str`, optional
705 Names of dependencies of the measurement plugin.
707 Returns
708 -------
709 config : `ForcedMeasurementTask.ConfigClass`
710 The resulting task configuration.
711 """
713 config = ForcedMeasurementTask.ConfigClass()
714 config.slots.centroid = "base_TransformedCentroid"
715 config.slots.shape = "base_TransformedShape"
716 config.slots.modelFlux = None
717 config.slots.apFlux = None
718 config.slots.psfFlux = None
719 config.slots.gaussianFlux = None
720 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
721 "base_TransformedShape")
722 return config
724 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
725 algMetadata=None):
726 """Create a configured instance of `ForcedMeasurementTask`.
728 Parameters
729 ----------
730 plugin : `str`, optional
731 Name of measurement plugin to enable. If `None`, a configuration
732 must be supplied as the ``config`` parameter. If both are
733 specified, ``config`` takes precedence.
734 dependencies : iterable of `str`, optional
735 Names of dependencies of the specified measurement plugin.
736 config : `SingleFrameMeasurementTask.ConfigClass`, optional
737 Configuration for the task. If `None`, a measurement plugin must
738 be supplied as the ``plugin`` paramter. If both are specified,
739 ``config`` takes precedence.
740 refSchema : `lsst.afw.table.Schema`, optional
741 Reference table schema. If `None`, a default schema is
742 generated.
743 algMetadata : `lsst.daf.base.PropertyList`, optional
744 Measurement algorithm metadata. If `None`, a default container
745 will be generated.
747 Returns
748 -------
749 task : `ForcedMeasurementTask`
750 A configured instance of the measurement task.
751 """
752 if config is None:
753 if plugin is None:
754 raise ValueError("Either plugin or config argument must not be None")
755 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
756 if refSchema is None:
757 refSchema = TestDataset.makeMinimalSchema()
758 if algMetadata is None:
759 algMetadata = lsst.daf.base.PropertyList()
760 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
763class TransformTestCase:
764 """Base class for testing measurement transformations.
766 Notes
767 -----
768 We test both that the transform itself operates successfully (fluxes are
769 converted to magnitudes, flags are propagated properly) and that the
770 transform is registered as the default for the appropriate measurement
771 algorithms.
773 In the simple case of one-measurement-per-transformation, the developer
774 need not directly write any tests themselves: simply customizing the class
775 variables is all that is required. More complex measurements (e.g.
776 multiple aperture fluxes) require extra effort.
777 """
778 name = "MeasurementTransformTest"
779 """The name used for the measurement algorithm (str).
781 Notes
782 -----
783 This determines the names of the fields in the resulting catalog. This
784 default should generally be fine, but subclasses can override if
785 required.
786 """
788 # These should be customized by subclassing.
789 controlClass = None
790 algorithmClass = None
791 transformClass = None
793 flagNames = ("flag",)
794 """Flags which may be set by the algorithm being tested (iterable of `str`).
795 """
797 # The plugin being tested should be registered under these names for
798 # single frame and forced measurement. Should be customized by
799 # subclassing.
800 singleFramePlugins = ()
801 forcedPlugins = ()
803 def setUp(self):
804 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200))
805 self.calexp = TestDataset.makeEmptyExposure(bbox)
806 self._setupTransform()
808 def tearDown(self):
809 del self.calexp
810 del self.inputCat
811 del self.mapper
812 del self.transform
813 del self.outputCat
815 def _populateCatalog(self, baseNames):
816 records = []
817 for flagValue in (True, False):
818 records.append(self.inputCat.addNew())
819 for baseName in baseNames:
820 for flagName in self.flagNames:
821 if records[-1].schema.join(baseName, flagName) in records[-1].schema:
822 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
823 self._setFieldsInRecords(records, baseName)
825 def _checkOutput(self, baseNames):
826 for inSrc, outSrc in zip(self.inputCat, self.outputCat):
827 for baseName in baseNames:
828 self._compareFieldsInRecords(inSrc, outSrc, baseName)
829 for flagName in self.flagNames:
830 keyName = outSrc.schema.join(baseName, flagName)
831 if keyName in inSrc.schema:
832 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
833 else:
834 self.assertFalse(keyName in outSrc.schema)
836 def _runTransform(self, doExtend=True):
837 if doExtend:
838 self.outputCat.extend(self.inputCat, mapper=self.mapper)
839 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib())
841 def testTransform(self, baseNames=None):
842 """Test the transformation on a catalog containing random data.
844 Parameters
845 ----------
846 baseNames : iterable of `str`
847 Iterable of the initial parts of measurement field names.
849 Notes
850 -----
851 We check that:
853 - An appropriate exception is raised on an attempt to transform
854 between catalogs with different numbers of rows;
855 - Otherwise, all appropriate conversions are properly appled and that
856 flags have been propagated.
858 The ``baseNames`` argument requires some explanation. This should be
859 an iterable of the leading parts of the field names for each
860 measurement; that is, everything that appears before ``_instFlux``,
861 ``_flag``, etc. In the simple case of a single measurement per plugin,
862 this is simply equal to ``self.name`` (thus measurements are stored as
863 ``self.name + "_instFlux"``, etc). More generally, the developer may
864 specify whatever iterable they require. For example, to handle
865 multiple apertures, we could have ``(self.name + "_0", self.name +
866 "_1", ...)``.
867 """
868 baseNames = baseNames or [self.name]
869 self._populateCatalog(baseNames)
870 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
871 self._runTransform()
872 self._checkOutput(baseNames)
874 def _checkRegisteredTransform(self, registry, name):
875 # If this is a Python-based transform, we can compare directly; if
876 # it's wrapped C++, we need to compare the wrapped class.
877 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
879 def testRegistration(self):
880 """Test that the transformation is appropriately registered.
881 """
882 for pluginName in self.singleFramePlugins:
883 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
884 for pluginName in self.forcedPlugins:
885 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
888class SingleFramePluginTransformSetupHelper:
890 def _setupTransform(self):
891 self.control = self.controlClass()
892 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema()
893 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
894 # it doesn't matter for this test since we won't actually use the plugins for anything besides
895 # defining the schema.
896 inputSchema.getAliasMap().set("slot_Centroid", "dummy")
897 inputSchema.getAliasMap().set("slot_Shape", "dummy")
898 self.algorithmClass(self.control, self.name, inputSchema)
899 inputSchema.getAliasMap().erase("slot_Centroid")
900 inputSchema.getAliasMap().erase("slot_Shape")
901 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema)
902 self.mapper = lsst.afw.table.SchemaMapper(inputSchema)
903 self.transform = self.transformClass(self.control, self.name, self.mapper)
904 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
907class ForcedPluginTransformSetupHelper:
909 def _setupTransform(self):
910 self.control = self.controlClass()
911 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(),
912 lsst.afw.table.SourceTable.makeMinimalSchema())
913 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
914 # it doesn't matter for this test since we won't actually use the plugins for anything besides
915 # defining the schema.
916 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
917 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
918 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
919 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
920 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
921 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
922 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
923 self.transform = self.transformClass(self.control, self.name, self.mapper)
924 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
927class FluxTransformTestCase(TransformTestCase):
929 def _setFieldsInRecords(self, records, name):
930 for record in records:
931 record[record.schema.join(name, 'instFlux')] = np.random.random()
932 record[record.schema.join(name, 'instFluxErr')] = np.random.random()
934 # Negative instFluxes should be converted to NaNs.
935 assert len(records) > 1
936 records[0][record.schema.join(name, 'instFlux')] = -1
938 def _compareFieldsInRecords(self, inSrc, outSrc, name):
939 instFluxName = inSrc.schema.join(name, 'instFlux')
940 instFluxErrName = inSrc.schema.join(name, 'instFluxErr')
941 if inSrc[instFluxName] > 0:
942 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
943 inSrc[instFluxErrName])
944 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value)
945 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
946 else:
947 # negative instFlux results in NaN magnitude, but can still have finite error
948 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
949 if np.isnan(inSrc[instFluxErrName]):
950 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
951 else:
952 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
953 inSrc[instFluxErrName])
954 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
957class CentroidTransformTestCase(TransformTestCase):
959 def _setFieldsInRecords(self, records, name):
960 for record in records:
961 record[record.schema.join(name, 'x')] = np.random.random()
962 record[record.schema.join(name, 'y')] = np.random.random()
963 # Some algorithms set no errors; some set only sigma on x & y; some provide
964 # a full covariance matrix. Set only those which exist in the schema.
965 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'):
966 fieldName = record.schema.join(name, fieldSuffix)
967 if fieldName in record.schema:
968 record[fieldName] = np.random.random()
970 def _compareFieldsInRecords(self, inSrc, outSrc, name):
971 centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
972 centroidResult = centroidResultKey.get(inSrc)
974 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
975 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
976 self.assertEqual(coordTruth, coord)
978 # If the centroid has an associated uncertainty matrix, the coordinate
979 # must have one too, and vice versa.
980 try:
981 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
982 ["ra", "dec"]).get(outSrc)
983 except lsst.pex.exceptions.NotFoundError:
984 self.assertFalse(centroidResultKey.getCentroidErr().isValid())
985 else:
986 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
987 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
988 centroidResult.getCentroidErr()),
989 transform.getLinear().getMatrix().transpose())
990 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)