Coverage for python/lsst/meas/base/tests.py: 13%
389 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:07 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:07 -0800
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
24import lsst.geom
25import lsst.afw.table
26import lsst.afw.image
27import lsst.afw.detection
28import lsst.afw.geom
29import lsst.pex.exceptions
31from .sfm import SingleFrameMeasurementTask
32from .forcedMeasurement import ForcedMeasurementTask
33from . import CentroidResultKey
35__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
36 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
37 "FluxTransformTestCase", "CentroidTransformTestCase")
40class BlendContext:
41 """Context manager which adds multiple overlapping sources and a parent.
43 Notes
44 -----
45 This is used as the return value for `TestDataset.addBlend`, and this is
46 the only way it should be used.
47 """
49 def __init__(self, owner):
50 self.owner = owner
51 self.parentRecord = self.owner.catalog.addNew()
52 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
53 self.children = []
55 def __enter__(self):
56 # BlendContext is its own context manager, so we just return self.
57 return self
59 def addChild(self, instFlux, centroid, shape=None):
60 """Add a child to the blend; return corresponding truth catalog record.
62 instFlux : `float`
63 Total instFlux of the source to be added.
64 centroid : `lsst.geom.Point2D`
65 Position of the source to be added.
66 shape : `lsst.afw.geom.Quadrupole`
67 Second moments of the source before PSF convolution. Note that
68 the truth catalog records post-convolution moments)
69 """
70 record, image = self.owner.addSource(instFlux, centroid, shape)
71 record.set(self.owner.keys["parent"], self.parentRecord.getId())
72 self.parentImage += image
73 self.children.append((record, image))
74 return record
76 def __exit__(self, type_, value, tb):
77 # We're not using the context manager for any kind of exception safety
78 # or guarantees; we just want the nice "with" statement syntax.
80 if type_ is not None:
81 # exception was raised; just skip all this and let it propagate
82 return
84 # On exit, compute and set the truth values for the parent object.
85 self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
86 # Compute instFlux from sum of component fluxes
87 instFlux = 0.0
88 for record, image in self.children:
89 instFlux += record.get(self.owner.keys["instFlux"])
90 self.parentRecord.set(self.owner.keys["instFlux"], instFlux)
91 # Compute centroid from instFlux-weighted mean of component centroids
92 x = 0.0
93 y = 0.0
94 for record, image in self.children:
95 w = record.get(self.owner.keys["instFlux"])/instFlux
96 x += record.get(self.owner.keys["centroid"].getX())*w
97 y += record.get(self.owner.keys["centroid"].getY())*w
98 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
99 # Compute shape from instFlux-weighted mean of offset component shapes
100 xx = 0.0
101 yy = 0.0
102 xy = 0.0
103 for record, image in self.children:
104 w = record.get(self.owner.keys["instFlux"])/instFlux
105 dx = record.get(self.owner.keys["centroid"].getX()) - x
106 dy = record.get(self.owner.keys["centroid"].getY()) - y
107 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
108 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
109 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
110 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
111 # Run detection on the parent image to get the parent Footprint.
112 self.owner._installFootprint(self.parentRecord, self.parentImage)
113 # Create perfect HeavyFootprints for all children; these will need to
114 # be modified later to account for the noise we'll add to the image.
115 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
116 for record, image in self.children:
117 deblend.getImage().getArray()[:, :] = image.getArray()
118 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
119 record.setFootprint(heavyFootprint)
122class TestDataset:
123 """A simulated dataset consisuting of test image and truth catalog.
125 TestDataset creates an idealized image made of pure Gaussians (including a
126 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints
127 that simulated the outputs of detection and deblending. Multiple noise
128 realizations can be created from the same underlying sources, allowing
129 uncertainty estimates to be verified via Monte Carlo.
131 Parameters
132 ----------
133 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
134 Bounding box of the test image.
135 threshold : `float`
136 Threshold absolute value used to determine footprints for
137 simulated sources. This thresholding will be applied before noise is
138 actually added to images (or before the noise level is even known), so
139 this will necessarily produce somewhat artificial footprints.
140 exposure : `lsst.afw.image.ExposureF`
141 The image to which test sources should be added. Ownership should
142 be considered transferred from the caller to the TestDataset.
143 Must have a Gaussian PSF for truth catalog shapes to be exact.
144 **kwds
145 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`.
147 Notes
148 -----
149 Typical usage:
151 .. code-block: py
153 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100,
154 100))
155 dataset = TestDataset(bbox)
156 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26))
157 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24),
158 shape=lsst.afw.geom.Quadrupole(8, 7, 2))
159 with dataset.addBlend() as family:
160 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72))
161 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
162 exposure, catalog = dataset.realize(noise=100.0,
163 schema=TestDataset.makeMinimalSchema())
164 """
166 @classmethod
167 def makeMinimalSchema(cls):
168 """Return the minimal schema needed to hold truth catalog fields.
170 Notes
171 -----
172 When `TestDataset.realize` is called, the schema must include at least
173 these fields. Usually it will include additional fields for
174 measurement algorithm outputs, allowing the same catalog to be used
175 for both truth values (the fields from the minimal schema) and the
176 measurements.
177 """
178 if not hasattr(cls, "_schema"):
179 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
180 cls.keys = {}
181 cls.keys["parent"] = schema.find("parent").key
182 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
183 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64,
184 doc="true instFlux", units="count")
185 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
186 schema, "truth", "true simulated centroid", "pixel"
187 )
188 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
189 schema, "truth", ['x', 'y'], "pixel"
190 )
191 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
192 doc="set if the object is a star")
193 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields(
194 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
195 )
196 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
197 doc="set if the object is a star")
198 schema.getAliasMap().set("slot_Shape", "truth")
199 schema.getAliasMap().set("slot_Centroid", "truth")
200 schema.getAliasMap().set("slot_ModelFlux", "truth")
201 cls._schema = schema
202 schema = lsst.afw.table.Schema(cls._schema)
203 schema.disconnectAliases()
204 return schema
206 @staticmethod
207 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
208 minRotation=None, maxRotation=None,
209 minRefShift=None, maxRefShift=None,
210 minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
211 """Return a perturbed version of the input WCS.
213 Create a new undistorted TAN WCS that is similar but not identical to
214 another, with random scaling, rotation, and offset (in both pixel
215 position and reference position).
217 Parameters
218 ----------
219 oldWcs : `lsst.afw.geom.SkyWcs`
220 The input WCS.
221 minScaleFactor : `float`
222 Minimum scale factor to apply to the input WCS.
223 maxScaleFactor : `float`
224 Maximum scale factor to apply to the input WCS.
225 minRotation : `lsst.geom.Angle` or `None`
226 Minimum rotation to apply to the input WCS. If `None`, defaults to
227 30 degrees.
228 maxRotation : `lsst.geom.Angle` or `None`
229 Minimum rotation to apply to the input WCS. If `None`, defaults to
230 60 degrees.
231 minRefShift : `lsst.geom.Angle` or `None`
232 Miniumum shift to apply to the input WCS reference value. If
233 `None`, defaults to 0.5 arcsec.
234 maxRefShift : `lsst.geom.Angle` or `None`
235 Miniumum shift to apply to the input WCS reference value. If
236 `None`, defaults to 1.0 arcsec.
237 minPixShift : `float`
238 Minimum shift to apply to the input WCS reference pixel.
239 maxPixShift : `float`
240 Maximum shift to apply to the input WCS reference pixel.
241 randomSeed : `int`
242 Random seed.
244 Returns
245 -------
246 newWcs : `lsst.afw.geom.SkyWcs`
247 A perturbed version of the input WCS.
249 Notes
250 -----
251 The maximum and minimum arguments are interpreted as absolute values
252 for a split range that covers both positive and negative values (as
253 this method is used in testing, it is typically most important to
254 avoid perturbations near zero). Scale factors are treated somewhat
255 differently: the actual scale factor is chosen between
256 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``)
257 and (``1/minScaleFactor``).
259 The default range for rotation is 30-60 degrees, and the default range
260 for reference shift is 0.5-1.0 arcseconds (these cannot be safely
261 included directly as default values because Angle objects are
262 mutable).
264 The random number generator is primed with the seed given. If
265 `None`, a seed is automatically chosen.
266 """
267 random_state = np.random.RandomState(randomSeed)
268 if minRotation is None:
269 minRotation = 30.0*lsst.geom.degrees
270 if maxRotation is None:
271 maxRotation = 60.0*lsst.geom.degrees
272 if minRefShift is None:
273 minRefShift = 0.5*lsst.geom.arcseconds
274 if maxRefShift is None:
275 maxRefShift = 1.0*lsst.geom.arcseconds
277 def splitRandom(min1, max1, min2=None, max2=None):
278 if min2 is None:
279 min2 = -max1
280 if max2 is None:
281 max2 = -min1
282 if random_state.uniform() > 0.5:
283 return float(random_state.uniform(min1, max1))
284 else:
285 return float(random_state.uniform(min2, max2))
286 # Generate random perturbations
287 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
288 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
289 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
290 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
291 pixShiftX = splitRandom(minPixShift, maxPixShift)
292 pixShiftY = splitRandom(minPixShift, maxPixShift)
293 # Compute new CD matrix
294 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
295 rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
296 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
297 newTransform = oldTransform*rTransform*sTransform
298 matrix = newTransform.getMatrix()
299 # Compute new coordinate reference pixel (CRVAL)
300 oldSkyOrigin = oldWcs.getSkyOrigin()
301 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
302 oldSkyOrigin.getDec() + refShiftDec)
303 # Compute new pixel reference pixel (CRPIX)
304 oldPixOrigin = oldWcs.getPixelOrigin()
305 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
306 oldPixOrigin.getY() + pixShiftY)
307 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
309 @staticmethod
310 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4):
311 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values.
313 Parameters
314 ----------
315 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
316 Bounding box of the image in image coordinates.
317 wcs : `lsst.afw.geom.SkyWcs`, optional
318 New WCS for the exposure (created from CRVAL and CDELT if `None`).
319 crval : `lsst.afw.geom.SpherePoint`, optional
320 ICRS center of the TAN WCS attached to the image. If `None`, (45
321 degrees, 45 degrees) is assumed.
322 cdelt : `lsst.geom.Angle`, optional
323 Pixel scale of the image. If `None`, 0.2 arcsec is assumed.
324 psfSigma : `float`, optional
325 Radius (sigma) of the Gaussian PSF attached to the image
326 psfDim : `int`, optional
327 Width and height of the image's Gaussian PSF attached to the image
328 calibration : `float`, optional
329 The spatially-constant calibration (in nJy/count) to set the
330 PhotoCalib of the exposure.
332 Returns
333 -------
334 exposure : `lsst.age.image.ExposureF`
335 An empty image.
336 """
337 if wcs is None:
338 if crval is None:
339 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
340 if cdelt is None:
341 cdelt = 0.2*lsst.geom.arcseconds
342 crpix = lsst.geom.Box2D(bbox).getCenter()
343 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
344 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
345 exposure = lsst.afw.image.ExposureF(bbox)
346 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
347 photoCalib = lsst.afw.image.PhotoCalib(calibration)
348 exposure.setWcs(wcs)
349 exposure.setPsf(psf)
350 exposure.setPhotoCalib(photoCalib)
351 return exposure
353 @staticmethod
354 def drawGaussian(bbox, instFlux, ellipse):
355 """Create an image of an elliptical Gaussian.
357 Parameters
358 ----------
359 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
360 Bounding box of image to create.
361 instFlux : `float`
362 Total instrumental flux of the Gaussian (normalized analytically,
363 not using pixel values).
364 ellipse : `lsst.afw.geom.Ellipse`
365 Defines the centroid and shape.
367 Returns
368 -------
369 image : `lsst.afw.image.ImageF`
370 An image of the Gaussian.
371 """
372 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
373 np.arange(bbox.getBeginY(), bbox.getEndY()))
374 t = ellipse.getGridTransform()
375 xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
376 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
377 image = lsst.afw.image.ImageF(bbox)
378 image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea())
379 return image
381 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
382 if exposure is None:
383 exposure = self.makeEmptyExposure(bbox, **kwds)
384 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
385 self.exposure = exposure
386 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter())
387 self.schema = self.makeMinimalSchema()
388 self.catalog = lsst.afw.table.SourceCatalog(self.schema)
390 def _installFootprint(self, record, image):
391 """Create simulated Footprint and add it to a truth catalog record.
392 """
393 # Run detection on the single-source image
394 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
395 # the call below to the FootprintSet ctor is actually a grow operation
396 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
397 # Update the full exposure's mask plane to indicate the detection
398 fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
399 # Attach the new footprint to the exposure
400 if len(fpSet.getFootprints()) > 1:
401 raise RuntimeError("Threshold value results in multiple Footprints for a single object")
402 if len(fpSet.getFootprints()) == 0:
403 raise RuntimeError("Threshold value results in zero Footprints for object")
404 record.setFootprint(fpSet.getFootprints()[0])
406 def addSource(self, instFlux, centroid, shape=None):
407 """Add a source to the simulation.
409 Parameters
410 ----------
411 instFlux : `float`
412 Total instFlux of the source to be added.
413 centroid : `lsst.geom.Point2D`
414 Position of the source to be added.
415 shape : `lsst.afw.geom.Quadrupole`
416 Second moments of the source before PSF convolution. Note that the
417 truth catalog records post-convolution moments. If `None`, a point
418 source will be added.
420 Returns
421 -------
422 record : `lsst.afw.table.SourceRecord`
423 A truth catalog record.
424 image : `lsst.afw.image.ImageF`
425 Single-source image corresponding to the new source.
426 """
427 # Create and set the truth catalog fields
428 record = self.catalog.addNew()
429 record.set(self.keys["instFlux"], instFlux)
430 record.set(self.keys["centroid"], centroid)
431 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
432 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
433 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
434 if shape is None:
435 record.set(self.keys["isStar"], True)
436 fullShape = self.psfShape
437 else:
438 record.set(self.keys["isStar"], False)
439 fullShape = shape.convolve(self.psfShape)
440 record.set(self.keys["shape"], fullShape)
441 # Create an image containing just this source
442 image = self.drawGaussian(self.exposure.getBBox(), instFlux,
443 lsst.afw.geom.Ellipse(fullShape, centroid))
444 # Generate a footprint for this source
445 self._installFootprint(record, image)
446 # Actually add the source to the full exposure
447 self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
448 return record, image
450 def addBlend(self):
451 """Return a context manager which can add a blend of multiple sources.
453 Notes
454 -----
455 Note that nothing stops you from creating overlapping sources just using the addSource() method,
456 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
457 produced by the detection and deblending pipelines.
459 Examples
460 --------
461 .. code-block: py
462 d = TestDataset(...)
463 with d.addBlend() as b:
464 b.addChild(flux1, centroid1)
465 b.addChild(flux2, centroid2, shape2)
466 """
467 return BlendContext(self)
469 def transform(self, wcs, **kwds):
470 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib.
472 Parameters
473 ----------
474 wcs : `lsst.afw.geom.SkyWcs`
475 WCS for the new dataset.
476 **kwds
477 Additional keyword arguments passed on to
478 `TestDataset.makeEmptyExposure`. If not specified, these revert
479 to the defaults for `~TestDataset.makeEmptyExposure`, not the
480 values in the current dataset.
482 Returns
483 -------
484 newDataset : `TestDataset`
485 Transformed copy of this dataset.
486 """
487 bboxD = lsst.geom.Box2D()
488 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
489 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
490 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
491 bboxI = lsst.geom.Box2I(bboxD)
492 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
493 oldPhotoCalib = self.exposure.getPhotoCalib()
494 newPhotoCalib = result.exposure.getPhotoCalib()
495 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter())
496 for record in self.catalog:
497 if record.get(self.keys["nChild"]):
498 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
499 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"]))
500 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude)
501 oldCentroid = record.get(self.keys["centroid"])
502 newCentroid = xyt.applyForward(oldCentroid)
503 if record.get(self.keys["isStar"]):
504 newDeconvolvedShape = None
505 else:
506 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
507 oldFullShape = record.get(self.keys["shape"])
508 oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
509 oldFullShape.getIxx() - oldPsfShape.getIxx(),
510 oldFullShape.getIyy() - oldPsfShape.getIyy(),
511 oldFullShape.getIxy() - oldPsfShape.getIxy(),
512 False
513 )
514 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
515 result.addSource(newFlux, newCentroid, newDeconvolvedShape)
516 return result
518 def realize(self, noise, schema, randomSeed=1):
519 r"""Simulate an exposure and detection catalog for this dataset.
521 The simulation includes noise, and the detection catalog includes
522 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s.
524 Parameters
525 ----------
526 noise : `float`
527 Standard deviation of noise to be added to the exposure. The
528 noise will be Gaussian and constant, appropriate for the
529 sky-limited regime.
530 schema : `lsst.afw.table.Schema`
531 Schema of the new catalog to be created. Must start with
532 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be
533 `True`), but typically contains fields for already-configured
534 measurement algorithms as well.
535 randomSeed : `int`, optional
536 Seed for the random number generator.
537 If `None`, a seed is chosen automatically.
539 Returns
540 -------
541 `exposure` : `lsst.afw.image.ExposureF`
542 Simulated image.
543 `catalog` : `lsst.afw.table.SourceCatalog`
544 Simulated detection catalog.
545 """
546 random_state = np.random.RandomState(randomSeed)
547 assert schema.contains(self.schema)
548 mapper = lsst.afw.table.SchemaMapper(self.schema)
549 mapper.addMinimalSchema(self.schema, True)
550 exposure = self.exposure.clone()
551 exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
552 exposure.getMaskedImage().getImage().getArray()[:, :] \
553 += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise
554 catalog = lsst.afw.table.SourceCatalog(schema)
555 catalog.extend(self.catalog, mapper=mapper)
556 # Loop over sources and generate new HeavyFootprints that divide up
557 # the noisy pixels, not the ideal no-noise pixels.
558 for record in catalog:
559 # parent objects have non-Heavy Footprints, which don't need to be
560 # updated after adding noise.
561 if record.getParent() == 0:
562 continue
563 # get flattened arrays that correspond to the no-noise and noisy
564 # parent images
565 parent = catalog.find(record.getParent())
566 footprint = parent.getFootprint()
567 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
568 footprint.spans.flatten(parentFluxArrayNoNoise,
569 self.exposure.getMaskedImage().getImage().getArray(),
570 self.exposure.getXY0())
571 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
572 footprint.spans.flatten(parentFluxArrayNoisy,
573 exposure.getMaskedImage().getImage().getArray(),
574 exposure.getXY0())
575 oldHeavy = record.getFootprint()
576 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
577 # N.B. this isn't a copy ctor - it's a copy from a vanilla
578 # Footprint, so it doesn't copy the arrays we don't want to
579 # change, and hence we have to do that ourselves below.
580 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
581 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
582 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
583 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
584 record.setFootprint(newHeavy)
585 return exposure, catalog
588class AlgorithmTestCase:
590 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
591 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`.
593 Only the specified plugin and its dependencies will be run; the
594 Centroid, Shape, and ModelFlux slots will be set to the truth fields
595 generated by the `TestDataset` class.
597 Parameters
598 ----------
599 plugin : `str`
600 Name of measurement plugin to enable.
601 dependencies : iterable of `str`, optional
602 Names of dependencies of the measurement plugin.
604 Returns
605 -------
606 config : `SingleFrameMeasurementTask.ConfigClass`
607 The resulting task configuration.
608 """
609 config = SingleFrameMeasurementTask.ConfigClass()
610 config.slots.centroid = "truth"
611 config.slots.shape = "truth"
612 config.slots.modelFlux = None
613 config.slots.apFlux = None
614 config.slots.psfFlux = None
615 config.slots.gaussianFlux = None
616 config.slots.calibFlux = None
617 config.plugins.names = (plugin,) + tuple(dependencies)
618 return config
620 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
621 algMetadata=None):
622 """Create a configured instance of `SingleFrameMeasurementTask`.
624 Parameters
625 ----------
626 plugin : `str`, optional
627 Name of measurement plugin to enable. If `None`, a configuration
628 must be supplied as the ``config`` parameter. If both are
629 specified, ``config`` takes precedence.
630 dependencies : iterable of `str`, optional
631 Names of dependencies of the specified measurement plugin.
632 config : `SingleFrameMeasurementTask.ConfigClass`, optional
633 Configuration for the task. If `None`, a measurement plugin must
634 be supplied as the ``plugin`` paramter. If both are specified,
635 ``config`` takes precedence.
636 schema : `lsst.afw.table.Schema`, optional
637 Measurement table schema. If `None`, a default schema is
638 generated.
639 algMetadata : `lsst.daf.base.PropertyList`, optional
640 Measurement algorithm metadata. If `None`, a default container
641 will be generated.
643 Returns
644 -------
645 task : `SingleFrameMeasurementTask`
646 A configured instance of the measurement task.
647 """
648 if config is None:
649 if plugin is None:
650 raise ValueError("Either plugin or config argument must not be None")
651 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
652 if schema is None:
653 schema = TestDataset.makeMinimalSchema()
654 # Clear all aliases so only those defined by config are set.
655 schema.setAliasMap(None)
656 if algMetadata is None:
657 algMetadata = lsst.daf.base.PropertyList()
658 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
660 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
661 """Create an instance of `ForcedMeasurementTask.ConfigClass`.
663 In addition to the plugins specified in the plugin and dependencies
664 arguments, the `TransformedCentroid` and `TransformedShape` plugins
665 will be run and used as the centroid and shape slots; these simply
666 transform the reference catalog centroid and shape to the measurement
667 coordinate system.
669 Parameters
670 ----------
671 plugin : `str`
672 Name of measurement plugin to enable.
673 dependencies : iterable of `str`, optional
674 Names of dependencies of the measurement plugin.
676 Returns
677 -------
678 config : `ForcedMeasurementTask.ConfigClass`
679 The resulting task configuration.
680 """
682 config = ForcedMeasurementTask.ConfigClass()
683 config.slots.centroid = "base_TransformedCentroid"
684 config.slots.shape = "base_TransformedShape"
685 config.slots.modelFlux = None
686 config.slots.apFlux = None
687 config.slots.psfFlux = None
688 config.slots.gaussianFlux = None
689 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
690 "base_TransformedShape")
691 return config
693 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
694 algMetadata=None):
695 """Create a configured instance of `ForcedMeasurementTask`.
697 Parameters
698 ----------
699 plugin : `str`, optional
700 Name of measurement plugin to enable. If `None`, a configuration
701 must be supplied as the ``config`` parameter. If both are
702 specified, ``config`` takes precedence.
703 dependencies : iterable of `str`, optional
704 Names of dependencies of the specified measurement plugin.
705 config : `SingleFrameMeasurementTask.ConfigClass`, optional
706 Configuration for the task. If `None`, a measurement plugin must
707 be supplied as the ``plugin`` paramter. If both are specified,
708 ``config`` takes precedence.
709 refSchema : `lsst.afw.table.Schema`, optional
710 Reference table schema. If `None`, a default schema is
711 generated.
712 algMetadata : `lsst.daf.base.PropertyList`, optional
713 Measurement algorithm metadata. If `None`, a default container
714 will be generated.
716 Returns
717 -------
718 task : `ForcedMeasurementTask`
719 A configured instance of the measurement task.
720 """
721 if config is None:
722 if plugin is None:
723 raise ValueError("Either plugin or config argument must not be None")
724 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
725 if refSchema is None:
726 refSchema = TestDataset.makeMinimalSchema()
727 if algMetadata is None:
728 algMetadata = lsst.daf.base.PropertyList()
729 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
732class TransformTestCase:
733 """Base class for testing measurement transformations.
735 Notes
736 -----
737 We test both that the transform itself operates successfully (fluxes are
738 converted to magnitudes, flags are propagated properly) and that the
739 transform is registered as the default for the appropriate measurement
740 algorithms.
742 In the simple case of one-measurement-per-transformation, the developer
743 need not directly write any tests themselves: simply customizing the class
744 variables is all that is required. More complex measurements (e.g.
745 multiple aperture fluxes) require extra effort.
746 """
747 name = "MeasurementTransformTest"
748 """The name used for the measurement algorithm (str).
750 Notes
751 -----
752 This determines the names of the fields in the resulting catalog. This
753 default should generally be fine, but subclasses can override if
754 required.
755 """
757 # These should be customized by subclassing.
758 controlClass = None
759 algorithmClass = None
760 transformClass = None
762 flagNames = ("flag",)
763 """Flags which may be set by the algorithm being tested (iterable of `str`).
764 """
766 # The plugin being tested should be registered under these names for
767 # single frame and forced measurement. Should be customized by
768 # subclassing.
769 singleFramePlugins = ()
770 forcedPlugins = ()
772 def setUp(self):
773 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200))
774 self.calexp = TestDataset.makeEmptyExposure(bbox)
775 self._setupTransform()
777 def tearDown(self):
778 del self.calexp
779 del self.inputCat
780 del self.mapper
781 del self.transform
782 del self.outputCat
784 def _populateCatalog(self, baseNames):
785 records = []
786 for flagValue in (True, False):
787 records.append(self.inputCat.addNew())
788 for baseName in baseNames:
789 for flagName in self.flagNames:
790 if records[-1].schema.join(baseName, flagName) in records[-1].schema:
791 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
792 self._setFieldsInRecords(records, baseName)
794 def _checkOutput(self, baseNames):
795 for inSrc, outSrc in zip(self.inputCat, self.outputCat):
796 for baseName in baseNames:
797 self._compareFieldsInRecords(inSrc, outSrc, baseName)
798 for flagName in self.flagNames:
799 keyName = outSrc.schema.join(baseName, flagName)
800 if keyName in inSrc.schema:
801 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
802 else:
803 self.assertFalse(keyName in outSrc.schema)
805 def _runTransform(self, doExtend=True):
806 if doExtend:
807 self.outputCat.extend(self.inputCat, mapper=self.mapper)
808 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib())
810 def testTransform(self, baseNames=None):
811 """Test the transformation on a catalog containing random data.
813 Parameters
814 ----------
815 baseNames : iterable of `str`
816 Iterable of the initial parts of measurement field names.
818 Notes
819 -----
820 We check that:
822 - An appropriate exception is raised on an attempt to transform
823 between catalogs with different numbers of rows;
824 - Otherwise, all appropriate conversions are properly appled and that
825 flags have been propagated.
827 The ``baseNames`` argument requires some explanation. This should be
828 an iterable of the leading parts of the field names for each
829 measurement; that is, everything that appears before ``_instFlux``,
830 ``_flag``, etc. In the simple case of a single measurement per plugin,
831 this is simply equal to ``self.name`` (thus measurements are stored as
832 ``self.name + "_instFlux"``, etc). More generally, the developer may
833 specify whatever iterable they require. For example, to handle
834 multiple apertures, we could have ``(self.name + "_0", self.name +
835 "_1", ...)``.
836 """
837 baseNames = baseNames or [self.name]
838 self._populateCatalog(baseNames)
839 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
840 self._runTransform()
841 self._checkOutput(baseNames)
843 def _checkRegisteredTransform(self, registry, name):
844 # If this is a Python-based transform, we can compare directly; if
845 # it's wrapped C++, we need to compare the wrapped class.
846 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
848 def testRegistration(self):
849 """Test that the transformation is appropriately registered.
850 """
851 for pluginName in self.singleFramePlugins:
852 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
853 for pluginName in self.forcedPlugins:
854 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
857class SingleFramePluginTransformSetupHelper:
859 def _setupTransform(self):
860 self.control = self.controlClass()
861 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema()
862 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
863 # it doesn't matter for this test since we won't actually use the plugins for anything besides
864 # defining the schema.
865 inputSchema.getAliasMap().set("slot_Centroid", "dummy")
866 inputSchema.getAliasMap().set("slot_Shape", "dummy")
867 self.algorithmClass(self.control, self.name, inputSchema)
868 inputSchema.getAliasMap().erase("slot_Centroid")
869 inputSchema.getAliasMap().erase("slot_Shape")
870 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema)
871 self.mapper = lsst.afw.table.SchemaMapper(inputSchema)
872 self.transform = self.transformClass(self.control, self.name, self.mapper)
873 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
876class ForcedPluginTransformSetupHelper:
878 def _setupTransform(self):
879 self.control = self.controlClass()
880 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(),
881 lsst.afw.table.SourceTable.makeMinimalSchema())
882 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
883 # it doesn't matter for this test since we won't actually use the plugins for anything besides
884 # defining the schema.
885 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
886 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
887 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
888 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
889 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
890 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
891 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
892 self.transform = self.transformClass(self.control, self.name, self.mapper)
893 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
896class FluxTransformTestCase(TransformTestCase):
898 def _setFieldsInRecords(self, records, name):
899 for record in records:
900 record[record.schema.join(name, 'instFlux')] = np.random.random()
901 record[record.schema.join(name, 'instFluxErr')] = np.random.random()
903 # Negative instFluxes should be converted to NaNs.
904 assert len(records) > 1
905 records[0][record.schema.join(name, 'instFlux')] = -1
907 def _compareFieldsInRecords(self, inSrc, outSrc, name):
908 instFluxName = inSrc.schema.join(name, 'instFlux')
909 instFluxErrName = inSrc.schema.join(name, 'instFluxErr')
910 if inSrc[instFluxName] > 0:
911 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
912 inSrc[instFluxErrName])
913 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value)
914 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
915 else:
916 # negative instFlux results in NaN magnitude, but can still have finite error
917 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
918 if np.isnan(inSrc[instFluxErrName]):
919 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
920 else:
921 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
922 inSrc[instFluxErrName])
923 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
926class CentroidTransformTestCase(TransformTestCase):
928 def _setFieldsInRecords(self, records, name):
929 for record in records:
930 record[record.schema.join(name, 'x')] = np.random.random()
931 record[record.schema.join(name, 'y')] = np.random.random()
932 # Some algorithms set no errors; some set only sigma on x & y; some provide
933 # a full covariance matrix. Set only those which exist in the schema.
934 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'):
935 fieldName = record.schema.join(name, fieldSuffix)
936 if fieldName in record.schema:
937 record[fieldName] = np.random.random()
939 def _compareFieldsInRecords(self, inSrc, outSrc, name):
940 centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
941 centroidResult = centroidResultKey.get(inSrc)
943 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
944 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
945 self.assertEqual(coordTruth, coord)
947 # If the centroid has an associated uncertainty matrix, the coordinate
948 # must have one too, and vice versa.
949 try:
950 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
951 ["ra", "dec"]).get(outSrc)
952 except lsst.pex.exceptions.NotFoundError:
953 self.assertFalse(centroidResultKey.getCentroidErr().isValid())
954 else:
955 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
956 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
957 centroidResult.getCentroidErr()),
958 transform.getLinear().getMatrix().transpose())
959 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)