lsst.meas.base g67924a670a+6e23f27fc3
Loading...
Searching...
No Matches
tests.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import warnings
23
24import numpy as np
25
26import lsst.geom
27import lsst.afw.table
28import lsst.afw.image
30import lsst.afw.geom
31import lsst.afw.coord
32import lsst.daf.base
34
35from .sfm import SingleFrameMeasurementTask
36from .forcedMeasurement import ForcedMeasurementTask
37from ._measBaseLib import CentroidResultKey
38
39__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
40 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
41 "FluxTransformTestCase", "CentroidTransformTestCase")
42
43
45 """Context manager which adds multiple overlapping sources and a parent.
46
47 Notes
48 -----
49 This is used as the return value for `TestDataset.addBlend`, and this is
50 the only way it should be used.
51 """
52
53 def __init__(self, owner):
54 self.owner = owner
55 self.parentRecord = self.owner.catalog.addNew()
56 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
57 self.children = []
58
59 def __enter__(self):
60 # BlendContext is its own context manager, so we just return self.
61 return self
62
63 def addChild(self, instFlux, centroid, shape=None):
64 """Add a child to the blend; return corresponding truth catalog record.
65
66 instFlux : `float`
67 Total instFlux of the source to be added.
68 centroid : `lsst.geom.Point2D`
69 Position of the source to be added.
70 shape : `lsst.afw.geom.Quadrupole`
71 Second moments of the source before PSF convolution. Note that
72 the truth catalog records post-convolution moments)
73 """
74 record, image = self.owner.addSource(instFlux, centroid, shape)
75 record.set(self.owner.keys["parent"], self.parentRecord.getId())
76 self.parentImage += image
77 self.children.append((record, image))
78 return record
79
80 def __exit__(self, type_, value, tb):
81 # We're not using the context manager for any kind of exception safety
82 # or guarantees; we just want the nice "with" statement syntax.
83
84 if type_ is not None:
85 # exception was raised; just skip all this and let it propagate
86 return
87
88 # On exit, compute and set the truth values for the parent object.
89 self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
90 # Compute instFlux from sum of component fluxes
91 instFlux = 0.0
92 for record, image in self.children:
93 instFlux += record.get(self.owner.keys["instFlux"])
94 self.parentRecord.set(self.owner.keys["instFlux"], instFlux)
95 # Compute centroid from instFlux-weighted mean of component centroids
96 x = 0.0
97 y = 0.0
98 for record, image in self.children:
99 w = record.get(self.owner.keys["instFlux"])/instFlux
100 x += record.get(self.owner.keys["centroid"].getX())*w
101 y += record.get(self.owner.keys["centroid"].getY())*w
102 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
103 # Compute shape from instFlux-weighted mean of offset component shapes
104 xx = 0.0
105 yy = 0.0
106 xy = 0.0
107 for record, image in self.children:
108 w = record.get(self.owner.keys["instFlux"])/instFlux
109 dx = record.get(self.owner.keys["centroid"].getX()) - x
110 dy = record.get(self.owner.keys["centroid"].getY()) - y
111 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
112 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
113 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
114 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
115 # Run detection on the parent image to get the parent Footprint.
116 self.owner._installFootprint(self.parentRecord, self.parentImage)
117 # Create perfect HeavyFootprints for all children; these will need to
118 # be modified later to account for the noise we'll add to the image.
119 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.maskedImage, True)
120 for record, image in self.children:
121 deblend.image.array[:, :] = image.array
122 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
123 record.setFootprint(heavyFootprint)
124
125
127 """A simulated dataset consisuting of test image and truth catalog.
128
129 TestDataset creates an idealized image made of pure Gaussians (including a
130 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints
131 that simulated the outputs of detection and deblending. Multiple noise
132 realizations can be created from the same underlying sources, allowing
133 uncertainty estimates to be verified via Monte Carlo.
134
135 Parameters
136 ----------
137 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
138 Bounding box of the test image.
139 threshold : `float`
140 Threshold absolute value used to determine footprints for
141 simulated sources. This thresholding will be applied before noise is
142 actually added to images (or before the noise level is even known), so
143 this will necessarily produce somewhat artificial footprints.
144 exposure : `lsst.afw.image.ExposureF`
145 The image to which test sources should be added. Ownership should
146 be considered transferred from the caller to the TestDataset.
147 Must have a Gaussian PSF for truth catalog shapes to be exact.
148 **kwds
149 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`.
150
151 Notes
152 -----
153 Typical usage:
154
155 .. code-block: py
156
158 100))
159 dataset = TestDataset(bbox)
160 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26))
161 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24),
162 shape=lsst.afw.geom.Quadrupole(8, 7, 2))
163 with dataset.addBlend() as family:
164 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72))
165 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
166 exposure, catalog = dataset.realize(noise=100.0,
167 schema=TestDataset.makeMinimalSchema())
168 """
169
170 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
171 if exposure is None:
172 exposure = self.makeEmptyExposure(bbox, **kwds)
173 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
174 self.exposure = exposure
175 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter())
178
179 @classmethod
181 """Return the minimal schema needed to hold truth catalog fields.
182
183 Notes
184 -----
185 When `TestDataset.realize` is called, the schema must include at least
186 these fields. Usually it will include additional fields for
187 measurement algorithm outputs, allowing the same catalog to be used
188 for both truth values (the fields from the minimal schema) and the
189 measurements.
190 """
191 if not hasattr(cls, "_schema"):
193 cls.keys = {}
194 cls.keys["parent"] = schema.find("parent").key
195 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
196 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64,
197 doc="true instFlux", units="count")
198 cls.keys["instFluxErr"] = schema.addField("truth_instFluxErr", type=np.float64,
199 doc="true instFluxErr", units="count")
200 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
201 schema, "truth", "true simulated centroid", "pixel"
202 )
203 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
204 schema, "truth", ['x', 'y'], "pixel"
205 )
206 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
207 doc="set if the object is a star")
209 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
210 )
211 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
212 doc="set if the object is a star")
213 schema.getAliasMap().set("slot_Shape", "truth")
214 schema.getAliasMap().set("slot_Centroid", "truth")
215 schema.getAliasMap().set("slot_ModelFlux", "truth")
216 cls._schema = schema
217 schema = lsst.afw.table.Schema(cls._schema)
218 schema.disconnectAliases()
219 return schema
220
221 @staticmethod
222 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
223 minRotation=None, maxRotation=None,
224 minRefShift=None, maxRefShift=None,
225 minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
226 """Return a perturbed version of the input WCS.
227
228 Create a new undistorted TAN WCS that is similar but not identical to
229 another, with random scaling, rotation, and offset (in both pixel
230 position and reference position).
231
232 Parameters
233 ----------
234 oldWcs : `lsst.afw.geom.SkyWcs`
235 The input WCS.
236 minScaleFactor : `float`
237 Minimum scale factor to apply to the input WCS.
238 maxScaleFactor : `float`
239 Maximum scale factor to apply to the input WCS.
240 minRotation : `lsst.geom.Angle` or `None`
241 Minimum rotation to apply to the input WCS. If `None`, defaults to
242 30 degrees.
243 maxRotation : `lsst.geom.Angle` or `None`
244 Minimum rotation to apply to the input WCS. If `None`, defaults to
245 60 degrees.
246 minRefShift : `lsst.geom.Angle` or `None`
247 Miniumum shift to apply to the input WCS reference value. If
248 `None`, defaults to 0.5 arcsec.
249 maxRefShift : `lsst.geom.Angle` or `None`
250 Miniumum shift to apply to the input WCS reference value. If
251 `None`, defaults to 1.0 arcsec.
252 minPixShift : `float`
253 Minimum shift to apply to the input WCS reference pixel.
254 maxPixShift : `float`
255 Maximum shift to apply to the input WCS reference pixel.
256 randomSeed : `int`
257 Random seed.
258
259 Returns
260 -------
261 newWcs : `lsst.afw.geom.SkyWcs`
262 A perturbed version of the input WCS.
263
264 Notes
265 -----
266 The maximum and minimum arguments are interpreted as absolute values
267 for a split range that covers both positive and negative values (as
268 this method is used in testing, it is typically most important to
269 avoid perturbations near zero). Scale factors are treated somewhat
270 differently: the actual scale factor is chosen between
271 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``)
272 and (``1/minScaleFactor``).
273
274 The default range for rotation is 30-60 degrees, and the default range
275 for reference shift is 0.5-1.0 arcseconds (these cannot be safely
276 included directly as default values because Angle objects are
277 mutable).
278
279 The random number generator is primed with the seed given. If
280 `None`, a seed is automatically chosen.
281 """
282 random_state = np.random.RandomState(randomSeed)
283 if minRotation is None:
284 minRotation = 30.0*lsst.geom.degrees
285 if maxRotation is None:
286 maxRotation = 60.0*lsst.geom.degrees
287 if minRefShift is None:
288 minRefShift = 0.5*lsst.geom.arcseconds
289 if maxRefShift is None:
290 maxRefShift = 1.0*lsst.geom.arcseconds
291
292 def splitRandom(min1, max1, min2=None, max2=None):
293 if min2 is None:
294 min2 = -max1
295 if max2 is None:
296 max2 = -min1
297 if random_state.uniform() > 0.5:
298 return float(random_state.uniform(min1, max1))
299 else:
300 return float(random_state.uniform(min2, max2))
301 # Generate random perturbations
302 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
303 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
304 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
305 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
306 pixShiftX = splitRandom(minPixShift, maxPixShift)
307 pixShiftY = splitRandom(minPixShift, maxPixShift)
308 # Compute new CD matrix
309 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
310 rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
311 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
312 newTransform = oldTransform*rTransform*sTransform
313 matrix = newTransform.getMatrix()
314 # Compute new coordinate reference pixel (CRVAL)
315 oldSkyOrigin = oldWcs.getSkyOrigin()
316 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
317 oldSkyOrigin.getDec() + refShiftDec)
318 # Compute new pixel reference pixel (CRPIX)
319 oldPixOrigin = oldWcs.getPixelOrigin()
320 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
321 oldPixOrigin.getY() + pixShiftY)
322 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
323
324 @staticmethod
325 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4):
326 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values.
327
328 Parameters
329 ----------
330 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
331 Bounding box of the image in image coordinates.
332 wcs : `lsst.afw.geom.SkyWcs`, optional
333 New WCS for the exposure (created from CRVAL and CDELT if `None`).
334 crval : `lsst.afw.geom.SpherePoint`, optional
335 ICRS center of the TAN WCS attached to the image. If `None`, (45
336 degrees, 45 degrees) is assumed.
337 cdelt : `lsst.geom.Angle`, optional
338 Pixel scale of the image. If `None`, 0.2 arcsec is assumed.
339 psfSigma : `float`, optional
340 Radius (sigma) of the Gaussian PSF attached to the image
341 psfDim : `int`, optional
342 Width and height of the image's Gaussian PSF attached to the image
343 calibration : `float`, optional
344 The spatially-constant calibration (in nJy/count) to set the
345 PhotoCalib of the exposure.
346
347 Returns
348 -------
349 exposure : `lsst.age.image.ExposureF`
350 An empty image.
351 """
352 if wcs is None:
353 if crval is None:
354 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
355 if cdelt is None:
356 cdelt = 0.2*lsst.geom.arcseconds
357 crpix = lsst.geom.Box2D(bbox).getCenter()
358 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
359 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
360 exposure = lsst.afw.image.ExposureF(bbox)
361 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
362 photoCalib = lsst.afw.image.PhotoCalib(calibration)
363 visitInfo = lsst.afw.image.VisitInfo(id=1234,
364 exposureTime=30.0,
365 date=lsst.daf.base.DateTime(60000.0),
366 observatory=lsst.afw.coord.Observatory(11.1*lsst.geom.degrees,
367 22.2*lsst.geom.degrees,
368 0.333))
369 exposure.setWcs(wcs)
370 exposure.setPsf(psf)
371 exposure.setPhotoCalib(photoCalib)
372 exposure.info.setVisitInfo(visitInfo)
373 return exposure
374
375 @staticmethod
376 def drawGaussian(bbox, instFlux, ellipse):
377 """Create an image of an elliptical Gaussian.
378
379 Parameters
380 ----------
381 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
382 Bounding box of image to create.
383 instFlux : `float`
384 Total instrumental flux of the Gaussian (normalized analytically,
385 not using pixel values).
386 ellipse : `lsst.afw.geom.Ellipse`
387 Defines the centroid and shape.
388
389 Returns
390 -------
391 image : `lsst.afw.image.ImageF`
392 An image of the Gaussian.
393 """
394 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
395 np.arange(bbox.getBeginY(), bbox.getEndY()))
396 t = ellipse.getGridTransform()
397 xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
398 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
399 image = lsst.afw.image.ImageF(bbox)
400 image.array[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea())
401 return image
402
403 def _installFootprint(self, record, image, setPeakSignificance=True):
404 """Create simulated Footprint and add it to a truth catalog record.
405 """
407 if setPeakSignificance:
408 schema.addField("significance", type=float,
409 doc="Ratio of peak value to configured standard deviation.")
410 # Run detection on the single-source image
411 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema)
412 # the call below to the FootprintSet ctor is actually a grow operation
413 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
414 if setPeakSignificance:
415 # This isn't a traditional significance, since we're using the VALUE
416 # threshold type, but it's the best we can do in that case.
417 for footprint in fpSet.getFootprints():
418 footprint.updatePeakSignificance(self.threshold.getValue())
419 # Update the full exposure's mask plane to indicate the detection
420 fpSet.setMask(self.exposure.mask, "DETECTED")
421 # Attach the new footprint to the exposure
422 if len(fpSet.getFootprints()) > 1:
423 raise RuntimeError("Threshold value results in multiple Footprints for a single object")
424 if len(fpSet.getFootprints()) == 0:
425 raise RuntimeError("Threshold value results in zero Footprints for object")
426 record.setFootprint(fpSet.getFootprints()[0])
427
428 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True):
429 """Add a source to the simulation.
430
431 To insert a point source with a given signal-to-noise (sn), the total
432 ``instFlux`` should be: ``sn*noise*psf_scale``, where ``noise`` is the
433 noise you will pass to ``realize()``, and
434 ``psf_scale=sqrt(4*pi*r^2)``, where ``r`` is the width of the PSF.
435
436 Parameters
437 ----------
438 instFlux : `float`
439 Total instFlux of the source to be added.
440 centroid : `lsst.geom.Point2D`
441 Position of the source to be added.
442 shape : `lsst.afw.geom.Quadrupole`
443 Second moments of the source before PSF convolution. Note that the
444 truth catalog records post-convolution moments. If `None`, a point
445 source will be added.
446 setPeakSignificance : `bool`
447 Set the ``significance`` field for peaks in the footprints?
448 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance``
449 for how this field is computed for real datasets.
450
451 Returns
452 -------
454 A truth catalog record.
455 image : `lsst.afw.image.ImageF`
456 Single-source image corresponding to the new source.
457 """
458 # Create and set the truth catalog fields
459 record = self.catalog.addNew()
460 record.set(self.keys["instFlux"], instFlux)
461 record.set(self.keys["instFluxErr"], 0)
462 record.set(self.keys["centroid"], centroid)
463 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
464 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
465 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
466 if shape is None:
467 record.set(self.keys["isStar"], True)
468 fullShape = self.psfShape
469 else:
470 record.set(self.keys["isStar"], False)
471 fullShape = shape.convolve(self.psfShape)
472 record.set(self.keys["shape"], fullShape)
473 # Create an image containing just this source
474 image = self.drawGaussian(self.exposure.getBBox(), instFlux,
475 lsst.afw.geom.Ellipse(fullShape, centroid))
476 # Generate a footprint for this source
477 self._installFootprint(record, image, setPeakSignificance)
478 # Actually add the source to the full exposure
479 self.exposure.image.array[:, :] += image.array
480 return record, image
481
482 def addBlend(self):
483 """Return a context manager which can add a blend of multiple sources.
484
485 Notes
486 -----
487 Note that nothing stops you from creating overlapping sources just using the addSource() method,
488 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
489 produced by the detection and deblending pipelines.
490
491 Examples
492 --------
493 .. code-block: py
494 d = TestDataset(...)
495 with d.addBlend() as b:
496 b.addChild(flux1, centroid1)
497 b.addChild(flux2, centroid2, shape2)
498 """
499 return BlendContext(self)
500
501 def transform(self, wcs, **kwds):
502 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib.
503
504 Parameters
505 ----------
507 WCS for the new dataset.
508 **kwds
509 Additional keyword arguments passed on to
510 `TestDataset.makeEmptyExposure`. If not specified, these revert
511 to the defaults for `~TestDataset.makeEmptyExposure`, not the
512 values in the current dataset.
513
514 Returns
515 -------
516 newDataset : `TestDataset`
517 Transformed copy of this dataset.
518 """
519 bboxD = lsst.geom.Box2D()
520 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
521 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
522 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
523 bboxI = lsst.geom.Box2I(bboxD)
524 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
525 oldPhotoCalib = self.exposure.getPhotoCalib()
526 newPhotoCalib = result.exposure.getPhotoCalib()
527 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter())
528 for record in self.catalog:
529 if record.get(self.keys["nChild"]):
530 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
531 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"]))
532 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude)
533 oldCentroid = record.get(self.keys["centroid"])
534 newCentroid = xyt.applyForward(oldCentroid)
535 if record.get(self.keys["isStar"]):
536 newDeconvolvedShape = None
537 else:
538 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
539 oldFullShape = record.get(self.keys["shape"])
540 oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
541 oldFullShape.getIxx() - oldPsfShape.getIxx(),
542 oldFullShape.getIyy() - oldPsfShape.getIyy(),
543 oldFullShape.getIxy() - oldPsfShape.getIxy(),
544 False
545 )
546 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
547 result.addSource(newFlux, newCentroid, newDeconvolvedShape)
548 return result
549
550 def realize(self, noise, schema, randomSeed=1):
551 r"""Simulate an exposure and detection catalog for this dataset.
552
553 The simulation includes noise, and the detection catalog includes
554 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s.
555
556 Parameters
557 ----------
558 noise : `float`
559 Standard deviation of noise to be added to the exposure. The
560 noise will be Gaussian and constant, appropriate for the
561 sky-limited regime.
562 schema : `lsst.afw.table.Schema`
563 Schema of the new catalog to be created. Must start with
564 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be
565 `True`), but typically contains fields for already-configured
566 measurement algorithms as well.
567 randomSeed : `int`, optional
568 Seed for the random number generator.
569 If `None`, a seed is chosen automatically.
570
571 Returns
572 -------
573 `exposure` : `lsst.afw.image.ExposureF`
574 Simulated image.
575 `catalog` : `lsst.afw.table.SourceCatalog`
576 Simulated detection catalog.
577 """
578 random_state = np.random.RandomState(randomSeed)
579 assert schema.contains(self.schema)
581 mapper.addMinimalSchema(self.schema, True)
582 exposure = self.exposure.clone()
583 exposure.variance.array[:, :] = noise**2
584 exposure.image.array[:, :] += random_state.randn(exposure.height, exposure.width)*noise
585 catalog = lsst.afw.table.SourceCatalog(schema)
586 catalog.extend(self.catalog, mapper=mapper)
587 # Loop over sources and generate new HeavyFootprints that divide up
588 # the noisy pixels, not the ideal no-noise pixels.
589 for record in catalog:
590 # parent objects have non-Heavy Footprints, which don't need to be
591 # updated after adding noise.
592 if record.getParent() == 0:
593 continue
594 # get flattened arrays that correspond to the no-noise and noisy
595 # parent images
596 parent = catalog.find(record.getParent())
597 footprint = parent.getFootprint()
598 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
599 footprint.spans.flatten(parentFluxArrayNoNoise, self.exposure.image.array, self.exposure.getXY0())
600 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
601 footprint.spans.flatten(parentFluxArrayNoisy, exposure.image.array, exposure.getXY0())
602 oldHeavy = record.getFootprint()
603 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
604 # N.B. this isn't a copy ctor - it's a copy from a vanilla
605 # Footprint, so it doesn't copy the arrays we don't want to
606 # change, and hence we have to do that ourselves below.
607 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
608 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
609 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
610 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
611 record.setFootprint(newHeavy)
612 return exposure, catalog
613
614
616 """Base class for tests of measurement tasks.
617 """
618 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
619 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`.
620
621 Only the specified plugin and its dependencies will be run; the
622 Centroid, Shape, and ModelFlux slots will be set to the truth fields
623 generated by the `TestDataset` class.
624
625 Parameters
626 ----------
627 plugin : `str`
628 Name of measurement plugin to enable.
629 dependencies : iterable of `str`, optional
630 Names of dependencies of the measurement plugin.
631
632 Returns
633 -------
634 config : `SingleFrameMeasurementTask.ConfigClass`
635 The resulting task configuration.
636 """
637 config = SingleFrameMeasurementTask.ConfigClass()
638 with warnings.catch_warnings():
639 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning)
640 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True)
641 config.slots.centroid = "truth"
642 config.slots.shape = "truth"
643 config.slots.modelFlux = None
644 config.slots.apFlux = None
645 config.slots.psfFlux = None
646 config.slots.gaussianFlux = None
647 config.slots.calibFlux = None
648 config.plugins.names = (plugin,) + tuple(dependencies)
649 return config
650
651 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
652 algMetadata=None):
653 """Create a configured instance of `SingleFrameMeasurementTask`.
654
655 Parameters
656 ----------
657 plugin : `str`, optional
658 Name of measurement plugin to enable. If `None`, a configuration
659 must be supplied as the ``config`` parameter. If both are
660 specified, ``config`` takes precedence.
661 dependencies : iterable of `str`, optional
662 Names of dependencies of the specified measurement plugin.
663 config : `SingleFrameMeasurementTask.ConfigClass`, optional
664 Configuration for the task. If `None`, a measurement plugin must
665 be supplied as the ``plugin`` paramter. If both are specified,
666 ``config`` takes precedence.
667 schema : `lsst.afw.table.Schema`, optional
668 Measurement table schema. If `None`, a default schema is
669 generated.
670 algMetadata : `lsst.daf.base.PropertyList`, optional
671 Measurement algorithm metadata. If `None`, a default container
672 will be generated.
673
674 Returns
675 -------
676 task : `SingleFrameMeasurementTask`
677 A configured instance of the measurement task.
678 """
679 if config is None:
680 if plugin is None:
681 raise ValueError("Either plugin or config argument must not be None")
682 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
683 if schema is None:
684 schema = TestDataset.makeMinimalSchema()
685 # Clear all aliases so only those defined by config are set.
686 schema.setAliasMap(None)
687 if algMetadata is None:
688 algMetadata = lsst.daf.base.PropertyList()
689 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
690
691 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
692 """Create an instance of `ForcedMeasurementTask.ConfigClass`.
693
694 In addition to the plugins specified in the plugin and dependencies
695 arguments, the `TransformedCentroid` and `TransformedShape` plugins
696 will be run and used as the centroid and shape slots; these simply
697 transform the reference catalog centroid and shape to the measurement
698 coordinate system.
699
700 Parameters
701 ----------
702 plugin : `str`
703 Name of measurement plugin to enable.
704 dependencies : iterable of `str`, optional
705 Names of dependencies of the measurement plugin.
706
707 Returns
708 -------
709 config : `ForcedMeasurementTask.ConfigClass`
710 The resulting task configuration.
711 """
712
713 config = ForcedMeasurementTask.ConfigClass()
714 config.slots.centroid = "base_TransformedCentroid"
715 config.slots.shape = "base_TransformedShape"
716 config.slots.modelFlux = None
717 config.slots.apFlux = None
718 config.slots.psfFlux = None
719 config.slots.gaussianFlux = None
720 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
721 "base_TransformedShape")
722 return config
723
724 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
725 algMetadata=None):
726 """Create a configured instance of `ForcedMeasurementTask`.
727
728 Parameters
729 ----------
730 plugin : `str`, optional
731 Name of measurement plugin to enable. If `None`, a configuration
732 must be supplied as the ``config`` parameter. If both are
733 specified, ``config`` takes precedence.
734 dependencies : iterable of `str`, optional
735 Names of dependencies of the specified measurement plugin.
736 config : `SingleFrameMeasurementTask.ConfigClass`, optional
737 Configuration for the task. If `None`, a measurement plugin must
738 be supplied as the ``plugin`` paramter. If both are specified,
739 ``config`` takes precedence.
740 refSchema : `lsst.afw.table.Schema`, optional
741 Reference table schema. If `None`, a default schema is
742 generated.
743 algMetadata : `lsst.daf.base.PropertyList`, optional
744 Measurement algorithm metadata. If `None`, a default container
745 will be generated.
746
747 Returns
748 -------
749 task : `ForcedMeasurementTask`
750 A configured instance of the measurement task.
751 """
752 if config is None:
753 if plugin is None:
754 raise ValueError("Either plugin or config argument must not be None")
755 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
756 if refSchema is None:
757 refSchema = TestDataset.makeMinimalSchema()
758 if algMetadata is None:
759 algMetadata = lsst.daf.base.PropertyList()
760 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
761
762
764 """Base class for testing measurement transformations.
765
766 Notes
767 -----
768 We test both that the transform itself operates successfully (fluxes are
769 converted to magnitudes, flags are propagated properly) and that the
770 transform is registered as the default for the appropriate measurement
771 algorithms.
772
773 In the simple case of one-measurement-per-transformation, the developer
774 need not directly write any tests themselves: simply customizing the class
775 variables is all that is required. More complex measurements (e.g.
776 multiple aperture fluxes) require extra effort.
777 """
778 name = "MeasurementTransformTest"
779 """The name used for the measurement algorithm (str).
780
781 Notes
782 -----
783 This determines the names of the fields in the resulting catalog. This
784 default should generally be fine, but subclasses can override if
785 required.
786 """
787
788 # These should be customized by subclassing.
789 controlClass = None
790 algorithmClass = None
791 transformClass = None
792
793 flagNames = ("flag",)
794 """Flags which may be set by the algorithm being tested (iterable of `str`).
795 """
796
797 # The plugin being tested should be registered under these names for
798 # single frame and forced measurement. Should be customized by
799 # subclassing.
800 singleFramePlugins = ()
801 forcedPlugins = ()
802
803 def setUp(self):
805 self.calexp = TestDataset.makeEmptyExposure(bbox)
806 self._setupTransform()
807
808 def tearDown(self):
809 del self.calexp
810 del self.inputCat
811 del self.mapper
812 del self.transform
813 del self.outputCat
814
815 def _populateCatalog(self, baseNames):
816 records = []
817 for flagValue in (True, False):
818 records.append(self.inputCat.addNew())
819 for baseName in baseNames:
820 for flagName in self.flagNames:
821 if records[-1].schema.join(baseName, flagName) in records[-1].schema:
822 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
823 self._setFieldsInRecords(records, baseName)
824
825 def _checkOutput(self, baseNames):
826 for inSrc, outSrc in zip(self.inputCat, self.outputCat):
827 for baseName in baseNames:
828 self._compareFieldsInRecords(inSrc, outSrc, baseName)
829 for flagName in self.flagNames:
830 keyName = outSrc.schema.join(baseName, flagName)
831 if keyName in inSrc.schema:
832 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
833 else:
834 self.assertFalse(keyName in outSrc.schema)
835
836 def _runTransform(self, doExtend=True):
837 if doExtend:
838 self.outputCat.extend(self.inputCat, mapper=self.mapper)
839 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib())
840
841 def testTransform(self, baseNames=None):
842 """Test the transformation on a catalog containing random data.
843
844 Parameters
845 ----------
846 baseNames : iterable of `str`
847 Iterable of the initial parts of measurement field names.
848
849 Notes
850 -----
851 We check that:
852
853 - An appropriate exception is raised on an attempt to transform
854 between catalogs with different numbers of rows;
855 - Otherwise, all appropriate conversions are properly appled and that
856 flags have been propagated.
857
858 The ``baseNames`` argument requires some explanation. This should be
859 an iterable of the leading parts of the field names for each
860 measurement; that is, everything that appears before ``_instFlux``,
861 ``_flag``, etc. In the simple case of a single measurement per plugin,
862 this is simply equal to ``self.name`` (thus measurements are stored as
863 ``self.name + "_instFlux"``, etc). More generally, the developer may
864 specify whatever iterable they require. For example, to handle
865 multiple apertures, we could have ``(self.name + "_0", self.name +
866 "_1", ...)``.
867 """
868 baseNames = baseNames or [self.name]
869 self._populateCatalog(baseNames)
872 self._checkOutput(baseNames)
873
874 def _checkRegisteredTransform(self, registry, name):
875 # If this is a Python-based transform, we can compare directly; if
876 # it's wrapped C++, we need to compare the wrapped class.
877 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
878
880 """Test that the transformation is appropriately registered.
881 """
882 for pluginName in self.singleFramePlugins:
883 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
884 for pluginName in self.forcedPlugins:
885 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
886
887
889
891 self.control = self.controlClass()
893 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
894 # it doesn't matter for this test since we won't actually use the plugins for anything besides
895 # defining the schema.
896 inputSchema.getAliasMap().set("slot_Centroid", "dummy")
897 inputSchema.getAliasMap().set("slot_Shape", "dummy")
898 self.algorithmClass(self.control, self.name, inputSchema)
899 inputSchema.getAliasMap().erase("slot_Centroid")
900 inputSchema.getAliasMap().erase("slot_Shape")
903 self.transform = self.transformClass(self.control, self.name, self.mapper)
904 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
905
906
908
910 self.control = self.controlClass()
913 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
914 # it doesn't matter for this test since we won't actually use the plugins for anything besides
915 # defining the schema.
916 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
917 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
918 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
919 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
920 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
921 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
922 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
923 self.transform = self.transformClass(self.control, self.name, self.mapper)
924 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
925
926
928
929 def _setFieldsInRecords(self, records, name):
930 for record in records:
931 record[record.schema.join(name, 'instFlux')] = np.random.random()
932 record[record.schema.join(name, 'instFluxErr')] = np.random.random()
933
934 # Negative instFluxes should be converted to NaNs.
935 assert len(records) > 1
936 records[0][record.schema.join(name, 'instFlux')] = -1
937
938 def _compareFieldsInRecords(self, inSrc, outSrc, name):
939 instFluxName = inSrc.schema.join(name, 'instFlux')
940 instFluxErrName = inSrc.schema.join(name, 'instFluxErr')
941 if inSrc[instFluxName] > 0:
942 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
943 inSrc[instFluxErrName])
944 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value)
945 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
946 else:
947 # negative instFlux results in NaN magnitude, but can still have finite error
948 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
949 if np.isnan(inSrc[instFluxErrName]):
950 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
951 else:
952 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
953 inSrc[instFluxErrName])
954 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
955
956
958
959 def _setFieldsInRecords(self, records, name):
960 for record in records:
961 record[record.schema.join(name, 'x')] = np.random.random()
962 record[record.schema.join(name, 'y')] = np.random.random()
963 # Some algorithms set no errors; some set only sigma on x & y; some provide
964 # a full covariance matrix. Set only those which exist in the schema.
965 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'):
966 fieldName = record.schema.join(name, fieldSuffix)
967 if fieldName in record.schema:
968 record[fieldName] = np.random.random()
969
970 def _compareFieldsInRecords(self, inSrc, outSrc, name):
971 centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
972 centroidResult = centroidResultKey.get(inSrc)
973
974 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
975 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
976 self.assertEqual(coordTruth, coord)
977
978 # If the centroid has an associated uncertainty matrix, the coordinate
979 # must have one too, and vice versa.
980 try:
981 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
982 ["ra", "dec"]).get(outSrc)
984 self.assertFalse(centroidResultKey.getCentroidErr().isValid())
985 else:
986 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
987 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
988 centroidResult.getCentroidErr()),
989 transform.getLinear().getMatrix().transpose())
990 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
static afw::table::Schema makeMinimalSchema()
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
static Schema makeMinimalSchema()
static LinearTransform makeRotation(Angle t) noexcept
static LinearTransform makeScaling(double s) noexcept
makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition tests.py:618
makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition tests.py:691
makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition tests.py:725
makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition tests.py:652
addChild(self, instFlux, centroid, shape=None)
Definition tests.py:63
__exit__(self, type_, value, tb)
Definition tests.py:80
_compareFieldsInRecords(self, inSrc, outSrc, name)
Definition tests.py:970
_compareFieldsInRecords(self, inSrc, outSrc, name)
Definition tests.py:938
_setFieldsInRecords(self, records, name)
Definition tests.py:929
makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=1)
Definition tests.py:225
realize(self, noise, schema, randomSeed=1)
Definition tests.py:550
addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True)
Definition tests.py:428
_installFootprint(self, record, image, setPeakSignificance=True)
Definition tests.py:403
drawGaussian(bbox, instFlux, ellipse)
Definition tests.py:376
makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4)
Definition tests.py:325
__init__(self, bbox, threshold=10.0, exposure=None, **kwds)
Definition tests.py:170
testTransform(self, baseNames=None)
Definition tests.py:841
_runTransform(self, doExtend=True)
Definition tests.py:836
_checkRegisteredTransform(self, registry, name)
Definition tests.py:874
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
Eigen::Matrix2d makeCdMatrix(lsst::geom::Angle const &scale, lsst::geom::Angle const &orientation=0 *lsst::geom::degrees, bool flipX=false)
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
lsst::geom::AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, lsst::geom::Point2D const &inPoint)