lsst.meas.base  14.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Macros Groups Pages
tests.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 from builtins import zip
25 from builtins import object
26 import numpy as np
27 
28 import lsst.afw.table
29 import lsst.afw.image
30 import lsst.afw.detection
31 import lsst.afw.geom
32 import lsst.afw.geom.ellipses
33 import lsst.afw.coord
34 import lsst.pex.exceptions
35 
36 from .sfm import SingleFrameMeasurementTask
37 from .forcedMeasurement import ForcedMeasurementTask
38 from . import CentroidResultKey
39 
40 __all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
41  "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
42  "FluxTransformTestCase", "CentroidTransformTestCase")
43 
44 
45 class BlendContext(object):
46  """!
47  A Python context manager used to add multiple overlapping sources along with a parent source
48  that represents all of them together.
49 
50  This is used as the return value for TestDataset.addBlend(), and this is the only way it should
51  be used. The only public method is addChild().
52  """
53 
54  def __init__(self, owner):
55  self.owner = owner
56  self.parentRecord = self.owner.catalog.addNew()
57  self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
58  self.children = []
59 
60  def __enter__(self):
61  # BlendContext is its own context manager, so we just return self.
62  return self
63 
64  def addChild(self, flux, centroid, shape=None):
65  """!
66  Add a child source to the blend, and return the truth catalog record that corresponds to it.
67 
68  @param[in] flux Total flux of the source to be added.
69  @param[in] centroid Position of the source to be added (lsst.afw.geom.Point2D).
70  @param[in] shape 2nd moments of the source before PSF convolution
71  (lsst.afw.geom.ellipses.Quadrupole). Note that the truth catalog
72  records post-convolution moments)
73  """
74  record, image = self.owner.addSource(flux, centroid, shape)
75  record.set(self.owner.keys["parent"], self.parentRecord.getId())
76  self.parentImage += image
77  self.children.append((record, image))
78  return record
79 
80  def __exit__(self, type_, value, tb):
81  # We're not using the context manager for any kind of exception safety or guarantees;
82  # we just want the nice "with" statement syntax.
83  if type_ is not None: # exception was raised; just skip all this and let it propagate
84  return
85  # On exit, we need to compute and set the truth values for the parent object.
86  self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
87  # Compute flux from sum of component fluxes
88  flux = 0.0
89  for record, image in self.children:
90  flux += record.get(self.owner.keys["flux"])
91  self.parentRecord.set(self.owner.keys["flux"], flux)
92  # Compute centroid from flux-weighted mean of component centroids
93  x = 0.0
94  y = 0.0
95  for record, image in self.children:
96  w = record.get(self.owner.keys["flux"])/flux
97  x += record.get(self.owner.keys["centroid"].getX())*w
98  y += record.get(self.owner.keys["centroid"].getY())*w
99  self.parentRecord.set(self.owner.keys["centroid"], lsst.afw.geom.Point2D(x, y))
100  # Compute shape from flux-weighted mean of offset component shapes
101  xx = 0.0
102  yy = 0.0
103  xy = 0.0
104  for record, image in self.children:
105  w = record.get(self.owner.keys["flux"])/flux
106  dx = record.get(self.owner.keys["centroid"].getX()) - x
107  dy = record.get(self.owner.keys["centroid"].getY()) - y
108  xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
109  yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
110  xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
111  self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.ellipses.Quadrupole(xx, yy, xy))
112  # Run detection on the parent image to get the parent Footprint.
113  self.owner._installFootprint(self.parentRecord, self.parentImage)
114  # Create perfect HeavyFootprints for all children; these will need to be modified later to account
115  # for the noise we'll add to the image.
116  deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
117  for record, image in self.children:
118  deblend.getImage().getArray()[:, :] = image.getArray()
119  heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
120  record.setFootprint(heavyFootprint)
121 
122 
123 class TestDataset(object):
124  """!
125  A simulated dataset consisting of a test image and an associated truth catalog.
126 
127  TestDataset creates an idealized image made of pure Gaussians (including a Gaussian PSF),
128  with simple noise and idealized Footprints/HeavyFootprints that simulated the outputs
129  of detection and deblending. Multiple noise realizations can be created from the same
130  underlying sources, allowing uncertainty estimates to be verified via Monte Carlo.
131 
132  Typical usage:
133  @code
134  bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0,0), lsst.afw.geom.Point2I(100, 100))
135  dataset = TestDataset(bbox)
136  dataset.addSource(flux=1E5, centroid=lsst.afw.geom.Point2D(25, 26))
137  dataset.addSource(flux=2E5, centroid=lsst.afw.geom.Point2D(75, 24),
138  shape=lsst.afw.geom.ellipses.Quadrupole(8, 7, 2))
139  with dataset.addBlend() as family:
140  family.addChild(flux=2E5, centroid=lsst.afw.geom.Point2D(50, 72))
141  family.addChild(flux=1.5E5, centroid=lsst.afw.geom.Point2D(51, 74))
142  exposure, catalog = dataset.realize(noise=100.0, schema=TestDataset.makeMinimalSchema())
143  @endcode
144  """
145 
146  @classmethod
148  """Return the minimal schema needed to hold truth catalog fields.
149 
150  When TestDataset.realize() is called, the schema must include at least these fields.
151  Usually it will include additional fields for measurement algorithm outputs, allowing
152  the same catalog to be used for both truth values (the fields from the minimal schema)
153  and the measurements.
154  """
155  if not hasattr(cls, "_schema"):
156  schema = lsst.afw.table.SourceTable.makeMinimalSchema()
157  cls.keys = {}
158  cls.keys["parent"] = schema.find("parent").key
159  cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
160  cls.keys["flux"] = schema.addField("truth_flux", type=np.float64, doc="true flux", units="count")
161  cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
162  schema, "truth", "true simulated centroid", "pixel"
163  )
164  cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
165  doc="set if the object is a star")
166  cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields(
167  schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
168  )
169  cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
170  doc="set if the object is a star")
171  schema.getAliasMap().set("slot_Shape", "truth")
172  schema.getAliasMap().set("slot_Centroid", "truth")
173  schema.getAliasMap().set("slot_ModelFlux", "truth")
174  schema.getCitizen().markPersistent()
175  cls._schema = schema
176  schema = lsst.afw.table.Schema(cls._schema)
177  schema.disconnectAliases()
178  return schema
179 
180  @staticmethod
181  def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
182  minRotation=None, maxRotation=None,
183  minRefShift=None, maxRefShift=None,
184  minPixShift=2.0, maxPixShift=4.0):
185  """!
186  Create a new undistorted TanWcs that is similar but not identical to another, with random
187  scaling, rotation, and offset (in both pixel position and reference position).
188 
189  The maximum and minimum arguments are interpreted as absolute values for a split
190  range that covers both positive and negative values (as this method is used
191  in testing, it is typically most important to avoid perturbations near zero).
192  Scale factors are treated somewhat differently: the actual scale factor is chosen between
193  minScaleFactor and maxScaleFactor OR (1/maxScaleFactor) and (1/minScaleFactor).
194 
195  The default range for rotation is 30-60 degrees, and the default range for reference shift
196  is 0.5-1.0 arcseconds (these cannot be safely included directly as default values because Angle
197  objects are mutable).
198  """
199  if minRotation is None:
200  minRotation = 30.0*lsst.afw.geom.degrees
201  if maxRotation is None:
202  maxRotation = 60.0*lsst.afw.geom.degrees
203  if minRefShift is None:
204  minRefShift = 0.5*lsst.afw.geom.arcseconds
205  if maxRefShift is None:
206  maxRefShift = 1.0*lsst.afw.geom.arcseconds
207 
208  def splitRandom(min1, max1, min2=None, max2=None):
209  if min2 is None:
210  min2 = -max1
211  if max2 is None:
212  max2 = -min1
213  if np.random.uniform() > 0.5:
214  return float(np.random.uniform(min1, max1))
215  else:
216  return float(np.random.uniform(min2, max2))
217  # Generate random perturbations
218  scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
219  rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.afw.geom.radians
220  refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.afw.geom.radians
221  refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.afw.geom.radians
222  pixShiftX = splitRandom(minPixShift, maxPixShift)
223  pixShiftY = splitRandom(minPixShift, maxPixShift)
224  # Compute new CD matrix
225  oldTransform = lsst.afw.geom.LinearTransform(oldWcs.getCDMatrix())
226  rTransform = lsst.afw.geom.LinearTransform.makeRotation(rotation)
227  sTransform = lsst.afw.geom.LinearTransform.makeScaling(scaleFactor)
228  newTransform = oldTransform*rTransform*sTransform
229  matrix = newTransform.getMatrix()
230  # Compute new coordinate reference pixel (CRVAL)
231  oldSkyOrigin = oldWcs.getSkyOrigin().toIcrs()
232  newSkyOrigin = lsst.afw.coord.IcrsCoord(oldSkyOrigin.getRa() + refShiftRa,
233  oldSkyOrigin.getDec() + refShiftDec)
234  # Compute new pixel reference pixel (CRPIX)
235  oldPixOrigin = oldWcs.getPixelOrigin()
236  newPixOrigin = lsst.afw.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
237  oldPixOrigin.getY() + pixShiftY)
238  return lsst.afw.image.makeWcs(newSkyOrigin, newPixOrigin,
239  matrix[0, 0], matrix[0, 1], matrix[1, 0], matrix[1, 1])
240 
241  @staticmethod
242  def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12):
243  """!
244  Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
245 
246  @param[in] bbox Bounding box of the image (image coordinates) as returned by makeCatalog.
247  @param[in] wcs New Wcs for the exposure (created from crval and cdelt if None).
248  @param[in] crval afw.coord.Coord: center of the TAN WCS attached to the image.
249  @param[in] cdelt afw.geom.Angle: pixel scale of the image
250  @param[in] psfSigma Radius (sigma) of the Gaussian PSF attached to the image
251  @param[in] psfDim Width and height of the image's Gaussian PSF attached to the image
252  @param[in] fluxMag0 Flux at magnitude zero (in e-) used to set the Calib of the exposure.
253  """
254  if wcs is None:
255  if crval is None:
256  crval = lsst.afw.coord.IcrsCoord(45.0*lsst.afw.geom.degrees, 45.0*lsst.afw.geom.degrees)
257  if cdelt is None:
258  cdelt = 0.2*lsst.afw.geom.arcseconds
259  crpix = lsst.afw.geom.Box2D(bbox).getCenter()
260  wcs = lsst.afw.image.makeWcs(crval, crpix, cdelt.asDegrees(), 0.0, 0.0, cdelt.asDegrees())
261  exposure = lsst.afw.image.ExposureF(bbox)
262  psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
263  calib = lsst.afw.image.Calib()
264  calib.setFluxMag0(fluxMag0)
265  exposure.setWcs(wcs)
266  exposure.setPsf(psf)
267  exposure.setCalib(calib)
268  return exposure
269 
270  @staticmethod
271  def drawGaussian(bbox, flux, ellipse):
272  """!
273  Create an image of an elliptical Gaussian.
274 
275  @param[in,out] bbox Bounding box of image to create.
276  @param[in] flux Total flux of the Gaussian (normalized analytically, not using pixel
277  values)
278  @param[in] ellipse lsst.afw.geom.ellipses.Ellipse holding the centroid and shape.
279  """
280  x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
281  np.arange(bbox.getBeginY(), bbox.getEndY()))
282  t = ellipse.getGridTransform()
283  xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
284  yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
285  image = lsst.afw.image.ImageF(bbox)
286  image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*flux/(2.0*ellipse.getCore().getArea())
287  return image
288 
289  def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
290  """!
291  Initialize the dataset.
292 
293  @param[in] bbox Bounding box of the test image.
294  @param[in] threshold Threshold absolute value used to determine footprints for
295  simulated sources. This thresholding will be applied before noise is
296  actually added to images (or before the noise level is even known), so
297  this will necessarily produce somewhat artificial footprints.
298  @param[in] exposure lsst.afw.image.ExposureF test sources should be added to. Ownership should
299  be considered transferred from the caller to the TestDataset.
300  Must have a Gaussian Psf for truth catalog shapes to be exact.
301  @param[in] **kwds Keyword arguments forwarded to makeEmptyExposure if exposure is None.
302  """
303  if exposure is None:
304  exposure = self.makeEmptyExposure(bbox, **kwds)
305  self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
306  self.exposure = exposure
307  self.psfShape = self.exposure.getPsf().computeShape()
308  self.schema = self.makeMinimalSchema()
309  self.catalog = lsst.afw.table.SourceCatalog(self.schema)
310 
311  def _installFootprint(self, record, image):
312  """Create a Footprint for a simulated source and add it to its truth catalog record.
313  """
314  # Run detection on the single-source image
315  fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
316  # the call below to the FootprintSet ctor is actually a grow operation
317  fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
318  # Update the full exposure's mask plane to indicate the detection
319  fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
320  # Attach the new footprint to the exposure
321  if len(fpSet.getFootprints()) > 1:
322  raise RuntimeError("Threshold value results in multiple Footprints for a single object")
323  if len(fpSet.getFootprints()) == 0:
324  raise RuntimeError("Threshold value results in zero Footprints for object")
325  record.setFootprint(fpSet.getFootprints()[0])
326 
327  def addSource(self, flux, centroid, shape=None):
328  """!
329  Add a source to the simulation
330 
331  @param[in] flux Total flux of the source to be added.
332  @param[in] centroid Position of the source to be added (lsst.afw.geom.Point2D).
333  @param[in] shape 2nd moments of the source before PSF convolution
334  (lsst.afw.geom.ellipses.Quadrupole). Note that the truth catalog
335  records post-convolution moments). If None, a point source
336  will be added.
337 
338  @return a truth catalog record and single-source image corresponding to the new source.
339  """
340  # Create and set the truth catalog fields
341  record = self.catalog.addNew()
342  record.set(self.keys["flux"], flux)
343  record.set(self.keys["centroid"], centroid)
344  if shape is None:
345  record.set(self.keys["isStar"], True)
346  fullShape = self.psfShape
347  else:
348  record.set(self.keys["isStar"], False)
349  fullShape = shape.convolve(self.psfShape)
350  record.set(self.keys["shape"], fullShape)
351  # Create an image containing just this source
352  image = self.drawGaussian(self.exposure.getBBox(), flux,
353  lsst.afw.geom.ellipses.Ellipse(fullShape, centroid))
354  # Generate a footprint for this source
355  self._installFootprint(record, image)
356  # Actually add the source to the full exposure
357  self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
358  return record, image
359 
360  def addBlend(self):
361  """!
362  Return a context manager that allows a blend of multiple sources to be added.
363 
364  Example:
365  @code
366  d = TestDataset(...)
367  with d.addBlend() as b:
368  b.addChild(flux1, centroid1)
369  b.addChild(flux2, centroid2, shape2)
370  @endcode
371 
372  Note that nothing stops you from creating overlapping sources just using the addSource() method,
373  but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
374  produced by the detection and deblending pipelines.
375  """
376  return BlendContext(self)
377 
378  def transform(self, wcs, **kwds):
379  """!
380  Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
381 
382  @param[in] wcs Wcs for the new dataset.
383  @param[in] **kwds Additional keyword arguments passed on to makeEmptyExposure. If not
384  specified, these revert to the defaults for makeEmptyExposure, not the
385  values in the current dataset.
386  """
387  bboxD = lsst.afw.geom.Box2D()
388  xyt = lsst.afw.image.XYTransformFromWcsPair(wcs, self.exposure.getWcs())
389  for corner in lsst.afw.geom.Box2D(self.exposure.getBBox()).getCorners():
390  bboxD.include(xyt.forwardTransform(lsst.afw.geom.Point2D(corner)))
391  bboxI = lsst.afw.geom.Box2I(bboxD)
392  result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
393  oldCalib = self.exposure.getCalib()
394  newCalib = result.exposure.getCalib()
395  oldPsfShape = self.exposure.getPsf().computeShape()
396  for record in self.catalog:
397  if record.get(self.keys["nChild"]):
398  raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
399  magnitude = oldCalib.getMagnitude(record.get(self.keys["flux"]))
400  newFlux = newCalib.getFlux(magnitude)
401  oldCentroid = record.get(self.keys["centroid"])
402  newCentroid = xyt.forwardTransform(oldCentroid)
403  if record.get(self.keys["isStar"]):
404  newDeconvolvedShape = None
405  else:
406  affine = xyt.linearizeForwardTransform(oldCentroid)
407  oldFullShape = record.get(self.keys["shape"])
408  oldDeconvolvedShape = lsst.afw.geom.ellipses.Quadrupole(
409  oldFullShape.getIxx() - oldPsfShape.getIxx(),
410  oldFullShape.getIyy() - oldPsfShape.getIyy(),
411  oldFullShape.getIxy() - oldPsfShape.getIxy(),
412  False
413  )
414  newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
415  result.addSource(newFlux, newCentroid, newDeconvolvedShape)
416  return result
417 
418  def realize(self, noise, schema):
419  """!
420  Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
421 
422  @param[in] noise Standard deviation of noise to be added to the exposure. The noise will be
423  Gaussian and constant, appropriate for the sky-limited regime.
424  @param[in] schema Schema of the new catalog to be created. Must start with self.schema (i.e.
425  schema.contains(self.schema) must be True), but typically contains fields for
426  already-configured measurement algorithms as well.
427 
428  @return a tuple of (exposure, catalog)
429  """
430  assert schema.contains(self.schema)
431  mapper = lsst.afw.table.SchemaMapper(self.schema)
432  mapper.addMinimalSchema(self.schema, True)
433  exposure = self.exposure.clone()
434  exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
435  exposure.getMaskedImage().getImage().getArray()[:, :] \
436  += np.random.randn(exposure.getHeight(), exposure.getWidth())*noise
437  catalog = lsst.afw.table.SourceCatalog(schema)
438  catalog.extend(self.catalog, mapper=mapper)
439  # Loop over sources and generate new HeavyFootprints that divide up the noisy pixels, not the
440  # ideal no-noise pixels.
441  for record in catalog:
442  # parent objects have non-Heavy Footprints, which don't need to be updated after adding noise.
443  if record.getParent() == 0:
444  continue
445  # get flattened arrays that correspond to the no-noise and noisy parent images
446  parent = catalog.find(record.getParent())
447  footprint = parent.getFootprint()
448  parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
449  footprint.spans.flatten(parentFluxArrayNoNoise,
450  self.exposure.getMaskedImage().getImage().getArray(),
451  self.exposure.getXY0())
452  parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
453  footprint.spans.flatten(parentFluxArrayNoisy,
454  exposure.getMaskedImage().getImage().getArray(),
455  exposure.getXY0())
456  oldHeavy = record.getFootprint()
457  fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
458  # n.b. this isn't a copy ctor - it's a copy from a vanilla Footprint, so it doesn't copy
459  # the arrays we don't want to change, and hence we have to do that ourselves below.
460  newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
461  newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
462  newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
463  newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
464  record.setFootprint(newHeavy)
465  return exposure, catalog
466 
467 
468 class AlgorithmTestCase(object):
469  # Some tests depend on the noise realization in the test data or from the
470  # np.random number generator. In most cases, they are testing that the
471  # measured flux lies within 2 sigma of the correct value, which we should
472  # expect to fail sometimes. Some -- but sadly not all -- of these cases
473  # have been marked with an "rng dependent" comment.
474  #
475  # We ensure these tests are provided with data which causes them to pass
476  # by seeding the np.RNG with this value. It can be over-ridden as
477  # necessary in subclasses.
478  randomSeed = 1234
479 
480  @classmethod
481  def setUpClass(cls):
482  np.random.seed(cls.randomSeed)
483 
484  def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
485  """Convenience function to create a Config instance for SingleFrameMeasurementTask
486 
487  The plugin and its dependencies will be the only plugins run, while the Centroid, Shape,
488  and ModelFlux slots will be set to the truth fields generated by the TestDataset class.
489  """
490  config = SingleFrameMeasurementTask.ConfigClass()
491  config.slots.centroid = "truth"
492  config.slots.shape = "truth"
493  config.slots.modelFlux = None
494  config.slots.apFlux = None
495  config.slots.psfFlux = None
496  config.slots.instFlux = None
497  config.slots.calibFlux = None
498  config.plugins.names = (plugin,) + tuple(dependencies)
499  return config
500 
501  def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
502  algMetadata=None):
503  """Convenience function to create a SingleFrameMeasurementTask with a simple configuration.
504  """
505  if config is None:
506  if plugin is None:
507  raise ValueError("Either plugin or config argument must not be None")
508  config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
509  if schema is None:
510  schema = TestDataset.makeMinimalSchema()
511  # Clear all aliases so only those defined by config are set.
512  schema.setAliasMap(None)
513  if algMetadata is None:
514  algMetadata = lsst.daf.base.PropertyList()
515  return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
516 
517  def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
518  """Convenience function to create a Config instance for ForcedMeasurementTask
519 
520  In addition to the plugins specified in the plugin and dependencies arguments,
521  the TransformedCentroid and TransformedShape plugins will be run and used as the
522  Centroid and Shape slots; these simply transform the reference catalog centroid
523  and shape to the measurement coordinate system.
524  """
525  config = ForcedMeasurementTask.ConfigClass()
526  config.slots.centroid = "base_TransformedCentroid"
527  config.slots.shape = "base_TransformedShape"
528  config.slots.modelFlux = None
529  config.slots.apFlux = None
530  config.slots.psfFlux = None
531  config.slots.instFlux = None
532  config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
533  "base_TransformedShape")
534  return config
535 
536  def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
537  algMetadata=None):
538  """Convenience function to create a ForcedMeasurementTask with a simple configuration.
539  """
540  if config is None:
541  if plugin is None:
542  raise ValueError("Either plugin or config argument must not be None")
543  config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
544  if refSchema is None:
545  refSchema = TestDataset.makeMinimalSchema()
546  if algMetadata is None:
547  algMetadata = lsst.daf.base.PropertyList()
548  return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
549 
550 
551 class TransformTestCase(object):
552  """!
553  Base class for testing measurement transformations.
554 
555  We test both that the transform itself operates successfully (fluxes are
556  converted to magnitudes, flags are propagated properly) and that the
557  transform is registered as the default for the appropriate measurement
558  algorithms.
559 
560  In the simple case of one-measurement-per-transformation, the developer
561  need not directly write any tests themselves: simply customizing the class
562  variables is all that is required. More complex measurements (e.g.
563  multiple aperture fluxes) require extra effort.
564  """
565  # The name used for the measurement algorithm; determines the names of the
566  # fields in the resulting catalog. This default should generally be fine,
567  # but subclasses can override if required.
568  name = "MeasurementTransformTest"
569 
570  # These should be customized by subclassing.
571  controlClass = None
572  algorithmClass = None
573  transformClass = None
574 
575  # Flags which may be set by the algorithm being tested. Can be customized
576  # in subclasses.
577  flagNames = ("flag",)
578 
579  # The plugin being tested should be registered under these names for
580  # single frame and forced measurement. Should be customized by
581  # subclassing.
582  singleFramePlugins = ()
583  forcedPlugins = ()
584 
585  def setUp(self):
586  bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0), lsst.afw.geom.Point2I(200, 200))
587  self.calexp = TestDataset.makeEmptyExposure(bbox)
588  self._setupTransform()
589 
590  def tearDown(self):
591  del self.calexp
592  del self.inputCat
593  del self.mapper
594  del self.transform
595  del self.outputCat
596 
597  def _populateCatalog(self, baseNames):
598  records = []
599  for flagValue in (True, False):
600  records.append(self.inputCat.addNew())
601  for baseName in baseNames:
602  for flagName in self.flagNames:
603  if records[-1].schema.join(baseName, flagName) in records[-1].schema:
604  records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
605  self._setFieldsInRecords(records, baseName)
606 
607  def _checkOutput(self, baseNames):
608  for inSrc, outSrc in zip(self.inputCat, self.outputCat):
609  for baseName in baseNames:
610  self._compareFieldsInRecords(inSrc, outSrc, baseName)
611  for flagName in self.flagNames:
612  keyName = outSrc.schema.join(baseName, flagName)
613  if keyName in inSrc.schema:
614  self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
615  else:
616  self.assertFalse(keyName in outSrc.schema)
617 
618  def _runTransform(self, doExtend=True):
619  if doExtend:
620  self.outputCat.extend(self.inputCat, mapper=self.mapper)
621  self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getCalib())
622 
623  def testTransform(self, baseNames=None):
624  """
625  Test the operation of the transformation on a catalog containing random data.
626 
627  We check that:
628 
629  * An appropriate exception is raised on an attempt to transform between catalogs with different
630  numbers of rows;
631  * Otherwise, all appropriate conversions are properly appled and that flags have been propagated.
632 
633  The `baseNames` argument requires some explanation. This should be an iterable of the leading parts of
634  the field names for each measurement; that is, everything that appears before `_flux`, `_flag`, etc.
635  In the simple case of a single measurement per plugin, this is simply equal to `self.name` (thus
636  measurements are stored as `self.name + "_flux"`, etc). More generally, the developer may specify
637  whatever iterable they require. For example, to handle multiple apertures, we could have
638  `(self.name + "_0", self.name + "_1", ...)`.
639 
640  @param[in] baseNames Iterable of the initial parts of measurement field names.
641  """
642  baseNames = baseNames or [self.name]
643  self._populateCatalog(baseNames)
644  self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
645  self._runTransform()
646  self._checkOutput(baseNames)
647 
648  def _checkRegisteredTransform(self, registry, name):
649  # If this is a Python-based transform, we can compare directly; if
650  # it's wrapped C++, we need to compare the wrapped class.
651  self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
652 
653  def testRegistration(self):
654  """
655  Test that the transformation is appropriately registered with the relevant measurement algorithms.
656  """
657  for pluginName in self.singleFramePlugins:
658  self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
659  for pluginName in self.forcedPlugins:
660  self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
661 
662 
664 
665  def _setupTransform(self):
666  self.control = self.controlClass()
667  inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema()
668  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
669  # it doesn't matter for this test since we won't actually use the plugins for anything besides
670  # defining the schema.
671  inputSchema.getAliasMap().set("slot_Centroid", "dummy")
672  inputSchema.getAliasMap().set("slot_Shape", "dummy")
673  self.algorithmClass(self.control, self.name, inputSchema)
674  inputSchema.getAliasMap().erase("slot_Centroid")
675  inputSchema.getAliasMap().erase("slot_Shape")
676  self.inputCat = lsst.afw.table.SourceCatalog(inputSchema)
677  self.mapper = lsst.afw.table.SchemaMapper(inputSchema)
678  self.transform = self.transformClass(self.control, self.name, self.mapper)
679  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
680 
681 
683 
684  def _setupTransform(self):
685  self.control = self.controlClass()
686  inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(),
687  lsst.afw.table.SourceTable.makeMinimalSchema())
688  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
689  # it doesn't matter for this test since we won't actually use the plugins for anything besides
690  # defining the schema.
691  inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
692  inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
693  self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
694  inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
695  inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
696  self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
697  self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
698  self.transform = self.transformClass(self.control, self.name, self.mapper)
699  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
700 
701 
703 
704  def _setFieldsInRecords(self, records, name):
705  for record in records:
706  record[record.schema.join(name, 'flux')] = np.random.random()
707  record[record.schema.join(name, 'fluxSigma')] = np.random.random()
708 
709  # Negative fluxes should be converted to NaNs.
710  assert len(records) > 1
711  records[0][record.schema.join(name, 'flux')] = -1
712 
713  def _compareFieldsInRecords(self, inSrc, outSrc, name):
714  fluxName, fluxSigmaName = inSrc.schema.join(name, 'flux'), inSrc.schema.join(name, 'fluxSigma')
715  if inSrc[fluxName] > 0:
716  mag, magErr = self.calexp.getCalib().getMagnitude(inSrc[fluxName], inSrc[fluxSigmaName])
717  self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag)
718  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], magErr)
719  else:
720  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
721  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
722 
723 
725 
726  def _setFieldsInRecords(self, records, name):
727  for record in records:
728  record[record.schema.join(name, 'x')] = np.random.random()
729  record[record.schema.join(name, 'y')] = np.random.random()
730  # Some algorithms set no errors; some set only sigma on x & y; some provide
731  # a full covariance matrix. Set only those which exist in the schema.
732  for fieldSuffix in ('xSigma', 'ySigma', 'x_y_Cov'):
733  fieldName = record.schema.join(name, fieldSuffix)
734  if fieldName in record.schema:
735  record[fieldName] = np.random.random()
736 
737  def _compareFieldsInRecords(self, inSrc, outSrc, name):
738  centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
739  centroidResult = centroidResultKey.get(inSrc)
740 
741  coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
742  coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
743  self.assertEqual(coordTruth, coord)
744 
745  # If the centroid has an associated uncertainty matrix, the coordinate
746  # must have one too, and vice versa.
747  try:
748  coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
749  ["ra", "dec"]).get(outSrc)
750  except lsst.pex.exceptions.NotFoundError:
751  self.assertFalse(centroidResultKey.getCentroidErr().isValid())
752  else:
753  transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.afw.geom.radians)
754  coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
755  centroidResult.getCentroidErr()),
756  transform.getLinear().getMatrix().transpose())
757  np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
def addBlend
Return a context manager that allows a blend of multiple sources to be added.
Definition: tests.py:360
def addSource
Add a source to the simulation.
Definition: tests.py:327
def addChild
Add a child source to the blend, and return the truth catalog record that corresponds to it...
Definition: tests.py:64
def drawGaussian
Create an image of an elliptical Gaussian.
Definition: tests.py:271
def makeEmptyExposure
Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
Definition: tests.py:242
def transform
Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
Definition: tests.py:378
A simulated dataset consisting of a test image and an associated truth catalog.
Definition: tests.py:123
def realize
Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
Definition: tests.py:418
def __init__
Initialize the dataset.
Definition: tests.py:289
def makePerturbedWcs
Create a new undistorted TanWcs that is similar but not identical to another, with random scaling...
Definition: tests.py:184
Base class for testing measurement transformations.
Definition: tests.py:551
A Python context manager used to add multiple overlapping sources along with a parent source that rep...
Definition: tests.py:45