lsst.meas.base  14.0-13-gd9a51ef+8
tests.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 from builtins import zip
25 from builtins import object
26 import numpy as np
27 
28 import lsst.afw.table
29 import lsst.afw.image
30 import lsst.afw.detection
31 import lsst.afw.geom
33 import lsst.afw.coord
35 
36 from .sfm import SingleFrameMeasurementTask
37 from .forcedMeasurement import ForcedMeasurementTask
38 from . import CentroidResultKey
39 
40 __all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
41  "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
42  "FluxTransformTestCase", "CentroidTransformTestCase")
43 
44 
45 class BlendContext(object):
46  """!
47  A Python context manager used to add multiple overlapping sources along with a parent source
48  that represents all of them together.
49 
50  This is used as the return value for TestDataset.addBlend(), and this is the only way it should
51  be used. The only public method is addChild().
52  """
53 
54  def __init__(self, owner):
55  self.owner = owner
56  self.parentRecord = self.owner.catalog.addNew()
57  self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
58  self.children = []
59 
60  def __enter__(self):
61  # BlendContext is its own context manager, so we just return self.
62  return self
63 
64  def addChild(self, flux, centroid, shape=None):
65  """!
66  Add a child source to the blend, and return the truth catalog record that corresponds to it.
67 
68  @param[in] flux Total flux of the source to be added.
69  @param[in] centroid Position of the source to be added (lsst.afw.geom.Point2D).
70  @param[in] shape 2nd moments of the source before PSF convolution
71  (lsst.afw.geom.ellipses.Quadrupole). Note that the truth catalog
72  records post-convolution moments)
73  """
74  record, image = self.owner.addSource(flux, centroid, shape)
75  record.set(self.owner.keys["parent"], self.parentRecord.getId())
76  self.parentImage += image
77  self.children.append((record, image))
78  return record
79 
80  def __exit__(self, type_, value, tb):
81  # We're not using the context manager for any kind of exception safety or guarantees;
82  # we just want the nice "with" statement syntax.
83  if type_ is not None: # exception was raised; just skip all this and let it propagate
84  return
85  # On exit, we need to compute and set the truth values for the parent object.
86  self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
87  # Compute flux from sum of component fluxes
88  flux = 0.0
89  for record, image in self.children:
90  flux += record.get(self.owner.keys["flux"])
91  self.parentRecord.set(self.owner.keys["flux"], flux)
92  # Compute centroid from flux-weighted mean of component centroids
93  x = 0.0
94  y = 0.0
95  for record, image in self.children:
96  w = record.get(self.owner.keys["flux"])/flux
97  x += record.get(self.owner.keys["centroid"].getX())*w
98  y += record.get(self.owner.keys["centroid"].getY())*w
99  self.parentRecord.set(self.owner.keys["centroid"], lsst.afw.geom.Point2D(x, y))
100  # Compute shape from flux-weighted mean of offset component shapes
101  xx = 0.0
102  yy = 0.0
103  xy = 0.0
104  for record, image in self.children:
105  w = record.get(self.owner.keys["flux"])/flux
106  dx = record.get(self.owner.keys["centroid"].getX()) - x
107  dy = record.get(self.owner.keys["centroid"].getY()) - y
108  xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
109  yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
110  xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
111  self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.ellipses.Quadrupole(xx, yy, xy))
112  # Run detection on the parent image to get the parent Footprint.
113  self.owner._installFootprint(self.parentRecord, self.parentImage)
114  # Create perfect HeavyFootprints for all children; these will need to be modified later to account
115  # for the noise we'll add to the image.
116  deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
117  for record, image in self.children:
118  deblend.getImage().getArray()[:, :] = image.getArray()
119  heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
120  record.setFootprint(heavyFootprint)
121 
122 
123 class TestDataset(object):
124  """!
125  A simulated dataset consisting of a test image and an associated truth catalog.
126 
127  TestDataset creates an idealized image made of pure Gaussians (including a Gaussian PSF),
128  with simple noise and idealized Footprints/HeavyFootprints that simulated the outputs
129  of detection and deblending. Multiple noise realizations can be created from the same
130  underlying sources, allowing uncertainty estimates to be verified via Monte Carlo.
131 
132  Typical usage:
133  @code
134  bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0,0), lsst.afw.geom.Point2I(100, 100))
135  dataset = TestDataset(bbox)
136  dataset.addSource(flux=1E5, centroid=lsst.afw.geom.Point2D(25, 26))
137  dataset.addSource(flux=2E5, centroid=lsst.afw.geom.Point2D(75, 24),
138  shape=lsst.afw.geom.ellipses.Quadrupole(8, 7, 2))
139  with dataset.addBlend() as family:
140  family.addChild(flux=2E5, centroid=lsst.afw.geom.Point2D(50, 72))
141  family.addChild(flux=1.5E5, centroid=lsst.afw.geom.Point2D(51, 74))
142  exposure, catalog = dataset.realize(noise=100.0, schema=TestDataset.makeMinimalSchema())
143  @endcode
144  """
145 
146  @classmethod
148  """Return the minimal schema needed to hold truth catalog fields.
149 
150  When TestDataset.realize() is called, the schema must include at least these fields.
151  Usually it will include additional fields for measurement algorithm outputs, allowing
152  the same catalog to be used for both truth values (the fields from the minimal schema)
153  and the measurements.
154  """
155  if not hasattr(cls, "_schema"):
157  cls.keys = {}
158  cls.keys["parent"] = schema.find("parent").key
159  cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
160  cls.keys["flux"] = schema.addField("truth_flux", type=np.float64, doc="true flux", units="count")
161  cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
162  schema, "truth", "true simulated centroid", "pixel"
163  )
164  cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
165  schema, "truth", ['x', 'y'], "pixel"
166  )
167  cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
168  doc="set if the object is a star")
170  schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
171  )
172  cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
173  doc="set if the object is a star")
174  schema.getAliasMap().set("slot_Shape", "truth")
175  schema.getAliasMap().set("slot_Centroid", "truth")
176  schema.getAliasMap().set("slot_ModelFlux", "truth")
177  schema.getCitizen().markPersistent()
178  cls._schema = schema
179  schema = lsst.afw.table.Schema(cls._schema)
180  schema.disconnectAliases()
181  return schema
182 
183  @staticmethod
184  def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
185  minRotation=None, maxRotation=None,
186  minRefShift=None, maxRefShift=None,
187  minPixShift=2.0, maxPixShift=4.0, randomSeed=None):
188  """!
189  Create a new undistorted TanWcs that is similar but not identical to another, with random
190  scaling, rotation, and offset (in both pixel position and reference position).
191 
192  The maximum and minimum arguments are interpreted as absolute values for a split
193  range that covers both positive and negative values (as this method is used
194  in testing, it is typically most important to avoid perturbations near zero).
195  Scale factors are treated somewhat differently: the actual scale factor is chosen between
196  minScaleFactor and maxScaleFactor OR (1/maxScaleFactor) and (1/minScaleFactor).
197 
198  The default range for rotation is 30-60 degrees, and the default range for reference shift
199  is 0.5-1.0 arcseconds (these cannot be safely included directly as default values because Angle
200  objects are mutable).
201 
202  The random number generator is primed with the seed given. If ``None``, a seed is
203  automatically chosen.
204  """
205  random_state = np.random.RandomState(randomSeed)
206  if minRotation is None:
207  minRotation = 30.0*lsst.afw.geom.degrees
208  if maxRotation is None:
209  maxRotation = 60.0*lsst.afw.geom.degrees
210  if minRefShift is None:
211  minRefShift = 0.5*lsst.afw.geom.arcseconds
212  if maxRefShift is None:
213  maxRefShift = 1.0*lsst.afw.geom.arcseconds
214 
215  def splitRandom(min1, max1, min2=None, max2=None):
216  if min2 is None:
217  min2 = -max1
218  if max2 is None:
219  max2 = -min1
220  if random_state.uniform() > 0.5:
221  return float(random_state.uniform(min1, max1))
222  else:
223  return float(random_state.uniform(min2, max2))
224  # Generate random perturbations
225  scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
226  rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.afw.geom.radians
227  refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.afw.geom.radians
228  refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.afw.geom.radians
229  pixShiftX = splitRandom(minPixShift, maxPixShift)
230  pixShiftY = splitRandom(minPixShift, maxPixShift)
231  # Compute new CD matrix
232  oldTransform = lsst.afw.geom.LinearTransform(oldWcs.getCDMatrix())
233  rTransform = lsst.afw.geom.LinearTransform.makeRotation(rotation)
234  sTransform = lsst.afw.geom.LinearTransform.makeScaling(scaleFactor)
235  newTransform = oldTransform*rTransform*sTransform
236  matrix = newTransform.getMatrix()
237  # Compute new coordinate reference pixel (CRVAL)
238  oldSkyOrigin = oldWcs.getSkyOrigin().toIcrs()
239  newSkyOrigin = lsst.afw.coord.IcrsCoord(oldSkyOrigin.getRa() + refShiftRa,
240  oldSkyOrigin.getDec() + refShiftDec)
241  # Compute new pixel reference pixel (CRPIX)
242  oldPixOrigin = oldWcs.getPixelOrigin()
243  newPixOrigin = lsst.afw.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
244  oldPixOrigin.getY() + pixShiftY)
245  return lsst.afw.image.makeWcs(newSkyOrigin, newPixOrigin,
246  matrix[0, 0], matrix[0, 1], matrix[1, 0], matrix[1, 1])
247 
248  @staticmethod
249  def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12):
250  """!
251  Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
252 
253  @param[in] bbox Bounding box of the image (image coordinates) as returned by makeCatalog.
254  @param[in] wcs New Wcs for the exposure (created from crval and cdelt if None).
255  @param[in] crval afw.coord.Coord: center of the TAN WCS attached to the image.
256  @param[in] cdelt afw.geom.Angle: pixel scale of the image
257  @param[in] psfSigma Radius (sigma) of the Gaussian PSF attached to the image
258  @param[in] psfDim Width and height of the image's Gaussian PSF attached to the image
259  @param[in] fluxMag0 Flux at magnitude zero (in e-) used to set the Calib of the exposure.
260  """
261  if wcs is None:
262  if crval is None:
263  crval = lsst.afw.coord.IcrsCoord(45.0*lsst.afw.geom.degrees, 45.0*lsst.afw.geom.degrees)
264  if cdelt is None:
265  cdelt = 0.2*lsst.afw.geom.arcseconds
266  crpix = lsst.afw.geom.Box2D(bbox).getCenter()
267  wcs = lsst.afw.image.makeWcs(crval, crpix, cdelt.asDegrees(), 0.0, 0.0, cdelt.asDegrees())
268  exposure = lsst.afw.image.ExposureF(bbox)
269  psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
270  calib = lsst.afw.image.Calib()
271  calib.setFluxMag0(fluxMag0)
272  exposure.setWcs(wcs)
273  exposure.setPsf(psf)
274  exposure.setCalib(calib)
275  return exposure
276 
277  @staticmethod
278  def drawGaussian(bbox, flux, ellipse):
279  """!
280  Create an image of an elliptical Gaussian.
281 
282  @param[in,out] bbox Bounding box of image to create.
283  @param[in] flux Total flux of the Gaussian (normalized analytically, not using pixel
284  values)
285  @param[in] ellipse lsst.afw.geom.ellipses.Ellipse holding the centroid and shape.
286  """
287  x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
288  np.arange(bbox.getBeginY(), bbox.getEndY()))
289  t = ellipse.getGridTransform()
290  xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
291  yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
292  image = lsst.afw.image.ImageF(bbox)
293  image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*flux/(2.0*ellipse.getCore().getArea())
294  return image
295 
296  def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
297  """!
298  Initialize the dataset.
299 
300  @param[in] bbox Bounding box of the test image.
301  @param[in] threshold Threshold absolute value used to determine footprints for
302  simulated sources. This thresholding will be applied before noise is
303  actually added to images (or before the noise level is even known), so
304  this will necessarily produce somewhat artificial footprints.
305  @param[in] exposure lsst.afw.image.ExposureF test sources should be added to. Ownership should
306  be considered transferred from the caller to the TestDataset.
307  Must have a Gaussian Psf for truth catalog shapes to be exact.
308  @param[in] **kwds Keyword arguments forwarded to makeEmptyExposure if exposure is None.
309  """
310  if exposure is None:
311  exposure = self.makeEmptyExposure(bbox, **kwds)
312  self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
313  self.exposure = exposure
314  self.psfShape = self.exposure.getPsf().computeShape()
315  self.schema = self.makeMinimalSchema()
317 
318  def _installFootprint(self, record, image):
319  """Create a Footprint for a simulated source and add it to its truth catalog record.
320  """
321  # Run detection on the single-source image
322  fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
323  # the call below to the FootprintSet ctor is actually a grow operation
324  fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
325  # Update the full exposure's mask plane to indicate the detection
326  fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
327  # Attach the new footprint to the exposure
328  if len(fpSet.getFootprints()) > 1:
329  raise RuntimeError("Threshold value results in multiple Footprints for a single object")
330  if len(fpSet.getFootprints()) == 0:
331  raise RuntimeError("Threshold value results in zero Footprints for object")
332  record.setFootprint(fpSet.getFootprints()[0])
333 
334  def addSource(self, flux, centroid, shape=None):
335  """!
336  Add a source to the simulation
337 
338  @param[in] flux Total flux of the source to be added.
339  @param[in] centroid Position of the source to be added (lsst.afw.geom.Point2D).
340  @param[in] shape 2nd moments of the source before PSF convolution
341  (lsst.afw.geom.ellipses.Quadrupole). Note that the truth catalog
342  records post-convolution moments). If None, a point source
343  will be added.
344 
345  @return a truth catalog record and single-source image corresponding to the new source.
346  """
347  # Create and set the truth catalog fields
348  record = self.catalog.addNew()
349  record.set(self.keys["flux"], flux)
350  record.set(self.keys["centroid"], centroid)
351  covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
352  covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
353  record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
354  if shape is None:
355  record.set(self.keys["isStar"], True)
356  fullShape = self.psfShape
357  else:
358  record.set(self.keys["isStar"], False)
359  fullShape = shape.convolve(self.psfShape)
360  record.set(self.keys["shape"], fullShape)
361  # Create an image containing just this source
362  image = self.drawGaussian(self.exposure.getBBox(), flux,
363  lsst.afw.geom.ellipses.Ellipse(fullShape, centroid))
364  # Generate a footprint for this source
365  self._installFootprint(record, image)
366  # Actually add the source to the full exposure
367  self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
368  return record, image
369 
370  def addBlend(self):
371  """!
372  Return a context manager that allows a blend of multiple sources to be added.
373 
374  Example:
375  @code
376  d = TestDataset(...)
377  with d.addBlend() as b:
378  b.addChild(flux1, centroid1)
379  b.addChild(flux2, centroid2, shape2)
380  @endcode
381 
382  Note that nothing stops you from creating overlapping sources just using the addSource() method,
383  but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
384  produced by the detection and deblending pipelines.
385  """
386  return BlendContext(self)
387 
388  def transform(self, wcs, **kwds):
389  """!
390  Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
391 
392  @param[in] wcs Wcs for the new dataset.
393  @param[in] **kwds Additional keyword arguments passed on to makeEmptyExposure. If not
394  specified, these revert to the defaults for makeEmptyExposure, not the
395  values in the current dataset.
396  """
397  bboxD = lsst.afw.geom.Box2D()
398  xyt = lsst.afw.image.XYTransformFromWcsPair(wcs, self.exposure.getWcs())
399  for corner in lsst.afw.geom.Box2D(self.exposure.getBBox()).getCorners():
400  bboxD.include(xyt.forwardTransform(lsst.afw.geom.Point2D(corner)))
401  bboxI = lsst.afw.geom.Box2I(bboxD)
402  result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
403  oldCalib = self.exposure.getCalib()
404  newCalib = result.exposure.getCalib()
405  oldPsfShape = self.exposure.getPsf().computeShape()
406  for record in self.catalog:
407  if record.get(self.keys["nChild"]):
408  raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
409  magnitude = oldCalib.getMagnitude(record.get(self.keys["flux"]))
410  newFlux = newCalib.getFlux(magnitude)
411  oldCentroid = record.get(self.keys["centroid"])
412  newCentroid = xyt.forwardTransform(oldCentroid)
413  if record.get(self.keys["isStar"]):
414  newDeconvolvedShape = None
415  else:
416  affine = xyt.linearizeForwardTransform(oldCentroid)
417  oldFullShape = record.get(self.keys["shape"])
418  oldDeconvolvedShape = lsst.afw.geom.ellipses.Quadrupole(
419  oldFullShape.getIxx() - oldPsfShape.getIxx(),
420  oldFullShape.getIyy() - oldPsfShape.getIyy(),
421  oldFullShape.getIxy() - oldPsfShape.getIxy(),
422  False
423  )
424  newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
425  result.addSource(newFlux, newCentroid, newDeconvolvedShape)
426  return result
427 
428  def realize(self, noise, schema, randomSeed=None):
429  """!
430  Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
431 
432  @param[in] noise Standard deviation of noise to be added to the exposure. The noise will be
433  Gaussian and constant, appropriate for the sky-limited regime.
434  @param[in] schema Schema of the new catalog to be created. Must start with self.schema (i.e.
435  schema.contains(self.schema) must be True), but typically contains fields for
436  already-configured measurement algorithms as well.
437  @param[in] randomSeed Seed for the random number generator. If None, a seed is chosen automatically.
438 
439  @return a tuple of (exposure, catalog)
440  """
441  random_state = np.random.RandomState(randomSeed)
442  assert schema.contains(self.schema)
443  mapper = lsst.afw.table.SchemaMapper(self.schema)
444  mapper.addMinimalSchema(self.schema, True)
445  exposure = self.exposure.clone()
446  exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
447  exposure.getMaskedImage().getImage().getArray()[:, :] \
448  += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise
449  catalog = lsst.afw.table.SourceCatalog(schema)
450  catalog.extend(self.catalog, mapper=mapper)
451  # Loop over sources and generate new HeavyFootprints that divide up the noisy pixels, not the
452  # ideal no-noise pixels.
453  for record in catalog:
454  # parent objects have non-Heavy Footprints, which don't need to be updated after adding noise.
455  if record.getParent() == 0:
456  continue
457  # get flattened arrays that correspond to the no-noise and noisy parent images
458  parent = catalog.find(record.getParent())
459  footprint = parent.getFootprint()
460  parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
461  footprint.spans.flatten(parentFluxArrayNoNoise,
462  self.exposure.getMaskedImage().getImage().getArray(),
463  self.exposure.getXY0())
464  parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
465  footprint.spans.flatten(parentFluxArrayNoisy,
466  exposure.getMaskedImage().getImage().getArray(),
467  exposure.getXY0())
468  oldHeavy = record.getFootprint()
469  fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
470  # n.b. this isn't a copy ctor - it's a copy from a vanilla Footprint, so it doesn't copy
471  # the arrays we don't want to change, and hence we have to do that ourselves below.
472  newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
473  newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
474  newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
475  newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
476  record.setFootprint(newHeavy)
477  return exposure, catalog
478 
479 
480 class AlgorithmTestCase(object):
481 
482  def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
483  """Convenience function to create a Config instance for SingleFrameMeasurementTask
484 
485  The plugin and its dependencies will be the only plugins run, while the Centroid, Shape,
486  and ModelFlux slots will be set to the truth fields generated by the TestDataset class.
487  """
488  config = SingleFrameMeasurementTask.ConfigClass()
489  config.slots.centroid = "truth"
490  config.slots.shape = "truth"
491  config.slots.modelFlux = None
492  config.slots.apFlux = None
493  config.slots.psfFlux = None
494  config.slots.instFlux = None
495  config.slots.calibFlux = None
496  config.plugins.names = (plugin,) + tuple(dependencies)
497  return config
498 
499  def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
500  algMetadata=None):
501  """Convenience function to create a SingleFrameMeasurementTask with a simple configuration.
502  """
503  if config is None:
504  if plugin is None:
505  raise ValueError("Either plugin or config argument must not be None")
506  config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
507  if schema is None:
508  schema = TestDataset.makeMinimalSchema()
509  # Clear all aliases so only those defined by config are set.
510  schema.setAliasMap(None)
511  if algMetadata is None:
512  algMetadata = lsst.daf.base.PropertyList()
513  return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
514 
515  def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
516  """Convenience function to create a Config instance for ForcedMeasurementTask
517 
518  In addition to the plugins specified in the plugin and dependencies arguments,
519  the TransformedCentroid and TransformedShape plugins will be run and used as the
520  Centroid and Shape slots; these simply transform the reference catalog centroid
521  and shape to the measurement coordinate system.
522  """
523  config = ForcedMeasurementTask.ConfigClass()
524  config.slots.centroid = "base_TransformedCentroid"
525  config.slots.shape = "base_TransformedShape"
526  config.slots.modelFlux = None
527  config.slots.apFlux = None
528  config.slots.psfFlux = None
529  config.slots.instFlux = None
530  config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
531  "base_TransformedShape")
532  return config
533 
534  def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
535  algMetadata=None):
536  """Convenience function to create a ForcedMeasurementTask with a simple configuration.
537  """
538  if config is None:
539  if plugin is None:
540  raise ValueError("Either plugin or config argument must not be None")
541  config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
542  if refSchema is None:
543  refSchema = TestDataset.makeMinimalSchema()
544  if algMetadata is None:
545  algMetadata = lsst.daf.base.PropertyList()
546  return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
547 
548 
549 class TransformTestCase(object):
550  """!
551  Base class for testing measurement transformations.
552 
553  We test both that the transform itself operates successfully (fluxes are
554  converted to magnitudes, flags are propagated properly) and that the
555  transform is registered as the default for the appropriate measurement
556  algorithms.
557 
558  In the simple case of one-measurement-per-transformation, the developer
559  need not directly write any tests themselves: simply customizing the class
560  variables is all that is required. More complex measurements (e.g.
561  multiple aperture fluxes) require extra effort.
562  """
563  # The name used for the measurement algorithm; determines the names of the
564  # fields in the resulting catalog. This default should generally be fine,
565  # but subclasses can override if required.
566  name = "MeasurementTransformTest"
567 
568  # These should be customized by subclassing.
569  controlClass = None
570  algorithmClass = None
571  transformClass = None
572 
573  # Flags which may be set by the algorithm being tested. Can be customized
574  # in subclasses.
575  flagNames = ("flag",)
576 
577  # The plugin being tested should be registered under these names for
578  # single frame and forced measurement. Should be customized by
579  # subclassing.
580  singleFramePlugins = ()
581  forcedPlugins = ()
582 
583  def setUp(self):
585  self.calexp = TestDataset.makeEmptyExposure(bbox)
586  self._setupTransform()
587 
588  def tearDown(self):
589  del self.calexp
590  del self.inputCat
591  del self.mapper
592  del self.transform
593  del self.outputCat
594 
595  def _populateCatalog(self, baseNames):
596  records = []
597  for flagValue in (True, False):
598  records.append(self.inputCat.addNew())
599  for baseName in baseNames:
600  for flagName in self.flagNames:
601  if records[-1].schema.join(baseName, flagName) in records[-1].schema:
602  records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
603  self._setFieldsInRecords(records, baseName)
604 
605  def _checkOutput(self, baseNames):
606  for inSrc, outSrc in zip(self.inputCat, self.outputCat):
607  for baseName in baseNames:
608  self._compareFieldsInRecords(inSrc, outSrc, baseName)
609  for flagName in self.flagNames:
610  keyName = outSrc.schema.join(baseName, flagName)
611  if keyName in inSrc.schema:
612  self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
613  else:
614  self.assertFalse(keyName in outSrc.schema)
615 
616  def _runTransform(self, doExtend=True):
617  if doExtend:
618  self.outputCat.extend(self.inputCat, mapper=self.mapper)
619  self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getCalib())
620 
621  def testTransform(self, baseNames=None):
622  """
623  Test the operation of the transformation on a catalog containing random data.
624 
625  We check that:
626 
627  * An appropriate exception is raised on an attempt to transform between catalogs with different
628  numbers of rows;
629  * Otherwise, all appropriate conversions are properly appled and that flags have been propagated.
630 
631  The `baseNames` argument requires some explanation. This should be an iterable of the leading parts of
632  the field names for each measurement; that is, everything that appears before `_flux`, `_flag`, etc.
633  In the simple case of a single measurement per plugin, this is simply equal to `self.name` (thus
634  measurements are stored as `self.name + "_flux"`, etc). More generally, the developer may specify
635  whatever iterable they require. For example, to handle multiple apertures, we could have
636  `(self.name + "_0", self.name + "_1", ...)`.
637 
638  @param[in] baseNames Iterable of the initial parts of measurement field names.
639  """
640  baseNames = baseNames or [self.name]
641  self._populateCatalog(baseNames)
642  self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
643  self._runTransform()
644  self._checkOutput(baseNames)
645 
646  def _checkRegisteredTransform(self, registry, name):
647  # If this is a Python-based transform, we can compare directly; if
648  # it's wrapped C++, we need to compare the wrapped class.
649  self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
650 
651  def testRegistration(self):
652  """
653  Test that the transformation is appropriately registered with the relevant measurement algorithms.
654  """
655  for pluginName in self.singleFramePlugins:
656  self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
657  for pluginName in self.forcedPlugins:
658  self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
659 
660 
662 
663  def _setupTransform(self):
664  self.control = self.controlClass()
666  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
667  # it doesn't matter for this test since we won't actually use the plugins for anything besides
668  # defining the schema.
669  inputSchema.getAliasMap().set("slot_Centroid", "dummy")
670  inputSchema.getAliasMap().set("slot_Shape", "dummy")
671  self.algorithmClass(self.control, self.name, inputSchema)
672  inputSchema.getAliasMap().erase("slot_Centroid")
673  inputSchema.getAliasMap().erase("slot_Shape")
676  self.transform = self.transformClass(self.control, self.name, self.mapper)
677  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
678 
679 
681 
682  def _setupTransform(self):
683  self.control = self.controlClass()
686  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
687  # it doesn't matter for this test since we won't actually use the plugins for anything besides
688  # defining the schema.
689  inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
690  inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
691  self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
692  inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
693  inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
694  self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
695  self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
696  self.transform = self.transformClass(self.control, self.name, self.mapper)
697  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
698 
699 
701 
702  def _setFieldsInRecords(self, records, name):
703  for record in records:
704  record[record.schema.join(name, 'flux')] = np.random.random()
705  record[record.schema.join(name, 'fluxSigma')] = np.random.random()
706 
707  # Negative fluxes should be converted to NaNs.
708  assert len(records) > 1
709  records[0][record.schema.join(name, 'flux')] = -1
710 
711  def _compareFieldsInRecords(self, inSrc, outSrc, name):
712  fluxName, fluxSigmaName = inSrc.schema.join(name, 'flux'), inSrc.schema.join(name, 'fluxSigma')
713  if inSrc[fluxName] > 0:
714  mag, magErr = self.calexp.getCalib().getMagnitude(inSrc[fluxName], inSrc[fluxSigmaName])
715  self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag)
716  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], magErr)
717  else:
718  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
719  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
720 
721 
723 
724  def _setFieldsInRecords(self, records, name):
725  for record in records:
726  record[record.schema.join(name, 'x')] = np.random.random()
727  record[record.schema.join(name, 'y')] = np.random.random()
728  # Some algorithms set no errors; some set only sigma on x & y; some provide
729  # a full covariance matrix. Set only those which exist in the schema.
730  for fieldSuffix in ('xSigma', 'ySigma', 'x_y_Cov'):
731  fieldName = record.schema.join(name, fieldSuffix)
732  if fieldName in record.schema:
733  record[fieldName] = np.random.random()
734 
735  def _compareFieldsInRecords(self, inSrc, outSrc, name):
736  centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
737  centroidResult = centroidResultKey.get(inSrc)
738 
739  coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
740  coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
741  self.assertEqual(coordTruth, coord)
742 
743  # If the centroid has an associated uncertainty matrix, the coordinate
744  # must have one too, and vice versa.
745  try:
746  coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
747  ["ra", "dec"]).get(outSrc)
749  self.assertFalse(centroidResultKey.getCentroidErr().isValid())
750  else:
751  transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.afw.geom.radians)
752  coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
753  centroidResult.getCentroidErr()),
754  transform.getLinear().getMatrix().transpose())
755  np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
def _runTransform(self, doExtend=True)
Definition: tests.py:616
def realize(self, noise, schema, randomSeed=None)
Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
Definition: tests.py:428
def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:482
def transform(self, wcs, kwds)
Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
Definition: tests.py:388
def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=None)
Create a new undistorted TanWcs that is similar but not identical to another, with random scaling...
Definition: tests.py:187
std::shared_ptr< Wcs > makeWcs(coord::Coord const &crval, geom::Point2D const &crpix, double CD11, double CD12, double CD21, double CD22)
def __init__(self, bbox, threshold=10.0, exposure=None, kwds)
Initialize the dataset.
Definition: tests.py:296
def _checkOutput(self, baseNames)
Definition: tests.py:605
A subtask for measuring the properties of sources on a single exposure, using an existing "reference"...
A subtask for measuring the properties of sources on a single exposure.
Definition: sfm.py:152
def addBlend(self)
Return a context manager that allows a blend of multiple sources to be added.
Definition: tests.py:370
def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12)
Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
Definition: tests.py:249
static LinearTransform makeScaling(double s)
def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition: tests.py:500
def __init__(self, owner)
Definition: tests.py:54
def testTransform(self, baseNames=None)
Definition: tests.py:621
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
def drawGaussian(bbox, flux, ellipse)
Create an image of an elliptical Gaussian.
Definition: tests.py:278
static LinearTransform makeRotation(Angle t)
def _installFootprint(self, record, image)
Definition: tests.py:318
def _checkRegisteredTransform(self, registry, name)
Definition: tests.py:646
def _populateCatalog(self, baseNames)
Definition: tests.py:595
def addSource(self, flux, centroid, shape=None)
Add a source to the simulation.
Definition: tests.py:334
static Schema makeMinimalSchema()
A simulated dataset consisting of a test image and an associated truth catalog.
Definition: tests.py:123
def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition: tests.py:535
def addChild(self, flux, centroid, shape=None)
Add a child source to the blend, and return the truth catalog record that corresponds to it...
Definition: tests.py:64
Base class for testing measurement transformations.
Definition: tests.py:549
def __exit__(self, type_, value, tb)
Definition: tests.py:80
def makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:515
A Python context manager used to add multiple overlapping sources along with a parent source that rep...
Definition: tests.py:45