lsst.meas.base  14.0-21-gb9e430a+5
tests.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 from builtins import zip
25 from builtins import object
26 import numpy as np
27 
28 import lsst.afw.table
29 import lsst.afw.image
30 import lsst.afw.detection
31 import lsst.afw.geom
32 import lsst.afw.coord
34 
35 from .sfm import SingleFrameMeasurementTask
36 from .forcedMeasurement import ForcedMeasurementTask
37 from . import CentroidResultKey
38 
39 __all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
40  "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
41  "FluxTransformTestCase", "CentroidTransformTestCase")
42 
43 
44 class BlendContext(object):
45  """!
46  A Python context manager used to add multiple overlapping sources along with a parent source
47  that represents all of them together.
48 
49  This is used as the return value for TestDataset.addBlend(), and this is the only way it should
50  be used. The only public method is addChild().
51  """
52 
53  def __init__(self, owner):
54  self.owner = owner
55  self.parentRecord = self.owner.catalog.addNew()
56  self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
57  self.children = []
58 
59  def __enter__(self):
60  # BlendContext is its own context manager, so we just return self.
61  return self
62 
63  def addChild(self, flux, centroid, shape=None):
64  """!
65  Add a child source to the blend, and return the truth catalog record that corresponds to it.
66 
67  @param[in] flux Total flux of the source to be added.
68  @param[in] centroid Position of the source to be added (lsst.afw.geom.Point2D).
69  @param[in] shape 2nd moments of the source before PSF convolution
70  (lsst.afw.geom.Quadrupole). Note that the truth catalog
71  records post-convolution moments)
72  """
73  record, image = self.owner.addSource(flux, centroid, shape)
74  record.set(self.owner.keys["parent"], self.parentRecord.getId())
75  self.parentImage += image
76  self.children.append((record, image))
77  return record
78 
79  def __exit__(self, type_, value, tb):
80  # We're not using the context manager for any kind of exception safety or guarantees;
81  # we just want the nice "with" statement syntax.
82  if type_ is not None: # exception was raised; just skip all this and let it propagate
83  return
84  # On exit, we need to compute and set the truth values for the parent object.
85  self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
86  # Compute flux from sum of component fluxes
87  flux = 0.0
88  for record, image in self.children:
89  flux += record.get(self.owner.keys["flux"])
90  self.parentRecord.set(self.owner.keys["flux"], flux)
91  # Compute centroid from flux-weighted mean of component centroids
92  x = 0.0
93  y = 0.0
94  for record, image in self.children:
95  w = record.get(self.owner.keys["flux"])/flux
96  x += record.get(self.owner.keys["centroid"].getX())*w
97  y += record.get(self.owner.keys["centroid"].getY())*w
98  self.parentRecord.set(self.owner.keys["centroid"], lsst.afw.geom.Point2D(x, y))
99  # Compute shape from flux-weighted mean of offset component shapes
100  xx = 0.0
101  yy = 0.0
102  xy = 0.0
103  for record, image in self.children:
104  w = record.get(self.owner.keys["flux"])/flux
105  dx = record.get(self.owner.keys["centroid"].getX()) - x
106  dy = record.get(self.owner.keys["centroid"].getY()) - y
107  xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
108  yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
109  xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
110  self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
111  # Run detection on the parent image to get the parent Footprint.
112  self.owner._installFootprint(self.parentRecord, self.parentImage)
113  # Create perfect HeavyFootprints for all children; these will need to be modified later to account
114  # for the noise we'll add to the image.
115  deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
116  for record, image in self.children:
117  deblend.getImage().getArray()[:, :] = image.getArray()
118  heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
119  record.setFootprint(heavyFootprint)
120 
121 
122 class TestDataset(object):
123  """!
124  A simulated dataset consisting of a test image and an associated truth catalog.
125 
126  TestDataset creates an idealized image made of pure Gaussians (including a Gaussian PSF),
127  with simple noise and idealized Footprints/HeavyFootprints that simulated the outputs
128  of detection and deblending. Multiple noise realizations can be created from the same
129  underlying sources, allowing uncertainty estimates to be verified via Monte Carlo.
130 
131  Typical usage:
132  @code
133  bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0,0), lsst.afw.geom.Point2I(100, 100))
134  dataset = TestDataset(bbox)
135  dataset.addSource(flux=1E5, centroid=lsst.afw.geom.Point2D(25, 26))
136  dataset.addSource(flux=2E5, centroid=lsst.afw.geom.Point2D(75, 24),
137  shape=lsst.afw.geom.Quadrupole(8, 7, 2))
138  with dataset.addBlend() as family:
139  family.addChild(flux=2E5, centroid=lsst.afw.geom.Point2D(50, 72))
140  family.addChild(flux=1.5E5, centroid=lsst.afw.geom.Point2D(51, 74))
141  exposure, catalog = dataset.realize(noise=100.0, schema=TestDataset.makeMinimalSchema())
142  @endcode
143  """
144 
145  @classmethod
147  """Return the minimal schema needed to hold truth catalog fields.
148 
149  When TestDataset.realize() is called, the schema must include at least these fields.
150  Usually it will include additional fields for measurement algorithm outputs, allowing
151  the same catalog to be used for both truth values (the fields from the minimal schema)
152  and the measurements.
153  """
154  if not hasattr(cls, "_schema"):
156  cls.keys = {}
157  cls.keys["parent"] = schema.find("parent").key
158  cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
159  cls.keys["flux"] = schema.addField("truth_flux", type=np.float64, doc="true flux", units="count")
160  cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
161  schema, "truth", "true simulated centroid", "pixel"
162  )
163  cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
164  schema, "truth", ['x', 'y'], "pixel"
165  )
166  cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
167  doc="set if the object is a star")
169  schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
170  )
171  cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
172  doc="set if the object is a star")
173  schema.getAliasMap().set("slot_Shape", "truth")
174  schema.getAliasMap().set("slot_Centroid", "truth")
175  schema.getAliasMap().set("slot_ModelFlux", "truth")
176  schema.getCitizen().markPersistent()
177  cls._schema = schema
178  schema = lsst.afw.table.Schema(cls._schema)
179  schema.disconnectAliases()
180  return schema
181 
182  @staticmethod
183  def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
184  minRotation=None, maxRotation=None,
185  minRefShift=None, maxRefShift=None,
186  minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
187  """!
188  Create a new undistorted TAN WCS that is similar but not identical to another, with random
189  scaling, rotation, and offset (in both pixel position and reference position).
190 
191  The maximum and minimum arguments are interpreted as absolute values for a split
192  range that covers both positive and negative values (as this method is used
193  in testing, it is typically most important to avoid perturbations near zero).
194  Scale factors are treated somewhat differently: the actual scale factor is chosen between
195  minScaleFactor and maxScaleFactor OR (1/maxScaleFactor) and (1/minScaleFactor).
196 
197  The default range for rotation is 30-60 degrees, and the default range for reference shift
198  is 0.5-1.0 arcseconds (these cannot be safely included directly as default values because Angle
199  objects are mutable).
200 
201  The random number generator is primed with the seed given. If ``None``, a seed is
202  automatically chosen.
203  """
204  random_state = np.random.RandomState(randomSeed)
205  if minRotation is None:
206  minRotation = 30.0*lsst.afw.geom.degrees
207  if maxRotation is None:
208  maxRotation = 60.0*lsst.afw.geom.degrees
209  if minRefShift is None:
210  minRefShift = 0.5*lsst.afw.geom.arcseconds
211  if maxRefShift is None:
212  maxRefShift = 1.0*lsst.afw.geom.arcseconds
213 
214  def splitRandom(min1, max1, min2=None, max2=None):
215  if min2 is None:
216  min2 = -max1
217  if max2 is None:
218  max2 = -min1
219  if random_state.uniform() > 0.5:
220  return float(random_state.uniform(min1, max1))
221  else:
222  return float(random_state.uniform(min2, max2))
223  # Generate random perturbations
224  scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
225  rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.afw.geom.radians
226  refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.afw.geom.radians
227  refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.afw.geom.radians
228  pixShiftX = splitRandom(minPixShift, maxPixShift)
229  pixShiftY = splitRandom(minPixShift, maxPixShift)
230  # Compute new CD matrix
231  oldTransform = lsst.afw.geom.LinearTransform(oldWcs.getCdMatrix())
232  rTransform = lsst.afw.geom.LinearTransform.makeRotation(rotation)
233  sTransform = lsst.afw.geom.LinearTransform.makeScaling(scaleFactor)
234  newTransform = oldTransform*rTransform*sTransform
235  matrix = newTransform.getMatrix()
236  # Compute new coordinate reference pixel (CRVAL)
237  oldSkyOrigin = oldWcs.getSkyOrigin()
238  newSkyOrigin = lsst.afw.coord.IcrsCoord(oldSkyOrigin.getRa() + refShiftRa,
239  oldSkyOrigin.getDec() + refShiftDec)
240  # Compute new pixel reference pixel (CRPIX)
241  oldPixOrigin = oldWcs.getPixelOrigin()
242  newPixOrigin = lsst.afw.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
243  oldPixOrigin.getY() + pixShiftY)
244  return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
245 
246  @staticmethod
247  def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12):
248  """!
249  Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
250 
251  @param[in] bbox Bounding box of the image (image coordinates) as returned by makeCatalog.
252  @param[in] wcs New Wcs for the exposure (created from crval and cdelt if None).
253  @param[in] crval afw.coord.IcrsCoord: center of the TAN WCS attached to the image.
254  @param[in] cdelt afw.geom.Angle: pixel scale of the image
255  @param[in] psfSigma Radius (sigma) of the Gaussian PSF attached to the image
256  @param[in] psfDim Width and height of the image's Gaussian PSF attached to the image
257  @param[in] fluxMag0 Flux at magnitude zero (in e-) used to set the Calib of the exposure.
258  """
259  if wcs is None:
260  if crval is None:
261  crval = lsst.afw.coord.IcrsCoord(45.0*lsst.afw.geom.degrees, 45.0*lsst.afw.geom.degrees)
262  if cdelt is None:
263  cdelt = 0.2*lsst.afw.geom.arcseconds
264  crpix = lsst.afw.geom.Box2D(bbox).getCenter()
265  wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
266  cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
267  exposure = lsst.afw.image.ExposureF(bbox)
268  psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
269  calib = lsst.afw.image.Calib()
270  calib.setFluxMag0(fluxMag0)
271  exposure.setWcs(wcs)
272  exposure.setPsf(psf)
273  exposure.setCalib(calib)
274  return exposure
275 
276  @staticmethod
277  def drawGaussian(bbox, flux, ellipse):
278  """!
279  Create an image of an elliptical Gaussian.
280 
281  @param[in,out] bbox Bounding box of image to create.
282  @param[in] flux Total flux of the Gaussian (normalized analytically, not using pixel
283  values)
284  @param[in] ellipse lsst.afw.geom.Ellipse holding the centroid and shape.
285  """
286  x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
287  np.arange(bbox.getBeginY(), bbox.getEndY()))
288  t = ellipse.getGridTransform()
289  xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
290  yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
291  image = lsst.afw.image.ImageF(bbox)
292  image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*flux/(2.0*ellipse.getCore().getArea())
293  return image
294 
295  def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
296  """!
297  Initialize the dataset.
298 
299  @param[in] bbox Bounding box of the test image.
300  @param[in] threshold Threshold absolute value used to determine footprints for
301  simulated sources. This thresholding will be applied before noise is
302  actually added to images (or before the noise level is even known), so
303  this will necessarily produce somewhat artificial footprints.
304  @param[in] exposure lsst.afw.image.ExposureF test sources should be added to. Ownership should
305  be considered transferred from the caller to the TestDataset.
306  Must have a Gaussian Psf for truth catalog shapes to be exact.
307  @param[in] **kwds Keyword arguments forwarded to makeEmptyExposure if exposure is None.
308  """
309  if exposure is None:
310  exposure = self.makeEmptyExposure(bbox, **kwds)
311  self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
312  self.exposure = exposure
313  self.psfShape = self.exposure.getPsf().computeShape()
314  self.schema = self.makeMinimalSchema()
316 
317  def _installFootprint(self, record, image):
318  """Create a Footprint for a simulated source and add it to its truth catalog record.
319  """
320  # Run detection on the single-source image
321  fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
322  # the call below to the FootprintSet ctor is actually a grow operation
323  fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
324  # Update the full exposure's mask plane to indicate the detection
325  fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
326  # Attach the new footprint to the exposure
327  if len(fpSet.getFootprints()) > 1:
328  raise RuntimeError("Threshold value results in multiple Footprints for a single object")
329  if len(fpSet.getFootprints()) == 0:
330  raise RuntimeError("Threshold value results in zero Footprints for object")
331  record.setFootprint(fpSet.getFootprints()[0])
332 
333  def addSource(self, flux, centroid, shape=None):
334  """!
335  Add a source to the simulation
336 
337  @param[in] flux Total flux of the source to be added.
338  @param[in] centroid Position of the source to be added (lsst.afw.geom.Point2D).
339  @param[in] shape 2nd moments of the source before PSF convolution
340  (lsst.afw.geom.Quadrupole). Note that the truth catalog
341  records post-convolution moments). If None, a point source
342  will be added.
343 
344  @return a truth catalog record and single-source image corresponding to the new source.
345  """
346  # Create and set the truth catalog fields
347  record = self.catalog.addNew()
348  record.set(self.keys["flux"], flux)
349  record.set(self.keys["centroid"], centroid)
350  covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
351  covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
352  record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
353  if shape is None:
354  record.set(self.keys["isStar"], True)
355  fullShape = self.psfShape
356  else:
357  record.set(self.keys["isStar"], False)
358  fullShape = shape.convolve(self.psfShape)
359  record.set(self.keys["shape"], fullShape)
360  # Create an image containing just this source
361  image = self.drawGaussian(self.exposure.getBBox(), flux,
362  lsst.afw.geom.Ellipse(fullShape, centroid))
363  # Generate a footprint for this source
364  self._installFootprint(record, image)
365  # Actually add the source to the full exposure
366  self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
367  return record, image
368 
369  def addBlend(self):
370  """!
371  Return a context manager that allows a blend of multiple sources to be added.
372 
373  Example:
374  @code
375  d = TestDataset(...)
376  with d.addBlend() as b:
377  b.addChild(flux1, centroid1)
378  b.addChild(flux2, centroid2, shape2)
379  @endcode
380 
381  Note that nothing stops you from creating overlapping sources just using the addSource() method,
382  but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
383  produced by the detection and deblending pipelines.
384  """
385  return BlendContext(self)
386 
387  def transform(self, wcs, **kwds):
388  """!
389  Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
390 
391  @param[in] wcs Wcs for the new dataset.
392  @param[in] **kwds Additional keyword arguments passed on to makeEmptyExposure. If not
393  specified, these revert to the defaults for makeEmptyExposure, not the
394  values in the current dataset.
395  """
396  bboxD = lsst.afw.geom.Box2D()
397  xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
398  for corner in lsst.afw.geom.Box2D(self.exposure.getBBox()).getCorners():
399  bboxD.include(xyt.applyForward(lsst.afw.geom.Point2D(corner)))
400  bboxI = lsst.afw.geom.Box2I(bboxD)
401  result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
402  oldCalib = self.exposure.getCalib()
403  newCalib = result.exposure.getCalib()
404  oldPsfShape = self.exposure.getPsf().computeShape()
405  for record in self.catalog:
406  if record.get(self.keys["nChild"]):
407  raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
408  magnitude = oldCalib.getMagnitude(record.get(self.keys["flux"]))
409  newFlux = newCalib.getFlux(magnitude)
410  oldCentroid = record.get(self.keys["centroid"])
411  newCentroid = xyt.applyForward(oldCentroid)
412  if record.get(self.keys["isStar"]):
413  newDeconvolvedShape = None
414  else:
415  affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
416  oldFullShape = record.get(self.keys["shape"])
417  oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
418  oldFullShape.getIxx() - oldPsfShape.getIxx(),
419  oldFullShape.getIyy() - oldPsfShape.getIyy(),
420  oldFullShape.getIxy() - oldPsfShape.getIxy(),
421  False
422  )
423  newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
424  result.addSource(newFlux, newCentroid, newDeconvolvedShape)
425  return result
426 
427  def realize(self, noise, schema, randomSeed=1):
428  """!
429  Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
430 
431  @param[in] noise Standard deviation of noise to be added to the exposure. The noise will be
432  Gaussian and constant, appropriate for the sky-limited regime.
433  @param[in] schema Schema of the new catalog to be created. Must start with self.schema (i.e.
434  schema.contains(self.schema) must be True), but typically contains fields for
435  already-configured measurement algorithms as well.
436  @param[in] randomSeed Seed for the random number generator. If None, a seed is chosen automatically.
437 
438  @return a tuple of (exposure, catalog)
439  """
440  random_state = np.random.RandomState(randomSeed)
441  assert schema.contains(self.schema)
442  mapper = lsst.afw.table.SchemaMapper(self.schema)
443  mapper.addMinimalSchema(self.schema, True)
444  exposure = self.exposure.clone()
445  exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
446  exposure.getMaskedImage().getImage().getArray()[:, :] \
447  += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise
448  catalog = lsst.afw.table.SourceCatalog(schema)
449  catalog.extend(self.catalog, mapper=mapper)
450  # Loop over sources and generate new HeavyFootprints that divide up the noisy pixels, not the
451  # ideal no-noise pixels.
452  for record in catalog:
453  # parent objects have non-Heavy Footprints, which don't need to be updated after adding noise.
454  if record.getParent() == 0:
455  continue
456  # get flattened arrays that correspond to the no-noise and noisy parent images
457  parent = catalog.find(record.getParent())
458  footprint = parent.getFootprint()
459  parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
460  footprint.spans.flatten(parentFluxArrayNoNoise,
461  self.exposure.getMaskedImage().getImage().getArray(),
462  self.exposure.getXY0())
463  parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
464  footprint.spans.flatten(parentFluxArrayNoisy,
465  exposure.getMaskedImage().getImage().getArray(),
466  exposure.getXY0())
467  oldHeavy = record.getFootprint()
468  fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
469  # n.b. this isn't a copy ctor - it's a copy from a vanilla Footprint, so it doesn't copy
470  # the arrays we don't want to change, and hence we have to do that ourselves below.
471  newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
472  newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
473  newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
474  newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
475  record.setFootprint(newHeavy)
476  return exposure, catalog
477 
478 
479 class AlgorithmTestCase(object):
480 
481  def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
482  """Convenience function to create a Config instance for SingleFrameMeasurementTask
483 
484  The plugin and its dependencies will be the only plugins run, while the Centroid, Shape,
485  and ModelFlux slots will be set to the truth fields generated by the TestDataset class.
486  """
487  config = SingleFrameMeasurementTask.ConfigClass()
488  config.slots.centroid = "truth"
489  config.slots.shape = "truth"
490  config.slots.modelFlux = None
491  config.slots.apFlux = None
492  config.slots.psfFlux = None
493  config.slots.instFlux = None
494  config.slots.calibFlux = None
495  config.plugins.names = (plugin,) + tuple(dependencies)
496  return config
497 
498  def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
499  algMetadata=None):
500  """Convenience function to create a SingleFrameMeasurementTask with a simple configuration.
501  """
502  if config is None:
503  if plugin is None:
504  raise ValueError("Either plugin or config argument must not be None")
505  config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
506  if schema is None:
507  schema = TestDataset.makeMinimalSchema()
508  # Clear all aliases so only those defined by config are set.
509  schema.setAliasMap(None)
510  if algMetadata is None:
511  algMetadata = lsst.daf.base.PropertyList()
512  return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
513 
514  def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
515  """Convenience function to create a Config instance for ForcedMeasurementTask
516 
517  In addition to the plugins specified in the plugin and dependencies arguments,
518  the TransformedCentroid and TransformedShape plugins will be run and used as the
519  Centroid and Shape slots; these simply transform the reference catalog centroid
520  and shape to the measurement coordinate system.
521  """
522  config = ForcedMeasurementTask.ConfigClass()
523  config.slots.centroid = "base_TransformedCentroid"
524  config.slots.shape = "base_TransformedShape"
525  config.slots.modelFlux = None
526  config.slots.apFlux = None
527  config.slots.psfFlux = None
528  config.slots.instFlux = None
529  config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
530  "base_TransformedShape")
531  return config
532 
533  def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
534  algMetadata=None):
535  """Convenience function to create a ForcedMeasurementTask with a simple configuration.
536  """
537  if config is None:
538  if plugin is None:
539  raise ValueError("Either plugin or config argument must not be None")
540  config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
541  if refSchema is None:
542  refSchema = TestDataset.makeMinimalSchema()
543  if algMetadata is None:
544  algMetadata = lsst.daf.base.PropertyList()
545  return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
546 
547 
548 class TransformTestCase(object):
549  """!
550  Base class for testing measurement transformations.
551 
552  We test both that the transform itself operates successfully (fluxes are
553  converted to magnitudes, flags are propagated properly) and that the
554  transform is registered as the default for the appropriate measurement
555  algorithms.
556 
557  In the simple case of one-measurement-per-transformation, the developer
558  need not directly write any tests themselves: simply customizing the class
559  variables is all that is required. More complex measurements (e.g.
560  multiple aperture fluxes) require extra effort.
561  """
562  # The name used for the measurement algorithm; determines the names of the
563  # fields in the resulting catalog. This default should generally be fine,
564  # but subclasses can override if required.
565  name = "MeasurementTransformTest"
566 
567  # These should be customized by subclassing.
568  controlClass = None
569  algorithmClass = None
570  transformClass = None
571 
572  # Flags which may be set by the algorithm being tested. Can be customized
573  # in subclasses.
574  flagNames = ("flag",)
575 
576  # The plugin being tested should be registered under these names for
577  # single frame and forced measurement. Should be customized by
578  # subclassing.
579  singleFramePlugins = ()
580  forcedPlugins = ()
581 
582  def setUp(self):
584  self.calexp = TestDataset.makeEmptyExposure(bbox)
585  self._setupTransform()
586 
587  def tearDown(self):
588  del self.calexp
589  del self.inputCat
590  del self.mapper
591  del self.transform
592  del self.outputCat
593 
594  def _populateCatalog(self, baseNames):
595  records = []
596  for flagValue in (True, False):
597  records.append(self.inputCat.addNew())
598  for baseName in baseNames:
599  for flagName in self.flagNames:
600  if records[-1].schema.join(baseName, flagName) in records[-1].schema:
601  records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
602  self._setFieldsInRecords(records, baseName)
603 
604  def _checkOutput(self, baseNames):
605  for inSrc, outSrc in zip(self.inputCat, self.outputCat):
606  for baseName in baseNames:
607  self._compareFieldsInRecords(inSrc, outSrc, baseName)
608  for flagName in self.flagNames:
609  keyName = outSrc.schema.join(baseName, flagName)
610  if keyName in inSrc.schema:
611  self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
612  else:
613  self.assertFalse(keyName in outSrc.schema)
614 
615  def _runTransform(self, doExtend=True):
616  if doExtend:
617  self.outputCat.extend(self.inputCat, mapper=self.mapper)
618  self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getCalib())
619 
620  def testTransform(self, baseNames=None):
621  """
622  Test the operation of the transformation on a catalog containing random data.
623 
624  We check that:
625 
626  * An appropriate exception is raised on an attempt to transform between catalogs with different
627  numbers of rows;
628  * Otherwise, all appropriate conversions are properly appled and that flags have been propagated.
629 
630  The `baseNames` argument requires some explanation. This should be an iterable of the leading parts of
631  the field names for each measurement; that is, everything that appears before `_flux`, `_flag`, etc.
632  In the simple case of a single measurement per plugin, this is simply equal to `self.name` (thus
633  measurements are stored as `self.name + "_flux"`, etc). More generally, the developer may specify
634  whatever iterable they require. For example, to handle multiple apertures, we could have
635  `(self.name + "_0", self.name + "_1", ...)`.
636 
637  @param[in] baseNames Iterable of the initial parts of measurement field names.
638  """
639  baseNames = baseNames or [self.name]
640  self._populateCatalog(baseNames)
641  self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
642  self._runTransform()
643  self._checkOutput(baseNames)
644 
645  def _checkRegisteredTransform(self, registry, name):
646  # If this is a Python-based transform, we can compare directly; if
647  # it's wrapped C++, we need to compare the wrapped class.
648  self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
649 
650  def testRegistration(self):
651  """
652  Test that the transformation is appropriately registered with the relevant measurement algorithms.
653  """
654  for pluginName in self.singleFramePlugins:
655  self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
656  for pluginName in self.forcedPlugins:
657  self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
658 
659 
661 
662  def _setupTransform(self):
663  self.control = self.controlClass()
665  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
666  # it doesn't matter for this test since we won't actually use the plugins for anything besides
667  # defining the schema.
668  inputSchema.getAliasMap().set("slot_Centroid", "dummy")
669  inputSchema.getAliasMap().set("slot_Shape", "dummy")
670  self.algorithmClass(self.control, self.name, inputSchema)
671  inputSchema.getAliasMap().erase("slot_Centroid")
672  inputSchema.getAliasMap().erase("slot_Shape")
675  self.transform = self.transformClass(self.control, self.name, self.mapper)
676  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
677 
678 
680 
681  def _setupTransform(self):
682  self.control = self.controlClass()
685  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
686  # it doesn't matter for this test since we won't actually use the plugins for anything besides
687  # defining the schema.
688  inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
689  inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
690  self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
691  inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
692  inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
693  self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
694  self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
695  self.transform = self.transformClass(self.control, self.name, self.mapper)
696  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
697 
698 
700 
701  def _setFieldsInRecords(self, records, name):
702  for record in records:
703  record[record.schema.join(name, 'flux')] = np.random.random()
704  record[record.schema.join(name, 'fluxSigma')] = np.random.random()
705 
706  # Negative fluxes should be converted to NaNs.
707  assert len(records) > 1
708  records[0][record.schema.join(name, 'flux')] = -1
709 
710  def _compareFieldsInRecords(self, inSrc, outSrc, name):
711  fluxName, fluxSigmaName = inSrc.schema.join(name, 'flux'), inSrc.schema.join(name, 'fluxSigma')
712  if inSrc[fluxName] > 0:
713  mag, magErr = self.calexp.getCalib().getMagnitude(inSrc[fluxName], inSrc[fluxSigmaName])
714  self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag)
715  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], magErr)
716  else:
717  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
718  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
719 
720 
722 
723  def _setFieldsInRecords(self, records, name):
724  for record in records:
725  record[record.schema.join(name, 'x')] = np.random.random()
726  record[record.schema.join(name, 'y')] = np.random.random()
727  # Some algorithms set no errors; some set only sigma on x & y; some provide
728  # a full covariance matrix. Set only those which exist in the schema.
729  for fieldSuffix in ('xSigma', 'ySigma', 'x_y_Cov'):
730  fieldName = record.schema.join(name, fieldSuffix)
731  if fieldName in record.schema:
732  record[fieldName] = np.random.random()
733 
734  def _compareFieldsInRecords(self, inSrc, outSrc, name):
735  centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
736  centroidResult = centroidResultKey.get(inSrc)
737 
738  coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
739  coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
740  self.assertEqual(coordTruth, coord)
741 
742  # If the centroid has an associated uncertainty matrix, the coordinate
743  # must have one too, and vice versa.
744  try:
745  coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
746  ["ra", "dec"]).get(outSrc)
748  self.assertFalse(centroidResultKey.getCentroidErr().isValid())
749  else:
750  transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.afw.geom.radians)
751  coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
752  centroidResult.getCentroidErr()),
753  transform.getLinear().getMatrix().transpose())
754  np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
def _runTransform(self, doExtend=True)
Definition: tests.py:615
def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:481
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
def transform(self, wcs, kwds)
Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
Definition: tests.py:387
def __init__(self, bbox, threshold=10.0, exposure=None, kwds)
Initialize the dataset.
Definition: tests.py:295
def _checkOutput(self, baseNames)
Definition: tests.py:604
A subtask for measuring the properties of sources on a single exposure, using an existing "reference"...
A subtask for measuring the properties of sources on a single exposure.
Definition: sfm.py:152
def addBlend(self)
Return a context manager that allows a blend of multiple sources to be added.
Definition: tests.py:369
def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12)
Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
Definition: tests.py:247
static LinearTransform makeScaling(double s)
def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition: tests.py:499
def realize(self, noise, schema, randomSeed=1)
Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
Definition: tests.py:427
std::shared_ptr< SkyWcs > makeSkyWcs(Point2D const &crpix, coord::IcrsCoord const &crval, Eigen::Matrix2d const &cdMatrix, std::string const &projection="TAN")
AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, Point2D const &inPoint)
def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=1)
Create a new undistorted TAN WCS that is similar but not identical to another, with random scaling...
Definition: tests.py:186
def __init__(self, owner)
Definition: tests.py:53
def testTransform(self, baseNames=None)
Definition: tests.py:620
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
def drawGaussian(bbox, flux, ellipse)
Create an image of an elliptical Gaussian.
Definition: tests.py:277
static LinearTransform makeRotation(Angle t)
def _installFootprint(self, record, image)
Definition: tests.py:317
def _checkRegisteredTransform(self, registry, name)
Definition: tests.py:645
def _populateCatalog(self, baseNames)
Definition: tests.py:594
def addSource(self, flux, centroid, shape=None)
Add a source to the simulation.
Definition: tests.py:333
static Schema makeMinimalSchema()
A simulated dataset consisting of a test image and an associated truth catalog.
Definition: tests.py:122
def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition: tests.py:534
def addChild(self, flux, centroid, shape=None)
Add a child source to the blend, and return the truth catalog record that corresponds to it...
Definition: tests.py:63
Eigen::Matrix2d makeCdMatrix(Angle const &scale, Angle const &orientation=0 *degrees, bool flipX=false)
Base class for testing measurement transformations.
Definition: tests.py:548
def __exit__(self, type_, value, tb)
Definition: tests.py:79
def makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:514
A Python context manager used to add multiple overlapping sources along with a parent source that rep...
Definition: tests.py:44