lsst.meas.base  16.0-13-gd9b1b71+1
tests.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 import numpy as np
25 
26 import lsst.geom
27 import lsst.afw.table
28 import lsst.afw.image
29 import lsst.afw.detection
30 import lsst.afw.geom
32 
33 from .sfm import SingleFrameMeasurementTask
34 from .forcedMeasurement import ForcedMeasurementTask
35 from . import CentroidResultKey
36 
37 __all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
38  "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
39  "FluxTransformTestCase", "CentroidTransformTestCase")
40 
41 
43  """!
44  A Python context manager used to add multiple overlapping sources along with a parent source
45  that represents all of them together.
46 
47  This is used as the return value for TestDataset.addBlend(), and this is the only way it should
48  be used. The only public method is addChild().
49  """
50 
51  def __init__(self, owner):
52  self.owner = owner
53  self.parentRecord = self.owner.catalog.addNew()
54  self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
55  self.children = []
56 
57  def __enter__(self):
58  # BlendContext is its own context manager, so we just return self.
59  return self
60 
61  def addChild(self, instFlux, centroid, shape=None):
62  """!
63  Add a child source to the blend, and return the truth catalog record that corresponds to it.
64 
65  @param[in] instFlux Total instFlux of the source to be added.
66  @param[in] centroid Position of the source to be added (lsst.geom.Point2D).
67  @param[in] shape 2nd moments of the source before PSF convolution
68  (lsst.afw.geom.Quadrupole). Note that the truth catalog
69  records post-convolution moments)
70  """
71  record, image = self.owner.addSource(instFlux, centroid, shape)
72  record.set(self.owner.keys["parent"], self.parentRecord.getId())
73  self.parentImage += image
74  self.children.append((record, image))
75  return record
76 
77  def __exit__(self, type_, value, tb):
78  # We're not using the context manager for any kind of exception safety or guarantees;
79  # we just want the nice "with" statement syntax.
80  if type_ is not None: # exception was raised; just skip all this and let it propagate
81  return
82  # On exit, we need to compute and set the truth values for the parent object.
83  self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
84  # Compute instFlux from sum of component fluxes
85  instFlux = 0.0
86  for record, image in self.children:
87  instFlux += record.get(self.owner.keys["instFlux"])
88  self.parentRecord.set(self.owner.keys["instFlux"], instFlux)
89  # Compute centroid from instFlux-weighted mean of component centroids
90  x = 0.0
91  y = 0.0
92  for record, image in self.children:
93  w = record.get(self.owner.keys["instFlux"])/instFlux
94  x += record.get(self.owner.keys["centroid"].getX())*w
95  y += record.get(self.owner.keys["centroid"].getY())*w
96  self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
97  # Compute shape from instFlux-weighted mean of offset component shapes
98  xx = 0.0
99  yy = 0.0
100  xy = 0.0
101  for record, image in self.children:
102  w = record.get(self.owner.keys["instFlux"])/instFlux
103  dx = record.get(self.owner.keys["centroid"].getX()) - x
104  dy = record.get(self.owner.keys["centroid"].getY()) - y
105  xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
106  yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
107  xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
108  self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
109  # Run detection on the parent image to get the parent Footprint.
110  self.owner._installFootprint(self.parentRecord, self.parentImage)
111  # Create perfect HeavyFootprints for all children; these will need to be modified later to account
112  # for the noise we'll add to the image.
113  deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
114  for record, image in self.children:
115  deblend.getImage().getArray()[:, :] = image.getArray()
116  heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
117  record.setFootprint(heavyFootprint)
118 
119 
121  """!
122  A simulated dataset consisting of a test image and an associated truth catalog.
123 
124  TestDataset creates an idealized image made of pure Gaussians (including a Gaussian PSF),
125  with simple noise and idealized Footprints/HeavyFootprints that simulated the outputs
126  of detection and deblending. Multiple noise realizations can be created from the same
127  underlying sources, allowing uncertainty estimates to be verified via Monte Carlo.
128 
129  Typical usage:
130  @code
131  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 100))
132  dataset = TestDataset(bbox)
133  dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26))
134  dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24),
135  shape=lsst.afw.geom.Quadrupole(8, 7, 2))
136  with dataset.addBlend() as family:
137  family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72))
138  family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
139  exposure, catalog = dataset.realize(noise=100.0, schema=TestDataset.makeMinimalSchema())
140  @endcode
141  """
142 
143  @classmethod
145  """Return the minimal schema needed to hold truth catalog fields.
146 
147  When TestDataset.realize() is called, the schema must include at least these fields.
148  Usually it will include additional fields for measurement algorithm outputs, allowing
149  the same catalog to be used for both truth values (the fields from the minimal schema)
150  and the measurements.
151  """
152  if not hasattr(cls, "_schema"):
154  cls.keys = {}
155  cls.keys["parent"] = schema.find("parent").key
156  cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
157  cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64,
158  doc="true instFlux", units="count")
159  cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
160  schema, "truth", "true simulated centroid", "pixel"
161  )
162  cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
163  schema, "truth", ['x', 'y'], "pixel"
164  )
165  cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
166  doc="set if the object is a star")
168  schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
169  )
170  cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
171  doc="set if the object is a star")
172  schema.getAliasMap().set("slot_Shape", "truth")
173  schema.getAliasMap().set("slot_Centroid", "truth")
174  schema.getAliasMap().set("slot_ModelFlux", "truth")
175  schema.getCitizen().markPersistent()
176  cls._schema = schema
177  schema = lsst.afw.table.Schema(cls._schema)
178  schema.disconnectAliases()
179  return schema
180 
181  @staticmethod
182  def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
183  minRotation=None, maxRotation=None,
184  minRefShift=None, maxRefShift=None,
185  minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
186  """!
187  Create a new undistorted TAN WCS that is similar but not identical to another, with random
188  scaling, rotation, and offset (in both pixel position and reference position).
189 
190  The maximum and minimum arguments are interpreted as absolute values for a split
191  range that covers both positive and negative values (as this method is used
192  in testing, it is typically most important to avoid perturbations near zero).
193  Scale factors are treated somewhat differently: the actual scale factor is chosen between
194  minScaleFactor and maxScaleFactor OR (1/maxScaleFactor) and (1/minScaleFactor).
195 
196  The default range for rotation is 30-60 degrees, and the default range for reference shift
197  is 0.5-1.0 arcseconds (these cannot be safely included directly as default values because Angle
198  objects are mutable).
199 
200  The random number generator is primed with the seed given. If ``None``, a seed is
201  automatically chosen.
202  """
203  random_state = np.random.RandomState(randomSeed)
204  if minRotation is None:
205  minRotation = 30.0*lsst.geom.degrees
206  if maxRotation is None:
207  maxRotation = 60.0*lsst.geom.degrees
208  if minRefShift is None:
209  minRefShift = 0.5*lsst.geom.arcseconds
210  if maxRefShift is None:
211  maxRefShift = 1.0*lsst.geom.arcseconds
212 
213  def splitRandom(min1, max1, min2=None, max2=None):
214  if min2 is None:
215  min2 = -max1
216  if max2 is None:
217  max2 = -min1
218  if random_state.uniform() > 0.5:
219  return float(random_state.uniform(min1, max1))
220  else:
221  return float(random_state.uniform(min2, max2))
222  # Generate random perturbations
223  scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
224  rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
225  refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
226  refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
227  pixShiftX = splitRandom(minPixShift, maxPixShift)
228  pixShiftY = splitRandom(minPixShift, maxPixShift)
229  # Compute new CD matrix
230  oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
231  rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
232  sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
233  newTransform = oldTransform*rTransform*sTransform
234  matrix = newTransform.getMatrix()
235  # Compute new coordinate reference pixel (CRVAL)
236  oldSkyOrigin = oldWcs.getSkyOrigin()
237  newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
238  oldSkyOrigin.getDec() + refShiftDec)
239  # Compute new pixel reference pixel (CRPIX)
240  oldPixOrigin = oldWcs.getPixelOrigin()
241  newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
242  oldPixOrigin.getY() + pixShiftY)
243  return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
244 
245  @staticmethod
246  def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12):
247  """!
248  Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
249 
250  @param[in] bbox Bounding box of the image (image coordinates) as returned by makeCatalog.
251  @param[in] wcs New Wcs for the exposure (created from crval and cdelt if None).
252  @param[in] crval afw.geom.SpherePoint: ICRS center of the TAN WCS attached to the image.
253  @param[in] cdelt afw.geom.Angle: pixel scale of the image
254  @param[in] psfSigma Radius (sigma) of the Gaussian PSF attached to the image
255  @param[in] psfDim Width and height of the image's Gaussian PSF attached to the image
256  @param[in] fluxMag0 Flux at magnitude zero (in e-) used to set the Calib of the exposure.
257  """
258  if wcs is None:
259  if crval is None:
260  crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
261  if cdelt is None:
262  cdelt = 0.2*lsst.geom.arcseconds
263  crpix = lsst.geom.Box2D(bbox).getCenter()
264  wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
265  cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
266  exposure = lsst.afw.image.ExposureF(bbox)
267  psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
268  calib = lsst.afw.image.Calib()
269  calib.setFluxMag0(fluxMag0)
270  exposure.setWcs(wcs)
271  exposure.setPsf(psf)
272  exposure.setCalib(calib)
273  return exposure
274 
275  @staticmethod
276  def drawGaussian(bbox, instFlux, ellipse):
277  """!
278  Create an image of an elliptical Gaussian.
279 
280  @param[in,out] bbox Bounding box of image to create.
281  @param[in] instFlux Total instFlux of the Gaussian (normalized analytically,
282  not using pixel values)
283  @param[in] ellipse lsst.afw.geom.Ellipse holding the centroid and shape.
284  """
285  x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
286  np.arange(bbox.getBeginY(), bbox.getEndY()))
287  t = ellipse.getGridTransform()
288  xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
289  yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
290  image = lsst.afw.image.ImageF(bbox)
291  image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea())
292  return image
293 
294  def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
295  """!
296  Initialize the dataset.
297 
298  @param[in] bbox Bounding box of the test image.
299  @param[in] threshold Threshold absolute value used to determine footprints for
300  simulated sources. This thresholding will be applied before noise is
301  actually added to images (or before the noise level is even known), so
302  this will necessarily produce somewhat artificial footprints.
303  @param[in] exposure lsst.afw.image.ExposureF test sources should be added to. Ownership should
304  be considered transferred from the caller to the TestDataset.
305  Must have a Gaussian Psf for truth catalog shapes to be exact.
306  @param[in] **kwds Keyword arguments forwarded to makeEmptyExposure if exposure is None.
307  """
308  if exposure is None:
309  exposure = self.makeEmptyExposure(bbox, **kwds)
310  self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
311  self.exposure = exposure
312  self.psfShape = self.exposure.getPsf().computeShape()
313  self.schema = self.makeMinimalSchema()
315 
316  def _installFootprint(self, record, image):
317  """Create a Footprint for a simulated source and add it to its truth catalog record.
318  """
319  # Run detection on the single-source image
320  fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
321  # the call below to the FootprintSet ctor is actually a grow operation
322  fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
323  # Update the full exposure's mask plane to indicate the detection
324  fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
325  # Attach the new footprint to the exposure
326  if len(fpSet.getFootprints()) > 1:
327  raise RuntimeError("Threshold value results in multiple Footprints for a single object")
328  if len(fpSet.getFootprints()) == 0:
329  raise RuntimeError("Threshold value results in zero Footprints for object")
330  record.setFootprint(fpSet.getFootprints()[0])
331 
332  def addSource(self, instFlux, centroid, shape=None):
333  """!
334  Add a source to the simulation
335 
336  @param[in] instFlux Total instFlux of the source to be added.
337  @param[in] centroid Position of the source to be added (lsst.geom.Point2D).
338  @param[in] shape 2nd moments of the source before PSF convolution
339  (lsst.afw.geom.Quadrupole). Note that the truth catalog
340  records post-convolution moments). If None, a point source
341  will be added.
342 
343  @return a truth catalog record and single-source image corresponding to the new source.
344  """
345  # Create and set the truth catalog fields
346  record = self.catalog.addNew()
347  record.set(self.keys["instFlux"], instFlux)
348  record.set(self.keys["centroid"], centroid)
349  covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
350  covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
351  record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
352  if shape is None:
353  record.set(self.keys["isStar"], True)
354  fullShape = self.psfShape
355  else:
356  record.set(self.keys["isStar"], False)
357  fullShape = shape.convolve(self.psfShape)
358  record.set(self.keys["shape"], fullShape)
359  # Create an image containing just this source
360  image = self.drawGaussian(self.exposure.getBBox(), instFlux,
361  lsst.afw.geom.Ellipse(fullShape, centroid))
362  # Generate a footprint for this source
363  self._installFootprint(record, image)
364  # Actually add the source to the full exposure
365  self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
366  return record, image
367 
368  def addBlend(self):
369  """!
370  Return a context manager that allows a blend of multiple sources to be added.
371 
372  Example:
373  @code
374  d = TestDataset(...)
375  with d.addBlend() as b:
376  b.addChild(flux1, centroid1)
377  b.addChild(flux2, centroid2, shape2)
378  @endcode
379 
380  Note that nothing stops you from creating overlapping sources just using the addSource() method,
381  but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
382  produced by the detection and deblending pipelines.
383  """
384  return BlendContext(self)
385 
386  def transform(self, wcs, **kwds):
387  """!
388  Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
389 
390  @param[in] wcs Wcs for the new dataset.
391  @param[in] **kwds Additional keyword arguments passed on to makeEmptyExposure. If not
392  specified, these revert to the defaults for makeEmptyExposure, not the
393  values in the current dataset.
394  """
395  bboxD = lsst.geom.Box2D()
396  xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
397  for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
398  bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
399  bboxI = lsst.geom.Box2I(bboxD)
400  result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
401  oldCalib = self.exposure.getCalib()
402  newCalib = result.exposure.getCalib()
403  oldPsfShape = self.exposure.getPsf().computeShape()
404  for record in self.catalog:
405  if record.get(self.keys["nChild"]):
406  raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
407  magnitude = oldCalib.getMagnitude(record.get(self.keys["instFlux"]))
408  newFlux = newCalib.getFlux(magnitude)
409  oldCentroid = record.get(self.keys["centroid"])
410  newCentroid = xyt.applyForward(oldCentroid)
411  if record.get(self.keys["isStar"]):
412  newDeconvolvedShape = None
413  else:
414  affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
415  oldFullShape = record.get(self.keys["shape"])
416  oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
417  oldFullShape.getIxx() - oldPsfShape.getIxx(),
418  oldFullShape.getIyy() - oldPsfShape.getIyy(),
419  oldFullShape.getIxy() - oldPsfShape.getIxy(),
420  False
421  )
422  newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
423  result.addSource(newFlux, newCentroid, newDeconvolvedShape)
424  return result
425 
426  def realize(self, noise, schema, randomSeed=1):
427  """!
428  Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
429 
430  @param[in] noise Standard deviation of noise to be added to the exposure. The noise will be
431  Gaussian and constant, appropriate for the sky-limited regime.
432  @param[in] schema Schema of the new catalog to be created. Must start with self.schema (i.e.
433  schema.contains(self.schema) must be True), but typically contains fields for
434  already-configured measurement algorithms as well.
435  @param[in] randomSeed Seed for the random number generator. If None, a seed is chosen automatically.
436 
437  @return a tuple of (exposure, catalog)
438  """
439  random_state = np.random.RandomState(randomSeed)
440  assert schema.contains(self.schema)
441  mapper = lsst.afw.table.SchemaMapper(self.schema)
442  mapper.addMinimalSchema(self.schema, True)
443  exposure = self.exposure.clone()
444  exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
445  exposure.getMaskedImage().getImage().getArray()[:, :] \
446  += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise
447  catalog = lsst.afw.table.SourceCatalog(schema)
448  catalog.extend(self.catalog, mapper=mapper)
449  # Loop over sources and generate new HeavyFootprints that divide up the noisy pixels, not the
450  # ideal no-noise pixels.
451  for record in catalog:
452  # parent objects have non-Heavy Footprints, which don't need to be updated after adding noise.
453  if record.getParent() == 0:
454  continue
455  # get flattened arrays that correspond to the no-noise and noisy parent images
456  parent = catalog.find(record.getParent())
457  footprint = parent.getFootprint()
458  parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
459  footprint.spans.flatten(parentFluxArrayNoNoise,
460  self.exposure.getMaskedImage().getImage().getArray(),
461  self.exposure.getXY0())
462  parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
463  footprint.spans.flatten(parentFluxArrayNoisy,
464  exposure.getMaskedImage().getImage().getArray(),
465  exposure.getXY0())
466  oldHeavy = record.getFootprint()
467  fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
468  # n.b. this isn't a copy ctor - it's a copy from a vanilla Footprint, so it doesn't copy
469  # the arrays we don't want to change, and hence we have to do that ourselves below.
470  newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
471  newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
472  newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
473  newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
474  record.setFootprint(newHeavy)
475  return exposure, catalog
476 
477 
479 
480  def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
481  """Convenience function to create a Config instance for SingleFrameMeasurementTask
482 
483  The plugin and its dependencies will be the only plugins run, while the Centroid, Shape,
484  and ModelFlux slots will be set to the truth fields generated by the TestDataset class.
485  """
486  config = SingleFrameMeasurementTask.ConfigClass()
487  config.slots.centroid = "truth"
488  config.slots.shape = "truth"
489  config.slots.modelFlux = None
490  config.slots.apFlux = None
491  config.slots.psfFlux = None
492  config.slots.gaussianFlux = None
493  config.slots.calibFlux = None
494  config.plugins.names = (plugin,) + tuple(dependencies)
495  return config
496 
497  def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
498  algMetadata=None):
499  """Convenience function to create a SingleFrameMeasurementTask with a simple configuration.
500  """
501  if config is None:
502  if plugin is None:
503  raise ValueError("Either plugin or config argument must not be None")
504  config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
505  if schema is None:
506  schema = TestDataset.makeMinimalSchema()
507  # Clear all aliases so only those defined by config are set.
508  schema.setAliasMap(None)
509  if algMetadata is None:
510  algMetadata = lsst.daf.base.PropertyList()
511  return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
512 
513  def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
514  """Convenience function to create a Config instance for ForcedMeasurementTask
515 
516  In addition to the plugins specified in the plugin and dependencies arguments,
517  the TransformedCentroid and TransformedShape plugins will be run and used as the
518  Centroid and Shape slots; these simply transform the reference catalog centroid
519  and shape to the measurement coordinate system.
520  """
521  config = ForcedMeasurementTask.ConfigClass()
522  config.slots.centroid = "base_TransformedCentroid"
523  config.slots.shape = "base_TransformedShape"
524  config.slots.modelFlux = None
525  config.slots.apFlux = None
526  config.slots.psfFlux = None
527  config.slots.gaussianFlux = None
528  config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
529  "base_TransformedShape")
530  return config
531 
532  def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
533  algMetadata=None):
534  """Convenience function to create a ForcedMeasurementTask with a simple configuration.
535  """
536  if config is None:
537  if plugin is None:
538  raise ValueError("Either plugin or config argument must not be None")
539  config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
540  if refSchema is None:
541  refSchema = TestDataset.makeMinimalSchema()
542  if algMetadata is None:
543  algMetadata = lsst.daf.base.PropertyList()
544  return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
545 
546 
548  """!
549  Base class for testing measurement transformations.
550 
551  We test both that the transform itself operates successfully (fluxes are
552  converted to magnitudes, flags are propagated properly) and that the
553  transform is registered as the default for the appropriate measurement
554  algorithms.
555 
556  In the simple case of one-measurement-per-transformation, the developer
557  need not directly write any tests themselves: simply customizing the class
558  variables is all that is required. More complex measurements (e.g.
559  multiple aperture fluxes) require extra effort.
560  """
561  # The name used for the measurement algorithm; determines the names of the
562  # fields in the resulting catalog. This default should generally be fine,
563  # but subclasses can override if required.
564  name = "MeasurementTransformTest"
565 
566  # These should be customized by subclassing.
567  controlClass = None
568  algorithmClass = None
569  transformClass = None
570 
571  # Flags which may be set by the algorithm being tested. Can be customized
572  # in subclasses.
573  flagNames = ("flag",)
574 
575  # The plugin being tested should be registered under these names for
576  # single frame and forced measurement. Should be customized by
577  # subclassing.
578  singleFramePlugins = ()
579  forcedPlugins = ()
580 
581  def setUp(self):
582  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200))
583  self.calexp = TestDataset.makeEmptyExposure(bbox)
584  self._setupTransform()
585 
586  def tearDown(self):
587  del self.calexp
588  del self.inputCat
589  del self.mapper
590  del self.transform
591  del self.outputCat
592 
593  def _populateCatalog(self, baseNames):
594  records = []
595  for flagValue in (True, False):
596  records.append(self.inputCat.addNew())
597  for baseName in baseNames:
598  for flagName in self.flagNames:
599  if records[-1].schema.join(baseName, flagName) in records[-1].schema:
600  records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
601  self._setFieldsInRecords(records, baseName)
602 
603  def _checkOutput(self, baseNames):
604  for inSrc, outSrc in zip(self.inputCat, self.outputCat):
605  for baseName in baseNames:
606  self._compareFieldsInRecords(inSrc, outSrc, baseName)
607  for flagName in self.flagNames:
608  keyName = outSrc.schema.join(baseName, flagName)
609  if keyName in inSrc.schema:
610  self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
611  else:
612  self.assertFalse(keyName in outSrc.schema)
613 
614  def _runTransform(self, doExtend=True):
615  if doExtend:
616  self.outputCat.extend(self.inputCat, mapper=self.mapper)
617  self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getCalib())
618 
619  def testTransform(self, baseNames=None):
620  """
621  Test the operation of the transformation on a catalog containing random data.
622 
623  We check that:
624 
625  * An appropriate exception is raised on an attempt to transform between catalogs with different
626  numbers of rows;
627  * Otherwise, all appropriate conversions are properly appled and that flags have been propagated.
628 
629  The `baseNames` argument requires some explanation. This should be an iterable of the leading parts of
630  the field names for each measurement; that is, everything that appears before `_instFlux`, `_flag`,
631  etc. In the simple case of a single measurement per plugin, this is simply equal to `self.name` (thus
632  measurements are stored as `self.name + "_instFlux"`, etc). More generally, the developer may specify
633  whatever iterable they require. For example, to handle multiple apertures, we could have
634  `(self.name + "_0", self.name + "_1", ...)`.
635 
636  @param[in] baseNames Iterable of the initial parts of measurement field names.
637  """
638  baseNames = baseNames or [self.name]
639  self._populateCatalog(baseNames)
640  self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
641  self._runTransform()
642  self._checkOutput(baseNames)
643 
644  def _checkRegisteredTransform(self, registry, name):
645  # If this is a Python-based transform, we can compare directly; if
646  # it's wrapped C++, we need to compare the wrapped class.
647  self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
648 
649  def testRegistration(self):
650  """
651  Test that the transformation is appropriately registered with the relevant measurement algorithms.
652  """
653  for pluginName in self.singleFramePlugins:
654  self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
655  for pluginName in self.forcedPlugins:
656  self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
657 
658 
660 
661  def _setupTransform(self):
662  self.control = self.controlClass()
664  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
665  # it doesn't matter for this test since we won't actually use the plugins for anything besides
666  # defining the schema.
667  inputSchema.getAliasMap().set("slot_Centroid", "dummy")
668  inputSchema.getAliasMap().set("slot_Shape", "dummy")
669  self.algorithmClass(self.control, self.name, inputSchema)
670  inputSchema.getAliasMap().erase("slot_Centroid")
671  inputSchema.getAliasMap().erase("slot_Shape")
674  self.transform = self.transformClass(self.control, self.name, self.mapper)
675  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
676 
677 
679 
680  def _setupTransform(self):
681  self.control = self.controlClass()
684  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
685  # it doesn't matter for this test since we won't actually use the plugins for anything besides
686  # defining the schema.
687  inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
688  inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
689  self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
690  inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
691  inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
692  self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
693  self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
694  self.transform = self.transformClass(self.control, self.name, self.mapper)
695  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
696 
697 
699 
700  def _setFieldsInRecords(self, records, name):
701  for record in records:
702  record[record.schema.join(name, 'instFlux')] = np.random.random()
703  record[record.schema.join(name, 'instFluxErr')] = np.random.random()
704 
705  # Negative instFluxes should be converted to NaNs.
706  assert len(records) > 1
707  records[0][record.schema.join(name, 'instFlux')] = -1
708 
709  def _compareFieldsInRecords(self, inSrc, outSrc, name):
710  instFluxName = inSrc.schema.join(name, 'instFlux')
711  instFluxErrName = inSrc.schema.join(name, 'instFluxErr')
712  if inSrc[instFluxName] > 0:
713  mag, magErr = self.calexp.getCalib().getMagnitude(inSrc[instFluxName], inSrc[instFluxErrName])
714  self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag)
715  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], magErr)
716  else:
717  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
718  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
719 
720 
722 
723  def _setFieldsInRecords(self, records, name):
724  for record in records:
725  record[record.schema.join(name, 'x')] = np.random.random()
726  record[record.schema.join(name, 'y')] = np.random.random()
727  # Some algorithms set no errors; some set only sigma on x & y; some provide
728  # a full covariance matrix. Set only those which exist in the schema.
729  for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'):
730  fieldName = record.schema.join(name, fieldSuffix)
731  if fieldName in record.schema:
732  record[fieldName] = np.random.random()
733 
734  def _compareFieldsInRecords(self, inSrc, outSrc, name):
735  centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
736  centroidResult = centroidResultKey.get(inSrc)
737 
738  coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
739  coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
740  self.assertEqual(coordTruth, coord)
741 
742  # If the centroid has an associated uncertainty matrix, the coordinate
743  # must have one too, and vice versa.
744  try:
745  coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
746  ["ra", "dec"]).get(outSrc)
748  self.assertFalse(centroidResultKey.getCentroidErr().isValid())
749  else:
750  transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
751  coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
752  centroidResult.getCentroidErr()),
753  transform.getLinear().getMatrix().transpose())
754  np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
def _runTransform(self, doExtend=True)
Definition: tests.py:614
def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:480
lsst::geom::AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, lsst::geom::Point2D const &inPoint)
def transform(self, wcs, kwds)
Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
Definition: tests.py:386
def __init__(self, bbox, threshold=10.0, exposure=None, kwds)
Initialize the dataset.
Definition: tests.py:294
def _checkOutput(self, baseNames)
Definition: tests.py:603
A subtask for measuring the properties of sources on a single exposure, using an existing "reference"...
A subtask for measuring the properties of sources on a single exposure.
Definition: sfm.py:152
def addBlend(self)
Return a context manager that allows a blend of multiple sources to be added.
Definition: tests.py:368
def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12)
Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
Definition: tests.py:246
def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition: tests.py:498
def realize(self, noise, schema, randomSeed=1)
Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
Definition: tests.py:426
def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=1)
Create a new undistorted TAN WCS that is similar but not identical to another, with random scaling...
Definition: tests.py:185
def __init__(self, owner)
Definition: tests.py:51
def testTransform(self, baseNames=None)
Definition: tests.py:619
def _installFootprint(self, record, image)
Definition: tests.py:316
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
def _checkRegisteredTransform(self, registry, name)
Definition: tests.py:644
def _populateCatalog(self, baseNames)
Definition: tests.py:593
static Schema makeMinimalSchema()
A simulated dataset consisting of a test image and an associated truth catalog.
Definition: tests.py:120
static LinearTransform makeScaling(double s) noexcept
def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition: tests.py:533
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
def drawGaussian(bbox, instFlux, ellipse)
Create an image of an elliptical Gaussian.
Definition: tests.py:276
std::shared_ptr< SkyWcs > makeSkyWcs(TransformPoint2ToPoint2 const &pixelsToFieldAngle, lsst::geom::Angle const &orientation, bool flipX, lsst::geom::SpherePoint const &boresight, std::string const &projection="TAN")
Eigen::Matrix2d makeCdMatrix(lsst::geom::Angle const &scale, lsst::geom::Angle const &orientation=0 *lsst::geom::degrees, bool flipX=false)
static LinearTransform makeRotation(Angle t) noexcept
def addSource(self, instFlux, centroid, shape=None)
Add a source to the simulation.
Definition: tests.py:332
def addChild(self, instFlux, centroid, shape=None)
Add a child source to the blend, and return the truth catalog record that corresponds to it...
Definition: tests.py:61
Base class for testing measurement transformations.
Definition: tests.py:547
def __exit__(self, type_, value, tb)
Definition: tests.py:77
def makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:513
A Python context manager used to add multiple overlapping sources along with a parent source that rep...
Definition: tests.py:42