lsst.meas.base  16.0-8-g2ce35ff+2
tests.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 import numpy as np
25 
26 import lsst.geom
27 import lsst.afw.table
28 import lsst.afw.image
29 import lsst.afw.detection
30 import lsst.afw.geom
32 
33 from .sfm import SingleFrameMeasurementTask
34 from .forcedMeasurement import ForcedMeasurementTask
35 from . import CentroidResultKey
36 
37 __all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
38  "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
39  "FluxTransformTestCase", "CentroidTransformTestCase")
40 
41 
43  """!
44  A Python context manager used to add multiple overlapping sources along with a parent source
45  that represents all of them together.
46 
47  This is used as the return value for TestDataset.addBlend(), and this is the only way it should
48  be used. The only public method is addChild().
49  """
50 
51  def __init__(self, owner):
52  self.owner = owner
53  self.parentRecord = self.owner.catalog.addNew()
54  self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
55  self.children = []
56 
57  def __enter__(self):
58  # BlendContext is its own context manager, so we just return self.
59  return self
60 
61  def addChild(self, flux, centroid, shape=None):
62  """!
63  Add a child source to the blend, and return the truth catalog record that corresponds to it.
64 
65  @param[in] flux Total flux of the source to be added.
66  @param[in] centroid Position of the source to be added (lsst.geom.Point2D).
67  @param[in] shape 2nd moments of the source before PSF convolution
68  (lsst.afw.geom.Quadrupole). Note that the truth catalog
69  records post-convolution moments)
70  """
71  record, image = self.owner.addSource(flux, centroid, shape)
72  record.set(self.owner.keys["parent"], self.parentRecord.getId())
73  self.parentImage += image
74  self.children.append((record, image))
75  return record
76 
77  def __exit__(self, type_, value, tb):
78  # We're not using the context manager for any kind of exception safety or guarantees;
79  # we just want the nice "with" statement syntax.
80  if type_ is not None: # exception was raised; just skip all this and let it propagate
81  return
82  # On exit, we need to compute and set the truth values for the parent object.
83  self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
84  # Compute flux from sum of component fluxes
85  flux = 0.0
86  for record, image in self.children:
87  flux += record.get(self.owner.keys["flux"])
88  self.parentRecord.set(self.owner.keys["flux"], flux)
89  # Compute centroid from flux-weighted mean of component centroids
90  x = 0.0
91  y = 0.0
92  for record, image in self.children:
93  w = record.get(self.owner.keys["flux"])/flux
94  x += record.get(self.owner.keys["centroid"].getX())*w
95  y += record.get(self.owner.keys["centroid"].getY())*w
96  self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
97  # Compute shape from flux-weighted mean of offset component shapes
98  xx = 0.0
99  yy = 0.0
100  xy = 0.0
101  for record, image in self.children:
102  w = record.get(self.owner.keys["flux"])/flux
103  dx = record.get(self.owner.keys["centroid"].getX()) - x
104  dy = record.get(self.owner.keys["centroid"].getY()) - y
105  xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
106  yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
107  xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
108  self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
109  # Run detection on the parent image to get the parent Footprint.
110  self.owner._installFootprint(self.parentRecord, self.parentImage)
111  # Create perfect HeavyFootprints for all children; these will need to be modified later to account
112  # for the noise we'll add to the image.
113  deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
114  for record, image in self.children:
115  deblend.getImage().getArray()[:, :] = image.getArray()
116  heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
117  record.setFootprint(heavyFootprint)
118 
119 
121  """!
122  A simulated dataset consisting of a test image and an associated truth catalog.
123 
124  TestDataset creates an idealized image made of pure Gaussians (including a Gaussian PSF),
125  with simple noise and idealized Footprints/HeavyFootprints that simulated the outputs
126  of detection and deblending. Multiple noise realizations can be created from the same
127  underlying sources, allowing uncertainty estimates to be verified via Monte Carlo.
128 
129  Typical usage:
130  @code
131  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 100))
132  dataset = TestDataset(bbox)
133  dataset.addSource(flux=1E5, centroid=lsst.geom.Point2D(25, 26))
134  dataset.addSource(flux=2E5, centroid=lsst.geom.Point2D(75, 24),
135  shape=lsst.afw.geom.Quadrupole(8, 7, 2))
136  with dataset.addBlend() as family:
137  family.addChild(flux=2E5, centroid=lsst.geom.Point2D(50, 72))
138  family.addChild(flux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
139  exposure, catalog = dataset.realize(noise=100.0, schema=TestDataset.makeMinimalSchema())
140  @endcode
141  """
142 
143  @classmethod
145  """Return the minimal schema needed to hold truth catalog fields.
146 
147  When TestDataset.realize() is called, the schema must include at least these fields.
148  Usually it will include additional fields for measurement algorithm outputs, allowing
149  the same catalog to be used for both truth values (the fields from the minimal schema)
150  and the measurements.
151  """
152  if not hasattr(cls, "_schema"):
154  cls.keys = {}
155  cls.keys["parent"] = schema.find("parent").key
156  cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
157  cls.keys["flux"] = schema.addField("truth_flux", type=np.float64, doc="true flux", units="count")
158  cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
159  schema, "truth", "true simulated centroid", "pixel"
160  )
161  cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
162  schema, "truth", ['x', 'y'], "pixel"
163  )
164  cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
165  doc="set if the object is a star")
167  schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
168  )
169  cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
170  doc="set if the object is a star")
171  schema.getAliasMap().set("slot_Shape", "truth")
172  schema.getAliasMap().set("slot_Centroid", "truth")
173  schema.getAliasMap().set("slot_ModelFlux", "truth")
174  schema.getCitizen().markPersistent()
175  cls._schema = schema
176  schema = lsst.afw.table.Schema(cls._schema)
177  schema.disconnectAliases()
178  return schema
179 
180  @staticmethod
181  def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
182  minRotation=None, maxRotation=None,
183  minRefShift=None, maxRefShift=None,
184  minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
185  """!
186  Create a new undistorted TAN WCS that is similar but not identical to another, with random
187  scaling, rotation, and offset (in both pixel position and reference position).
188 
189  The maximum and minimum arguments are interpreted as absolute values for a split
190  range that covers both positive and negative values (as this method is used
191  in testing, it is typically most important to avoid perturbations near zero).
192  Scale factors are treated somewhat differently: the actual scale factor is chosen between
193  minScaleFactor and maxScaleFactor OR (1/maxScaleFactor) and (1/minScaleFactor).
194 
195  The default range for rotation is 30-60 degrees, and the default range for reference shift
196  is 0.5-1.0 arcseconds (these cannot be safely included directly as default values because Angle
197  objects are mutable).
198 
199  The random number generator is primed with the seed given. If ``None``, a seed is
200  automatically chosen.
201  """
202  random_state = np.random.RandomState(randomSeed)
203  if minRotation is None:
204  minRotation = 30.0*lsst.geom.degrees
205  if maxRotation is None:
206  maxRotation = 60.0*lsst.geom.degrees
207  if minRefShift is None:
208  minRefShift = 0.5*lsst.geom.arcseconds
209  if maxRefShift is None:
210  maxRefShift = 1.0*lsst.geom.arcseconds
211 
212  def splitRandom(min1, max1, min2=None, max2=None):
213  if min2 is None:
214  min2 = -max1
215  if max2 is None:
216  max2 = -min1
217  if random_state.uniform() > 0.5:
218  return float(random_state.uniform(min1, max1))
219  else:
220  return float(random_state.uniform(min2, max2))
221  # Generate random perturbations
222  scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
223  rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
224  refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
225  refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
226  pixShiftX = splitRandom(minPixShift, maxPixShift)
227  pixShiftY = splitRandom(minPixShift, maxPixShift)
228  # Compute new CD matrix
229  oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
230  rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
231  sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
232  newTransform = oldTransform*rTransform*sTransform
233  matrix = newTransform.getMatrix()
234  # Compute new coordinate reference pixel (CRVAL)
235  oldSkyOrigin = oldWcs.getSkyOrigin()
236  newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
237  oldSkyOrigin.getDec() + refShiftDec)
238  # Compute new pixel reference pixel (CRPIX)
239  oldPixOrigin = oldWcs.getPixelOrigin()
240  newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
241  oldPixOrigin.getY() + pixShiftY)
242  return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
243 
244  @staticmethod
245  def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12):
246  """!
247  Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
248 
249  @param[in] bbox Bounding box of the image (image coordinates) as returned by makeCatalog.
250  @param[in] wcs New Wcs for the exposure (created from crval and cdelt if None).
251  @param[in] crval afw.geom.SpherePoint: ICRS center of the TAN WCS attached to the image.
252  @param[in] cdelt afw.geom.Angle: pixel scale of the image
253  @param[in] psfSigma Radius (sigma) of the Gaussian PSF attached to the image
254  @param[in] psfDim Width and height of the image's Gaussian PSF attached to the image
255  @param[in] fluxMag0 Flux at magnitude zero (in e-) used to set the Calib of the exposure.
256  """
257  if wcs is None:
258  if crval is None:
259  crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
260  if cdelt is None:
261  cdelt = 0.2*lsst.geom.arcseconds
262  crpix = lsst.geom.Box2D(bbox).getCenter()
263  wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
264  cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
265  exposure = lsst.afw.image.ExposureF(bbox)
266  psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
267  calib = lsst.afw.image.Calib()
268  calib.setFluxMag0(fluxMag0)
269  exposure.setWcs(wcs)
270  exposure.setPsf(psf)
271  exposure.setCalib(calib)
272  return exposure
273 
274  @staticmethod
275  def drawGaussian(bbox, flux, ellipse):
276  """!
277  Create an image of an elliptical Gaussian.
278 
279  @param[in,out] bbox Bounding box of image to create.
280  @param[in] flux Total flux of the Gaussian (normalized analytically, not using pixel
281  values)
282  @param[in] ellipse lsst.afw.geom.Ellipse holding the centroid and shape.
283  """
284  x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
285  np.arange(bbox.getBeginY(), bbox.getEndY()))
286  t = ellipse.getGridTransform()
287  xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
288  yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
289  image = lsst.afw.image.ImageF(bbox)
290  image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*flux/(2.0*ellipse.getCore().getArea())
291  return image
292 
293  def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
294  """!
295  Initialize the dataset.
296 
297  @param[in] bbox Bounding box of the test image.
298  @param[in] threshold Threshold absolute value used to determine footprints for
299  simulated sources. This thresholding will be applied before noise is
300  actually added to images (or before the noise level is even known), so
301  this will necessarily produce somewhat artificial footprints.
302  @param[in] exposure lsst.afw.image.ExposureF test sources should be added to. Ownership should
303  be considered transferred from the caller to the TestDataset.
304  Must have a Gaussian Psf for truth catalog shapes to be exact.
305  @param[in] **kwds Keyword arguments forwarded to makeEmptyExposure if exposure is None.
306  """
307  if exposure is None:
308  exposure = self.makeEmptyExposure(bbox, **kwds)
309  self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
310  self.exposure = exposure
311  self.psfShape = self.exposure.getPsf().computeShape()
312  self.schema = self.makeMinimalSchema()
314 
315  def _installFootprint(self, record, image):
316  """Create a Footprint for a simulated source and add it to its truth catalog record.
317  """
318  # Run detection on the single-source image
319  fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
320  # the call below to the FootprintSet ctor is actually a grow operation
321  fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
322  # Update the full exposure's mask plane to indicate the detection
323  fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
324  # Attach the new footprint to the exposure
325  if len(fpSet.getFootprints()) > 1:
326  raise RuntimeError("Threshold value results in multiple Footprints for a single object")
327  if len(fpSet.getFootprints()) == 0:
328  raise RuntimeError("Threshold value results in zero Footprints for object")
329  record.setFootprint(fpSet.getFootprints()[0])
330 
331  def addSource(self, flux, centroid, shape=None):
332  """!
333  Add a source to the simulation
334 
335  @param[in] flux Total flux of the source to be added.
336  @param[in] centroid Position of the source to be added (lsst.geom.Point2D).
337  @param[in] shape 2nd moments of the source before PSF convolution
338  (lsst.afw.geom.Quadrupole). Note that the truth catalog
339  records post-convolution moments). If None, a point source
340  will be added.
341 
342  @return a truth catalog record and single-source image corresponding to the new source.
343  """
344  # Create and set the truth catalog fields
345  record = self.catalog.addNew()
346  record.set(self.keys["flux"], flux)
347  record.set(self.keys["centroid"], centroid)
348  covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
349  covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
350  record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
351  if shape is None:
352  record.set(self.keys["isStar"], True)
353  fullShape = self.psfShape
354  else:
355  record.set(self.keys["isStar"], False)
356  fullShape = shape.convolve(self.psfShape)
357  record.set(self.keys["shape"], fullShape)
358  # Create an image containing just this source
359  image = self.drawGaussian(self.exposure.getBBox(), flux,
360  lsst.afw.geom.Ellipse(fullShape, centroid))
361  # Generate a footprint for this source
362  self._installFootprint(record, image)
363  # Actually add the source to the full exposure
364  self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
365  return record, image
366 
367  def addBlend(self):
368  """!
369  Return a context manager that allows a blend of multiple sources to be added.
370 
371  Example:
372  @code
373  d = TestDataset(...)
374  with d.addBlend() as b:
375  b.addChild(flux1, centroid1)
376  b.addChild(flux2, centroid2, shape2)
377  @endcode
378 
379  Note that nothing stops you from creating overlapping sources just using the addSource() method,
380  but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
381  produced by the detection and deblending pipelines.
382  """
383  return BlendContext(self)
384 
385  def transform(self, wcs, **kwds):
386  """!
387  Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
388 
389  @param[in] wcs Wcs for the new dataset.
390  @param[in] **kwds Additional keyword arguments passed on to makeEmptyExposure. If not
391  specified, these revert to the defaults for makeEmptyExposure, not the
392  values in the current dataset.
393  """
394  bboxD = lsst.geom.Box2D()
395  xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
396  for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
397  bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
398  bboxI = lsst.geom.Box2I(bboxD)
399  result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
400  oldCalib = self.exposure.getCalib()
401  newCalib = result.exposure.getCalib()
402  oldPsfShape = self.exposure.getPsf().computeShape()
403  for record in self.catalog:
404  if record.get(self.keys["nChild"]):
405  raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
406  magnitude = oldCalib.getMagnitude(record.get(self.keys["flux"]))
407  newFlux = newCalib.getFlux(magnitude)
408  oldCentroid = record.get(self.keys["centroid"])
409  newCentroid = xyt.applyForward(oldCentroid)
410  if record.get(self.keys["isStar"]):
411  newDeconvolvedShape = None
412  else:
413  affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
414  oldFullShape = record.get(self.keys["shape"])
415  oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
416  oldFullShape.getIxx() - oldPsfShape.getIxx(),
417  oldFullShape.getIyy() - oldPsfShape.getIyy(),
418  oldFullShape.getIxy() - oldPsfShape.getIxy(),
419  False
420  )
421  newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
422  result.addSource(newFlux, newCentroid, newDeconvolvedShape)
423  return result
424 
425  def realize(self, noise, schema, randomSeed=1):
426  """!
427  Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
428 
429  @param[in] noise Standard deviation of noise to be added to the exposure. The noise will be
430  Gaussian and constant, appropriate for the sky-limited regime.
431  @param[in] schema Schema of the new catalog to be created. Must start with self.schema (i.e.
432  schema.contains(self.schema) must be True), but typically contains fields for
433  already-configured measurement algorithms as well.
434  @param[in] randomSeed Seed for the random number generator. If None, a seed is chosen automatically.
435 
436  @return a tuple of (exposure, catalog)
437  """
438  random_state = np.random.RandomState(randomSeed)
439  assert schema.contains(self.schema)
440  mapper = lsst.afw.table.SchemaMapper(self.schema)
441  mapper.addMinimalSchema(self.schema, True)
442  exposure = self.exposure.clone()
443  exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
444  exposure.getMaskedImage().getImage().getArray()[:, :] \
445  += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise
446  catalog = lsst.afw.table.SourceCatalog(schema)
447  catalog.extend(self.catalog, mapper=mapper)
448  # Loop over sources and generate new HeavyFootprints that divide up the noisy pixels, not the
449  # ideal no-noise pixels.
450  for record in catalog:
451  # parent objects have non-Heavy Footprints, which don't need to be updated after adding noise.
452  if record.getParent() == 0:
453  continue
454  # get flattened arrays that correspond to the no-noise and noisy parent images
455  parent = catalog.find(record.getParent())
456  footprint = parent.getFootprint()
457  parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
458  footprint.spans.flatten(parentFluxArrayNoNoise,
459  self.exposure.getMaskedImage().getImage().getArray(),
460  self.exposure.getXY0())
461  parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
462  footprint.spans.flatten(parentFluxArrayNoisy,
463  exposure.getMaskedImage().getImage().getArray(),
464  exposure.getXY0())
465  oldHeavy = record.getFootprint()
466  fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
467  # n.b. this isn't a copy ctor - it's a copy from a vanilla Footprint, so it doesn't copy
468  # the arrays we don't want to change, and hence we have to do that ourselves below.
469  newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
470  newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
471  newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
472  newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
473  record.setFootprint(newHeavy)
474  return exposure, catalog
475 
476 
478 
479  def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
480  """Convenience function to create a Config instance for SingleFrameMeasurementTask
481 
482  The plugin and its dependencies will be the only plugins run, while the Centroid, Shape,
483  and ModelFlux slots will be set to the truth fields generated by the TestDataset class.
484  """
485  config = SingleFrameMeasurementTask.ConfigClass()
486  config.slots.centroid = "truth"
487  config.slots.shape = "truth"
488  config.slots.modelFlux = None
489  config.slots.apFlux = None
490  config.slots.psfFlux = None
491  config.slots.instFlux = None
492  config.slots.calibFlux = None
493  config.plugins.names = (plugin,) + tuple(dependencies)
494  return config
495 
496  def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
497  algMetadata=None):
498  """Convenience function to create a SingleFrameMeasurementTask with a simple configuration.
499  """
500  if config is None:
501  if plugin is None:
502  raise ValueError("Either plugin or config argument must not be None")
503  config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
504  if schema is None:
505  schema = TestDataset.makeMinimalSchema()
506  # Clear all aliases so only those defined by config are set.
507  schema.setAliasMap(None)
508  if algMetadata is None:
509  algMetadata = lsst.daf.base.PropertyList()
510  return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
511 
512  def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
513  """Convenience function to create a Config instance for ForcedMeasurementTask
514 
515  In addition to the plugins specified in the plugin and dependencies arguments,
516  the TransformedCentroid and TransformedShape plugins will be run and used as the
517  Centroid and Shape slots; these simply transform the reference catalog centroid
518  and shape to the measurement coordinate system.
519  """
520  config = ForcedMeasurementTask.ConfigClass()
521  config.slots.centroid = "base_TransformedCentroid"
522  config.slots.shape = "base_TransformedShape"
523  config.slots.modelFlux = None
524  config.slots.apFlux = None
525  config.slots.psfFlux = None
526  config.slots.instFlux = None
527  config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
528  "base_TransformedShape")
529  return config
530 
531  def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
532  algMetadata=None):
533  """Convenience function to create a ForcedMeasurementTask with a simple configuration.
534  """
535  if config is None:
536  if plugin is None:
537  raise ValueError("Either plugin or config argument must not be None")
538  config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
539  if refSchema is None:
540  refSchema = TestDataset.makeMinimalSchema()
541  if algMetadata is None:
542  algMetadata = lsst.daf.base.PropertyList()
543  return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
544 
545 
547  """!
548  Base class for testing measurement transformations.
549 
550  We test both that the transform itself operates successfully (fluxes are
551  converted to magnitudes, flags are propagated properly) and that the
552  transform is registered as the default for the appropriate measurement
553  algorithms.
554 
555  In the simple case of one-measurement-per-transformation, the developer
556  need not directly write any tests themselves: simply customizing the class
557  variables is all that is required. More complex measurements (e.g.
558  multiple aperture fluxes) require extra effort.
559  """
560  # The name used for the measurement algorithm; determines the names of the
561  # fields in the resulting catalog. This default should generally be fine,
562  # but subclasses can override if required.
563  name = "MeasurementTransformTest"
564 
565  # These should be customized by subclassing.
566  controlClass = None
567  algorithmClass = None
568  transformClass = None
569 
570  # Flags which may be set by the algorithm being tested. Can be customized
571  # in subclasses.
572  flagNames = ("flag",)
573 
574  # The plugin being tested should be registered under these names for
575  # single frame and forced measurement. Should be customized by
576  # subclassing.
577  singleFramePlugins = ()
578  forcedPlugins = ()
579 
580  def setUp(self):
581  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200))
582  self.calexp = TestDataset.makeEmptyExposure(bbox)
583  self._setupTransform()
584 
585  def tearDown(self):
586  del self.calexp
587  del self.inputCat
588  del self.mapper
589  del self.transform
590  del self.outputCat
591 
592  def _populateCatalog(self, baseNames):
593  records = []
594  for flagValue in (True, False):
595  records.append(self.inputCat.addNew())
596  for baseName in baseNames:
597  for flagName in self.flagNames:
598  if records[-1].schema.join(baseName, flagName) in records[-1].schema:
599  records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
600  self._setFieldsInRecords(records, baseName)
601 
602  def _checkOutput(self, baseNames):
603  for inSrc, outSrc in zip(self.inputCat, self.outputCat):
604  for baseName in baseNames:
605  self._compareFieldsInRecords(inSrc, outSrc, baseName)
606  for flagName in self.flagNames:
607  keyName = outSrc.schema.join(baseName, flagName)
608  if keyName in inSrc.schema:
609  self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
610  else:
611  self.assertFalse(keyName in outSrc.schema)
612 
613  def _runTransform(self, doExtend=True):
614  if doExtend:
615  self.outputCat.extend(self.inputCat, mapper=self.mapper)
616  self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getCalib())
617 
618  def testTransform(self, baseNames=None):
619  """
620  Test the operation of the transformation on a catalog containing random data.
621 
622  We check that:
623 
624  * An appropriate exception is raised on an attempt to transform between catalogs with different
625  numbers of rows;
626  * Otherwise, all appropriate conversions are properly appled and that flags have been propagated.
627 
628  The `baseNames` argument requires some explanation. This should be an iterable of the leading parts of
629  the field names for each measurement; that is, everything that appears before `_flux`, `_flag`, etc.
630  In the simple case of a single measurement per plugin, this is simply equal to `self.name` (thus
631  measurements are stored as `self.name + "_flux"`, etc). More generally, the developer may specify
632  whatever iterable they require. For example, to handle multiple apertures, we could have
633  `(self.name + "_0", self.name + "_1", ...)`.
634 
635  @param[in] baseNames Iterable of the initial parts of measurement field names.
636  """
637  baseNames = baseNames or [self.name]
638  self._populateCatalog(baseNames)
639  self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
640  self._runTransform()
641  self._checkOutput(baseNames)
642 
643  def _checkRegisteredTransform(self, registry, name):
644  # If this is a Python-based transform, we can compare directly; if
645  # it's wrapped C++, we need to compare the wrapped class.
646  self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
647 
648  def testRegistration(self):
649  """
650  Test that the transformation is appropriately registered with the relevant measurement algorithms.
651  """
652  for pluginName in self.singleFramePlugins:
653  self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
654  for pluginName in self.forcedPlugins:
655  self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
656 
657 
659 
660  def _setupTransform(self):
661  self.control = self.controlClass()
663  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
664  # it doesn't matter for this test since we won't actually use the plugins for anything besides
665  # defining the schema.
666  inputSchema.getAliasMap().set("slot_Centroid", "dummy")
667  inputSchema.getAliasMap().set("slot_Shape", "dummy")
668  self.algorithmClass(self.control, self.name, inputSchema)
669  inputSchema.getAliasMap().erase("slot_Centroid")
670  inputSchema.getAliasMap().erase("slot_Shape")
673  self.transform = self.transformClass(self.control, self.name, self.mapper)
674  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
675 
676 
678 
679  def _setupTransform(self):
680  self.control = self.controlClass()
683  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
684  # it doesn't matter for this test since we won't actually use the plugins for anything besides
685  # defining the schema.
686  inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
687  inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
688  self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
689  inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
690  inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
691  self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
692  self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
693  self.transform = self.transformClass(self.control, self.name, self.mapper)
694  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
695 
696 
698 
699  def _setFieldsInRecords(self, records, name):
700  for record in records:
701  record[record.schema.join(name, 'flux')] = np.random.random()
702  record[record.schema.join(name, 'fluxSigma')] = np.random.random()
703 
704  # Negative fluxes should be converted to NaNs.
705  assert len(records) > 1
706  records[0][record.schema.join(name, 'flux')] = -1
707 
708  def _compareFieldsInRecords(self, inSrc, outSrc, name):
709  fluxName, fluxSigmaName = inSrc.schema.join(name, 'flux'), inSrc.schema.join(name, 'fluxSigma')
710  if inSrc[fluxName] > 0:
711  mag, magErr = self.calexp.getCalib().getMagnitude(inSrc[fluxName], inSrc[fluxSigmaName])
712  self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag)
713  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], magErr)
714  else:
715  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
716  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
717 
718 
720 
721  def _setFieldsInRecords(self, records, name):
722  for record in records:
723  record[record.schema.join(name, 'x')] = np.random.random()
724  record[record.schema.join(name, 'y')] = np.random.random()
725  # Some algorithms set no errors; some set only sigma on x & y; some provide
726  # a full covariance matrix. Set only those which exist in the schema.
727  for fieldSuffix in ('xSigma', 'ySigma', 'x_y_Cov'):
728  fieldName = record.schema.join(name, fieldSuffix)
729  if fieldName in record.schema:
730  record[fieldName] = np.random.random()
731 
732  def _compareFieldsInRecords(self, inSrc, outSrc, name):
733  centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
734  centroidResult = centroidResultKey.get(inSrc)
735 
736  coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
737  coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
738  self.assertEqual(coordTruth, coord)
739 
740  # If the centroid has an associated uncertainty matrix, the coordinate
741  # must have one too, and vice versa.
742  try:
743  coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
744  ["ra", "dec"]).get(outSrc)
746  self.assertFalse(centroidResultKey.getCentroidErr().isValid())
747  else:
748  transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
749  coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
750  centroidResult.getCentroidErr()),
751  transform.getLinear().getMatrix().transpose())
752  np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
def _runTransform(self, doExtend=True)
Definition: tests.py:613
def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:479
lsst::geom::AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, lsst::geom::Point2D const &inPoint)
def transform(self, wcs, kwds)
Create a copy of the dataset transformed to a new WCS, with new Psf and Calib.
Definition: tests.py:385
def __init__(self, bbox, threshold=10.0, exposure=None, kwds)
Initialize the dataset.
Definition: tests.py:293
def _checkOutput(self, baseNames)
Definition: tests.py:602
A subtask for measuring the properties of sources on a single exposure, using an existing "reference"...
A subtask for measuring the properties of sources on a single exposure.
Definition: sfm.py:152
def addBlend(self)
Return a context manager that allows a blend of multiple sources to be added.
Definition: tests.py:367
def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, fluxMag0=1E12)
Create an Exposure, with a Calib, Wcs, and Psf, but no pixel values set.
Definition: tests.py:245
def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition: tests.py:497
def realize(self, noise, schema, randomSeed=1)
Create a simulated with noise and a simulated post-detection catalog with (Heavy)Footprints.
Definition: tests.py:425
def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=1)
Create a new undistorted TAN WCS that is similar but not identical to another, with random scaling...
Definition: tests.py:184
def __init__(self, owner)
Definition: tests.py:51
def testTransform(self, baseNames=None)
Definition: tests.py:618
def drawGaussian(bbox, flux, ellipse)
Create an image of an elliptical Gaussian.
Definition: tests.py:275
def _installFootprint(self, record, image)
Definition: tests.py:315
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
def _checkRegisteredTransform(self, registry, name)
Definition: tests.py:643
def _populateCatalog(self, baseNames)
Definition: tests.py:592
def addSource(self, flux, centroid, shape=None)
Add a source to the simulation.
Definition: tests.py:331
static Schema makeMinimalSchema()
A simulated dataset consisting of a test image and an associated truth catalog.
Definition: tests.py:120
static LinearTransform makeScaling(double s) noexcept
def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition: tests.py:532
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
std::shared_ptr< SkyWcs > makeSkyWcs(TransformPoint2ToPoint2 const &pixelsToFieldAngle, lsst::geom::Angle const &orientation, bool flipX, lsst::geom::SpherePoint const &boresight, std::string const &projection="TAN")
Eigen::Matrix2d makeCdMatrix(lsst::geom::Angle const &scale, lsst::geom::Angle const &orientation=0 *lsst::geom::degrees, bool flipX=false)
static LinearTransform makeRotation(Angle t) noexcept
def addChild(self, flux, centroid, shape=None)
Add a child source to the blend, and return the truth catalog record that corresponds to it...
Definition: tests.py:61
Base class for testing measurement transformations.
Definition: tests.py:546
def __exit__(self, type_, value, tb)
Definition: tests.py:77
def makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:512
A Python context manager used to add multiple overlapping sources along with a parent source that rep...
Definition: tests.py:42