lsst.meas.base  17.0.1-7-g35889ee
tests.py
Go to the documentation of this file.
1 # This file is part of meas_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import numpy as np
23 
24 import lsst.geom
25 import lsst.afw.table
26 import lsst.afw.image
27 import lsst.afw.detection
28 import lsst.afw.geom
30 
31 from .sfm import SingleFrameMeasurementTask
32 from .forcedMeasurement import ForcedMeasurementTask
33 from . import CentroidResultKey
34 
35 __all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase",
36  "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper",
37  "FluxTransformTestCase", "CentroidTransformTestCase")
38 
39 
41  """Context manager which adds multiple overlapping sources and a parent.
42 
43  Notes
44  -----
45  This is used as the return value for `TestDataset.addBlend`, and this is
46  the only way it should be used.
47  """
48 
49  def __init__(self, owner):
50  self.owner = owner
51  self.parentRecord = self.owner.catalog.addNew()
52  self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox())
53  self.children = []
54 
55  def __enter__(self):
56  # BlendContext is its own context manager, so we just return self.
57  return self
58 
59  def addChild(self, instFlux, centroid, shape=None):
60  """Add a child to the blend; return corresponding truth catalog record.
61 
62  instFlux : `float`
63  Total instFlux of the source to be added.
64  centroid : `lsst.geom.Point2D`
65  Position of the source to be added.
66  shape : `lsst.afw.geom.Quadrupole`
67  Second moments of the source before PSF convolution. Note that
68  the truth catalog records post-convolution moments)
69  """
70  record, image = self.owner.addSource(instFlux, centroid, shape)
71  record.set(self.owner.keys["parent"], self.parentRecord.getId())
72  self.parentImage += image
73  self.children.append((record, image))
74  return record
75 
76  def __exit__(self, type_, value, tb):
77  # We're not using the context manager for any kind of exception safety
78  # or guarantees; we just want the nice "with" statement syntax.
79 
80  if type_ is not None:
81  # exception was raised; just skip all this and let it propagate
82  return
83 
84  # On exit, compute and set the truth values for the parent object.
85  self.parentRecord.set(self.owner.keys["nChild"], len(self.children))
86  # Compute instFlux from sum of component fluxes
87  instFlux = 0.0
88  for record, image in self.children:
89  instFlux += record.get(self.owner.keys["instFlux"])
90  self.parentRecord.set(self.owner.keys["instFlux"], instFlux)
91  # Compute centroid from instFlux-weighted mean of component centroids
92  x = 0.0
93  y = 0.0
94  for record, image in self.children:
95  w = record.get(self.owner.keys["instFlux"])/instFlux
96  x += record.get(self.owner.keys["centroid"].getX())*w
97  y += record.get(self.owner.keys["centroid"].getY())*w
98  self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y))
99  # Compute shape from instFlux-weighted mean of offset component shapes
100  xx = 0.0
101  yy = 0.0
102  xy = 0.0
103  for record, image in self.children:
104  w = record.get(self.owner.keys["instFlux"])/instFlux
105  dx = record.get(self.owner.keys["centroid"].getX()) - x
106  dy = record.get(self.owner.keys["centroid"].getY()) - y
107  xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w
108  yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w
109  xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w
110  self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy))
111  # Run detection on the parent image to get the parent Footprint.
112  self.owner._installFootprint(self.parentRecord, self.parentImage)
113  # Create perfect HeavyFootprints for all children; these will need to
114  # be modified later to account for the noise we'll add to the image.
115  deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True)
116  for record, image in self.children:
117  deblend.getImage().getArray()[:, :] = image.getArray()
118  heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend)
119  record.setFootprint(heavyFootprint)
120 
121 
123  """A simulated dataset consisuting of test image and truth catalog.
124 
125  TestDataset creates an idealized image made of pure Gaussians (including a
126  Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints
127  that simulated the outputs of detection and deblending. Multiple noise
128  realizations can be created from the same underlying sources, allowing
129  uncertainty estimates to be verified via Monte Carlo.
130 
131  Parameters
132  ----------
133  bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
134  Bounding box of the test image.
135  threshold : `float`
136  Threshold absolute value used to determine footprints for
137  simulated sources. This thresholding will be applied before noise is
138  actually added to images (or before the noise level is even known), so
139  this will necessarily produce somewhat artificial footprints.
140  exposure : `lsst.afw.image.ExposureF`
141  The image to which test sources should be added. Ownership should
142  be considered transferred from the caller to the TestDataset.
143  Must have a Gaussian PSF for truth catalog shapes to be exact.
144  **kwds
145  Keyword arguments forwarded to makeEmptyExposure if exposure is `None`.
146 
147  Notes
148  -----
149  Typical usage:
150 
151  .. code-block: py
152 
153  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100,
154  100))
155  dataset = TestDataset(bbox)
156  dataset.addSource(flux=1E5, centroid=lsst.geom.Point2D(25, 26))
157  dataset.addSource(flux=2E5, centroid=lsst.geom.Point2D(75, 24),
158  shape=lsst.afw.geom.Quadrupole(8, 7, 2))
159  with dataset.addBlend() as family:
160  family.addChild(flux=2E5, centroid=lsst.geom.Point2D(50, 72))
161  family.addChild(flux=1.5E5, centroid=lsst.geom.Point2D(51, 74))
162  exposure, catalog = dataset.realize(noise=100.0,
163  schema=TestDataset.makeMinimalSchema())
164  """
165 
166  @classmethod
168  """Return the minimal schema needed to hold truth catalog fields.
169 
170  Notes
171  -----
172  When `TestDataset.realize` is called, the schema must include at least
173  these fields. Usually it will include additional fields for
174  measurement algorithm outputs, allowing the same catalog to be used
175  for both truth values (the fields from the minimal schema) and the
176  measurements.
177  """
178  if not hasattr(cls, "_schema"):
180  cls.keys = {}
181  cls.keys["parent"] = schema.find("parent").key
182  cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32)
183  cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64,
184  doc="true instFlux", units="count")
185  cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields(
186  schema, "truth", "true simulated centroid", "pixel"
187  )
188  cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields(
189  schema, "truth", ['x', 'y'], "pixel"
190  )
191  cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag",
192  doc="set if the object is a star")
194  schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL
195  )
196  cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag",
197  doc="set if the object is a star")
198  schema.getAliasMap().set("slot_Shape", "truth")
199  schema.getAliasMap().set("slot_Centroid", "truth")
200  schema.getAliasMap().set("slot_ModelFlux", "truth")
201  schema.getCitizen().markPersistent()
202  cls._schema = schema
203  schema = lsst.afw.table.Schema(cls._schema)
204  schema.disconnectAliases()
205  return schema
206 
207  @staticmethod
208  def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5,
209  minRotation=None, maxRotation=None,
210  minRefShift=None, maxRefShift=None,
211  minPixShift=2.0, maxPixShift=4.0, randomSeed=1):
212  """Return a perturbed version of the input WCS.
213 
214  Create a new undistorted TAN WCS that is similar but not identical to
215  another, with random scaling, rotation, and offset (in both pixel
216  position and reference position).
217 
218  Parameters
219  ----------
220  oldWcs : `lsst.afw.geom.SkyWcs`
221  The input WCS.
222  minScaleFactor : `float`
223  Minimum scale factor to apply to the input WCS.
224  maxScaleFactor : `float`
225  Maximum scale factor to apply to the input WCS.
226  minRotation : `lsst.geom.Angle` or `None`
227  Minimum rotation to apply to the input WCS. If `None`, defaults to
228  30 degrees.
229  maxRotation : `lsst.geom.Angle` or `None`
230  Minimum rotation to apply to the input WCS. If `None`, defaults to
231  60 degrees.
232  minRefShift : `lsst.geom.Angle` or `None`
233  Miniumum shift to apply to the input WCS reference value. If
234  `None`, defaults to 0.5 arcsec.
235  maxRefShift : `lsst.geom.Angle` or `None`
236  Miniumum shift to apply to the input WCS reference value. If
237  `None`, defaults to 1.0 arcsec.
238  minPixShift : `float`
239  Minimum shift to apply to the input WCS reference pixel.
240  maxPixShift : `float`
241  Maximum shift to apply to the input WCS reference pixel.
242  randomSeed : `int`
243  Random seed.
244 
245  Returns
246  -------
247  newWcs : `lsst.afw.geom.SkyWcs`
248  A perturbed version of the input WCS.
249 
250  Notes
251  -----
252  The maximum and minimum arguments are interpreted as absolute values
253  for a split range that covers both positive and negative values (as
254  this method is used in testing, it is typically most important to
255  avoid perturbations near zero). Scale factors are treated somewhat
256  differently: the actual scale factor is chosen between
257  ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``)
258  and (``1/minScaleFactor``).
259 
260  The default range for rotation is 30-60 degrees, and the default range
261  for reference shift is 0.5-1.0 arcseconds (these cannot be safely
262  included directly as default values because Angle objects are
263  mutable).
264 
265  The random number generator is primed with the seed given. If
266  `None`, a seed is automatically chosen.
267  """
268  random_state = np.random.RandomState(randomSeed)
269  if minRotation is None:
270  minRotation = 30.0*lsst.geom.degrees
271  if maxRotation is None:
272  maxRotation = 60.0*lsst.geom.degrees
273  if minRefShift is None:
274  minRefShift = 0.5*lsst.geom.arcseconds
275  if maxRefShift is None:
276  maxRefShift = 1.0*lsst.geom.arcseconds
277 
278  def splitRandom(min1, max1, min2=None, max2=None):
279  if min2 is None:
280  min2 = -max1
281  if max2 is None:
282  max2 = -min1
283  if random_state.uniform() > 0.5:
284  return float(random_state.uniform(min1, max1))
285  else:
286  return float(random_state.uniform(min2, max2))
287  # Generate random perturbations
288  scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor)
289  rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians
290  refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
291  refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians
292  pixShiftX = splitRandom(minPixShift, maxPixShift)
293  pixShiftY = splitRandom(minPixShift, maxPixShift)
294  # Compute new CD matrix
295  oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix())
296  rTransform = lsst.geom.LinearTransform.makeRotation(rotation)
297  sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor)
298  newTransform = oldTransform*rTransform*sTransform
299  matrix = newTransform.getMatrix()
300  # Compute new coordinate reference pixel (CRVAL)
301  oldSkyOrigin = oldWcs.getSkyOrigin()
302  newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa,
303  oldSkyOrigin.getDec() + refShiftDec)
304  # Compute new pixel reference pixel (CRPIX)
305  oldPixOrigin = oldWcs.getPixelOrigin()
306  newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX,
307  oldPixOrigin.getY() + pixShiftY)
308  return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix)
309 
310  @staticmethod
311  def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4):
312  """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values.
313 
314  Parameters
315  ----------
316  bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
317  Bounding box of the image in image coordinates.
318  wcs : `lsst.afw.geom.SkyWcs`, optional
319  New WCS for the exposure (created from CRVAL and CDELT if `None`).
320  crval : `lsst.afw.geom.SpherePoint`, optional
321  ICRS center of the TAN WCS attached to the image. If `None`, (45
322  degrees, 45 degrees) is assumed.
323  cdelt : `lsst.geom.Angle`, optional
324  Pixel scale of the image. If `None`, 0.2 arcsec is assumed.
325  psfSigma : `float`, optional
326  Radius (sigma) of the Gaussian PSF attached to the image
327  psfDim : `int`, optional
328  Width and height of the image's Gaussian PSF attached to the image
329  calibration : `float`, optional
330  The spatially-constant calibration (in nJy/count) to set the
331  PhotoCalib of the exposure.
332 
333  Returns
334  -------
335  exposure : `lsst.age.image.ExposureF`
336  An empty image.
337  """
338  if wcs is None:
339  if crval is None:
340  crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees)
341  if cdelt is None:
342  cdelt = 0.2*lsst.geom.arcseconds
343  crpix = lsst.geom.Box2D(bbox).getCenter()
344  wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval,
345  cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt))
346  exposure = lsst.afw.image.ExposureF(bbox)
347  psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma)
348  photoCalib = lsst.afw.image.PhotoCalib(calibration)
349  exposure.setWcs(wcs)
350  exposure.setPsf(psf)
351  exposure.setPhotoCalib(photoCalib)
352  return exposure
353 
354  @staticmethod
355  def drawGaussian(bbox, instFlux, ellipse):
356  """Create an image of an elliptical Gaussian.
357 
358  Parameters
359  ----------
360  bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
361  Bounding box of image to create.
362  instFlux : `float`
363  Total instrumental flux of the Gaussian (normalized analytically,
364  not using pixel values).
365  ellipse : `lsst.afw.geom.Ellipse`
366  Defines the centroid and shape.
367 
368  Returns
369  -------
370  image : `lsst.afw.image.ImageF`
371  An image of the Gaussian.
372  """
373  x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()),
374  np.arange(bbox.getBeginY(), bbox.getEndY()))
375  t = ellipse.getGridTransform()
376  xt = t[t.XX] * x + t[t.XY] * y + t[t.X]
377  yt = t[t.YX] * x + t[t.YY] * y + t[t.Y]
378  image = lsst.afw.image.ImageF(bbox)
379  image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea())
380  return image
381 
382  def __init__(self, bbox, threshold=10.0, exposure=None, **kwds):
383  if exposure is None:
384  exposure = self.makeEmptyExposure(bbox, **kwds)
385  self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE)
386  self.exposure = exposure
387  self.psfShape = self.exposure.getPsf().computeShape()
388  self.schema = self.makeMinimalSchema()
390 
391  def _installFootprint(self, record, image):
392  """Create simulated Footprint and add it to a truth catalog record.
393  """
394  # Run detection on the single-source image
395  fpSet = lsst.afw.detection.FootprintSet(image, self.threshold)
396  # the call below to the FootprintSet ctor is actually a grow operation
397  fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True)
398  # Update the full exposure's mask plane to indicate the detection
399  fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED")
400  # Attach the new footprint to the exposure
401  if len(fpSet.getFootprints()) > 1:
402  raise RuntimeError("Threshold value results in multiple Footprints for a single object")
403  if len(fpSet.getFootprints()) == 0:
404  raise RuntimeError("Threshold value results in zero Footprints for object")
405  record.setFootprint(fpSet.getFootprints()[0])
406 
407  def addSource(self, instFlux, centroid, shape=None):
408  """Add a source to the simulation.
409 
410  Parameters
411  ----------
412  instFlux : `float`
413  Total instFlux of the source to be added.
414  centroid : `lsst.geom.Point2D`
415  Position of the source to be added.
416  shape : `lsst.afw.geom.Quadrupole`
417  Second moments of the source before PSF convolution. Note that the
418  truth catalog records post-convolution moments. If `None`, a point
419  source will be added.
420 
421  Returns
422  -------
423  record : `lsst.afw.table.SourceRecord`
424  A truth catalog record.
425  image : `lsst.afw.image.ImageF`
426  Single-source image corresponding to the new source.
427  """
428  # Create and set the truth catalog fields
429  record = self.catalog.addNew()
430  record.set(self.keys["instFlux"], instFlux)
431  record.set(self.keys["centroid"], centroid)
432  covariance = np.random.normal(0, 0.1, 4).reshape(2, 2)
433  covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov
434  record.set(self.keys["centroid_sigma"], covariance.astype(np.float32))
435  if shape is None:
436  record.set(self.keys["isStar"], True)
437  fullShape = self.psfShape
438  else:
439  record.set(self.keys["isStar"], False)
440  fullShape = shape.convolve(self.psfShape)
441  record.set(self.keys["shape"], fullShape)
442  # Create an image containing just this source
443  image = self.drawGaussian(self.exposure.getBBox(), instFlux,
444  lsst.afw.geom.Ellipse(fullShape, centroid))
445  # Generate a footprint for this source
446  self._installFootprint(record, image)
447  # Actually add the source to the full exposure
448  self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray()
449  return record, image
450 
451  def addBlend(self):
452  """Return a context manager which can add a blend of multiple sources.
453 
454  Notes
455  -----
456  Note that nothing stops you from creating overlapping sources just using the addSource() method,
457  but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type
458  produced by the detection and deblending pipelines.
459 
460  Examples
461  --------
462  .. code-block: py
463  d = TestDataset(...)
464  with d.addBlend() as b:
465  b.addChild(flux1, centroid1)
466  b.addChild(flux2, centroid2, shape2)
467  """
468  return BlendContext(self)
469 
470  def transform(self, wcs, **kwds):
471  """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib.
472 
473  Parameters
474  ----------
475  wcs : `lsst.afw.geom.SkyWcs`
476  WCS for the new dataset.
477  **kwds
478  Additional keyword arguments passed on to
479  `TestDataset.makeEmptyExposure`. If not specified, these revert
480  to the defaults for `~TestDataset.makeEmptyExposure`, not the
481  values in the current dataset.
482 
483  Returns
484  -------
485  newDataset : `TestDataset`
486  Transformed copy of this dataset.
487  """
488  bboxD = lsst.geom.Box2D()
489  xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs)
490  for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners():
491  bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner)))
492  bboxI = lsst.geom.Box2I(bboxD)
493  result = TestDataset(bbox=bboxI, wcs=wcs, **kwds)
494  oldPhotoCalib = self.exposure.getPhotoCalib()
495  newPhotoCalib = result.exposure.getPhotoCalib()
496  oldPsfShape = self.exposure.getPsf().computeShape()
497  for record in self.catalog:
498  if record.get(self.keys["nChild"]):
499  raise NotImplementedError("Transforming blended sources in TestDatasets is not supported")
500  magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"]))
501  newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude)
502  oldCentroid = record.get(self.keys["centroid"])
503  newCentroid = xyt.applyForward(oldCentroid)
504  if record.get(self.keys["isStar"]):
505  newDeconvolvedShape = None
506  else:
507  affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid)
508  oldFullShape = record.get(self.keys["shape"])
509  oldDeconvolvedShape = lsst.afw.geom.Quadrupole(
510  oldFullShape.getIxx() - oldPsfShape.getIxx(),
511  oldFullShape.getIyy() - oldPsfShape.getIyy(),
512  oldFullShape.getIxy() - oldPsfShape.getIxy(),
513  False
514  )
515  newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear())
516  result.addSource(newFlux, newCentroid, newDeconvolvedShape)
517  return result
518 
519  def realize(self, noise, schema, randomSeed=1):
520  r"""Simulate an exposure and detection catalog for this dataset.
521 
522  The simulation includes noise, and the detection catalog includes
523  `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s.
524 
525  Parameters
526  ----------
527  noise : `float`
528  Standard deviation of noise to be added to the exposure. The
529  noise will be Gaussian and constant, appropriate for the
530  sky-limited regime.
531  schema : `lsst.afw.table.Schema`
532  Schema of the new catalog to be created. Must start with
533  ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be
534  `True`), but typically contains fields for already-configured
535  measurement algorithms as well.
536  randomSeed : `int`, optional
537  Seed for the random number generator.
538  If `None`, a seed is chosen automatically.
539 
540  Returns
541  -------
542  `exposure` : `lsst.afw.image.ExposureF`
543  Simulated image.
544  `catalog` : `lsst.afw.table.SourceCatalog`
545  Simulated detection catalog.
546  """
547  random_state = np.random.RandomState(randomSeed)
548  assert schema.contains(self.schema)
549  mapper = lsst.afw.table.SchemaMapper(self.schema)
550  mapper.addMinimalSchema(self.schema, True)
551  exposure = self.exposure.clone()
552  exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2
553  exposure.getMaskedImage().getImage().getArray()[:, :] \
554  += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise
555  catalog = lsst.afw.table.SourceCatalog(schema)
556  catalog.extend(self.catalog, mapper=mapper)
557  # Loop over sources and generate new HeavyFootprints that divide up
558  # the noisy pixels, not the ideal no-noise pixels.
559  for record in catalog:
560  # parent objects have non-Heavy Footprints, which don't need to be
561  # updated after adding noise.
562  if record.getParent() == 0:
563  continue
564  # get flattened arrays that correspond to the no-noise and noisy
565  # parent images
566  parent = catalog.find(record.getParent())
567  footprint = parent.getFootprint()
568  parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32)
569  footprint.spans.flatten(parentFluxArrayNoNoise,
570  self.exposure.getMaskedImage().getImage().getArray(),
571  self.exposure.getXY0())
572  parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32)
573  footprint.spans.flatten(parentFluxArrayNoisy,
574  exposure.getMaskedImage().getImage().getArray(),
575  exposure.getXY0())
576  oldHeavy = record.getFootprint()
577  fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise)
578  # N.B. this isn't a copy ctor - it's a copy from a vanilla
579  # Footprint, so it doesn't copy the arrays we don't want to
580  # change, and hence we have to do that ourselves below.
581  newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy)
582  newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction
583  newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray()
584  newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray()
585  record.setFootprint(newHeavy)
586  return exposure, catalog
587 
588 
590 
591  def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()):
592  """Create an instance of `SingleFrameMeasurementTask.ConfigClass`.
593 
594  Only the specified plugin and its dependencies will be run; the
595  Centroid, Shape, and ModelFlux slots will be set to the truth fields
596  generated by the `TestDataset` class.
597 
598  Parameters
599  ----------
600  plugin : `str`
601  Name of measurement plugin to enable.
602  dependencies : iterable of `str`, optional
603  Names of dependencies of the measurement plugin.
604 
605  Returns
606  -------
607  config : `SingleFrameMeasurementTask.ConfigClass`
608  The resulting task configuration.
609  """
610  config = SingleFrameMeasurementTask.ConfigClass()
611  config.slots.centroid = "truth"
612  config.slots.shape = "truth"
613  config.slots.modelFlux = None
614  config.slots.apFlux = None
615  config.slots.psfFlux = None
616  config.slots.gaussianFlux = None
617  config.slots.calibFlux = None
618  config.plugins.names = (plugin,) + tuple(dependencies)
619  return config
620 
621  def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None,
622  algMetadata=None):
623  """Create a configured instance of `SingleFrameMeasurementTask`.
624 
625  Parameters
626  ----------
627  plugin : `str`, optional
628  Name of measurement plugin to enable. If `None`, a configuration
629  must be supplied as the ``config`` parameter. If both are
630  specified, ``config`` takes precedence.
631  dependencies : iterable of `str`, optional
632  Names of dependencies of the specified measurement plugin.
633  config : `SingleFrameMeasurementTask.ConfigClass`, optional
634  Configuration for the task. If `None`, a measurement plugin must
635  be supplied as the ``plugin`` paramter. If both are specified,
636  ``config`` takes precedence.
637  schema : `lsst.afw.table.Schema`, optional
638  Measurement table schema. If `None`, a default schema is
639  generated.
640  algMetadata : `lsst.daf.base.PropertyList`, optional
641  Measurement algorithm metadata. If `None`, a default container
642  will be generated.
643 
644  Returns
645  -------
646  task : `SingleFrameMeasurementTask`
647  A configured instance of the measurement task.
648  """
649  if config is None:
650  if plugin is None:
651  raise ValueError("Either plugin or config argument must not be None")
652  config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies)
653  if schema is None:
654  schema = TestDataset.makeMinimalSchema()
655  # Clear all aliases so only those defined by config are set.
656  schema.setAliasMap(None)
657  if algMetadata is None:
658  algMetadata = lsst.daf.base.PropertyList()
659  return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config)
660 
661  def makeForcedMeasurementConfig(self, plugin=None, dependencies=()):
662  """Create an instance of `ForcedMeasurementTask.ConfigClass`.
663 
664  In addition to the plugins specified in the plugin and dependencies
665  arguments, the `TransformedCentroid` and `TransformedShape` plugins
666  will be run and used as the centroid and shape slots; these simply
667  transform the reference catalog centroid and shape to the measurement
668  coordinate system.
669 
670  Parameters
671  ----------
672  plugin : `str`
673  Name of measurement plugin to enable.
674  dependencies : iterable of `str`, optional
675  Names of dependencies of the measurement plugin.
676 
677  Returns
678  -------
679  config : `ForcedMeasurementTask.ConfigClass`
680  The resulting task configuration.
681  """
682 
683  config = ForcedMeasurementTask.ConfigClass()
684  config.slots.centroid = "base_TransformedCentroid"
685  config.slots.shape = "base_TransformedShape"
686  config.slots.modelFlux = None
687  config.slots.apFlux = None
688  config.slots.psfFlux = None
689  config.slots.gaussianFlux = None
690  config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid",
691  "base_TransformedShape")
692  return config
693 
694  def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None,
695  algMetadata=None):
696  """Create a configured instance of `ForcedMeasurementTask`.
697 
698  Parameters
699  ----------
700  plugin : `str`, optional
701  Name of measurement plugin to enable. If `None`, a configuration
702  must be supplied as the ``config`` parameter. If both are
703  specified, ``config`` takes precedence.
704  dependencies : iterable of `str`, optional
705  Names of dependencies of the specified measurement plugin.
706  config : `SingleFrameMeasurementTask.ConfigClass`, optional
707  Configuration for the task. If `None`, a measurement plugin must
708  be supplied as the ``plugin`` paramter. If both are specified,
709  ``config`` takes precedence.
710  refSchema : `lsst.afw.table.Schema`, optional
711  Reference table schema. If `None`, a default schema is
712  generated.
713  algMetadata : `lsst.daf.base.PropertyList`, optional
714  Measurement algorithm metadata. If `None`, a default container
715  will be generated.
716 
717  Returns
718  -------
719  task : `ForcedMeasurementTask`
720  A configured instance of the measurement task.
721  """
722  if config is None:
723  if plugin is None:
724  raise ValueError("Either plugin or config argument must not be None")
725  config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies)
726  if refSchema is None:
727  refSchema = TestDataset.makeMinimalSchema()
728  if algMetadata is None:
729  algMetadata = lsst.daf.base.PropertyList()
730  return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config)
731 
732 
734  """Base class for testing measurement transformations.
735 
736  Notes
737  -----
738  We test both that the transform itself operates successfully (fluxes are
739  converted to magnitudes, flags are propagated properly) and that the
740  transform is registered as the default for the appropriate measurement
741  algorithms.
742 
743  In the simple case of one-measurement-per-transformation, the developer
744  need not directly write any tests themselves: simply customizing the class
745  variables is all that is required. More complex measurements (e.g.
746  multiple aperture fluxes) require extra effort.
747  """
748  name = "MeasurementTransformTest"
749  """The name used for the measurement algorithm (str).
750 
751  Notes
752  -----
753  This determines the names of the fields in the resulting catalog. This
754  default should generally be fine, but subclasses can override if
755  required.
756  """
757 
758  # These should be customized by subclassing.
759  controlClass = None
760  algorithmClass = None
761  transformClass = None
762 
763  flagNames = ("flag",)
764  """Flags which may be set by the algorithm being tested (iterable of `str`).
765  """
766 
767  # The plugin being tested should be registered under these names for
768  # single frame and forced measurement. Should be customized by
769  # subclassing.
770  singleFramePlugins = ()
771  forcedPlugins = ()
772 
773  def setUp(self):
774  bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200))
775  self.calexp = TestDataset.makeEmptyExposure(bbox)
776  self._setupTransform()
777 
778  def tearDown(self):
779  del self.calexp
780  del self.inputCat
781  del self.mapper
782  del self.transform
783  del self.outputCat
784 
785  def _populateCatalog(self, baseNames):
786  records = []
787  for flagValue in (True, False):
788  records.append(self.inputCat.addNew())
789  for baseName in baseNames:
790  for flagName in self.flagNames:
791  if records[-1].schema.join(baseName, flagName) in records[-1].schema:
792  records[-1].set(records[-1].schema.join(baseName, flagName), flagValue)
793  self._setFieldsInRecords(records, baseName)
794 
795  def _checkOutput(self, baseNames):
796  for inSrc, outSrc in zip(self.inputCat, self.outputCat):
797  for baseName in baseNames:
798  self._compareFieldsInRecords(inSrc, outSrc, baseName)
799  for flagName in self.flagNames:
800  keyName = outSrc.schema.join(baseName, flagName)
801  if keyName in inSrc.schema:
802  self.assertEqual(outSrc.get(keyName), inSrc.get(keyName))
803  else:
804  self.assertFalse(keyName in outSrc.schema)
805 
806  def _runTransform(self, doExtend=True):
807  if doExtend:
808  self.outputCat.extend(self.inputCat, mapper=self.mapper)
809  self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib())
810 
811  def testTransform(self, baseNames=None):
812  """Test the transformation on a catalog containing random data.
813 
814  Parameters
815  ----------
816  baseNames : iterable of `str`
817  Iterable of the initial parts of measurement field names.
818 
819  Notes
820  -----
821  We check that:
822 
823  - An appropriate exception is raised on an attempt to transform
824  between catalogs with different numbers of rows;
825  - Otherwise, all appropriate conversions are properly appled and that
826  flags have been propagated.
827 
828  The ``baseNames`` argument requires some explanation. This should be
829  an iterable of the leading parts of the field names for each
830  measurement; that is, everything that appears before ``_instFlux``,
831  ``_flag``, etc. In the simple case of a single measurement per plugin,
832  this is simply equal to ``self.name`` (thus measurements are stored as
833  ``self.name + "_instFlux"``, etc). More generally, the developer may
834  specify whatever iterable they require. For example, to handle
835  multiple apertures, we could have ``(self.name + "_0", self.name +
836  "_1", ...)``.
837  """
838  baseNames = baseNames or [self.name]
839  self._populateCatalog(baseNames)
840  self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False)
841  self._runTransform()
842  self._checkOutput(baseNames)
843 
844  def _checkRegisteredTransform(self, registry, name):
845  # If this is a Python-based transform, we can compare directly; if
846  # it's wrapped C++, we need to compare the wrapped class.
847  self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass)
848 
849  def testRegistration(self):
850  """Test that the transformation is appropriately registered.
851  """
852  for pluginName in self.singleFramePlugins:
853  self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName)
854  for pluginName in self.forcedPlugins:
855  self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName)
856 
857 
859 
860  def _setupTransform(self):
861  self.control = self.controlClass()
863  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
864  # it doesn't matter for this test since we won't actually use the plugins for anything besides
865  # defining the schema.
866  inputSchema.getAliasMap().set("slot_Centroid", "dummy")
867  inputSchema.getAliasMap().set("slot_Shape", "dummy")
868  self.algorithmClass(self.control, self.name, inputSchema)
869  inputSchema.getAliasMap().erase("slot_Centroid")
870  inputSchema.getAliasMap().erase("slot_Shape")
873  self.transform = self.transformClass(self.control, self.name, self.mapper)
874  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
875 
876 
878 
879  def _setupTransform(self):
880  self.control = self.controlClass()
883  # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined;
884  # it doesn't matter for this test since we won't actually use the plugins for anything besides
885  # defining the schema.
886  inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy")
887  inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy")
888  self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList())
889  inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid")
890  inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape")
891  self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema())
892  self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema())
893  self.transform = self.transformClass(self.control, self.name, self.mapper)
894  self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema())
895 
896 
898 
899  def _setFieldsInRecords(self, records, name):
900  for record in records:
901  record[record.schema.join(name, 'instFlux')] = np.random.random()
902  record[record.schema.join(name, 'instFluxErr')] = np.random.random()
903 
904  # Negative instFluxes should be converted to NaNs.
905  assert len(records) > 1
906  records[0][record.schema.join(name, 'instFlux')] = -1
907 
908  def _compareFieldsInRecords(self, inSrc, outSrc, name):
909  instFluxName = inSrc.schema.join(name, 'instFlux')
910  instFluxErrName = inSrc.schema.join(name, 'instFluxErr')
911  if inSrc[instFluxName] > 0:
912  mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
913  inSrc[instFluxErrName])
914  self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value)
915  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
916  else:
917  # negative instFlux results in NaN magnitude, but can still have finite error
918  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')]))
919  if np.isnan(inSrc[instFluxErrName]):
920  self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')]))
921  else:
922  mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName],
923  inSrc[instFluxErrName])
924  self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error)
925 
926 
928 
929  def _setFieldsInRecords(self, records, name):
930  for record in records:
931  record[record.schema.join(name, 'x')] = np.random.random()
932  record[record.schema.join(name, 'y')] = np.random.random()
933  # Some algorithms set no errors; some set only sigma on x & y; some provide
934  # a full covariance matrix. Set only those which exist in the schema.
935  for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'):
936  fieldName = record.schema.join(name, fieldSuffix)
937  if fieldName in record.schema:
938  record[fieldName] = np.random.random()
939 
940  def _compareFieldsInRecords(self, inSrc, outSrc, name):
941  centroidResultKey = CentroidResultKey(inSrc.schema[self.name])
942  centroidResult = centroidResultKey.get(inSrc)
943 
944  coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc)
945  coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid())
946  self.assertEqual(coordTruth, coord)
947 
948  # If the centroid has an associated uncertainty matrix, the coordinate
949  # must have one too, and vice versa.
950  try:
951  coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name],
952  ["ra", "dec"]).get(outSrc)
954  self.assertFalse(centroidResultKey.getCentroidErr().isValid())
955  else:
956  transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians)
957  coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(),
958  centroidResult.getCentroidErr()),
959  transform.getLinear().getMatrix().transpose())
960  np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)
def _runTransform(self, doExtend=True)
Definition: tests.py:806
def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:591
lsst::geom::AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, lsst::geom::Point2D const &inPoint)
def transform(self, wcs, kwds)
Definition: tests.py:470
def __init__(self, bbox, threshold=10.0, exposure=None, kwds)
Definition: tests.py:382
def _checkOutput(self, baseNames)
Definition: tests.py:795
def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, algMetadata=None)
Definition: tests.py:622
def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4)
Definition: tests.py:311
def realize(self, noise, schema, randomSeed=1)
Definition: tests.py:519
def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, minRotation=None, maxRotation=None, minRefShift=None, maxRefShift=None, minPixShift=2.0, maxPixShift=4.0, randomSeed=1)
Definition: tests.py:211
def __init__(self, owner)
Definition: tests.py:49
def testTransform(self, baseNames=None)
Definition: tests.py:811
def _installFootprint(self, record, image)
Definition: tests.py:391
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
def _checkRegisteredTransform(self, registry, name)
Definition: tests.py:844
def _populateCatalog(self, baseNames)
Definition: tests.py:785
static Schema makeMinimalSchema()
static LinearTransform makeScaling(double s) noexcept
def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, algMetadata=None)
Definition: tests.py:695
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
def drawGaussian(bbox, instFlux, ellipse)
Definition: tests.py:355
std::shared_ptr< SkyWcs > makeSkyWcs(TransformPoint2ToPoint2 const &pixelsToFieldAngle, lsst::geom::Angle const &orientation, bool flipX, lsst::geom::SpherePoint const &boresight, std::string const &projection="TAN")
Eigen::Matrix2d makeCdMatrix(lsst::geom::Angle const &scale, lsst::geom::Angle const &orientation=0 *lsst::geom::degrees, bool flipX=false)
static LinearTransform makeRotation(Angle t) noexcept
def addSource(self, instFlux, centroid, shape=None)
Definition: tests.py:407
def addChild(self, instFlux, centroid, shape=None)
Definition: tests.py:59
def __exit__(self, type_, value, tb)
Definition: tests.py:76
def makeForcedMeasurementConfig(self, plugin=None, dependencies=())
Definition: tests.py:661