lsst.pipe.tasks  16.0-5-g81851deb
dcrModel.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 from lsst.afw.coord.refraction import differentialRefraction
25 import lsst.afw.geom as afwGeom
26 from lsst.afw.geom import AffineTransform
27 from lsst.afw.geom import makeTransform
28 import lsst.afw.image as afwImage
29 import lsst.afw.math as afwMath
30 from lsst.geom import radians
31 
32 __all__ = ["DcrModel", "applyDcr", "calculateDcr", "calculateImageParallacticAngle"]
33 
34 
35 class DcrModel:
36  """A model of the true sky after correcting chromatic effects.
37 
38  Attributes
39  ----------
40  dcrNumSubfilters : `int`
41  Number of sub-filters used to model chromatic effects within a band.
42  filterInfo : `lsst.afw.image.Filter`
43  The filter definition, set in the current instruments' obs package.
44  modelImages : `list` of `lsst.afw.image.MaskedImage`
45  A list of masked images, each containing the model for one subfilter
46 
47  Parameters
48  ----------
49  modelImages : `list` of `lsst.afw.image.MaskedImage`
50  A list of masked images, each containing the model for one subfilter.
51  filterInfo : `lsst.afw.image.Filter`, optional
52  The filter definition, set in the current instruments' obs package.
53  Required for any calculation of DCR, including making matched templates.
54 
55  Notes
56  -----
57  The ``DcrModel`` contains an estimate of the true sky, at a higher
58  wavelength resolution than the input observations. It can be forward-
59  modeled to produce Differential Chromatic Refraction (DCR) matched
60  templates for a given ``Exposure``, and provides utilities for conditioning
61  the model in ``dcrAssembleCoadd`` to avoid oscillating solutions between
62  iterations of forward modeling or between the subfilters of the model.
63  """
64 
65  def __init__(self, modelImages, filterInfo=None):
66  self.dcrNumSubfilters = len(modelImages)
67  self.filterInfo = filterInfo
68  self.modelImages = modelImages
69 
70  @classmethod
71  def fromImage(cls, maskedImage, dcrNumSubfilters, filterInfo=None):
72  """Initialize a DcrModel by dividing a coadd between the subfilters.
73 
74  Parameters
75  ----------
76  maskedImage : `lsst.afw.image.MaskedImage`
77  Input coadded image to divide equally between the subfilters.
78  dcrNumSubfilters : `int`
79  Number of sub-filters used to model chromatic effects within a band.
80  filterInfo : None, optional
81  The filter definition, set in the current instruments' obs package.
82  Required for any calculation of DCR, including making matched templates.
83 
84  Returns
85  -------
86  dcrModel : `lsst.pipe.tasks.DcrModel`
87  Best fit model of the true sky after correcting chromatic effects.
88  """
89  # NANs will potentially contaminate the entire image,
90  # depending on the shift or convolution type used.
91  model = maskedImage.clone()
92  badPixels = np.isnan(model.image.array) | np.isnan(model.variance.array)
93  model.image.array[badPixels] = 0.
94  model.variance.array[badPixels] = 0.
95  model.image.array /= dcrNumSubfilters
96  model.variance.array /= dcrNumSubfilters
97  model.mask.array[badPixels] = model.mask.getPlaneBitMask("NO_DATA")
98  modelImages = [model, ]
99  for subfilter in range(1, dcrNumSubfilters):
100  modelImages.append(model.clone())
101  return cls(modelImages, filterInfo)
102 
103  @classmethod
104  def fromDataRef(cls, dataRef, dcrNumSubfilters):
105  """Load an existing DcrModel from a repository.
106 
107  Parameters
108  ----------
109  dataRef : `lsst.daf.persistence.ButlerDataRef`
110  Data reference defining the patch for coaddition and the
111  reference Warp
112  dcrNumSubfilters : `int`
113  Number of sub-filters used to model chromatic effects within a band.
114 
115  Returns
116  -------
117  dcrModel : `lsst.pipe.tasks.DcrModel`
118  Best fit model of the true sky after correcting chromatic effects.
119  """
120  modelImages = []
121  filterInfo = None
122  for subfilter in range(dcrNumSubfilters):
123  dcrCoadd = dataRef.get("dcrCoadd", subfilter=subfilter, numSubfilters=dcrNumSubfilters)
124  if filterInfo is None:
125  filterInfo = dcrCoadd.getFilter()
126  modelImages.append(dcrCoadd.maskedImage)
127  return cls(modelImages, filterInfo)
128 
129  def __len__(self):
130  """Return the number of subfilters.
131 
132  Returns
133  -------
134  dcrNumSubfilters : `int`
135  The number of DCR subfilters in the model.
136  """
137  return self.dcrNumSubfilters
138 
139  def __getitem__(self, subfilter):
140  """Iterate over the subfilters of the DCR model.
141 
142  Parameters
143  ----------
144  subfilter : `int`
145  Index of the current ``subfilter`` within the full band.
146  Negative indices are allowed, and count in reverse order
147  from the highest ``subfilter``.
148 
149  Returns
150  -------
151  modelImage : `lsst.afw.image.MaskedImage`
152  The DCR model for the given ``subfilter``.
153 
154  Raises
155  ------
156  IndexError
157  If the requested ``subfilter`` is greater or equal to the number
158  of subfilters in the model.
159  """
160  if np.abs(subfilter) >= len(self):
161  raise IndexError("subfilter out of bounds.")
162  return self.modelImages[subfilter]
163 
164  def __setitem__(self, subfilter, maskedImage):
165  """Update the model image for one subfilter.
166 
167  Parameters
168  ----------
169  subfilter : `int`
170  Index of the current subfilter within the full band.
171  maskedImage : `lsst.afw.image.MaskedImage`
172  The DCR model to set for the given ``subfilter``.
173 
174  Raises
175  ------
176  IndexError
177  If the requested ``subfilter`` is greater or equal to the number
178  of subfilters in the model.
179  ValueError
180  If the bounding box of the new image does not match.
181  """
182  if np.abs(subfilter) >= len(self):
183  raise IndexError("subfilter out of bounds.")
184  if maskedImage.getBBox() != self.getBBox():
185  raise ValueError("The bounding box of a subfilter must not change.")
186  self.modelImages[subfilter] = maskedImage
187 
188  def getBBox(self):
189  """Return the common bounding box of each subfilter image.
190 
191  Returns
192  -------
193  bbox : `lsst.afw.geom.Box2I`
194  Bounding box of the DCR model.
195  """
196  return self[0].getBBox()
197 
198  def getReferenceImage(self, bbox=None):
199  """Create a simple template from the DCR model.
200 
201  Parameters
202  ----------
203  bbox : `lsst.afw.geom.Box2I`, optional
204  Sub-region of the coadd. Returns the entire image if None.
205 
206  Returns
207  -------
208  templateImage : `numpy.ndarray`
209  The template with no chromatic effects applied.
210  """
211  return np.mean([model[bbox].image.array for model in self], axis=0)
212 
213  def assign(self, dcrSubModel, bbox=None):
214  """Update a sub-region of the ``DcrModel`` with new values.
215 
216  Parameters
217  ----------
218  dcrSubModel : `lsst.pipe.tasks.DcrModel`
219  New model of the true scene after correcting chromatic effects.
220  bbox : `lsst.afw.geom.Box2I`, optional
221  Sub-region of the coadd.
222  Defaults to the bounding box of ``dcrSubModel``.
223 
224  Raises
225  ------
226  ValueError
227  If the new model has a different number of subfilters.
228  """
229  if len(dcrSubModel) != len(self):
230  raise ValueError("The number of DCR subfilters must be the same "
231  "between the old and new models.")
232  if bbox is None:
233  bbox = self.getBBox()
234  for model, subModel in zip(self, dcrSubModel):
235  model.assign(subModel[bbox], bbox)
236 
237  def buildMatchedTemplate(self, warpCtrl, exposure=None, visitInfo=None, bbox=None, wcs=None, mask=None):
238  """Create a DCR-matched template for an exposure.
239 
240  Parameters
241  ----------
242  warpCtrl : `lsst.afw.math.WarpingControl`
243  Configuration settings for warping an image
244  exposure : `lsst.afw.image.Exposure`, optional
245  The input exposure to build a matched template for.
246  May be omitted if all of the metadata is supplied separately
247  visitInfo : `lsst.afw.image.VisitInfo`, optional
248  Metadata for the exposure. Ignored if ``exposure`` is set.
249  bbox : `lsst.afw.geom.Box2I`, optional
250  Sub-region of the coadd. Ignored if ``exposure`` is set.
251  wcs : `lsst.afw.geom.SkyWcs`, optional
252  Coordinate system definition (wcs) for the exposure.
253  Ignored if ``exposure`` is set.
254  mask : `lsst.afw.image.Mask`, optional
255  reference mask to use for the template image.
256  Ignored if ``exposure`` is set.
257 
258  Returns
259  -------
260  templateImage : `lsst.afw.image.maskedImageF`
261  The DCR-matched template
262 
263  Raises
264  ------
265  ValueError
266  If ``filterInfo`` is not set.
267  ValueError
268  If neither ``exposure`` or all of ``visitInfo``, ``bbox``, and ``wcs`` are set.
269  """
270  if self.filterInfo is None:
271  raise ValueError("'filterInfo' must be set for the DcrModel in order to calculate DCR.")
272  if exposure is not None:
273  visitInfo = exposure.getInfo().getVisitInfo()
274  bbox = exposure.getBBox()
275  wcs = exposure.getInfo().getWcs()
276  elif visitInfo is None or bbox is None or wcs is None:
277  raise ValueError("Either exposure or visitInfo, bbox, and wcs must be set.")
278  dcrShift = calculateDcr(visitInfo, wcs, self.filterInfo, len(self))
279  templateImage = afwImage.MaskedImageF(bbox)
280  for subfilter, dcr in enumerate(dcrShift):
281  templateImage += applyDcr(self[subfilter][bbox], dcr, warpCtrl)
282  if mask is not None:
283  templateImage.setMask(mask[bbox])
284  return templateImage
285 
286  def conditionDcrModel(self, subfilter, newModel, bbox, gain=1.):
287  """Average two iterations' solutions to reduce oscillations.
288 
289  Parameters
290  ----------
291  subfilter : `int`
292  Index of the current subfilter within the full band.
293  newModel : `lsst.afw.image.MaskedImage`
294  The new DCR model for one subfilter from the current iteration.
295  Values in ``newModel`` that are extreme compared with the last
296  iteration are modified in place.
297  bbox : `lsst.afw.geom.Box2I`
298  Sub-region of the coadd
299  gain : `float`, optional
300  Additional weight to apply to the model from the current iteration.
301  Defaults to 1.0, which gives equal weight to both solutions.
302  """
303  # Calculate weighted averages of the image and variance planes.
304  # Note that ``newModel *= gain`` would multiply the variance by ``gain**2``
305  newModel.image *= gain
306  newModel.image += self[subfilter][bbox].image
307  newModel.image /= 1. + gain
308  newModel.variance *= gain
309  newModel.variance += self[subfilter][bbox].variance
310  newModel.variance /= 1. + gain
311 
312  def clampModel(self, subfilter, newModel, bbox, statsCtrl, regularizeSigma, modelClampFactor,
313  convergenceMaskPlanes="DETECTED"):
314  """Restrict large variations in the model between iterations.
315 
316  Parameters
317  ----------
318  subfilter : `int`
319  Index of the current subfilter within the full band.
320  newModel : `lsst.afw.image.MaskedImage`
321  The new DCR model for one subfilter from the current iteration.
322  Values in ``newModel`` that are extreme compared with the last
323  iteration are modified in place.
324  bbox : `lsst.afw.geom.Box2I`
325  Sub-region to coadd
326  statsCtrl : `lsst.afw.math.StatisticsControl`
327  Statistics control object for coadd
328  regularizeSigma : `float`
329  Threshold to exclude noise-like pixels from regularization.
330  modelClampFactor : `float`
331  Maximum relative change of the model allowed between iterations.
332  convergenceMaskPlanes : `list` of `str`, or `str`, optional
333  Mask planes to use to calculate convergence.
334  Default value is set in ``calculateNoiseCutoff`` if not supplied.
335  """
336  newImage = newModel.image.array
337  oldImage = self[subfilter][bbox].image.array
338  noiseCutoff = self.calculateNoiseCutoff(newModel, statsCtrl, regularizeSigma,
339  convergenceMaskPlanes=convergenceMaskPlanes)
340  # Catch any invalid values
341  nanPixels = np.isnan(newImage)
342  newImage[nanPixels] = 0.
343  infPixels = np.isinf(newImage)
344  newImage[infPixels] = oldImage[infPixels]*modelClampFactor
345  # Clip pixels that have very high amplitude, compared with the previous iteration.
346  clampPixels = np.abs(newImage - oldImage) > (np.abs(oldImage*(modelClampFactor - 1)) +
347  noiseCutoff)
348  # Set high amplitude pixels to a multiple or fraction of the old model value,
349  # depending on whether the new model is higher or lower than the old
350  highPixels = newImage > oldImage
351  newImage[clampPixels & highPixels] = oldImage[clampPixels & highPixels]*modelClampFactor
352  newImage[clampPixels & ~highPixels] = oldImage[clampPixels & ~highPixels]/modelClampFactor
353 
354  def regularizeModel(self, bbox, mask, statsCtrl, regularizeSigma, clampFrequency,
355  convergenceMaskPlanes="DETECTED"):
356  """Restrict large variations in the model between subfilters.
357 
358  Any flux subtracted by the restriction is accumulated from all
359  subfilters, and divided evenly to each afterwards in order to preserve
360  total flux.
361 
362  Parameters
363  ----------
364  bbox : `lsst.afw.geom.Box2I`
365  Sub-region to coadd
366  mask : `lsst.afw.image.Mask`
367  Reference mask to use for all model planes.
368  statsCtrl : `lsst.afw.math.StatisticsControl`
369  Statistics control object for coadd
370  regularizeSigma : `float`
371  Threshold to exclude noise-like pixels from regularization.
372  clampFrequency : `float`
373  Maximum relative change of the model allowed between subfilters.
374  convergenceMaskPlanes : `list` of `str`, or `str`, optional
375  Mask planes to use to calculate convergence. (Default is "DETECTED")
376  Default value is set in ``calculateNoiseCutoff`` if not supplied.
377  """
378  templateImage = self.getReferenceImage(bbox)
379  excess = np.zeros_like(templateImage)
380  for model in self:
381  noiseCutoff = self.calculateNoiseCutoff(model, statsCtrl, regularizeSigma,
382  convergenceMaskPlanes=convergenceMaskPlanes,
383  mask=mask[bbox])
384  modelVals = model.image.array
385  highPixels = (modelVals > (templateImage*clampFrequency + noiseCutoff))
386  excess[highPixels] += modelVals[highPixels] - templateImage[highPixels]*clampFrequency
387  modelVals[highPixels] = templateImage[highPixels]*clampFrequency
388  lowPixels = (modelVals < templateImage/clampFrequency - noiseCutoff)
389  excess[lowPixels] += modelVals[lowPixels] - templateImage[lowPixels]/clampFrequency
390  modelVals[lowPixels] = templateImage[lowPixels]/clampFrequency
391  excess /= len(self)
392  for model in self:
393  model.image.array += excess
394 
395  def calculateNoiseCutoff(self, maskedImage, statsCtrl, regularizeSigma,
396  convergenceMaskPlanes="DETECTED", mask=None):
397  """Helper function to calculate the background noise level of an image.
398 
399  Parameters
400  ----------
401  maskedImage : `lsst.afw.image.MaskedImage`
402  The input image to evaluate the background noise properties.
403  statsCtrl : `lsst.afw.math.StatisticsControl`
404  Statistics control object for coadd
405  regularizeSigma : `float`
406  Threshold to exclude noise-like pixels from regularization.
407  convergenceMaskPlanes : `list` of `str`, or `str`
408  Mask planes to use to calculate convergence.
409  mask : `lsst.afw.image.Mask`, Optional
410  Optional alternate mask
411 
412  Returns
413  -------
414  noiseCutoff : `float`
415  The threshold value to treat pixels as noise in an image..
416  """
417  convergeMask = maskedImage.mask.getPlaneBitMask(convergenceMaskPlanes)
418  if mask is None:
419  mask = maskedImage.mask
420  backgroundPixels = mask.array & (statsCtrl.getAndMask() | convergeMask) == 0
421  noiseCutoff = regularizeSigma*np.std(maskedImage.image.array[backgroundPixels])
422  return noiseCutoff
423 
424 
425 def applyDcr(maskedImage, dcr, warpCtrl, bbox=None, useInverse=False):
426  """Shift a masked image.
427 
428  Parameters
429  ----------
430  maskedImage : `lsst.afw.image.MaskedImage`
431  The input masked image to shift.
432  dcr : `lsst.afw.geom.Extent2I`
433  Shift calculated with ``calculateDcr``.
434  warpCtrl : `lsst.afw.math.WarpingControl`
435  Configuration settings for warping an image
436  bbox : `lsst.afw.geom.Box2I`, optional
437  Sub-region of the masked image to shift.
438  Shifts the entire image if None (Default).
439  useInverse : `bool`, optional
440  Use the reverse of ``dcr`` for the shift. Default: False
441 
442  Returns
443  -------
444  `lsst.afw.image.maskedImageF`
445  A masked image, with the pixels within the bounding box shifted.
446  """
447  padValue = afwImage.pixel.SinglePixelF(0., maskedImage.mask.getPlaneBitMask("NO_DATA"), 0)
448  if bbox is None:
449  bbox = maskedImage.getBBox()
450  shiftedImage = afwImage.MaskedImageF(bbox)
451  transform = makeTransform(AffineTransform((-1.0 if useInverse else 1.0)*dcr))
452  afwMath.warpImage(shiftedImage, maskedImage[bbox],
453  transform, warpCtrl, padValue=padValue)
454  return shiftedImage
455 
456 
457 def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters):
458  """Calculate the shift in pixels of an exposure due to DCR.
459 
460  Parameters
461  ----------
462  visitInfo : `lsst.afw.image.VisitInfo`
463  Metadata for the exposure.
464  wcs : `lsst.afw.geom.SkyWcs`
465  Coordinate system definition (wcs) for the exposure.
466  filterInfo : `lsst.afw.image.Filter`
467  The filter definition, set in the current instruments' obs package.
468  dcrNumSubfilters : `int`
469  Number of sub-filters used to model chromatic effects within a band.
470 
471  Returns
472  -------
473  `lsst.afw.geom.Extent2I`
474  The 2D shift due to DCR, in pixels.
475  """
476  rotation = calculateImageParallacticAngle(visitInfo, wcs)
477  dcrShift = []
478  lambdaEff = filterInfo.getFilterProperty().getLambdaEff()
479  for wl0, wl1 in wavelengthGenerator(filterInfo, dcrNumSubfilters):
480  # Note that diffRefractAmp can be negative, since it's relative to the midpoint of the full band
481  diffRefractAmp0 = differentialRefraction(wl0, lambdaEff,
482  elevation=visitInfo.getBoresightAzAlt().getLatitude(),
483  observatory=visitInfo.getObservatory(),
484  weather=visitInfo.getWeather())
485  diffRefractAmp1 = differentialRefraction(wl1, lambdaEff,
486  elevation=visitInfo.getBoresightAzAlt().getLatitude(),
487  observatory=visitInfo.getObservatory(),
488  weather=visitInfo.getWeather())
489  diffRefractAmp = (diffRefractAmp0 + diffRefractAmp1)/2.
490  diffRefractPix = diffRefractAmp.asArcseconds()/wcs.getPixelScale().asArcseconds()
491  dcrShift.append(afwGeom.Extent2D(diffRefractPix*np.cos(rotation.asRadians()),
492  diffRefractPix*np.sin(rotation.asRadians())))
493  return dcrShift
494 
495 
496 def calculateImageParallacticAngle(visitInfo, wcs):
497  """Calculate the total sky rotation angle of an exposure.
498 
499  Parameters
500  ----------
501  visitInfo : `lsst.afw.image.VisitInfo`
502  Metadata for the exposure.
503  wcs : `lsst.afw.geom.SkyWcs`
504  Coordinate system definition (wcs) for the exposure.
505 
506  Returns
507  -------
508  `lsst.geom.Angle`
509  The rotation of the image axis, East from North.
510  Equal to the parallactic angle plus any additional rotation of the
511  coordinate system.
512  A rotation angle of 0 degrees is defined with
513  North along the +y axis and East along the +x axis.
514  A rotation angle of 90 degrees is defined with
515  North along the +x axis and East along the -y axis.
516  """
517  parAngle = visitInfo.getBoresightParAngle().asRadians()
518  cd = wcs.getCdMatrix()
519  cdAngle = (np.arctan2(-cd[0, 1], cd[0, 0]) + np.arctan2(cd[1, 0], cd[1, 1]))/2.
520  rotAngle = (cdAngle + parAngle)*radians
521  return rotAngle
522 
523 
524 def wavelengthGenerator(filterInfo, dcrNumSubfilters):
525  """Iterate over the wavelength endpoints of subfilters.
526 
527  Parameters
528  ----------
529  filterInfo : `lsst.afw.image.Filter`
530  The filter definition, set in the current instruments' obs package.
531  dcrNumSubfilters : `int`
532  Number of sub-filters used to model chromatic effects within a band.
533 
534  Yields
535  ------
536  `tuple` of two `float`
537  The next set of wavelength endpoints for a subfilter, in nm.
538  """
539  lambdaMin = filterInfo.getFilterProperty().getLambdaMin()
540  lambdaMax = filterInfo.getFilterProperty().getLambdaMax()
541  wlStep = (lambdaMax - lambdaMin)/dcrNumSubfilters
542  for wl in np.linspace(lambdaMin, lambdaMax, dcrNumSubfilters, endpoint=False):
543  yield (wl, wl + wlStep)
def calculateNoiseCutoff(self, maskedImage, statsCtrl, regularizeSigma, convergenceMaskPlanes="DETECTED", mask=None)
Definition: dcrModel.py:396
def conditionDcrModel(self, subfilter, newModel, bbox, gain=1.)
Definition: dcrModel.py:286
def __init__(self, modelImages, filterInfo=None)
Definition: dcrModel.py:65
def regularizeModel(self, bbox, mask, statsCtrl, regularizeSigma, clampFrequency, convergenceMaskPlanes="DETECTED")
Definition: dcrModel.py:355
def clampModel(self, subfilter, newModel, bbox, statsCtrl, regularizeSigma, modelClampFactor, convergenceMaskPlanes="DETECTED")
Definition: dcrModel.py:313
def fromDataRef(cls, dataRef, dcrNumSubfilters)
Definition: dcrModel.py:104
def calculateImageParallacticAngle(visitInfo, wcs)
Definition: dcrModel.py:496
def __getitem__(self, subfilter)
Definition: dcrModel.py:139
def __setitem__(self, subfilter, maskedImage)
Definition: dcrModel.py:164
def assign(self, dcrSubModel, bbox=None)
Definition: dcrModel.py:213
def applyDcr(maskedImage, dcr, warpCtrl, bbox=None, useInverse=False)
Definition: dcrModel.py:425
def wavelengthGenerator(filterInfo, dcrNumSubfilters)
Definition: dcrModel.py:524
def buildMatchedTemplate(self, warpCtrl, exposure=None, visitInfo=None, bbox=None, wcs=None, mask=None)
Definition: dcrModel.py:237
def fromImage(cls, maskedImage, dcrNumSubfilters, filterInfo=None)
Definition: dcrModel.py:71
def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters)
Definition: dcrModel.py:457
def getReferenceImage(self, bbox=None)
Definition: dcrModel.py:198