lsst.ip.diffim  22.0.0-12-g9aaea118+107b621308
getTemplate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 
25 import lsst.afw.image as afwImage
26 import lsst.geom as geom
27 import lsst.afw.geom as afwGeom
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 from lsst.daf.butler import DeferredDatasetHandle
31 from lsst.ip.diffim.dcrModel import DcrModel
32 
33 __all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
34  "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
35 
36 
37 class GetCoaddAsTemplateConfig(pexConfig.Config):
38  templateBorderSize = pexConfig.Field(
39  dtype=int,
40  default=10,
41  doc="Number of pixels to grow the requested template image to account for warping"
42  )
43  coaddName = pexConfig.Field(
44  doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
45  dtype=str,
46  default="deep",
47  )
48  numSubfilters = pexConfig.Field(
49  doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'",
50  dtype=int,
51  default=3,
52  )
53  effectiveWavelength = pexConfig.Field(
54  doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'",
55  optional=True,
56  dtype=float,
57  )
58  bandwidth = pexConfig.Field(
59  doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'",
60  optional=True,
61  dtype=float,
62  )
63  warpType = pexConfig.Field(
64  doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
65  dtype=str,
66  default="direct",
67  )
68 
69  def validate(self):
70  if self.coaddNamecoaddName == 'dcr':
71  if self.effectiveWavelengtheffectiveWavelength is None or self.bandwidthbandwidth is None:
72  raise ValueError("The effective wavelength and bandwidth of the physical filter "
73  "must be set in the getTemplate config for DCR coadds. "
74  "Required until transmission curves are used in DM-13668.")
75 
76 
77 class GetCoaddAsTemplateTask(pipeBase.Task):
78  """Subtask to retrieve coadd for use as an image difference template.
79 
80  This is the default getTemplate Task to be run as a subtask by
81  ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
82  ``runGen3()``.
83 
84  Notes
85  -----
86  From the given skymap, the closest tract is selected; multiple tracts are
87  not supported. The assembled template inherits the WCS of the selected
88  skymap tract and the resolution of the template exposures. Overlapping box
89  regions of the input template patches are pixel by pixel copied into the
90  assembled template image. There is no warping or pixel resampling.
91 
92  Pixels with no overlap of any available input patches are set to ``nan`` value
93  and ``NO_DATA`` flagged.
94  """
95 
96  ConfigClass = GetCoaddAsTemplateConfig
97  _DefaultName = "GetCoaddAsTemplateTask"
98 
99  def runDataRef(self, exposure, sensorRef, templateIdList=None):
100  """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
101  that overlaps the science exposure.
102 
103  Parameters
104  ----------
105  exposure: `lsst.afw.image.Exposure`
106  an exposure for which to generate an overlapping template
107  sensorRef : TYPE
108  a Butler data reference that can be used to obtain coadd data
109  templateIdList : TYPE, optional
110  list of data ids, unused here, in the case of coadd template
111 
112  Returns
113  -------
114  result : `lsst.pipe.base.Struct`
115  - ``exposure`` : `lsst.afw.image.ExposureF`
116  a template coadd exposure assembled out of patches
117  - ``sources`` : None for this subtask
118  """
119  skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
120  tractInfo, patchList, skyCorners = self.getOverlapPatchListgetOverlapPatchList(exposure, skyMap)
121 
122  availableCoaddRefs = dict()
123  for patchInfo in patchList:
124  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
125  patchArgDict = dict(
126  datasetType=self.getCoaddDatasetNamegetCoaddDatasetName() + "_sub",
127  bbox=patchInfo.getOuterBBox(),
128  tract=tractInfo.getId(),
129  patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
130  subfilter=0,
131  numSubfilters=self.config.numSubfilters,
132  )
133 
134  if sensorRef.datasetExists(**patchArgDict):
135  self.log.info("Reading patch %s", patchArgDict)
136  availableCoaddRefs[patchNumber] = patchArgDict
137 
138  templateExposure = self.runrun(
139  tractInfo, patchList, skyCorners, availableCoaddRefs,
140  sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
141  )
142  return pipeBase.Struct(exposure=templateExposure, sources=None)
143 
144  def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
145  """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
146  that overlaps the science exposure.
147 
148  Parameters
149  ----------
150  exposure : `lsst.afw.image.Exposure`
151  The science exposure to define the sky region of the template coadd.
152  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
153  Butler like object that supports getting data by DatasetRef.
154  skyMapRef : `lsst.daf.butler.DatasetRef`
155  Reference to SkyMap object that corresponds to the template coadd.
156  coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
157  Iterable of references to the available template coadd patches.
158 
159  Returns
160  -------
161  result : `lsst.pipe.base.Struct`
162  - ``exposure`` : `lsst.afw.image.ExposureF`
163  a template coadd exposure assembled out of patches
164  - ``sources`` : `None` for this subtask
165  """
166  skyMap = butlerQC.get(skyMapRef)
167  coaddExposureRefs = butlerQC.get(coaddExposureRefs)
168  tracts = [ref.dataId['tract'] for ref in coaddExposureRefs]
169  if tracts.count(tracts[0]) == len(tracts):
170  tractInfo = skyMap[tracts[0]]
171  else:
172  raise RuntimeError("Templates constructed from multiple Tracts not yet supported")
173 
174  detectorBBox = exposure.getBBox()
175  detectorWcs = exposure.getWcs()
176  detectorCorners = detectorWcs.pixelToSky(geom.Box2D(detectorBBox).getCorners())
177  validPolygon = exposure.getInfo().getValidPolygon()
178  detectorPolygon = validPolygon if validPolygon else geom.Box2D(detectorBBox)
179 
180  availableCoaddRefs = dict()
181  overlappingArea = 0
182  for coaddRef in coaddExposureRefs:
183  dataId = coaddRef.dataId
184  patchWcs = skyMap[dataId['tract']].getWcs()
185  patchBBox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
186  patchCorners = patchWcs.pixelToSky(geom.Box2D(patchBBox).getCorners())
187  patchPolygon = afwGeom.Polygon(detectorWcs.skyToPixel(patchCorners))
188  if patchPolygon.intersection(detectorPolygon):
189  overlappingArea += patchPolygon.intersectionSingle(detectorPolygon).calculateArea()
190  if self.config.coaddName == 'dcr':
191  self.log.info("Using template input tract=%s, patch=%s, subfilter=%s",
192  dataId['tract'], dataId['patch'], dataId['subfilter'])
193  if dataId['patch'] in availableCoaddRefs:
194  availableCoaddRefs[dataId['patch']].append(coaddRef)
195  else:
196  availableCoaddRefs[dataId['patch']] = [coaddRef, ]
197  else:
198  self.log.info("Using template input tract=%s, patch=%s",
199  dataId['tract'], dataId['patch'])
200  availableCoaddRefs[dataId['patch']] = coaddRef
201 
202  if overlappingArea == 0:
203  templateExposure = None
204  pixGood = 0
205  self.log.warning("No overlapping template patches found")
206  else:
207  patchList = [tractInfo[patch] for patch in availableCoaddRefs.keys()]
208  templateExposure = self.runrun(tractInfo, patchList, detectorCorners, availableCoaddRefs,
209  visitInfo=exposure.getInfo().getVisitInfo())
210  # Count the number of pixels with the NO_DATA mask bit set
211  # counting NaN pixels is insufficient because pixels without data are often intepolated over)
212  pixNoData = np.count_nonzero(templateExposure.mask.array
213  & templateExposure.mask.getPlaneBitMask('NO_DATA'))
214  pixGood = templateExposure.getBBox().getArea() - pixNoData
215  self.log.info("template has %d good pixels (%.1f%%)", pixGood,
216  100*pixGood/templateExposure.getBBox().getArea())
217  return pipeBase.Struct(exposure=templateExposure, sources=None, area=pixGood)
218 
219  def getOverlapPatchList(self, exposure, skyMap):
220  """Select the relevant tract and its patches that overlap with the science exposure.
221 
222  Parameters
223  ----------
224  exposure : `lsst.afw.image.Exposure`
225  The science exposure to define the sky region of the template coadd.
226 
227  skyMap : `lsst.skymap.BaseSkyMap`
228  SkyMap object that corresponds to the template coadd.
229 
230  Returns
231  -------
232  result : `tuple` of
233  - ``tractInfo`` : `lsst.skymap.TractInfo`
234  The selected tract.
235  - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
236  List of all overlap patches of the selected tract.
237  - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
238  Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
239  """
240  expWcs = exposure.getWcs()
241  expBoxD = geom.Box2D(exposure.getBBox())
242  expBoxD.grow(self.config.templateBorderSize)
243  ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
244  tractInfo = skyMap.findTract(ctrSkyPos)
245  self.log.info("Using skyMap tract %s", tractInfo.getId())
246  skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
247  patchList = tractInfo.findPatchList(skyCorners)
248 
249  if not patchList:
250  raise RuntimeError("No suitable tract found")
251 
252  self.log.info("Assembling %d coadd patches", len(patchList))
253  self.log.info("exposure dimensions=%s", exposure.getDimensions())
254 
255  return (tractInfo, patchList, skyCorners)
256 
257  def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
258  sensorRef=None, visitInfo=None):
259  """Gen2 and gen3 shared code: determination of exposure dimensions and
260  copying of pixels from overlapping patch regions.
261 
262  Parameters
263  ----------
264  skyMap : `lsst.skymap.BaseSkyMap`
265  SkyMap object that corresponds to the template coadd.
266  tractInfo : `lsst.skymap.TractInfo`
267  The selected tract.
268  patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
269  Patches to consider for making the template exposure.
270  skyCorners : list of `lsst.geom.SpherePoint`
271  Sky corner coordinates to be covered by the template exposure.
272  availableCoaddRefs : `dict` [`int`]
273  Dictionary of spatially relevant retrieved coadd patches,
274  indexed by their sequential patch number. In Gen3 mode, values are
275  `lsst.daf.butler.DeferredDatasetHandle` and ``.get()`` is called,
276  in Gen2 mode, ``sensorRef.get(**coaddef)`` is called to retrieve the coadd.
277  sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
278  Butler data reference to get coadd data.
279  Must be `None` for Gen3.
280  visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
281  VisitInfo to make dcr model.
282 
283  Returns
284  -------
285  templateExposure: `lsst.afw.image.ExposureF`
286  The created template exposure.
287  """
288  coaddWcs = tractInfo.getWcs()
289 
290  # compute coadd bbox
291  coaddBBox = geom.Box2D()
292  for skyPos in skyCorners:
293  coaddBBox.include(coaddWcs.skyToPixel(skyPos))
294  coaddBBox = geom.Box2I(coaddBBox)
295  self.log.info("coadd dimensions=%s", coaddBBox.getDimensions())
296 
297  coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
298  coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
299  nPatchesFound = 0
300  coaddFilterLabel = None
301  coaddPsf = None
302  coaddPhotoCalib = None
303  for patchInfo in patchList:
304  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
305  patchSubBBox = patchInfo.getOuterBBox()
306  patchSubBBox.clip(coaddBBox)
307  if patchNumber not in availableCoaddRefs:
308  self.log.warning("skip patch=%d; patch does not exist for this coadd", patchNumber)
309  continue
310  if patchSubBBox.isEmpty():
311  if isinstance(availableCoaddRefs[patchNumber], DeferredDatasetHandle):
312  tract = availableCoaddRefs[patchNumber].dataId['tract']
313  else:
314  tract = availableCoaddRefs[patchNumber]['tract']
315  self.log.info("skip tract=%d patch=%d; no overlapping pixels", tract, patchNumber)
316  continue
317 
318  if self.config.coaddName == 'dcr':
319  patchInnerBBox = patchInfo.getInnerBBox()
320  patchInnerBBox.clip(coaddBBox)
321  if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
322  self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels.",
323  availableCoaddRefs[patchNumber])
324  continue
325  self.log.info("Constructing DCR-matched template for patch %s",
326  availableCoaddRefs[patchNumber])
327 
328  if sensorRef:
329  dcrModel = DcrModel.fromDataRef(sensorRef,
330  self.config.effectiveWavelength,
331  self.config.bandwidth,
332  **availableCoaddRefs[patchNumber])
333  else:
334  dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber],
335  self.config.effectiveWavelength,
336  self.config.bandwidth)
337  # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
338  # Each patch has significant overlap, and the contaminated edge pixels in
339  # a new patch will overwrite good pixels in the overlap region from
340  # previous patches.
341  # Shrink the BBox to remove the contaminated pixels,
342  # but make sure it is only the overlap region that is reduced.
343  dcrBBox = geom.Box2I(patchSubBBox)
344  dcrBBox.grow(-self.config.templateBorderSize)
345  dcrBBox.include(patchInnerBBox)
346  coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
347  wcs=coaddWcs,
348  visitInfo=visitInfo)
349  else:
350  if sensorRef is None:
351  # Gen3
352  coaddPatch = availableCoaddRefs[patchNumber].get()
353  else:
354  # Gen2
355  coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
356  nPatchesFound += 1
357 
358  # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
359  # calculation from caller code. Gen3 also does not do this.
360  overlapBox = coaddPatch.getBBox()
361  overlapBox.clip(coaddBBox)
362  coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
363 
364  if coaddFilterLabel is None:
365  coaddFilterLabel = coaddPatch.getFilterLabel()
366 
367  # Retrieve the PSF for this coadd tract, if not already retrieved
368  if coaddPsf is None and coaddPatch.hasPsf():
369  coaddPsf = coaddPatch.getPsf()
370 
371  # Retrieve the calibration for this coadd tract, if not already retrieved
372  if coaddPhotoCalib is None:
373  coaddPhotoCalib = coaddPatch.getPhotoCalib()
374 
375  if coaddPhotoCalib is None:
376  raise RuntimeError("No coadd PhotoCalib found!")
377  if nPatchesFound == 0:
378  raise RuntimeError("No patches found!")
379  if coaddPsf is None:
380  raise RuntimeError("No coadd Psf found!")
381 
382  coaddExposure.setPhotoCalib(coaddPhotoCalib)
383  coaddExposure.setPsf(coaddPsf)
384  coaddExposure.setFilterLabel(coaddFilterLabel)
385  return coaddExposure
386 
388  """Return coadd name for given task config
389 
390  Returns
391  -------
392  CoaddDatasetName : `string`
393 
394  TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
395  """
396  warpType = self.config.warpType
397  suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
398  return self.config.coaddName + "Coadd" + suffix
399 
400 
401 class GetCalexpAsTemplateConfig(pexConfig.Config):
402  doAddCalexpBackground = pexConfig.Field(
403  dtype=bool,
404  default=True,
405  doc="Add background to calexp before processing it."
406  )
407 
408 
409 class GetCalexpAsTemplateTask(pipeBase.Task):
410  """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
411  for use as an image difference template. Only gen2 supported.
412 
413  To be run as a subtask by pipe.tasks.ImageDifferenceTask.
414  Intended for use with simulations and surveys that repeatedly visit the same pointing.
415  This code was originally part of Winter2013ImageDifferenceTask.
416  """
417 
418  ConfigClass = GetCalexpAsTemplateConfig
419  _DefaultName = "GetCalexpAsTemplateTask"
420 
421  def run(self, exposure, sensorRef, templateIdList):
422  """Return a calexp exposure with based on input sensorRef.
423 
424  Construct a dataId based on the sensorRef.dataId combined
425  with the specifications from the first dataId in templateIdList
426 
427  Parameters
428  ----------
429  exposure : `lsst.afw.image.Exposure`
430  exposure (unused)
431  sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
432  Data reference of the calexp(s) to subtract from.
433  templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
434  Data reference of the template calexp to be subtraced.
435  Can be incomplete, fields are initialized from `sensorRef`.
436  If there are multiple items, only the first one is used.
437 
438  Returns
439  -------
440  result : `struct`
441 
442  return a pipeBase.Struct:
443 
444  - ``exposure`` : a template calexp
445  - ``sources`` : source catalog measured on the template
446  """
447 
448  if len(templateIdList) == 0:
449  raise RuntimeError("No template data reference supplied.")
450  if len(templateIdList) > 1:
451  self.log.warning("Multiple template data references supplied. Using the first one only.")
452 
453  templateId = sensorRef.dataId.copy()
454  templateId.update(templateIdList[0])
455 
456  self.log.info("Fetching calexp (%s) as template.", templateId)
457 
458  butler = sensorRef.getButler()
459  template = butler.get(datasetType="calexp", dataId=templateId)
460  if self.config.doAddCalexpBackground:
461  templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
462  mi = template.getMaskedImage()
463  mi += templateBg.getImage()
464 
465  if not template.hasPsf():
466  raise pipeBase.TaskError("Template has no psf")
467 
468  templateSources = butler.get(datasetType="src", dataId=templateId)
469  return pipeBase.Struct(exposure=template,
470  sources=templateSources)
471 
472  def runDataRef(self, *args, **kwargs):
473  return self.runrun(*args, **kwargs)
474 
475  def runQuantum(self, **kwargs):
476  raise NotImplementedError("Calexp template is not supported with gen3 middleware")
def run(self, exposure, sensorRef, templateIdList)
Definition: getTemplate.py:421
def runDataRef(self, exposure, sensorRef, templateIdList=None)
Definition: getTemplate.py:99
def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, sensorRef=None, visitInfo=None)
Definition: getTemplate.py:258
def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs)
Definition: getTemplate.py:144