lsst.ip.diffim  19.0.0-13-g8db0348+3
getTemplate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 
25 import lsst.afw.image as afwImage
26 import lsst.geom as geom
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 from lsst.ip.diffim.dcrModel import DcrModel
30 
31 __all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
32  "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
33 
34 
35 class GetCoaddAsTemplateConfig(pexConfig.Config):
36  templateBorderSize = pexConfig.Field(
37  dtype=int,
38  default=10,
39  doc="Number of pixels to grow the requested template image to account for warping"
40  )
41  coaddName = pexConfig.Field(
42  doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
43  dtype=str,
44  default="deep",
45  )
46  numSubfilters = pexConfig.Field(
47  doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'",
48  dtype=int,
49  default=3,
50  )
51  warpType = pexConfig.Field(
52  doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
53  dtype=str,
54  default="direct",
55  )
56 
57 
58 class GetCoaddAsTemplateTask(pipeBase.Task):
59  """Subtask to retrieve coadd for use as an image difference template.
60 
61  This is the default getTemplate Task to be run as a subtask by
62  ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
63  ``runGen3()``.
64 
65  Notes
66  -----
67  From the given skymap, the closest tract is selected; multiple tracts are
68  not supported. The assembled template inherits the WCS of the selected
69  skymap tract and the resolution of the template exposures. Overlapping box
70  regions of the input template patches are pixel by pixel copied into the
71  assembled template image. There is no warping or pixel resampling.
72 
73  Pixels with no overlap of any available input patches are set to ``nan`` value
74  and ``NO_DATA`` flagged.
75  """
76 
77  ConfigClass = GetCoaddAsTemplateConfig
78  _DefaultName = "GetCoaddAsTemplateTask"
79 
80  def runDataRef(self, exposure, sensorRef, templateIdList=None):
81  """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
82  that overlaps the science exposure.
83 
84  Parameters
85  ----------
86  exposure: `lsst.afw.image.Exposure`
87  an exposure for which to generate an overlapping template
88  sensorRef : TYPE
89  a Butler data reference that can be used to obtain coadd data
90  templateIdList : TYPE, optional
91  list of data ids, unused here, in the case of coadd template
92 
93  Returns
94  -------
95  result : `lsst.pipe.base.Struct`
96  - ``exposure`` : `lsst.afw.image.ExposureF`
97  a template coadd exposure assembled out of patches
98  - ``sources`` : None for this subtask
99  """
100  skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
101  tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
102 
103  availableCoaddRefs = dict()
104  for patchInfo in patchList:
105  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
106  patchArgDict = dict(
107  datasetType=self.getCoaddDatasetName() + "_sub",
108  bbox=patchInfo.getOuterBBox(),
109  tract=tractInfo.getId(),
110  patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
111  numSubfilters=self.config.numSubfilters,
112  )
113 
114  if self.config.coaddName != 'dcr' and sensorRef.datasetExists(**patchArgDict):
115  self.log.info("Reading patch %s" % patchArgDict)
116  availableCoaddRefs[patchNumber] = patchArgDict
117 
118  templateExposure = self.run(
119  tractInfo, patchList, skyCorners, availableCoaddRefs,
120  sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
121  )
122  return pipeBase.Struct(exposure=templateExposure, sources=None)
123 
124  def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
125  """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
126  that overlaps the science exposure.
127 
128  Parameters
129  ----------
130  exposure : `lsst.afw.image.Exposure`
131  The science exposure to define the sky region of the template coadd.
132  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
133  Butler like object that supports getting data by DatasetRef.
134  skyMapRef : `lsst.daf.butler.DatasetRef`
135  Reference to SkyMap object that corresponds to the template coadd.
136  coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
137  Iterable of references to the available template coadd patches.
138 
139  Returns
140  -------
141  result : `lsst.pipe.base.Struct`
142  - ``exposure`` : `lsst.afw.image.ExposureF`
143  a template coadd exposure assembled out of patches
144  - ``sources`` : `None` for this subtask
145  """
146  skyMap = butlerQC.get(skyMapRef)
147  tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
148  patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList)
149 
150  availableCoaddRefs = dict()
151  for coaddRef in coaddExposureRefs:
152  dataId = coaddRef.datasetRef.dataId
153  if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter:
154  self.log.info("Using template input tract=%s, patch=%s" %
155  (tractInfo.getId(), dataId['patch']))
156  availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef)
157 
158  templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs)
159  return pipeBase.Struct(exposure=templateExposure, sources=None)
160 
161  def getOverlapPatchList(self, exposure, skyMap):
162  """Select the relevant tract and its patches that overlap with the science exposure.
163 
164  Parameters
165  ----------
166  exposure : `lsst.afw.image.Exposure`
167  The science exposure to define the sky region of the template coadd.
168 
169  skyMap : `lsst.skymap.BaseSkyMap`
170  SkyMap object that corresponds to the template coadd.
171 
172  Returns
173  -------
174  result : `tuple` of
175  - ``tractInfo`` : `lsst.skymap.TractInfo`
176  The selected tract.
177  - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
178  List of all overlap patches of the selected tract.
179  - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
180  Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
181  """
182  expWcs = exposure.getWcs()
183  expBoxD = geom.Box2D(exposure.getBBox())
184  expBoxD.grow(self.config.templateBorderSize)
185  ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
186  tractInfo = skyMap.findTract(ctrSkyPos)
187  self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
188  skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
189  patchList = tractInfo.findPatchList(skyCorners)
190 
191  if not patchList:
192  raise RuntimeError("No suitable tract found")
193 
194  self.log.info("Assembling %s coadd patches" % (len(patchList),))
195  self.log.info("exposure dimensions=%s" % exposure.getDimensions())
196 
197  return (tractInfo, patchList, skyCorners)
198 
199  def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
200  sensorRef=None, visitInfo=None):
201  """Gen2 and gen3 shared code: determination of exposure dimensions and
202  copying of pixels from overlapping patch regions.
203 
204  Parameters
205  ----------
206  skyMap : `lsst.skymap.BaseSkyMap`
207  SkyMap object that corresponds to the template coadd.
208  tractInfo : `lsst.skymap.TractInfo`
209  The selected tract.
210  patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
211  Patches to consider for making the template exposure.
212  skyCorners : list of `lsst.geom.SpherePoint`
213  Sky corner coordinates to be covered by the template exposure.
214  availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
215  `dict` (Gen2)
216  Dictionary of spatially relevant retrieved coadd patches,
217  indexed by their sequential patch number. In Gen3 mode, .get() is called,
218  in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
219  sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
220  TODO DM-22952 Butler data reference to get coadd data.
221  Must be `None` for Gen3.
222  visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
223  TODO DM-22952 VisitInfo to make dcr model.
224 
225  Returns
226  -------
227  templateExposure: `lsst.afw.image.ExposureF`
228  The created template exposure.
229  """
230  coaddWcs = tractInfo.getWcs()
231 
232  # compute coadd bbox
233  coaddBBox = geom.Box2D()
234  for skyPos in skyCorners:
235  coaddBBox.include(coaddWcs.skyToPixel(skyPos))
236  coaddBBox = geom.Box2I(coaddBBox)
237  self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
238 
239  coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
240  coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
241  nPatchesFound = 0
242  coaddFilter = None
243  coaddPsf = None
244  coaddPhotoCalib = None
245  for patchInfo in patchList:
246  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
247  patchSubBBox = patchInfo.getOuterBBox()
248  patchSubBBox.clip(coaddBBox)
249  patchArgDict = dict(
250  datasetType=self.getCoaddDatasetName() + "_sub",
251  bbox=patchSubBBox,
252  tract=tractInfo.getId(),
253  patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
254  numSubfilters=self.config.numSubfilters,
255  )
256  if patchSubBBox.isEmpty():
257  self.log.info(f"skip tract={patchArgDict['tract']}, "
258  f"patch={patchNumber}; no overlapping pixels")
259  continue
260 
261  # TODO DM-22952
262  # Dcr coadd support is gen2 only
263  # Under gen3, sensorRef will be None and execution is stopped in runQuantum if dcr coadd is
264  # configured
265  if self.config.coaddName == 'dcr':
266  if not sensorRef.datasetExists(subfilter=0, **patchArgDict):
267  self.log.warn("%(datasetType)s, tract=%(tract)s, patch=%(patch)s,"
268  " numSubfilters=%(numSubfilters)s, subfilter=0 does not exist"
269  % patchArgDict)
270  continue
271  patchInnerBBox = patchInfo.getInnerBBox()
272  patchInnerBBox.clip(coaddBBox)
273  if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
274  self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." % patchArgDict)
275  continue
276  self.log.info("Constructing DCR-matched template for patch %s" % patchArgDict)
277 
278  dcrModel = DcrModel.fromDataRef(sensorRef, **patchArgDict)
279  # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
280  # Each patch has significant overlap, and the contaminated edge pixels in
281  # a new patch will overwrite good pixels in the overlap region from
282  # previous patches.
283  # Shrink the BBox to remove the contaminated pixels,
284  # but make sure it is only the overlap region that is reduced.
285  dcrBBox = geom.Box2I(patchSubBBox)
286  dcrBBox.grow(-self.config.templateBorderSize)
287  dcrBBox.include(patchInnerBBox)
288  coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
289  wcs=coaddWcs,
290  visitInfo=visitInfo)
291  else:
292  if patchNumber not in availableCoaddRefs:
293  self.log.warn(f"{patchArgDict['datasetType']}, "
294  f"tract={patchArgDict['tract']}, patch={patchNumber} does not exist")
295  continue
296  if sensorRef is None:
297  # Gen3
298  coaddPatch = availableCoaddRefs[patchNumber].get()
299  else:
300  # Gen2
301  coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
302  nPatchesFound += 1
303 
304  # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
305  # calculation from caller code. Gen3 also does not do this.
306  overlapBox = coaddPatch.getBBox()
307  overlapBox.clip(coaddBBox)
308  coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
309 
310  if coaddFilter is None:
311  coaddFilter = coaddPatch.getFilter()
312 
313  # Retrieve the PSF for this coadd tract, if not already retrieved
314  if coaddPsf is None and coaddPatch.hasPsf():
315  coaddPsf = coaddPatch.getPsf()
316 
317  # Retrieve the calibration for this coadd tract, if not already retrieved
318  if coaddPhotoCalib is None:
319  coaddPhotoCalib = coaddPatch.getPhotoCalib()
320 
321  if coaddPhotoCalib is None:
322  raise RuntimeError("No coadd PhotoCalib found!")
323  if nPatchesFound == 0:
324  raise RuntimeError("No patches found!")
325  if coaddPsf is None:
326  raise RuntimeError("No coadd Psf found!")
327 
328  coaddExposure.setPhotoCalib(coaddPhotoCalib)
329  coaddExposure.setPsf(coaddPsf)
330  coaddExposure.setFilter(coaddFilter)
331  return coaddExposure
332 
334  """Return coadd name for given task config
335 
336  Returns
337  -------
338  CoaddDatasetName : `string`
339 
340  TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
341  """
342  warpType = self.config.warpType
343  suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
344  return self.config.coaddName + "Coadd" + suffix
345 
346 
347 class GetCalexpAsTemplateConfig(pexConfig.Config):
348  doAddCalexpBackground = pexConfig.Field(
349  dtype=bool,
350  default=True,
351  doc="Add background to calexp before processing it."
352  )
353 
354 
355 class GetCalexpAsTemplateTask(pipeBase.Task):
356  """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
357  for use as an image difference template. Only gen2 supported.
358 
359  To be run as a subtask by pipe.tasks.ImageDifferenceTask.
360  Intended for use with simulations and surveys that repeatedly visit the same pointing.
361  This code was originally part of Winter2013ImageDifferenceTask.
362  """
363 
364  ConfigClass = GetCalexpAsTemplateConfig
365  _DefaultName = "GetCalexpAsTemplateTask"
366 
367  def run(self, exposure, sensorRef, templateIdList):
368  """Return a calexp exposure with based on input sensorRef.
369 
370  Construct a dataId based on the sensorRef.dataId combined
371  with the specifications from the first dataId in templateIdList
372 
373  Parameters
374  ----------
375  exposure : `lsst.afw.image.Exposure`
376  exposure (unused)
377  sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
378  Data reference of the calexp(s) to subtract from.
379  templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
380  Data reference of the template calexp to be subtraced.
381  Can be incomplete, fields are initialized from `sensorRef`.
382  If there are multiple items, only the first one is used.
383 
384  Returns
385  -------
386  result : `struct`
387 
388  return a pipeBase.Struct:
389 
390  - ``exposure`` : a template calexp
391  - ``sources`` : source catalog measured on the template
392  """
393 
394  if len(templateIdList) == 0:
395  raise RuntimeError("No template data reference supplied.")
396  if len(templateIdList) > 1:
397  self.log.warn("Multiple template data references supplied. Using the first one only.")
398 
399  templateId = sensorRef.dataId.copy()
400  templateId.update(templateIdList[0])
401 
402  self.log.info("Fetching calexp (%s) as template." % (templateId))
403 
404  butler = sensorRef.getButler()
405  template = butler.get(datasetType="calexp", dataId=templateId)
406  if self.config.doAddCalexpBackground:
407  templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
408  mi = template.getMaskedImage()
409  mi += templateBg.getImage()
410 
411  if not template.hasPsf():
412  raise pipeBase.TaskError("Template has no psf")
413 
414  templateSources = butler.get(datasetType="src", dataId=templateId)
415  return pipeBase.Struct(exposure=template,
416  sources=templateSources)
417 
418  def runDataRef(self, *args, **kwargs):
419  return self.run(*args, **kwargs)
420 
421  def runQuantum(self, **kwargs):
422  raise NotImplementedError("Calexp template is not supported with gen3 middleware")
def run(self, exposure, sensorRef, templateIdList)
Definition: getTemplate.py:367
def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs)
Definition: getTemplate.py:124
def runDataRef(self, exposure, sensorRef, templateIdList=None)
Definition: getTemplate.py:80
def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, sensorRef=None, visitInfo=None)
Definition: getTemplate.py:200