lsst.ip.diffim  20.0.0-7-gcda7bf1+8720e7b1f5
getTemplate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 
25 import lsst.afw.image as afwImage
26 import lsst.geom as geom
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 from lsst.ip.diffim.dcrModel import DcrModel
30 
31 __all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
32  "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
33 
34 
35 class GetCoaddAsTemplateConfig(pexConfig.Config):
36  templateBorderSize = pexConfig.Field(
37  dtype=int,
38  default=10,
39  doc="Number of pixels to grow the requested template image to account for warping"
40  )
41  coaddName = pexConfig.Field(
42  doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
43  dtype=str,
44  default="deep",
45  )
46  numSubfilters = pexConfig.Field(
47  doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'",
48  dtype=int,
49  default=3,
50  )
51  warpType = pexConfig.Field(
52  doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
53  dtype=str,
54  default="direct",
55  )
56 
57 
58 class GetCoaddAsTemplateTask(pipeBase.Task):
59  """Subtask to retrieve coadd for use as an image difference template.
60 
61  This is the default getTemplate Task to be run as a subtask by
62  ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
63  ``runGen3()``.
64 
65  Notes
66  -----
67  From the given skymap, the closest tract is selected; multiple tracts are
68  not supported. The assembled template inherits the WCS of the selected
69  skymap tract and the resolution of the template exposures. Overlapping box
70  regions of the input template patches are pixel by pixel copied into the
71  assembled template image. There is no warping or pixel resampling.
72 
73  Pixels with no overlap of any available input patches are set to ``nan`` value
74  and ``NO_DATA`` flagged.
75  """
76 
77  ConfigClass = GetCoaddAsTemplateConfig
78  _DefaultName = "GetCoaddAsTemplateTask"
79 
80  def runDataRef(self, exposure, sensorRef, templateIdList=None):
81  """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
82  that overlaps the science exposure.
83 
84  Parameters
85  ----------
86  exposure: `lsst.afw.image.Exposure`
87  an exposure for which to generate an overlapping template
88  sensorRef : TYPE
89  a Butler data reference that can be used to obtain coadd data
90  templateIdList : TYPE, optional
91  list of data ids, unused here, in the case of coadd template
92 
93  Returns
94  -------
95  result : `lsst.pipe.base.Struct`
96  - ``exposure`` : `lsst.afw.image.ExposureF`
97  a template coadd exposure assembled out of patches
98  - ``sources`` : None for this subtask
99  """
100  skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
101  tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
102 
103  availableCoaddRefs = dict()
104  for patchInfo in patchList:
105  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
106  patchArgDict = dict(
107  datasetType=self.getCoaddDatasetName() + "_sub",
108  bbox=patchInfo.getOuterBBox(),
109  tract=tractInfo.getId(),
110  patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
111  subfilter=0,
112  numSubfilters=self.config.numSubfilters,
113  )
114 
115  if sensorRef.datasetExists(**patchArgDict):
116  self.log.info("Reading patch %s" % patchArgDict)
117  availableCoaddRefs[patchNumber] = patchArgDict
118 
119  templateExposure = self.run(
120  tractInfo, patchList, skyCorners, availableCoaddRefs,
121  sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
122  )
123  return pipeBase.Struct(exposure=templateExposure, sources=None)
124 
125  def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
126  """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
127  that overlaps the science exposure.
128 
129  Parameters
130  ----------
131  exposure : `lsst.afw.image.Exposure`
132  The science exposure to define the sky region of the template coadd.
133  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
134  Butler like object that supports getting data by DatasetRef.
135  skyMapRef : `lsst.daf.butler.DatasetRef`
136  Reference to SkyMap object that corresponds to the template coadd.
137  coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
138  Iterable of references to the available template coadd patches.
139 
140  Returns
141  -------
142  result : `lsst.pipe.base.Struct`
143  - ``exposure`` : `lsst.afw.image.ExposureF`
144  a template coadd exposure assembled out of patches
145  - ``sources`` : `None` for this subtask
146  """
147  skyMap = butlerQC.get(skyMapRef)
148  tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
149  patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList)
150 
151  availableCoaddRefs = dict()
152  for coaddRef in coaddExposureRefs:
153  dataId = coaddRef.datasetRef.dataId
154  if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter:
155  if self.config.coaddName == 'dcr':
156  self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" %
157  (tractInfo.getId(), dataId['patch'], dataId['subfilter']))
158  if dataId['patch'] in availableCoaddRefs:
159  availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef))
160  else:
161  availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ]
162  else:
163  self.log.info("Using template input tract=%s, patch=%s" %
164  (tractInfo.getId(), dataId['patch']))
165  availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef)
166 
167  templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs,
168  visitInfo=exposure.getInfo().getVisitInfo())
169  return pipeBase.Struct(exposure=templateExposure, sources=None)
170 
171  def getOverlapPatchList(self, exposure, skyMap):
172  """Select the relevant tract and its patches that overlap with the science exposure.
173 
174  Parameters
175  ----------
176  exposure : `lsst.afw.image.Exposure`
177  The science exposure to define the sky region of the template coadd.
178 
179  skyMap : `lsst.skymap.BaseSkyMap`
180  SkyMap object that corresponds to the template coadd.
181 
182  Returns
183  -------
184  result : `tuple` of
185  - ``tractInfo`` : `lsst.skymap.TractInfo`
186  The selected tract.
187  - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
188  List of all overlap patches of the selected tract.
189  - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
190  Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
191  """
192  expWcs = exposure.getWcs()
193  expBoxD = geom.Box2D(exposure.getBBox())
194  expBoxD.grow(self.config.templateBorderSize)
195  ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
196  tractInfo = skyMap.findTract(ctrSkyPos)
197  self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
198  skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
199  patchList = tractInfo.findPatchList(skyCorners)
200 
201  if not patchList:
202  raise RuntimeError("No suitable tract found")
203 
204  self.log.info("Assembling %s coadd patches" % (len(patchList),))
205  self.log.info("exposure dimensions=%s" % exposure.getDimensions())
206 
207  return (tractInfo, patchList, skyCorners)
208 
209  def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
210  sensorRef=None, visitInfo=None):
211  """Gen2 and gen3 shared code: determination of exposure dimensions and
212  copying of pixels from overlapping patch regions.
213 
214  Parameters
215  ----------
216  skyMap : `lsst.skymap.BaseSkyMap`
217  SkyMap object that corresponds to the template coadd.
218  tractInfo : `lsst.skymap.TractInfo`
219  The selected tract.
220  patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
221  Patches to consider for making the template exposure.
222  skyCorners : list of `lsst.geom.SpherePoint`
223  Sky corner coordinates to be covered by the template exposure.
224  availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
225  `dict` (Gen2)
226  Dictionary of spatially relevant retrieved coadd patches,
227  indexed by their sequential patch number. In Gen3 mode, .get() is called,
228  in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
229  sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
230  Butler data reference to get coadd data.
231  Must be `None` for Gen3.
232  visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
233  VisitInfo to make dcr model.
234 
235  Returns
236  -------
237  templateExposure: `lsst.afw.image.ExposureF`
238  The created template exposure.
239  """
240  coaddWcs = tractInfo.getWcs()
241 
242  # compute coadd bbox
243  coaddBBox = geom.Box2D()
244  for skyPos in skyCorners:
245  coaddBBox.include(coaddWcs.skyToPixel(skyPos))
246  coaddBBox = geom.Box2I(coaddBBox)
247  self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
248 
249  coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
250  coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
251  nPatchesFound = 0
252  coaddFilter = None
253  coaddPsf = None
254  coaddPhotoCalib = None
255  for patchInfo in patchList:
256  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
257  patchSubBBox = patchInfo.getOuterBBox()
258  patchSubBBox.clip(coaddBBox)
259  if patchNumber not in availableCoaddRefs:
260  self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd")
261  continue
262  if patchSubBBox.isEmpty():
263  self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, "
264  f"patch={patchNumber}; no overlapping pixels")
265  continue
266 
267  if self.config.coaddName == 'dcr':
268  patchInnerBBox = patchInfo.getInnerBBox()
269  patchInnerBBox.clip(coaddBBox)
270  if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
271  self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels."
272  % availableCoaddRefs[patchNumber])
273  continue
274  self.log.info("Constructing DCR-matched template for patch %s"
275  % availableCoaddRefs[patchNumber])
276 
277  if sensorRef:
278  dcrModel = DcrModel.fromDataRef(sensorRef, **availableCoaddRefs[patchNumber])
279  else:
280  dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber])
281  # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
282  # Each patch has significant overlap, and the contaminated edge pixels in
283  # a new patch will overwrite good pixels in the overlap region from
284  # previous patches.
285  # Shrink the BBox to remove the contaminated pixels,
286  # but make sure it is only the overlap region that is reduced.
287  dcrBBox = geom.Box2I(patchSubBBox)
288  dcrBBox.grow(-self.config.templateBorderSize)
289  dcrBBox.include(patchInnerBBox)
290  coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
291  wcs=coaddWcs,
292  visitInfo=visitInfo)
293  else:
294  if sensorRef is None:
295  # Gen3
296  coaddPatch = availableCoaddRefs[patchNumber].get()
297  else:
298  # Gen2
299  coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
300  nPatchesFound += 1
301 
302  # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
303  # calculation from caller code. Gen3 also does not do this.
304  overlapBox = coaddPatch.getBBox()
305  overlapBox.clip(coaddBBox)
306  coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
307 
308  if coaddFilter is None:
309  coaddFilter = coaddPatch.getFilter()
310 
311  # Retrieve the PSF for this coadd tract, if not already retrieved
312  if coaddPsf is None and coaddPatch.hasPsf():
313  coaddPsf = coaddPatch.getPsf()
314 
315  # Retrieve the calibration for this coadd tract, if not already retrieved
316  if coaddPhotoCalib is None:
317  coaddPhotoCalib = coaddPatch.getPhotoCalib()
318 
319  if coaddPhotoCalib is None:
320  raise RuntimeError("No coadd PhotoCalib found!")
321  if nPatchesFound == 0:
322  raise RuntimeError("No patches found!")
323  if coaddPsf is None:
324  raise RuntimeError("No coadd Psf found!")
325 
326  coaddExposure.setPhotoCalib(coaddPhotoCalib)
327  coaddExposure.setPsf(coaddPsf)
328  coaddExposure.setFilter(coaddFilter)
329  return coaddExposure
330 
332  """Return coadd name for given task config
333 
334  Returns
335  -------
336  CoaddDatasetName : `string`
337 
338  TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
339  """
340  warpType = self.config.warpType
341  suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
342  return self.config.coaddName + "Coadd" + suffix
343 
344 
345 class GetCalexpAsTemplateConfig(pexConfig.Config):
346  doAddCalexpBackground = pexConfig.Field(
347  dtype=bool,
348  default=True,
349  doc="Add background to calexp before processing it."
350  )
351 
352 
353 class GetCalexpAsTemplateTask(pipeBase.Task):
354  """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
355  for use as an image difference template. Only gen2 supported.
356 
357  To be run as a subtask by pipe.tasks.ImageDifferenceTask.
358  Intended for use with simulations and surveys that repeatedly visit the same pointing.
359  This code was originally part of Winter2013ImageDifferenceTask.
360  """
361 
362  ConfigClass = GetCalexpAsTemplateConfig
363  _DefaultName = "GetCalexpAsTemplateTask"
364 
365  def run(self, exposure, sensorRef, templateIdList):
366  """Return a calexp exposure with based on input sensorRef.
367 
368  Construct a dataId based on the sensorRef.dataId combined
369  with the specifications from the first dataId in templateIdList
370 
371  Parameters
372  ----------
373  exposure : `lsst.afw.image.Exposure`
374  exposure (unused)
375  sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
376  Data reference of the calexp(s) to subtract from.
377  templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
378  Data reference of the template calexp to be subtraced.
379  Can be incomplete, fields are initialized from `sensorRef`.
380  If there are multiple items, only the first one is used.
381 
382  Returns
383  -------
384  result : `struct`
385 
386  return a pipeBase.Struct:
387 
388  - ``exposure`` : a template calexp
389  - ``sources`` : source catalog measured on the template
390  """
391 
392  if len(templateIdList) == 0:
393  raise RuntimeError("No template data reference supplied.")
394  if len(templateIdList) > 1:
395  self.log.warn("Multiple template data references supplied. Using the first one only.")
396 
397  templateId = sensorRef.dataId.copy()
398  templateId.update(templateIdList[0])
399 
400  self.log.info("Fetching calexp (%s) as template." % (templateId))
401 
402  butler = sensorRef.getButler()
403  template = butler.get(datasetType="calexp", dataId=templateId)
404  if self.config.doAddCalexpBackground:
405  templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
406  mi = template.getMaskedImage()
407  mi += templateBg.getImage()
408 
409  if not template.hasPsf():
410  raise pipeBase.TaskError("Template has no psf")
411 
412  templateSources = butler.get(datasetType="src", dataId=templateId)
413  return pipeBase.Struct(exposure=template,
414  sources=templateSources)
415 
416  def runDataRef(self, *args, **kwargs):
417  return self.run(*args, **kwargs)
418 
419  def runQuantum(self, **kwargs):
420  raise NotImplementedError("Calexp template is not supported with gen3 middleware")
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateTask.runQuantum
def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs)
Definition: getTemplate.py:125
lsst::afw::image
lsst::ip::diffim.getTemplate.GetCalexpAsTemplateConfig
Definition: getTemplate.py:345
lsst::ip::diffim.getTemplate.GetCalexpAsTemplateTask.runQuantum
def runQuantum(self, **kwargs)
Definition: getTemplate.py:419
lsst::ip::diffim.dcrModel
Definition: dcrModel.py:1
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateTask.getCoaddDatasetName
def getCoaddDatasetName(self)
Definition: getTemplate.py:331
lsst::ip::diffim.getTemplate.GetCalexpAsTemplateTask
Definition: getTemplate.py:353
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateConfig
Definition: getTemplate.py:35
lsst::ip::diffim.getTemplate.GetCalexpAsTemplateTask.run
def run(self, exposure, sensorRef, templateIdList)
Definition: getTemplate.py:365
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateTask.getOverlapPatchList
def getOverlapPatchList(self, exposure, skyMap)
Definition: getTemplate.py:171
lsst::pex::config
lsst::ip::diffim.getTemplate.GetCalexpAsTemplateTask.runDataRef
def runDataRef(self, *args, **kwargs)
Definition: getTemplate.py:416
lsst::geom
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateTask.runDataRef
def runDataRef(self, exposure, sensorRef, templateIdList=None)
Definition: getTemplate.py:80
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateTask
Definition: getTemplate.py:58
lsst::geom::Box2I
lsst::ip::diffim.getTemplate.GetCoaddAsTemplateTask.run
def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, sensorRef=None, visitInfo=None)
Definition: getTemplate.py:209
lsst::geom::Box2D