lsst.ip.diffim  21.0.0-16-gec61338+5610ae7019
getTemplate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 
25 import lsst.afw.image as afwImage
26 import lsst.geom as geom
27 import lsst.sphgeom as sphgeom
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 from lsst.ip.diffim.dcrModel import DcrModel
31 
32 __all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
33  "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
34 
35 
36 class GetCoaddAsTemplateConfig(pexConfig.Config):
37  templateBorderSize = pexConfig.Field(
38  dtype=int,
39  default=10,
40  doc="Number of pixels to grow the requested template image to account for warping"
41  )
42  coaddName = pexConfig.Field(
43  doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
44  dtype=str,
45  default="deep",
46  )
47  numSubfilters = pexConfig.Field(
48  doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'",
49  dtype=int,
50  default=3,
51  )
52  effectiveWavelength = pexConfig.Field(
53  doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'",
54  optional=True,
55  dtype=float,
56  )
57  bandwidth = pexConfig.Field(
58  doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'",
59  optional=True,
60  dtype=float,
61  )
62  warpType = pexConfig.Field(
63  doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
64  dtype=str,
65  default="direct",
66  )
67 
68  def validate(self):
69  if self.coaddNamecoaddName == 'dcr':
70  if self.effectiveWavelengtheffectiveWavelength is None or self.bandwidthbandwidth is None:
71  raise ValueError("The effective wavelength and bandwidth of the physical filter "
72  "must be set in the getTemplate config for DCR coadds. "
73  "Required until transmission curves are used in DM-13668.")
74 
75 
76 class GetCoaddAsTemplateTask(pipeBase.Task):
77  """Subtask to retrieve coadd for use as an image difference template.
78 
79  This is the default getTemplate Task to be run as a subtask by
80  ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
81  ``runGen3()``.
82 
83  Notes
84  -----
85  From the given skymap, the closest tract is selected; multiple tracts are
86  not supported. The assembled template inherits the WCS of the selected
87  skymap tract and the resolution of the template exposures. Overlapping box
88  regions of the input template patches are pixel by pixel copied into the
89  assembled template image. There is no warping or pixel resampling.
90 
91  Pixels with no overlap of any available input patches are set to ``nan`` value
92  and ``NO_DATA`` flagged.
93  """
94 
95  ConfigClass = GetCoaddAsTemplateConfig
96  _DefaultName = "GetCoaddAsTemplateTask"
97 
98  def runDataRef(self, exposure, sensorRef, templateIdList=None):
99  """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
100  that overlaps the science exposure.
101 
102  Parameters
103  ----------
104  exposure: `lsst.afw.image.Exposure`
105  an exposure for which to generate an overlapping template
106  sensorRef : TYPE
107  a Butler data reference that can be used to obtain coadd data
108  templateIdList : TYPE, optional
109  list of data ids, unused here, in the case of coadd template
110 
111  Returns
112  -------
113  result : `lsst.pipe.base.Struct`
114  - ``exposure`` : `lsst.afw.image.ExposureF`
115  a template coadd exposure assembled out of patches
116  - ``sources`` : None for this subtask
117  """
118  skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
119  tractInfo, patchList, skyCorners = self.getOverlapPatchListgetOverlapPatchList(exposure, skyMap)
120 
121  availableCoaddRefs = dict()
122  for patchInfo in patchList:
123  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
124  patchArgDict = dict(
125  datasetType=self.getCoaddDatasetNamegetCoaddDatasetName() + "_sub",
126  bbox=patchInfo.getOuterBBox(),
127  tract=tractInfo.getId(),
128  patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
129  subfilter=0,
130  numSubfilters=self.config.numSubfilters,
131  )
132 
133  if sensorRef.datasetExists(**patchArgDict):
134  self.log.info("Reading patch %s" % patchArgDict)
135  availableCoaddRefs[patchNumber] = patchArgDict
136 
137  templateExposure = self.runrun(
138  tractInfo, patchList, skyCorners, availableCoaddRefs,
139  sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
140  )
141  return pipeBase.Struct(exposure=templateExposure, sources=None)
142 
143  def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
144  """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
145  that overlaps the science exposure.
146 
147  Parameters
148  ----------
149  exposure : `lsst.afw.image.Exposure`
150  The science exposure to define the sky region of the template coadd.
151  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
152  Butler like object that supports getting data by DatasetRef.
153  skyMapRef : `lsst.daf.butler.DatasetRef`
154  Reference to SkyMap object that corresponds to the template coadd.
155  coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
156  Iterable of references to the available template coadd patches.
157 
158  Returns
159  -------
160  result : `lsst.pipe.base.Struct`
161  - ``exposure`` : `lsst.afw.image.ExposureF`
162  a template coadd exposure assembled out of patches
163  - ``sources`` : `None` for this subtask
164  """
165  skyMap = butlerQC.get(skyMapRef)
166  coaddExposureRefs = butlerQC.get(coaddExposureRefs)
167  tracts = [ref.dataId['tract'] for ref in coaddExposureRefs]
168  if tracts.count(tracts[0]) == len(tracts):
169  tractInfo = skyMap[tracts[0]]
170  else:
171  raise RuntimeError("Templates constructed from multiple Tracts not yet supported")
172 
173  detectorBBox = exposure.getBBox()
174  detectorWcs = exposure.getWcs()
175  detectorCorners = detectorWcs.pixelToSky(geom.Box2D(detectorBBox).getCorners())
176 
177  availableCoaddRefs = dict()
178  for coaddRef in coaddExposureRefs:
179  dataId = coaddRef.dataId
180  patchWcs = skyMap[dataId['tract']].getWcs()
181  patchBBox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
182  if self.bboxIntersectsCornersbboxIntersectsCorners(patchBBox, patchWcs, detectorCorners):
183  if self.config.coaddName == 'dcr':
184  self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" %
185  (dataId['tract'], dataId['patch'], dataId['subfilter']))
186  if dataId['patch'] in availableCoaddRefs:
187  availableCoaddRefs[dataId['patch']].append(coaddRef)
188  else:
189  availableCoaddRefs[dataId['patch']] = [coaddRef, ]
190  else:
191  self.log.info("Using template input tract=%s, patch=%s" %
192  (dataId['tract'], dataId['patch']))
193  availableCoaddRefs[dataId['patch']] = coaddRef
194 
195  patchList = [tractInfo[patch] for patch in availableCoaddRefs.keys()]
196  templateExposure = self.runrun(tractInfo, patchList, detectorCorners, availableCoaddRefs,
197  visitInfo=exposure.getInfo().getVisitInfo())
198  return pipeBase.Struct(exposure=templateExposure, sources=None)
199 
200  def getOverlapPatchList(self, exposure, skyMap):
201  """Select the relevant tract and its patches that overlap with the science exposure.
202 
203  Parameters
204  ----------
205  exposure : `lsst.afw.image.Exposure`
206  The science exposure to define the sky region of the template coadd.
207 
208  skyMap : `lsst.skymap.BaseSkyMap`
209  SkyMap object that corresponds to the template coadd.
210 
211  Returns
212  -------
213  result : `tuple` of
214  - ``tractInfo`` : `lsst.skymap.TractInfo`
215  The selected tract.
216  - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
217  List of all overlap patches of the selected tract.
218  - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
219  Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
220  """
221  expWcs = exposure.getWcs()
222  expBoxD = geom.Box2D(exposure.getBBox())
223  expBoxD.grow(self.config.templateBorderSize)
224  ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
225  tractInfo = skyMap.findTract(ctrSkyPos)
226  self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
227  skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
228  patchList = tractInfo.findPatchList(skyCorners)
229 
230  if not patchList:
231  raise RuntimeError("No suitable tract found")
232 
233  self.log.info("Assembling %s coadd patches" % (len(patchList),))
234  self.log.info("exposure dimensions=%s" % exposure.getDimensions())
235 
236  return (tractInfo, patchList, skyCorners)
237 
238  def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
239  sensorRef=None, visitInfo=None):
240  """Gen2 and gen3 shared code: determination of exposure dimensions and
241  copying of pixels from overlapping patch regions.
242 
243  Parameters
244  ----------
245  skyMap : `lsst.skymap.BaseSkyMap`
246  SkyMap object that corresponds to the template coadd.
247  tractInfo : `lsst.skymap.TractInfo`
248  The selected tract.
249  patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
250  Patches to consider for making the template exposure.
251  skyCorners : list of `lsst.geom.SpherePoint`
252  Sky corner coordinates to be covered by the template exposure.
253  availableCoaddRefs : `dict` [`int`]
254  Dictionary of spatially relevant retrieved coadd patches,
255  indexed by their sequential patch number. In Gen3 mode, values are
256  `lsst.daf.butler.DeferredDatasetHandle` and ``.get()`` is called,
257  in Gen2 mode, ``sensorRef.get(**coaddef)`` is called to retrieve the coadd.
258  sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
259  Butler data reference to get coadd data.
260  Must be `None` for Gen3.
261  visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
262  VisitInfo to make dcr model.
263 
264  Returns
265  -------
266  templateExposure: `lsst.afw.image.ExposureF`
267  The created template exposure.
268  """
269  coaddWcs = tractInfo.getWcs()
270 
271  # compute coadd bbox
272  coaddBBox = geom.Box2D()
273  for skyPos in skyCorners:
274  coaddBBox.include(coaddWcs.skyToPixel(skyPos))
275  coaddBBox = geom.Box2I(coaddBBox)
276  self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
277 
278  coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
279  coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
280  nPatchesFound = 0
281  coaddFilterLabel = None
282  coaddPsf = None
283  coaddPhotoCalib = None
284  for patchInfo in patchList:
285  patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
286  patchSubBBox = patchInfo.getOuterBBox()
287  patchSubBBox.clip(coaddBBox)
288  if patchNumber not in availableCoaddRefs:
289  self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd")
290  continue
291  if patchSubBBox.isEmpty():
292  self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, "
293  f"patch={patchNumber}; no overlapping pixels")
294  continue
295 
296  if self.config.coaddName == 'dcr':
297  patchInnerBBox = patchInfo.getInnerBBox()
298  patchInnerBBox.clip(coaddBBox)
299  if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
300  self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels."
301  % availableCoaddRefs[patchNumber])
302  continue
303  self.log.info("Constructing DCR-matched template for patch %s"
304  % availableCoaddRefs[patchNumber])
305 
306  if sensorRef:
307  dcrModel = DcrModel.fromDataRef(sensorRef,
308  self.config.effectiveWavelength,
309  self.config.bandwidth,
310  **availableCoaddRefs[patchNumber])
311  else:
312  dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber],
313  self.config.effectiveWavelength,
314  self.config.bandwidth)
315  # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
316  # Each patch has significant overlap, and the contaminated edge pixels in
317  # a new patch will overwrite good pixels in the overlap region from
318  # previous patches.
319  # Shrink the BBox to remove the contaminated pixels,
320  # but make sure it is only the overlap region that is reduced.
321  dcrBBox = geom.Box2I(patchSubBBox)
322  dcrBBox.grow(-self.config.templateBorderSize)
323  dcrBBox.include(patchInnerBBox)
324  coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
325  wcs=coaddWcs,
326  visitInfo=visitInfo)
327  else:
328  if sensorRef is None:
329  # Gen3
330  coaddPatch = availableCoaddRefs[patchNumber].get()
331  else:
332  # Gen2
333  coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
334  nPatchesFound += 1
335 
336  # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
337  # calculation from caller code. Gen3 also does not do this.
338  overlapBox = coaddPatch.getBBox()
339  overlapBox.clip(coaddBBox)
340  coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
341 
342  if coaddFilterLabel is None:
343  coaddFilterLabel = coaddPatch.getFilterLabel()
344 
345  # Retrieve the PSF for this coadd tract, if not already retrieved
346  if coaddPsf is None and coaddPatch.hasPsf():
347  coaddPsf = coaddPatch.getPsf()
348 
349  # Retrieve the calibration for this coadd tract, if not already retrieved
350  if coaddPhotoCalib is None:
351  coaddPhotoCalib = coaddPatch.getPhotoCalib()
352 
353  if coaddPhotoCalib is None:
354  raise RuntimeError("No coadd PhotoCalib found!")
355  if nPatchesFound == 0:
356  raise RuntimeError("No patches found!")
357  if coaddPsf is None:
358  raise RuntimeError("No coadd Psf found!")
359 
360  coaddExposure.setPhotoCalib(coaddPhotoCalib)
361  coaddExposure.setPsf(coaddPsf)
362  coaddExposure.setFilterLabel(coaddFilterLabel)
363  return coaddExposure
364 
366  """Return coadd name for given task config
367 
368  Returns
369  -------
370  CoaddDatasetName : `string`
371 
372  TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
373  """
374  warpType = self.config.warpType
375  suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
376  return self.config.coaddName + "Coadd" + suffix
377 
378  def bboxIntersectsCorners(self, bbox, wcs, otherCorners):
379  """Returns true if the bbox with wcs intersects otherCorners
380 
381  Parameters
382  ----------
383  bbox : `lsst.geom.Box2I`
384  specifying the bounding box of test region
385  wcs : `lsst.afw.geom.SkyWcs`
386  specifying the WCS of test region
387  otherCorners : `list` of `lsst.geom.SpherePoint`
388  ICRS coordinates specifying boundary of the other sky region
389 
390  Returns
391  -------
392  result: `bool`
393  Does bbox/wcs intersect other corners?
394  """
395  bboxCorners = wcs.pixelToSky(geom.Box2D(bbox).getCorners())
396  bboxPolygon = sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in bboxCorners])
397  otherPolygon = sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in otherCorners])
398  return otherPolygon.intersects(bboxPolygon)
399 
400 
401 class GetCalexpAsTemplateConfig(pexConfig.Config):
402  doAddCalexpBackground = pexConfig.Field(
403  dtype=bool,
404  default=True,
405  doc="Add background to calexp before processing it."
406  )
407 
408 
409 class GetCalexpAsTemplateTask(pipeBase.Task):
410  """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
411  for use as an image difference template. Only gen2 supported.
412 
413  To be run as a subtask by pipe.tasks.ImageDifferenceTask.
414  Intended for use with simulations and surveys that repeatedly visit the same pointing.
415  This code was originally part of Winter2013ImageDifferenceTask.
416  """
417 
418  ConfigClass = GetCalexpAsTemplateConfig
419  _DefaultName = "GetCalexpAsTemplateTask"
420 
421  def run(self, exposure, sensorRef, templateIdList):
422  """Return a calexp exposure with based on input sensorRef.
423 
424  Construct a dataId based on the sensorRef.dataId combined
425  with the specifications from the first dataId in templateIdList
426 
427  Parameters
428  ----------
429  exposure : `lsst.afw.image.Exposure`
430  exposure (unused)
431  sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
432  Data reference of the calexp(s) to subtract from.
433  templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
434  Data reference of the template calexp to be subtraced.
435  Can be incomplete, fields are initialized from `sensorRef`.
436  If there are multiple items, only the first one is used.
437 
438  Returns
439  -------
440  result : `struct`
441 
442  return a pipeBase.Struct:
443 
444  - ``exposure`` : a template calexp
445  - ``sources`` : source catalog measured on the template
446  """
447 
448  if len(templateIdList) == 0:
449  raise RuntimeError("No template data reference supplied.")
450  if len(templateIdList) > 1:
451  self.log.warn("Multiple template data references supplied. Using the first one only.")
452 
453  templateId = sensorRef.dataId.copy()
454  templateId.update(templateIdList[0])
455 
456  self.log.info("Fetching calexp (%s) as template." % (templateId))
457 
458  butler = sensorRef.getButler()
459  template = butler.get(datasetType="calexp", dataId=templateId)
460  if self.config.doAddCalexpBackground:
461  templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
462  mi = template.getMaskedImage()
463  mi += templateBg.getImage()
464 
465  if not template.hasPsf():
466  raise pipeBase.TaskError("Template has no psf")
467 
468  templateSources = butler.get(datasetType="src", dataId=templateId)
469  return pipeBase.Struct(exposure=template,
470  sources=templateSources)
471 
472  def runDataRef(self, *args, **kwargs):
473  return self.runrun(*args, **kwargs)
474 
475  def runQuantum(self, **kwargs):
476  raise NotImplementedError("Calexp template is not supported with gen3 middleware")
def run(self, exposure, sensorRef, templateIdList)
Definition: getTemplate.py:421
def bboxIntersectsCorners(self, bbox, wcs, otherCorners)
Definition: getTemplate.py:378
def runDataRef(self, exposure, sensorRef, templateIdList=None)
Definition: getTemplate.py:98
def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, sensorRef=None, visitInfo=None)
Definition: getTemplate.py:239
def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs)
Definition: getTemplate.py:143