Coverage for python/lsst/ip/diffim/getTemplate.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.geom as geom
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from lsst.ip.diffim.dcrModel import DcrModel
31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
35class GetCoaddAsTemplateConfig(pexConfig.Config):
36 templateBorderSize = pexConfig.Field(
37 dtype=int,
38 default=10,
39 doc="Number of pixels to grow the requested template image to account for warping"
40 )
41 coaddName = pexConfig.Field(
42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
43 dtype=str,
44 default="deep",
45 )
46 numSubfilters = pexConfig.Field(
47 doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'",
48 dtype=int,
49 default=3,
50 )
51 effectiveWavelength = pexConfig.Field(
52 doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'",
53 optional=True,
54 dtype=float,
55 )
56 bandwidth = pexConfig.Field(
57 doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'",
58 optional=True,
59 dtype=float,
60 )
61 warpType = pexConfig.Field(
62 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
63 dtype=str,
64 default="direct",
65 )
67 def validate(self):
68 if self.coaddName == 'dcr':
69 if self.effectiveWavelength is None or self.bandwidth is None:
70 raise ValueError("The effective wavelength and bandwidth of the physical filter "
71 "must be set in the getTemplate config for DCR coadds. "
72 "Required until transmission curves are used in DM-13668.")
75class GetCoaddAsTemplateTask(pipeBase.Task):
76 """Subtask to retrieve coadd for use as an image difference template.
78 This is the default getTemplate Task to be run as a subtask by
79 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
80 ``runGen3()``.
82 Notes
83 -----
84 From the given skymap, the closest tract is selected; multiple tracts are
85 not supported. The assembled template inherits the WCS of the selected
86 skymap tract and the resolution of the template exposures. Overlapping box
87 regions of the input template patches are pixel by pixel copied into the
88 assembled template image. There is no warping or pixel resampling.
90 Pixels with no overlap of any available input patches are set to ``nan`` value
91 and ``NO_DATA`` flagged.
92 """
94 ConfigClass = GetCoaddAsTemplateConfig
95 _DefaultName = "GetCoaddAsTemplateTask"
97 def runDataRef(self, exposure, sensorRef, templateIdList=None):
98 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
99 that overlaps the science exposure.
101 Parameters
102 ----------
103 exposure: `lsst.afw.image.Exposure`
104 an exposure for which to generate an overlapping template
105 sensorRef : TYPE
106 a Butler data reference that can be used to obtain coadd data
107 templateIdList : TYPE, optional
108 list of data ids, unused here, in the case of coadd template
110 Returns
111 -------
112 result : `lsst.pipe.base.Struct`
113 - ``exposure`` : `lsst.afw.image.ExposureF`
114 a template coadd exposure assembled out of patches
115 - ``sources`` : None for this subtask
116 """
117 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
118 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
120 availableCoaddRefs = dict()
121 for patchInfo in patchList:
122 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
123 patchArgDict = dict(
124 datasetType=self.getCoaddDatasetName() + "_sub",
125 bbox=patchInfo.getOuterBBox(),
126 tract=tractInfo.getId(),
127 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
128 subfilter=0,
129 numSubfilters=self.config.numSubfilters,
130 )
132 if sensorRef.datasetExists(**patchArgDict):
133 self.log.info("Reading patch %s" % patchArgDict)
134 availableCoaddRefs[patchNumber] = patchArgDict
136 templateExposure = self.run(
137 tractInfo, patchList, skyCorners, availableCoaddRefs,
138 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
139 )
140 return pipeBase.Struct(exposure=templateExposure, sources=None)
142 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
143 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
144 that overlaps the science exposure.
146 Parameters
147 ----------
148 exposure : `lsst.afw.image.Exposure`
149 The science exposure to define the sky region of the template coadd.
150 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
151 Butler like object that supports getting data by DatasetRef.
152 skyMapRef : `lsst.daf.butler.DatasetRef`
153 Reference to SkyMap object that corresponds to the template coadd.
154 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
155 Iterable of references to the available template coadd patches.
157 Returns
158 -------
159 result : `lsst.pipe.base.Struct`
160 - ``exposure`` : `lsst.afw.image.ExposureF`
161 a template coadd exposure assembled out of patches
162 - ``sources`` : `None` for this subtask
163 """
164 skyMap = butlerQC.get(skyMapRef)
165 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
166 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList)
168 availableCoaddRefs = dict()
169 for coaddRef in coaddExposureRefs:
170 dataId = coaddRef.datasetRef.dataId
171 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter:
172 if self.config.coaddName == 'dcr':
173 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" %
174 (tractInfo.getId(), dataId['patch'], dataId['subfilter']))
175 if dataId['patch'] in availableCoaddRefs:
176 availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef))
177 else:
178 availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ]
179 else:
180 self.log.info("Using template input tract=%s, patch=%s" %
181 (tractInfo.getId(), dataId['patch']))
182 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef)
184 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs,
185 visitInfo=exposure.getInfo().getVisitInfo())
186 return pipeBase.Struct(exposure=templateExposure, sources=None)
188 def getOverlapPatchList(self, exposure, skyMap):
189 """Select the relevant tract and its patches that overlap with the science exposure.
191 Parameters
192 ----------
193 exposure : `lsst.afw.image.Exposure`
194 The science exposure to define the sky region of the template coadd.
196 skyMap : `lsst.skymap.BaseSkyMap`
197 SkyMap object that corresponds to the template coadd.
199 Returns
200 -------
201 result : `tuple` of
202 - ``tractInfo`` : `lsst.skymap.TractInfo`
203 The selected tract.
204 - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
205 List of all overlap patches of the selected tract.
206 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
207 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
208 """
209 expWcs = exposure.getWcs()
210 expBoxD = geom.Box2D(exposure.getBBox())
211 expBoxD.grow(self.config.templateBorderSize)
212 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
213 tractInfo = skyMap.findTract(ctrSkyPos)
214 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
215 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
216 patchList = tractInfo.findPatchList(skyCorners)
218 if not patchList:
219 raise RuntimeError("No suitable tract found")
221 self.log.info("Assembling %s coadd patches" % (len(patchList),))
222 self.log.info("exposure dimensions=%s" % exposure.getDimensions())
224 return (tractInfo, patchList, skyCorners)
226 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
227 sensorRef=None, visitInfo=None):
228 """Gen2 and gen3 shared code: determination of exposure dimensions and
229 copying of pixels from overlapping patch regions.
231 Parameters
232 ----------
233 skyMap : `lsst.skymap.BaseSkyMap`
234 SkyMap object that corresponds to the template coadd.
235 tractInfo : `lsst.skymap.TractInfo`
236 The selected tract.
237 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
238 Patches to consider for making the template exposure.
239 skyCorners : list of `lsst.geom.SpherePoint`
240 Sky corner coordinates to be covered by the template exposure.
241 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
242 `dict` (Gen2)
243 Dictionary of spatially relevant retrieved coadd patches,
244 indexed by their sequential patch number. In Gen3 mode, .get() is called,
245 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
246 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
247 Butler data reference to get coadd data.
248 Must be `None` for Gen3.
249 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
250 VisitInfo to make dcr model.
252 Returns
253 -------
254 templateExposure: `lsst.afw.image.ExposureF`
255 The created template exposure.
256 """
257 coaddWcs = tractInfo.getWcs()
259 # compute coadd bbox
260 coaddBBox = geom.Box2D()
261 for skyPos in skyCorners:
262 coaddBBox.include(coaddWcs.skyToPixel(skyPos))
263 coaddBBox = geom.Box2I(coaddBBox)
264 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
266 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
267 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
268 nPatchesFound = 0
269 coaddFilter = None
270 coaddPsf = None
271 coaddPhotoCalib = None
272 for patchInfo in patchList:
273 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
274 patchSubBBox = patchInfo.getOuterBBox()
275 patchSubBBox.clip(coaddBBox)
276 if patchNumber not in availableCoaddRefs:
277 self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd")
278 continue
279 if patchSubBBox.isEmpty():
280 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, "
281 f"patch={patchNumber}; no overlapping pixels")
282 continue
284 if self.config.coaddName == 'dcr':
285 patchInnerBBox = patchInfo.getInnerBBox()
286 patchInnerBBox.clip(coaddBBox)
287 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
288 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels."
289 % availableCoaddRefs[patchNumber])
290 continue
291 self.log.info("Constructing DCR-matched template for patch %s"
292 % availableCoaddRefs[patchNumber])
294 if sensorRef:
295 dcrModel = DcrModel.fromDataRef(sensorRef,
296 self.config.effectiveWavelength,
297 self.config.bandwidth,
298 **availableCoaddRefs[patchNumber])
299 else:
300 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber],
301 self.config.effectiveWavelength,
302 self.config.bandwidth)
303 # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
304 # Each patch has significant overlap, and the contaminated edge pixels in
305 # a new patch will overwrite good pixels in the overlap region from
306 # previous patches.
307 # Shrink the BBox to remove the contaminated pixels,
308 # but make sure it is only the overlap region that is reduced.
309 dcrBBox = geom.Box2I(patchSubBBox)
310 dcrBBox.grow(-self.config.templateBorderSize)
311 dcrBBox.include(patchInnerBBox)
312 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
313 wcs=coaddWcs,
314 visitInfo=visitInfo)
315 else:
316 if sensorRef is None:
317 # Gen3
318 coaddPatch = availableCoaddRefs[patchNumber].get()
319 else:
320 # Gen2
321 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
322 nPatchesFound += 1
324 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
325 # calculation from caller code. Gen3 also does not do this.
326 overlapBox = coaddPatch.getBBox()
327 overlapBox.clip(coaddBBox)
328 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
330 if coaddFilter is None:
331 coaddFilter = coaddPatch.getFilter()
333 # Retrieve the PSF for this coadd tract, if not already retrieved
334 if coaddPsf is None and coaddPatch.hasPsf():
335 coaddPsf = coaddPatch.getPsf()
337 # Retrieve the calibration for this coadd tract, if not already retrieved
338 if coaddPhotoCalib is None:
339 coaddPhotoCalib = coaddPatch.getPhotoCalib()
341 if coaddPhotoCalib is None:
342 raise RuntimeError("No coadd PhotoCalib found!")
343 if nPatchesFound == 0:
344 raise RuntimeError("No patches found!")
345 if coaddPsf is None:
346 raise RuntimeError("No coadd Psf found!")
348 coaddExposure.setPhotoCalib(coaddPhotoCalib)
349 coaddExposure.setPsf(coaddPsf)
350 coaddExposure.setFilter(coaddFilter)
351 return coaddExposure
353 def getCoaddDatasetName(self):
354 """Return coadd name for given task config
356 Returns
357 -------
358 CoaddDatasetName : `string`
360 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
361 """
362 warpType = self.config.warpType
363 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
364 return self.config.coaddName + "Coadd" + suffix
367class GetCalexpAsTemplateConfig(pexConfig.Config):
368 doAddCalexpBackground = pexConfig.Field(
369 dtype=bool,
370 default=True,
371 doc="Add background to calexp before processing it."
372 )
375class GetCalexpAsTemplateTask(pipeBase.Task):
376 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
377 for use as an image difference template. Only gen2 supported.
379 To be run as a subtask by pipe.tasks.ImageDifferenceTask.
380 Intended for use with simulations and surveys that repeatedly visit the same pointing.
381 This code was originally part of Winter2013ImageDifferenceTask.
382 """
384 ConfigClass = GetCalexpAsTemplateConfig
385 _DefaultName = "GetCalexpAsTemplateTask"
387 def run(self, exposure, sensorRef, templateIdList):
388 """Return a calexp exposure with based on input sensorRef.
390 Construct a dataId based on the sensorRef.dataId combined
391 with the specifications from the first dataId in templateIdList
393 Parameters
394 ----------
395 exposure : `lsst.afw.image.Exposure`
396 exposure (unused)
397 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
398 Data reference of the calexp(s) to subtract from.
399 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
400 Data reference of the template calexp to be subtraced.
401 Can be incomplete, fields are initialized from `sensorRef`.
402 If there are multiple items, only the first one is used.
404 Returns
405 -------
406 result : `struct`
408 return a pipeBase.Struct:
410 - ``exposure`` : a template calexp
411 - ``sources`` : source catalog measured on the template
412 """
414 if len(templateIdList) == 0:
415 raise RuntimeError("No template data reference supplied.")
416 if len(templateIdList) > 1:
417 self.log.warn("Multiple template data references supplied. Using the first one only.")
419 templateId = sensorRef.dataId.copy()
420 templateId.update(templateIdList[0])
422 self.log.info("Fetching calexp (%s) as template." % (templateId))
424 butler = sensorRef.getButler()
425 template = butler.get(datasetType="calexp", dataId=templateId)
426 if self.config.doAddCalexpBackground:
427 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
428 mi = template.getMaskedImage()
429 mi += templateBg.getImage()
431 if not template.hasPsf():
432 raise pipeBase.TaskError("Template has no psf")
434 templateSources = butler.get(datasetType="src", dataId=templateId)
435 return pipeBase.Struct(exposure=template,
436 sources=templateSources)
438 def runDataRef(self, *args, **kwargs):
439 return self.run(*args, **kwargs)
441 def runQuantum(self, **kwargs):
442 raise NotImplementedError("Calexp template is not supported with gen3 middleware")