Coverage for python/lsst/ip/diffim/getTemplate.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.geom as geom
27import lsst.sphgeom as sphgeom
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30from lsst.ip.diffim.dcrModel import DcrModel
32__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
33 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
36class GetCoaddAsTemplateConfig(pexConfig.Config):
37 templateBorderSize = pexConfig.Field(
38 dtype=int,
39 default=10,
40 doc="Number of pixels to grow the requested template image to account for warping"
41 )
42 coaddName = pexConfig.Field(
43 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
44 dtype=str,
45 default="deep",
46 )
47 numSubfilters = pexConfig.Field(
48 doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'",
49 dtype=int,
50 default=3,
51 )
52 effectiveWavelength = pexConfig.Field(
53 doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'",
54 optional=True,
55 dtype=float,
56 )
57 bandwidth = pexConfig.Field(
58 doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'",
59 optional=True,
60 dtype=float,
61 )
62 warpType = pexConfig.Field(
63 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
64 dtype=str,
65 default="direct",
66 )
68 def validate(self):
69 if self.coaddName == 'dcr':
70 if self.effectiveWavelength is None or self.bandwidth is None:
71 raise ValueError("The effective wavelength and bandwidth of the physical filter "
72 "must be set in the getTemplate config for DCR coadds. "
73 "Required until transmission curves are used in DM-13668.")
76class GetCoaddAsTemplateTask(pipeBase.Task):
77 """Subtask to retrieve coadd for use as an image difference template.
79 This is the default getTemplate Task to be run as a subtask by
80 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
81 ``runGen3()``.
83 Notes
84 -----
85 From the given skymap, the closest tract is selected; multiple tracts are
86 not supported. The assembled template inherits the WCS of the selected
87 skymap tract and the resolution of the template exposures. Overlapping box
88 regions of the input template patches are pixel by pixel copied into the
89 assembled template image. There is no warping or pixel resampling.
91 Pixels with no overlap of any available input patches are set to ``nan`` value
92 and ``NO_DATA`` flagged.
93 """
95 ConfigClass = GetCoaddAsTemplateConfig
96 _DefaultName = "GetCoaddAsTemplateTask"
98 def runDataRef(self, exposure, sensorRef, templateIdList=None):
99 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
100 that overlaps the science exposure.
102 Parameters
103 ----------
104 exposure: `lsst.afw.image.Exposure`
105 an exposure for which to generate an overlapping template
106 sensorRef : TYPE
107 a Butler data reference that can be used to obtain coadd data
108 templateIdList : TYPE, optional
109 list of data ids, unused here, in the case of coadd template
111 Returns
112 -------
113 result : `lsst.pipe.base.Struct`
114 - ``exposure`` : `lsst.afw.image.ExposureF`
115 a template coadd exposure assembled out of patches
116 - ``sources`` : None for this subtask
117 """
118 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
119 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
121 availableCoaddRefs = dict()
122 for patchInfo in patchList:
123 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
124 patchArgDict = dict(
125 datasetType=self.getCoaddDatasetName() + "_sub",
126 bbox=patchInfo.getOuterBBox(),
127 tract=tractInfo.getId(),
128 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
129 subfilter=0,
130 numSubfilters=self.config.numSubfilters,
131 )
133 if sensorRef.datasetExists(**patchArgDict):
134 self.log.info("Reading patch %s" % patchArgDict)
135 availableCoaddRefs[patchNumber] = patchArgDict
137 templateExposure = self.run(
138 tractInfo, patchList, skyCorners, availableCoaddRefs,
139 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
140 )
141 return pipeBase.Struct(exposure=templateExposure, sources=None)
143 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
144 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
145 that overlaps the science exposure.
147 Parameters
148 ----------
149 exposure : `lsst.afw.image.Exposure`
150 The science exposure to define the sky region of the template coadd.
151 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
152 Butler like object that supports getting data by DatasetRef.
153 skyMapRef : `lsst.daf.butler.DatasetRef`
154 Reference to SkyMap object that corresponds to the template coadd.
155 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
156 Iterable of references to the available template coadd patches.
158 Returns
159 -------
160 result : `lsst.pipe.base.Struct`
161 - ``exposure`` : `lsst.afw.image.ExposureF`
162 a template coadd exposure assembled out of patches
163 - ``sources`` : `None` for this subtask
164 """
165 skyMap = butlerQC.get(skyMapRef)
166 coaddExposureRefs = butlerQC.get(coaddExposureRefs)
167 tracts = [ref.dataId['tract'] for ref in coaddExposureRefs]
168 if tracts.count(tracts[0]) == len(tracts):
169 tractInfo = skyMap[tracts[0]]
170 else:
171 raise RuntimeError("Templates constructed from multiple Tracts not yet supported")
173 detectorBBox = exposure.getBBox()
174 detectorWcs = exposure.getWcs()
175 detectorCorners = detectorWcs.pixelToSky(geom.Box2D(detectorBBox).getCorners())
177 availableCoaddRefs = dict()
178 for coaddRef in coaddExposureRefs:
179 dataId = coaddRef.dataId
180 patchWcs = skyMap[dataId['tract']].getWcs()
181 patchBBox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
182 if self.bboxIntersectsCorners(patchBBox, patchWcs, detectorCorners):
183 if self.config.coaddName == 'dcr':
184 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" %
185 (dataId['tract'], dataId['patch'], dataId['subfilter']))
186 if dataId['patch'] in availableCoaddRefs:
187 availableCoaddRefs[dataId['patch']].append(coaddRef)
188 else:
189 availableCoaddRefs[dataId['patch']] = [coaddRef, ]
190 else:
191 self.log.info("Using template input tract=%s, patch=%s" %
192 (dataId['tract'], dataId['patch']))
193 availableCoaddRefs[dataId['patch']] = coaddRef
195 patchList = [tractInfo[patch] for patch in availableCoaddRefs.keys()]
196 templateExposure = self.run(tractInfo, patchList, detectorCorners, availableCoaddRefs,
197 visitInfo=exposure.getInfo().getVisitInfo())
198 return pipeBase.Struct(exposure=templateExposure, sources=None)
200 def getOverlapPatchList(self, exposure, skyMap):
201 """Select the relevant tract and its patches that overlap with the science exposure.
203 Parameters
204 ----------
205 exposure : `lsst.afw.image.Exposure`
206 The science exposure to define the sky region of the template coadd.
208 skyMap : `lsst.skymap.BaseSkyMap`
209 SkyMap object that corresponds to the template coadd.
211 Returns
212 -------
213 result : `tuple` of
214 - ``tractInfo`` : `lsst.skymap.TractInfo`
215 The selected tract.
216 - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
217 List of all overlap patches of the selected tract.
218 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
219 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
220 """
221 expWcs = exposure.getWcs()
222 expBoxD = geom.Box2D(exposure.getBBox())
223 expBoxD.grow(self.config.templateBorderSize)
224 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
225 tractInfo = skyMap.findTract(ctrSkyPos)
226 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
227 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
228 patchList = tractInfo.findPatchList(skyCorners)
230 if not patchList:
231 raise RuntimeError("No suitable tract found")
233 self.log.info("Assembling %s coadd patches" % (len(patchList),))
234 self.log.info("exposure dimensions=%s" % exposure.getDimensions())
236 return (tractInfo, patchList, skyCorners)
238 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
239 sensorRef=None, visitInfo=None):
240 """Gen2 and gen3 shared code: determination of exposure dimensions and
241 copying of pixels from overlapping patch regions.
243 Parameters
244 ----------
245 skyMap : `lsst.skymap.BaseSkyMap`
246 SkyMap object that corresponds to the template coadd.
247 tractInfo : `lsst.skymap.TractInfo`
248 The selected tract.
249 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
250 Patches to consider for making the template exposure.
251 skyCorners : list of `lsst.geom.SpherePoint`
252 Sky corner coordinates to be covered by the template exposure.
253 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
254 `dict` (Gen2)
255 Dictionary of spatially relevant retrieved coadd patches,
256 indexed by their sequential patch number. In Gen3 mode, .get() is called,
257 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
258 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
259 Butler data reference to get coadd data.
260 Must be `None` for Gen3.
261 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
262 VisitInfo to make dcr model.
264 Returns
265 -------
266 templateExposure: `lsst.afw.image.ExposureF`
267 The created template exposure.
268 """
269 coaddWcs = tractInfo.getWcs()
271 # compute coadd bbox
272 coaddBBox = geom.Box2D()
273 for skyPos in skyCorners:
274 coaddBBox.include(coaddWcs.skyToPixel(skyPos))
275 coaddBBox = geom.Box2I(coaddBBox)
276 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
278 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
279 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
280 nPatchesFound = 0
281 coaddFilterLabel = None
282 coaddPsf = None
283 coaddPhotoCalib = None
284 for patchInfo in patchList:
285 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
286 patchSubBBox = patchInfo.getOuterBBox()
287 patchSubBBox.clip(coaddBBox)
288 if patchNumber not in availableCoaddRefs:
289 self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd")
290 continue
291 if patchSubBBox.isEmpty():
292 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, "
293 f"patch={patchNumber}; no overlapping pixels")
294 continue
296 if self.config.coaddName == 'dcr':
297 patchInnerBBox = patchInfo.getInnerBBox()
298 patchInnerBBox.clip(coaddBBox)
299 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
300 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels."
301 % availableCoaddRefs[patchNumber])
302 continue
303 self.log.info("Constructing DCR-matched template for patch %s"
304 % availableCoaddRefs[patchNumber])
306 if sensorRef:
307 dcrModel = DcrModel.fromDataRef(sensorRef,
308 self.config.effectiveWavelength,
309 self.config.bandwidth,
310 **availableCoaddRefs[patchNumber])
311 else:
312 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber],
313 self.config.effectiveWavelength,
314 self.config.bandwidth)
315 # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
316 # Each patch has significant overlap, and the contaminated edge pixels in
317 # a new patch will overwrite good pixels in the overlap region from
318 # previous patches.
319 # Shrink the BBox to remove the contaminated pixels,
320 # but make sure it is only the overlap region that is reduced.
321 dcrBBox = geom.Box2I(patchSubBBox)
322 dcrBBox.grow(-self.config.templateBorderSize)
323 dcrBBox.include(patchInnerBBox)
324 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
325 wcs=coaddWcs,
326 visitInfo=visitInfo)
327 else:
328 if sensorRef is None:
329 # Gen3
330 coaddPatch = availableCoaddRefs[patchNumber].get()
331 else:
332 # Gen2
333 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
334 nPatchesFound += 1
336 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
337 # calculation from caller code. Gen3 also does not do this.
338 overlapBox = coaddPatch.getBBox()
339 overlapBox.clip(coaddBBox)
340 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
342 if coaddFilterLabel is None:
343 coaddFilterLabel = coaddPatch.getFilterLabel()
345 # Retrieve the PSF for this coadd tract, if not already retrieved
346 if coaddPsf is None and coaddPatch.hasPsf():
347 coaddPsf = coaddPatch.getPsf()
349 # Retrieve the calibration for this coadd tract, if not already retrieved
350 if coaddPhotoCalib is None:
351 coaddPhotoCalib = coaddPatch.getPhotoCalib()
353 if coaddPhotoCalib is None:
354 raise RuntimeError("No coadd PhotoCalib found!")
355 if nPatchesFound == 0:
356 raise RuntimeError("No patches found!")
357 if coaddPsf is None:
358 raise RuntimeError("No coadd Psf found!")
360 coaddExposure.setPhotoCalib(coaddPhotoCalib)
361 coaddExposure.setPsf(coaddPsf)
362 coaddExposure.setFilterLabel(coaddFilterLabel)
363 return coaddExposure
365 def getCoaddDatasetName(self):
366 """Return coadd name for given task config
368 Returns
369 -------
370 CoaddDatasetName : `string`
372 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
373 """
374 warpType = self.config.warpType
375 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
376 return self.config.coaddName + "Coadd" + suffix
378 def bboxIntersectsCorners(self, bbox, wcs, otherCorners):
379 """Returns true if the bbox with wcs intersects otherCorners
381 Parameters:
382 -----------
383 bbox : `lsst.geom.Box2I`
384 specifying the bounding box of test region
385 wcs : `lsst.afw.geom.SkyWcs`
386 specifying the WCS of test region
387 otherCorners : `list` of `lsst.geom.SpherePoint`
388 ICRS coordinates specifying boundary of the other sky region
390 Returns:
391 --------
392 result: `bool`
393 Does bbox/wcs intersect other corners?
394 """
395 bboxCorners = wcs.pixelToSky(geom.Box2D(bbox).getCorners())
396 bboxPolygon = sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in bboxCorners])
397 otherPolygon = sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in otherCorners])
398 return otherPolygon.intersects(bboxPolygon)
401class GetCalexpAsTemplateConfig(pexConfig.Config):
402 doAddCalexpBackground = pexConfig.Field(
403 dtype=bool,
404 default=True,
405 doc="Add background to calexp before processing it."
406 )
409class GetCalexpAsTemplateTask(pipeBase.Task):
410 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
411 for use as an image difference template. Only gen2 supported.
413 To be run as a subtask by pipe.tasks.ImageDifferenceTask.
414 Intended for use with simulations and surveys that repeatedly visit the same pointing.
415 This code was originally part of Winter2013ImageDifferenceTask.
416 """
418 ConfigClass = GetCalexpAsTemplateConfig
419 _DefaultName = "GetCalexpAsTemplateTask"
421 def run(self, exposure, sensorRef, templateIdList):
422 """Return a calexp exposure with based on input sensorRef.
424 Construct a dataId based on the sensorRef.dataId combined
425 with the specifications from the first dataId in templateIdList
427 Parameters
428 ----------
429 exposure : `lsst.afw.image.Exposure`
430 exposure (unused)
431 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
432 Data reference of the calexp(s) to subtract from.
433 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
434 Data reference of the template calexp to be subtraced.
435 Can be incomplete, fields are initialized from `sensorRef`.
436 If there are multiple items, only the first one is used.
438 Returns
439 -------
440 result : `struct`
442 return a pipeBase.Struct:
444 - ``exposure`` : a template calexp
445 - ``sources`` : source catalog measured on the template
446 """
448 if len(templateIdList) == 0:
449 raise RuntimeError("No template data reference supplied.")
450 if len(templateIdList) > 1:
451 self.log.warn("Multiple template data references supplied. Using the first one only.")
453 templateId = sensorRef.dataId.copy()
454 templateId.update(templateIdList[0])
456 self.log.info("Fetching calexp (%s) as template." % (templateId))
458 butler = sensorRef.getButler()
459 template = butler.get(datasetType="calexp", dataId=templateId)
460 if self.config.doAddCalexpBackground:
461 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
462 mi = template.getMaskedImage()
463 mi += templateBg.getImage()
465 if not template.hasPsf():
466 raise pipeBase.TaskError("Template has no psf")
468 templateSources = butler.get(datasetType="src", dataId=templateId)
469 return pipeBase.Struct(exposure=template,
470 sources=templateSources)
472 def runDataRef(self, *args, **kwargs):
473 return self.run(*args, **kwargs)
475 def runQuantum(self, **kwargs):
476 raise NotImplementedError("Calexp template is not supported with gen3 middleware")