Coverage for python/lsst/ip/diffim/getTemplate.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.geom as geom
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from lsst.ip.diffim.dcrModel import DcrModel
31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
35class GetCoaddAsTemplateConfig(pexConfig.Config):
36 templateBorderSize = pexConfig.Field(
37 dtype=int,
38 default=10,
39 doc="Number of pixels to grow the requested template image to account for warping"
40 )
41 coaddName = pexConfig.Field(
42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
43 dtype=str,
44 default="deep",
45 )
46 numSubfilters = pexConfig.Field(
47 doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'",
48 dtype=int,
49 default=3,
50 )
51 warpType = pexConfig.Field(
52 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
53 dtype=str,
54 default="direct",
55 )
58class GetCoaddAsTemplateTask(pipeBase.Task):
59 """Subtask to retrieve coadd for use as an image difference template.
61 This is the default getTemplate Task to be run as a subtask by
62 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
63 ``runGen3()``.
65 Notes
66 -----
67 From the given skymap, the closest tract is selected; multiple tracts are
68 not supported. The assembled template inherits the WCS of the selected
69 skymap tract and the resolution of the template exposures. Overlapping box
70 regions of the input template patches are pixel by pixel copied into the
71 assembled template image. There is no warping or pixel resampling.
73 Pixels with no overlap of any available input patches are set to ``nan`` value
74 and ``NO_DATA`` flagged.
75 """
77 ConfigClass = GetCoaddAsTemplateConfig
78 _DefaultName = "GetCoaddAsTemplateTask"
80 def runDataRef(self, exposure, sensorRef, templateIdList=None):
81 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
82 that overlaps the science exposure.
84 Parameters
85 ----------
86 exposure: `lsst.afw.image.Exposure`
87 an exposure for which to generate an overlapping template
88 sensorRef : TYPE
89 a Butler data reference that can be used to obtain coadd data
90 templateIdList : TYPE, optional
91 list of data ids, unused here, in the case of coadd template
93 Returns
94 -------
95 result : `lsst.pipe.base.Struct`
96 - ``exposure`` : `lsst.afw.image.ExposureF`
97 a template coadd exposure assembled out of patches
98 - ``sources`` : None for this subtask
99 """
100 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
101 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
103 availableCoaddRefs = dict()
104 for patchInfo in patchList:
105 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
106 patchArgDict = dict(
107 datasetType=self.getCoaddDatasetName() + "_sub",
108 bbox=patchInfo.getOuterBBox(),
109 tract=tractInfo.getId(),
110 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
111 subfilter=0,
112 numSubfilters=self.config.numSubfilters,
113 )
115 if sensorRef.datasetExists(**patchArgDict):
116 self.log.info("Reading patch %s" % patchArgDict)
117 availableCoaddRefs[patchNumber] = patchArgDict
119 templateExposure = self.run(
120 tractInfo, patchList, skyCorners, availableCoaddRefs,
121 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
122 )
123 return pipeBase.Struct(exposure=templateExposure, sources=None)
125 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
126 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
127 that overlaps the science exposure.
129 Parameters
130 ----------
131 exposure : `lsst.afw.image.Exposure`
132 The science exposure to define the sky region of the template coadd.
133 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
134 Butler like object that supports getting data by DatasetRef.
135 skyMapRef : `lsst.daf.butler.DatasetRef`
136 Reference to SkyMap object that corresponds to the template coadd.
137 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
138 Iterable of references to the available template coadd patches.
140 Returns
141 -------
142 result : `lsst.pipe.base.Struct`
143 - ``exposure`` : `lsst.afw.image.ExposureF`
144 a template coadd exposure assembled out of patches
145 - ``sources`` : `None` for this subtask
146 """
147 skyMap = butlerQC.get(skyMapRef)
148 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
149 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList)
151 availableCoaddRefs = dict()
152 for coaddRef in coaddExposureRefs:
153 dataId = coaddRef.datasetRef.dataId
154 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter:
155 if self.config.coaddName == 'dcr':
156 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" %
157 (tractInfo.getId(), dataId['patch'], dataId['subfilter']))
158 if dataId['patch'] in availableCoaddRefs:
159 availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef))
160 else:
161 availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ]
162 else:
163 self.log.info("Using template input tract=%s, patch=%s" %
164 (tractInfo.getId(), dataId['patch']))
165 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef)
167 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs,
168 visitInfo=exposure.getInfo().getVisitInfo())
169 return pipeBase.Struct(exposure=templateExposure, sources=None)
171 def getOverlapPatchList(self, exposure, skyMap):
172 """Select the relevant tract and its patches that overlap with the science exposure.
174 Parameters
175 ----------
176 exposure : `lsst.afw.image.Exposure`
177 The science exposure to define the sky region of the template coadd.
179 skyMap : `lsst.skymap.BaseSkyMap`
180 SkyMap object that corresponds to the template coadd.
182 Returns
183 -------
184 result : `tuple` of
185 - ``tractInfo`` : `lsst.skymap.TractInfo`
186 The selected tract.
187 - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
188 List of all overlap patches of the selected tract.
189 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
190 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
191 """
192 expWcs = exposure.getWcs()
193 expBoxD = geom.Box2D(exposure.getBBox())
194 expBoxD.grow(self.config.templateBorderSize)
195 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
196 tractInfo = skyMap.findTract(ctrSkyPos)
197 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
198 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
199 patchList = tractInfo.findPatchList(skyCorners)
201 if not patchList:
202 raise RuntimeError("No suitable tract found")
204 self.log.info("Assembling %s coadd patches" % (len(patchList),))
205 self.log.info("exposure dimensions=%s" % exposure.getDimensions())
207 return (tractInfo, patchList, skyCorners)
209 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
210 sensorRef=None, visitInfo=None):
211 """Gen2 and gen3 shared code: determination of exposure dimensions and
212 copying of pixels from overlapping patch regions.
214 Parameters
215 ----------
216 skyMap : `lsst.skymap.BaseSkyMap`
217 SkyMap object that corresponds to the template coadd.
218 tractInfo : `lsst.skymap.TractInfo`
219 The selected tract.
220 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
221 Patches to consider for making the template exposure.
222 skyCorners : list of `lsst.geom.SpherePoint`
223 Sky corner coordinates to be covered by the template exposure.
224 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
225 `dict` (Gen2)
226 Dictionary of spatially relevant retrieved coadd patches,
227 indexed by their sequential patch number. In Gen3 mode, .get() is called,
228 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
229 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
230 Butler data reference to get coadd data.
231 Must be `None` for Gen3.
232 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
233 VisitInfo to make dcr model.
235 Returns
236 -------
237 templateExposure: `lsst.afw.image.ExposureF`
238 The created template exposure.
239 """
240 coaddWcs = tractInfo.getWcs()
242 # compute coadd bbox
243 coaddBBox = geom.Box2D()
244 for skyPos in skyCorners:
245 coaddBBox.include(coaddWcs.skyToPixel(skyPos))
246 coaddBBox = geom.Box2I(coaddBBox)
247 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
249 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
250 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
251 nPatchesFound = 0
252 coaddFilter = None
253 coaddPsf = None
254 coaddPhotoCalib = None
255 for patchInfo in patchList:
256 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
257 patchSubBBox = patchInfo.getOuterBBox()
258 patchSubBBox.clip(coaddBBox)
259 if patchNumber not in availableCoaddRefs:
260 self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd")
261 continue
262 if patchSubBBox.isEmpty():
263 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, "
264 f"patch={patchNumber}; no overlapping pixels")
265 continue
267 if self.config.coaddName == 'dcr':
268 patchInnerBBox = patchInfo.getInnerBBox()
269 patchInnerBBox.clip(coaddBBox)
270 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
271 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels."
272 % availableCoaddRefs[patchNumber])
273 continue
274 self.log.info("Constructing DCR-matched template for patch %s"
275 % availableCoaddRefs[patchNumber])
277 if sensorRef:
278 dcrModel = DcrModel.fromDataRef(sensorRef, **availableCoaddRefs[patchNumber])
279 else:
280 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber])
281 # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
282 # Each patch has significant overlap, and the contaminated edge pixels in
283 # a new patch will overwrite good pixels in the overlap region from
284 # previous patches.
285 # Shrink the BBox to remove the contaminated pixels,
286 # but make sure it is only the overlap region that is reduced.
287 dcrBBox = geom.Box2I(patchSubBBox)
288 dcrBBox.grow(-self.config.templateBorderSize)
289 dcrBBox.include(patchInnerBBox)
290 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
291 wcs=coaddWcs,
292 visitInfo=visitInfo)
293 else:
294 if sensorRef is None:
295 # Gen3
296 coaddPatch = availableCoaddRefs[patchNumber].get()
297 else:
298 # Gen2
299 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
300 nPatchesFound += 1
302 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
303 # calculation from caller code. Gen3 also does not do this.
304 overlapBox = coaddPatch.getBBox()
305 overlapBox.clip(coaddBBox)
306 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
308 if coaddFilter is None:
309 coaddFilter = coaddPatch.getFilter()
311 # Retrieve the PSF for this coadd tract, if not already retrieved
312 if coaddPsf is None and coaddPatch.hasPsf():
313 coaddPsf = coaddPatch.getPsf()
315 # Retrieve the calibration for this coadd tract, if not already retrieved
316 if coaddPhotoCalib is None:
317 coaddPhotoCalib = coaddPatch.getPhotoCalib()
319 if coaddPhotoCalib is None:
320 raise RuntimeError("No coadd PhotoCalib found!")
321 if nPatchesFound == 0:
322 raise RuntimeError("No patches found!")
323 if coaddPsf is None:
324 raise RuntimeError("No coadd Psf found!")
326 coaddExposure.setPhotoCalib(coaddPhotoCalib)
327 coaddExposure.setPsf(coaddPsf)
328 coaddExposure.setFilter(coaddFilter)
329 return coaddExposure
331 def getCoaddDatasetName(self):
332 """Return coadd name for given task config
334 Returns
335 -------
336 CoaddDatasetName : `string`
338 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
339 """
340 warpType = self.config.warpType
341 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
342 return self.config.coaddName + "Coadd" + suffix
345class GetCalexpAsTemplateConfig(pexConfig.Config):
346 doAddCalexpBackground = pexConfig.Field(
347 dtype=bool,
348 default=True,
349 doc="Add background to calexp before processing it."
350 )
353class GetCalexpAsTemplateTask(pipeBase.Task):
354 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
355 for use as an image difference template. Only gen2 supported.
357 To be run as a subtask by pipe.tasks.ImageDifferenceTask.
358 Intended for use with simulations and surveys that repeatedly visit the same pointing.
359 This code was originally part of Winter2013ImageDifferenceTask.
360 """
362 ConfigClass = GetCalexpAsTemplateConfig
363 _DefaultName = "GetCalexpAsTemplateTask"
365 def run(self, exposure, sensorRef, templateIdList):
366 """Return a calexp exposure with based on input sensorRef.
368 Construct a dataId based on the sensorRef.dataId combined
369 with the specifications from the first dataId in templateIdList
371 Parameters
372 ----------
373 exposure : `lsst.afw.image.Exposure`
374 exposure (unused)
375 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
376 Data reference of the calexp(s) to subtract from.
377 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
378 Data reference of the template calexp to be subtraced.
379 Can be incomplete, fields are initialized from `sensorRef`.
380 If there are multiple items, only the first one is used.
382 Returns
383 -------
384 result : `struct`
386 return a pipeBase.Struct:
388 - ``exposure`` : a template calexp
389 - ``sources`` : source catalog measured on the template
390 """
392 if len(templateIdList) == 0:
393 raise RuntimeError("No template data reference supplied.")
394 if len(templateIdList) > 1:
395 self.log.warn("Multiple template data references supplied. Using the first one only.")
397 templateId = sensorRef.dataId.copy()
398 templateId.update(templateIdList[0])
400 self.log.info("Fetching calexp (%s) as template." % (templateId))
402 butler = sensorRef.getButler()
403 template = butler.get(datasetType="calexp", dataId=templateId)
404 if self.config.doAddCalexpBackground:
405 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
406 mi = template.getMaskedImage()
407 mi += templateBg.getImage()
409 if not template.hasPsf():
410 raise pipeBase.TaskError("Template has no psf")
412 templateSources = butler.get(datasetType="src", dataId=templateId)
413 return pipeBase.Struct(exposure=template,
414 sources=templateSources)
416 def runDataRef(self, *args, **kwargs):
417 return self.run(*args, **kwargs)
419 def runQuantum(self, **kwargs):
420 raise NotImplementedError("Calexp template is not supported with gen3 middleware")