Coverage for python/lsst/ip/diffim/getTemplate.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.geom as geom
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from lsst.ip.diffim.dcrModel import DcrModel
31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
35class GetCoaddAsTemplateConfig(pexConfig.Config):
36 templateBorderSize = pexConfig.Field(
37 dtype=int,
38 default=10,
39 doc="Number of pixels to grow the requested template image to account for warping"
40 )
41 coaddName = pexConfig.Field(
42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
43 dtype=str,
44 default="deep",
45 )
46 numSubfilters = pexConfig.Field(
47 doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'",
48 dtype=int,
49 default=3,
50 )
51 warpType = pexConfig.Field(
52 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
53 dtype=str,
54 default="direct",
55 )
58class GetCoaddAsTemplateTask(pipeBase.Task):
59 """Subtask to retrieve coadd for use as an image difference template.
61 This is the default getTemplate Task to be run as a subtask by
62 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
63 ``runGen3()``.
65 Notes
66 -----
67 From the given skymap, the closest tract is selected; multiple tracts are
68 not supported. The assembled template inherits the WCS of the selected
69 skymap tract and the resolution of the template exposures. Overlapping box
70 regions of the input template patches are pixel by pixel copied into the
71 assembled template image. There is no warping or pixel resampling.
73 Pixels with no overlap of any available input patches are set to ``nan`` value
74 and ``NO_DATA`` flagged.
75 """
77 ConfigClass = GetCoaddAsTemplateConfig
78 _DefaultName = "GetCoaddAsTemplateTask"
80 def runDataRef(self, exposure, sensorRef, templateIdList=None):
81 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
82 that overlaps the science exposure.
84 Parameters
85 ----------
86 exposure: `lsst.afw.image.Exposure`
87 an exposure for which to generate an overlapping template
88 sensorRef : TYPE
89 a Butler data reference that can be used to obtain coadd data
90 templateIdList : TYPE, optional
91 list of data ids, unused here, in the case of coadd template
93 Returns
94 -------
95 result : `lsst.pipe.base.Struct`
96 - ``exposure`` : `lsst.afw.image.ExposureF`
97 a template coadd exposure assembled out of patches
98 - ``sources`` : None for this subtask
99 """
100 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
101 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
103 availableCoaddRefs = dict()
104 for patchInfo in patchList:
105 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
106 patchArgDict = dict(
107 datasetType=self.getCoaddDatasetName() + "_sub",
108 bbox=patchInfo.getOuterBBox(),
109 tract=tractInfo.getId(),
110 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
111 numSubfilters=self.config.numSubfilters,
112 )
114 if self.config.coaddName != 'dcr' and sensorRef.datasetExists(**patchArgDict):
115 self.log.info("Reading patch %s" % patchArgDict)
116 availableCoaddRefs[patchNumber] = patchArgDict
118 templateExposure = self.run(
119 tractInfo, patchList, skyCorners, availableCoaddRefs,
120 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
121 )
122 return pipeBase.Struct(exposure=templateExposure, sources=None)
124 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
125 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
126 that overlaps the science exposure.
128 Parameters
129 ----------
130 exposure : `lsst.afw.image.Exposure`
131 The science exposure to define the sky region of the template coadd.
132 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
133 Butler like object that supports getting data by DatasetRef.
134 skyMapRef : `lsst.daf.butler.DatasetRef`
135 Reference to SkyMap object that corresponds to the template coadd.
136 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
137 Iterable of references to the available template coadd patches.
139 Returns
140 -------
141 result : `lsst.pipe.base.Struct`
142 - ``exposure`` : `lsst.afw.image.ExposureF`
143 a template coadd exposure assembled out of patches
144 - ``sources`` : `None` for this subtask
145 """
146 skyMap = butlerQC.get(skyMapRef)
147 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
148 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList)
150 availableCoaddRefs = dict()
151 for coaddRef in coaddExposureRefs:
152 dataId = coaddRef.datasetRef.dataId
153 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter:
154 self.log.info("Using template input tract=%s, patch=%s" %
155 (tractInfo.getId(), dataId['patch']))
156 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef)
158 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs)
159 return pipeBase.Struct(exposure=templateExposure, sources=None)
161 def getOverlapPatchList(self, exposure, skyMap):
162 """Select the relevant tract and its patches that overlap with the science exposure.
164 Parameters
165 ----------
166 exposure : `lsst.afw.image.Exposure`
167 The science exposure to define the sky region of the template coadd.
169 skyMap : `lsst.skymap.BaseSkyMap`
170 SkyMap object that corresponds to the template coadd.
172 Returns
173 -------
174 result : `tuple` of
175 - ``tractInfo`` : `lsst.skymap.TractInfo`
176 The selected tract.
177 - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
178 List of all overlap patches of the selected tract.
179 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
180 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
181 """
182 expWcs = exposure.getWcs()
183 expBoxD = geom.Box2D(exposure.getBBox())
184 expBoxD.grow(self.config.templateBorderSize)
185 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
186 tractInfo = skyMap.findTract(ctrSkyPos)
187 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
188 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
189 patchList = tractInfo.findPatchList(skyCorners)
191 if not patchList:
192 raise RuntimeError("No suitable tract found")
194 self.log.info("Assembling %s coadd patches" % (len(patchList),))
195 self.log.info("exposure dimensions=%s" % exposure.getDimensions())
197 return (tractInfo, patchList, skyCorners)
199 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
200 sensorRef=None, visitInfo=None):
201 """Gen2 and gen3 shared code: determination of exposure dimensions and
202 copying of pixels from overlapping patch regions.
204 Parameters
205 ----------
206 skyMap : `lsst.skymap.BaseSkyMap`
207 SkyMap object that corresponds to the template coadd.
208 tractInfo : `lsst.skymap.TractInfo`
209 The selected tract.
210 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
211 Patches to consider for making the template exposure.
212 skyCorners : list of `lsst.geom.SpherePoint`
213 Sky corner coordinates to be covered by the template exposure.
214 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
215 `dict` (Gen2)
216 Dictionary of spatially relevant retrieved coadd patches,
217 indexed by their sequential patch number. In Gen3 mode, .get() is called,
218 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
219 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
220 TODO DM-22952 Butler data reference to get coadd data.
221 Must be `None` for Gen3.
222 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
223 TODO DM-22952 VisitInfo to make dcr model.
225 Returns
226 -------
227 templateExposure: `lsst.afw.image.ExposureF`
228 The created template exposure.
229 """
230 coaddWcs = tractInfo.getWcs()
232 # compute coadd bbox
233 coaddBBox = geom.Box2D()
234 for skyPos in skyCorners:
235 coaddBBox.include(coaddWcs.skyToPixel(skyPos))
236 coaddBBox = geom.Box2I(coaddBBox)
237 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
239 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
240 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
241 nPatchesFound = 0
242 coaddFilter = None
243 coaddPsf = None
244 coaddPhotoCalib = None
245 for patchInfo in patchList:
246 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
247 patchSubBBox = patchInfo.getOuterBBox()
248 patchSubBBox.clip(coaddBBox)
249 patchArgDict = dict(
250 datasetType=self.getCoaddDatasetName() + "_sub",
251 bbox=patchSubBBox,
252 tract=tractInfo.getId(),
253 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
254 numSubfilters=self.config.numSubfilters,
255 )
256 if patchSubBBox.isEmpty():
257 self.log.info(f"skip tract={patchArgDict['tract']}, "
258 f"patch={patchNumber}; no overlapping pixels")
259 continue
261 # TODO DM-22952
262 # Dcr coadd support is gen2 only
263 # Under gen3, sensorRef will be None and execution is stopped in runQuantum if dcr coadd is
264 # configured
265 if self.config.coaddName == 'dcr':
266 if not sensorRef.datasetExists(subfilter=0, **patchArgDict):
267 self.log.warn("%(datasetType)s, tract=%(tract)s, patch=%(patch)s,"
268 " numSubfilters=%(numSubfilters)s, subfilter=0 does not exist"
269 % patchArgDict)
270 continue
271 patchInnerBBox = patchInfo.getInnerBBox()
272 patchInnerBBox.clip(coaddBBox)
273 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
274 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." % patchArgDict)
275 continue
276 self.log.info("Constructing DCR-matched template for patch %s" % patchArgDict)
278 dcrModel = DcrModel.fromDataRef(sensorRef, **patchArgDict)
279 # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
280 # Each patch has significant overlap, and the contaminated edge pixels in
281 # a new patch will overwrite good pixels in the overlap region from
282 # previous patches.
283 # Shrink the BBox to remove the contaminated pixels,
284 # but make sure it is only the overlap region that is reduced.
285 dcrBBox = geom.Box2I(patchSubBBox)
286 dcrBBox.grow(-self.config.templateBorderSize)
287 dcrBBox.include(patchInnerBBox)
288 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
289 wcs=coaddWcs,
290 visitInfo=visitInfo)
291 else:
292 if patchNumber not in availableCoaddRefs:
293 self.log.warn(f"{patchArgDict['datasetType']}, "
294 f"tract={patchArgDict['tract']}, patch={patchNumber} does not exist")
295 continue
296 if sensorRef is None:
297 # Gen3
298 coaddPatch = availableCoaddRefs[patchNumber].get()
299 else:
300 # Gen2
301 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
302 nPatchesFound += 1
304 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
305 # calculation from caller code. Gen3 also does not do this.
306 overlapBox = coaddPatch.getBBox()
307 overlapBox.clip(coaddBBox)
308 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
310 if coaddFilter is None:
311 coaddFilter = coaddPatch.getFilter()
313 # Retrieve the PSF for this coadd tract, if not already retrieved
314 if coaddPsf is None and coaddPatch.hasPsf():
315 coaddPsf = coaddPatch.getPsf()
317 # Retrieve the calibration for this coadd tract, if not already retrieved
318 if coaddPhotoCalib is None:
319 coaddPhotoCalib = coaddPatch.getPhotoCalib()
321 if coaddPhotoCalib is None:
322 raise RuntimeError("No coadd PhotoCalib found!")
323 if nPatchesFound == 0:
324 raise RuntimeError("No patches found!")
325 if coaddPsf is None:
326 raise RuntimeError("No coadd Psf found!")
328 coaddExposure.setPhotoCalib(coaddPhotoCalib)
329 coaddExposure.setPsf(coaddPsf)
330 coaddExposure.setFilter(coaddFilter)
331 return coaddExposure
333 def getCoaddDatasetName(self):
334 """Return coadd name for given task config
336 Returns
337 -------
338 CoaddDatasetName : `string`
340 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
341 """
342 warpType = self.config.warpType
343 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
344 return self.config.coaddName + "Coadd" + suffix
347class GetCalexpAsTemplateConfig(pexConfig.Config):
348 doAddCalexpBackground = pexConfig.Field(
349 dtype=bool,
350 default=True,
351 doc="Add background to calexp before processing it."
352 )
355class GetCalexpAsTemplateTask(pipeBase.Task):
356 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
357 for use as an image difference template. Only gen2 supported.
359 To be run as a subtask by pipe.tasks.ImageDifferenceTask.
360 Intended for use with simulations and surveys that repeatedly visit the same pointing.
361 This code was originally part of Winter2013ImageDifferenceTask.
362 """
364 ConfigClass = GetCalexpAsTemplateConfig
365 _DefaultName = "GetCalexpAsTemplateTask"
367 def run(self, exposure, sensorRef, templateIdList):
368 """Return a calexp exposure with based on input sensorRef.
370 Construct a dataId based on the sensorRef.dataId combined
371 with the specifications from the first dataId in templateIdList
373 Parameters
374 ----------
375 exposure : `lsst.afw.image.Exposure`
376 exposure (unused)
377 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
378 Data reference of the calexp(s) to subtract from.
379 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
380 Data reference of the template calexp to be subtraced.
381 Can be incomplete, fields are initialized from `sensorRef`.
382 If there are multiple items, only the first one is used.
384 Returns
385 -------
386 result : `struct`
388 return a pipeBase.Struct:
390 - ``exposure`` : a template calexp
391 - ``sources`` : source catalog measured on the template
392 """
394 if len(templateIdList) == 0:
395 raise RuntimeError("No template data reference supplied.")
396 if len(templateIdList) > 1:
397 self.log.warn("Multiple template data references supplied. Using the first one only.")
399 templateId = sensorRef.dataId.copy()
400 templateId.update(templateIdList[0])
402 self.log.info("Fetching calexp (%s) as template." % (templateId))
404 butler = sensorRef.getButler()
405 template = butler.get(datasetType="calexp", dataId=templateId)
406 if self.config.doAddCalexpBackground:
407 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
408 mi = template.getMaskedImage()
409 mi += templateBg.getImage()
411 if not template.hasPsf():
412 raise pipeBase.TaskError("Template has no psf")
414 templateSources = butler.get(datasetType="src", dataId=templateId)
415 return pipeBase.Struct(exposure=template,
416 sources=templateSources)
418 def runDataRef(self, *args, **kwargs):
419 return self.run(*args, **kwargs)
421 def runQuantum(self, **kwargs):
422 raise NotImplementedError("Calexp template is not supported with gen3 middleware")