Coverage for python/lsst/ip/diffim/getTemplate.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.geom as geom
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from lsst.ip.diffim.dcrModel import DcrModel
31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig",
32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"]
35class GetCoaddAsTemplateConfig(pexConfig.Config):
36 templateBorderSize = pexConfig.Field(
37 dtype=int,
38 default=10,
39 doc="Number of pixels to grow the requested template image to account for warping"
40 )
41 coaddName = pexConfig.Field(
42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'",
43 dtype=str,
44 default="deep",
45 )
46 numSubfilters = pexConfig.Field(
47 doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'",
48 dtype=int,
49 default=3,
50 )
51 warpType = pexConfig.Field(
52 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'",
53 dtype=str,
54 default="direct",
55 )
58class GetCoaddAsTemplateTask(pipeBase.Task):
59 """Subtask to retrieve coadd for use as an image difference template.
61 This is the default getTemplate Task to be run as a subtask by
62 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and
63 ``runGen3()``.
65 Notes
66 -----
67 From the given skymap, the closest tract is selected; multiple tracts are
68 not supported. The assembled template inherits the WCS of the selected
69 skymap tract and the resolution of the template exposures. Overlapping box
70 regions of the input template patches are pixel by pixel copied into the
71 assembled template image. There is no warping or pixel resampling.
73 Pixels with no overlap of any available input patches are set to ``nan`` value
74 and ``NO_DATA`` flagged.
75 """
77 ConfigClass = GetCoaddAsTemplateConfig
78 _DefaultName = "GetCoaddAsTemplateTask"
80 def runDataRef(self, exposure, sensorRef, templateIdList=None):
81 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure
82 that overlaps the science exposure.
84 Parameters
85 ----------
86 exposure: `lsst.afw.image.Exposure`
87 an exposure for which to generate an overlapping template
88 sensorRef : TYPE
89 a Butler data reference that can be used to obtain coadd data
90 templateIdList : TYPE, optional
91 list of data ids, unused here, in the case of coadd template
93 Returns
94 -------
95 result : `lsst.pipe.base.Struct`
96 - ``exposure`` : `lsst.afw.image.ExposureF`
97 a template coadd exposure assembled out of patches
98 - ``sources`` : None for this subtask
99 """
100 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap")
101 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
103 availableCoaddRefs = dict()
104 for patchInfo in patchList:
105 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
106 patchArgDict = dict(
107 datasetType=self.getCoaddDatasetName() + "_sub",
108 bbox=patchInfo.getOuterBBox(),
109 tract=tractInfo.getId(),
110 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
111 subfilter=0,
112 numSubfilters=self.config.numSubfilters,
113 )
115 if sensorRef.datasetExists(**patchArgDict):
116 self.log.info("Reading patch %s" % patchArgDict)
117 availableCoaddRefs[patchNumber] = patchArgDict
119 templateExposure = self.run(
120 tractInfo, patchList, skyCorners, availableCoaddRefs,
121 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo()
122 )
123 return pipeBase.Struct(exposure=templateExposure, sources=None)
125 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs):
126 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure
127 that overlaps the science exposure.
129 Parameters
130 ----------
131 exposure : `lsst.afw.image.Exposure`
132 The science exposure to define the sky region of the template coadd.
133 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
134 Butler like object that supports getting data by DatasetRef.
135 skyMapRef : `lsst.daf.butler.DatasetRef`
136 Reference to SkyMap object that corresponds to the template coadd.
137 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef`
138 Iterable of references to the available template coadd patches.
140 Returns
141 -------
142 result : `lsst.pipe.base.Struct`
143 - ``exposure`` : `lsst.afw.image.ExposureF`
144 a template coadd exposure assembled out of patches
145 - ``sources`` : `None` for this subtask
146 """
147 skyMap = butlerQC.get(skyMapRef)
148 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap)
149 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList)
151 availableCoaddRefs = dict()
152 for coaddRef in coaddExposureRefs:
153 dataId = coaddRef.datasetRef.dataId
154 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter:
155 if self.config.coaddName == 'dcr':
156 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" %
157 (tractInfo.getId(), dataId['patch'], dataId['subfilter']))
158 if dataId['patch'] in availableCoaddRefs:
159 availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef))
160 else:
161 availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ]
162 else:
163 self.log.info("Using template input tract=%s, patch=%s" %
164 (tractInfo.getId(), dataId['patch']))
165 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef)
167 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs,
168 visitInfo=exposure.getInfo().getVisitInfo())
169 return pipeBase.Struct(exposure=templateExposure, sources=None)
171 def getOverlapPatchList(self, exposure, skyMap):
172 """Select the relevant tract and its patches that overlap with the science exposure.
174 Parameters
175 ----------
176 exposure : `lsst.afw.image.Exposure`
177 The science exposure to define the sky region of the template coadd.
179 skyMap : `lsst.skymap.BaseSkyMap`
180 SkyMap object that corresponds to the template coadd.
182 Returns
183 -------
184 result : `tuple` of
185 - ``tractInfo`` : `lsst.skymap.TractInfo`
186 The selected tract.
187 - ``patchList`` : `list` of `lsst.skymap.PatchInfo`
188 List of all overlap patches of the selected tract.
189 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint`
190 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`.
191 """
192 expWcs = exposure.getWcs()
193 expBoxD = geom.Box2D(exposure.getBBox())
194 expBoxD.grow(self.config.templateBorderSize)
195 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
196 tractInfo = skyMap.findTract(ctrSkyPos)
197 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),))
198 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()]
199 patchList = tractInfo.findPatchList(skyCorners)
201 if not patchList:
202 raise RuntimeError("No suitable tract found")
204 self.log.info("Assembling %s coadd patches" % (len(patchList),))
205 self.log.info("exposure dimensions=%s" % exposure.getDimensions())
207 return (tractInfo, patchList, skyCorners)
209 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs,
210 sensorRef=None, visitInfo=None):
211 """Gen2 and gen3 shared code: determination of exposure dimensions and
212 copying of pixels from overlapping patch regions.
214 Parameters
215 ----------
216 skyMap : `lsst.skymap.BaseSkyMap`
217 SkyMap object that corresponds to the template coadd.
218 tractInfo : `lsst.skymap.TractInfo`
219 The selected tract.
220 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo`
221 Patches to consider for making the template exposure.
222 skyCorners : list of `lsst.geom.SpherePoint`
223 Sky corner coordinates to be covered by the template exposure.
224 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3)
225 `dict` (Gen2)
226 Dictionary of spatially relevant retrieved coadd patches,
227 indexed by their sequential patch number. In Gen3 mode, .get() is called,
228 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd.
229 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only
230 Butler data reference to get coadd data.
231 Must be `None` for Gen3.
232 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only
233 VisitInfo to make dcr model.
235 Returns
236 -------
237 templateExposure: `lsst.afw.image.ExposureF`
238 The created template exposure.
239 """
240 coaddWcs = tractInfo.getWcs()
242 # compute coadd bbox
243 coaddBBox = geom.Box2D()
244 for skyPos in skyCorners:
245 coaddBBox.include(coaddWcs.skyToPixel(skyPos))
246 coaddBBox = geom.Box2I(coaddBBox)
247 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions())
249 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
250 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
251 nPatchesFound = 0
252 coaddFilter = None
253 coaddPsf = None
254 coaddPhotoCalib = None
255 for patchInfo in patchList:
256 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo)
257 patchSubBBox = patchInfo.getOuterBBox()
258 patchSubBBox.clip(coaddBBox)
259 if patchSubBBox.isEmpty():
260 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, "
261 f"patch={patchNumber}; no overlapping pixels")
262 continue
264 if self.config.coaddName == 'dcr':
265 patchInnerBBox = patchInfo.getInnerBBox()
266 patchInnerBBox.clip(coaddBBox)
267 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize:
268 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels."
269 % availableCoaddRefs[patchNumber])
270 continue
271 self.log.info("Constructing DCR-matched template for patch %s"
272 % availableCoaddRefs[patchNumber])
274 if sensorRef:
275 dcrModel = DcrModel.fromDataRef(sensorRef, **availableCoaddRefs[patchNumber])
276 else:
277 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber])
278 # The edge pixels of the DcrCoadd may contain artifacts due to missing data.
279 # Each patch has significant overlap, and the contaminated edge pixels in
280 # a new patch will overwrite good pixels in the overlap region from
281 # previous patches.
282 # Shrink the BBox to remove the contaminated pixels,
283 # but make sure it is only the overlap region that is reduced.
284 dcrBBox = geom.Box2I(patchSubBBox)
285 dcrBBox.grow(-self.config.templateBorderSize)
286 dcrBBox.include(patchInnerBBox)
287 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox,
288 wcs=coaddWcs,
289 visitInfo=visitInfo)
290 else:
291 if sensorRef is None:
292 # Gen3
293 coaddPatch = availableCoaddRefs[patchNumber].get()
294 else:
295 # Gen2
296 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber])
297 nPatchesFound += 1
299 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox
300 # calculation from caller code. Gen3 also does not do this.
301 overlapBox = coaddPatch.getBBox()
302 overlapBox.clip(coaddBBox)
303 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox)
305 if coaddFilter is None:
306 coaddFilter = coaddPatch.getFilter()
308 # Retrieve the PSF for this coadd tract, if not already retrieved
309 if coaddPsf is None and coaddPatch.hasPsf():
310 coaddPsf = coaddPatch.getPsf()
312 # Retrieve the calibration for this coadd tract, if not already retrieved
313 if coaddPhotoCalib is None:
314 coaddPhotoCalib = coaddPatch.getPhotoCalib()
316 if coaddPhotoCalib is None:
317 raise RuntimeError("No coadd PhotoCalib found!")
318 if nPatchesFound == 0:
319 raise RuntimeError("No patches found!")
320 if coaddPsf is None:
321 raise RuntimeError("No coadd Psf found!")
323 coaddExposure.setPhotoCalib(coaddPhotoCalib)
324 coaddExposure.setPsf(coaddPsf)
325 coaddExposure.setFilter(coaddFilter)
326 return coaddExposure
328 def getCoaddDatasetName(self):
329 """Return coadd name for given task config
331 Returns
332 -------
333 CoaddDatasetName : `string`
335 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985)
336 """
337 warpType = self.config.warpType
338 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
339 return self.config.coaddName + "Coadd" + suffix
342class GetCalexpAsTemplateConfig(pexConfig.Config):
343 doAddCalexpBackground = pexConfig.Field(
344 dtype=bool,
345 default=True,
346 doc="Add background to calexp before processing it."
347 )
350class GetCalexpAsTemplateTask(pipeBase.Task):
351 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef
352 for use as an image difference template. Only gen2 supported.
354 To be run as a subtask by pipe.tasks.ImageDifferenceTask.
355 Intended for use with simulations and surveys that repeatedly visit the same pointing.
356 This code was originally part of Winter2013ImageDifferenceTask.
357 """
359 ConfigClass = GetCalexpAsTemplateConfig
360 _DefaultName = "GetCalexpAsTemplateTask"
362 def run(self, exposure, sensorRef, templateIdList):
363 """Return a calexp exposure with based on input sensorRef.
365 Construct a dataId based on the sensorRef.dataId combined
366 with the specifications from the first dataId in templateIdList
368 Parameters
369 ----------
370 exposure : `lsst.afw.image.Exposure`
371 exposure (unused)
372 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
373 Data reference of the calexp(s) to subtract from.
374 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
375 Data reference of the template calexp to be subtraced.
376 Can be incomplete, fields are initialized from `sensorRef`.
377 If there are multiple items, only the first one is used.
379 Returns
380 -------
381 result : `struct`
383 return a pipeBase.Struct:
385 - ``exposure`` : a template calexp
386 - ``sources`` : source catalog measured on the template
387 """
389 if len(templateIdList) == 0:
390 raise RuntimeError("No template data reference supplied.")
391 if len(templateIdList) > 1:
392 self.log.warn("Multiple template data references supplied. Using the first one only.")
394 templateId = sensorRef.dataId.copy()
395 templateId.update(templateIdList[0])
397 self.log.info("Fetching calexp (%s) as template." % (templateId))
399 butler = sensorRef.getButler()
400 template = butler.get(datasetType="calexp", dataId=templateId)
401 if self.config.doAddCalexpBackground:
402 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId)
403 mi = template.getMaskedImage()
404 mi += templateBg.getImage()
406 if not template.hasPsf():
407 raise pipeBase.TaskError("Template has no psf")
409 templateSources = butler.get(datasetType="src", dataId=templateId)
410 return pipeBase.Struct(exposure=template,
411 sources=templateSources)
413 def runDataRef(self, *args, **kwargs):
414 return self.run(*args, **kwargs)
416 def runQuantum(self, **kwargs):
417 raise NotImplementedError("Calexp template is not supported with gen3 middleware")