lsst.pipe.tasks  13.0-56-g21e2fbf+3
 All Classes Namespaces Files Functions Variables Groups Pages
coaddBase.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 import numpy
24 
25 import lsst.pex.config as pexConfig
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.pipe.base as pipeBase
29 import lsst.meas.algorithms as measAlg
30 import lsst.log as log
31 
32 from lsst.afw.fits import FitsError
33 from lsst.coadd.utils import CoaddDataIdContainer
34 from .selectImages import WcsSelectImagesTask, SelectStruct
35 from .coaddInputRecorder import CoaddInputRecorderTask
36 
37 try:
38  from lsst.meas.mosaic import applyMosaicResults
39 except ImportError:
40  applyMosaicResults = None
41 
42 __all__ = ["CoaddBaseTask", "getSkyInfo"]
43 
44 
45 class CoaddBaseConfig(pexConfig.Config):
46  """!Configuration parameters for CoaddBaseTask
47 
48  \anchor CoaddBaseConfig_
49 
50  \brief Configuration parameters shared between MakeCoaddTempExp and AssembleCoadd
51  """
52  coaddName = pexConfig.Field(
53  doc="Coadd name: typically one of deep or goodSeeing.",
54  dtype=str,
55  default="deep",
56  )
57  select = pexConfig.ConfigurableField(
58  doc="Image selection subtask.",
59  target=WcsSelectImagesTask,
60  )
61  badMaskPlanes = pexConfig.ListField(
62  dtype=str,
63  doc="Mask planes that, if set, the associated pixel should not be included in the coaddTempExp.",
64  default=("NO_DATA",),
65  )
66  inputRecorder = pexConfig.ConfigurableField(
67  doc="Subtask that helps fill CoaddInputs catalogs added to the final Exposure",
68  target=CoaddInputRecorderTask
69  )
70  doPsfMatch = pexConfig.Field(
71  dtype=bool,
72  doc="Match to modelPsf? Deprecated. Sets makePsfMatched=True, makeDirect=False",
73  default=False
74  )
75  modelPsf = measAlg.GaussianPsfFactory.makeField(doc="Model Psf factory")
76  doApplyUberCal = pexConfig.Field(
77  dtype=bool,
78  doc="Apply meas_mosaic ubercal results to input calexps?",
79  default=False
80  )
81  makeDirect = pexConfig.Field(
82  doc="Make direct Warp/Coadds",
83  dtype=bool,
84  default=True,
85  )
86  makePsfMatched = pexConfig.Field(
87  doc="Make Psf-Matched Warp/Coadd?",
88  dtype=bool,
89  default=False,
90  )
91 
92  def validate(self):
93  pexConfig.Config.validate(self)
94  if not self.makePsfMatched and not self.makeDirect:
95  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
96  if self.doPsfMatch:
97  # Courtesy backwards compatibility.
98  # Configs do not have loggers
99  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
100  self.makePsfMatched = True
101  self.makeDirect = False
102 
103 
104 class CoaddTaskRunner(pipeBase.TaskRunner):
105 
106  @staticmethod
107  def getTargetList(parsedCmd, **kwargs):
108  return pipeBase.TaskRunner.getTargetList(parsedCmd, selectDataList=parsedCmd.selectId.dataList,
109  **kwargs)
110 
111 
112 class CoaddBaseTask(pipeBase.CmdLineTask):
113  """!Base class for coaddition.
114 
115  Subclasses must specify _DefaultName
116  """
117  ConfigClass = CoaddBaseConfig
118  RunnerClass = CoaddTaskRunner
119 
120  def __init__(self, *args, **kwargs):
121  pipeBase.Task.__init__(self, *args, **kwargs)
122  self.makeSubtask("select")
123  self.makeSubtask("inputRecorder")
124 
125  def selectExposures(self, patchRef, skyInfo=None, selectDataList=[]):
126  """!
127  \brief Select exposures to coadd
128 
129  Get the corners of the bbox supplied in skyInfo using \ref afwGeom.Box2D and convert the pixel
130  positions of the bbox corners to sky coordinates using \ref skyInfo.wcs.pixelToSky. Use the
131  \ref WcsSelectImagesTask_ "WcsSelectImagesTask" to select exposures that lie inside the patch
132  indicated by the dataRef.
133 
134  \param[in] patchRef data reference for sky map patch. Must include keys "tract", "patch",
135  plus the camera-specific filter key (e.g. "filter" or "band")
136  \param[in] skyInfo geometry for the patch; output from getSkyInfo
137  \return a list of science exposures to coadd, as butler data references
138  """
139  if skyInfo is None:
140  skyInfo = self.getSkyInfo(patchRef)
141  cornerPosList = afwGeom.Box2D(skyInfo.bbox).getCorners()
142  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
143  return self.select.runDataRef(patchRef, coordList, selectDataList=selectDataList).dataRefList
144 
145  def getSkyInfo(self, patchRef):
146  """!
147  \brief Use \ref getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox
148  of the patch.
149 
150  \param[in] patchRef data reference for sky map. Must include keys "tract" and "patch"
151 
152  \return pipe_base Struct containing:
153  - skyMap: sky map
154  - tractInfo: information for chosen tract of sky map
155  - patchInfo: information about chosen patch of tract
156  - wcs: WCS of tract
157  - bbox: outer bbox of patch, as an afwGeom Box2I
158  """
159  return getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
160 
161  def getCalExp(self, dataRef, bgSubtracted):
162  """!Return one "calexp" calibrated exposure
163 
164  @param[in] dataRef a sensor-level data reference
165  @param[in] bgSubtracted return calexp with background subtracted? If False get the
166  calexp's background background model and add it to the calexp.
167  @return calibrated exposure
168 
169  If config.doApplyUberCal, meas_mosaic calibrations will be applied to
170  the returned exposure using applyMosaicResults.
171  """
172  exposure = dataRef.get("calexp", immediate=True)
173  if not bgSubtracted:
174  background = dataRef.get("calexpBackground", immediate=True)
175  mi = exposure.getMaskedImage()
176  mi += background.getImage()
177  del mi
178  if not self.config.doApplyUberCal:
179  return exposure
180  if applyMosaicResults is None:
181  raise RuntimeError(
182  "Cannot use improved calibrations for %s because meas_mosaic could not be imported."
183  % dataRef.dataId
184  )
185  else:
186  applyMosaicResults(dataRef, calexp=exposure)
187  return exposure
188 
189  def getCoaddDatasetName(self, warpType="direct"):
190  """Return coadd name for given warpType and task config
191 
192  Parameters
193  ----------
194  warpType : string
195  Either 'direct' or 'psfMatched'
196 
197  Returns
198  -------
199  CoaddDatasetName : `string`
200  """
201  suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:]
202  return self.config.coaddName + "Coadd" + suffix
203 
204  def getTempExpDatasetName(self, warpType="direct"):
205  """Return warp name for given warpType and task config
206 
207  Parameters
208  ----------
209  warpType : string
210  Either 'direct' or 'psfMatched'
211 
212  Returns
213  -------
214  WarpDatasetName : `string`
215  """
216  return self.config.coaddName + "Coadd_" + warpType + "Warp"
217 
218  def getWarpTypeList(self):
219  """Return list of requested warp types per the config.
220  """
221  warpTypeList = []
222  if self.config.makeDirect:
223  warpTypeList.append("direct")
224  if self.config.makePsfMatched:
225  warpTypeList.append("psfMatched")
226  return warpTypeList
227 
228  @classmethod
229  def _makeArgumentParser(cls):
230  """Create an argument parser
231  """
232  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
233  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2",
234  ContainerClass=CoaddDataIdContainer)
235  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
236  ContainerClass=SelectDataIdContainer)
237  return parser
238 
239  def _getConfigName(self):
240  """Return the name of the config dataset
241  """
242  return "%s_%s_config" % (self.config.coaddName, self._DefaultName)
243 
244  def _getMetadataName(self):
245  """Return the name of the metadata dataset
246  """
247  return "%s_%s_metadata" % (self.config.coaddName, self._DefaultName)
248 
249  def getBadPixelMask(self):
250  """!
251  \brief Convenience method to provide the bitmask from the mask plane names
252  """
253  return afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)
254 
255 
256 class SelectDataIdContainer(pipeBase.DataIdContainer):
257  """!
258  \brief A dataId container for inputs to be selected.
259 
260  Read the header (including the size and Wcs) for all specified
261  inputs and pass those along, ultimately for the SelectImagesTask.
262  This is most useful when used with multiprocessing, as input headers are
263  only read once.
264  """
265 
266  def makeDataRefList(self, namespace):
267  """Add a dataList containing useful information for selecting images"""
268  super(SelectDataIdContainer, self).makeDataRefList(namespace)
269  self.dataList = []
270  for ref in self.refList:
271  try:
272  md = ref.get("calexp_md", immediate=True)
273  wcs = afwImage.makeWcs(md)
274  data = SelectStruct(dataRef=ref, wcs=wcs, dims=(md.get("NAXIS1"), md.get("NAXIS2")))
275  except FitsError as e:
276  namespace.log.warn("Unable to construct Wcs from %s" % (ref.dataId))
277  continue
278  self.dataList.append(data)
279 
280 
281 def getSkyInfo(coaddName, patchRef):
282  """!
283  \brief Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded.
284 
285  \param[in] coaddName coadd name; typically one of deep or goodSeeing
286  \param[in] patchRef data reference for sky map. Must include keys "tract" and "patch"
287 
288  \return pipe_base Struct containing:
289  - skyMap: sky map
290  - tractInfo: information for chosen tract of sky map
291  - patchInfo: information about chosen patch of tract
292  - wcs: WCS of tract
293  - bbox: outer bbox of patch, as an afwGeom Box2I
294  """
295  skyMap = patchRef.get(coaddName + "Coadd_skyMap")
296  tractId = patchRef.dataId["tract"]
297  tractInfo = skyMap[tractId]
298 
299  # patch format is "xIndex,yIndex"
300  patchIndex = tuple(int(i) for i in patchRef.dataId["patch"].split(","))
301  patchInfo = tractInfo.getPatchInfo(patchIndex)
302 
303  return pipeBase.Struct(
304  skyMap=skyMap,
305  tractInfo=tractInfo,
306  patchInfo=patchInfo,
307  wcs=tractInfo.getWcs(),
308  bbox=patchInfo.getOuterBBox(),
309  )
310 
311 
312 def scaleVariance(maskedImage, maskPlanes, log=None):
313  """!
314  \brief Scale the variance in a maskedImage
315 
316  The variance plane in a convolved or warped image (or a coadd derived
317  from warped images) does not accurately reflect the noise properties of
318  the image because variance has been lost to covariance. This function
319  attempts to correct for this by scaling the variance plane to match
320  the observed variance in the image. This is not perfect (because we're
321  not tracking the covariance) but it's simple and is often good enough.
322 
323  @param maskedImage MaskedImage to operate on; variance will be scaled
324  @param maskPlanes List of mask planes for pixels to reject
325  @param log Log for reporting the renormalization factor; or None
326  @return renormalisation factor
327  """
328  variance = maskedImage.getVariance()
329  sigNoise = maskedImage.getImage().getArray()/numpy.sqrt(variance.getArray())
330  maskVal = maskedImage.getMask().getPlaneBitMask(maskPlanes)
331  good = (maskedImage.getMask().getArray() & maskVal) == 0
332  # Robust measurement of stdev
333  q1, q3 = numpy.percentile(sigNoise[good], (25, 75))
334  stdev = 0.74*(q3 - q1)
335  ratio = stdev**2
336  if log:
337  log.info("Renormalizing variance by %f" % (ratio,))
338  variance *= ratio
339  return ratio
Base class for coaddition.
Definition: coaddBase.py:112
def getBadPixelMask
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:249
Configuration parameters for CoaddBaseTask.
Definition: coaddBase.py:45
A dataId container for inputs to be selected.
Definition: coaddBase.py:256
def getCalExp
Return one &quot;calexp&quot; calibrated exposure.
Definition: coaddBase.py:161
def scaleVariance
Scale the variance in a maskedImage.
Definition: coaddBase.py:312
def selectExposures
Select exposures to coadd.
Definition: coaddBase.py:125
def getSkyInfo
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:145
def getSkyInfo
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:281