lsst.pipe.tasks  14.0-34-g85a33b94+3
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.log as log
31 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
32 from .coaddBase import CoaddBaseTask
33 from .warpAndPsfMatch import WarpAndPsfMatchTask
34 from .coaddHelpers import groupPatchExposures, getGroupDataRef
35 
36 __all__ = ["MakeCoaddTempExpTask"]
37 
38 
39 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
40  """Config for MakeCoaddTempExpTask
41  """
42  warpAndPsfMatch = pexConfig.ConfigurableField(
43  target=WarpAndPsfMatchTask,
44  doc="Task to warp and PSF-match calexp",
45  )
46  doWrite = pexConfig.Field(
47  doc="persist <coaddName>Coadd_<warpType>Warp",
48  dtype=bool,
49  default=True,
50  )
51  bgSubtracted = pexConfig.Field(
52  doc="Work with a background subtracted calexp?",
53  dtype=bool,
54  default=True,
55  )
56  coaddPsf = pexConfig.ConfigField(
57  doc="Configuration for CoaddPsf",
58  dtype=CoaddPsfConfig,
59  )
60  makeDirect = pexConfig.Field(
61  doc="Make direct Warp/Coadds",
62  dtype=bool,
63  default=True,
64  )
65  makePsfMatched = pexConfig.Field(
66  doc="Make Psf-Matched Warp/Coadd?",
67  dtype=bool,
68  default=False,
69  )
70  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
71 
72  def validate(self):
73  CoaddBaseTask.ConfigClass.validate(self)
74  if not self.makePsfMatched and not self.makeDirect:
75  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
76  if self.doPsfMatch:
77  # Backwards compatibility.
78  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
79  self.makePsfMatched = True
80  self.makeDirect = False
81 
82 
83 
89 
90 
92  """!Warp and optionally PSF-Match calexps onto an a common projection.
93 
94  @anchor MakeCoaddTempExpTask_
95 
96  @section pipe_tasks_makeCoaddTempExp_Contents Contents
97 
98  - @ref pipe_tasks_makeCoaddTempExp_Purpose
99  - @ref pipe_tasks_makeCoaddTempExp_Initialize
100  - @ref pipe_tasks_makeCoaddTempExp_IO
101  - @ref pipe_tasks_makeCoaddTempExp_Config
102  - @ref pipe_tasks_makeCoaddTempExp_Debug
103  - @ref pipe_tasks_makeCoaddTempExp_Example
104 
105  @section pipe_tasks_makeCoaddTempExp_Purpose Description
106 
107  Warp and optionally PSF-Match calexps onto a common projection, by
108  performing the following operations:
109  - Group calexps by visit/run
110  - For each visit, generate a Warp by calling method @ref makeTempExp.
111  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
112  on each visit
113 
114  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
115 
116  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
117 
118  @copydoc \_\_init\_\_
119 
120  This task has one special keyword argument: passing reuse=True will cause
121  the task to skip the creation of warps that are already present in the
122  output repositories.
123 
124  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
125 
126  This task is primarily designed to be run from the command line.
127 
128  The main method is `run`, which takes a single butler data reference for the patch(es)
129  to process.
130 
131  @copydoc run
132 
133  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
134  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
135  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
136  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
137 
138  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
139 
140  See @ref MakeCoaddTempExpConfig and parameters inherited from
141  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
142 
143  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
144 
145  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
146  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
147  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
148  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
149  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
150  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
151 
152  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
153  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
154  In general, for a set of input images, this config should equal the FWHM of the visit
155  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
156  of the other config options offer a reasonable starting point.
157  The following list presents the most common problems that arise from a misconfigured
158  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
159  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
160  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
161  Problem: Explanation. *Solution*
162 
163  *Troublshooting PSF-Matching Configuration:*
164  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
165  For example:_
166 
167  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
168 
169  Note that increasing the kernel size also increases runtime.
170  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
171  for matching kernel. _Provide the matcher with more data by either increasing
172  the spatial sampling by decreasing the spatial cell size,_
173 
174  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
175  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
176 
177  _or increasing the padding around the Science PSF, for example:_
178 
179  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
180 
181  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
182  matching kernel dimensions, thus increasing the number of pixels available to fit
183  after convolving the PSF with the matching kernel.
184  Optionally, for debugging the effects of padding, the level of padding may be manually
185  controlled by setting turning off the automatic padding and setting the number
186  of pixels by which to pad the PSF:
187 
188  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
189  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
190 
191  - Deconvolution: Matching a large PSF to a smaller PSF produces
192  a telltale noise pattern which looks like ripples or a brain.
193  _Increase the size of the requested model PSF. For example:_
194 
195  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
196 
197  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
198  _Increase the width of the Gaussian basis functions. For example:_
199 
200  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
201  # from default [0.7, 1.5, 3.0]
202 
203 
204  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
205 
206  MakeCoaddTempExpTask has no debug output, but its subtasks do.
207 
208  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
209 
210  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
211  into the larger Data Release Processing.
212  Set up by running:
213 
214  setup ci_hsc
215  cd $CI_HSC_DIR
216  # if not built already:
217  python $(which scons) # this will take a while
218 
219  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
220  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
221  output respository with the desired SkyMap. The command,
222 
223  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
224  --id patch=5,4 tract=0 filter=HSC-I \
225  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
226  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
227  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
228 
229  writes a direct and PSF-Matched Warp to
230  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
231  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
232  respectively.
233 
234  @note PSF-Matching in this particular dataset would benefit from adding
235  `--configfile ./matchingConfig.py` to
236  the command line arguments where `matchingConfig.py` is defined by:
237 
238  echo "
239  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
240  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
241 
242 
243  Add the option `--help` to see more options.
244  """
245  ConfigClass = MakeCoaddTempExpConfig
246  _DefaultName = "makeCoaddTempExp"
247 
248  def __init__(self, reuse=False, **kwargs):
249  CoaddBaseTask.__init__(self, **kwargs)
250  self.reuse = reuse
251  self.makeSubtask("warpAndPsfMatch")
252 
253  @pipeBase.timeMethod
254  def run(self, patchRef, selectDataList=[]):
255  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
256 
257  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
258  plus the camera-specific filter key (e.g. "filter" or "band")
259  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
260  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
261  warps are requested.
262 
263  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
264 
265  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
266  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
267  (assembleCoadd should determine zeropoint scaling without referring to it).
268  """
269  skyInfo = self.getSkyInfo(patchRef)
270 
271  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
272  if self.config.makePsfMatched and not self.config.makeDirect:
273  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
274  else:
275  primaryWarpDataset = self.getTempExpDatasetName("direct")
276 
277  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
278  if len(calExpRefList) == 0:
279  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
280  return None
281  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
282  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
283  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
284 
285  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
286  primaryWarpDataset)
287  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
288 
289  dataRefList = []
290  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
291  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
292  tempExpTuple, groupData.keys)
293  if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
294  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
295  dataRefList.append(tempExpRef)
296  continue
297  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
298 
299  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
300  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
301  # of the visit in the list.
302  try:
303  visitId = int(tempExpRef.dataId["visit"])
304  except (KeyError, ValueError):
305  visitId = i
306 
307  exps = self.createTempExp(calexpRefList, skyInfo, visitId).exposures
308 
309  if any(exps.values()):
310  dataRefList.append(tempExpRef)
311  else:
312  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
313 
314  if self.config.doWrite:
315  for (warpType, exposure) in exps.items(): # compatible w/ Py3
316  if exposure is not None:
317  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
318  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
319 
320  return dataRefList
321 
322  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
323  """Create a Warp from inputs
324 
325  We iterate over the multiple calexps in a single exposure to construct
326  the warp (previously called a coaddTempExp) of that exposure to the
327  supplied tract/patch.
328 
329  Pixels that receive no pixels are set to NAN; this is not correct
330  (violates LSST algorithms group policy), but will be fixed up by
331  interpolating after the coaddition.
332 
333  @param calexpRefList: List of data references for calexps that (may)
334  overlap the patch of interest
335  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
336  information about the patch
337  @param visitId: integer identifier for visit, for the table that will
338  produce the CoaddPsf
339  @return a pipeBase Struct containing:
340  - exposures: a dictionary containing the warps requested:
341  "direct": direct warp if config.makeDirect
342  "psfMatched": PSF-matched warp if config.makePsfMatched
343  """
344  warpTypeList = self.getWarpTypeList()
345 
346  totGoodPix = {warpType: 0 for warpType in warpTypeList}
347  didSetMetadata = {warpType: False for warpType in warpTypeList}
348  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
349  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
350  for warpType in warpTypeList}
351 
352  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
353  for calExpInd, calExpRef in enumerate(calexpRefList):
354  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
355  calExpInd+1, len(calexpRefList), calExpRef.dataId)
356  try:
357  ccdId = calExpRef.get("ccdExposureId", immediate=True)
358  except Exception:
359  ccdId = calExpInd
360  try:
361  # We augment the dataRef here with the tract, which is harmless for loading things
362  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
363  # which do.
364  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
365  tract=skyInfo.tractInfo.getId())
366  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
367  except Exception as e:
368  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
369  continue
370 
371  if self.config.doApplySkyCorr:
372  self.applySkyCorr(calExpRef, calExp)
373 
374  try:
375  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
376  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
377  makeDirect=self.config.makeDirect,
378  makePsfMatched=self.config.makePsfMatched)
379  except Exception as e:
380  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
381  continue
382  try:
383  numGoodPix = {warpType: 0 for warpType in warpTypeList}
384  for warpType in warpTypeList:
385  exposure = warpedAndMatched.getDict()[warpType]
386  if exposure is None:
387  continue
388  coaddTempExp = coaddTempExps[warpType]
389  if didSetMetadata[warpType]:
390  mimg = exposure.getMaskedImage()
391  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
392  exposure.getCalib().getFluxMag0()[0])
393  del mimg
394  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
395  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
396  totGoodPix[warpType] += numGoodPix[warpType]
397  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
398  calExpRef.dataId, numGoodPix[warpType],
399  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
400  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
401  coaddTempExp.setCalib(exposure.getCalib())
402  coaddTempExp.setFilter(exposure.getFilter())
403  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
404  coaddTempExp.setPsf(exposure.getPsf())
405  didSetMetadata[warpType] = True
406 
407  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
408  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
409 
410  except Exception as e:
411  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
412  continue
413 
414  for warpType in warpTypeList:
415  self.log.info("%sWarp has %d good pixels (%.1f%%)",
416  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
417 
418  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
419  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
420  if warpType == "direct":
421  coaddTempExps[warpType].setPsf(
422  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
423  self.config.coaddPsf.makeControl()))
424  else:
425  # No good pixels. Exposure still empty
426  coaddTempExps[warpType] = None
427 
428  result = pipeBase.Struct(exposures=coaddTempExps)
429  return result
430 
431  def _prepareEmptyExposure(cls, skyInfo):
432  """Produce an empty exposure for a given patch"""
433  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
434  exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
435  .getPlaneBitMask("NO_DATA"), numpy.inf)
436  return exp
437 
438  def getWarpTypeList(self):
439  """Return list of requested warp types per the config.
440  """
441  warpTypeList = []
442  if self.config.makeDirect:
443  warpTypeList.append("direct")
444  if self.config.makePsfMatched:
445  warpTypeList.append("psfMatched")
446  return warpTypeList
447 
448  def applySkyCorr(self, dataRef, calexp):
449  """Apply correction to the sky background level
450 
451  Sky corrections can be generated with the 'skyCorrection.py'
452  executable in pipe_drivers. Because the sky model used by that
453  code extends over the entire focal plane, this can produce
454  better sky subtraction.
455 
456  The calexp is updated in-place.
457 
458  Parameters
459  ----------
460  dataRef : `lsst.daf.persistence.ButlerDataRef`
461  Data reference for calexp.
462  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
463  Calibrated exposure.
464  """
465  bg = dataRef.get("skyCorr")
466  if isinstance(calexp, afwImage.Exposure):
467  calexp = calexp.getMaskedImage()
468  calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
Definition: coaddBase.py:139
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63