lsst.pipe.tasks  16.0-23-ge8a9b866+3
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 
24 import lsst.pex.config as pexConfig
25 import lsst.afw.image as afwImage
26 import lsst.coadd.utils as coaddUtils
27 import lsst.pipe.base as pipeBase
28 import lsst.log as log
29 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
30 from .coaddBase import CoaddBaseTask
31 from .warpAndPsfMatch import WarpAndPsfMatchTask
32 from .coaddHelpers import groupPatchExposures, getGroupDataRef
33 
34 __all__ = ["MakeCoaddTempExpTask"]
35 
36 
37 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
38  """Config for MakeCoaddTempExpTask
39  """
40  warpAndPsfMatch = pexConfig.ConfigurableField(
41  target=WarpAndPsfMatchTask,
42  doc="Task to warp and PSF-match calexp",
43  )
44  doWrite = pexConfig.Field(
45  doc="persist <coaddName>Coadd_<warpType>Warp",
46  dtype=bool,
47  default=True,
48  )
49  bgSubtracted = pexConfig.Field(
50  doc="Work with a background subtracted calexp?",
51  dtype=bool,
52  default=True,
53  )
54  coaddPsf = pexConfig.ConfigField(
55  doc="Configuration for CoaddPsf",
56  dtype=CoaddPsfConfig,
57  )
58  makeDirect = pexConfig.Field(
59  doc="Make direct Warp/Coadds",
60  dtype=bool,
61  default=True,
62  )
63  makePsfMatched = pexConfig.Field(
64  doc="Make Psf-Matched Warp/Coadd?",
65  dtype=bool,
66  default=False,
67  )
68  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
69 
70  def validate(self):
71  CoaddBaseTask.ConfigClass.validate(self)
72  if not self.makePsfMatched and not self.makeDirect:
73  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
74  if self.doPsfMatch:
75  # Backwards compatibility.
76  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
77  self.makePsfMatched = True
78  self.makeDirect = False
79 
80  def setDefaults(self):
81  CoaddBaseTask.ConfigClass.setDefaults(self)
82  self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
83 
84 
90 
91 
93  """!Warp and optionally PSF-Match calexps onto an a common projection.
94 
95  @anchor MakeCoaddTempExpTask_
96 
97  @section pipe_tasks_makeCoaddTempExp_Contents Contents
98 
99  - @ref pipe_tasks_makeCoaddTempExp_Purpose
100  - @ref pipe_tasks_makeCoaddTempExp_Initialize
101  - @ref pipe_tasks_makeCoaddTempExp_IO
102  - @ref pipe_tasks_makeCoaddTempExp_Config
103  - @ref pipe_tasks_makeCoaddTempExp_Debug
104  - @ref pipe_tasks_makeCoaddTempExp_Example
105 
106  @section pipe_tasks_makeCoaddTempExp_Purpose Description
107 
108  Warp and optionally PSF-Match calexps onto a common projection, by
109  performing the following operations:
110  - Group calexps by visit/run
111  - For each visit, generate a Warp by calling method @ref makeTempExp.
112  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
113  on each visit
114 
115  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
116 
117  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
118 
119  @copydoc \_\_init\_\_
120 
121  This task has one special keyword argument: passing reuse=True will cause
122  the task to skip the creation of warps that are already present in the
123  output repositories.
124 
125  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
126 
127  This task is primarily designed to be run from the command line.
128 
129  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
130  to process.
131 
132  @copydoc run
133 
134  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
135  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
136  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
137  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
138 
139  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
140 
141  See @ref MakeCoaddTempExpConfig and parameters inherited from
142  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
143 
144  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
145 
146  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
147  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
148  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
149  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
150  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
151  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
152 
153  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
154  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
155  In general, for a set of input images, this config should equal the FWHM of the visit
156  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
157  of the other config options offer a reasonable starting point.
158  The following list presents the most common problems that arise from a misconfigured
159  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
160  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
161  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
162  Problem: Explanation. *Solution*
163 
164  *Troublshooting PSF-Matching Configuration:*
165  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
166  For example:_
167 
168  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
169 
170  Note that increasing the kernel size also increases runtime.
171  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
172  for matching kernel. _Provide the matcher with more data by either increasing
173  the spatial sampling by decreasing the spatial cell size,_
174 
175  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
176  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
177 
178  _or increasing the padding around the Science PSF, for example:_
179 
180  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
181 
182  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
183  matching kernel dimensions, thus increasing the number of pixels available to fit
184  after convolving the PSF with the matching kernel.
185  Optionally, for debugging the effects of padding, the level of padding may be manually
186  controlled by setting turning off the automatic padding and setting the number
187  of pixels by which to pad the PSF:
188 
189  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
190  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
191 
192  - Deconvolution: Matching a large PSF to a smaller PSF produces
193  a telltale noise pattern which looks like ripples or a brain.
194  _Increase the size of the requested model PSF. For example:_
195 
196  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
197 
198  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
199  _Increase the width of the Gaussian basis functions. For example:_
200 
201  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
202  # from default [0.7, 1.5, 3.0]
203 
204 
205  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
206 
207  MakeCoaddTempExpTask has no debug output, but its subtasks do.
208 
209  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
210 
211  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
212  into the larger Data Release Processing.
213  Set up by running:
214 
215  setup ci_hsc
216  cd $CI_HSC_DIR
217  # if not built already:
218  python $(which scons) # this will take a while
219 
220  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
221  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
222  output respository with the desired SkyMap. The command,
223 
224  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
225  --id patch=5,4 tract=0 filter=HSC-I \
226  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
227  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
228  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
229 
230  writes a direct and PSF-Matched Warp to
231  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
232  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
233  respectively.
234 
235  @note PSF-Matching in this particular dataset would benefit from adding
236  `--configfile ./matchingConfig.py` to
237  the command line arguments where `matchingConfig.py` is defined by:
238 
239  echo "
240  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
241  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
242 
243 
244  Add the option `--help` to see more options.
245  """
246  ConfigClass = MakeCoaddTempExpConfig
247  _DefaultName = "makeCoaddTempExp"
248 
249  def __init__(self, reuse=False, **kwargs):
250  CoaddBaseTask.__init__(self, **kwargs)
251  self.reuse = reuse
252  self.makeSubtask("warpAndPsfMatch")
253 
254  @pipeBase.timeMethod
255  def runDataRef(self, patchRef, selectDataList=[]):
256  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
257 
258  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
259  plus the camera-specific filter key (e.g. "filter" or "band")
260  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
261  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
262  warps are requested.
263 
264  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
265 
266  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
267  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
268  (assembleCoadd should determine zeropoint scaling without referring to it).
269  """
270  skyInfo = self.getSkyInfo(patchRef)
271 
272  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
273  if self.config.makePsfMatched and not self.config.makeDirect:
274  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
275  else:
276  primaryWarpDataset = self.getTempExpDatasetName("direct")
277 
278  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
279  if len(calExpRefList) == 0:
280  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
281  return None
282  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
283  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
284  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
285 
286  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
287  primaryWarpDataset)
288  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
289 
290  dataRefList = []
291  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
292  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
293  tempExpTuple, groupData.keys)
294  if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
295  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
296  dataRefList.append(tempExpRef)
297  continue
298  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
299 
300  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
301  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
302  # of the visit in the list.
303  try:
304  visitId = int(tempExpRef.dataId["visit"])
305  except (KeyError, ValueError):
306  visitId = i
307 
308  exps = self.run(calexpRefList, skyInfo, visitId).exposures
309 
310  if any(exps.values()):
311  dataRefList.append(tempExpRef)
312  else:
313  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
314 
315  if self.config.doWrite:
316  for (warpType, exposure) in exps.items(): # compatible w/ Py3
317  if exposure is not None:
318  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
319  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
320 
321  return dataRefList
322 
323  def run(self, calexpRefList, skyInfo, visitId=0):
324  """Create a Warp from inputs
325 
326  We iterate over the multiple calexps in a single exposure to construct
327  the warp (previously called a coaddTempExp) of that exposure to the
328  supplied tract/patch.
329 
330  Pixels that receive no pixels are set to NAN; this is not correct
331  (violates LSST algorithms group policy), but will be fixed up by
332  interpolating after the coaddition.
333 
334  @param calexpRefList: List of data references for calexps that (may)
335  overlap the patch of interest
336  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
337  information about the patch
338  @param visitId: integer identifier for visit, for the table that will
339  produce the CoaddPsf
340  @return a pipeBase Struct containing:
341  - exposures: a dictionary containing the warps requested:
342  "direct": direct warp if config.makeDirect
343  "psfMatched": PSF-matched warp if config.makePsfMatched
344  """
345  warpTypeList = self.getWarpTypeList()
346 
347  totGoodPix = {warpType: 0 for warpType in warpTypeList}
348  didSetMetadata = {warpType: False for warpType in warpTypeList}
349  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
350  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
351  for warpType in warpTypeList}
352 
353  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
354  for calExpInd, calExpRef in enumerate(calexpRefList):
355  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
356  calExpInd+1, len(calexpRefList), calExpRef.dataId)
357  try:
358  ccdId = calExpRef.get("ccdExposureId", immediate=True)
359  except Exception:
360  ccdId = calExpInd
361  try:
362  # We augment the dataRef here with the tract, which is harmless for loading things
363  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
364  # which do.
365  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
366  tract=skyInfo.tractInfo.getId())
367  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
368  except Exception as e:
369  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
370  continue
371 
372  if self.config.doApplySkyCorr:
373  self.applySkyCorr(calExpRef, calExp)
374 
375  try:
376  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
377  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
378  makeDirect=self.config.makeDirect,
379  makePsfMatched=self.config.makePsfMatched)
380  except Exception as e:
381  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
382  continue
383  try:
384  numGoodPix = {warpType: 0 for warpType in warpTypeList}
385  for warpType in warpTypeList:
386  exposure = warpedAndMatched.getDict()[warpType]
387  if exposure is None:
388  continue
389  coaddTempExp = coaddTempExps[warpType]
390  if didSetMetadata[warpType]:
391  mimg = exposure.getMaskedImage()
392  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
393  exposure.getCalib().getFluxMag0()[0])
394  del mimg
395  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
396  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
397  totGoodPix[warpType] += numGoodPix[warpType]
398  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
399  calExpRef.dataId, numGoodPix[warpType],
400  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
401  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
402  coaddTempExp.setCalib(exposure.getCalib())
403  coaddTempExp.setFilter(exposure.getFilter())
404  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
405  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
406  coaddTempExp.setPsf(exposure.getPsf())
407  didSetMetadata[warpType] = True
408 
409  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
410  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
411 
412  except Exception as e:
413  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
414  continue
415 
416  for warpType in warpTypeList:
417  self.log.info("%sWarp has %d good pixels (%.1f%%)",
418  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
419 
420  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
421  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
422  if warpType == "direct":
423  coaddTempExps[warpType].setPsf(
424  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
425  self.config.coaddPsf.makeControl()))
426  else:
427  # No good pixels. Exposure still empty
428  coaddTempExps[warpType] = None
429 
430  result = pipeBase.Struct(exposures=coaddTempExps)
431  return result
432 
433  @staticmethod
434  def _prepareEmptyExposure(skyInfo):
435  """Produce an empty exposure for a given patch"""
436  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
437  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
438  .getPlaneBitMask("NO_DATA"), numpy.inf)
439  return exp
440 
441  def getWarpTypeList(self):
442  """Return list of requested warp types per the config.
443  """
444  warpTypeList = []
445  if self.config.makeDirect:
446  warpTypeList.append("direct")
447  if self.config.makePsfMatched:
448  warpTypeList.append("psfMatched")
449  return warpTypeList
450 
451  def applySkyCorr(self, dataRef, calexp):
452  """Apply correction to the sky background level
453 
454  Sky corrections can be generated with the 'skyCorrection.py'
455  executable in pipe_drivers. Because the sky model used by that
456  code extends over the entire focal plane, this can produce
457  better sky subtraction.
458 
459  The calexp is updated in-place.
460 
461  Parameters
462  ----------
463  dataRef : `lsst.daf.persistence.ButlerDataRef`
464  Data reference for calexp.
465  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
466  Calibrated exposure.
467  """
468  bg = dataRef.get("skyCorr")
469  if isinstance(calexp, afwImage.Exposure):
470  calexp = calexp.getMaskedImage()
471  calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:171
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:94
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:127
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:221
def run(self, calexpRefList, skyInfo, visitId=0)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:107
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
Definition: coaddBase.py:143
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60