lsst.pipe.tasks  14.0-54-gb4e84373+2
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.log as log
31 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
32 from .coaddBase import CoaddBaseTask
33 from .warpAndPsfMatch import WarpAndPsfMatchTask
34 from .coaddHelpers import groupPatchExposures, getGroupDataRef
35 
36 __all__ = ["MakeCoaddTempExpTask"]
37 
38 
39 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
40  """Config for MakeCoaddTempExpTask
41  """
42  warpAndPsfMatch = pexConfig.ConfigurableField(
43  target=WarpAndPsfMatchTask,
44  doc="Task to warp and PSF-match calexp",
45  )
46  doWrite = pexConfig.Field(
47  doc="persist <coaddName>Coadd_<warpType>Warp",
48  dtype=bool,
49  default=True,
50  )
51  bgSubtracted = pexConfig.Field(
52  doc="Work with a background subtracted calexp?",
53  dtype=bool,
54  default=True,
55  )
56  coaddPsf = pexConfig.ConfigField(
57  doc="Configuration for CoaddPsf",
58  dtype=CoaddPsfConfig,
59  )
60  makeDirect = pexConfig.Field(
61  doc="Make direct Warp/Coadds",
62  dtype=bool,
63  default=True,
64  )
65  makePsfMatched = pexConfig.Field(
66  doc="Make Psf-Matched Warp/Coadd?",
67  dtype=bool,
68  default=False,
69  )
70  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
71 
72  def validate(self):
73  CoaddBaseTask.ConfigClass.validate(self)
74  if not self.makePsfMatched and not self.makeDirect:
75  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
76  if self.doPsfMatch:
77  # Backwards compatibility.
78  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
79  self.makePsfMatched = True
80  self.makeDirect = False
81 
82  def setDefaults(self):
83  CoaddBaseTask.ConfigClass.setDefaults(self)
84  self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
85 
86 
92 
93 
95  """!Warp and optionally PSF-Match calexps onto an a common projection.
96 
97  @anchor MakeCoaddTempExpTask_
98 
99  @section pipe_tasks_makeCoaddTempExp_Contents Contents
100 
101  - @ref pipe_tasks_makeCoaddTempExp_Purpose
102  - @ref pipe_tasks_makeCoaddTempExp_Initialize
103  - @ref pipe_tasks_makeCoaddTempExp_IO
104  - @ref pipe_tasks_makeCoaddTempExp_Config
105  - @ref pipe_tasks_makeCoaddTempExp_Debug
106  - @ref pipe_tasks_makeCoaddTempExp_Example
107 
108  @section pipe_tasks_makeCoaddTempExp_Purpose Description
109 
110  Warp and optionally PSF-Match calexps onto a common projection, by
111  performing the following operations:
112  - Group calexps by visit/run
113  - For each visit, generate a Warp by calling method @ref makeTempExp.
114  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
115  on each visit
116 
117  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
118 
119  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
120 
121  @copydoc \_\_init\_\_
122 
123  This task has one special keyword argument: passing reuse=True will cause
124  the task to skip the creation of warps that are already present in the
125  output repositories.
126 
127  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
128 
129  This task is primarily designed to be run from the command line.
130 
131  The main method is `run`, which takes a single butler data reference for the patch(es)
132  to process.
133 
134  @copydoc run
135 
136  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
137  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
138  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
139  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
140 
141  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
142 
143  See @ref MakeCoaddTempExpConfig and parameters inherited from
144  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
145 
146  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
147 
148  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
149  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
150  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
151  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
152  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
153  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
154 
155  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
156  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
157  In general, for a set of input images, this config should equal the FWHM of the visit
158  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
159  of the other config options offer a reasonable starting point.
160  The following list presents the most common problems that arise from a misconfigured
161  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
162  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
163  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
164  Problem: Explanation. *Solution*
165 
166  *Troublshooting PSF-Matching Configuration:*
167  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
168  For example:_
169 
170  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
171 
172  Note that increasing the kernel size also increases runtime.
173  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
174  for matching kernel. _Provide the matcher with more data by either increasing
175  the spatial sampling by decreasing the spatial cell size,_
176 
177  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
178  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
179 
180  _or increasing the padding around the Science PSF, for example:_
181 
182  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
183 
184  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
185  matching kernel dimensions, thus increasing the number of pixels available to fit
186  after convolving the PSF with the matching kernel.
187  Optionally, for debugging the effects of padding, the level of padding may be manually
188  controlled by setting turning off the automatic padding and setting the number
189  of pixels by which to pad the PSF:
190 
191  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
192  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
193 
194  - Deconvolution: Matching a large PSF to a smaller PSF produces
195  a telltale noise pattern which looks like ripples or a brain.
196  _Increase the size of the requested model PSF. For example:_
197 
198  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
199 
200  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
201  _Increase the width of the Gaussian basis functions. For example:_
202 
203  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
204  # from default [0.7, 1.5, 3.0]
205 
206 
207  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
208 
209  MakeCoaddTempExpTask has no debug output, but its subtasks do.
210 
211  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
212 
213  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
214  into the larger Data Release Processing.
215  Set up by running:
216 
217  setup ci_hsc
218  cd $CI_HSC_DIR
219  # if not built already:
220  python $(which scons) # this will take a while
221 
222  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
223  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
224  output respository with the desired SkyMap. The command,
225 
226  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
227  --id patch=5,4 tract=0 filter=HSC-I \
228  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
229  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
230  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
231 
232  writes a direct and PSF-Matched Warp to
233  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
234  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
235  respectively.
236 
237  @note PSF-Matching in this particular dataset would benefit from adding
238  `--configfile ./matchingConfig.py` to
239  the command line arguments where `matchingConfig.py` is defined by:
240 
241  echo "
242  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
243  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
244 
245 
246  Add the option `--help` to see more options.
247  """
248  ConfigClass = MakeCoaddTempExpConfig
249  _DefaultName = "makeCoaddTempExp"
250 
251  def __init__(self, reuse=False, **kwargs):
252  CoaddBaseTask.__init__(self, **kwargs)
253  self.reuse = reuse
254  self.makeSubtask("warpAndPsfMatch")
255 
256  @pipeBase.timeMethod
257  def run(self, patchRef, selectDataList=[]):
258  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
259 
260  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
261  plus the camera-specific filter key (e.g. "filter" or "band")
262  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
263  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
264  warps are requested.
265 
266  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
267 
268  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
269  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
270  (assembleCoadd should determine zeropoint scaling without referring to it).
271  """
272  skyInfo = self.getSkyInfo(patchRef)
273 
274  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
275  if self.config.makePsfMatched and not self.config.makeDirect:
276  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
277  else:
278  primaryWarpDataset = self.getTempExpDatasetName("direct")
279 
280  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
281  if len(calExpRefList) == 0:
282  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
283  return None
284  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
285  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
286  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
287 
288  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
289  primaryWarpDataset)
290  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
291 
292  dataRefList = []
293  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
294  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
295  tempExpTuple, groupData.keys)
296  if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
297  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
298  dataRefList.append(tempExpRef)
299  continue
300  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
301 
302  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
303  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
304  # of the visit in the list.
305  try:
306  visitId = int(tempExpRef.dataId["visit"])
307  except (KeyError, ValueError):
308  visitId = i
309 
310  exps = self.createTempExp(calexpRefList, skyInfo, visitId).exposures
311 
312  if any(exps.values()):
313  dataRefList.append(tempExpRef)
314  else:
315  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
316 
317  if self.config.doWrite:
318  for (warpType, exposure) in exps.items(): # compatible w/ Py3
319  if exposure is not None:
320  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
321  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
322 
323  return dataRefList
324 
325  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
326  """Create a Warp from inputs
327 
328  We iterate over the multiple calexps in a single exposure to construct
329  the warp (previously called a coaddTempExp) of that exposure to the
330  supplied tract/patch.
331 
332  Pixels that receive no pixels are set to NAN; this is not correct
333  (violates LSST algorithms group policy), but will be fixed up by
334  interpolating after the coaddition.
335 
336  @param calexpRefList: List of data references for calexps that (may)
337  overlap the patch of interest
338  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
339  information about the patch
340  @param visitId: integer identifier for visit, for the table that will
341  produce the CoaddPsf
342  @return a pipeBase Struct containing:
343  - exposures: a dictionary containing the warps requested:
344  "direct": direct warp if config.makeDirect
345  "psfMatched": PSF-matched warp if config.makePsfMatched
346  """
347  warpTypeList = self.getWarpTypeList()
348 
349  totGoodPix = {warpType: 0 for warpType in warpTypeList}
350  didSetMetadata = {warpType: False for warpType in warpTypeList}
351  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
352  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
353  for warpType in warpTypeList}
354 
355  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
356  for calExpInd, calExpRef in enumerate(calexpRefList):
357  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
358  calExpInd+1, len(calexpRefList), calExpRef.dataId)
359  try:
360  ccdId = calExpRef.get("ccdExposureId", immediate=True)
361  except Exception:
362  ccdId = calExpInd
363  try:
364  # We augment the dataRef here with the tract, which is harmless for loading things
365  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
366  # which do.
367  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
368  tract=skyInfo.tractInfo.getId())
369  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
370  except Exception as e:
371  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
372  continue
373 
374  if self.config.doApplySkyCorr:
375  self.applySkyCorr(calExpRef, calExp)
376 
377  try:
378  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
379  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
380  makeDirect=self.config.makeDirect,
381  makePsfMatched=self.config.makePsfMatched)
382  except Exception as e:
383  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
384  continue
385  try:
386  numGoodPix = {warpType: 0 for warpType in warpTypeList}
387  for warpType in warpTypeList:
388  exposure = warpedAndMatched.getDict()[warpType]
389  if exposure is None:
390  continue
391  coaddTempExp = coaddTempExps[warpType]
392  if didSetMetadata[warpType]:
393  mimg = exposure.getMaskedImage()
394  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
395  exposure.getCalib().getFluxMag0()[0])
396  del mimg
397  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
398  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
399  totGoodPix[warpType] += numGoodPix[warpType]
400  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
401  calExpRef.dataId, numGoodPix[warpType],
402  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
403  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
404  coaddTempExp.setCalib(exposure.getCalib())
405  coaddTempExp.setFilter(exposure.getFilter())
406  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
407  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
408  coaddTempExp.setPsf(exposure.getPsf())
409  didSetMetadata[warpType] = True
410 
411  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
412  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
413 
414  except Exception as e:
415  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
416  continue
417 
418  for warpType in warpTypeList:
419  self.log.info("%sWarp has %d good pixels (%.1f%%)",
420  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
421 
422  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
423  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
424  if warpType == "direct":
425  coaddTempExps[warpType].setPsf(
426  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
427  self.config.coaddPsf.makeControl()))
428  else:
429  # No good pixels. Exposure still empty
430  coaddTempExps[warpType] = None
431 
432  result = pipeBase.Struct(exposures=coaddTempExps)
433  return result
434 
435  @staticmethod
436  def _prepareEmptyExposure(skyInfo):
437  """Produce an empty exposure for a given patch"""
438  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
439  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
440  .getPlaneBitMask("NO_DATA"), numpy.inf)
441  return exp
442 
443  def getWarpTypeList(self):
444  """Return list of requested warp types per the config.
445  """
446  warpTypeList = []
447  if self.config.makeDirect:
448  warpTypeList.append("direct")
449  if self.config.makePsfMatched:
450  warpTypeList.append("psfMatched")
451  return warpTypeList
452 
453  def applySkyCorr(self, dataRef, calexp):
454  """Apply correction to the sky background level
455 
456  Sky corrections can be generated with the 'skyCorrection.py'
457  executable in pipe_drivers. Because the sky model used by that
458  code extends over the entire focal plane, this can produce
459  better sky subtraction.
460 
461  The calexp is updated in-place.
462 
463  Parameters
464  ----------
465  dataRef : `lsst.daf.persistence.ButlerDataRef`
466  Data reference for calexp.
467  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
468  Calibrated exposure.
469  """
470  bg = dataRef.get("skyCorr")
471  if isinstance(calexp, afwImage.Exposure):
472  calexp = calexp.getMaskedImage()
473  calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:174
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:97
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:130
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:189
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:224
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:110
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
Definition: coaddBase.py:146
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63