lsst.pipe.tasks  14.0-19-g29094cd5
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.log as log
31 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
32 from .coaddBase import CoaddBaseTask
33 from .warpAndPsfMatch import WarpAndPsfMatchTask
34 from .coaddHelpers import groupPatchExposures, getGroupDataRef
35 
36 __all__ = ["MakeCoaddTempExpTask"]
37 
38 
39 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
40  """Config for MakeCoaddTempExpTask
41  """
42  warpAndPsfMatch = pexConfig.ConfigurableField(
43  target=WarpAndPsfMatchTask,
44  doc="Task to warp and PSF-match calexp",
45  )
46  doWrite = pexConfig.Field(
47  doc="persist <coaddName>Coadd_<warpType>Warp",
48  dtype=bool,
49  default=True,
50  )
51  bgSubtracted = pexConfig.Field(
52  doc="Work with a background subtracted calexp?",
53  dtype=bool,
54  default=True,
55  )
56  coaddPsf = pexConfig.ConfigField(
57  doc="Configuration for CoaddPsf",
58  dtype=CoaddPsfConfig,
59  )
60  makeDirect = pexConfig.Field(
61  doc="Make direct Warp/Coadds",
62  dtype=bool,
63  default=True,
64  )
65  makePsfMatched = pexConfig.Field(
66  doc="Make Psf-Matched Warp/Coadd?",
67  dtype=bool,
68  default=False,
69  )
70 
71  def validate(self):
72  CoaddBaseTask.ConfigClass.validate(self)
73  if not self.makePsfMatched and not self.makeDirect:
74  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
75  if self.doPsfMatch:
76  # Backwards compatibility.
77  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
78  self.makePsfMatched = True
79  self.makeDirect = False
80 
81 
82 
88 
89 
91  """!Warp and optionally PSF-Match calexps onto an a common projection.
92 
93  @anchor MakeCoaddTempExpTask_
94 
95  @section pipe_tasks_makeCoaddTempExp_Contents Contents
96 
97  - @ref pipe_tasks_makeCoaddTempExp_Purpose
98  - @ref pipe_tasks_makeCoaddTempExp_Initialize
99  - @ref pipe_tasks_makeCoaddTempExp_IO
100  - @ref pipe_tasks_makeCoaddTempExp_Config
101  - @ref pipe_tasks_makeCoaddTempExp_Debug
102  - @ref pipe_tasks_makeCoaddTempExp_Example
103 
104  @section pipe_tasks_makeCoaddTempExp_Purpose Description
105 
106  Warp and optionally PSF-Match calexps onto a common projection, by
107  performing the following operations:
108  - Group calexps by visit/run
109  - For each visit, generate a Warp by calling method @ref makeTempExp.
110  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
111  on each visit
112 
113  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
114 
115  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
116 
117  @copydoc \_\_init\_\_
118 
119  This task has one special keyword argument: passing reuse=True will cause
120  the task to skip the creation of warps that are already present in the
121  output repositories.
122 
123  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
124 
125  This task is primarily designed to be run from the command line.
126 
127  The main method is `run`, which takes a single butler data reference for the patch(es)
128  to process.
129 
130  @copydoc run
131 
132  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
133  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
134  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
135  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
136 
137  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
138 
139  See @ref MakeCoaddTempExpConfig and parameters inherited from
140  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
141 
142  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
143 
144  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
145  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
146  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
147  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
148  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
149  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
150 
151  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
152  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
153  In general, for a set of input images, this config should equal the FWHM of the visit
154  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
155  of the other config options offer a reasonable starting point.
156  The following list presents the most common problems that arise from a misconfigured
157  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
158  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
159  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
160  Problem: Explanation. *Solution*
161 
162  *Troublshooting PSF-Matching Configuration:*
163  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
164  For example:_
165 
166  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
167 
168  Note that increasing the kernel size also increases runtime.
169  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
170  for matching kernel. _Provide the matcher with more data by either increasing
171  the spatial sampling by decreasing the spatial cell size,_
172 
173  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
174  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
175 
176  _or increasing the padding around the Science PSF, for example:_
177 
178  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
179 
180  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
181  matching kernel dimensions, thus increasing the number of pixels available to fit
182  after convolving the PSF with the matching kernel.
183  Optionally, for debugging the effects of padding, the level of padding may be manually
184  controlled by setting turning off the automatic padding and setting the number
185  of pixels by which to pad the PSF:
186 
187  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
188  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
189 
190  - Deconvolution: Matching a large PSF to a smaller PSF produces
191  a telltale noise pattern which looks like ripples or a brain.
192  _Increase the size of the requested model PSF. For example:_
193 
194  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
195 
196  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
197  _Increase the width of the Gaussian basis functions. For example:_
198 
199  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
200  # from default [0.7, 1.5, 3.0]
201 
202 
203  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
204 
205  MakeCoaddTempExpTask has no debug output, but its subtasks do.
206 
207  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
208 
209  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
210  into the larger Data Release Processing.
211  Set up by running:
212 
213  setup ci_hsc
214  cd $CI_HSC_DIR
215  # if not built already:
216  python $(which scons) # this will take a while
217 
218  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
219  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
220  output respository with the desired SkyMap. The command,
221 
222  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
223  --id patch=5,4 tract=0 filter=HSC-I \
224  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
225  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
226  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
227 
228  writes a direct and PSF-Matched Warp to
229  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
230  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
231  respectively.
232 
233  @note PSF-Matching in this particular dataset would benefit from adding
234  `--configfile ./matchingConfig.py` to
235  the command line arguments where `matchingConfig.py` is defined by:
236 
237  echo "
238  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
239  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
240 
241 
242  Add the option `--help` to see more options.
243  """
244  ConfigClass = MakeCoaddTempExpConfig
245  _DefaultName = "makeCoaddTempExp"
246 
247  def __init__(self, reuse=False, **kwargs):
248  CoaddBaseTask.__init__(self, **kwargs)
249  self.reuse = reuse
250  self.makeSubtask("warpAndPsfMatch")
251 
252  @pipeBase.timeMethod
253  def run(self, patchRef, selectDataList=[]):
254  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
255 
256  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
257  plus the camera-specific filter key (e.g. "filter" or "band")
258  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
259  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
260  warps are requested.
261 
262  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
263 
264  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
265  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
266  (assembleCoadd should determine zeropoint scaling without referring to it).
267  """
268  skyInfo = self.getSkyInfo(patchRef)
269 
270  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
271  if self.config.makePsfMatched and not self.config.makeDirect:
272  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
273  else:
274  primaryWarpDataset = self.getTempExpDatasetName("direct")
275 
276  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
277  if len(calExpRefList) == 0:
278  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
279  return None
280  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
281  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
282  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
283 
284  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
285  primaryWarpDataset)
286  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
287 
288  dataRefList = []
289  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
290  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
291  tempExpTuple, groupData.keys)
292  if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
293  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
294  dataRefList.append(tempExpRef)
295  continue
296  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
297 
298  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
299  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
300  # of the visit in the list.
301  try:
302  visitId = int(tempExpRef.dataId["visit"])
303  except (KeyError, ValueError):
304  visitId = i
305 
306  exps = self.createTempExp(calexpRefList, skyInfo, visitId).exposures
307 
308  if any(exps.values()):
309  dataRefList.append(tempExpRef)
310  else:
311  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
312 
313  if self.config.doWrite:
314  for (warpType, exposure) in exps.items(): # compatible w/ Py3
315  if exposure is not None:
316  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
317  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
318 
319  return dataRefList
320 
321  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
322  """Create a Warp from inputs
323 
324  We iterate over the multiple calexps in a single exposure to construct
325  the warp (previously called a coaddTempExp) of that exposure to the
326  supplied tract/patch.
327 
328  Pixels that receive no pixels are set to NAN; this is not correct
329  (violates LSST algorithms group policy), but will be fixed up by
330  interpolating after the coaddition.
331 
332  @param calexpRefList: List of data references for calexps that (may)
333  overlap the patch of interest
334  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
335  information about the patch
336  @param visitId: integer identifier for visit, for the table that will
337  produce the CoaddPsf
338  @return a pipeBase Struct containing:
339  - exposures: a dictionary containing the warps requested:
340  "direct": direct warp if config.makeDirect
341  "psfMatched": PSF-matched warp if config.makePsfMatched
342  """
343  warpTypeList = self.getWarpTypeList()
344 
345  totGoodPix = {warpType: 0 for warpType in warpTypeList}
346  didSetMetadata = {warpType: False for warpType in warpTypeList}
347  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
348  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
349  for warpType in warpTypeList}
350 
351  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
352  for calExpInd, calExpRef in enumerate(calexpRefList):
353  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
354  calExpInd+1, len(calexpRefList), calExpRef.dataId)
355  try:
356  ccdId = calExpRef.get("ccdExposureId", immediate=True)
357  except Exception:
358  ccdId = calExpInd
359  try:
360  # We augment the dataRef here with the tract, which is harmless for loading things
361  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
362  # which do.
363  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
364  tract=skyInfo.tractInfo.getId())
365  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
366  except Exception as e:
367  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
368  continue
369  try:
370  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
371  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
372  makeDirect=self.config.makeDirect,
373  makePsfMatched=self.config.makePsfMatched)
374  except Exception as e:
375  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
376  continue
377  try:
378  numGoodPix = {warpType: 0 for warpType in warpTypeList}
379  for warpType in warpTypeList:
380  exposure = warpedAndMatched.getDict()[warpType]
381  if exposure is None:
382  continue
383  coaddTempExp = coaddTempExps[warpType]
384  if didSetMetadata[warpType]:
385  mimg = exposure.getMaskedImage()
386  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
387  exposure.getCalib().getFluxMag0()[0])
388  del mimg
389  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
390  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
391  totGoodPix[warpType] += numGoodPix[warpType]
392  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
393  calExpRef.dataId, numGoodPix[warpType],
394  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
395  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
396  coaddTempExp.setCalib(exposure.getCalib())
397  coaddTempExp.setFilter(exposure.getFilter())
398  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
399  coaddTempExp.setPsf(exposure.getPsf())
400  didSetMetadata[warpType] = True
401 
402  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
403  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
404 
405  except Exception as e:
406  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
407  continue
408 
409  for warpType in warpTypeList:
410  self.log.info("%sWarp has %d good pixels (%.1f%%)",
411  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
412 
413  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
414  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
415  if warpType == "direct":
416  coaddTempExps[warpType].setPsf(
417  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
418  self.config.coaddPsf.makeControl()))
419  else:
420  # No good pixels. Exposure still empty
421  coaddTempExps[warpType] = None
422 
423  result = pipeBase.Struct(exposures=coaddTempExps)
424  return result
425 
426  def _prepareEmptyExposure(cls, skyInfo):
427  """Produce an empty exposure for a given patch"""
428  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
429  exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
430  .getPlaneBitMask("NO_DATA"), numpy.inf)
431  return exp
432 
433  def getWarpTypeList(self):
434  """Return list of requested warp types per the config.
435  """
436  warpTypeList = []
437  if self.config.makeDirect:
438  warpTypeList.append("direct")
439  if self.config.makePsfMatched:
440  warpTypeList.append("psfMatched")
441  return warpTypeList
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
Definition: coaddBase.py:139
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63