lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.log as log
31 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
32 from .coaddBase import CoaddBaseTask
33 from .warpAndPsfMatch import WarpAndPsfMatchTask
34 from .coaddHelpers import groupPatchExposures, getGroupDataRef
35 
36 __all__ = ["MakeCoaddTempExpTask"]
37 
38 
39 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
40  """Config for MakeCoaddTempExpTask
41  """
42  warpAndPsfMatch = pexConfig.ConfigurableField(
43  target=WarpAndPsfMatchTask,
44  doc="Task to warp and PSF-match calexp",
45  )
46  doWrite = pexConfig.Field(
47  doc="persist <coaddName>Coadd_<warpType>Warp",
48  dtype=bool,
49  default=True,
50  )
51  doOverwrite = pexConfig.Field(
52  doc="overwrite <coaddName>Coadd_<warpType>Warp; If False, continue if the file exists on disk",
53  dtype=bool,
54  default=True,
55  )
56  bgSubtracted = pexConfig.Field(
57  doc="Work with a background subtracted calexp?",
58  dtype=bool,
59  default=True,
60  )
61  coaddPsf = pexConfig.ConfigField(
62  doc="Configuration for CoaddPsf",
63  dtype=CoaddPsfConfig,
64  )
65  makeDirect = pexConfig.Field(
66  doc="Make direct Warp/Coadds",
67  dtype=bool,
68  default=True,
69  )
70  makePsfMatched = pexConfig.Field(
71  doc="Make Psf-Matched Warp/Coadd?",
72  dtype=bool,
73  default=False,
74  )
75 
76  def validate(self):
77  CoaddBaseTask.ConfigClass.validate(self)
78  if not self.makePsfMatched and not self.makeDirect:
79  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
80  if self.doPsfMatch:
81  # Backwards compatibility.
82  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
83  self.makePsfMatched = True
84  self.makeDirect = False
85 
86 
87 
93 
94 
96  """!Warp and optionally PSF-Match calexps onto an a common projection.
97 
98  @anchor MakeCoaddTempExpTask_
99 
100  @section pipe_tasks_makeCoaddTempExp_Contents Contents
101 
102  - @ref pipe_tasks_makeCoaddTempExp_Purpose
103  - @ref pipe_tasks_makeCoaddTempExp_Initialize
104  - @ref pipe_tasks_makeCoaddTempExp_IO
105  - @ref pipe_tasks_makeCoaddTempExp_Config
106  - @ref pipe_tasks_makeCoaddTempExp_Debug
107  - @ref pipe_tasks_makeCoaddTempExp_Example
108 
109  @section pipe_tasks_makeCoaddTempExp_Purpose Description
110 
111  Warp and optionally PSF-Match calexps onto a common projection, by
112  performing the following operations:
113  - Group calexps by visit/run
114  - For each visit, generate a Warp by calling method @ref makeTempExp.
115  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
116  on each visit
117 
118  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
119 
120  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
121 
122  @copydoc \_\_init\_\_
123 
124  This task has no special keyword arguments.
125 
126  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
127 
128  This task is primarily designed to be run from the command line.
129 
130  The main method is `run`, which takes a single butler data reference for the patch(es)
131  to process.
132 
133  @copydoc run
134 
135  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
136  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
137  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
138  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
139 
140  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
141 
142  See @ref MakeCoaddTempExpConfig and parameters inherited from
143  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
144 
145  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
146 
147  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
148  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
149  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
150  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
151  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
152  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
153 
154  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
155  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
156  In general, for a set of input images, this config should equal the FWHM of the visit
157  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
158  of the other config options offer a reasonable starting point.
159  The following list presents the most common problems that arise from a misconfigured
160  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
161  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
162  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
163  Problem: Explanation. *Solution*
164 
165  *Troublshooting PSF-Matching Configuration:*
166  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
167  For example:_
168 
169  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
170 
171  Note that increasing the kernel size also increases runtime.
172  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
173  for matching kernel. _Provide the matcher with more data by either increasing
174  the spatial sampling by decreasing the spatial cell size,_
175 
176  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
177  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
178 
179  _or increasing the padding around the Science PSF, for example:_
180 
181  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
182 
183  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
184  matching kernel dimensions, thus increasing the number of pixels available to fit
185  after convolving the PSF with the matching kernel.
186  Optionally, for debugging the effects of padding, the level of padding may be manually
187  controlled by setting turning off the automatic padding and setting the number
188  of pixels by which to pad the PSF:
189 
190  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
191  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
192 
193  - Deconvolution: Matching a large PSF to a smaller PSF produces
194  a telltale noise pattern which looks like ripples or a brain.
195  _Increase the size of the requested model PSF. For example:_
196 
197  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
198 
199  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
200  _Increase the width of the Gaussian basis functions. For example:_
201 
202  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
203  # from default [0.7, 1.5, 3.0]
204 
205 
206  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
207 
208  MakeCoaddTempExpTask has no debug output, but its subtasks do.
209 
210  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
211 
212  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
213  into the larger Data Release Processing.
214  Set up by running:
215 
216  setup ci_hsc
217  cd $CI_HSC_DIR
218  # if not built already:
219  python $(which scons) # this will take a while
220 
221  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
222  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
223  output respository with the desired SkyMap. The command,
224 
225  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
226  --id patch=5,4 tract=0 filter=HSC-I \
227  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
228  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
229  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
230 
231  writes a direct and PSF-Matched Warp to
232  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
233  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
234  respectively.
235 
236  @note PSF-Matching in this particular dataset would benefit from adding
237  `--configfile ./matchingConfig.py` to
238  the command line arguments where `matchingConfig.py` is defined by:
239 
240  echo "
241  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
242  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
243 
244 
245  Add the option `--help` to see more options.
246  """
247  ConfigClass = MakeCoaddTempExpConfig
248  _DefaultName = "makeCoaddTempExp"
249 
250  def __init__(self, *args, **kwargs):
251  CoaddBaseTask.__init__(self, *args, **kwargs)
252  self.makeSubtask("warpAndPsfMatch")
253 
254  @pipeBase.timeMethod
255  def run(self, patchRef, selectDataList=[]):
256  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
257 
258  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
259  plus the camera-specific filter key (e.g. "filter" or "band")
260  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
261  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
262  warps are requested.
263 
264  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
265 
266  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
267  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
268  (assembleCoadd should determine zeropoint scaling without referring to it).
269  """
270  skyInfo = self.getSkyInfo(patchRef)
271 
272  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
273  if self.config.makePsfMatched and not self.config.makeDirect:
274  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
275  else:
276  primaryWarpDataset = self.getTempExpDatasetName("direct")
277 
278  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
279  if len(calExpRefList) == 0:
280  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
281  return None
282  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
283  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
284  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
285 
286  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
287  primaryWarpDataset)
288  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
289 
290  dataRefList = []
291  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
292  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
293  tempExpTuple, groupData.keys)
294  if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=primaryWarpDataset):
295  self.log.info("Warp %s exists; skipping", tempExpRef.dataId)
296  dataRefList.append(tempExpRef)
297  continue
298  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
299 
300  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
301  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
302  # of the visit in the list.
303  try:
304  visitId = int(tempExpRef.dataId["visit"])
305  except (KeyError, ValueError):
306  visitId = i
307 
308  exps = self.createTempExp(calexpRefList, skyInfo, visitId).exposures
309 
310  if any(exps.values()):
311  dataRefList.append(tempExpRef)
312  else:
313  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
314 
315  if self.config.doWrite:
316  for (warpType, exposure) in exps.items(): # compatible w/ Py3
317  if exposure is not None:
318  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
319  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
320 
321  return dataRefList
322 
323  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
324  """Create a Warp from inputs
325 
326  We iterate over the multiple calexps in a single exposure to construct
327  the warp (previously called a coaddTempExp) of that exposure to the
328  supplied tract/patch.
329 
330  Pixels that receive no pixels are set to NAN; this is not correct
331  (violates LSST algorithms group policy), but will be fixed up by
332  interpolating after the coaddition.
333 
334  @param calexpRefList: List of data references for calexps that (may)
335  overlap the patch of interest
336  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
337  information about the patch
338  @param visitId: integer identifier for visit, for the table that will
339  produce the CoaddPsf
340  @return a pipeBase Struct containing:
341  - exposures: a dictionary containing the warps requested:
342  "direct": direct warp if config.makeDirect
343  "psfMatched": PSF-matched warp if config.makePsfMatched
344  """
345  warpTypeList = self.getWarpTypeList()
346 
347  totGoodPix = {warpType: 0 for warpType in warpTypeList}
348  didSetMetadata = {warpType: False for warpType in warpTypeList}
349  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
350  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
351  for warpType in warpTypeList}
352 
353  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
354  for calExpInd, calExpRef in enumerate(calexpRefList):
355  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
356  calExpInd+1, len(calexpRefList), calExpRef.dataId)
357  try:
358  ccdId = calExpRef.get("ccdExposureId", immediate=True)
359  except Exception:
360  ccdId = calExpInd
361  try:
362  # We augment the dataRef here with the tract, which is harmless for loading things
363  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
364  # which do.
365  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
366  tract=skyInfo.tractInfo.getId())
367  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
368  except Exception as e:
369  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
370  continue
371  try:
372  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
373  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
374  makeDirect=self.config.makeDirect,
375  makePsfMatched=self.config.makePsfMatched)
376  except Exception as e:
377  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
378  continue
379  try:
380  numGoodPix = {warpType: 0 for warpType in warpTypeList}
381  for warpType in warpTypeList:
382  exposure = warpedAndMatched.getDict()[warpType]
383  if exposure is None:
384  continue
385  coaddTempExp = coaddTempExps[warpType]
386  if didSetMetadata[warpType]:
387  mimg = exposure.getMaskedImage()
388  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
389  exposure.getCalib().getFluxMag0()[0])
390  del mimg
391  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
392  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
393  totGoodPix[warpType] += numGoodPix[warpType]
394  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
395  calExpRef.dataId, numGoodPix[warpType],
396  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
397  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
398  coaddTempExp.setCalib(exposure.getCalib())
399  coaddTempExp.setFilter(exposure.getFilter())
400  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
401  coaddTempExp.setPsf(exposure.getPsf())
402  didSetMetadata[warpType] = True
403 
404  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
405  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
406 
407  except Exception as e:
408  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
409  continue
410 
411  for warpType in warpTypeList:
412  self.log.info("%sWarp has %d good pixels (%.1f%%)",
413  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
414 
415  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
416  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
417  if warpType == "direct":
418  coaddTempExps[warpType].setPsf(
419  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
420  self.config.coaddPsf.makeControl()))
421  else:
422  # No good pixels. Exposure still empty
423  coaddTempExps[warpType] = None
424 
425  result = pipeBase.Struct(exposures=coaddTempExps)
426  return result
427 
428  def _prepareEmptyExposure(cls, skyInfo):
429  """Produce an empty exposure for a given patch"""
430  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
431  exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
432  .getPlaneBitMask("NO_DATA"), numpy.inf)
433  return exp
434 
435  def getWarpTypeList(self):
436  """Return list of requested warp types per the config.
437  """
438  warpTypeList = []
439  if self.config.makeDirect:
440  warpTypeList.append("direct")
441  if self.config.makePsfMatched:
442  warpTypeList.append("psfMatched")
443  return warpTypeList
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
Definition: coaddBase.py:139
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63