lsst.pipe.tasks  13.0-37-g58c8d4e+3
 All Classes Namespaces Files Functions Variables Groups Pages
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 from lsst.meas.algorithms import CoaddPsf
31 from .coaddBase import CoaddBaseTask
32 from .warpAndPsfMatch import WarpAndPsfMatchTask
33 from .coaddHelpers import groupPatchExposures, getGroupDataRef
34 
35 __all__ = ["MakeCoaddTempExpTask"]
36 
37 
38 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
39  """Config for MakeCoaddTempExpTask
40  """
41  warpAndPsfMatch = pexConfig.ConfigurableField(
42  target=WarpAndPsfMatchTask,
43  doc="Task to warp and PSF-match calexp",
44  )
45  doWrite = pexConfig.Field(
46  doc="persist <coaddName>Coadd_<warpType>Warp",
47  dtype=bool,
48  default=True,
49  )
50  doOverwrite = pexConfig.Field(
51  doc="overwrite <coaddName>Coadd_<warpType>Warp; If False, continue if the file exists on disk",
52  dtype=bool,
53  default=True,
54  )
55  bgSubtracted = pexConfig.Field(
56  doc="Work with a background subtracted calexp?",
57  dtype=bool,
58  default=True,
59  )
60 
61 ## \addtogroup LSST_task_documentation
62 ## \{
63 ## \page MakeCoaddTempExpTask
64 ## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask"
65 ## \copybrief MakeCoaddTempExpTask
66 ## \}
67 
68 
69 class MakeCoaddTempExpTask(CoaddBaseTask):
70  """!Warp and optionally PSF-Match calexps onto an a common projection.
71 
72  @anchor MakeCoaddTempExpTask_
73 
74  @section pipe_tasks_makeCoaddTempExp_Contents Contents
75 
76  - @ref pipe_tasks_makeCoaddTempExp_Purpose
77  - @ref pipe_tasks_makeCoaddTempExp_Initialize
78  - @ref pipe_tasks_makeCoaddTempExp_IO
79  - @ref pipe_tasks_makeCoaddTempExp_Config
80  - @ref pipe_tasks_makeCoaddTempExp_Debug
81  - @ref pipe_tasks_makeCoaddTempExp_Example
82 
83  @section pipe_tasks_makeCoaddTempExp_Purpose Description
84 
85  Warp and optionally PSF-Match calexps onto a common projection, by
86  performing the following operations:
87  - Group calexps by visit/run
88  - For each visit, generate a Warp by calling method @ref makeTempExp.
89  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
90  on each visit
91 
92  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
93 
94  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
95 
96  @copydoc \_\_init\_\_
97 
98  This task has no special keyword arguments.
99 
100  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
101 
102  This task is primarily designed to be run from the command line.
103 
104  The main method is `run`, which takes a single butler data reference for the patch(es)
105  to process.
106 
107  @copydoc run
108 
109  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
110  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
111  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
112  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
113 
114  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
115 
116  See @ref MakeCoaddTempExpConfig and parameters inherited from
117  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
118 
119  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
120 
121  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
122  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
123  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
124  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
125  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
126  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
127 
128  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
129  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
130  In general, for a set of input images, this config should equal the FWHM of the visit
131  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
132  of the other config options offer a reasonable starting point.
133  The following list presents the most common problems that arise from a misconfigured
134  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
135  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
136  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
137  Problem: Explanation. *Solution*
138 
139  *Troublshooting PSF-Matching Configuration:*
140  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
141  For example:_
142 
143  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
144 
145  Note that increasing the kernel size also increases runtime.
146  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
147  for matching kernel. _Provide the matcher with more data by either increasing
148  the spatial sampling by decreasing the spatial cell size,_
149 
150  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
151  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
152 
153  _or increasing the padding around the Science PSF, for example:_
154 
155  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
156 
157  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
158  matching kernel dimensions, thus increasing the number of pixels available to fit
159  after convolving the PSF with the matching kernel.
160  Optionally, for debugging the effects of padding, the level of padding may be manually
161  controlled by setting turning off the automatic padding and setting the number
162  of pixels by which to pad the PSF:
163 
164  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
165  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
166 
167  - Deconvolution: Matching a large PSF to a smaller PSF produces
168  a telltale noise pattern which looks like ripples or a brain.
169  _Increase the size of the requested model PSF. For example:_
170 
171  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
172 
173  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
174  _Increase the width of the Gaussian basis functions. For example:_
175 
176  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
177  # from default [0.7, 1.5, 3.0]
178 
179 
180  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
181 
182  MakeCoaddTempExpTask has no debug output, but its subtasks do.
183 
184  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
185 
186  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
187  into the larger Data Release Processing.
188  Set up by running:
189 
190  setup ci_hsc
191  cd $CI_HSC_DIR
192  # if not built already:
193  python $(which scons) # this will take a while
194 
195  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
196  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
197  output respository with the desired SkyMap. The command,
198 
199  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
200  --id patch=5,4 tract=0 filter=HSC-I \
201  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
202  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
203  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
204 
205  writes a direct and PSF-Matched Warp to
206  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
207  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
208  respectively.
209 
210  @note PSF-Matching in this particular dataset would benefit from adding
211  `--configfile ./matchingConfig.py` to
212  the command line arguments where `matchingConfig.py` is defined by:
213 
214  echo "
215  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
216  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
217 
218 
219  Add the option `--help` to see more options.
220  """
221  ConfigClass = MakeCoaddTempExpConfig
222  _DefaultName = "makeCoaddTempExp"
223 
224  def __init__(self, *args, **kwargs):
225  CoaddBaseTask.__init__(self, *args, **kwargs)
226  self.makeSubtask("warpAndPsfMatch")
227 
228  @pipeBase.timeMethod
229  def run(self, patchRef, selectDataList=[]):
230  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
231 
232  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
233  plus the camera-specific filter key (e.g. "filter" or "band")
234  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
235  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
236  warps are requested.
237 
238  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
239 
240  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
241  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
242  (assembleCoadd should determine zeropoint scaling without referring to it).
243  """
244  skyInfo = self.getSkyInfo(patchRef)
245 
246  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
247  if self.config.makePsfMatched and not self.config.makeDirect:
248  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
249  else:
250  primaryWarpDataset = self.getTempExpDatasetName("direct")
251 
252  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
253  if len(calExpRefList) == 0:
254  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
255  return None
256  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
257  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
258  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
259 
260  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
261  primaryWarpDataset)
262  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
263 
264  dataRefList = []
265  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
266  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
267  tempExpTuple, groupData.keys)
268  if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=primaryWarpDataset):
269  self.log.info("Warp %s exists; skipping", tempExpRef.dataId)
270  dataRefList.append(tempExpRef)
271  continue
272  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
273 
274  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
275  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
276  # of the visit in the list.
277  try:
278  visitId = int(tempExpRef.dataId["visit"])
279  except (KeyError, ValueError):
280  visitId = i
281 
282  exps = self.createTempExp(calexpRefList, skyInfo, visitId).exposures
283 
284  if any(exps.values()):
285  dataRefList.append(tempExpRef)
286  else:
287  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
288 
289  if self.config.doWrite:
290  for (warpType, exposure) in exps.items(): # compatible w/ Py3
291  if exposure is not None:
292  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
293  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
294 
295  return dataRefList
296 
297  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
298  """Create a Warp from inputs
299 
300  We iterate over the multiple calexps in a single exposure to construct
301  the warp (previously called a coaddTempExp) of that exposure to the
302  supplied tract/patch.
303 
304  Pixels that receive no pixels are set to NAN; this is not correct
305  (violates LSST algorithms group policy), but will be fixed up by
306  interpolating after the coaddition.
307 
308  @param calexpRefList: List of data references for calexps that (may)
309  overlap the patch of interest
310  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
311  information about the patch
312  @param visitId: integer identifier for visit, for the table that will
313  produce the CoaddPsf
314  @return a pipeBase Struct containing:
315  - exposures: a dictionary containing the warps requested:
316  "direct": direct warp if config.makeDirect
317  "psfMatched": PSF-matched warp if config.makePsfMatched
318  """
319  warpTypeList = self.getWarpTypeList()
320 
321  totGoodPix = {warpType: 0 for warpType in warpTypeList}
322  didSetMetadata = {warpType: False for warpType in warpTypeList}
323  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
324  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
325  for warpType in warpTypeList}
326 
327  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
328  for calExpInd, calExpRef in enumerate(calexpRefList):
329  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
330  calExpInd+1, len(calexpRefList), calExpRef.dataId)
331  try:
332  ccdId = calExpRef.get("ccdExposureId", immediate=True)
333  except Exception:
334  ccdId = calExpInd
335  try:
336  # We augment the dataRef here with the tract, which is harmless for loading things
337  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
338  # which do.
339  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
340  tract=skyInfo.tractInfo.getId())
341  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
342  except Exception as e:
343  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
344  continue
345  try:
346  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
347  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
348  makeDirect=self.config.makeDirect,
349  makePsfMatched=self.config.makePsfMatched)
350  except Exception as e:
351  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
352  continue
353  try:
354  numGoodPix = {warpType: 0 for warpType in warpTypeList}
355  for warpType in warpTypeList:
356  exposure = warpedAndMatched.getDict()[warpType]
357  if exposure is None:
358  continue
359  coaddTempExp = coaddTempExps[warpType]
360  if didSetMetadata[warpType]:
361  mimg = exposure.getMaskedImage()
362  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
363  exposure.getCalib().getFluxMag0()[0])
364  del mimg
365  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
366  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
367  totGoodPix[warpType] += numGoodPix[warpType]
368  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
369  calExpRef.dataId, numGoodPix[warpType],
370  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
371  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
372  coaddTempExp.setCalib(exposure.getCalib())
373  coaddTempExp.setFilter(exposure.getFilter())
374  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
375  coaddTempExp.setPsf(exposure.getPsf())
376  didSetMetadata[warpType] = True
377 
378  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
379  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
380 
381  except Exception as e:
382  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
383  continue
384 
385  for warpType in warpTypeList:
386  self.log.info("%sWarp has %d good pixels (%.1f%%)",
387  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
388 
389  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
390  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
391  if warpType == "direct":
392  coaddTempExps[warpType].setPsf(
393  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs))
394  else:
395  # No good pixels. Exposure still empty
396  coaddTempExps[warpType] = None
397 
398  result = pipeBase.Struct(exposures=coaddTempExps)
399  return result
400 
401  def _prepareEmptyExposure(cls, skyInfo):
402  """Produce an empty exposure for a given patch"""
403  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
404  exp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf)
405  return exp
def run
Produce &lt;coaddName&gt;Coadd_&lt;warpType&gt;Warp images by warping and optionally PSF-matching.
Warp and optionally PSF-Match calexps onto an a common projection.