lsst.pipe.tasks  13.0-56-g21e2fbf+3
 All Classes Namespaces Files Functions Variables Groups Pages
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
31 from .coaddBase import CoaddBaseTask
32 from .warpAndPsfMatch import WarpAndPsfMatchTask
33 from .coaddHelpers import groupPatchExposures, getGroupDataRef
34 
35 __all__ = ["MakeCoaddTempExpTask"]
36 
37 
38 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
39  """Config for MakeCoaddTempExpTask
40  """
41  warpAndPsfMatch = pexConfig.ConfigurableField(
42  target=WarpAndPsfMatchTask,
43  doc="Task to warp and PSF-match calexp",
44  )
45  doWrite = pexConfig.Field(
46  doc="persist <coaddName>Coadd_<warpType>Warp",
47  dtype=bool,
48  default=True,
49  )
50  doOverwrite = pexConfig.Field(
51  doc="overwrite <coaddName>Coadd_<warpType>Warp; If False, continue if the file exists on disk",
52  dtype=bool,
53  default=True,
54  )
55  bgSubtracted = pexConfig.Field(
56  doc="Work with a background subtracted calexp?",
57  dtype=bool,
58  default=True,
59  )
60  coaddPsf = pexConfig.ConfigField(
61  doc="Configuration for CoaddPsf",
62  dtype=CoaddPsfConfig,
63  )
64 
65 ## \addtogroup LSST_task_documentation
66 ## \{
67 ## \page MakeCoaddTempExpTask
68 ## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask"
69 ## \copybrief MakeCoaddTempExpTask
70 ## \}
71 
72 
73 class MakeCoaddTempExpTask(CoaddBaseTask):
74  """!Warp and optionally PSF-Match calexps onto an a common projection.
75 
76  @anchor MakeCoaddTempExpTask_
77 
78  @section pipe_tasks_makeCoaddTempExp_Contents Contents
79 
80  - @ref pipe_tasks_makeCoaddTempExp_Purpose
81  - @ref pipe_tasks_makeCoaddTempExp_Initialize
82  - @ref pipe_tasks_makeCoaddTempExp_IO
83  - @ref pipe_tasks_makeCoaddTempExp_Config
84  - @ref pipe_tasks_makeCoaddTempExp_Debug
85  - @ref pipe_tasks_makeCoaddTempExp_Example
86 
87  @section pipe_tasks_makeCoaddTempExp_Purpose Description
88 
89  Warp and optionally PSF-Match calexps onto a common projection, by
90  performing the following operations:
91  - Group calexps by visit/run
92  - For each visit, generate a Warp by calling method @ref makeTempExp.
93  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
94  on each visit
95 
96  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
97 
98  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
99 
100  @copydoc \_\_init\_\_
101 
102  This task has no special keyword arguments.
103 
104  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
105 
106  This task is primarily designed to be run from the command line.
107 
108  The main method is `run`, which takes a single butler data reference for the patch(es)
109  to process.
110 
111  @copydoc run
112 
113  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
114  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
115  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
116  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
117 
118  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
119 
120  See @ref MakeCoaddTempExpConfig and parameters inherited from
121  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
122 
123  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
124 
125  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
126  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
127  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
128  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
129  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
130  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
131 
132  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
133  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
134  In general, for a set of input images, this config should equal the FWHM of the visit
135  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
136  of the other config options offer a reasonable starting point.
137  The following list presents the most common problems that arise from a misconfigured
138  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
139  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
140  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
141  Problem: Explanation. *Solution*
142 
143  *Troublshooting PSF-Matching Configuration:*
144  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
145  For example:_
146 
147  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
148 
149  Note that increasing the kernel size also increases runtime.
150  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
151  for matching kernel. _Provide the matcher with more data by either increasing
152  the spatial sampling by decreasing the spatial cell size,_
153 
154  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
155  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
156 
157  _or increasing the padding around the Science PSF, for example:_
158 
159  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
160 
161  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
162  matching kernel dimensions, thus increasing the number of pixels available to fit
163  after convolving the PSF with the matching kernel.
164  Optionally, for debugging the effects of padding, the level of padding may be manually
165  controlled by setting turning off the automatic padding and setting the number
166  of pixels by which to pad the PSF:
167 
168  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
169  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
170 
171  - Deconvolution: Matching a large PSF to a smaller PSF produces
172  a telltale noise pattern which looks like ripples or a brain.
173  _Increase the size of the requested model PSF. For example:_
174 
175  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
176 
177  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
178  _Increase the width of the Gaussian basis functions. For example:_
179 
180  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
181  # from default [0.7, 1.5, 3.0]
182 
183 
184  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
185 
186  MakeCoaddTempExpTask has no debug output, but its subtasks do.
187 
188  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
189 
190  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
191  into the larger Data Release Processing.
192  Set up by running:
193 
194  setup ci_hsc
195  cd $CI_HSC_DIR
196  # if not built already:
197  python $(which scons) # this will take a while
198 
199  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
200  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
201  output respository with the desired SkyMap. The command,
202 
203  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
204  --id patch=5,4 tract=0 filter=HSC-I \
205  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
206  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
207  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
208 
209  writes a direct and PSF-Matched Warp to
210  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
211  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
212  respectively.
213 
214  @note PSF-Matching in this particular dataset would benefit from adding
215  `--configfile ./matchingConfig.py` to
216  the command line arguments where `matchingConfig.py` is defined by:
217 
218  echo "
219  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
220  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
221 
222 
223  Add the option `--help` to see more options.
224  """
225  ConfigClass = MakeCoaddTempExpConfig
226  _DefaultName = "makeCoaddTempExp"
227 
228  def __init__(self, *args, **kwargs):
229  CoaddBaseTask.__init__(self, *args, **kwargs)
230  self.makeSubtask("warpAndPsfMatch")
231 
232  @pipeBase.timeMethod
233  def run(self, patchRef, selectDataList=[]):
234  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
235 
236  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
237  plus the camera-specific filter key (e.g. "filter" or "band")
238  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
239  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
240  warps are requested.
241 
242  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
243 
244  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
245  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
246  (assembleCoadd should determine zeropoint scaling without referring to it).
247  """
248  skyInfo = self.getSkyInfo(patchRef)
249 
250  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
251  if self.config.makePsfMatched and not self.config.makeDirect:
252  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
253  else:
254  primaryWarpDataset = self.getTempExpDatasetName("direct")
255 
256  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
257  if len(calExpRefList) == 0:
258  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
259  return None
260  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
261  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
262  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
263 
264  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
265  primaryWarpDataset)
266  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
267 
268  dataRefList = []
269  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
270  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
271  tempExpTuple, groupData.keys)
272  if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=primaryWarpDataset):
273  self.log.info("Warp %s exists; skipping", tempExpRef.dataId)
274  dataRefList.append(tempExpRef)
275  continue
276  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
277 
278  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
279  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
280  # of the visit in the list.
281  try:
282  visitId = int(tempExpRef.dataId["visit"])
283  except (KeyError, ValueError):
284  visitId = i
285 
286  exps = self.createTempExp(calexpRefList, skyInfo, visitId).exposures
287 
288  if any(exps.values()):
289  dataRefList.append(tempExpRef)
290  else:
291  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
292 
293  if self.config.doWrite:
294  for (warpType, exposure) in exps.items(): # compatible w/ Py3
295  if exposure is not None:
296  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
297  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
298 
299  return dataRefList
300 
301  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
302  """Create a Warp from inputs
303 
304  We iterate over the multiple calexps in a single exposure to construct
305  the warp (previously called a coaddTempExp) of that exposure to the
306  supplied tract/patch.
307 
308  Pixels that receive no pixels are set to NAN; this is not correct
309  (violates LSST algorithms group policy), but will be fixed up by
310  interpolating after the coaddition.
311 
312  @param calexpRefList: List of data references for calexps that (may)
313  overlap the patch of interest
314  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
315  information about the patch
316  @param visitId: integer identifier for visit, for the table that will
317  produce the CoaddPsf
318  @return a pipeBase Struct containing:
319  - exposures: a dictionary containing the warps requested:
320  "direct": direct warp if config.makeDirect
321  "psfMatched": PSF-matched warp if config.makePsfMatched
322  """
323  warpTypeList = self.getWarpTypeList()
324 
325  totGoodPix = {warpType: 0 for warpType in warpTypeList}
326  didSetMetadata = {warpType: False for warpType in warpTypeList}
327  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
328  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
329  for warpType in warpTypeList}
330 
331  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
332  for calExpInd, calExpRef in enumerate(calexpRefList):
333  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
334  calExpInd+1, len(calexpRefList), calExpRef.dataId)
335  try:
336  ccdId = calExpRef.get("ccdExposureId", immediate=True)
337  except Exception:
338  ccdId = calExpInd
339  try:
340  # We augment the dataRef here with the tract, which is harmless for loading things
341  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
342  # which do.
343  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
344  tract=skyInfo.tractInfo.getId())
345  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
346  except Exception as e:
347  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
348  continue
349  try:
350  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
351  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
352  makeDirect=self.config.makeDirect,
353  makePsfMatched=self.config.makePsfMatched)
354  except Exception as e:
355  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
356  continue
357  try:
358  numGoodPix = {warpType: 0 for warpType in warpTypeList}
359  for warpType in warpTypeList:
360  exposure = warpedAndMatched.getDict()[warpType]
361  if exposure is None:
362  continue
363  coaddTempExp = coaddTempExps[warpType]
364  if didSetMetadata[warpType]:
365  mimg = exposure.getMaskedImage()
366  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
367  exposure.getCalib().getFluxMag0()[0])
368  del mimg
369  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
370  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
371  totGoodPix[warpType] += numGoodPix[warpType]
372  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
373  calExpRef.dataId, numGoodPix[warpType],
374  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
375  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
376  coaddTempExp.setCalib(exposure.getCalib())
377  coaddTempExp.setFilter(exposure.getFilter())
378  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
379  coaddTempExp.setPsf(exposure.getPsf())
380  didSetMetadata[warpType] = True
381 
382  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
383  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
384 
385  except Exception as e:
386  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
387  continue
388 
389  for warpType in warpTypeList:
390  self.log.info("%sWarp has %d good pixels (%.1f%%)",
391  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
392 
393  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
394  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
395  if warpType == "direct":
396  coaddTempExps[warpType].setPsf(
397  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
398  self.config.coaddPsf.makeControl()))
399  else:
400  # No good pixels. Exposure still empty
401  coaddTempExps[warpType] = None
402 
403  result = pipeBase.Struct(exposures=coaddTempExps)
404  return result
405 
406  def _prepareEmptyExposure(cls, skyInfo):
407  """Produce an empty exposure for a given patch"""
408  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
409  exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
410  .getPlaneBitMask("NO_DATA"), numpy.inf)
411  return exp
def run
Produce &lt;coaddName&gt;Coadd_&lt;warpType&gt;Warp images by warping and optionally PSF-matching.
Warp and optionally PSF-Match calexps onto an a common projection.