lsst.pipe.tasks  16.0-61-gb2b2650a+1
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 
24 import lsst.pex.config as pexConfig
25 import lsst.daf.persistence as dafPersist
26 import lsst.afw.image as afwImage
27 import lsst.coadd.utils as coaddUtils
28 import lsst.pipe.base as pipeBase
29 import lsst.log as log
30 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
31 from .coaddBase import CoaddBaseTask, makeSkyInfo
32 from .warpAndPsfMatch import WarpAndPsfMatchTask
33 from .coaddHelpers import groupPatchExposures, getGroupDataRef
34 
35 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
36 
37 
38 class MissingExposureError(Exception):
39  """Raised when data cannot be retrieved for an exposure.
40  When processing patches, sometimes one exposure is missing; this lets us
41  distinguish bewteen that case, and other errors.
42  """
43  pass
44 
45 
46 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
47  """Config for MakeCoaddTempExpTask
48  """
49  warpAndPsfMatch = pexConfig.ConfigurableField(
50  target=WarpAndPsfMatchTask,
51  doc="Task to warp and PSF-match calexp",
52  )
53  doWrite = pexConfig.Field(
54  doc="persist <coaddName>Coadd_<warpType>Warp",
55  dtype=bool,
56  default=True,
57  )
58  bgSubtracted = pexConfig.Field(
59  doc="Work with a background subtracted calexp?",
60  dtype=bool,
61  default=True,
62  )
63  coaddPsf = pexConfig.ConfigField(
64  doc="Configuration for CoaddPsf",
65  dtype=CoaddPsfConfig,
66  )
67  makeDirect = pexConfig.Field(
68  doc="Make direct Warp/Coadds",
69  dtype=bool,
70  default=True,
71  )
72  makePsfMatched = pexConfig.Field(
73  doc="Make Psf-Matched Warp/Coadd?",
74  dtype=bool,
75  default=False,
76  )
77  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
78 
79  def validate(self):
80  CoaddBaseTask.ConfigClass.validate(self)
81  if not self.makePsfMatched and not self.makeDirect:
82  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
83  if self.doPsfMatch:
84  # Backwards compatibility.
85  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
86  self.makePsfMatched = True
87  self.makeDirect = False
88 
89  def setDefaults(self):
90  CoaddBaseTask.ConfigClass.setDefaults(self)
91  self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
92 
93 
99 
100 
102  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
103 
104  @anchor MakeCoaddTempExpTask_
105 
106  @section pipe_tasks_makeCoaddTempExp_Contents Contents
107 
108  - @ref pipe_tasks_makeCoaddTempExp_Purpose
109  - @ref pipe_tasks_makeCoaddTempExp_Initialize
110  - @ref pipe_tasks_makeCoaddTempExp_IO
111  - @ref pipe_tasks_makeCoaddTempExp_Config
112  - @ref pipe_tasks_makeCoaddTempExp_Debug
113  - @ref pipe_tasks_makeCoaddTempExp_Example
114 
115  @section pipe_tasks_makeCoaddTempExp_Purpose Description
116 
117  Warp and optionally PSF-Match calexps onto a common projection, by
118  performing the following operations:
119  - Group calexps by visit/run
120  - For each visit, generate a Warp by calling method @ref makeTempExp.
121  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
122  on each visit
123 
124  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
125 
126  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
127 
128  @copydoc \_\_init\_\_
129 
130  This task has one special keyword argument: passing reuse=True will cause
131  the task to skip the creation of warps that are already present in the
132  output repositories.
133 
134  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
135 
136  This task is primarily designed to be run from the command line.
137 
138  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
139  to process.
140 
141  @copydoc run
142 
143  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
144  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
145  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
146  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
147 
148  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
149 
150  See @ref MakeCoaddTempExpConfig and parameters inherited from
151  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
152 
153  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
154 
155  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
156  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
157  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
158  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
159  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
160  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
161 
162  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
163  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
164  In general, for a set of input images, this config should equal the FWHM of the visit
165  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
166  of the other config options offer a reasonable starting point.
167  The following list presents the most common problems that arise from a misconfigured
168  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
169  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
170  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
171  Problem: Explanation. *Solution*
172 
173  *Troublshooting PSF-Matching Configuration:*
174  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
175  For example:_
176 
177  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
178 
179  Note that increasing the kernel size also increases runtime.
180  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
181  for matching kernel. _Provide the matcher with more data by either increasing
182  the spatial sampling by decreasing the spatial cell size,_
183 
184  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
185  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
186 
187  _or increasing the padding around the Science PSF, for example:_
188 
189  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
190 
191  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
192  matching kernel dimensions, thus increasing the number of pixels available to fit
193  after convolving the PSF with the matching kernel.
194  Optionally, for debugging the effects of padding, the level of padding may be manually
195  controlled by setting turning off the automatic padding and setting the number
196  of pixels by which to pad the PSF:
197 
198  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
199  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
200 
201  - Deconvolution: Matching a large PSF to a smaller PSF produces
202  a telltale noise pattern which looks like ripples or a brain.
203  _Increase the size of the requested model PSF. For example:_
204 
205  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
206 
207  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
208  _Increase the width of the Gaussian basis functions. For example:_
209 
210  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
211  # from default [0.7, 1.5, 3.0]
212 
213 
214  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
215 
216  MakeCoaddTempExpTask has no debug output, but its subtasks do.
217 
218  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
219 
220  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
221  into the larger Data Release Processing.
222  Set up by running:
223 
224  setup ci_hsc
225  cd $CI_HSC_DIR
226  # if not built already:
227  python $(which scons) # this will take a while
228 
229  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
230  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
231  output respository with the desired SkyMap. The command,
232 
233  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
234  --id patch=5,4 tract=0 filter=HSC-I \
235  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
236  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
237  --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
238 
239  writes a direct and PSF-Matched Warp to
240  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
241  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
242  respectively.
243 
244  @note PSF-Matching in this particular dataset would benefit from adding
245  `--configfile ./matchingConfig.py` to
246  the command line arguments where `matchingConfig.py` is defined by:
247 
248  echo "
249  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
250  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
251 
252 
253  Add the option `--help` to see more options.
254  """
255  ConfigClass = MakeCoaddTempExpConfig
256  _DefaultName = "makeCoaddTempExp"
257 
258  def __init__(self, reuse=False, **kwargs):
259  CoaddBaseTask.__init__(self, **kwargs)
260  self.reuse = reuse
261  self.makeSubtask("warpAndPsfMatch")
262 
263  @pipeBase.timeMethod
264  def runDataRef(self, patchRef, selectDataList=[]):
265  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
266 
267  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
268  plus the camera-specific filter key (e.g. "filter" or "band")
269  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
270  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
271  warps are requested.
272 
273  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
274 
275  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
276  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
277  (assembleCoadd should determine zeropoint scaling without referring to it).
278  """
279  skyInfo = self.getSkyInfo(patchRef)
280 
281  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
282  if self.config.makePsfMatched and not self.config.makeDirect:
283  primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
284  else:
285  primaryWarpDataset = self.getTempExpDatasetName("direct")
286 
287  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
288  if len(calExpRefList) == 0:
289  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
290  return None
291  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
292  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
293  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
294 
295  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
296  primaryWarpDataset)
297  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
298 
299  dataRefList = []
300  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
301  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
302  tempExpTuple, groupData.keys)
303  if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
304  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
305  dataRefList.append(tempExpRef)
306  continue
307  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
308 
309  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
310  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
311  # of the visit in the list.
312  try:
313  visitId = int(tempExpRef.dataId["visit"])
314  except (KeyError, ValueError):
315  visitId = i
316 
317  calExpList = []
318  ccdIdList = []
319  dataIdList = []
320 
321  for calExpInd, calExpRef in enumerate(calexpRefList):
322  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
323  calExpRef.dataId)
324  try:
325  ccdId = calExpRef.get("ccdExposureId", immediate=True)
326  except Exception:
327  ccdId = calExpInd
328  try:
329  # We augment the dataRef here with the tract, which is harmless for loading things
330  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
331  # which do.
332  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
333  tract=skyInfo.tractInfo.getId())
334  calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
335  except Exception as e:
336  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
337  continue
338 
339  if self.config.doApplySkyCorr:
340  self.applySkyCorr(calExpRef, calExp)
341 
342  calExpList.append(calExp)
343  ccdIdList.append(ccdId)
344  dataIdList.append(calExpRef.dataId)
345 
346  exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
347 
348  if any(exps.values()):
349  dataRefList.append(tempExpRef)
350  else:
351  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
352 
353  if self.config.doWrite:
354  for (warpType, exposure) in exps.items(): # compatible w/ Py3
355  if exposure is not None:
356  self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
357  tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
358 
359  return dataRefList
360 
361  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
362  """Create a Warp from inputs
363 
364  We iterate over the multiple calexps in a single exposure to construct
365  the warp (previously called a coaddTempExp) of that exposure to the
366  supplied tract/patch.
367 
368  Pixels that receive no pixels are set to NAN; this is not correct
369  (violates LSST algorithms group policy), but will be fixed up by
370  interpolating after the coaddition.
371 
372  @param calexpRefList: List of data references for calexps that (may)
373  overlap the patch of interest
374  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
375  information about the patch
376  @param visitId: integer identifier for visit, for the table that will
377  produce the CoaddPsf
378  @return a pipeBase Struct containing:
379  - exposures: a dictionary containing the warps requested:
380  "direct": direct warp if config.makeDirect
381  "psfMatched": PSF-matched warp if config.makePsfMatched
382  """
383  warpTypeList = self.getWarpTypeList()
384 
385  totGoodPix = {warpType: 0 for warpType in warpTypeList}
386  didSetMetadata = {warpType: False for warpType in warpTypeList}
387  coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
388  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
389  for warpType in warpTypeList}
390 
391  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
392  if dataIdList is None:
393  dataIdList = ccdIdList
394 
395  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
396  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
397  calExpInd+1, len(calExpList), dataId)
398 
399  try:
400  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
401  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
402  makeDirect=self.config.makeDirect,
403  makePsfMatched=self.config.makePsfMatched)
404  except Exception as e:
405  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
406  continue
407  try:
408  numGoodPix = {warpType: 0 for warpType in warpTypeList}
409  for warpType in warpTypeList:
410  exposure = warpedAndMatched.getDict()[warpType]
411  if exposure is None:
412  continue
413  coaddTempExp = coaddTempExps[warpType]
414  if didSetMetadata[warpType]:
415  mimg = exposure.getMaskedImage()
416  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
417  exposure.getCalib().getFluxMag0()[0])
418  del mimg
419  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
420  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
421  totGoodPix[warpType] += numGoodPix[warpType]
422  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
423  dataId, numGoodPix[warpType],
424  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
425  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
426  coaddTempExp.setCalib(exposure.getCalib())
427  coaddTempExp.setFilter(exposure.getFilter())
428  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
429  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
430  coaddTempExp.setPsf(exposure.getPsf())
431  didSetMetadata[warpType] = True
432 
433  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
434  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
435 
436  except Exception as e:
437  self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
438  continue
439 
440  for warpType in warpTypeList:
441  self.log.info("%sWarp has %d good pixels (%.1f%%)",
442  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
443 
444  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
445  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
446  if warpType == "direct":
447  coaddTempExps[warpType].setPsf(
448  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
449  self.config.coaddPsf.makeControl()))
450  else:
451  # No good pixels. Exposure still empty
452  coaddTempExps[warpType] = None
453 
454  result = pipeBase.Struct(exposures=coaddTempExps)
455  return result
456 
457  def getCalibratedExposure(self, dataRef, bgSubtracted):
458  """Return one calibrated Exposure, possibly with an updated SkyWcs.
459 
460  @param[in] dataRef a sensor-level data reference
461  @param[in] bgSubtracted return calexp with background subtracted? If False get the
462  calexp's background background model and add it to the calexp.
463  @return calibrated exposure
464 
465  @raises MissingExposureError If data for the exposure is not available.
466 
467  If config.doApplyUberCal, the exposure will be photometrically
468  calibrated via the `jointcal_photoCalib` dataset and have its SkyWcs
469  updated to the `jointcal_wcs`, otherwise it will be calibrated via the
470  Exposure's own Calib and have the original SkyWcs.
471  """
472  try:
473  exposure = dataRef.get("calexp", immediate=True)
474  except dafPersist.NoResults as e:
475  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
476 
477  if not bgSubtracted:
478  background = dataRef.get("calexpBackground", immediate=True)
479  mi = exposure.getMaskedImage()
480  mi += background.getImage()
481  del mi
482 
483  # TODO: this is needed until DM-10153 is done and Calib is gone
484  referenceFlux = 1e23 * 10**(48.6 / -2.5) * 1e9
485  if self.config.doApplyUberCal:
486  if self.config.useMeasMosaic:
487  from lsst.meas.mosaic import applyMosaicResultsExposure
488  # NOTE: this changes exposure in-place, updating its Calib and Wcs.
489  # Save the calibration error, as it gets overwritten with zero.
490  fluxMag0Err = exposure.getCalib().getFluxMag0()[1]
491  try:
492  applyMosaicResultsExposure(dataRef, calexp=exposure)
493  except dafPersist.NoResults as e:
494  raise MissingExposureError('Mosaic calibration not found: %s ' % str(e)) from e
495  fluxMag0 = exposure.getCalib().getFluxMag0()[0]
496  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0,
497  referenceFlux*fluxMag0Err/fluxMag0**2,
498  exposure.getBBox())
499  else:
500  photoCalib = dataRef.get("jointcal_photoCalib")
501  skyWcs = dataRef.get("jointcal_wcs")
502  exposure.setWcs(skyWcs)
503  else:
504  fluxMag0 = exposure.getCalib().getFluxMag0()
505  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0[0],
506  referenceFlux*fluxMag0[1]/fluxMag0[0]**2,
507  exposure.getBBox())
508 
509  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
510  includeScaleUncertainty=self.config.includeCalibVar)
511  exposure.maskedImage /= photoCalib.getCalibrationMean()
512  exposure.setCalib(afwImage.Calib(photoCalib.getInstFluxAtZeroMagnitude()))
513  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
514  # exposure.setCalib(afwImage.Calib(1.0))
515  return exposure
516 
517  @staticmethod
518  def _prepareEmptyExposure(skyInfo):
519  """Produce an empty exposure for a given patch"""
520  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
521  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
522  .getPlaneBitMask("NO_DATA"), numpy.inf)
523  return exp
524 
525  def getWarpTypeList(self):
526  """Return list of requested warp types per the config.
527  """
528  warpTypeList = []
529  if self.config.makeDirect:
530  warpTypeList.append("direct")
531  if self.config.makePsfMatched:
532  warpTypeList.append("psfMatched")
533  return warpTypeList
534 
535  def applySkyCorr(self, dataRef, calexp):
536  """Apply correction to the sky background level
537 
538  Sky corrections can be generated with the 'skyCorrection.py'
539  executable in pipe_drivers. Because the sky model used by that
540  code extends over the entire focal plane, this can produce
541  better sky subtraction.
542 
543  The calexp is updated in-place.
544 
545  Parameters
546  ----------
547  dataRef : `lsst.daf.persistence.ButlerDataRef`
548  Data reference for calexp.
549  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
550  Calibrated exposure.
551  """
552  bg = dataRef.get("skyCorr")
553  if isinstance(calexp, afwImage.Exposure):
554  calexp = calexp.getMaskedImage()
555  calexp -= bg.getImage()
556 
557 
558 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig):
559  calExpList = pipeBase.InputDatasetField(
560  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
561  name="calexp",
562  storageClass="ExposureF",
563  dimensions=("Visit", "Detector")
564  )
565  backgroundList = pipeBase.InputDatasetField(
566  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
567  name="calexpBackground",
568  storageClass="Background",
569  dimensions=("Visit", "Detector")
570  )
571  skyCorrList = pipeBase.InputDatasetField(
572  doc="SkyCorr",
573  name="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
574  storageClass="Background",
575  dimensions=("Visit", "Detector")
576  )
577  skyMap = pipeBase.InputDatasetField(
578  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
579  nameTemplate="{coaddName}Coadd_skyMap",
580  storageClass="SkyMap",
581  dimensions=("SkyMap",),
582  scalar=True
583  )
584  direct = pipeBase.OutputDatasetField(
585  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
586  "calexps onto the skyMap patch geometry."),
587  nameTemplate="{coaddName}Coadd_directWarp",
588  storageClass="ExposureF",
589  dimensions=("Tract", "Patch", "SkyMap", "Visit"),
590  scalar=True
591  )
592  psfMatched = pipeBase.OutputDatasetField(
593  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
594  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
595  nameTemplate="{coaddName}Coadd_psfMatchedWarp",
596  storageClass="ExposureF",
597  dimensions=("Tract", "Patch", "SkyMap", "Visit"),
598  scalar=True
599  )
600 
601  def setDefaults(self):
602  super().setDefaults()
603  self.formatTemplateNames({"coaddName": "deep"})
604  self.quantum.dimensions = ("Tract", "Patch", "SkyMap", "Visit")
605 
606  def validate(self):
607  super().validate()
608  # TODO: Remove this constraint after DM-17062
609  if self.doApplyUberCal:
610  raise RuntimeError("Gen3 MakeWarpTask cannot apply meas_mosaic or jointcal results."
611  "Please set doApplyUbercal=False.")
612 
613 
614 class MakeWarpTask(MakeCoaddTempExpTask, pipeBase.PipelineTask):
615  """Warp and optionally PSF-Match calexps onto an a common projection
616 
617  First Draft of a Gen3 compatible MakeWarpTask which
618  currently does not handle doApplyUberCal=True.
619  """
620  ConfigClass = MakeWarpConfig
621  _DefaultName = "makeWarp"
622 
623  @classmethod
624  def getInputDatasetTypes(cls, config):
625  """Return input dataset type descriptors
626 
627  Remove input dataset types not used by the Task
628  """
629  inputTypeDict = super().getInputDatasetTypes(config)
630  if config.bgSubtracted:
631  inputTypeDict.pop("backgroundList", None)
632  if not config.doApplySkyCorr:
633  inputTypeDict.pop("skyCorr", None)
634  return inputTypeDict
635 
636  @classmethod
637  def getOutputDatasetTypes(cls, config):
638  """Return output dataset type descriptors
639 
640  Remove output dataset types not produced by the Task
641  """
642  outputTypeDict = super().getOutputDatasetTypes(config)
643  if not config.makeDirect:
644  outputTypeDict.pop("direct", None)
645  if not config.makePsfMatched:
646  outputTypeDict.pop("psfMatched", None)
647  return outputTypeDict
648 
649  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
650  """Construct warps for requested warp type for single epoch
651 
652  PipelineTask (Gen3) entry point to warp and optionally PSF-match
653  calexps. This method is analogous to `runDataRef`, it prepares all
654  the data products to be passed to `run`.
655  Return a Struct with only requested warpTypes controlled by the configs
656  makePsfMatched and makeDirect.
657 
658  Parameters
659  ----------
660  inputData : `dict`
661  Keys are the names of the configs describing input dataset types.
662  Values are input Python-domain data objects (or lists of objects)
663  retrieved from data butler.
664  inputDataIds : `dict`
665  Keys are the names of the configs describing input dataset types.
666  Values are DataIds (or lists of DataIds) that task consumes for
667  corresponding dataset type.
668  outputDataIds : `dict`
669  Keys are the names of the configs describing input dataset types.
670  Values are DataIds (or lists of DataIds) that task is to produce
671  for corresponding dataset type.
672  butler : `lsst.daf.butler.Butler`
673  Gen3 Butler object for fetching additional data products before
674  running the Task
675 
676  Returns
677  -------
678  result : `lsst.pipe.base.Struct`
679  Result struct with components:
680 
681  - ``direct`` : (optional) direct Warp Exposure
682  (``lsst.afw.image.Exposure``)
683  - ``psfMatched``: (optional) PSF-Matched Warp Exposure
684  (``lsst.afw.image.Exposure``)
685  """
686  # Construct skyInfo expected by `run`
687  skyMap = inputData["skyMap"]
688  outputDataId = next(iter(outputDataIds.values()))
689  inputData['skyInfo'] = makeSkyInfo(skyMap,
690  tractId=outputDataId['tract'],
691  patchId=outputDataId['patch'])
692 
693  # Construct list of DataIds expected by `run`
694  dataIdList = inputDataIds['calExpList']
695  inputData['dataIdList'] = dataIdList
696 
697  # Construct list of ccdExposureIds expected by `run`
698  inputData['ccdIdList'] = [butler.registry.packDataId("VisitDetector", dataId)
699  for dataId in dataIdList]
700 
701  # Extract integer visitId requested by `run`
702  visits = [dataId['visit'] for dataId in dataIdList]
703  assert(all(visits[0] == visit for visit in visits))
704  inputData["visitId"] = visits[0]
705 
706  self.prepareCalibratedExposures(**inputData)
707  results = self.run(**inputData)
708  return pipeBase.Struct(**results.exposures)
709 
710  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, **kwargs):
711  """Calibrate and add backgrounds to input calExpList in place
712 
713  TODO DM-17062: apply jointcal/meas_mosaic here
714 
715  Parameters
716  ----------
717  calExpList : `list` of `lsst.afw.image.Exposure`
718  Sequence of calexps to be modified in place
719  backgroundList : `list` of `lsst.afw.math.backgroundList`
720  Sequence of backgrounds to be added back in if bgSubtracted=False
721  skyCorrList : `list` of `lsst.afw.math.backgroundList`
722  Sequence of background corrections to be subtracted if doApplySkyCorr=True
723  """
724  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
725  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
726  for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
727  mi = calexp.maskedImage
728  if not self.config.bgSubtracted:
729  mi += background.getImage()
730  if self.config.doApplySkyCorr:
731  mi -= skyCorr.getImage()
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:149
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:100
def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None, kwargs)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:249
Warp and optionally PSF-Match calexps onto an a common projection.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, kwargs)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:199
def getCalibratedExposure(self, dataRef, bgSubtracted)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:113
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60