lsst.pipe.tasks  21.0.0-35-g64f566ff+b27e5ef93e
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 
24 import lsst.pex.config as pexConfig
25 import lsst.daf.persistence as dafPersist
26 import lsst.afw.image as afwImage
27 import lsst.coadd.utils as coaddUtils
28 import lsst.pipe.base as pipeBase
30 import lsst.log as log
31 import lsst.utils as utils
32 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
33 from lsst.skymap import BaseSkyMap
34 from .coaddBase import CoaddBaseTask, makeSkyInfo
35 from .warpAndPsfMatch import WarpAndPsfMatchTask
36 from .coaddHelpers import groupPatchExposures, getGroupDataRef
37 
38 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
39 
40 
41 class MissingExposureError(Exception):
42  """Raised when data cannot be retrieved for an exposure.
43  When processing patches, sometimes one exposure is missing; this lets us
44  distinguish bewteen that case, and other errors.
45  """
46  pass
47 
48 
49 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
50  """Config for MakeCoaddTempExpTask
51  """
52  warpAndPsfMatch = pexConfig.ConfigurableField(
53  target=WarpAndPsfMatchTask,
54  doc="Task to warp and PSF-match calexp",
55  )
56  doWrite = pexConfig.Field(
57  doc="persist <coaddName>Coadd_<warpType>Warp",
58  dtype=bool,
59  default=True,
60  )
61  bgSubtracted = pexConfig.Field(
62  doc="Work with a background subtracted calexp?",
63  dtype=bool,
64  default=True,
65  )
66  coaddPsf = pexConfig.ConfigField(
67  doc="Configuration for CoaddPsf",
68  dtype=CoaddPsfConfig,
69  )
70  makeDirect = pexConfig.Field(
71  doc="Make direct Warp/Coadds",
72  dtype=bool,
73  default=True,
74  )
75  makePsfMatched = pexConfig.Field(
76  doc="Make Psf-Matched Warp/Coadd?",
77  dtype=bool,
78  default=False,
79  )
80 
81  doWriteEmptyWarps = pexConfig.Field(
82  dtype=bool,
83  default=False,
84  doc="Write out warps even if they are empty"
85  )
86 
87  hasFakes = pexConfig.Field(
88  doc="Should be set to True if fake sources have been inserted into the input data.",
89  dtype=bool,
90  default=False,
91  )
92  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
93 
94  def validate(self):
95  CoaddBaseTask.ConfigClass.validate(self)
96  if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
97  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
98  if self.doPsfMatch:
99  # Backwards compatibility.
100  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
101  self.makePsfMatchedmakePsfMatched = True
102  self.makeDirectmakeDirect = False
103 
104  def setDefaults(self):
105  CoaddBaseTask.ConfigClass.setDefaults(self)
106  self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
107 
108 
114 
115 
117  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
118 
119  @anchor MakeCoaddTempExpTask_
120 
121  @section pipe_tasks_makeCoaddTempExp_Contents Contents
122 
123  - @ref pipe_tasks_makeCoaddTempExp_Purpose
124  - @ref pipe_tasks_makeCoaddTempExp_Initialize
125  - @ref pipe_tasks_makeCoaddTempExp_IO
126  - @ref pipe_tasks_makeCoaddTempExp_Config
127  - @ref pipe_tasks_makeCoaddTempExp_Debug
128  - @ref pipe_tasks_makeCoaddTempExp_Example
129 
130  @section pipe_tasks_makeCoaddTempExp_Purpose Description
131 
132  Warp and optionally PSF-Match calexps onto a common projection, by
133  performing the following operations:
134  - Group calexps by visit/run
135  - For each visit, generate a Warp by calling method @ref makeTempExp.
136  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
137  on each visit
138 
139  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
140 
141  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
142 
143  @copydoc \_\_init\_\_
144 
145  This task has one special keyword argument: passing reuse=True will cause
146  the task to skip the creation of warps that are already present in the
147  output repositories.
148 
149  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
150 
151  This task is primarily designed to be run from the command line.
152 
153  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
154  to process.
155 
156  @copydoc run
157 
158  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
159  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
160  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
161  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
162 
163  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
164 
165  See @ref MakeCoaddTempExpConfig and parameters inherited from
166  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
167 
168  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
169 
170  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
171  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
172  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
173  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
174  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
175  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
176 
177  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
178  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
179  In general, for a set of input images, this config should equal the FWHM of the visit
180  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
181  of the other config options offer a reasonable starting point.
182  The following list presents the most common problems that arise from a misconfigured
183  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
184  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
185  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
186  Problem: Explanation. *Solution*
187 
188  *Troublshooting PSF-Matching Configuration:*
189  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
190  For example:_
191 
192  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
193 
194  Note that increasing the kernel size also increases runtime.
195  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
196  for matching kernel. _Provide the matcher with more data by either increasing
197  the spatial sampling by decreasing the spatial cell size,_
198 
199  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
200  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
201 
202  _or increasing the padding around the Science PSF, for example:_
203 
204  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
205 
206  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
207  matching kernel dimensions, thus increasing the number of pixels available to fit
208  after convolving the PSF with the matching kernel.
209  Optionally, for debugging the effects of padding, the level of padding may be manually
210  controlled by setting turning off the automatic padding and setting the number
211  of pixels by which to pad the PSF:
212 
213  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
214  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
215 
216  - Deconvolution: Matching a large PSF to a smaller PSF produces
217  a telltale noise pattern which looks like ripples or a brain.
218  _Increase the size of the requested model PSF. For example:_
219 
220  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
221 
222  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
223  _Increase the width of the Gaussian basis functions. For example:_
224 
225  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
226  # from default [0.7, 1.5, 3.0]
227 
228 
229  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
230 
231  MakeCoaddTempExpTask has no debug output, but its subtasks do.
232 
233  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
234 
235  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
236  into the larger Data Release Processing.
237  Set up by running:
238 
239  setup ci_hsc
240  cd $CI_HSC_DIR
241  # if not built already:
242  python $(which scons) # this will take a while
243 
244  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
245  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
246  output respository with the desired SkyMap. The command,
247 
248  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
249  --id patch=5,4 tract=0 filter=HSC-I \
250  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
251  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
252  --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
253  makePsfMatched=True modelPsf.defaultFwhm=11
254 
255  writes a direct and PSF-Matched Warp to
256  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
257  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
258  respectively.
259 
260  @note PSF-Matching in this particular dataset would benefit from adding
261  `--configfile ./matchingConfig.py` to
262  the command line arguments where `matchingConfig.py` is defined by:
263 
264  echo "
265  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
266  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
267 
268 
269  Add the option `--help` to see more options.
270  """
271  ConfigClass = MakeCoaddTempExpConfig
272  _DefaultName = "makeCoaddTempExp"
273 
274  def __init__(self, reuse=False, **kwargs):
275  CoaddBaseTask.__init__(self, **kwargs)
276  self.reusereuse = reuse
277  self.makeSubtask("warpAndPsfMatch")
278  if self.config.hasFakes:
279  self.calexpTypecalexpType = "fakes_calexp"
280  else:
281  self.calexpTypecalexpType = "calexp"
282 
283  @pipeBase.timeMethod
284  def runDataRef(self, patchRef, selectDataList=[]):
285  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
286 
287  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
288  plus the camera-specific filter key (e.g. "filter" or "band")
289  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
290  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
291  warps are requested.
292 
293  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
294 
295  @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
296  with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
297  (assembleCoadd should determine zeropoint scaling without referring to it).
298  """
299  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
300 
301  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
302  if self.config.makePsfMatched and not self.config.makeDirect:
303  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
304  else:
305  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
306 
307  calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
308 
309  if len(calExpRefList) == 0:
310  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
311  return None
312  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
313  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
314  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
315 
316  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
317  primaryWarpDataset)
318  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
319 
320  dataRefList = []
321  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
322  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
323  tempExpTuple, groupData.keys)
324  if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
325  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
326  dataRefList.append(tempExpRef)
327  continue
328  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
329 
330  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
331  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
332  # of the visit in the list.
333  try:
334  visitId = int(tempExpRef.dataId["visit"])
335  except (KeyError, ValueError):
336  visitId = i
337 
338  calExpList = []
339  ccdIdList = []
340  dataIdList = []
341 
342  for calExpInd, calExpRef in enumerate(calexpRefList):
343  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
344  calExpRef.dataId)
345  try:
346  ccdId = calExpRef.get("ccdExposureId", immediate=True)
347  except Exception:
348  ccdId = calExpInd
349  try:
350  # We augment the dataRef here with the tract, which is harmless for loading things
351  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
352  # which do.
353  calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
354  dataId=calExpRef.dataId,
355  tract=skyInfo.tractInfo.getId())
356  calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
357  except Exception as e:
358  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
359  continue
360 
361  if self.config.doApplySkyCorr:
362  self.applySkyCorrapplySkyCorr(calExpRef, calExp)
363 
364  calExpList.append(calExp)
365  ccdIdList.append(ccdId)
366  dataIdList.append(calExpRef.dataId)
367 
368  exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
369 
370  if any(exps.values()):
371  dataRefList.append(tempExpRef)
372  else:
373  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
374 
375  if self.config.doWrite:
376  for (warpType, exposure) in exps.items(): # compatible w/ Py3
377  if exposure is not None:
378  self.log.info("Persisting %s" % self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
379  tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
380 
381  return dataRefList
382 
383  @pipeBase.timeMethod
384  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None):
385  """Create a Warp from inputs
386 
387  We iterate over the multiple calexps in a single exposure to construct
388  the warp (previously called a coaddTempExp) of that exposure to the
389  supplied tract/patch.
390 
391  Pixels that receive no pixels are set to NAN; this is not correct
392  (violates LSST algorithms group policy), but will be fixed up by
393  interpolating after the coaddition.
394 
395  @param calexpRefList: List of data references for calexps that (may)
396  overlap the patch of interest
397  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
398  information about the patch
399  @param visitId: integer identifier for visit, for the table that will
400  produce the CoaddPsf
401  @return a pipeBase Struct containing:
402  - exposures: a dictionary containing the warps requested:
403  "direct": direct warp if config.makeDirect
404  "psfMatched": PSF-matched warp if config.makePsfMatched
405  """
406  warpTypeList = self.getWarpTypeListgetWarpTypeList()
407 
408  totGoodPix = {warpType: 0 for warpType in warpTypeList}
409  didSetMetadata = {warpType: False for warpType in warpTypeList}
410  coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
411  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
412  for warpType in warpTypeList}
413 
414  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
415  if dataIdList is None:
416  dataIdList = ccdIdList
417 
418  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
419  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
420  calExpInd+1, len(calExpList), dataId)
421 
422  try:
423  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
424  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
425  makeDirect=self.config.makeDirect,
426  makePsfMatched=self.config.makePsfMatched)
427  except Exception as e:
428  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
429  continue
430  try:
431  numGoodPix = {warpType: 0 for warpType in warpTypeList}
432  for warpType in warpTypeList:
433  exposure = warpedAndMatched.getDict()[warpType]
434  if exposure is None:
435  continue
436  coaddTempExp = coaddTempExps[warpType]
437  if didSetMetadata[warpType]:
438  mimg = exposure.getMaskedImage()
439  mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
440  / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
441  del mimg
442  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
443  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
444  totGoodPix[warpType] += numGoodPix[warpType]
445  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
446  dataId, numGoodPix[warpType],
447  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
448  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
449  coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
450  coaddTempExp.setFilterLabel(exposure.getFilterLabel())
451  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
452  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
453  coaddTempExp.setPsf(exposure.getPsf())
454  didSetMetadata[warpType] = True
455 
456  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
457  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
458 
459  except Exception as e:
460  self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
461  continue
462 
463  for warpType in warpTypeList:
464  self.log.info("%sWarp has %d good pixels (%.1f%%)",
465  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
466 
467  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
468  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
469  if warpType == "direct":
470  coaddTempExps[warpType].setPsf(
471  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
472  self.config.coaddPsf.makeControl()))
473  else:
474  if not self.config.doWriteEmptyWarps:
475  # No good pixels. Exposure still empty
476  coaddTempExps[warpType] = None
477 
478  result = pipeBase.Struct(exposures=coaddTempExps)
479  return result
480 
481  def getCalibratedExposure(self, dataRef, bgSubtracted):
482  """Return one calibrated Exposure, possibly with an updated SkyWcs.
483 
484  @param[in] dataRef a sensor-level data reference
485  @param[in] bgSubtracted return calexp with background subtracted? If False get the
486  calexp's background background model and add it to the calexp.
487  @return calibrated exposure
488 
489  @raises MissingExposureError If data for the exposure is not available.
490 
491  If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
492  (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
493  `name_photoCalib` dataset. Otherwise, the photometric calibration is
494  retrieved from the processed exposure. When
495  `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
496  is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
497  Otherwise, the astrometric calibration is taken from the processed
498  exposure.
499  """
500  try:
501  exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
502  except dafPersist.NoResults as e:
503  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
504 
505  if not bgSubtracted:
506  background = dataRef.get("calexpBackground", immediate=True)
507  mi = exposure.getMaskedImage()
508  mi += background.getImage()
509  del mi
510 
511  if self.config.doApplyExternalPhotoCalib:
512  source = f"{self.config.externalPhotoCalibName}_photoCalib"
513  self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
514  photoCalib = dataRef.get(source)
515  exposure.setPhotoCalib(photoCalib)
516  else:
517  photoCalib = exposure.getPhotoCalib()
518 
519  if self.config.doApplyExternalSkyWcs:
520  source = f"{self.config.externalSkyWcsName}_wcs"
521  self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
522  skyWcs = dataRef.get(source)
523  exposure.setWcs(skyWcs)
524 
525  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
526  includeScaleUncertainty=self.config.includeCalibVar)
527  exposure.maskedImage /= photoCalib.getCalibrationMean()
528  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
529  # exposure.setCalib(afwImage.Calib(1.0))
530  return exposure
531 
532  @staticmethod
533  def _prepareEmptyExposure(skyInfo):
534  """Produce an empty exposure for a given patch"""
535  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
536  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
537  .getPlaneBitMask("NO_DATA"), numpy.inf)
538  return exp
539 
540  def getWarpTypeList(self):
541  """Return list of requested warp types per the config.
542  """
543  warpTypeList = []
544  if self.config.makeDirect:
545  warpTypeList.append("direct")
546  if self.config.makePsfMatched:
547  warpTypeList.append("psfMatched")
548  return warpTypeList
549 
550  def applySkyCorr(self, dataRef, calexp):
551  """Apply correction to the sky background level
552 
553  Sky corrections can be generated with the 'skyCorrection.py'
554  executable in pipe_drivers. Because the sky model used by that
555  code extends over the entire focal plane, this can produce
556  better sky subtraction.
557 
558  The calexp is updated in-place.
559 
560  Parameters
561  ----------
562  dataRef : `lsst.daf.persistence.ButlerDataRef`
563  Data reference for calexp.
564  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
565  Calibrated exposure.
566  """
567  bg = dataRef.get("skyCorr")
568  self.log.debug("Applying sky correction to %s", dataRef.dataId)
569  if isinstance(calexp, afwImage.Exposure):
570  calexp = calexp.getMaskedImage()
571  calexp -= bg.getImage()
572 
573 
574 class MakeWarpConnections(pipeBase.PipelineTaskConnections,
575  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
576  defaultTemplates={"coaddName": "deep"}):
577  calExpList = cT.Input(
578  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
579  name="calexp",
580  storageClass="ExposureF",
581  dimensions=("instrument", "visit", "detector"),
582  multiple=True,
583  )
584  backgroundList = cT.Input(
585  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
586  name="calexpBackground",
587  storageClass="Background",
588  dimensions=("instrument", "visit", "detector"),
589  multiple=True,
590  )
591  skyCorrList = cT.Input(
592  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
593  name="skyCorr",
594  storageClass="Background",
595  dimensions=("instrument", "visit", "detector"),
596  multiple=True,
597  )
598  skyMap = cT.Input(
599  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
600  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
601  storageClass="SkyMap",
602  dimensions=("skymap",),
603  )
604  direct = cT.Output(
605  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
606  "calexps onto the skyMap patch geometry."),
607  name="{coaddName}Coadd_directWarp",
608  storageClass="ExposureF",
609  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
610  )
611  psfMatched = cT.Output(
612  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
613  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
614  name="{coaddName}Coadd_psfMatchedWarp",
615  storageClass="ExposureF",
616  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
617  )
618 
619  def __init__(self, *, config=None):
620  super().__init__(config=config)
621  if config.bgSubtracted:
622  self.inputs.remove("backgroundList")
623  if not config.doApplySkyCorr:
624  self.inputs.remove("skyCorrList")
625  if not config.makeDirect:
626  self.outputs.remove("direct")
627  if not config.makePsfMatched:
628  self.outputs.remove("psfMatched")
629 
630 
631 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
632  pipelineConnections=MakeWarpConnections):
633 
634  def validate(self):
635  super().validate()
636  # TODO: Remove this constraint after DM-17062
637  if self.doApplyExternalPhotoCalib:
638  raise RuntimeError("Gen3 MakeWarpTask cannot apply external PhotoCalib results. "
639  "Please set doApplyExternalPhotoCalib=False.")
640  if self.doApplyExternalSkyWcs:
641  raise RuntimeError("Gen3 MakeWarpTask cannot apply external SkyWcs results. "
642  "Please set doApplyExternalSkyWcs=False.")
643 
644 
645 class MakeWarpTask(MakeCoaddTempExpTask, pipeBase.PipelineTask):
646  """Warp and optionally PSF-Match calexps onto an a common projection
647 
648  First Draft of a Gen3 compatible MakeWarpTask which
649  currently does not handle doApplyExternalPhotoCalib=True or
650  doApplyExternalSkyWcs=True.
651  """
652  ConfigClass = MakeWarpConfig
653  _DefaultName = "makeWarp"
654 
655  @utils.inheritDoc(pipeBase.PipelineTask)
656  def runQuantum(self, butlerQC, inputRefs, outputRefs):
657  """
658  Notes
659  ----
660  Construct warps for requested warp type for single epoch
661 
662  PipelineTask (Gen3) entry point to warp and optionally PSF-match
663  calexps. This method is analogous to `runDataRef`.
664  """
665  # Read in all inputs.
666  inputs = butlerQC.get(inputRefs)
667 
668  # Construct skyInfo expected by `run`. We remove the SkyMap itself
669  # from the dictionary so we can pass it as kwargs later.
670  skyMap = inputs.pop("skyMap")
671  quantumDataId = butlerQC.quantum.dataId
672  skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
673 
674  # Construct list of input DataIds expected by `run`
675  dataIdList = [ref.dataId for ref in inputRefs.calExpList]
676 
677  # Construct list of packed integer IDs expected by `run`
678  ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
679 
680  # Extract integer visitId requested by `run`
681  visits = [dataId['visit'] for dataId in dataIdList]
682  assert(all(visits[0] == visit for visit in visits))
683  visitId = visits[0]
684 
685  self.prepareCalibratedExposures(**inputs)
686  results = self.run(**inputs, visitId=visitId, ccdIdList=ccdIdList, dataIdList=dataIdList,
687  skyInfo=skyInfo)
688  if self.config.makeDirect:
689  butlerQC.put(results.exposures["direct"], outputRefs.direct)
690  if self.config.makePsfMatched:
691  butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
692 
693  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None):
694  """Calibrate and add backgrounds to input calExpList in place
695 
696  TODO DM-17062: apply jointcal/meas_mosaic here
697 
698  Parameters
699  ----------
700  calExpList : `list` of `lsst.afw.image.Exposure`
701  Sequence of calexps to be modified in place
702  backgroundList : `list` of `lsst.afw.math.backgroundList`
703  Sequence of backgrounds to be added back in if bgSubtracted=False
704  skyCorrList : `list` of `lsst.afw.math.backgroundList`
705  Sequence of background corrections to be subtracted if doApplySkyCorr=True
706  """
707  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
708  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
709  for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
710  mi = calexp.maskedImage
711  if not self.config.bgSubtracted:
712  mi += background.getImage()
713  if self.config.doApplySkyCorr:
714  mi -= skyCorr.getImage()
Base class for coaddition.
Definition: coaddBase.py:131
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:194
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:144
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:180
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:164
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:229
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60