lsst.pipe.tasks  21.0.0-154-gbe4cc4e5+80d40069b5
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 import logging
24 
25 import lsst.pex.config as pexConfig
26 import lsst.daf.persistence as dafPersist
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as connectionTypes
31 import lsst.utils as utils
32 import lsst.geom
33 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34 from lsst.skymap import BaseSkyMap
35 from lsst.utils.timer import timeMethod
36 from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37 from .warpAndPsfMatch import WarpAndPsfMatchTask
38 from .coaddHelpers import groupPatchExposures, getGroupDataRef
39 from collections.abc import Iterable
40 
41 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
42 
43 log = logging.getLogger(__name__.partition(".")[2])
44 
45 
46 class MissingExposureError(Exception):
47  """Raised when data cannot be retrieved for an exposure.
48  When processing patches, sometimes one exposure is missing; this lets us
49  distinguish bewteen that case, and other errors.
50  """
51  pass
52 
53 
54 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
55  """Config for MakeCoaddTempExpTask
56  """
57  warpAndPsfMatch = pexConfig.ConfigurableField(
58  target=WarpAndPsfMatchTask,
59  doc="Task to warp and PSF-match calexp",
60  )
61  doWrite = pexConfig.Field(
62  doc="persist <coaddName>Coadd_<warpType>Warp",
63  dtype=bool,
64  default=True,
65  )
66  bgSubtracted = pexConfig.Field(
67  doc="Work with a background subtracted calexp?",
68  dtype=bool,
69  default=True,
70  )
71  coaddPsf = pexConfig.ConfigField(
72  doc="Configuration for CoaddPsf",
73  dtype=CoaddPsfConfig,
74  )
75  makeDirect = pexConfig.Field(
76  doc="Make direct Warp/Coadds",
77  dtype=bool,
78  default=True,
79  )
80  makePsfMatched = pexConfig.Field(
81  doc="Make Psf-Matched Warp/Coadd?",
82  dtype=bool,
83  default=False,
84  )
85 
86  doWriteEmptyWarps = pexConfig.Field(
87  dtype=bool,
88  default=False,
89  doc="Write out warps even if they are empty"
90  )
91 
92  hasFakes = pexConfig.Field(
93  doc="Should be set to True if fake sources have been inserted into the input data.",
94  dtype=bool,
95  default=False,
96  )
97  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
98 
99  def validate(self):
100  CoaddBaseTask.ConfigClass.validate(self)
101  if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
102  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
103  if self.doPsfMatch:
104  # Backwards compatibility.
105  log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
106  self.makePsfMatchedmakePsfMatched = True
107  self.makeDirectmakeDirect = False
108 
109  def setDefaults(self):
110  CoaddBaseTask.ConfigClass.setDefaults(self)
111  self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
112 
113 
119 
120 
122  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
123 
124  @anchor MakeCoaddTempExpTask_
125 
126  @section pipe_tasks_makeCoaddTempExp_Contents Contents
127 
128  - @ref pipe_tasks_makeCoaddTempExp_Purpose
129  - @ref pipe_tasks_makeCoaddTempExp_Initialize
130  - @ref pipe_tasks_makeCoaddTempExp_IO
131  - @ref pipe_tasks_makeCoaddTempExp_Config
132  - @ref pipe_tasks_makeCoaddTempExp_Debug
133  - @ref pipe_tasks_makeCoaddTempExp_Example
134 
135  @section pipe_tasks_makeCoaddTempExp_Purpose Description
136 
137  Warp and optionally PSF-Match calexps onto a common projection, by
138  performing the following operations:
139  - Group calexps by visit/run
140  - For each visit, generate a Warp by calling method @ref makeTempExp.
141  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
142  on each visit
143 
144  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
145 
146  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
147 
148  @copydoc \_\_init\_\_
149 
150  This task has one special keyword argument: passing reuse=True will cause
151  the task to skip the creation of warps that are already present in the
152  output repositories.
153 
154  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
155 
156  This task is primarily designed to be run from the command line.
157 
158  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
159  to process.
160 
161  @copydoc run
162 
163  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
164  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
165  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
166  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
167 
168  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
169 
170  See @ref MakeCoaddTempExpConfig and parameters inherited from
171  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
172 
173  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
174 
175  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
176  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
177  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
178  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
179  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
180  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
181 
182  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
183  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
184  In general, for a set of input images, this config should equal the FWHM of the visit
185  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
186  of the other config options offer a reasonable starting point.
187  The following list presents the most common problems that arise from a misconfigured
188  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
189  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
190  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
191  Problem: Explanation. *Solution*
192 
193  *Troublshooting PSF-Matching Configuration:*
194  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
195  For example:_
196 
197  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
198 
199  Note that increasing the kernel size also increases runtime.
200  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
201  for matching kernel. _Provide the matcher with more data by either increasing
202  the spatial sampling by decreasing the spatial cell size,_
203 
204  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
205  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
206 
207  _or increasing the padding around the Science PSF, for example:_
208 
209  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
210 
211  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
212  matching kernel dimensions, thus increasing the number of pixels available to fit
213  after convolving the PSF with the matching kernel.
214  Optionally, for debugging the effects of padding, the level of padding may be manually
215  controlled by setting turning off the automatic padding and setting the number
216  of pixels by which to pad the PSF:
217 
218  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
219  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
220 
221  - Deconvolution: Matching a large PSF to a smaller PSF produces
222  a telltale noise pattern which looks like ripples or a brain.
223  _Increase the size of the requested model PSF. For example:_
224 
225  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
226 
227  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
228  _Increase the width of the Gaussian basis functions. For example:_
229 
230  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
231  # from default [0.7, 1.5, 3.0]
232 
233 
234  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
235 
236  MakeCoaddTempExpTask has no debug output, but its subtasks do.
237 
238  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
239 
240  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
241  into the larger Data Release Processing.
242  Set up by running:
243 
244  setup ci_hsc
245  cd $CI_HSC_DIR
246  # if not built already:
247  python $(which scons) # this will take a while
248 
249  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
250  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
251  output respository with the desired SkyMap. The command,
252 
253  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
254  --id patch=5,4 tract=0 filter=HSC-I \
255  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
256  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
257  --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
258  makePsfMatched=True modelPsf.defaultFwhm=11
259 
260  writes a direct and PSF-Matched Warp to
261  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
262  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
263  respectively.
264 
265  @note PSF-Matching in this particular dataset would benefit from adding
266  `--configfile ./matchingConfig.py` to
267  the command line arguments where `matchingConfig.py` is defined by:
268 
269  echo "
270  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
271  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
272 
273 
274  Add the option `--help` to see more options.
275  """
276  ConfigClass = MakeCoaddTempExpConfig
277  _DefaultName = "makeCoaddTempExp"
278 
279  def __init__(self, reuse=False, **kwargs):
280  CoaddBaseTask.__init__(self, **kwargs)
281  self.reusereuse = reuse
282  self.makeSubtask("warpAndPsfMatch")
283  if self.config.hasFakes:
284  self.calexpTypecalexpType = "fakes_calexp"
285  else:
286  self.calexpTypecalexpType = "calexp"
287 
288  @timeMethod
289  def runDataRef(self, patchRef, selectDataList=[]):
290  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
291 
292  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
293  plus the camera-specific filter key (e.g. "filter" or "band")
294  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
295  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
296  warps are requested.
297 
298  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
299 
300  @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
301  with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
302  (assembleCoadd should determine zeropoint scaling without referring to it).
303  """
304  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
305 
306  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
307  if self.config.makePsfMatched and not self.config.makeDirect:
308  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
309  else:
310  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
311 
312  calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
313 
314  if len(calExpRefList) == 0:
315  self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
316  return None
317  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
318  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
319  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
320 
321  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
322  primaryWarpDataset)
323  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
324 
325  dataRefList = []
326  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
327  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
328  tempExpTuple, groupData.keys)
329  if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
330  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
331  dataRefList.append(tempExpRef)
332  continue
333  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
334 
335  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
336  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
337  # of the visit in the list.
338  try:
339  visitId = int(tempExpRef.dataId["visit"])
340  except (KeyError, ValueError):
341  visitId = i
342 
343  calExpList = []
344  ccdIdList = []
345  dataIdList = []
346 
347  for calExpInd, calExpRef in enumerate(calexpRefList):
348  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
349  calExpRef.dataId)
350  try:
351  ccdId = calExpRef.get("ccdExposureId", immediate=True)
352  except Exception:
353  ccdId = calExpInd
354  try:
355  # We augment the dataRef here with the tract, which is harmless for loading things
356  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
357  # which do.
358  calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
359  dataId=calExpRef.dataId,
360  tract=skyInfo.tractInfo.getId())
361  calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
362  except Exception as e:
363  self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
364  continue
365 
366  if self.config.doApplySkyCorr:
367  self.applySkyCorrapplySkyCorr(calExpRef, calExp)
368 
369  calExpList.append(calExp)
370  ccdIdList.append(ccdId)
371  dataIdList.append(calExpRef.dataId)
372 
373  exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
374 
375  if any(exps.values()):
376  dataRefList.append(tempExpRef)
377  else:
378  self.log.warning("Warp %s could not be created", tempExpRef.dataId)
379 
380  if self.config.doWrite:
381  for (warpType, exposure) in exps.items(): # compatible w/ Py3
382  if exposure is not None:
383  self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
384  tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
385 
386  return dataRefList
387 
388  @timeMethod
389  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
390  """Create a Warp from inputs
391 
392  We iterate over the multiple calexps in a single exposure to construct
393  the warp (previously called a coaddTempExp) of that exposure to the
394  supplied tract/patch.
395 
396  Pixels that receive no pixels are set to NAN; this is not correct
397  (violates LSST algorithms group policy), but will be fixed up by
398  interpolating after the coaddition.
399 
400  @param calexpRefList: List of data references for calexps that (may)
401  overlap the patch of interest
402  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
403  information about the patch
404  @param visitId: integer identifier for visit, for the table that will
405  produce the CoaddPsf
406  @return a pipeBase Struct containing:
407  - exposures: a dictionary containing the warps requested:
408  "direct": direct warp if config.makeDirect
409  "psfMatched": PSF-matched warp if config.makePsfMatched
410  """
411  warpTypeList = self.getWarpTypeListgetWarpTypeList()
412 
413  totGoodPix = {warpType: 0 for warpType in warpTypeList}
414  didSetMetadata = {warpType: False for warpType in warpTypeList}
415  coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
416  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
417  for warpType in warpTypeList}
418 
419  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
420  if dataIdList is None:
421  dataIdList = ccdIdList
422 
423  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
424  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
425  calExpInd+1, len(calExpList), dataId)
426 
427  try:
428  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
429  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
430  makeDirect=self.config.makeDirect,
431  makePsfMatched=self.config.makePsfMatched)
432  except Exception as e:
433  self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
434  continue
435  try:
436  numGoodPix = {warpType: 0 for warpType in warpTypeList}
437  for warpType in warpTypeList:
438  exposure = warpedAndMatched.getDict()[warpType]
439  if exposure is None:
440  continue
441  coaddTempExp = coaddTempExps[warpType]
442  if didSetMetadata[warpType]:
443  mimg = exposure.getMaskedImage()
444  mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
445  / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
446  del mimg
447  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
448  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
449  totGoodPix[warpType] += numGoodPix[warpType]
450  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
451  dataId, numGoodPix[warpType],
452  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
453  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
454  coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
455  coaddTempExp.setFilterLabel(exposure.getFilterLabel())
456  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
457  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
458  coaddTempExp.setPsf(exposure.getPsf())
459  didSetMetadata[warpType] = True
460 
461  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
462  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
463 
464  except Exception as e:
465  self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
466  continue
467 
468  for warpType in warpTypeList:
469  self.log.info("%sWarp has %d good pixels (%.1f%%)",
470  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
471 
472  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
473  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
474  if warpType == "direct":
475  coaddTempExps[warpType].setPsf(
476  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
477  self.config.coaddPsf.makeControl()))
478  else:
479  if not self.config.doWriteEmptyWarps:
480  # No good pixels. Exposure still empty
481  coaddTempExps[warpType] = None
482  # NoWorkFound is unnecessary as the downstream tasks will
483  # adjust the quantum accordingly, and it prevents gen2
484  # MakeCoaddTempExp from continuing to loop over visits.
485 
486  result = pipeBase.Struct(exposures=coaddTempExps)
487  return result
488 
489  def getCalibratedExposure(self, dataRef, bgSubtracted):
490  """Return one calibrated Exposure, possibly with an updated SkyWcs.
491 
492  @param[in] dataRef a sensor-level data reference
493  @param[in] bgSubtracted return calexp with background subtracted? If False get the
494  calexp's background background model and add it to the calexp.
495  @return calibrated exposure
496 
497  @raises MissingExposureError If data for the exposure is not available.
498 
499  If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
500  (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
501  `name_photoCalib` dataset. Otherwise, the photometric calibration is
502  retrieved from the processed exposure. When
503  `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
504  is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
505  Otherwise, the astrometric calibration is taken from the processed
506  exposure.
507  """
508  try:
509  exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
510  except dafPersist.NoResults as e:
511  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
512 
513  if not bgSubtracted:
514  background = dataRef.get("calexpBackground", immediate=True)
515  mi = exposure.getMaskedImage()
516  mi += background.getImage()
517  del mi
518 
519  if self.config.doApplyExternalPhotoCalib:
520  source = f"{self.config.externalPhotoCalibName}_photoCalib"
521  self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
522  photoCalib = dataRef.get(source)
523  exposure.setPhotoCalib(photoCalib)
524  else:
525  photoCalib = exposure.getPhotoCalib()
526 
527  if self.config.doApplyExternalSkyWcs:
528  source = f"{self.config.externalSkyWcsName}_wcs"
529  self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
530  skyWcs = dataRef.get(source)
531  exposure.setWcs(skyWcs)
532 
533  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
534  includeScaleUncertainty=self.config.includeCalibVar)
535  exposure.maskedImage /= photoCalib.getCalibrationMean()
536  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
537  # exposure.setCalib(afwImage.Calib(1.0))
538  return exposure
539 
540  @staticmethod
541  def _prepareEmptyExposure(skyInfo):
542  """Produce an empty exposure for a given patch"""
543  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
544  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
545  .getPlaneBitMask("NO_DATA"), numpy.inf)
546  return exp
547 
548  def getWarpTypeList(self):
549  """Return list of requested warp types per the config.
550  """
551  warpTypeList = []
552  if self.config.makeDirect:
553  warpTypeList.append("direct")
554  if self.config.makePsfMatched:
555  warpTypeList.append("psfMatched")
556  return warpTypeList
557 
558  def applySkyCorr(self, dataRef, calexp):
559  """Apply correction to the sky background level
560 
561  Sky corrections can be generated with the 'skyCorrection.py'
562  executable in pipe_drivers. Because the sky model used by that
563  code extends over the entire focal plane, this can produce
564  better sky subtraction.
565 
566  The calexp is updated in-place.
567 
568  Parameters
569  ----------
570  dataRef : `lsst.daf.persistence.ButlerDataRef`
571  Data reference for calexp.
572  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
573  Calibrated exposure.
574  """
575  bg = dataRef.get("skyCorr")
576  self.log.debug("Applying sky correction to %s", dataRef.dataId)
577  if isinstance(calexp, afwImage.Exposure):
578  calexp = calexp.getMaskedImage()
579  calexp -= bg.getImage()
580 
581 
582 class MakeWarpConnections(pipeBase.PipelineTaskConnections,
583  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
584  defaultTemplates={"coaddName": "deep",
585  "skyWcsName": "jointcal",
586  "photoCalibName": "fgcm",
587  "calexpType": ""}):
588  calExpList = connectionTypes.Input(
589  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
590  name="{calexpType}calexp",
591  storageClass="ExposureF",
592  dimensions=("instrument", "visit", "detector"),
593  multiple=True,
594  deferLoad=True,
595  )
596  backgroundList = connectionTypes.Input(
597  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
598  name="calexpBackground",
599  storageClass="Background",
600  dimensions=("instrument", "visit", "detector"),
601  multiple=True,
602  )
603  skyCorrList = connectionTypes.Input(
604  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
605  name="skyCorr",
606  storageClass="Background",
607  dimensions=("instrument", "visit", "detector"),
608  multiple=True,
609  )
610  skyMap = connectionTypes.Input(
611  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
612  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
613  storageClass="SkyMap",
614  dimensions=("skymap",),
615  )
616  externalSkyWcsTractCatalog = connectionTypes.Input(
617  doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
618  "id for the catalog id, sorted on id for fast lookup."),
619  name="{skyWcsName}SkyWcsCatalog",
620  storageClass="ExposureCatalog",
621  dimensions=("instrument", "visit", "tract"),
622  )
623  externalSkyWcsGlobalCatalog = connectionTypes.Input(
624  doc=("Per-visit wcs calibrations computed globally (with no tract information). "
625  "These catalogs use the detector id for the catalog id, sorted on id for "
626  "fast lookup."),
627  name="{skyWcsName}SkyWcsCatalog",
628  storageClass="ExposureCatalog",
629  dimensions=("instrument", "visit"),
630  )
631  externalPhotoCalibTractCatalog = connectionTypes.Input(
632  doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
633  "detector id for the catalog id, sorted on id for fast lookup."),
634  name="{photoCalibName}PhotoCalibCatalog",
635  storageClass="ExposureCatalog",
636  dimensions=("instrument", "visit", "tract"),
637  )
638  externalPhotoCalibGlobalCatalog = connectionTypes.Input(
639  doc=("Per-visit photometric calibrations computed globally (with no tract "
640  "information). These catalogs use the detector id for the catalog id, "
641  "sorted on id for fast lookup."),
642  name="{photoCalibName}PhotoCalibCatalog",
643  storageClass="ExposureCatalog",
644  dimensions=("instrument", "visit"),
645  )
646  direct = connectionTypes.Output(
647  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
648  "calexps onto the skyMap patch geometry."),
649  name="{coaddName}Coadd_directWarp",
650  storageClass="ExposureF",
651  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
652  )
653  psfMatched = connectionTypes.Output(
654  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
655  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
656  name="{coaddName}Coadd_psfMatchedWarp",
657  storageClass="ExposureF",
658  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
659  )
660  # TODO DM-28769, have selectImages subtask indicate which connections they need:
661  wcsList = connectionTypes.Input(
662  doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
663  name="{calexpType}calexp.wcs",
664  storageClass="Wcs",
665  dimensions=("instrument", "visit", "detector"),
666  multiple=True,
667  )
668  bboxList = connectionTypes.Input(
669  doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
670  name="{calexpType}calexp.bbox",
671  storageClass="Box2I",
672  dimensions=("instrument", "visit", "detector"),
673  multiple=True,
674  )
675  srcList = connectionTypes.Input(
676  doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
677  name="src",
678  storageClass="SourceCatalog",
679  dimensions=("instrument", "visit", "detector"),
680  multiple=True,
681  )
682  psfList = connectionTypes.Input(
683  doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
684  name="{calexpType}calexp.psf",
685  storageClass="Psf",
686  dimensions=("instrument", "visit", "detector"),
687  multiple=True,
688  )
689 
690  def __init__(self, *, config=None):
691  super().__init__(config=config)
692  if config.bgSubtracted:
693  self.inputs.remove("backgroundList")
694  if not config.doApplySkyCorr:
695  self.inputs.remove("skyCorrList")
696  if config.doApplyExternalSkyWcs:
697  if config.useGlobalExternalSkyWcs:
698  self.inputs.remove("externalSkyWcsTractCatalog")
699  else:
700  self.inputs.remove("externalSkyWcsGlobalCatalog")
701  else:
702  self.inputs.remove("externalSkyWcsTractCatalog")
703  self.inputs.remove("externalSkyWcsGlobalCatalog")
704  if config.doApplyExternalPhotoCalib:
705  if config.useGlobalExternalPhotoCalib:
706  self.inputs.remove("externalPhotoCalibTractCatalog")
707  else:
708  self.inputs.remove("externalPhotoCalibGlobalCatalog")
709  else:
710  self.inputs.remove("externalPhotoCalibTractCatalog")
711  self.inputs.remove("externalPhotoCalibGlobalCatalog")
712  if not config.makeDirect:
713  self.outputs.remove("direct")
714  if not config.makePsfMatched:
715  self.outputs.remove("psfMatched")
716  # TODO DM-28769: add connection per selectImages connections
717  # instead of removing if not PsfWcsSelectImagesTask here:
718  if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
719  self.inputs.remove("srcList")
720  if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
721  self.inputs.remove("psfList")
722 
723 
724 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
725  pipelineConnections=MakeWarpConnections):
726 
727  def validate(self):
728  super().validate()
729 
730 
731 class MakeWarpTask(MakeCoaddTempExpTask):
732  """Warp and optionally PSF-Match calexps onto an a common projection
733  """
734  ConfigClass = MakeWarpConfig
735  _DefaultName = "makeWarp"
736 
737  @utils.inheritDoc(pipeBase.PipelineTask)
738  def runQuantum(self, butlerQC, inputRefs, outputRefs):
739  """
740  Notes
741  ----
742  Construct warps for requested warp type for single epoch
743 
744  PipelineTask (Gen3) entry point to warp and optionally PSF-match
745  calexps. This method is analogous to `runDataRef`.
746  """
747 
748  # Ensure all input lists are in same detector order as the calExpList
749  detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
750  inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
751 
752  # Read in all inputs.
753  inputs = butlerQC.get(inputRefs)
754 
755  # Construct skyInfo expected by `run`. We remove the SkyMap itself
756  # from the dictionary so we can pass it as kwargs later.
757  skyMap = inputs.pop("skyMap")
758  quantumDataId = butlerQC.quantum.dataId
759  skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
760 
761  # Construct list of input DataIds expected by `run`
762  dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
763  # Construct list of packed integer IDs expected by `run`
764  ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
765 
766  # Run the selector and filter out calexps that were not selected
767  # primarily because they do not overlap the patch
768  cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
769  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
770  goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
771  inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
772 
773  # Read from disk only the selected calexps
774  inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
775 
776  # Extract integer visitId requested by `run`
777  visits = [dataId['visit'] for dataId in dataIdList]
778  visitId = visits[0]
779 
780  if self.config.doApplyExternalSkyWcs:
781  if self.config.useGlobalExternalSkyWcs:
782  externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
783  else:
784  externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
785  else:
786  externalSkyWcsCatalog = None
787 
788  if self.config.doApplyExternalPhotoCalib:
789  if self.config.useGlobalExternalPhotoCalib:
790  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
791  else:
792  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
793  else:
794  externalPhotoCalibCatalog = None
795 
796  completeIndices = self.prepareCalibratedExposures(**inputs,
797  externalSkyWcsCatalog=externalSkyWcsCatalog,
798  externalPhotoCalibCatalog=externalPhotoCalibCatalog)
799  # Redo the input selection with inputs with complete wcs/photocalib info.
800  inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
801 
802  results = self.run(**inputs, visitId=visitId,
803  ccdIdList=[ccdIdList[i] for i in goodIndices],
804  dataIdList=[dataIdList[i] for i in goodIndices],
805  skyInfo=skyInfo)
806  if self.config.makeDirect and results.exposures["direct"] is not None:
807  butlerQC.put(results.exposures["direct"], outputRefs.direct)
808  if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
809  butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
810 
811  def filterInputs(self, indices, inputs):
812  """Return task inputs with their lists filtered by indices
813 
814  Parameters
815  ----------
816  indices : `list` of integers
817  inputs : `dict` of `list` of input connections to be passed to run
818  """
819  for key in inputs.keys():
820  # Only down-select on list inputs
821  if isinstance(inputs[key], list):
822  inputs[key] = [inputs[key][ind] for ind in indices]
823  return inputs
824 
825  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
826  externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
827  **kwargs):
828  """Calibrate and add backgrounds to input calExpList in place
829 
830  Parameters
831  ----------
832  calExpList : `list` of `lsst.afw.image.Exposure`
833  Sequence of calexps to be modified in place
834  backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
835  Sequence of backgrounds to be added back in if bgSubtracted=False
836  skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
837  Sequence of background corrections to be subtracted if doApplySkyCorr=True
838  externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
839  Exposure catalog with external skyWcs to be applied
840  if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
841  for the catalog id, sorted on id for fast lookup.
842  externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
843  Exposure catalog with external photoCalib to be applied
844  if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
845  id for the catalog id, sorted on id for fast lookup.
846 
847  Returns
848  -------
849  indices : `list` [`int`]
850  Indices of calExpList and friends that have valid photoCalib/skyWcs
851  """
852  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
853  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
854 
855  includeCalibVar = self.config.includeCalibVar
856 
857  indices = []
858  for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
859  backgroundList,
860  skyCorrList)):
861  mi = calexp.maskedImage
862  if not self.config.bgSubtracted:
863  mi += background.getImage()
864 
865  if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
866  detectorId = calexp.getInfo().getDetector().getId()
867 
868  # Find the external photoCalib
869  if externalPhotoCalibCatalog is not None:
870  row = externalPhotoCalibCatalog.find(detectorId)
871  if row is None:
872  self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
873  "and will not be used in the warp.", detectorId)
874  continue
875  photoCalib = row.getPhotoCalib()
876  if photoCalib is None:
877  self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
878  "and will not be used in the warp.", detectorId)
879  continue
880  calexp.setPhotoCalib(photoCalib)
881  else:
882  photoCalib = calexp.getPhotoCalib()
883  if photoCalib is None:
884  self.log.warning("Detector id %s has None for photoCalib in the calexp "
885  "and will not be used in the warp.", detectorId)
886  continue
887 
888  # Find and apply external skyWcs
889  if externalSkyWcsCatalog is not None:
890  row = externalSkyWcsCatalog.find(detectorId)
891  if row is None:
892  self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
893  "and will not be used in the warp.", detectorId)
894  continue
895  skyWcs = row.getWcs()
896  if skyWcs is None:
897  self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
898  "and will not be used in the warp.", detectorId)
899  continue
900  calexp.setWcs(skyWcs)
901  else:
902  skyWcs = calexp.getWcs()
903  if skyWcs is None:
904  self.log.warning("Detector id %s has None for skyWcs in the calexp "
905  "and will not be used in the warp.", detectorId)
906  continue
907 
908  # Calibrate the image
909  calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
910  includeScaleUncertainty=includeCalibVar)
911  calexp.maskedImage /= photoCalib.getCalibrationMean()
912  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
913  # exposure.setCalib(afwImage.Calib(1.0))
914 
915  # Apply skycorr
916  if self.config.doApplySkyCorr:
917  mi -= skyCorr.getImage()
918 
919  indices.append(index)
920 
921  return indices
922 
923 
924 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
925  """Reorder inputRefs per outputSortKeyOrder
926 
927  Any inputRefs which are lists will be resorted per specified key e.g.,
928  'detector.' Only iterables will be reordered, and values can be of type
929  `lsst.pipe.base.connections.DeferredDatasetRef` or
930  `lsst.daf.butler.core.datasets.ref.DatasetRef`.
931  Returned lists of refs have the same length as the outputSortKeyOrder.
932  If an outputSortKey not in the inputRef, then it will be padded with None.
933  If an inputRef contains an inputSortKey that is not in the
934  outputSortKeyOrder it will be removed.
935 
936  Parameters
937  ----------
938  inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
939  Input references to be reordered and padded.
940  outputSortKeyOrder : iterable
941  Iterable of values to be compared with inputRef's dataId[dataIdKey]
942  dataIdKey : `str`
943  dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
944 
945  Returns:
946  --------
947  inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
948  Quantized Connection with sorted DatasetRef values sorted if iterable.
949  """
950  for connectionName, refs in inputRefs:
951  if isinstance(refs, Iterable):
952  if hasattr(refs[0], "dataId"):
953  inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
954  else:
955  inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
956  if inputSortKeyOrder != outputSortKeyOrder:
957  setattr(inputRefs, connectionName,
958  reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
959  return inputRefs
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60