lsst.pipe.tasks  21.0.0-167-gb57ef11d+e02347d8c1
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 import logging
24 
25 import lsst.pex.config as pexConfig
26 import lsst.daf.persistence as dafPersist
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as connectionTypes
31 import lsst.utils as utils
32 import lsst.geom
33 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34 from lsst.skymap import BaseSkyMap
35 from lsst.utils.timer import timeMethod
36 from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37 from .warpAndPsfMatch import WarpAndPsfMatchTask
38 from .coaddHelpers import groupPatchExposures, getGroupDataRef
39 from collections.abc import Iterable
40 
41 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
42 
43 log = logging.getLogger(__name__.partition(".")[2])
44 
45 
46 class MissingExposureError(Exception):
47  """Raised when data cannot be retrieved for an exposure.
48  When processing patches, sometimes one exposure is missing; this lets us
49  distinguish bewteen that case, and other errors.
50  """
51  pass
52 
53 
54 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
55  """Config for MakeCoaddTempExpTask
56  """
57  warpAndPsfMatch = pexConfig.ConfigurableField(
58  target=WarpAndPsfMatchTask,
59  doc="Task to warp and PSF-match calexp",
60  )
61  doWrite = pexConfig.Field(
62  doc="persist <coaddName>Coadd_<warpType>Warp",
63  dtype=bool,
64  default=True,
65  )
66  bgSubtracted = pexConfig.Field(
67  doc="Work with a background subtracted calexp?",
68  dtype=bool,
69  default=True,
70  )
71  coaddPsf = pexConfig.ConfigField(
72  doc="Configuration for CoaddPsf",
73  dtype=CoaddPsfConfig,
74  )
75  makeDirect = pexConfig.Field(
76  doc="Make direct Warp/Coadds",
77  dtype=bool,
78  default=True,
79  )
80  makePsfMatched = pexConfig.Field(
81  doc="Make Psf-Matched Warp/Coadd?",
82  dtype=bool,
83  default=False,
84  )
85 
86  doWriteEmptyWarps = pexConfig.Field(
87  dtype=bool,
88  default=False,
89  doc="Write out warps even if they are empty"
90  )
91 
92  hasFakes = pexConfig.Field(
93  doc="Should be set to True if fake sources have been inserted into the input data.",
94  dtype=bool,
95  default=False,
96  )
97  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
98 
99  def validate(self):
100  CoaddBaseTask.ConfigClass.validate(self)
101  if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
102  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
103  if self.doPsfMatch:
104  # Backwards compatibility.
105  log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
106  self.makePsfMatchedmakePsfMatched = True
107  self.makeDirectmakeDirect = False
108 
109  def setDefaults(self):
110  CoaddBaseTask.ConfigClass.setDefaults(self)
111  self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
112 
113 
119 
120 
122  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
123 
124  @anchor MakeCoaddTempExpTask_
125 
126  @section pipe_tasks_makeCoaddTempExp_Contents Contents
127 
128  - @ref pipe_tasks_makeCoaddTempExp_Purpose
129  - @ref pipe_tasks_makeCoaddTempExp_Initialize
130  - @ref pipe_tasks_makeCoaddTempExp_IO
131  - @ref pipe_tasks_makeCoaddTempExp_Config
132  - @ref pipe_tasks_makeCoaddTempExp_Debug
133  - @ref pipe_tasks_makeCoaddTempExp_Example
134 
135  @section pipe_tasks_makeCoaddTempExp_Purpose Description
136 
137  Warp and optionally PSF-Match calexps onto a common projection, by
138  performing the following operations:
139  - Group calexps by visit/run
140  - For each visit, generate a Warp by calling method @ref makeTempExp.
141  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
142  on each visit
143 
144  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
145 
146  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
147 
148  @copydoc \_\_init\_\_
149 
150  This task has one special keyword argument: passing reuse=True will cause
151  the task to skip the creation of warps that are already present in the
152  output repositories.
153 
154  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
155 
156  This task is primarily designed to be run from the command line.
157 
158  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
159  to process.
160 
161  @copydoc run
162 
163  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
164  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
165  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
166  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
167 
168  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
169 
170  See @ref MakeCoaddTempExpConfig and parameters inherited from
171  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
172 
173  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
174 
175  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
176  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
177  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
178  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
179  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
180  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
181 
182  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
183  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
184  In general, for a set of input images, this config should equal the FWHM of the visit
185  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
186  of the other config options offer a reasonable starting point.
187  The following list presents the most common problems that arise from a misconfigured
188  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
189  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
190  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
191  Problem: Explanation. *Solution*
192 
193  *Troublshooting PSF-Matching Configuration:*
194  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
195  For example:_
196 
197  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
198 
199  Note that increasing the kernel size also increases runtime.
200  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
201  for matching kernel. _Provide the matcher with more data by either increasing
202  the spatial sampling by decreasing the spatial cell size,_
203 
204  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
205  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
206 
207  _or increasing the padding around the Science PSF, for example:_
208 
209  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
210 
211  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
212  matching kernel dimensions, thus increasing the number of pixels available to fit
213  after convolving the PSF with the matching kernel.
214  Optionally, for debugging the effects of padding, the level of padding may be manually
215  controlled by setting turning off the automatic padding and setting the number
216  of pixels by which to pad the PSF:
217 
218  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
219  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
220 
221  - Deconvolution: Matching a large PSF to a smaller PSF produces
222  a telltale noise pattern which looks like ripples or a brain.
223  _Increase the size of the requested model PSF. For example:_
224 
225  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
226 
227  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
228  _Increase the width of the Gaussian basis functions. For example:_
229 
230  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
231  # from default [0.7, 1.5, 3.0]
232 
233 
234  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
235 
236  MakeCoaddTempExpTask has no debug output, but its subtasks do.
237 
238  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
239 
240  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
241  into the larger Data Release Processing.
242  Set up by running:
243 
244  setup ci_hsc
245  cd $CI_HSC_DIR
246  # if not built already:
247  python $(which scons) # this will take a while
248 
249  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
250  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
251  output respository with the desired SkyMap. The command,
252 
253  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
254  --id patch=5,4 tract=0 filter=HSC-I \
255  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
256  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
257  --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
258  makePsfMatched=True modelPsf.defaultFwhm=11
259 
260  writes a direct and PSF-Matched Warp to
261  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
262  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
263  respectively.
264 
265  @note PSF-Matching in this particular dataset would benefit from adding
266  `--configfile ./matchingConfig.py` to
267  the command line arguments where `matchingConfig.py` is defined by:
268 
269  echo "
270  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
271  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
272 
273 
274  Add the option `--help` to see more options.
275  """
276  ConfigClass = MakeCoaddTempExpConfig
277  _DefaultName = "makeCoaddTempExp"
278 
279  def __init__(self, reuse=False, **kwargs):
280  CoaddBaseTask.__init__(self, **kwargs)
281  self.reusereuse = reuse
282  self.makeSubtask("warpAndPsfMatch")
283  if self.config.hasFakes:
284  self.calexpTypecalexpType = "fakes_calexp"
285  else:
286  self.calexpTypecalexpType = "calexp"
287 
288  @timeMethod
289  def runDataRef(self, patchRef, selectDataList=[]):
290  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
291 
292  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
293  plus the camera-specific filter key (e.g. "filter" or "band")
294  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
295  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
296  warps are requested.
297 
298  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
299 
300  @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
301  with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
302  (assembleCoadd should determine zeropoint scaling without referring to it).
303  """
304  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
305 
306  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
307  if self.config.makePsfMatched and not self.config.makeDirect:
308  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
309  else:
310  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
311 
312  calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
313 
314  if len(calExpRefList) == 0:
315  self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
316  return None
317  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
318  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
319  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
320 
321  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
322  primaryWarpDataset)
323  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
324 
325  dataRefList = []
326  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
327  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
328  tempExpTuple, groupData.keys)
329  if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
330  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
331  dataRefList.append(tempExpRef)
332  continue
333  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
334 
335  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
336  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
337  # of the visit in the list.
338  try:
339  visitId = int(tempExpRef.dataId["visit"])
340  except (KeyError, ValueError):
341  visitId = i
342 
343  calExpList = []
344  ccdIdList = []
345  dataIdList = []
346 
347  for calExpInd, calExpRef in enumerate(calexpRefList):
348  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
349  calExpRef.dataId)
350  try:
351  ccdId = calExpRef.get("ccdExposureId", immediate=True)
352  except Exception:
353  ccdId = calExpInd
354  try:
355  # We augment the dataRef here with the tract, which is harmless for loading things
356  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
357  # which do.
358  calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
359  dataId=calExpRef.dataId,
360  tract=skyInfo.tractInfo.getId())
361  calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
362  except Exception as e:
363  self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
364  continue
365 
366  if self.config.doApplySkyCorr:
367  self.applySkyCorrapplySkyCorr(calExpRef, calExp)
368 
369  calExpList.append(calExp)
370  ccdIdList.append(ccdId)
371  dataIdList.append(calExpRef.dataId)
372 
373  exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
374 
375  if any(exps.values()):
376  dataRefList.append(tempExpRef)
377  else:
378  self.log.warning("Warp %s could not be created", tempExpRef.dataId)
379 
380  if self.config.doWrite:
381  for (warpType, exposure) in exps.items(): # compatible w/ Py3
382  if exposure is not None:
383  self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
384  tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
385 
386  return dataRefList
387 
388  @timeMethod
389  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
390  """Create a Warp from inputs
391 
392  We iterate over the multiple calexps in a single exposure to construct
393  the warp (previously called a coaddTempExp) of that exposure to the
394  supplied tract/patch.
395 
396  Pixels that receive no pixels are set to NAN; this is not correct
397  (violates LSST algorithms group policy), but will be fixed up by
398  interpolating after the coaddition.
399 
400  @param calexpRefList: List of data references for calexps that (may)
401  overlap the patch of interest
402  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
403  information about the patch
404  @param visitId: integer identifier for visit, for the table that will
405  produce the CoaddPsf
406  @return a pipeBase Struct containing:
407  - exposures: a dictionary containing the warps requested:
408  "direct": direct warp if config.makeDirect
409  "psfMatched": PSF-matched warp if config.makePsfMatched
410  """
411  warpTypeList = self.getWarpTypeListgetWarpTypeList()
412 
413  totGoodPix = {warpType: 0 for warpType in warpTypeList}
414  didSetMetadata = {warpType: False for warpType in warpTypeList}
415  coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
416  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
417  for warpType in warpTypeList}
418 
419  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
420  if dataIdList is None:
421  dataIdList = ccdIdList
422 
423  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
424  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
425  calExpInd+1, len(calExpList), dataId)
426 
427  try:
428  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
429  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
430  makeDirect=self.config.makeDirect,
431  makePsfMatched=self.config.makePsfMatched)
432  except Exception as e:
433  self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
434  continue
435  try:
436  numGoodPix = {warpType: 0 for warpType in warpTypeList}
437  for warpType in warpTypeList:
438  exposure = warpedAndMatched.getDict()[warpType]
439  if exposure is None:
440  continue
441  coaddTempExp = coaddTempExps[warpType]
442  if didSetMetadata[warpType]:
443  mimg = exposure.getMaskedImage()
444  mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
445  / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
446  del mimg
447  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
448  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
449  totGoodPix[warpType] += numGoodPix[warpType]
450  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
451  dataId, numGoodPix[warpType],
452  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
453  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
454  coaddTempExp.info.id = exposure.info.id
455  coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
456  coaddTempExp.setFilterLabel(exposure.getFilterLabel())
457  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
458  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
459  coaddTempExp.setPsf(exposure.getPsf())
460  didSetMetadata[warpType] = True
461 
462  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
463  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
464 
465  except Exception as e:
466  self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
467  continue
468 
469  for warpType in warpTypeList:
470  self.log.info("%sWarp has %d good pixels (%.1f%%)",
471  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
472 
473  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
474  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
475  if warpType == "direct":
476  coaddTempExps[warpType].setPsf(
477  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
478  self.config.coaddPsf.makeControl()))
479  else:
480  if not self.config.doWriteEmptyWarps:
481  # No good pixels. Exposure still empty
482  coaddTempExps[warpType] = None
483  # NoWorkFound is unnecessary as the downstream tasks will
484  # adjust the quantum accordingly, and it prevents gen2
485  # MakeCoaddTempExp from continuing to loop over visits.
486 
487  result = pipeBase.Struct(exposures=coaddTempExps)
488  return result
489 
490  def getCalibratedExposure(self, dataRef, bgSubtracted):
491  """Return one calibrated Exposure, possibly with an updated SkyWcs.
492 
493  @param[in] dataRef a sensor-level data reference
494  @param[in] bgSubtracted return calexp with background subtracted? If False get the
495  calexp's background background model and add it to the calexp.
496  @return calibrated exposure
497 
498  @raises MissingExposureError If data for the exposure is not available.
499 
500  If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
501  (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
502  `name_photoCalib` dataset. Otherwise, the photometric calibration is
503  retrieved from the processed exposure. When
504  `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
505  is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
506  Otherwise, the astrometric calibration is taken from the processed
507  exposure.
508  """
509  try:
510  exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
511  except dafPersist.NoResults as e:
512  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
513 
514  if not bgSubtracted:
515  background = dataRef.get("calexpBackground", immediate=True)
516  mi = exposure.getMaskedImage()
517  mi += background.getImage()
518  del mi
519 
520  if self.config.doApplyExternalPhotoCalib:
521  source = f"{self.config.externalPhotoCalibName}_photoCalib"
522  self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
523  photoCalib = dataRef.get(source)
524  exposure.setPhotoCalib(photoCalib)
525  else:
526  photoCalib = exposure.getPhotoCalib()
527 
528  if self.config.doApplyExternalSkyWcs:
529  source = f"{self.config.externalSkyWcsName}_wcs"
530  self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
531  skyWcs = dataRef.get(source)
532  exposure.setWcs(skyWcs)
533 
534  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
535  includeScaleUncertainty=self.config.includeCalibVar)
536  exposure.maskedImage /= photoCalib.getCalibrationMean()
537  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
538  # exposure.setCalib(afwImage.Calib(1.0))
539  return exposure
540 
541  @staticmethod
542  def _prepareEmptyExposure(skyInfo):
543  """Produce an empty exposure for a given patch"""
544  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546  .getPlaneBitMask("NO_DATA"), numpy.inf)
547  return exp
548 
549  def getWarpTypeList(self):
550  """Return list of requested warp types per the config.
551  """
552  warpTypeList = []
553  if self.config.makeDirect:
554  warpTypeList.append("direct")
555  if self.config.makePsfMatched:
556  warpTypeList.append("psfMatched")
557  return warpTypeList
558 
559  def applySkyCorr(self, dataRef, calexp):
560  """Apply correction to the sky background level
561 
562  Sky corrections can be generated with the 'skyCorrection.py'
563  executable in pipe_drivers. Because the sky model used by that
564  code extends over the entire focal plane, this can produce
565  better sky subtraction.
566 
567  The calexp is updated in-place.
568 
569  Parameters
570  ----------
571  dataRef : `lsst.daf.persistence.ButlerDataRef`
572  Data reference for calexp.
573  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
574  Calibrated exposure.
575  """
576  bg = dataRef.get("skyCorr")
577  self.log.debug("Applying sky correction to %s", dataRef.dataId)
578  if isinstance(calexp, afwImage.Exposure):
579  calexp = calexp.getMaskedImage()
580  calexp -= bg.getImage()
581 
582 
583 class MakeWarpConnections(pipeBase.PipelineTaskConnections,
584  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
585  defaultTemplates={"coaddName": "deep",
586  "skyWcsName": "jointcal",
587  "photoCalibName": "fgcm",
588  "calexpType": ""}):
589  calExpList = connectionTypes.Input(
590  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
591  name="{calexpType}calexp",
592  storageClass="ExposureF",
593  dimensions=("instrument", "visit", "detector"),
594  multiple=True,
595  deferLoad=True,
596  )
597  backgroundList = connectionTypes.Input(
598  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
599  name="calexpBackground",
600  storageClass="Background",
601  dimensions=("instrument", "visit", "detector"),
602  multiple=True,
603  )
604  skyCorrList = connectionTypes.Input(
605  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
606  name="skyCorr",
607  storageClass="Background",
608  dimensions=("instrument", "visit", "detector"),
609  multiple=True,
610  )
611  skyMap = connectionTypes.Input(
612  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
613  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
614  storageClass="SkyMap",
615  dimensions=("skymap",),
616  )
617  externalSkyWcsTractCatalog = connectionTypes.Input(
618  doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
619  "id for the catalog id, sorted on id for fast lookup."),
620  name="{skyWcsName}SkyWcsCatalog",
621  storageClass="ExposureCatalog",
622  dimensions=("instrument", "visit", "tract"),
623  )
624  externalSkyWcsGlobalCatalog = connectionTypes.Input(
625  doc=("Per-visit wcs calibrations computed globally (with no tract information). "
626  "These catalogs use the detector id for the catalog id, sorted on id for "
627  "fast lookup."),
628  name="{skyWcsName}SkyWcsCatalog",
629  storageClass="ExposureCatalog",
630  dimensions=("instrument", "visit"),
631  )
632  externalPhotoCalibTractCatalog = connectionTypes.Input(
633  doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
634  "detector id for the catalog id, sorted on id for fast lookup."),
635  name="{photoCalibName}PhotoCalibCatalog",
636  storageClass="ExposureCatalog",
637  dimensions=("instrument", "visit", "tract"),
638  )
639  externalPhotoCalibGlobalCatalog = connectionTypes.Input(
640  doc=("Per-visit photometric calibrations computed globally (with no tract "
641  "information). These catalogs use the detector id for the catalog id, "
642  "sorted on id for fast lookup."),
643  name="{photoCalibName}PhotoCalibCatalog",
644  storageClass="ExposureCatalog",
645  dimensions=("instrument", "visit"),
646  )
647  direct = connectionTypes.Output(
648  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
649  "calexps onto the skyMap patch geometry."),
650  name="{coaddName}Coadd_directWarp",
651  storageClass="ExposureF",
652  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
653  )
654  psfMatched = connectionTypes.Output(
655  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
656  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
657  name="{coaddName}Coadd_psfMatchedWarp",
658  storageClass="ExposureF",
659  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
660  )
661  # TODO DM-28769, have selectImages subtask indicate which connections they need:
662  wcsList = connectionTypes.Input(
663  doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
664  name="{calexpType}calexp.wcs",
665  storageClass="Wcs",
666  dimensions=("instrument", "visit", "detector"),
667  multiple=True,
668  )
669  bboxList = connectionTypes.Input(
670  doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
671  name="{calexpType}calexp.bbox",
672  storageClass="Box2I",
673  dimensions=("instrument", "visit", "detector"),
674  multiple=True,
675  )
676  srcList = connectionTypes.Input(
677  doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
678  name="src",
679  storageClass="SourceCatalog",
680  dimensions=("instrument", "visit", "detector"),
681  multiple=True,
682  )
683  psfList = connectionTypes.Input(
684  doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
685  name="{calexpType}calexp.psf",
686  storageClass="Psf",
687  dimensions=("instrument", "visit", "detector"),
688  multiple=True,
689  )
690 
691  def __init__(self, *, config=None):
692  super().__init__(config=config)
693  if config.bgSubtracted:
694  self.inputs.remove("backgroundList")
695  if not config.doApplySkyCorr:
696  self.inputs.remove("skyCorrList")
697  if config.doApplyExternalSkyWcs:
698  if config.useGlobalExternalSkyWcs:
699  self.inputs.remove("externalSkyWcsTractCatalog")
700  else:
701  self.inputs.remove("externalSkyWcsGlobalCatalog")
702  else:
703  self.inputs.remove("externalSkyWcsTractCatalog")
704  self.inputs.remove("externalSkyWcsGlobalCatalog")
705  if config.doApplyExternalPhotoCalib:
706  if config.useGlobalExternalPhotoCalib:
707  self.inputs.remove("externalPhotoCalibTractCatalog")
708  else:
709  self.inputs.remove("externalPhotoCalibGlobalCatalog")
710  else:
711  self.inputs.remove("externalPhotoCalibTractCatalog")
712  self.inputs.remove("externalPhotoCalibGlobalCatalog")
713  if not config.makeDirect:
714  self.outputs.remove("direct")
715  if not config.makePsfMatched:
716  self.outputs.remove("psfMatched")
717  # TODO DM-28769: add connection per selectImages connections
718  # instead of removing if not PsfWcsSelectImagesTask here:
719  if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
720  self.inputs.remove("srcList")
721  if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
722  self.inputs.remove("psfList")
723 
724 
725 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
726  pipelineConnections=MakeWarpConnections):
727 
728  def validate(self):
729  super().validate()
730 
731 
732 class MakeWarpTask(MakeCoaddTempExpTask):
733  """Warp and optionally PSF-Match calexps onto an a common projection
734  """
735  ConfigClass = MakeWarpConfig
736  _DefaultName = "makeWarp"
737 
738  @utils.inheritDoc(pipeBase.PipelineTask)
739  def runQuantum(self, butlerQC, inputRefs, outputRefs):
740  """
741  Notes
742  ----
743  Construct warps for requested warp type for single epoch
744 
745  PipelineTask (Gen3) entry point to warp and optionally PSF-match
746  calexps. This method is analogous to `runDataRef`.
747  """
748 
749  # Ensure all input lists are in same detector order as the calExpList
750  detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
751  inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
752 
753  # Read in all inputs.
754  inputs = butlerQC.get(inputRefs)
755 
756  # Construct skyInfo expected by `run`. We remove the SkyMap itself
757  # from the dictionary so we can pass it as kwargs later.
758  skyMap = inputs.pop("skyMap")
759  quantumDataId = butlerQC.quantum.dataId
760  skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
761 
762  # Construct list of input DataIds expected by `run`
763  dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
764  # Construct list of packed integer IDs expected by `run`
765  ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
766 
767  # Run the selector and filter out calexps that were not selected
768  # primarily because they do not overlap the patch
769  cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
770  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
771  goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
772  inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
773 
774  # Read from disk only the selected calexps
775  inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
776 
777  # Extract integer visitId requested by `run`
778  visits = [dataId['visit'] for dataId in dataIdList]
779  visitId = visits[0]
780 
781  if self.config.doApplyExternalSkyWcs:
782  if self.config.useGlobalExternalSkyWcs:
783  externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
784  else:
785  externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
786  else:
787  externalSkyWcsCatalog = None
788 
789  if self.config.doApplyExternalPhotoCalib:
790  if self.config.useGlobalExternalPhotoCalib:
791  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
792  else:
793  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
794  else:
795  externalPhotoCalibCatalog = None
796 
797  completeIndices = self.prepareCalibratedExposures(**inputs,
798  externalSkyWcsCatalog=externalSkyWcsCatalog,
799  externalPhotoCalibCatalog=externalPhotoCalibCatalog)
800  # Redo the input selection with inputs with complete wcs/photocalib info.
801  inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
802 
803  results = self.run(**inputs, visitId=visitId,
804  ccdIdList=[ccdIdList[i] for i in goodIndices],
805  dataIdList=[dataIdList[i] for i in goodIndices],
806  skyInfo=skyInfo)
807  if self.config.makeDirect and results.exposures["direct"] is not None:
808  butlerQC.put(results.exposures["direct"], outputRefs.direct)
809  if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
810  butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
811 
812  def filterInputs(self, indices, inputs):
813  """Return task inputs with their lists filtered by indices
814 
815  Parameters
816  ----------
817  indices : `list` of integers
818  inputs : `dict` of `list` of input connections to be passed to run
819  """
820  for key in inputs.keys():
821  # Only down-select on list inputs
822  if isinstance(inputs[key], list):
823  inputs[key] = [inputs[key][ind] for ind in indices]
824  return inputs
825 
826  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
827  externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
828  **kwargs):
829  """Calibrate and add backgrounds to input calExpList in place
830 
831  Parameters
832  ----------
833  calExpList : `list` of `lsst.afw.image.Exposure`
834  Sequence of calexps to be modified in place
835  backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
836  Sequence of backgrounds to be added back in if bgSubtracted=False
837  skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
838  Sequence of background corrections to be subtracted if doApplySkyCorr=True
839  externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
840  Exposure catalog with external skyWcs to be applied
841  if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
842  for the catalog id, sorted on id for fast lookup.
843  externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
844  Exposure catalog with external photoCalib to be applied
845  if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
846  id for the catalog id, sorted on id for fast lookup.
847 
848  Returns
849  -------
850  indices : `list` [`int`]
851  Indices of calExpList and friends that have valid photoCalib/skyWcs
852  """
853  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
854  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
855 
856  includeCalibVar = self.config.includeCalibVar
857 
858  indices = []
859  for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
860  backgroundList,
861  skyCorrList)):
862  mi = calexp.maskedImage
863  if not self.config.bgSubtracted:
864  mi += background.getImage()
865 
866  if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
867  detectorId = calexp.getInfo().getDetector().getId()
868 
869  # Find the external photoCalib
870  if externalPhotoCalibCatalog is not None:
871  row = externalPhotoCalibCatalog.find(detectorId)
872  if row is None:
873  self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
874  "and will not be used in the warp.", detectorId)
875  continue
876  photoCalib = row.getPhotoCalib()
877  if photoCalib is None:
878  self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
879  "and will not be used in the warp.", detectorId)
880  continue
881  calexp.setPhotoCalib(photoCalib)
882  else:
883  photoCalib = calexp.getPhotoCalib()
884  if photoCalib is None:
885  self.log.warning("Detector id %s has None for photoCalib in the calexp "
886  "and will not be used in the warp.", detectorId)
887  continue
888 
889  # Find and apply external skyWcs
890  if externalSkyWcsCatalog is not None:
891  row = externalSkyWcsCatalog.find(detectorId)
892  if row is None:
893  self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
894  "and will not be used in the warp.", detectorId)
895  continue
896  skyWcs = row.getWcs()
897  if skyWcs is None:
898  self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
899  "and will not be used in the warp.", detectorId)
900  continue
901  calexp.setWcs(skyWcs)
902  else:
903  skyWcs = calexp.getWcs()
904  if skyWcs is None:
905  self.log.warning("Detector id %s has None for skyWcs in the calexp "
906  "and will not be used in the warp.", detectorId)
907  continue
908 
909  # Calibrate the image
910  calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
911  includeScaleUncertainty=includeCalibVar)
912  calexp.maskedImage /= photoCalib.getCalibrationMean()
913  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
914  # exposure.setCalib(afwImage.Calib(1.0))
915 
916  # Apply skycorr
917  if self.config.doApplySkyCorr:
918  mi -= skyCorr.getImage()
919 
920  indices.append(index)
921 
922  return indices
923 
924 
925 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
926  """Reorder inputRefs per outputSortKeyOrder
927 
928  Any inputRefs which are lists will be resorted per specified key e.g.,
929  'detector.' Only iterables will be reordered, and values can be of type
930  `lsst.pipe.base.connections.DeferredDatasetRef` or
931  `lsst.daf.butler.core.datasets.ref.DatasetRef`.
932  Returned lists of refs have the same length as the outputSortKeyOrder.
933  If an outputSortKey not in the inputRef, then it will be padded with None.
934  If an inputRef contains an inputSortKey that is not in the
935  outputSortKeyOrder it will be removed.
936 
937  Parameters
938  ----------
939  inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
940  Input references to be reordered and padded.
941  outputSortKeyOrder : iterable
942  Iterable of values to be compared with inputRef's dataId[dataIdKey]
943  dataIdKey : `str`
944  dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
945 
946  Returns:
947  --------
948  inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
949  Quantized Connection with sorted DatasetRef values sorted if iterable.
950  """
951  for connectionName, refs in inputRefs:
952  if isinstance(refs, Iterable):
953  if hasattr(refs[0], "dataId"):
954  inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
955  else:
956  inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
957  if inputSortKeyOrder != outputSortKeyOrder:
958  setattr(inputRefs, connectionName,
959  reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
960  return inputRefs
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60