lsst.pipe.tasks  v23.0.x-g0162f352a6+f2fbba1123
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 import logging
24 
25 import lsst.pex.config as pexConfig
26 import lsst.daf.persistence as dafPersist
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as connectionTypes
31 import lsst.utils as utils
32 import lsst.geom
33 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34 from lsst.skymap import BaseSkyMap
35 from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36 from .warpAndPsfMatch import WarpAndPsfMatchTask
37 from .coaddHelpers import groupPatchExposures, getGroupDataRef
38 from collections.abc import Iterable
39 
40 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
41 
42 log = logging.getLogger(__name__.partition(".")[2])
43 
44 
45 class MissingExposureError(Exception):
46  """Raised when data cannot be retrieved for an exposure.
47  When processing patches, sometimes one exposure is missing; this lets us
48  distinguish bewteen that case, and other errors.
49  """
50  pass
51 
52 
53 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
54  """Config for MakeCoaddTempExpTask
55  """
56  warpAndPsfMatch = pexConfig.ConfigurableField(
57  target=WarpAndPsfMatchTask,
58  doc="Task to warp and PSF-match calexp",
59  )
60  doWrite = pexConfig.Field(
61  doc="persist <coaddName>Coadd_<warpType>Warp",
62  dtype=bool,
63  default=True,
64  )
65  bgSubtracted = pexConfig.Field(
66  doc="Work with a background subtracted calexp?",
67  dtype=bool,
68  default=True,
69  )
70  coaddPsf = pexConfig.ConfigField(
71  doc="Configuration for CoaddPsf",
72  dtype=CoaddPsfConfig,
73  )
74  makeDirect = pexConfig.Field(
75  doc="Make direct Warp/Coadds",
76  dtype=bool,
77  default=True,
78  )
79  makePsfMatched = pexConfig.Field(
80  doc="Make Psf-Matched Warp/Coadd?",
81  dtype=bool,
82  default=False,
83  )
84 
85  doWriteEmptyWarps = pexConfig.Field(
86  dtype=bool,
87  default=False,
88  doc="Write out warps even if they are empty"
89  )
90 
91  hasFakes = pexConfig.Field(
92  doc="Should be set to True if fake sources have been inserted into the input data.",
93  dtype=bool,
94  default=False,
95  )
96  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
97 
98  def validate(self):
99  CoaddBaseTask.ConfigClass.validate(self)
100  if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
101  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
102  if self.doPsfMatch:
103  # Backwards compatibility.
104  log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
105  self.makePsfMatchedmakePsfMatched = True
106  self.makeDirectmakeDirect = False
107 
108  def setDefaults(self):
109  CoaddBaseTask.ConfigClass.setDefaults(self)
110  self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
111 
112 
118 
119 
121  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
122 
123  @anchor MakeCoaddTempExpTask_
124 
125  @section pipe_tasks_makeCoaddTempExp_Contents Contents
126 
127  - @ref pipe_tasks_makeCoaddTempExp_Purpose
128  - @ref pipe_tasks_makeCoaddTempExp_Initialize
129  - @ref pipe_tasks_makeCoaddTempExp_IO
130  - @ref pipe_tasks_makeCoaddTempExp_Config
131  - @ref pipe_tasks_makeCoaddTempExp_Debug
132  - @ref pipe_tasks_makeCoaddTempExp_Example
133 
134  @section pipe_tasks_makeCoaddTempExp_Purpose Description
135 
136  Warp and optionally PSF-Match calexps onto a common projection, by
137  performing the following operations:
138  - Group calexps by visit/run
139  - For each visit, generate a Warp by calling method @ref makeTempExp.
140  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
141  on each visit
142 
143  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
144 
145  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
146 
147  @copydoc \_\_init\_\_
148 
149  This task has one special keyword argument: passing reuse=True will cause
150  the task to skip the creation of warps that are already present in the
151  output repositories.
152 
153  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
154 
155  This task is primarily designed to be run from the command line.
156 
157  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
158  to process.
159 
160  @copydoc run
161 
162  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
163  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
164  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
165  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
166 
167  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
168 
169  See @ref MakeCoaddTempExpConfig and parameters inherited from
170  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
171 
172  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
173 
174  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
175  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
176  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
177  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
178  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
179  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
180 
181  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
182  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
183  In general, for a set of input images, this config should equal the FWHM of the visit
184  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
185  of the other config options offer a reasonable starting point.
186  The following list presents the most common problems that arise from a misconfigured
187  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
188  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
189  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
190  Problem: Explanation. *Solution*
191 
192  *Troublshooting PSF-Matching Configuration:*
193  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
194  For example:_
195 
196  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
197 
198  Note that increasing the kernel size also increases runtime.
199  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
200  for matching kernel. _Provide the matcher with more data by either increasing
201  the spatial sampling by decreasing the spatial cell size,_
202 
203  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
204  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
205 
206  _or increasing the padding around the Science PSF, for example:_
207 
208  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
209 
210  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
211  matching kernel dimensions, thus increasing the number of pixels available to fit
212  after convolving the PSF with the matching kernel.
213  Optionally, for debugging the effects of padding, the level of padding may be manually
214  controlled by setting turning off the automatic padding and setting the number
215  of pixels by which to pad the PSF:
216 
217  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
218  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
219 
220  - Deconvolution: Matching a large PSF to a smaller PSF produces
221  a telltale noise pattern which looks like ripples or a brain.
222  _Increase the size of the requested model PSF. For example:_
223 
224  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
225 
226  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
227  _Increase the width of the Gaussian basis functions. For example:_
228 
229  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
230  # from default [0.7, 1.5, 3.0]
231 
232 
233  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
234 
235  MakeCoaddTempExpTask has no debug output, but its subtasks do.
236 
237  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
238 
239  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
240  into the larger Data Release Processing.
241  Set up by running:
242 
243  setup ci_hsc
244  cd $CI_HSC_DIR
245  # if not built already:
246  python $(which scons) # this will take a while
247 
248  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
249  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
250  output respository with the desired SkyMap. The command,
251 
252  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
253  --id patch=5,4 tract=0 filter=HSC-I \
254  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
255  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
256  --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
257  makePsfMatched=True modelPsf.defaultFwhm=11
258 
259  writes a direct and PSF-Matched Warp to
260  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
261  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
262  respectively.
263 
264  @note PSF-Matching in this particular dataset would benefit from adding
265  `--configfile ./matchingConfig.py` to
266  the command line arguments where `matchingConfig.py` is defined by:
267 
268  echo "
269  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
270  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
271 
272 
273  Add the option `--help` to see more options.
274  """
275  ConfigClass = MakeCoaddTempExpConfig
276  _DefaultName = "makeCoaddTempExp"
277 
278  def __init__(self, reuse=False, **kwargs):
279  CoaddBaseTask.__init__(self, **kwargs)
280  self.reusereuse = reuse
281  self.makeSubtask("warpAndPsfMatch")
282  if self.config.hasFakes:
283  self.calexpTypecalexpType = "fakes_calexp"
284  else:
285  self.calexpTypecalexpType = "calexp"
286 
287  @pipeBase.timeMethod
288  def runDataRef(self, patchRef, selectDataList=[]):
289  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
290 
291  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
292  plus the camera-specific filter key (e.g. "filter" or "band")
293  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
294  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
295  warps are requested.
296 
297  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
298 
299  @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
300  with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
301  (assembleCoadd should determine zeropoint scaling without referring to it).
302  """
303  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
304 
305  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
306  if self.config.makePsfMatched and not self.config.makeDirect:
307  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
308  else:
309  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
310 
311  calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
312 
313  if len(calExpRefList) == 0:
314  self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
315  return None
316  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
317  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
318  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
319 
320  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
321  primaryWarpDataset)
322  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
323 
324  dataRefList = []
325  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
326  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
327  tempExpTuple, groupData.keys)
328  if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
329  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
330  dataRefList.append(tempExpRef)
331  continue
332  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
333 
334  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
335  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
336  # of the visit in the list.
337  try:
338  visitId = int(tempExpRef.dataId["visit"])
339  except (KeyError, ValueError):
340  visitId = i
341 
342  calExpList = []
343  ccdIdList = []
344  dataIdList = []
345 
346  for calExpInd, calExpRef in enumerate(calexpRefList):
347  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
348  calExpRef.dataId)
349  try:
350  ccdId = calExpRef.get("ccdExposureId", immediate=True)
351  except Exception:
352  ccdId = calExpInd
353  try:
354  # We augment the dataRef here with the tract, which is harmless for loading things
355  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
356  # which do.
357  calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
358  dataId=calExpRef.dataId,
359  tract=skyInfo.tractInfo.getId())
360  calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
361  except Exception as e:
362  self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
363  continue
364 
365  if self.config.doApplySkyCorr:
366  self.applySkyCorrapplySkyCorr(calExpRef, calExp)
367 
368  calExpList.append(calExp)
369  ccdIdList.append(ccdId)
370  dataIdList.append(calExpRef.dataId)
371 
372  exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
373 
374  if any(exps.values()):
375  dataRefList.append(tempExpRef)
376  else:
377  self.log.warning("Warp %s could not be created", tempExpRef.dataId)
378 
379  if self.config.doWrite:
380  for (warpType, exposure) in exps.items(): # compatible w/ Py3
381  if exposure is not None:
382  self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
383  tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
384 
385  return dataRefList
386 
387  @pipeBase.timeMethod
388  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
389  """Create a Warp from inputs
390 
391  We iterate over the multiple calexps in a single exposure to construct
392  the warp (previously called a coaddTempExp) of that exposure to the
393  supplied tract/patch.
394 
395  Pixels that receive no pixels are set to NAN; this is not correct
396  (violates LSST algorithms group policy), but will be fixed up by
397  interpolating after the coaddition.
398 
399  @param calexpRefList: List of data references for calexps that (may)
400  overlap the patch of interest
401  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
402  information about the patch
403  @param visitId: integer identifier for visit, for the table that will
404  produce the CoaddPsf
405  @return a pipeBase Struct containing:
406  - exposures: a dictionary containing the warps requested:
407  "direct": direct warp if config.makeDirect
408  "psfMatched": PSF-matched warp if config.makePsfMatched
409  """
410  warpTypeList = self.getWarpTypeListgetWarpTypeList()
411 
412  totGoodPix = {warpType: 0 for warpType in warpTypeList}
413  didSetMetadata = {warpType: False for warpType in warpTypeList}
414  coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
415  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
416  for warpType in warpTypeList}
417 
418  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
419  if dataIdList is None:
420  dataIdList = ccdIdList
421 
422  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
423  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
424  calExpInd+1, len(calExpList), dataId)
425 
426  try:
427  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
428  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
429  makeDirect=self.config.makeDirect,
430  makePsfMatched=self.config.makePsfMatched)
431  except Exception as e:
432  self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
433  continue
434  try:
435  numGoodPix = {warpType: 0 for warpType in warpTypeList}
436  for warpType in warpTypeList:
437  exposure = warpedAndMatched.getDict()[warpType]
438  if exposure is None:
439  continue
440  coaddTempExp = coaddTempExps[warpType]
441  if didSetMetadata[warpType]:
442  mimg = exposure.getMaskedImage()
443  mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
444  / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
445  del mimg
446  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
447  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
448  totGoodPix[warpType] += numGoodPix[warpType]
449  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
450  dataId, numGoodPix[warpType],
451  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
452  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
453  coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
454  coaddTempExp.setFilterLabel(exposure.getFilterLabel())
455  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
456  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
457  coaddTempExp.setPsf(exposure.getPsf())
458  didSetMetadata[warpType] = True
459 
460  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
461  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
462 
463  except Exception as e:
464  self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
465  continue
466 
467  for warpType in warpTypeList:
468  self.log.info("%sWarp has %d good pixels (%.1f%%)",
469  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
470 
471  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
472  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
473  if warpType == "direct":
474  coaddTempExps[warpType].setPsf(
475  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
476  self.config.coaddPsf.makeControl()))
477  else:
478  if not self.config.doWriteEmptyWarps:
479  # No good pixels. Exposure still empty
480  coaddTempExps[warpType] = None
481  # NoWorkFound is unnecessary as the downstream tasks will
482  # adjust the quantum accordingly, and it prevents gen2
483  # MakeCoaddTempExp from continuing to loop over visits.
484 
485  result = pipeBase.Struct(exposures=coaddTempExps)
486  return result
487 
488  def getCalibratedExposure(self, dataRef, bgSubtracted):
489  """Return one calibrated Exposure, possibly with an updated SkyWcs.
490 
491  @param[in] dataRef a sensor-level data reference
492  @param[in] bgSubtracted return calexp with background subtracted? If False get the
493  calexp's background background model and add it to the calexp.
494  @return calibrated exposure
495 
496  @raises MissingExposureError If data for the exposure is not available.
497 
498  If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
499  (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
500  `name_photoCalib` dataset. Otherwise, the photometric calibration is
501  retrieved from the processed exposure. When
502  `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
503  is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
504  Otherwise, the astrometric calibration is taken from the processed
505  exposure.
506  """
507  try:
508  exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
509  except dafPersist.NoResults as e:
510  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
511 
512  if not bgSubtracted:
513  background = dataRef.get("calexpBackground", immediate=True)
514  mi = exposure.getMaskedImage()
515  mi += background.getImage()
516  del mi
517 
518  if self.config.doApplyExternalPhotoCalib:
519  source = f"{self.config.externalPhotoCalibName}_photoCalib"
520  self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
521  photoCalib = dataRef.get(source)
522  exposure.setPhotoCalib(photoCalib)
523  else:
524  photoCalib = exposure.getPhotoCalib()
525 
526  if self.config.doApplyExternalSkyWcs:
527  source = f"{self.config.externalSkyWcsName}_wcs"
528  self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
529  skyWcs = dataRef.get(source)
530  exposure.setWcs(skyWcs)
531 
532  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
533  includeScaleUncertainty=self.config.includeCalibVar)
534  exposure.maskedImage /= photoCalib.getCalibrationMean()
535  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
536  # exposure.setCalib(afwImage.Calib(1.0))
537  return exposure
538 
539  @staticmethod
540  def _prepareEmptyExposure(skyInfo):
541  """Produce an empty exposure for a given patch"""
542  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
543  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
544  .getPlaneBitMask("NO_DATA"), numpy.inf)
545  return exp
546 
547  def getWarpTypeList(self):
548  """Return list of requested warp types per the config.
549  """
550  warpTypeList = []
551  if self.config.makeDirect:
552  warpTypeList.append("direct")
553  if self.config.makePsfMatched:
554  warpTypeList.append("psfMatched")
555  return warpTypeList
556 
557  def applySkyCorr(self, dataRef, calexp):
558  """Apply correction to the sky background level
559 
560  Sky corrections can be generated with the 'skyCorrection.py'
561  executable in pipe_drivers. Because the sky model used by that
562  code extends over the entire focal plane, this can produce
563  better sky subtraction.
564 
565  The calexp is updated in-place.
566 
567  Parameters
568  ----------
569  dataRef : `lsst.daf.persistence.ButlerDataRef`
570  Data reference for calexp.
571  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
572  Calibrated exposure.
573  """
574  bg = dataRef.get("skyCorr")
575  self.log.debug("Applying sky correction to %s", dataRef.dataId)
576  if isinstance(calexp, afwImage.Exposure):
577  calexp = calexp.getMaskedImage()
578  calexp -= bg.getImage()
579 
580 
581 class MakeWarpConnections(pipeBase.PipelineTaskConnections,
582  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
583  defaultTemplates={"coaddName": "deep",
584  "skyWcsName": "jointcal",
585  "photoCalibName": "fgcm",
586  "calexpType": ""}):
587  calExpList = connectionTypes.Input(
588  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
589  name="{calexpType}calexp",
590  storageClass="ExposureF",
591  dimensions=("instrument", "visit", "detector"),
592  multiple=True,
593  deferLoad=True,
594  )
595  backgroundList = connectionTypes.Input(
596  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
597  name="calexpBackground",
598  storageClass="Background",
599  dimensions=("instrument", "visit", "detector"),
600  multiple=True,
601  )
602  skyCorrList = connectionTypes.Input(
603  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
604  name="skyCorr",
605  storageClass="Background",
606  dimensions=("instrument", "visit", "detector"),
607  multiple=True,
608  )
609  skyMap = connectionTypes.Input(
610  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
611  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
612  storageClass="SkyMap",
613  dimensions=("skymap",),
614  )
615  externalSkyWcsTractCatalog = connectionTypes.Input(
616  doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
617  "id for the catalog id, sorted on id for fast lookup."),
618  name="{skyWcsName}SkyWcsCatalog",
619  storageClass="ExposureCatalog",
620  dimensions=("instrument", "visit", "tract"),
621  )
622  externalSkyWcsGlobalCatalog = connectionTypes.Input(
623  doc=("Per-visit wcs calibrations computed globally (with no tract information). "
624  "These catalogs use the detector id for the catalog id, sorted on id for "
625  "fast lookup."),
626  name="{skyWcsName}SkyWcsCatalog",
627  storageClass="ExposureCatalog",
628  dimensions=("instrument", "visit"),
629  )
630  externalPhotoCalibTractCatalog = connectionTypes.Input(
631  doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
632  "detector id for the catalog id, sorted on id for fast lookup."),
633  name="{photoCalibName}PhotoCalibCatalog",
634  storageClass="ExposureCatalog",
635  dimensions=("instrument", "visit", "tract"),
636  )
637  externalPhotoCalibGlobalCatalog = connectionTypes.Input(
638  doc=("Per-visit photometric calibrations computed globally (with no tract "
639  "information). These catalogs use the detector id for the catalog id, "
640  "sorted on id for fast lookup."),
641  name="{photoCalibName}PhotoCalibCatalog",
642  storageClass="ExposureCatalog",
643  dimensions=("instrument", "visit"),
644  )
645  direct = connectionTypes.Output(
646  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
647  "calexps onto the skyMap patch geometry."),
648  name="{coaddName}Coadd_directWarp",
649  storageClass="ExposureF",
650  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
651  )
652  psfMatched = connectionTypes.Output(
653  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
654  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
655  name="{coaddName}Coadd_psfMatchedWarp",
656  storageClass="ExposureF",
657  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
658  )
659  # TODO DM-28769, have selectImages subtask indicate which connections they need:
660  wcsList = connectionTypes.Input(
661  doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
662  name="{calexpType}calexp.wcs",
663  storageClass="Wcs",
664  dimensions=("instrument", "visit", "detector"),
665  multiple=True,
666  )
667  bboxList = connectionTypes.Input(
668  doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
669  name="{calexpType}calexp.bbox",
670  storageClass="Box2I",
671  dimensions=("instrument", "visit", "detector"),
672  multiple=True,
673  )
674  visitSummary = connectionTypes.Input(
675  doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
676  name="{calexpType}visitSummary",
677  storageClass="ExposureCatalog",
678  dimensions=("instrument", "visit",),
679  )
680 
681  def __init__(self, *, config=None):
682  super().__init__(config=config)
683  if config.bgSubtracted:
684  self.inputs.remove("backgroundList")
685  if not config.doApplySkyCorr:
686  self.inputs.remove("skyCorrList")
687  if config.doApplyExternalSkyWcs:
688  if config.useGlobalExternalSkyWcs:
689  self.inputs.remove("externalSkyWcsTractCatalog")
690  else:
691  self.inputs.remove("externalSkyWcsGlobalCatalog")
692  else:
693  self.inputs.remove("externalSkyWcsTractCatalog")
694  self.inputs.remove("externalSkyWcsGlobalCatalog")
695  if config.doApplyExternalPhotoCalib:
696  if config.useGlobalExternalPhotoCalib:
697  self.inputs.remove("externalPhotoCalibTractCatalog")
698  else:
699  self.inputs.remove("externalPhotoCalibGlobalCatalog")
700  else:
701  self.inputs.remove("externalPhotoCalibTractCatalog")
702  self.inputs.remove("externalPhotoCalibGlobalCatalog")
703  if not config.makeDirect:
704  self.outputs.remove("direct")
705  if not config.makePsfMatched:
706  self.outputs.remove("psfMatched")
707  # TODO DM-28769: add connection per selectImages connections
708  if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
709  self.inputs.remove("visitSummary")
710 
711 
712 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
713  pipelineConnections=MakeWarpConnections):
714 
715  def validate(self):
716  super().validate()
717 
718 
719 class MakeWarpTask(MakeCoaddTempExpTask):
720  """Warp and optionally PSF-Match calexps onto an a common projection
721  """
722  ConfigClass = MakeWarpConfig
723  _DefaultName = "makeWarp"
724 
725  @utils.inheritDoc(pipeBase.PipelineTask)
726  def runQuantum(self, butlerQC, inputRefs, outputRefs):
727  """
728  Notes
729  ----
730  Construct warps for requested warp type for single epoch
731 
732  PipelineTask (Gen3) entry point to warp and optionally PSF-match
733  calexps. This method is analogous to `runDataRef`.
734  """
735 
736  # Ensure all input lists are in same detector order as the calExpList
737  detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
738  inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
739 
740  # Read in all inputs.
741  inputs = butlerQC.get(inputRefs)
742 
743  # Construct skyInfo expected by `run`. We remove the SkyMap itself
744  # from the dictionary so we can pass it as kwargs later.
745  skyMap = inputs.pop("skyMap")
746  quantumDataId = butlerQC.quantum.dataId
747  skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
748 
749  # Construct list of input DataIds expected by `run`
750  dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
751  # Construct list of packed integer IDs expected by `run`
752  ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
753 
754  # Run the selector and filter out calexps that were not selected
755  # primarily because they do not overlap the patch
756  cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
757  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
758  goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
759  inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
760 
761  # Read from disk only the selected calexps
762  inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
763 
764  # Extract integer visitId requested by `run`
765  visits = [dataId['visit'] for dataId in dataIdList]
766  visitId = visits[0]
767 
768  if self.config.doApplyExternalSkyWcs:
769  if self.config.useGlobalExternalSkyWcs:
770  externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
771  else:
772  externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
773  else:
774  externalSkyWcsCatalog = None
775 
776  if self.config.doApplyExternalPhotoCalib:
777  if self.config.useGlobalExternalPhotoCalib:
778  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
779  else:
780  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
781  else:
782  externalPhotoCalibCatalog = None
783 
784  completeIndices = self.prepareCalibratedExposures(**inputs,
785  externalSkyWcsCatalog=externalSkyWcsCatalog,
786  externalPhotoCalibCatalog=externalPhotoCalibCatalog)
787  # Redo the input selection with inputs with complete wcs/photocalib info.
788  inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
789 
790  results = self.run(**inputs, visitId=visitId,
791  ccdIdList=[ccdIdList[i] for i in goodIndices],
792  dataIdList=[dataIdList[i] for i in goodIndices],
793  skyInfo=skyInfo)
794  if self.config.makeDirect and results.exposures["direct"] is not None:
795  butlerQC.put(results.exposures["direct"], outputRefs.direct)
796  if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
797  butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
798 
799  def filterInputs(self, indices, inputs):
800  """Return task inputs with their lists filtered by indices
801 
802  Parameters
803  ----------
804  indices : `list` of integers
805  inputs : `dict` of `list` of input connections to be passed to run
806  """
807  for key in inputs.keys():
808  # Only down-select on list inputs
809  if isinstance(inputs[key], list):
810  inputs[key] = [inputs[key][ind] for ind in indices]
811  return inputs
812 
813  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
814  externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
815  **kwargs):
816  """Calibrate and add backgrounds to input calExpList in place
817 
818  Parameters
819  ----------
820  calExpList : `list` of `lsst.afw.image.Exposure`
821  Sequence of calexps to be modified in place
822  backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
823  Sequence of backgrounds to be added back in if bgSubtracted=False
824  skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
825  Sequence of background corrections to be subtracted if doApplySkyCorr=True
826  externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
827  Exposure catalog with external skyWcs to be applied
828  if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
829  for the catalog id, sorted on id for fast lookup.
830  externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
831  Exposure catalog with external photoCalib to be applied
832  if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
833  id for the catalog id, sorted on id for fast lookup.
834 
835  Returns
836  -------
837  indices : `list` [`int`]
838  Indices of calExpList and friends that have valid photoCalib/skyWcs
839  """
840  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
841  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
842 
843  includeCalibVar = self.config.includeCalibVar
844 
845  indices = []
846  for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
847  backgroundList,
848  skyCorrList)):
849  mi = calexp.maskedImage
850  if not self.config.bgSubtracted:
851  mi += background.getImage()
852 
853  if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
854  detectorId = calexp.getInfo().getDetector().getId()
855 
856  # Find the external photoCalib
857  if externalPhotoCalibCatalog is not None:
858  row = externalPhotoCalibCatalog.find(detectorId)
859  if row is None:
860  self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
861  "and will not be used in the warp.", detectorId)
862  continue
863  photoCalib = row.getPhotoCalib()
864  if photoCalib is None:
865  self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
866  "and will not be used in the warp.", detectorId)
867  continue
868  calexp.setPhotoCalib(photoCalib)
869  else:
870  photoCalib = calexp.getPhotoCalib()
871  if photoCalib is None:
872  self.log.warning("Detector id %s has None for photoCalib in the calexp "
873  "and will not be used in the warp.", detectorId)
874  continue
875 
876  # Find and apply external skyWcs
877  if externalSkyWcsCatalog is not None:
878  row = externalSkyWcsCatalog.find(detectorId)
879  if row is None:
880  self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
881  "and will not be used in the warp.", detectorId)
882  continue
883  skyWcs = row.getWcs()
884  if skyWcs is None:
885  self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
886  "and will not be used in the warp.", detectorId)
887  continue
888  calexp.setWcs(skyWcs)
889  else:
890  skyWcs = calexp.getWcs()
891  if skyWcs is None:
892  self.log.warning("Detector id %s has None for skyWcs in the calexp "
893  "and will not be used in the warp.", detectorId)
894  continue
895 
896  # Calibrate the image
897  calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
898  includeScaleUncertainty=includeCalibVar)
899  calexp.maskedImage /= photoCalib.getCalibrationMean()
900  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
901  # exposure.setCalib(afwImage.Calib(1.0))
902 
903  # Apply skycorr
904  if self.config.doApplySkyCorr:
905  mi -= skyCorr.getImage()
906 
907  indices.append(index)
908 
909  return indices
910 
911 
912 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
913  """Reorder inputRefs per outputSortKeyOrder
914 
915  Any inputRefs which are lists will be resorted per specified key e.g.,
916  'detector.' Only iterables will be reordered, and values can be of type
917  `lsst.pipe.base.connections.DeferredDatasetRef` or
918  `lsst.daf.butler.core.datasets.ref.DatasetRef`.
919  Returned lists of refs have the same length as the outputSortKeyOrder.
920  If an outputSortKey not in the inputRef, then it will be padded with None.
921  If an inputRef contains an inputSortKey that is not in the
922  outputSortKeyOrder it will be removed.
923 
924  Parameters
925  ----------
926  inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
927  Input references to be reordered and padded.
928  outputSortKeyOrder : iterable
929  Iterable of values to be compared with inputRef's dataId[dataIdKey]
930  dataIdKey : `str`
931  dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
932 
933  Returns:
934  --------
935  inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
936  Quantized Connection with sorted DatasetRef values sorted if iterable.
937  """
938  for connectionName, refs in inputRefs:
939  if isinstance(refs, Iterable):
940  if hasattr(refs[0], "dataId"):
941  inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
942  else:
943  inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
944  if inputSortKeyOrder != outputSortKeyOrder:
945  setattr(inputRefs, connectionName,
946  reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
947  return inputRefs
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60