lsst.pipe.tasks  21.0.0-125-g25893231+6d8227318b
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 
24 import lsst.pex.config as pexConfig
25 import lsst.daf.persistence as dafPersist
26 import lsst.afw.image as afwImage
27 import lsst.coadd.utils as coaddUtils
28 import lsst.pipe.base as pipeBase
29 import lsst.pipe.base.connectionTypes as connectionTypes
30 import lsst.log as log
31 import lsst.utils as utils
32 import lsst.geom
33 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34 from lsst.skymap import BaseSkyMap
35 from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36 from .warpAndPsfMatch import WarpAndPsfMatchTask
37 from .coaddHelpers import groupPatchExposures, getGroupDataRef
38 from collections.abc import Iterable
39 
40 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
41 
42 
43 class MissingExposureError(Exception):
44  """Raised when data cannot be retrieved for an exposure.
45  When processing patches, sometimes one exposure is missing; this lets us
46  distinguish bewteen that case, and other errors.
47  """
48  pass
49 
50 
51 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
52  """Config for MakeCoaddTempExpTask
53  """
54  warpAndPsfMatch = pexConfig.ConfigurableField(
55  target=WarpAndPsfMatchTask,
56  doc="Task to warp and PSF-match calexp",
57  )
58  doWrite = pexConfig.Field(
59  doc="persist <coaddName>Coadd_<warpType>Warp",
60  dtype=bool,
61  default=True,
62  )
63  bgSubtracted = pexConfig.Field(
64  doc="Work with a background subtracted calexp?",
65  dtype=bool,
66  default=True,
67  )
68  coaddPsf = pexConfig.ConfigField(
69  doc="Configuration for CoaddPsf",
70  dtype=CoaddPsfConfig,
71  )
72  makeDirect = pexConfig.Field(
73  doc="Make direct Warp/Coadds",
74  dtype=bool,
75  default=True,
76  )
77  makePsfMatched = pexConfig.Field(
78  doc="Make Psf-Matched Warp/Coadd?",
79  dtype=bool,
80  default=False,
81  )
82 
83  doWriteEmptyWarps = pexConfig.Field(
84  dtype=bool,
85  default=False,
86  doc="Write out warps even if they are empty"
87  )
88 
89  hasFakes = pexConfig.Field(
90  doc="Should be set to True if fake sources have been inserted into the input data.",
91  dtype=bool,
92  default=False,
93  )
94  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
95 
96  def validate(self):
97  CoaddBaseTask.ConfigClass.validate(self)
98  if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
99  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
100  if self.doPsfMatch:
101  # Backwards compatibility.
102  log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
103  self.makePsfMatchedmakePsfMatched = True
104  self.makeDirectmakeDirect = False
105 
106  def setDefaults(self):
107  CoaddBaseTask.ConfigClass.setDefaults(self)
108  self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
109 
110 
116 
117 
119  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
120 
121  @anchor MakeCoaddTempExpTask_
122 
123  @section pipe_tasks_makeCoaddTempExp_Contents Contents
124 
125  - @ref pipe_tasks_makeCoaddTempExp_Purpose
126  - @ref pipe_tasks_makeCoaddTempExp_Initialize
127  - @ref pipe_tasks_makeCoaddTempExp_IO
128  - @ref pipe_tasks_makeCoaddTempExp_Config
129  - @ref pipe_tasks_makeCoaddTempExp_Debug
130  - @ref pipe_tasks_makeCoaddTempExp_Example
131 
132  @section pipe_tasks_makeCoaddTempExp_Purpose Description
133 
134  Warp and optionally PSF-Match calexps onto a common projection, by
135  performing the following operations:
136  - Group calexps by visit/run
137  - For each visit, generate a Warp by calling method @ref makeTempExp.
138  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
139  on each visit
140 
141  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
142 
143  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
144 
145  @copydoc \_\_init\_\_
146 
147  This task has one special keyword argument: passing reuse=True will cause
148  the task to skip the creation of warps that are already present in the
149  output repositories.
150 
151  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
152 
153  This task is primarily designed to be run from the command line.
154 
155  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
156  to process.
157 
158  @copydoc run
159 
160  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
161  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
162  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
163  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
164 
165  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
166 
167  See @ref MakeCoaddTempExpConfig and parameters inherited from
168  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
169 
170  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
171 
172  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
173  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
174  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
175  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
176  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
177  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
178 
179  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
180  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
181  In general, for a set of input images, this config should equal the FWHM of the visit
182  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
183  of the other config options offer a reasonable starting point.
184  The following list presents the most common problems that arise from a misconfigured
185  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
186  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
187  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
188  Problem: Explanation. *Solution*
189 
190  *Troublshooting PSF-Matching Configuration:*
191  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
192  For example:_
193 
194  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
195 
196  Note that increasing the kernel size also increases runtime.
197  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
198  for matching kernel. _Provide the matcher with more data by either increasing
199  the spatial sampling by decreasing the spatial cell size,_
200 
201  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
202  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
203 
204  _or increasing the padding around the Science PSF, for example:_
205 
206  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
207 
208  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
209  matching kernel dimensions, thus increasing the number of pixels available to fit
210  after convolving the PSF with the matching kernel.
211  Optionally, for debugging the effects of padding, the level of padding may be manually
212  controlled by setting turning off the automatic padding and setting the number
213  of pixels by which to pad the PSF:
214 
215  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
216  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
217 
218  - Deconvolution: Matching a large PSF to a smaller PSF produces
219  a telltale noise pattern which looks like ripples or a brain.
220  _Increase the size of the requested model PSF. For example:_
221 
222  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
223 
224  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
225  _Increase the width of the Gaussian basis functions. For example:_
226 
227  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
228  # from default [0.7, 1.5, 3.0]
229 
230 
231  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
232 
233  MakeCoaddTempExpTask has no debug output, but its subtasks do.
234 
235  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
236 
237  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
238  into the larger Data Release Processing.
239  Set up by running:
240 
241  setup ci_hsc
242  cd $CI_HSC_DIR
243  # if not built already:
244  python $(which scons) # this will take a while
245 
246  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
247  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
248  output respository with the desired SkyMap. The command,
249 
250  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
251  --id patch=5,4 tract=0 filter=HSC-I \
252  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
253  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
254  --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
255  makePsfMatched=True modelPsf.defaultFwhm=11
256 
257  writes a direct and PSF-Matched Warp to
258  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
259  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
260  respectively.
261 
262  @note PSF-Matching in this particular dataset would benefit from adding
263  `--configfile ./matchingConfig.py` to
264  the command line arguments where `matchingConfig.py` is defined by:
265 
266  echo "
267  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
268  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
269 
270 
271  Add the option `--help` to see more options.
272  """
273  ConfigClass = MakeCoaddTempExpConfig
274  _DefaultName = "makeCoaddTempExp"
275 
276  def __init__(self, reuse=False, **kwargs):
277  CoaddBaseTask.__init__(self, **kwargs)
278  self.reusereuse = reuse
279  self.makeSubtask("warpAndPsfMatch")
280  if self.config.hasFakes:
281  self.calexpTypecalexpType = "fakes_calexp"
282  else:
283  self.calexpTypecalexpType = "calexp"
284 
285  @pipeBase.timeMethod
286  def runDataRef(self, patchRef, selectDataList=[]):
287  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
288 
289  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
290  plus the camera-specific filter key (e.g. "filter" or "band")
291  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
292  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
293  warps are requested.
294 
295  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
296 
297  @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
298  with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
299  (assembleCoadd should determine zeropoint scaling without referring to it).
300  """
301  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
302 
303  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
304  if self.config.makePsfMatched and not self.config.makeDirect:
305  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
306  else:
307  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
308 
309  calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
310 
311  if len(calExpRefList) == 0:
312  self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
313  return None
314  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
315  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
316  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
317 
318  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
319  primaryWarpDataset)
320  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
321 
322  dataRefList = []
323  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
324  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
325  tempExpTuple, groupData.keys)
326  if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
327  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
328  dataRefList.append(tempExpRef)
329  continue
330  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
331 
332  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
333  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
334  # of the visit in the list.
335  try:
336  visitId = int(tempExpRef.dataId["visit"])
337  except (KeyError, ValueError):
338  visitId = i
339 
340  calExpList = []
341  ccdIdList = []
342  dataIdList = []
343 
344  for calExpInd, calExpRef in enumerate(calexpRefList):
345  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
346  calExpRef.dataId)
347  try:
348  ccdId = calExpRef.get("ccdExposureId", immediate=True)
349  except Exception:
350  ccdId = calExpInd
351  try:
352  # We augment the dataRef here with the tract, which is harmless for loading things
353  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
354  # which do.
355  calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
356  dataId=calExpRef.dataId,
357  tract=skyInfo.tractInfo.getId())
358  calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
359  except Exception as e:
360  self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
361  continue
362 
363  if self.config.doApplySkyCorr:
364  self.applySkyCorrapplySkyCorr(calExpRef, calExp)
365 
366  calExpList.append(calExp)
367  ccdIdList.append(ccdId)
368  dataIdList.append(calExpRef.dataId)
369 
370  exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
371 
372  if any(exps.values()):
373  dataRefList.append(tempExpRef)
374  else:
375  self.log.warning("Warp %s could not be created", tempExpRef.dataId)
376 
377  if self.config.doWrite:
378  for (warpType, exposure) in exps.items(): # compatible w/ Py3
379  if exposure is not None:
380  self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
381  tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
382 
383  return dataRefList
384 
385  @pipeBase.timeMethod
386  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
387  """Create a Warp from inputs
388 
389  We iterate over the multiple calexps in a single exposure to construct
390  the warp (previously called a coaddTempExp) of that exposure to the
391  supplied tract/patch.
392 
393  Pixels that receive no pixels are set to NAN; this is not correct
394  (violates LSST algorithms group policy), but will be fixed up by
395  interpolating after the coaddition.
396 
397  @param calexpRefList: List of data references for calexps that (may)
398  overlap the patch of interest
399  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
400  information about the patch
401  @param visitId: integer identifier for visit, for the table that will
402  produce the CoaddPsf
403  @return a pipeBase Struct containing:
404  - exposures: a dictionary containing the warps requested:
405  "direct": direct warp if config.makeDirect
406  "psfMatched": PSF-matched warp if config.makePsfMatched
407  """
408  warpTypeList = self.getWarpTypeListgetWarpTypeList()
409 
410  totGoodPix = {warpType: 0 for warpType in warpTypeList}
411  didSetMetadata = {warpType: False for warpType in warpTypeList}
412  coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
413  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
414  for warpType in warpTypeList}
415 
416  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
417  if dataIdList is None:
418  dataIdList = ccdIdList
419 
420  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
421  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
422  calExpInd+1, len(calExpList), dataId)
423 
424  try:
425  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
426  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
427  makeDirect=self.config.makeDirect,
428  makePsfMatched=self.config.makePsfMatched)
429  except Exception as e:
430  self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
431  continue
432  try:
433  numGoodPix = {warpType: 0 for warpType in warpTypeList}
434  for warpType in warpTypeList:
435  exposure = warpedAndMatched.getDict()[warpType]
436  if exposure is None:
437  continue
438  coaddTempExp = coaddTempExps[warpType]
439  if didSetMetadata[warpType]:
440  mimg = exposure.getMaskedImage()
441  mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
442  / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
443  del mimg
444  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
445  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
446  totGoodPix[warpType] += numGoodPix[warpType]
447  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
448  dataId, numGoodPix[warpType],
449  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
450  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
451  coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
452  coaddTempExp.setFilterLabel(exposure.getFilterLabel())
453  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
454  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
455  coaddTempExp.setPsf(exposure.getPsf())
456  didSetMetadata[warpType] = True
457 
458  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
459  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
460 
461  except Exception as e:
462  self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
463  continue
464 
465  for warpType in warpTypeList:
466  self.log.info("%sWarp has %d good pixels (%.1f%%)",
467  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
468 
469  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
470  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
471  if warpType == "direct":
472  coaddTempExps[warpType].setPsf(
473  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
474  self.config.coaddPsf.makeControl()))
475  else:
476  if not self.config.doWriteEmptyWarps:
477  # No good pixels. Exposure still empty
478  coaddTempExps[warpType] = None
479  # NoWorkFound is unnecessary as the downstream tasks will
480  # adjust the quantum accordingly, and it prevents gen2
481  # MakeCoaddTempExp from continuing to loop over visits.
482 
483  result = pipeBase.Struct(exposures=coaddTempExps)
484  return result
485 
486  def getCalibratedExposure(self, dataRef, bgSubtracted):
487  """Return one calibrated Exposure, possibly with an updated SkyWcs.
488 
489  @param[in] dataRef a sensor-level data reference
490  @param[in] bgSubtracted return calexp with background subtracted? If False get the
491  calexp's background background model and add it to the calexp.
492  @return calibrated exposure
493 
494  @raises MissingExposureError If data for the exposure is not available.
495 
496  If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
497  (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
498  `name_photoCalib` dataset. Otherwise, the photometric calibration is
499  retrieved from the processed exposure. When
500  `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
501  is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
502  Otherwise, the astrometric calibration is taken from the processed
503  exposure.
504  """
505  try:
506  exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
507  except dafPersist.NoResults as e:
508  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
509 
510  if not bgSubtracted:
511  background = dataRef.get("calexpBackground", immediate=True)
512  mi = exposure.getMaskedImage()
513  mi += background.getImage()
514  del mi
515 
516  if self.config.doApplyExternalPhotoCalib:
517  source = f"{self.config.externalPhotoCalibName}_photoCalib"
518  self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
519  photoCalib = dataRef.get(source)
520  exposure.setPhotoCalib(photoCalib)
521  else:
522  photoCalib = exposure.getPhotoCalib()
523 
524  if self.config.doApplyExternalSkyWcs:
525  source = f"{self.config.externalSkyWcsName}_wcs"
526  self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
527  skyWcs = dataRef.get(source)
528  exposure.setWcs(skyWcs)
529 
530  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
531  includeScaleUncertainty=self.config.includeCalibVar)
532  exposure.maskedImage /= photoCalib.getCalibrationMean()
533  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
534  # exposure.setCalib(afwImage.Calib(1.0))
535  return exposure
536 
537  @staticmethod
538  def _prepareEmptyExposure(skyInfo):
539  """Produce an empty exposure for a given patch"""
540  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
541  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
542  .getPlaneBitMask("NO_DATA"), numpy.inf)
543  return exp
544 
545  def getWarpTypeList(self):
546  """Return list of requested warp types per the config.
547  """
548  warpTypeList = []
549  if self.config.makeDirect:
550  warpTypeList.append("direct")
551  if self.config.makePsfMatched:
552  warpTypeList.append("psfMatched")
553  return warpTypeList
554 
555  def applySkyCorr(self, dataRef, calexp):
556  """Apply correction to the sky background level
557 
558  Sky corrections can be generated with the 'skyCorrection.py'
559  executable in pipe_drivers. Because the sky model used by that
560  code extends over the entire focal plane, this can produce
561  better sky subtraction.
562 
563  The calexp is updated in-place.
564 
565  Parameters
566  ----------
567  dataRef : `lsst.daf.persistence.ButlerDataRef`
568  Data reference for calexp.
569  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
570  Calibrated exposure.
571  """
572  bg = dataRef.get("skyCorr")
573  self.log.debug("Applying sky correction to %s", dataRef.dataId)
574  if isinstance(calexp, afwImage.Exposure):
575  calexp = calexp.getMaskedImage()
576  calexp -= bg.getImage()
577 
578 
579 class MakeWarpConnections(pipeBase.PipelineTaskConnections,
580  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
581  defaultTemplates={"coaddName": "deep",
582  "skyWcsName": "jointcal",
583  "photoCalibName": "fgcm",
584  "calexpType": ""}):
585  calExpList = connectionTypes.Input(
586  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
587  name="{calexpType}calexp",
588  storageClass="ExposureF",
589  dimensions=("instrument", "visit", "detector"),
590  multiple=True,
591  deferLoad=True,
592  )
593  backgroundList = connectionTypes.Input(
594  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
595  name="calexpBackground",
596  storageClass="Background",
597  dimensions=("instrument", "visit", "detector"),
598  multiple=True,
599  )
600  skyCorrList = connectionTypes.Input(
601  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
602  name="skyCorr",
603  storageClass="Background",
604  dimensions=("instrument", "visit", "detector"),
605  multiple=True,
606  )
607  skyMap = connectionTypes.Input(
608  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
609  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
610  storageClass="SkyMap",
611  dimensions=("skymap",),
612  )
613  externalSkyWcsTractCatalog = connectionTypes.Input(
614  doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
615  "id for the catalog id, sorted on id for fast lookup."),
616  name="{skyWcsName}SkyWcsCatalog",
617  storageClass="ExposureCatalog",
618  dimensions=("instrument", "visit", "tract"),
619  )
620  externalSkyWcsGlobalCatalog = connectionTypes.Input(
621  doc=("Per-visit wcs calibrations computed globally (with no tract information). "
622  "These catalogs use the detector id for the catalog id, sorted on id for "
623  "fast lookup."),
624  name="{skyWcsName}SkyWcsCatalog",
625  storageClass="ExposureCatalog",
626  dimensions=("instrument", "visit"),
627  )
628  externalPhotoCalibTractCatalog = connectionTypes.Input(
629  doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
630  "detector id for the catalog id, sorted on id for fast lookup."),
631  name="{photoCalibName}PhotoCalibCatalog",
632  storageClass="ExposureCatalog",
633  dimensions=("instrument", "visit", "tract"),
634  )
635  externalPhotoCalibGlobalCatalog = connectionTypes.Input(
636  doc=("Per-visit photometric calibrations computed globally (with no tract "
637  "information). These catalogs use the detector id for the catalog id, "
638  "sorted on id for fast lookup."),
639  name="{photoCalibName}PhotoCalibCatalog",
640  storageClass="ExposureCatalog",
641  dimensions=("instrument", "visit"),
642  )
643  direct = connectionTypes.Output(
644  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
645  "calexps onto the skyMap patch geometry."),
646  name="{coaddName}Coadd_directWarp",
647  storageClass="ExposureF",
648  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
649  )
650  psfMatched = connectionTypes.Output(
651  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
652  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
653  name="{coaddName}Coadd_psfMatchedWarp",
654  storageClass="ExposureF",
655  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
656  )
657  # TODO DM-28769, have selectImages subtask indicate which connections they need:
658  wcsList = connectionTypes.Input(
659  doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
660  name="{calexpType}calexp.wcs",
661  storageClass="Wcs",
662  dimensions=("instrument", "visit", "detector"),
663  multiple=True,
664  )
665  bboxList = connectionTypes.Input(
666  doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
667  name="{calexpType}calexp.bbox",
668  storageClass="Box2I",
669  dimensions=("instrument", "visit", "detector"),
670  multiple=True,
671  )
672  srcList = connectionTypes.Input(
673  doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
674  name="src",
675  storageClass="SourceCatalog",
676  dimensions=("instrument", "visit", "detector"),
677  multiple=True,
678  )
679  psfList = connectionTypes.Input(
680  doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
681  name="{calexpType}calexp.psf",
682  storageClass="Psf",
683  dimensions=("instrument", "visit", "detector"),
684  multiple=True,
685  )
686 
687  def __init__(self, *, config=None):
688  super().__init__(config=config)
689  if config.bgSubtracted:
690  self.inputs.remove("backgroundList")
691  if not config.doApplySkyCorr:
692  self.inputs.remove("skyCorrList")
693  if config.doApplyExternalSkyWcs:
694  if config.useGlobalExternalSkyWcs:
695  self.inputs.remove("externalSkyWcsTractCatalog")
696  else:
697  self.inputs.remove("externalSkyWcsGlobalCatalog")
698  else:
699  self.inputs.remove("externalSkyWcsTractCatalog")
700  self.inputs.remove("externalSkyWcsGlobalCatalog")
701  if config.doApplyExternalPhotoCalib:
702  if config.useGlobalExternalPhotoCalib:
703  self.inputs.remove("externalPhotoCalibTractCatalog")
704  else:
705  self.inputs.remove("externalPhotoCalibGlobalCatalog")
706  else:
707  self.inputs.remove("externalPhotoCalibTractCatalog")
708  self.inputs.remove("externalPhotoCalibGlobalCatalog")
709  if not config.makeDirect:
710  self.outputs.remove("direct")
711  if not config.makePsfMatched:
712  self.outputs.remove("psfMatched")
713  # TODO DM-28769: add connection per selectImages connections
714  # instead of removing if not PsfWcsSelectImagesTask here:
715  if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
716  self.inputs.remove("srcList")
717  if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
718  self.inputs.remove("psfList")
719 
720 
721 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
722  pipelineConnections=MakeWarpConnections):
723 
724  def validate(self):
725  super().validate()
726 
727 
728 class MakeWarpTask(MakeCoaddTempExpTask):
729  """Warp and optionally PSF-Match calexps onto an a common projection
730  """
731  ConfigClass = MakeWarpConfig
732  _DefaultName = "makeWarp"
733 
734  @utils.inheritDoc(pipeBase.PipelineTask)
735  def runQuantum(self, butlerQC, inputRefs, outputRefs):
736  """
737  Notes
738  ----
739  Construct warps for requested warp type for single epoch
740 
741  PipelineTask (Gen3) entry point to warp and optionally PSF-match
742  calexps. This method is analogous to `runDataRef`.
743  """
744 
745  # Ensure all input lists are in same detector order as the calExpList
746  detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
747  inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
748 
749  # Read in all inputs.
750  inputs = butlerQC.get(inputRefs)
751 
752  # Construct skyInfo expected by `run`. We remove the SkyMap itself
753  # from the dictionary so we can pass it as kwargs later.
754  skyMap = inputs.pop("skyMap")
755  quantumDataId = butlerQC.quantum.dataId
756  skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
757 
758  # Construct list of input DataIds expected by `run`
759  dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
760  # Construct list of packed integer IDs expected by `run`
761  ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
762 
763  # Run the selector and filter out calexps that were not selected
764  # primarily because they do not overlap the patch
765  cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
766  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
767  goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
768  inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
769 
770  # Read from disk only the selected calexps
771  inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
772 
773  # Extract integer visitId requested by `run`
774  visits = [dataId['visit'] for dataId in dataIdList]
775  visitId = visits[0]
776 
777  if self.config.doApplyExternalSkyWcs:
778  if self.config.useGlobalExternalSkyWcs:
779  externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
780  else:
781  externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
782  else:
783  externalSkyWcsCatalog = None
784 
785  if self.config.doApplyExternalPhotoCalib:
786  if self.config.useGlobalExternalPhotoCalib:
787  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
788  else:
789  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
790  else:
791  externalPhotoCalibCatalog = None
792 
793  completeIndices = self.prepareCalibratedExposures(**inputs,
794  externalSkyWcsCatalog=externalSkyWcsCatalog,
795  externalPhotoCalibCatalog=externalPhotoCalibCatalog)
796  # Redo the input selection with inputs with complete wcs/photocalib info.
797  inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
798 
799  results = self.run(**inputs, visitId=visitId,
800  ccdIdList=[ccdIdList[i] for i in goodIndices],
801  dataIdList=[dataIdList[i] for i in goodIndices],
802  skyInfo=skyInfo)
803  if self.config.makeDirect and results.exposures["direct"] is not None:
804  butlerQC.put(results.exposures["direct"], outputRefs.direct)
805  if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
806  butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
807 
808  def filterInputs(self, indices, inputs):
809  """Return task inputs with their lists filtered by indices
810 
811  Parameters
812  ----------
813  indices : `list` of integers
814  inputs : `dict` of `list` of input connections to be passed to run
815  """
816  for key in inputs.keys():
817  # Only down-select on list inputs
818  if isinstance(inputs[key], list):
819  inputs[key] = [inputs[key][ind] for ind in indices]
820  return inputs
821 
822  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
823  externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
824  **kwargs):
825  """Calibrate and add backgrounds to input calExpList in place
826 
827  Parameters
828  ----------
829  calExpList : `list` of `lsst.afw.image.Exposure`
830  Sequence of calexps to be modified in place
831  backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
832  Sequence of backgrounds to be added back in if bgSubtracted=False
833  skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
834  Sequence of background corrections to be subtracted if doApplySkyCorr=True
835  externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
836  Exposure catalog with external skyWcs to be applied
837  if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
838  for the catalog id, sorted on id for fast lookup.
839  externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
840  Exposure catalog with external photoCalib to be applied
841  if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
842  id for the catalog id, sorted on id for fast lookup.
843 
844  Returns
845  -------
846  indices : `list` [`int`]
847  Indices of calExpList and friends that have valid photoCalib/skyWcs
848  """
849  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
850  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
851 
852  includeCalibVar = self.config.includeCalibVar
853 
854  indices = []
855  for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
856  backgroundList,
857  skyCorrList)):
858  mi = calexp.maskedImage
859  if not self.config.bgSubtracted:
860  mi += background.getImage()
861 
862  if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
863  detectorId = calexp.getInfo().getDetector().getId()
864 
865  # Find the external photoCalib
866  if externalPhotoCalibCatalog is not None:
867  row = externalPhotoCalibCatalog.find(detectorId)
868  if row is None:
869  self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
870  "and will not be used in the warp.", detectorId)
871  continue
872  photoCalib = row.getPhotoCalib()
873  if photoCalib is None:
874  self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
875  "and will not be used in the warp.", detectorId)
876  continue
877  calexp.setPhotoCalib(photoCalib)
878  else:
879  photoCalib = calexp.getPhotoCalib()
880  if photoCalib is None:
881  self.log.warning("Detector id %s has None for photoCalib in the calexp "
882  "and will not be used in the warp.", detectorId)
883  continue
884 
885  # Find and apply external skyWcs
886  if externalSkyWcsCatalog is not None:
887  row = externalSkyWcsCatalog.find(detectorId)
888  if row is None:
889  self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
890  "and will not be used in the warp.", detectorId)
891  continue
892  skyWcs = row.getWcs()
893  if skyWcs is None:
894  self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
895  "and will not be used in the warp.", detectorId)
896  continue
897  calexp.setWcs(skyWcs)
898  else:
899  skyWcs = calexp.getWcs()
900  if skyWcs is None:
901  self.log.warning("Detector id %s has None for skyWcs in the calexp "
902  "and will not be used in the warp.", detectorId)
903  continue
904 
905  # Calibrate the image
906  calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
907  includeScaleUncertainty=includeCalibVar)
908  calexp.maskedImage /= photoCalib.getCalibrationMean()
909  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
910  # exposure.setCalib(afwImage.Calib(1.0))
911 
912  # Apply skycorr
913  if self.config.doApplySkyCorr:
914  mi -= skyCorr.getImage()
915 
916  indices.append(index)
917 
918  return indices
919 
920 
921 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
922  """Reorder inputRefs per outputSortKeyOrder
923 
924  Any inputRefs which are lists will be resorted per specified key e.g.,
925  'detector.' Only iterables will be reordered, and values can be of type
926  `lsst.pipe.base.connections.DeferredDatasetRef` or
927  `lsst.daf.butler.core.datasets.ref.DatasetRef`.
928  Returned lists of refs have the same length as the outputSortKeyOrder.
929  If an outputSortKey not in the inputRef, then it will be padded with None.
930  If an inputRef contains an inputSortKey that is not in the
931  outputSortKeyOrder it will be removed.
932 
933  Parameters
934  ----------
935  inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
936  Input references to be reordered and padded.
937  outputSortKeyOrder : iterable
938  Iterable of values to be compared with inputRef's dataId[dataIdKey]
939  dataIdKey : `str`
940  dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
941 
942  Returns:
943  --------
944  inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
945  Quantized Connection with sorted DatasetRef values sorted if iterable.
946  """
947  for connectionName, refs in inputRefs:
948  if isinstance(refs, Iterable):
949  if hasattr(refs[0], "dataId"):
950  inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
951  else:
952  inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
953  if inputSortKeyOrder != outputSortKeyOrder:
954  setattr(inputRefs, connectionName,
955  reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
956  return inputRefs
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60