lsst.pipe.tasks  21.0.0-51-gd3b42663+3149e5afc4
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy
23 
24 import lsst.pex.config as pexConfig
25 import lsst.daf.persistence as dafPersist
26 import lsst.afw.image as afwImage
27 import lsst.coadd.utils as coaddUtils
28 import lsst.pipe.base as pipeBase
29 import lsst.pipe.base.connectionTypes as connectionTypes
30 import lsst.log as log
31 import lsst.utils as utils
32 import lsst.geom
33 from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34 from lsst.skymap import BaseSkyMap
35 from .coaddBase import CoaddBaseTask, makeSkyInfo
36 from .warpAndPsfMatch import WarpAndPsfMatchTask
37 from .coaddHelpers import groupPatchExposures, getGroupDataRef
38 
39 __all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
40 
41 
42 class MissingExposureError(Exception):
43  """Raised when data cannot be retrieved for an exposure.
44  When processing patches, sometimes one exposure is missing; this lets us
45  distinguish bewteen that case, and other errors.
46  """
47  pass
48 
49 
50 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
51  """Config for MakeCoaddTempExpTask
52  """
53  warpAndPsfMatch = pexConfig.ConfigurableField(
54  target=WarpAndPsfMatchTask,
55  doc="Task to warp and PSF-match calexp",
56  )
57  doWrite = pexConfig.Field(
58  doc="persist <coaddName>Coadd_<warpType>Warp",
59  dtype=bool,
60  default=True,
61  )
62  bgSubtracted = pexConfig.Field(
63  doc="Work with a background subtracted calexp?",
64  dtype=bool,
65  default=True,
66  )
67  coaddPsf = pexConfig.ConfigField(
68  doc="Configuration for CoaddPsf",
69  dtype=CoaddPsfConfig,
70  )
71  makeDirect = pexConfig.Field(
72  doc="Make direct Warp/Coadds",
73  dtype=bool,
74  default=True,
75  )
76  makePsfMatched = pexConfig.Field(
77  doc="Make Psf-Matched Warp/Coadd?",
78  dtype=bool,
79  default=False,
80  )
81 
82  doWriteEmptyWarps = pexConfig.Field(
83  dtype=bool,
84  default=False,
85  doc="Write out warps even if they are empty"
86  )
87 
88  hasFakes = pexConfig.Field(
89  doc="Should be set to True if fake sources have been inserted into the input data.",
90  dtype=bool,
91  default=False,
92  )
93  doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
94 
95  def validate(self):
96  CoaddBaseTask.ConfigClass.validate(self)
97  if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
98  raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
99  if self.doPsfMatch:
100  # Backwards compatibility.
101  log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
102  self.makePsfMatchedmakePsfMatched = True
103  self.makeDirectmakeDirect = False
104 
105  def setDefaults(self):
106  CoaddBaseTask.ConfigClass.setDefaults(self)
107  self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
108 
109 
115 
116 
118  r"""!Warp and optionally PSF-Match calexps onto an a common projection.
119 
120  @anchor MakeCoaddTempExpTask_
121 
122  @section pipe_tasks_makeCoaddTempExp_Contents Contents
123 
124  - @ref pipe_tasks_makeCoaddTempExp_Purpose
125  - @ref pipe_tasks_makeCoaddTempExp_Initialize
126  - @ref pipe_tasks_makeCoaddTempExp_IO
127  - @ref pipe_tasks_makeCoaddTempExp_Config
128  - @ref pipe_tasks_makeCoaddTempExp_Debug
129  - @ref pipe_tasks_makeCoaddTempExp_Example
130 
131  @section pipe_tasks_makeCoaddTempExp_Purpose Description
132 
133  Warp and optionally PSF-Match calexps onto a common projection, by
134  performing the following operations:
135  - Group calexps by visit/run
136  - For each visit, generate a Warp by calling method @ref makeTempExp.
137  makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
138  on each visit
139 
140  The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
141 
142  @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
143 
144  @copydoc \_\_init\_\_
145 
146  This task has one special keyword argument: passing reuse=True will cause
147  the task to skip the creation of warps that are already present in the
148  output repositories.
149 
150  @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
151 
152  This task is primarily designed to be run from the command line.
153 
154  The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
155  to process.
156 
157  @copydoc run
158 
159  WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
160  Only two types are available: direct (for regular Warps/Coadds) and psfMatched
161  (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
162  for generating likelihood Coadds with Warps that have been correlated with their own PSF.
163 
164  @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
165 
166  See @ref MakeCoaddTempExpConfig and parameters inherited from
167  @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
168 
169  @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
170 
171  To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
172  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
173  is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
174  The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
175  dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
176  padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
177 
178  *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
179  `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
180  In general, for a set of input images, this config should equal the FWHM of the visit
181  with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
182  of the other config options offer a reasonable starting point.
183  The following list presents the most common problems that arise from a misconfigured
184  @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
185  and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
186  ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
187  Problem: Explanation. *Solution*
188 
189  *Troublshooting PSF-Matching Configuration:*
190  - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
191  For example:_
192 
193  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
194 
195  Note that increasing the kernel size also increases runtime.
196  - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
197  for matching kernel. _Provide the matcher with more data by either increasing
198  the spatial sampling by decreasing the spatial cell size,_
199 
200  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
201  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
202 
203  _or increasing the padding around the Science PSF, for example:_
204 
205  config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
206 
207  Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
208  matching kernel dimensions, thus increasing the number of pixels available to fit
209  after convolving the PSF with the matching kernel.
210  Optionally, for debugging the effects of padding, the level of padding may be manually
211  controlled by setting turning off the automatic padding and setting the number
212  of pixels by which to pad the PSF:
213 
214  config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
215  config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
216 
217  - Deconvolution: Matching a large PSF to a smaller PSF produces
218  a telltale noise pattern which looks like ripples or a brain.
219  _Increase the size of the requested model PSF. For example:_
220 
221  config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
222 
223  - High frequency (sometimes checkered) noise: The matching basis functions are too small.
224  _Increase the width of the Gaussian basis functions. For example:_
225 
226  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
227  # from default [0.7, 1.5, 3.0]
228 
229 
230  @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
231 
232  MakeCoaddTempExpTask has no debug output, but its subtasks do.
233 
234  @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
235 
236  This example uses the package ci_hsc to show how MakeCoaddTempExp fits
237  into the larger Data Release Processing.
238  Set up by running:
239 
240  setup ci_hsc
241  cd $CI_HSC_DIR
242  # if not built already:
243  python $(which scons) # this will take a while
244 
245  The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
246  (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
247  output respository with the desired SkyMap. The command,
248 
249  makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
250  --id patch=5,4 tract=0 filter=HSC-I \
251  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
252  --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
253  --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
254  makePsfMatched=True modelPsf.defaultFwhm=11
255 
256  writes a direct and PSF-Matched Warp to
257  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
258  - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
259  respectively.
260 
261  @note PSF-Matching in this particular dataset would benefit from adding
262  `--configfile ./matchingConfig.py` to
263  the command line arguments where `matchingConfig.py` is defined by:
264 
265  echo "
266  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
267  config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
268 
269 
270  Add the option `--help` to see more options.
271  """
272  ConfigClass = MakeCoaddTempExpConfig
273  _DefaultName = "makeCoaddTempExp"
274 
275  def __init__(self, reuse=False, **kwargs):
276  CoaddBaseTask.__init__(self, **kwargs)
277  self.reusereuse = reuse
278  self.makeSubtask("warpAndPsfMatch")
279  if self.config.hasFakes:
280  self.calexpTypecalexpType = "fakes_calexp"
281  else:
282  self.calexpTypecalexpType = "calexp"
283 
284  @pipeBase.timeMethod
285  def runDataRef(self, patchRef, selectDataList=[]):
286  """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
287 
288  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
289  plus the camera-specific filter key (e.g. "filter" or "band")
290  @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
291  if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
292  warps are requested.
293 
294  @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
295 
296  @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
297  with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
298  (assembleCoadd should determine zeropoint scaling without referring to it).
299  """
300  skyInfo = self.getSkyInfogetSkyInfo(patchRef)
301 
302  # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
303  if self.config.makePsfMatched and not self.config.makeDirect:
304  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
305  else:
306  primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
307 
308  calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
309 
310  if len(calExpRefList) == 0:
311  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
312  return None
313  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
314  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
315  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
316 
317  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
318  primaryWarpDataset)
319  self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
320 
321  dataRefList = []
322  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
323  tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
324  tempExpTuple, groupData.keys)
325  if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
326  self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
327  dataRefList.append(tempExpRef)
328  continue
329  self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
330 
331  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
332  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
333  # of the visit in the list.
334  try:
335  visitId = int(tempExpRef.dataId["visit"])
336  except (KeyError, ValueError):
337  visitId = i
338 
339  calExpList = []
340  ccdIdList = []
341  dataIdList = []
342 
343  for calExpInd, calExpRef in enumerate(calexpRefList):
344  self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
345  calExpRef.dataId)
346  try:
347  ccdId = calExpRef.get("ccdExposureId", immediate=True)
348  except Exception:
349  ccdId = calExpInd
350  try:
351  # We augment the dataRef here with the tract, which is harmless for loading things
352  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
353  # which do.
354  calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
355  dataId=calExpRef.dataId,
356  tract=skyInfo.tractInfo.getId())
357  calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
358  except Exception as e:
359  self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
360  continue
361 
362  if self.config.doApplySkyCorr:
363  self.applySkyCorrapplySkyCorr(calExpRef, calExp)
364 
365  calExpList.append(calExp)
366  ccdIdList.append(ccdId)
367  dataIdList.append(calExpRef.dataId)
368 
369  exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
370 
371  if any(exps.values()):
372  dataRefList.append(tempExpRef)
373  else:
374  self.log.warn("Warp %s could not be created", tempExpRef.dataId)
375 
376  if self.config.doWrite:
377  for (warpType, exposure) in exps.items(): # compatible w/ Py3
378  if exposure is not None:
379  self.log.info("Persisting %s" % self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
380  tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
381 
382  return dataRefList
383 
384  @pipeBase.timeMethod
385  def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
386  """Create a Warp from inputs
387 
388  We iterate over the multiple calexps in a single exposure to construct
389  the warp (previously called a coaddTempExp) of that exposure to the
390  supplied tract/patch.
391 
392  Pixels that receive no pixels are set to NAN; this is not correct
393  (violates LSST algorithms group policy), but will be fixed up by
394  interpolating after the coaddition.
395 
396  @param calexpRefList: List of data references for calexps that (may)
397  overlap the patch of interest
398  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
399  information about the patch
400  @param visitId: integer identifier for visit, for the table that will
401  produce the CoaddPsf
402  @return a pipeBase Struct containing:
403  - exposures: a dictionary containing the warps requested:
404  "direct": direct warp if config.makeDirect
405  "psfMatched": PSF-matched warp if config.makePsfMatched
406  """
407  warpTypeList = self.getWarpTypeListgetWarpTypeList()
408 
409  totGoodPix = {warpType: 0 for warpType in warpTypeList}
410  didSetMetadata = {warpType: False for warpType in warpTypeList}
411  coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
412  inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
413  for warpType in warpTypeList}
414 
415  modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
416  if dataIdList is None:
417  dataIdList = ccdIdList
418 
419  for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
420  self.log.info("Processing calexp %d of %d for this Warp: id=%s",
421  calExpInd+1, len(calExpList), dataId)
422 
423  try:
424  warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
425  wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
426  makeDirect=self.config.makeDirect,
427  makePsfMatched=self.config.makePsfMatched)
428  except Exception as e:
429  self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
430  continue
431  try:
432  numGoodPix = {warpType: 0 for warpType in warpTypeList}
433  for warpType in warpTypeList:
434  exposure = warpedAndMatched.getDict()[warpType]
435  if exposure is None:
436  continue
437  coaddTempExp = coaddTempExps[warpType]
438  if didSetMetadata[warpType]:
439  mimg = exposure.getMaskedImage()
440  mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
441  / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
442  del mimg
443  numGoodPix[warpType] = coaddUtils.copyGoodPixels(
444  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
445  totGoodPix[warpType] += numGoodPix[warpType]
446  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
447  dataId, numGoodPix[warpType],
448  100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
449  if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
450  coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
451  coaddTempExp.setFilterLabel(exposure.getFilterLabel())
452  coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
453  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
454  coaddTempExp.setPsf(exposure.getPsf())
455  didSetMetadata[warpType] = True
456 
457  # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
458  inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
459 
460  except Exception as e:
461  self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
462  continue
463 
464  for warpType in warpTypeList:
465  self.log.info("%sWarp has %d good pixels (%.1f%%)",
466  warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
467 
468  if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
469  inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
470  if warpType == "direct":
471  coaddTempExps[warpType].setPsf(
472  CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
473  self.config.coaddPsf.makeControl()))
474  else:
475  if not self.config.doWriteEmptyWarps:
476  # No good pixels. Exposure still empty
477  coaddTempExps[warpType] = None
478 
479  result = pipeBase.Struct(exposures=coaddTempExps)
480  return result
481 
482  def getCalibratedExposure(self, dataRef, bgSubtracted):
483  """Return one calibrated Exposure, possibly with an updated SkyWcs.
484 
485  @param[in] dataRef a sensor-level data reference
486  @param[in] bgSubtracted return calexp with background subtracted? If False get the
487  calexp's background background model and add it to the calexp.
488  @return calibrated exposure
489 
490  @raises MissingExposureError If data for the exposure is not available.
491 
492  If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
493  (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
494  `name_photoCalib` dataset. Otherwise, the photometric calibration is
495  retrieved from the processed exposure. When
496  `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
497  is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
498  Otherwise, the astrometric calibration is taken from the processed
499  exposure.
500  """
501  try:
502  exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
503  except dafPersist.NoResults as e:
504  raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
505 
506  if not bgSubtracted:
507  background = dataRef.get("calexpBackground", immediate=True)
508  mi = exposure.getMaskedImage()
509  mi += background.getImage()
510  del mi
511 
512  if self.config.doApplyExternalPhotoCalib:
513  source = f"{self.config.externalPhotoCalibName}_photoCalib"
514  self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
515  photoCalib = dataRef.get(source)
516  exposure.setPhotoCalib(photoCalib)
517  else:
518  photoCalib = exposure.getPhotoCalib()
519 
520  if self.config.doApplyExternalSkyWcs:
521  source = f"{self.config.externalSkyWcsName}_wcs"
522  self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
523  skyWcs = dataRef.get(source)
524  exposure.setWcs(skyWcs)
525 
526  exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
527  includeScaleUncertainty=self.config.includeCalibVar)
528  exposure.maskedImage /= photoCalib.getCalibrationMean()
529  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
530  # exposure.setCalib(afwImage.Calib(1.0))
531  return exposure
532 
533  @staticmethod
534  def _prepareEmptyExposure(skyInfo):
535  """Produce an empty exposure for a given patch"""
536  exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
537  exp.getMaskedImage().set(numpy.nan, afwImage.Mask
538  .getPlaneBitMask("NO_DATA"), numpy.inf)
539  return exp
540 
541  def getWarpTypeList(self):
542  """Return list of requested warp types per the config.
543  """
544  warpTypeList = []
545  if self.config.makeDirect:
546  warpTypeList.append("direct")
547  if self.config.makePsfMatched:
548  warpTypeList.append("psfMatched")
549  return warpTypeList
550 
551  def applySkyCorr(self, dataRef, calexp):
552  """Apply correction to the sky background level
553 
554  Sky corrections can be generated with the 'skyCorrection.py'
555  executable in pipe_drivers. Because the sky model used by that
556  code extends over the entire focal plane, this can produce
557  better sky subtraction.
558 
559  The calexp is updated in-place.
560 
561  Parameters
562  ----------
563  dataRef : `lsst.daf.persistence.ButlerDataRef`
564  Data reference for calexp.
565  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
566  Calibrated exposure.
567  """
568  bg = dataRef.get("skyCorr")
569  self.log.debug("Applying sky correction to %s", dataRef.dataId)
570  if isinstance(calexp, afwImage.Exposure):
571  calexp = calexp.getMaskedImage()
572  calexp -= bg.getImage()
573 
574 
575 class MakeWarpConnections(pipeBase.PipelineTaskConnections,
576  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
577  defaultTemplates={"coaddName": "deep",
578  "skyWcsName": "jointcal",
579  "photoCalibName": "fgcmcal"}):
580  calExpList = connectionTypes.Input(
581  doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
582  name="calexp",
583  storageClass="ExposureF",
584  dimensions=("instrument", "visit", "detector"),
585  multiple=True,
586  deferLoad=True,
587  )
588  backgroundList = connectionTypes.Input(
589  doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
590  name="calexpBackground",
591  storageClass="Background",
592  dimensions=("instrument", "visit", "detector"),
593  multiple=True,
594  )
595  skyCorrList = connectionTypes.Input(
596  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
597  name="skyCorr",
598  storageClass="Background",
599  dimensions=("instrument", "visit", "detector"),
600  multiple=True,
601  )
602  skyMap = connectionTypes.Input(
603  doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
604  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
605  storageClass="SkyMap",
606  dimensions=("skymap",),
607  )
608  externalSkyWcsTractCatalog = connectionTypes.Input(
609  doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
610  "id for the catalog id, sorted on id for fast lookup."),
611  name="{skyWcsName}SkyWcsCatalog",
612  storageClass="ExposureCatalog",
613  dimensions=("instrument", "visit", "tract"),
614  )
615  externalSkyWcsGlobalCatalog = connectionTypes.Input(
616  doc=("Per-visit wcs calibrations computed globally (with no tract information). "
617  "These catalogs use the detector id for the catalog id, sorted on id for "
618  "fast lookup."),
619  name="{skyWcsName}SkyWcsCatalog",
620  storageClass="ExposureCatalog",
621  dimensions=("instrument", "visit"),
622  )
623  externalPhotoCalibTractCatalog = connectionTypes.Input(
624  doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
625  "detector id for the catalog id, sorted on id for fast lookup."),
626  name="{photoCalibName}PhotoCalibCatalog",
627  storageClass="ExposureCatalog",
628  dimensions=("instrument", "visit", "tract"),
629  )
630  externalPhotoCalibGlobalCatalog = connectionTypes.Input(
631  doc=("Per-visit photometric calibrations computed globally (with no tract "
632  "information). These catalogs use the detector id for the catalog id, "
633  "sorted on id for fast lookup."),
634  name="{photoCalibName}PhotoCalibCatalog",
635  storageClass="ExposureCatalog",
636  dimensions=("instrument", "visit"),
637  )
638  direct = connectionTypes.Output(
639  doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
640  "calexps onto the skyMap patch geometry."),
641  name="{coaddName}Coadd_directWarp",
642  storageClass="ExposureF",
643  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
644  )
645  psfMatched = connectionTypes.Output(
646  doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
647  "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
648  name="{coaddName}Coadd_psfMatchedWarp",
649  storageClass="ExposureF",
650  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
651  )
652  # TODO DM-28769, have selectImages subtask indicate which connections they need:
653  wcsList = connectionTypes.Input(
654  doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
655  name="calexp.wcs",
656  storageClass="Wcs",
657  dimensions=("instrument", "visit", "detector"),
658  multiple=True,
659  )
660  bboxList = connectionTypes.Input(
661  doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
662  name="calexp.bbox",
663  storageClass="Box2I",
664  dimensions=("instrument", "visit", "detector"),
665  multiple=True,
666  )
667  srcList = connectionTypes.Input(
668  doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
669  name="src",
670  storageClass="SourceCatalog",
671  dimensions=("instrument", "visit", "detector"),
672  multiple=True,
673  )
674  psfList = connectionTypes.Input(
675  doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
676  name="calexp.psf",
677  storageClass="Psf",
678  dimensions=("instrument", "visit", "detector"),
679  multiple=True,
680  )
681 
682  def __init__(self, *, config=None):
683  super().__init__(config=config)
684  if config.bgSubtracted:
685  self.inputs.remove("backgroundList")
686  if not config.doApplySkyCorr:
687  self.inputs.remove("skyCorrList")
688  if config.doApplyExternalSkyWcs:
689  if config.useGlobalExternalSkyWcs:
690  self.inputs.remove("externalSkyWcsTractCatalog")
691  else:
692  self.inputs.remove("externalSkyWcsGlobalCatalog")
693  else:
694  self.inputs.remove("externalSkyWcsTractCatalog")
695  self.inputs.remove("externalSkyWcsGlobalCatalog")
696  if config.doApplyExternalPhotoCalib:
697  if config.useGlobalExternalPhotoCalib:
698  self.inputs.remove("externalPhotoCalibTractCatalog")
699  else:
700  self.inputs.remove("externalPhotoCalibGlobalCatalog")
701  else:
702  self.inputs.remove("externalPhotoCalibTractCatalog")
703  self.inputs.remove("externalPhotoCalibGlobalCatalog")
704  if not config.makeDirect:
705  self.outputs.remove("direct")
706  if not config.makePsfMatched:
707  self.outputs.remove("psfMatched")
708  # TODO DM-28769: add connection per selectImages connections
709  # instead of removing if not PsfWcsSelectImagesTask here:
710  if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
711  self.inputs.remove("srcList")
712  if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
713  self.inputs.remove("psfList")
714 
715 
716 class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
717  pipelineConnections=MakeWarpConnections):
718 
719  def validate(self):
720  super().validate()
721 
722 
723 class MakeWarpTask(MakeCoaddTempExpTask):
724  """Warp and optionally PSF-Match calexps onto an a common projection
725  """
726  ConfigClass = MakeWarpConfig
727  _DefaultName = "makeWarp"
728 
729  @utils.inheritDoc(pipeBase.PipelineTask)
730  def runQuantum(self, butlerQC, inputRefs, outputRefs):
731  """
732  Notes
733  ----
734  Construct warps for requested warp type for single epoch
735 
736  PipelineTask (Gen3) entry point to warp and optionally PSF-match
737  calexps. This method is analogous to `runDataRef`.
738  """
739  # Read in all inputs.
740  inputs = butlerQC.get(inputRefs)
741 
742  # Construct skyInfo expected by `run`. We remove the SkyMap itself
743  # from the dictionary so we can pass it as kwargs later.
744  skyMap = inputs.pop("skyMap")
745  quantumDataId = butlerQC.quantum.dataId
746  skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
747 
748  # Construct list of input DataIds expected by `run`
749  dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
750  # Construct list of packed integer IDs expected by `run`
751  ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
752 
753  # Run the selector and filter out calexps that were not selected
754  # primarily because they do not overlap the patch
755  cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
756  coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
757  goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
758  inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
759 
760  # Read from disk only the selected calexps
761  inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
762 
763  # Extract integer visitId requested by `run`
764  visits = [dataId['visit'] for dataId in dataIdList]
765  visitId = visits[0]
766 
767  if self.config.doApplyExternalSkyWcs:
768  if self.config.useGlobalExternalSkyWcs:
769  externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
770  else:
771  externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
772  else:
773  externalSkyWcsCatalog = None
774 
775  if self.config.doApplyExternalPhotoCalib:
776  if self.config.useGlobalExternalPhotoCalib:
777  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
778  else:
779  externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
780  else:
781  externalPhotoCalibCatalog = None
782 
783  self.prepareCalibratedExposures(**inputs, externalSkyWcsCatalog=externalSkyWcsCatalog,
784  externalPhotoCalibCatalog=externalPhotoCalibCatalog)
785 
786  results = self.run(**inputs, visitId=visitId,
787  ccdIdList=[ccdIdList[i] for i in goodIndices],
788  dataIdList=[dataIdList[i] for i in goodIndices],
789  skyInfo=skyInfo)
790  if self.config.makeDirect:
791  butlerQC.put(results.exposures["direct"], outputRefs.direct)
792  if self.config.makePsfMatched:
793  butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
794 
795  def filterInputs(self, indices, inputs):
796  """Return task inputs with their lists filtered by indices
797 
798  Parameters
799  ----------
800  indices : `list` of integers
801  inputs : `dict` of `list` of input connections to be passed to run
802  """
803  for key in inputs.keys():
804  # Only down-select on list inputs
805  if isinstance(inputs[key], list):
806  inputs[key] = [inputs[key][ind] for ind in indices]
807  return inputs
808 
809  def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
810  externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
811  **kwargs):
812  """Calibrate and add backgrounds to input calExpList in place
813 
814  Parameters
815  ----------
816  calExpList : `list` of `lsst.afw.image.Exposure`
817  Sequence of calexps to be modified in place
818  backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
819  Sequence of backgrounds to be added back in if bgSubtracted=False
820  skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
821  Sequence of background corrections to be subtracted if doApplySkyCorr=True
822  externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
823  Exposure catalog with external skyWcs to be applied
824  if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
825  for the catalog id, sorted on id for fast lookup.
826  externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
827  Exposure catalog with external photoCalib to be applied
828  if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
829  id for the catalog id, sorted on id for fast lookup.
830  """
831  backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
832  skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
833 
834  includeCalibVar = self.config.includeCalibVar
835 
836  for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
837  mi = calexp.maskedImage
838  if not self.config.bgSubtracted:
839  mi += background.getImage()
840 
841  if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
842  detectorId = calexp.getInfo().getDetector().getId()
843 
844  # Find the external photoCalib
845  if externalPhotoCalibCatalog is not None:
846  row = externalPhotoCalibCatalog.find(detectorId)
847  if row is None:
848  raise RuntimeError(f"Detector id {detectorId} not found in "
849  f"externalPhotoCalibCatalog.")
850  photoCalib = row.getPhotoCalib()
851  if photoCalib is None:
852  raise RuntimeError(f"Detector id {detectorId} has None for photoCalib "
853  f"in externalPhotoCalibCatalog.")
854  else:
855  photoCalib = calexp.getPhotoCalib()
856 
857  # Find and apply external skyWcs
858  if externalSkyWcsCatalog is not None:
859  row = externalSkyWcsCatalog.find(detectorId)
860  if row is None:
861  raise RuntimeError(f"Detector id {detectorId} not found in externalSkyWcsCatalog.")
862  skyWcs = row.getWcs()
863  if skyWcs is None:
864  raise RuntimeError(f"Detector id {detectorId} has None for WCS "
865  f" in externalSkyWcsCatalog.")
866  calexp.setWcs(skyWcs)
867 
868  # Calibrate the image
869  calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
870  includeScaleUncertainty=includeCalibVar)
871  calexp.maskedImage /= photoCalib.getCalibrationMean()
872  # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
873  # exposure.setCalib(afwImage.Calib(1.0))
874 
875  # Apply skycorr
876  if self.config.doApplySkyCorr:
877  mi -= skyCorr.getImage()
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60