lsst.pipe.tasks  13.0-29-g7046ce1+1
 All Classes Namespaces Files Functions Variables Groups Pages
makeCoaddTempExp.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 import lsst.pipe.base as pipeBase
30 from lsst.meas.algorithms import CoaddPsf
31 from .coaddBase import CoaddBaseTask
32 from .warpAndPsfMatch import WarpAndPsfMatchTask
33 from .coaddHelpers import groupPatchExposures, getGroupDataRef
34 
35 __all__ = ["MakeCoaddTempExpTask"]
36 
37 
38 class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
39  """Config for MakeCoaddTempExpTask
40  """
41  warpAndPsfMatch = pexConfig.ConfigurableField(
42  target=WarpAndPsfMatchTask,
43  doc="Task to warp and PSF-match calexp",
44  )
45  doWrite = pexConfig.Field(
46  doc="persist <coaddName>Coadd_tempExp",
47  dtype=bool,
48  default=True,
49  )
50  doOverwrite = pexConfig.Field(
51  doc="overwrite <coaddName>Coadd_tempExp; If False, continue if the file exists on disk",
52  dtype=bool,
53  default=True,
54  )
55  bgSubtracted = pexConfig.Field(
56  doc="Work with a background subtracted calexp?",
57  dtype=bool,
58  default=True,
59  )
60 
61 
62 class MakeCoaddTempExpTask(CoaddBaseTask):
63  """Task to produce <coaddName>Coadd_tempExp images
64  """
65  ConfigClass = MakeCoaddTempExpConfig
66  _DefaultName = "makeCoaddTempExp"
67 
68  def __init__(self, *args, **kwargs):
69  CoaddBaseTask.__init__(self, *args, **kwargs)
70  self.makeSubtask("warpAndPsfMatch")
71 
72  @pipeBase.timeMethod
73  def run(self, patchRef, selectDataList=[]):
74  """Produce <coaddName>Coadd_tempExp images
75 
76  <coaddName>Coadd_tempExp are produced by PSF-matching (optional) and warping.
77 
78  @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
79  plus the camera-specific filter key (e.g. "filter" or "band")
80  @return: dataRefList: a list of data references for the new <coaddName>Coadd_tempExp
81 
82  @warning: this task assumes that all exposures in a coaddTempExp have the same filter.
83 
84  @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
85  with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
86  (assembleCoadd should determine zeropoint scaling without referring to it).
87  """
88  skyInfo = self.getSkyInfo(patchRef)
89 
90  calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
91  if len(calExpRefList) == 0:
92  self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
93  return None
94  self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
95  calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
96  self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
97 
98  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
99  self.getTempExpDatasetName())
100  self.log.info("Processing %d tempExps for patch %s", len(groupData.groups), patchRef.dataId)
101 
102  dataRefList = []
103  for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
104  tempExpRef = getGroupDataRef(patchRef.getButler(), self.getTempExpDatasetName(),
105  tempExpTuple, groupData.keys)
106  if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=self.getTempExpDatasetName()):
107  self.log.info("tempCoaddExp %s exists; skipping", tempExpRef.dataId)
108  dataRefList.append(tempExpRef)
109  continue
110  self.log.info("Processing tempExp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
111 
112  # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
113  # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
114  # of the visit in the list.
115  try:
116  visitId = int(tempExpRef.dataId["visit"])
117  except (KeyError, ValueError):
118  visitId = i
119 
120  exp = self.createTempExp(calexpRefList, skyInfo, visitId)
121  if exp is not None:
122  dataRefList.append(tempExpRef)
123  if self.config.doWrite:
124  self.writeCoaddOutput(tempExpRef, exp, "tempExp")
125  else:
126  self.log.warn("tempExp %s could not be created", tempExpRef.dataId)
127  return dataRefList
128 
129  def createTempExp(self, calexpRefList, skyInfo, visitId=0):
130  """Create a tempExp from inputs
131 
132  We iterate over the multiple calexps in a single exposure to construct
133  the warp ("tempExp") of that exposure to the supplied tract/patch.
134 
135  Pixels that receive no pixels are set to NAN; this is not correct
136  (violates LSST algorithms group policy), but will be fixed up by
137  interpolating after the coaddition.
138 
139  @param calexpRefList: List of data references for calexps that (may)
140  overlap the patch of interest
141  @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
142  information about the patch
143  @param visitId: integer identifier for visit, for the table that will
144  produce the CoaddPsf
145  @return warped exposure, or None if no pixels overlap
146  """
147  inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
148  coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
149  coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf)
150  totGoodPix = 0
151  didSetMetadata = False
152  modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None
153  for calExpInd, calExpRef in enumerate(calexpRefList):
154  self.log.info("Processing calexp %d of %d for this tempExp: id=%s",
155  calExpInd+1, len(calexpRefList), calExpRef.dataId)
156  try:
157  ccdId = calExpRef.get("ccdExposureId", immediate=True)
158  except Exception:
159  ccdId = calExpInd
160  numGoodPix = 0
161  try:
162  # We augment the dataRef here with the tract, which is harmless for loading things
163  # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
164  # which do.
165  calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
166  tract=skyInfo.tractInfo.getId())
167  calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
168  exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
169  maxBBox=skyInfo.bbox).exposure
170  if didSetMetadata:
171  mimg = exposure.getMaskedImage()
172  mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0])
173  del mimg
174  numGoodPix = coaddUtils.copyGoodPixels(
175  coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
176  totGoodPix += numGoodPix
177  self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%)",
178  calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea())
179  if numGoodPix > 0 and not didSetMetadata:
180  coaddTempExp.setCalib(exposure.getCalib())
181  coaddTempExp.setFilter(exposure.getFilter())
182  # PSF replaced with CoaddPsf after loop if and only if creating direct warp
183  coaddTempExp.setPsf(exposure.getPsf())
184  didSetMetadata = True
185  except Exception as e:
186  self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
187  continue
188  inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
189 
190  inputRecorder.finish(coaddTempExp, totGoodPix)
191  if totGoodPix > 0 and didSetMetadata and not self.config.doPsfMatch:
192  coaddTempExp.setPsf(CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs))
193 
194  self.log.info("coaddTempExp has %d good pixels (%.1f%%)",
195  totGoodPix, 100.0*totGoodPix/skyInfo.bbox.getArea())
196  return coaddTempExp if totGoodPix > 0 and didSetMetadata else None