lsst.pipe.tasks  15.0-3-ga659d1f3
coaddInputRecorder.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import object
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.afw.table as afwTable
28 import lsst.afw.image as afwImage
29 import lsst.pipe.base as pipeBase
30 from lsst.meas.algorithms import CoaddPsf, makeCoaddApCorrMap
31 
32 __all__ = ["CoaddInputRecorderTask"]
33 
34 
35 class CoaddInputRecorderConfig(pexConfig.Config):
36  """Config for CoaddInputRecorderTask
37 
38  The inputRecorder section of the various coadd tasks' configs should generally agree,
39  or the schemas created by earlier tasks (like MakeCoaddTempExpTask) will not contain
40  the fields filled by later tasks (like AssembleCoaddTask).
41  """
42 
43  saveEmptyCcds = pexConfig.Field(
44  dtype=bool, default=False, optional=False,
45  doc=("Add records for CCDs we iterated over but did not add a coaddTempExp"
46  " due to a lack of unmasked pixels in the coadd footprint.")
47  )
48  saveErrorCcds = pexConfig.Field(
49  dtype=bool, default=False, optional=False,
50  doc=("Add records for CCDs we iterated over but did not add a coaddTempExp"
51  " due to an exception (often due to the calexp not being found on disk).")
52  )
53  saveVisitGoodPix = pexConfig.Field(
54  dtype=bool, default=True, optional=False,
55  doc=("Save the total number of good pixels in each coaddTempExp (redundant with a sum of"
56  " good pixels in associated CCDs)")
57  )
58  saveCcdWeights = pexConfig.Field(
59  dtype=bool, default=True, optional=False,
60  doc=("Save weights in the CCDs table as well as the visits table?"
61  " (This is necessary for easy construction of CoaddPsf, but otherwise duplicate information.)")
62  )
63 
64 
66  """A helper class for CoaddInputRecorderTask, managing the CoaddInputs object for that single
67  CoaddTempExp. This will contain a single 'visit' record for the CoaddTempExp and a number of 'ccd'
68  records.
69 
70  Should generally be created by calling CoaddInputRecorderTask.makeCoaddTempExp().
71  """
72 
73  def __init__(self, task, visitId, num=0):
74  """Constructor
75 
76  @param task The CoaddInputRecorderTask that is utilising us
77  @param visitId Identifier (integer) for the visit
78  @param num Number of CCDs for this visit that overlap this
79  patch (for reserving memory)
80  """
81  self.task = task
82  self.coaddInputs = self.task.makeCoaddInputs()
83  self.coaddInputs.visits.reserve(1)
84  if num > 0:
85  self.coaddInputs.ccds.reserve(num)
86  self.visitRecord = self.coaddInputs.visits.addNew()
87  self.visitRecord.setId(visitId)
88 
89  def addCalExp(self, calExp, ccdId, nGoodPix):
90  """Add a 'ccd' record for a calexp just added to the CoaddTempExp
91 
92  @param[in] calExp Calibrated exposure just added to the CoaddTempExp, or None in case of
93  failures that should nonetheless be tracked. Should be the original
94  calexp, in that it should contain the original Psf and Wcs, not the
95  warped and/or matched ones.
96  @param[in] ccdId A unique numeric ID for the Exposure.
97  @param[in] nGoodPix Number of good pixels this image will contribute to the CoaddTempExp.
98  If saveEmptyCcds is not set and this value is zero, no record will be
99  added.
100  """
101  if nGoodPix == 0 and not self.task.config.saveEmptyCcds:
102  return
103  record = self.coaddInputs.ccds.addNew()
104  record.setId(ccdId)
105  record.setL(self.task.ccdVisitKey, self.visitRecord.getId())
106  try:
107  record.setI(self.task.ccdCcdKey, calExp.getDetector().getId())
108  except:
109  self.task.log.warn("Error getting detector serial number in visit %d; using -1"
110  % self.visitRecord.getId())
111  record.setI(self.task.ccdCcdKey, -1)
112  record.setI(self.task.ccdGoodPixKey, nGoodPix)
113  if calExp is not None:
114  self._setExposureInfoInRecord(exposure=calExp, record=record)
115  if self.task.config.saveCcdWeights:
116  record.setD(self.task.ccdWeightKey, 1.0) # No weighting or overlap when warping
117  record.set(self.task.ccdFilterKey, calExp.getFilter().getName())
118 
119  def finish(self, coaddTempExp, nGoodPix=None):
120  """Finish creating the CoaddInputs for a CoaddTempExp.
121 
122  @param[in,out] coaddTempExp Exposure object from which to obtain the PSF, WCS, and bounding
123  box for the entry in the 'visits' table. On return, the completed
124  CoaddInputs object will be attached to it.
125  @param[in] nGoodPix Total number of good pixels in the CoaddTempExp; ignored unless
126  saveVisitGoodPix is true.
127  """
128  self._setExposureInfoInRecord(exposure=coaddTempExp, record=self.visitRecord)
129  if self.task.config.saveVisitGoodPix:
130  self.visitRecord.setI(self.task.visitGoodPixKey, nGoodPix)
131  coaddTempExp.getInfo().setCoaddInputs(self.coaddInputs)
132  wcs = coaddTempExp.getWcs()
133  if False:
134  # This causes a test failure, pending fix in issue HSC-802
135  coaddTempExp.setPsf(CoaddPsf(self.coaddInputs.ccds, wcs))
136  apCorrMap = makeCoaddApCorrMap(self.coaddInputs.ccds, coaddTempExp.getBBox(afwImage.PARENT), wcs)
137  coaddTempExp.getInfo().setApCorrMap(apCorrMap)
138 
139  def _setExposureInfoInRecord(self, exposure, record):
140  """Set exposure info and bbox in an ExposureTable record
141 
142  @param[in] exposure exposure whose info is to be recorded
143  @param[in,out] record record of an ExposureTable to set
144  """
145  info = exposure.getInfo()
146  record.setPsf(info.getPsf())
147  record.setWcs(info.getWcs())
148  record.setCalib(info.getCalib())
149  record.setApCorrMap(info.getApCorrMap())
150  record.setValidPolygon(info.getValidPolygon())
151  record.setVisitInfo(info.getVisitInfo())
152  record.setBBox(exposure.getBBox())
153  record.setTransmissionCurve(info.getTransmissionCurve())
154 
155 
156 
157 class CoaddInputRecorderTask(pipeBase.Task):
158  """Subtask that handles filling a CoaddInputs object for a coadd exposure, tracking the CCDs and
159  visits that went into a coadd.
160 
161  The interface here is a little messy, but I think this is at least partly a product of a bit of
162  messiness in the coadd code it's plugged into. I hope #2590 might result in a better design.
163  """
164 
165  ConfigClass = CoaddInputRecorderConfig
166 
167  def __init__(self, *args, **kwargs):
168  pipeBase.Task.__init__(self, *args, **kwargs)
169  self.visitSchema = afwTable.ExposureTable.makeMinimalSchema()
170  if self.config.saveVisitGoodPix:
171  self.visitGoodPixKey = self.visitSchema.addField("goodpix", type=numpy.int32,
172  doc="Number of good pixels in the coaddTempExp")
173  self.visitWeightKey = self.visitSchema.addField("weight", type=float,
174  doc="Weight for this visit in the coadd")
175  self.ccdSchema = afwTable.ExposureTable.makeMinimalSchema()
176  self.ccdCcdKey = self.ccdSchema.addField("ccd", type=numpy.int32, doc="cameraGeom CCD serial number")
177  self.ccdVisitKey = self.ccdSchema.addField("visit", type=numpy.int64,
178  doc="Foreign key for the visits (coaddTempExp) catalog")
179  self.ccdGoodPixKey = self.ccdSchema.addField("goodpix", type=numpy.int32,
180  doc="Number of good pixels in this CCD")
181  if self.config.saveCcdWeights:
182  self.ccdWeightKey = self.ccdSchema.addField("weight", type=float,
183  doc="Weight for this visit in the coadd")
184  self.visitFilterKey = self.visitSchema.addField("filter", type=str, size=32,
185  doc="Filter associated with this visit.")
186  self.ccdFilterKey = self.ccdSchema.addField("filter", type=str, size=32,
187  doc="Filter associated with this visit.")
188 
189  def makeCoaddTempExpRecorder(self, visitId, num=0):
190  """Return a CoaddTempExpInputRecorder instance to help with saving a CoaddTempExp's inputs.
191 
192  The visitId may be any number that is unique for each CoaddTempExp that goes into a coadd,
193  but ideally should be something more meaningful that can be used to reconstruct a data ID.
194  """
195  return CoaddTempExpInputRecorder(self, visitId, num=num)
196 
197  def makeCoaddInputs(self):
198  """Create a CoaddInputs object with schemas defined by the task configuration"""
199  return afwImage.CoaddInputs(self.visitSchema, self.ccdSchema)
200 
201  def addVisitToCoadd(self, coaddInputs, coaddTempExp, weight):
202  """Called by AssembleCoaddTask when adding (a subset of) a coaddTempExp to a coadd. The
203  base class impementation extracts the CoaddInputs from the coaddTempExp and appends
204  them to the given coaddInputs, filling in the weight column(s).
205 
206  Note that the passed coaddTempExp may be a subimage, but that this method will only be
207  called for the first subimage
208 
209  Returns the record for the visit to allow subclasses to fill in additional fields.
210  Warns and returns None if the inputRecorder catalogs for the coaddTempExp are not usable.
211  """
212  tempExpInputs = coaddTempExp.getInfo().getCoaddInputs()
213  if len(tempExpInputs.visits) != 1:
214  self.log.warn("CoaddInputs for coaddTempExp should have exactly one record in visits table "
215  "(found %d). CoaddInputs for this visit will not be saved."
216  % len(tempExpInputs.visits))
217  return None
218  inputVisitRecord = tempExpInputs.visits[0]
219  outputVisitRecord = coaddInputs.visits.addNew()
220  outputVisitRecord.assign(inputVisitRecord)
221  outputVisitRecord.setD(self.visitWeightKey, weight)
222  outputVisitRecord.set(self.visitFilterKey, coaddTempExp.getFilter().getName())
223  for inputCcdRecord in tempExpInputs.ccds:
224  if inputCcdRecord.getL(self.ccdVisitKey) != inputVisitRecord.getId():
225  self.log.warn("CoaddInputs for coaddTempExp with id %d contains CCDs with visit=%d. "
226  "CoaddInputs may be unreliable."
227  % (inputVisitRecord.getId(), inputCcdRecord.getL(self.ccdVisitKey)))
228  outputCcdRecord = coaddInputs.ccds.addNew()
229  outputCcdRecord.assign(inputCcdRecord)
230  if self.config.saveCcdWeights:
231  outputCcdRecord.setD(self.ccdWeightKey, weight)
232  outputCcdRecord.set(self.ccdFilterKey, coaddTempExp.getFilter().getName())
233  return inputVisitRecord
def addVisitToCoadd(self, coaddInputs, coaddTempExp, weight)