lsst.pipe.tasks  14.0-27-g4464a260+1
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import os
26 import numpy
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
40 from .interpImage import InterpImageTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms import SourceDetectionTask
44 
45 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
46 
47 
48 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
49  """!
50 \anchor AssembleCoaddConfig_
51 
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53  """
54  warpType = pexConfig.Field(
55  doc="Warp name: one of 'direct' or 'psfMatched'",
56  dtype=str,
57  default="direct",
58  )
59  subregionSize = pexConfig.ListField(
60  dtype=int,
61  doc="Width, height of stack subregion size; "
62  "make small enough that a full stack of images will fit into memory at once.",
63  length=2,
64  default=(2000, 2000),
65  )
66  statistic = pexConfig.Field(
67  dtype=str,
68  doc="Main stacking statistic for aggregating over the epochs.",
69  default="MEANCLIP",
70  )
71  doSigmaClip = pexConfig.Field(
72  dtype=bool,
73  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
74  default=False,
75  )
76  sigmaClip = pexConfig.Field(
77  dtype=float,
78  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
79  default=3.0,
80  )
81  clipIter = pexConfig.Field(
82  dtype=int,
83  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
84  default=2,
85  )
86  scaleZeroPoint = pexConfig.ConfigurableField(
87  target=ScaleZeroPointTask,
88  doc="Task to adjust the photometric zero point of the coadd temp exposures",
89  )
90  doInterp = pexConfig.Field(
91  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
92  dtype=bool,
93  default=True,
94  )
95  interpImage = pexConfig.ConfigurableField(
96  target=InterpImageTask,
97  doc="Task to interpolate (and extrapolate) over NaN pixels",
98  )
99  doWrite = pexConfig.Field(
100  doc="Persist coadd?",
101  dtype=bool,
102  default=True,
103  )
104  doNImage = pexConfig.Field(
105  doc="Create image of number of contributing exposures for each pixel",
106  dtype=bool,
107  default=False,
108  )
109  maskPropagationThresholds = pexConfig.DictField(
110  keytype=str,
111  itemtype=float,
112  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
113  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
114  "would have contributed exceeds this value."),
115  default={"SAT": 0.1},
116  )
117  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
118  doc="Mask planes to remove before coadding")
119  #
120  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
121  # To make this useful you *must* also configure the flags.pixel algorithm, for example
122  # by adding
123  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
124  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
125  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
126  #
127  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
128  doc="Set mask and flag bits for bright objects?")
129  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
130  doc="Name of mask bit used for bright objects")
131 
132  coaddPsf = pexConfig.ConfigField(
133  doc="Configuration for CoaddPsf",
134  dtype=measAlg.CoaddPsfConfig,
135  )
136 
137  def setDefaults(self):
138  CoaddBaseTask.ConfigClass.setDefaults(self)
139  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
140 
141  def validate(self):
142  CoaddBaseTask.ConfigClass.validate(self)
143  if self.doPsfMatch:
144  # Backwards compatibility.
145  # Configs do not have loggers
146  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
147  self.warpType = 'psfMatched'
148  if self.doSigmaClip and self.statistic != "MEANCLIP":
149  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
150  self.statistic = "MEANCLIP"
151  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
152  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
153  "compute and set a non-zero coadd variance estimate." % (self.statistic))
154 
155  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
156  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
157  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
158  if str(k) not in unstackableStats]
159  raise ValueError("statistic %s is not allowed. Please choose one of %s."
160  % (self.statistic, stackableStats))
161 
162 
163 
170  """!
171 \anchor AssembleCoaddTask_
172 
173 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
174 
175 \section pipe_tasks_assembleCoadd_Contents Contents
176  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
177  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
178  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
179  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
180  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
181  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
182 
183 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
184 
185 \copybrief AssembleCoaddTask_
186 
187 We want to assemble a coadded image from a set of Warps (also called
188 coadded temporary exposures or coaddTempExps.
189 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
190 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
191 Warps that cover the specified patch (pointed at by dataRef).
192 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
193 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
194 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
195 config.makePsfMatched set which of the warp types will be coadded.
196 The coadd is computed as a mean with optional outlier rejection.
197 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
198 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
199 
200 AssembleCoaddTask uses several sub-tasks. These are
201 <DL>
202  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
203  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
204  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
205  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
206 </DL>
207 You can retarget these subtasks if you wish.
208 
209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
210 \copydoc \_\_init\_\_
211 
212 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
213 \copydoc run
214 
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
216 See \ref AssembleCoaddConfig_
217 
218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
219 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
220 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
221 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
222 the documetation for the subtasks for further information.
223 
224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
225 
226 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
227 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
228 a data reference to the tract patch and filter to be coadded (specified using
229 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
230 Warps to attempt to coadd (specified using
231 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
232 that cover the specified tract and patch will be coadded. A list of the available optional
233 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
234 \code
235 assembleCoadd.py --help
236 \endcode
237 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
238 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
239 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
240 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
241 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
242 <DL>
243  <DT>processCcd</DT>
244  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
245  <DT>makeSkyMap</DT>
246  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
247  <DT>makeCoaddTempExp</DT>
248  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
249 </DL>
250 We can perform all of these steps by running
251 \code
252 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
253 \endcode
254 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
255 follows:
256 \code
257 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
258 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
259 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
260 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
261 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
262 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
263 --selectId visit=903988 ccd=24
264 \endcode
265 that will process the HSC-I band data. The results are written in
266 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
267 
268 You may also choose to run:
269 \code
270 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
271 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
272 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
273 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
274 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
275 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
276 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
277 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
278 \endcode
279 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
280 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
281 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
282  """
283  ConfigClass = AssembleCoaddConfig
284  _DefaultName = "assembleCoadd"
285 
286  def __init__(self, *args, **kwargs):
287  """!
288  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
289  & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
290  """
291  CoaddBaseTask.__init__(self, *args, **kwargs)
292  self.makeSubtask("interpImage")
293  self.makeSubtask("scaleZeroPoint")
294 
295  if self.config.doMaskBrightObjects:
296  mask = afwImage.Mask()
297  try:
298  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
299  except pexExceptions.LsstCppException:
300  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
301  mask.getMaskPlaneDict().keys())
302  del mask
303 
304  self.warpType = self.config.warpType
305 
306  @pipeBase.timeMethod
307  def run(self, dataRef, selectDataList=[]):
308  """!
309  \brief Assemble a coadd from a set of Warps
310 
311  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
312  match the photometric zeropoint to a reference Warp. Assemble the Warps using
313  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
314  exposure.
315 
316  \anchor runParams
317  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
318  (if config.autoReference=False). Used to access the following data products:
319  - [in] self.config.coaddName + "Coadd_skyMap"
320  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
321  - [out] self.config.coaddName + "Coadd"
322  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
323  selected from this list based on overlap with the patch defined by dataRef.
324 
325  \return a pipeBase.Struct with fields:
326  - coaddExposure: coadded exposure
327  - nImage: exposure count image
328  """
329  skyInfo = self.getSkyInfo(dataRef)
330  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
331  if len(calExpRefList) == 0:
332  self.log.warn("No exposures to coadd")
333  return
334  self.log.info("Coadding %d exposures", len(calExpRefList))
335 
336  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
337  inputData = self.prepareInputs(tempExpRefList)
338  self.log.info("Found %d %s", len(inputData.tempExpRefList),
339  self.getTempExpDatasetName(self.warpType))
340  if len(inputData.tempExpRefList) == 0:
341  self.log.warn("No coadd temporary exposures found")
342  return
343 
344  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
345 
346  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
347  inputData.weightList, supplementaryData=supplementaryData)
348 
349  if self.config.doInterp:
350  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
351  # The variance must be positive; work around for DM-3201.
352  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
353  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
354 
355  if self.config.doMaskBrightObjects:
356  brightObjectMasks = self.readBrightObjectMasks(dataRef)
357  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
358 
359  if self.config.doWrite:
360  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
361  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
362  if retStruct.nImage is not None:
363  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
364 
365  return retStruct
366 
367  def makeSupplementaryData(self, dataRef, selectDataList):
368  """!
369  \brief Make additional inputs to assemble() specific to subclasses.
370 
371  Available to be implemented by subclasses only if they need the
372  coadd dataRef for performing preliminary processing before
373  assembling the coadd.
374  """
375  pass
376 
377  def getTempExpRefList(self, patchRef, calExpRefList):
378  """!
379  \brief Generate list data references corresponding to warped exposures that lie within the
380  patch to be coadded.
381 
382  \param[in] patchRef: Data reference for patch
383  \param[in] calExpRefList: List of data references for input calexps
384  \return List of Warp/CoaddTempExp data references
385  """
386  butler = patchRef.getButler()
387  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
388  self.getTempExpDatasetName(self.warpType))
389  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
390  g, groupData.keys) for
391  g in groupData.groups.keys()]
392  return tempExpRefList
393 
394  def prepareInputs(self, refList):
395  """!
396  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
397  for the photometric zero point.
398 
399  Each Warp has its own photometric zeropoint and background variance. Before coadding these
400  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
401  weight for each Warp.
402 
403  \param[in] refList: List of data references to tempExp
404  \return Struct:
405  - tempExprefList: List of data references to tempExp
406  - weightList: List of weightings
407  - imageScalerList: List of image scalers
408  """
409  statsCtrl = afwMath.StatisticsControl()
410  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
411  statsCtrl.setNumIter(self.config.clipIter)
412  statsCtrl.setAndMask(self.getBadPixelMask())
413  statsCtrl.setNanSafe(True)
414  # compute tempExpRefList: a list of tempExpRef that actually exist
415  # and weightList: a list of the weight of the associated coadd tempExp
416  # and imageScalerList: a list of scale factors for the associated coadd tempExp
417  tempExpRefList = []
418  weightList = []
419  imageScalerList = []
420  tempExpName = self.getTempExpDatasetName(self.warpType)
421  for tempExpRef in refList:
422  if not tempExpRef.datasetExists(tempExpName):
423  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
424  continue
425 
426  tempExp = tempExpRef.get(tempExpName, immediate=True)
427  maskedImage = tempExp.getMaskedImage()
428  imageScaler = self.scaleZeroPoint.computeImageScaler(
429  exposure=tempExp,
430  dataRef=tempExpRef,
431  )
432  try:
433  imageScaler.scaleMaskedImage(maskedImage)
434  except Exception as e:
435  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
436  continue
437  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
438  afwMath.MEANCLIP, statsCtrl)
439  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
440  weight = 1.0 / float(meanVar)
441  if not numpy.isfinite(weight):
442  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
443  continue
444  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
445 
446  del maskedImage
447  del tempExp
448 
449  tempExpRefList.append(tempExpRef)
450  weightList.append(weight)
451  imageScalerList.append(imageScaler)
452 
453  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
454  imageScalerList=imageScalerList)
455 
456  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
457  altMaskList=None, mask=None, supplementaryData=None):
458  """!
459  \anchor AssembleCoaddTask.assemble_
460 
461  \brief Assemble a coadd from input warps
462 
463  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
464  large area), the assembly is performed over small areas on the image at a time in order to
465  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
466  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
467  statistic specified. Set the edge bits the coadd mask based on the weight map.
468 
469  \param[in] skyInfo: Patch geometry information, from getSkyInfo
470  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
471  \param[in] imageScalerList: List of image scalers
472  \param[in] weightList: List of weights
473  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
474  \param[in] mask: Mask to ignore when coadding
475  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
476  Only used by subclasses that implement makeSupplementaryData and override assemble.
477  \return pipeBase.Struct with coaddExposure, nImage if requested
478  """
479  tempExpName = self.getTempExpDatasetName(self.warpType)
480  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
481  if mask is None:
482  mask = self.getBadPixelMask()
483 
484  statsCtrl = afwMath.StatisticsControl()
485  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
486  statsCtrl.setNumIter(self.config.clipIter)
487  statsCtrl.setAndMask(mask)
488  statsCtrl.setNanSafe(True)
489  statsCtrl.setWeighted(True)
490  statsCtrl.setCalcErrorFromInputVariance(True)
491  for plane, threshold in self.config.maskPropagationThresholds.items():
492  bit = afwImage.Mask.getMaskPlane(plane)
493  statsCtrl.setMaskPropagationThreshold(bit, threshold)
494 
495  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
496 
497  if altMaskList is None:
498  altMaskList = [None]*len(tempExpRefList)
499 
500  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
501  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
502  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
503  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
504  coaddMaskedImage = coaddExposure.getMaskedImage()
505  subregionSizeArr = self.config.subregionSize
506  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
507  # if nImage is requested, create a zero one which can be passed to assembleSubregion
508  if self.config.doNImage:
509  nImage = afwImage.ImageU(skyInfo.bbox)
510  else:
511  nImage = None
512  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
513  try:
514  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
515  weightList, altMaskList, statsFlags, statsCtrl,
516  nImage=nImage)
517  except Exception as e:
518  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
519 
520  self.setEdge(coaddMaskedImage.getMask())
521  self.setInexactPsf(coaddMaskedImage.getMask())
522  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
523  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
524  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
525  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
526 
527  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
528  """!
529  \brief Set the metadata for the coadd
530 
531  This basic implementation simply sets the filter from the
532  first input.
533 
534  \param[in] coaddExposure: The target image for the coadd
535  \param[in] tempExpRefList: List of data references to tempExp
536  \param[in] weightList: List of weights
537  """
538  assert len(tempExpRefList) == len(weightList), "Length mismatch"
539  tempExpName = self.getTempExpDatasetName(self.warpType)
540  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
541  # (and we need more than just the PropertySet that contains the header), which is not possible
542  # with the current butler (see #2777).
543  tempExpList = [tempExpRef.get(tempExpName + "_sub",
544  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
545  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
546  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
547 
548  coaddExposure.setFilter(tempExpList[0].getFilter())
549  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
550  coaddInputs.ccds.reserve(numCcds)
551  coaddInputs.visits.reserve(len(tempExpList))
552 
553  for tempExp, weight in zip(tempExpList, weightList):
554  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
555  coaddInputs.visits.sort()
556  if self.warpType == "psfMatched":
557  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
558  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
559  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
560  # having the maximum width (sufficient because square)
561  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
562  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
563  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
564  else:
565  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
566  self.config.coaddPsf.makeControl())
567  coaddExposure.setPsf(psf)
568  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
569  coaddExposure.getWcs())
570  coaddExposure.getInfo().setApCorrMap(apCorrMap)
571 
572  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
573  altMaskList, statsFlags, statsCtrl, nImage=None):
574  """!
575  \brief Assemble the coadd for a sub-region.
576 
577  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
578  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
579  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
580  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
581  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
582  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
583 
584  \param[in] coaddExposure: The target image for the coadd
585  \param[in] bbox: Sub-region to coadd
586  \param[in] tempExpRefList: List of data reference to tempExp
587  \param[in] imageScalerList: List of image scalers
588  \param[in] weightList: List of weights
589  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
590  \param[in] statsFlags: afwMath.Property object for statistic for coadd
591  \param[in] statsCtrl: Statistics control object for coadd
592  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
593  """
594  self.log.debug("Computing coadd over %s", bbox)
595  tempExpName = self.getTempExpDatasetName(self.warpType)
596  coaddMaskedImage = coaddExposure.getMaskedImage()
597  coaddMaskedImage.getMask().addMaskPlane("CLIPPED")
598  maskedImageList = []
599  if nImage is not None:
600  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
601  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
602  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
603  maskedImage = exposure.getMaskedImage()
604  if altMask:
605  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
606  maskedImage.getMask().swap(altMaskSub)
607  imageScaler.scaleMaskedImage(maskedImage)
608 
609  # Add 1 for each pixel which is not excluded by the exclude mask.
610  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
611  if nImage is not None:
612  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
613  if self.config.removeMaskPlanes:
614  mask = maskedImage.getMask()
615  for maskPlane in self.config.removeMaskPlanes:
616  try:
617  mask &= ~mask.getPlaneBitMask(maskPlane)
618  except Exception as e:
619  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
620 
621  maskedImageList.append(maskedImage)
622 
623  with self.timer("stack"):
624  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
625  coaddMaskedImage.getMask().getPlaneBitMask("CLIPPED"),
626  coaddMaskedImage.getMask().getPlaneBitMask("NO_DATA"))
627  coaddMaskedImage.assign(coaddSubregion, bbox)
628  if nImage is not None:
629  nImage.assign(subNImage, bbox)
630 
631  def readBrightObjectMasks(self, dataRef):
632  """Returns None on failure"""
633  try:
634  return dataRef.get("brightObjectMask", immediate=True)
635  except Exception as e:
636  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
637  return None
638 
639  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
640  """Set the bright object masks
641 
642  exposure: Exposure under consideration
643  dataId: Data identifier dict for patch
644  brightObjectMasks: afwTable of bright objects to mask
645  """
646  #
647  # Check the metadata specifying the tract/patch/filter
648  #
649  if brightObjectMasks is None:
650  self.log.warn("Unable to apply bright object mask: none supplied")
651  return
652  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
653  md = brightObjectMasks.table.getMetadata()
654  for k in dataId:
655  if not md.exists(k):
656  self.log.warn("Expected to see %s in metadata", k)
657  else:
658  if md.get(k) != dataId[k]:
659  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
660 
661  mask = exposure.getMaskedImage().getMask()
662  wcs = exposure.getWcs()
663  plateScale = wcs.pixelScale().asArcseconds()
664 
665  for rec in brightObjectMasks:
666  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
667  if rec["type"] == "box":
668  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
669  width = rec["width"].asArcseconds()/plateScale # convert to pixels
670  height = rec["height"].asArcseconds()/plateScale # convert to pixels
671 
672  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
673  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
674 
675  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
676  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
677  spans = afwGeom.SpanSet(bbox)
678  elif rec["type"] == "circle":
679  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
680  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
681  else:
682  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
683  continue
684  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
685 
686  def setEdge(self, mask):
687  """Set EDGE bits as SENSOR_EDGE
688 
689  The EDGE pixels from the individual CCDs have printed through into the
690  coadd, but EDGE means "we couldn't search for sources in this area
691  because we couldn't convolve by the PSF near the edge of the image",
692  so this mask plane needs to be converted to something else if we want
693  to keep them. We do want to be able to identify pixels near the edge
694  of the detector because they will have an inexact `CoaddPsf`. We
695  rename EDGE pixels as SENSOR_EDGE.
696 
697  Parameters
698  ----------
699  mask : `lsst.afw.image.Mask`
700  Coadded exposure's mask, modified in-place.
701  """
702  mask.addMaskPlane("SENSOR_EDGE")
703  edge = mask.getPlaneBitMask("EDGE")
704  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE")
705  array = mask.getArray()
706  selected = (array & edge > 0)
707  array[selected] |= sensorEdge
708  array[selected] &= ~edge
709 
710  def setInexactPsf(self, mask):
711  """Set INEXACT_PSF mask plane
712 
713  If any of the input images isn't represented in the coadd (due to
714  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
715  these pixels.
716 
717  Parameters
718  ----------
719  mask : `lsst.afw.image.Mask`
720  Coadded exposure's mask, modified in-place.
721  """
722  mask.addMaskPlane("INEXACT_PSF")
723  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
724  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip gaps
725  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
726  array = mask.getArray()
727  selected = array & (sensorEdge | clipped) > 0
728  array[selected] |= inexactPsf
729 
730  @classmethod
731  def _makeArgumentParser(cls):
732  """!
733  \brief Create an argument parser
734  """
735  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
736  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
737  cls.ConfigClass().warpType + "Warp",
738  help="data ID, e.g. --id tract=12345 patch=1,2",
739  ContainerClass=AssembleCoaddDataIdContainer)
740  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
741  ContainerClass=SelectDataIdContainer)
742  return parser
743 
744 
745 def _subBBoxIter(bbox, subregionSize):
746  """!
747  \brief Iterate over subregions of a bbox
748 
749  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
750  \param[in] subregionSize: size of sub-bboxes
751 
752  \return subBBox: next sub-bounding box of size subregionSize or smaller;
753  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
754  but it will never be empty
755  """
756  if bbox.isEmpty():
757  raise RuntimeError("bbox %s is empty" % (bbox,))
758  if subregionSize[0] < 1 or subregionSize[1] < 1:
759  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
760 
761  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
762  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
763  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
764  subBBox.clip(bbox)
765  if subBBox.isEmpty():
766  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
767  (bbox, subregionSize, colShift, rowShift))
768  yield subBBox
769 
770 
771 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
772  """!
773  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
774  """
775 
776  def makeDataRefList(self, namespace):
777  """!
778  \brief Make self.refList from self.idList.
779  """
780  datasetType = namespace.config.coaddName + "Coadd"
781  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
782 
783  for dataId in self.idList:
784  # tract and patch are required
785  for key in keysCoadd:
786  if key not in dataId:
787  raise RuntimeError("--id must include " + key)
788 
789  dataRef = namespace.butler.dataRef(
790  datasetType=datasetType,
791  dataId=dataId,
792  )
793  self.refList.append(dataRef)
794 
795 
796 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
797  """!
798  \brief Function to count the number of pixels with a specific mask in a footprint.
799 
800  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
801  have bitmask set but do not have ignoreMask set. Return the count.
802 
803  \param[in] mask: mask to define intersection region by.
804  \parma[in] footprint: footprint to define the intersection region by.
805  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
806  \param[in] ignoreMask: pixels to not consider.
807  \return count of number of pixels in footprint with specified mask.
808  """
809  bbox = footprint.getBBox()
810  bbox.clip(mask.getBBox(afwImage.PARENT))
811  fp = afwImage.Mask(bbox)
812  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
813  footprint.spans.setMask(fp, bitmask)
814  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
815  (subMask.getArray() & ignoreMask) == 0).sum()
816 
817 
819  """!
820 \anchor SafeClipAssembleCoaddConfig
821 
822 \brief Configuration parameters for the SafeClipAssembleCoaddTask
823  """
824  clipDetection = pexConfig.ConfigurableField(
825  target=SourceDetectionTask,
826  doc="Detect sources on difference between unclipped and clipped coadd")
827  minClipFootOverlap = pexConfig.Field(
828  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
829  dtype=float,
830  default=0.6
831  )
832  minClipFootOverlapSingle = pexConfig.Field(
833  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
834  "clipped when only one visit overlaps",
835  dtype=float,
836  default=0.5
837  )
838  minClipFootOverlapDouble = pexConfig.Field(
839  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
840  "clipped when two visits overlap",
841  dtype=float,
842  default=0.45
843  )
844  maxClipFootOverlapDouble = pexConfig.Field(
845  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
846  "considering two visits",
847  dtype=float,
848  default=0.15
849  )
850  minBigOverlap = pexConfig.Field(
851  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
852  "when labeling clipped footprints",
853  dtype=int,
854  default=100
855  )
856 
857  def setDefaults(self):
858  # The numeric values for these configuration parameters were empirically determined, future work
859  # may further refine them.
860  AssembleCoaddConfig.setDefaults(self)
861  self.clipDetection.doTempLocalBackground = False
862  self.clipDetection.reEstimateBackground = False
863  self.clipDetection.returnOriginalFootprints = False
864  self.clipDetection.thresholdPolarity = "both"
865  self.clipDetection.thresholdValue = 2
866  self.clipDetection.nSigmaToGrow = 2
867  self.clipDetection.minPixels = 4
868  self.clipDetection.isotropicGrow = True
869  self.clipDetection.thresholdType = "pixel_stdev"
870  self.sigmaClip = 1.5
871  self.clipIter = 3
872  self.statistic = "MEAN"
873 
874  def validate(self):
875  if self.doSigmaClip:
876  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
877  "Ignoring doSigmaClip.")
878  self.doSigmaClip = False
879  if self.statistic != "MEAN":
880  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
881  "(%s chosen). Please set statistic to MEAN."
882  % (self.statistic))
883  AssembleCoaddTask.ConfigClass.validate(self)
884 
885 
886 
892 
893 
895  """!
896  \anchor SafeClipAssembleCoaddTask_
897 
898  \brief Assemble a coadded image from a set of coadded temporary exposures,
899  being careful to clip & flag areas with potential artifacts.
900 
901  \section pipe_tasks_assembleCoadd_Contents Contents
902  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
903  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
904  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
905  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
906  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
907  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
908 
909  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
910 
911  \copybrief SafeClipAssembleCoaddTask
912 
913  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
914  SafeClipAssembleCoaddTask subtasks that task.
915  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
916  outliers).
917  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
918  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
919  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
920  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
921  that there is an outlier and ii. this outlier appears on only one or two images.
922  Such regions will not contribute to the final coadd.
923  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
924  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
925  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
926  surveys.
927 
928  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
929  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
930  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
931 
932  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
933  \copydoc \_\_init\_\_
934 
935  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
936  \copydoc run
937 
938  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
939  See \ref SafeClipAssembleCoaddConfig
940 
941  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
942  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
943  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
944  files.
945  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
946  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
947  for further information.
948 
949  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
950  SafeClipAssembleCoaddTask
951 
952  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
953  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
954  '--legacyCoadd'.
955  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
956  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
957  with a list of coaddTempExps to attempt to coadd (specified using
958  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
959  Only the coaddTempExps that cover the specified tract and patch will be coadded.
960  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
961  command line argument:
962  \code
963  assembleCoadd.py --help
964  \endcode
965  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
966  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
967  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
968  packages.
969  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
970  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
971  <DL>
972  <DT>processCcd</DT>
973  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
974  <DT>makeSkyMap</DT>
975  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
976  <DT>makeCoaddTempExp</DT>
977  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
978  </DL>
979  We can perform all of these steps by running
980  \code
981  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
982  \endcode
983  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
984  as follows:
985  \code
986  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
987  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
988  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
989  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
990  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
991  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
992  --selectId visit=903988 ccd=24
993  \endcode
994  This will process the HSC-I band data. The results are written in
995  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
996 
997  You may also choose to run:
998  \code
999  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1000  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1001  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1002  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1003  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1004  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1005  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1006  --selectId visit=903346 ccd=12
1007  \endcode
1008  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1009  discussed in \ref pipeTasks_multiBand.
1010  """
1011  ConfigClass = SafeClipAssembleCoaddConfig
1012  _DefaultName = "safeClipAssembleCoadd"
1013 
1014  def __init__(self, *args, **kwargs):
1015  """!
1016  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1017  """
1018  AssembleCoaddTask.__init__(self, *args, **kwargs)
1019  schema = afwTable.SourceTable.makeMinimalSchema()
1020  self.makeSubtask("clipDetection", schema=schema)
1021 
1022  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1023  """!
1024  \brief Assemble the coadd for a region
1025 
1026  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1027  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1028  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1029  overlap between the clipped region and a source.
1030  This leaves us with a set of footprints from the difference image that have been identified as having
1031  occured on just one or two individual visits. However, these footprints were generated from a
1032  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1033  footprints acrosss the coadd difference in this process.
1034  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1035  these are big footprints.
1036  Combine the small and big clipped footprints and mark them on a new bad mask plane
1037  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1038  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1039  bad mask plane.
1040 
1041  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1042  parent task.
1043 
1044  @param skyInfo: Patch geometry information, from getSkyInfo
1045  @param tempExpRefList: List of data reference to tempExp
1046  @param imageScalerList: List of image scalers
1047  @param weightList: List of weights
1048  return pipeBase.Struct with coaddExposure, nImage
1049  """
1050  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1051  mask = exp.getMaskedImage().getMask()
1052  mask.addMaskPlane("CLIPPED")
1053 
1054  result = self.detectClip(exp, tempExpRefList)
1055 
1056  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1057 
1058  # Go to individual visits for big footprints
1059  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1060  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1061  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1062  maskClipValue, maskDetValue)
1063 
1064  # Create mask of the current clipped footprints
1065  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1066  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1067 
1068  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1069  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1070  maskClip |= maskClipBig
1071 
1072  # Assemble coadd from base class, but ignoring CLIPPED pixels
1073  badMaskPlanes = self.config.badMaskPlanes[:]
1074  badMaskPlanes.append("CLIPPED")
1075  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1076  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1077  result.tempExpClipList, mask=badPixelMask)
1078 
1079  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1080  """!
1081  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1082 
1083  Generate a difference image between clipped and unclipped coadds.
1084  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1085  Return the difference image.
1086 
1087  @param skyInfo: Patch geometry information, from getSkyInfo
1088  @param tempExpRefList: List of data reference to tempExp
1089  @param imageScalerList: List of image scalers
1090  @param weightList: List of weights
1091  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1092  """
1093  # Clone and upcast self.config because current self.config is frozen
1094  config = AssembleCoaddConfig()
1095  # getattr necessary because subtasks do not survive Config.toDict()
1096  configIntersection = {k: getattr(self.config, k)
1097  for k, v in self.config.toDict().items() if (k in config.keys())}
1098  config.update(**configIntersection)
1099 
1100  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1101  config.statistic = 'MEAN'
1102  task = AssembleCoaddTask(config=config)
1103  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1104 
1105  config.statistic = 'MEANCLIP'
1106  task = AssembleCoaddTask(config=config)
1107  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1108 
1109  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1110  coaddDiff -= coaddClip.getMaskedImage()
1111  exp = afwImage.ExposureF(coaddDiff)
1112  exp.setPsf(coaddMean.getPsf())
1113  return exp
1114 
1115  def detectClip(self, exp, tempExpRefList):
1116  """!
1117  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1118 
1119  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1120  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1121  fraction to thresholds set in the config.
1122  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1123  two).
1124  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1125  the coaddTempExp.
1126  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1127  with the clipped footprints and a list of new masks for the coaddTempExps.
1128 
1129  \param[in] exp: Exposure to run detection on
1130  \param[in] tempExpRefList: List of data reference to tempExp
1131  \return struct containing:
1132  - clippedFootprints: list of clipped footprints
1133  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1134  - tempExpClipList: list of new masks for tempExp
1135  """
1136  mask = exp.getMaskedImage().getMask()
1137  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1138  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1139  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1140  # Merge positive and negative together footprints together
1141  fpSet.positive.merge(fpSet.negative)
1142  footprints = fpSet.positive
1143  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1144  ignoreMask = self.getBadPixelMask()
1145 
1146  clipFootprints = []
1147  clipIndices = []
1148 
1149  # build a list with a mask for each visit which can be modified with clipping information
1150  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.warpType),
1151  immediate=True).getMaskedImage().getMask() for
1152  tmpExpRef in tempExpRefList]
1153 
1154  for footprint in footprints.getFootprints():
1155  nPixel = footprint.getArea()
1156  overlap = [] # hold the overlap with each visit
1157  maskList = [] # which visit mask match
1158  indexList = [] # index of visit in global list
1159  for i, tmpExpMask in enumerate(tempExpClipList):
1160  # Determine the overlap with the footprint
1161  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1162  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1163  totPixel = nPixel - ignore
1164 
1165  # If we have more bad pixels than detection skip
1166  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1167  continue
1168  overlap.append(overlapDet/float(totPixel))
1169  maskList.append(tmpExpMask)
1170  indexList.append(i)
1171 
1172  overlap = numpy.array(overlap)
1173  if not len(overlap):
1174  continue
1175 
1176  keep = False # Should this footprint be marked as clipped?
1177  keepIndex = [] # Which tempExps does the clipped footprint belong to
1178 
1179  # If footprint only has one overlap use a lower threshold
1180  if len(overlap) == 1:
1181  if overlap[0] > self.config.minClipFootOverlapSingle:
1182  keep = True
1183  keepIndex = [0]
1184  else:
1185  # This is the general case where only visit should be clipped
1186  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1187  if len(clipIndex) == 1:
1188  keep = True
1189  keepIndex = [clipIndex[0]]
1190 
1191  # Test if there are clipped objects that overlap two different visits
1192  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1193  if len(clipIndex) == 2 and len(overlap) > 3:
1194  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1195  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1196  keep = True
1197  keepIndex = clipIndex
1198 
1199  if not keep:
1200  continue
1201 
1202  for index in keepIndex:
1203  footprint.spans.setMask(maskList[index], maskClipValue)
1204 
1205  clipIndices.append(numpy.array(indexList)[keepIndex])
1206  clipFootprints.append(footprint)
1207 
1208  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1209  tempExpClipList=tempExpClipList)
1210 
1211  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1212  """!
1213  \brief Find footprints from individual tempExp footprints for large footprints.
1214 
1215  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1216  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1217  significantly with each source in all the coaddTempExps.
1218 
1219  \param[in] tempExpClipList: List of tempExp masks with clipping information
1220  \param[in] clipFootprints: List of clipped footprints
1221  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1222  \param[in] maskClipValue: Mask value of clipped pixels
1223  \param[in] maskClipValue: Mask value of detected pixels
1224  \return list of big footprints
1225  """
1226  bigFootprintsCoadd = []
1227  ignoreMask = self.getBadPixelMask()
1228  for index, tmpExpMask in enumerate(tempExpClipList):
1229 
1230  # Create list of footprints from the DETECTED pixels
1231  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1232  afwImage.PARENT, True)
1233  maskVisitDet &= maskDetValue
1234  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1235 
1236  # build a mask of clipped footprints that are in this visit
1237  clippedFootprintsVisit = []
1238  for foot, clipIndex in zip(clipFootprints, clipIndices):
1239  if index not in clipIndex:
1240  continue
1241  clippedFootprintsVisit.append(foot)
1242  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1243  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1244 
1245  bigFootprintsVisit = []
1246  for foot in visitFootprints.getFootprints():
1247  if foot.getArea() < self.config.minBigOverlap:
1248  continue
1249  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1250  if nCount > self.config.minBigOverlap:
1251  bigFootprintsVisit.append(foot)
1252  bigFootprintsCoadd.append(foot)
1253 
1254  # Update single visit masks
1255  maskVisitClip.clearAllMaskPlanes()
1256  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1257  tmpExpMask |= maskVisitClip
1258 
1259  return bigFootprintsCoadd
1260 
1261 
1263  assembleStaticSkyModel = pexConfig.ConfigurableField(
1264  target=AssembleCoaddTask,
1265  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1266  " naive/first-iteration model of the static sky.",
1267  )
1268  detect = pexConfig.ConfigurableField(
1269  target=SourceDetectionTask,
1270  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1271  )
1272  maxNumEpochs = pexConfig.Field(
1273  doc="Maximum number of epochs/visits in which an artifact candidate can appear and still be masked. "
1274  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1275  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1276  "than maxNumEpochs, the artifact candidate is persistant rather than transient and not masked.",
1277  dtype=int,
1278  default=2
1279  )
1280  spatialThreshold = pexConfig.RangeField(
1281  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1282  "temporal criteria. If 0, clip all. If 1, clip none.",
1283  dtype=float,
1284  default=0.5,
1285  min=0., max=1.,
1286  inclusiveMin=True, inclusiveMax=True
1287  )
1288 
1289  def setDefaults(self):
1290  AssembleCoaddConfig.setDefaults(self)
1291  self.statistic = 'MEAN'
1292  self.assembleStaticSkyModel.warpType = 'psfMatched'
1293  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1294  self.assembleStaticSkyModel.sigmaClip = 1.5
1295  self.assembleStaticSkyModel.clipIter = 3
1296  self.assembleStaticSkyModel.doWrite = False
1297  self.detect.doTempLocalBackground = False
1298  self.detect.reEstimateBackground = False
1299  self.detect.returnOriginalFootprints = False
1300  self.detect.thresholdPolarity = "both"
1301  self.detect.thresholdValue = 5
1302  self.detect.nSigmaToGrow = 2
1303  self.detect.minPixels = 4
1304  self.detect.isotropicGrow = True
1305  self.detect.thresholdType = "pixel_stdev"
1306 
1307 
1308 
1314 
1316  """!
1317  \anchor CompareWarpAssembleCoaddTask_
1318 
1319  \brief Assemble a compareWarp coadded image from a set of warps
1320  by masking artifacts detected by comparing PSF-matched warps
1321 
1322  \section pipe_tasks_assembleCoadd_Contents Contents
1323  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1324  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1325  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1326  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1327  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1328  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1329 
1330  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1331 
1332  \copybrief CompareWarpAssembleCoaddTask
1333 
1334  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1335  outliers).
1336  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1337  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1338  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1339  pixels in the individual warps suspected to contain an artifact.
1340  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1341  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1342  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1343  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1344  such as bright stars.
1345  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1346  The temporalThreshold is the maximum fraction of epochs that the deviation can
1347  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1348  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1349  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1350  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1351  These regions will not contribute to the final coadd.
1352  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1353  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1354  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1355  surveys.
1356 
1357  CompareWarpAssembleCoaddTask sub-classes
1358  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1359  as a subtask to generate the TemplateCoadd (the model of the static sky)
1360 
1361  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1362  \copydoc \_\_init\_\_
1363 
1364  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1365  \copydoc run
1366 
1367  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1368  See \ref CompareWarpAssembleCoaddConfig
1369 
1370  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1371  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1372  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1373  files.
1374 
1375  This task supports the following debug variables:
1376  <dl>
1377  <dt>`saveCountIm`
1378  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1379  <dt> `saveAltMask`
1380  <dd> If True then save the new masks with CLIPPED planes as fits files to the `figPath`
1381  <dt> `figPath`
1382  <dd> Path to save the debug fits images and figures
1383  </dl>
1384 
1385  For example, put something like:
1386  @code{.py}
1387  import lsstDebug
1388  def DebugInfo(name):
1389  di = lsstDebug.getInfo(name)
1390  if name == "lsst.pipe.tasks.assembleCoadd":
1391  di.saveCountIm = True
1392  di.saveAltMask = True
1393  di.figPath = "/desired/path/to/debugging/output/images"
1394  return di
1395  lsstDebug.Info = DebugInfo
1396  @endcode
1397  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1398  flag.
1399  Some subtasks may have their own debug variables; see individual Task
1400  documentation
1401 
1402  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1403  CompareWarpAssembleCoaddTask
1404 
1405  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1406  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1407  '--compareWarpCoadd'.
1408  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1409  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1410  with a list of coaddTempExps to attempt to coadd (specified using
1411  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1412  Only the warps that cover the specified tract and patch will be coadded.
1413  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1414  command line argument:
1415  \code
1416  assembleCoadd.py --help
1417  \endcode
1418  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1419  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1420  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1421  packages.
1422  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1423  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1424  <DL>
1425  <DT>processCcd</DT>
1426  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1427  <DT>makeSkyMap</DT>
1428  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1429  <DT>makeCoaddTempExp</DT>
1430  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1431  </DL>
1432  We can perform all of these steps by running
1433  \code
1434  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1435  \endcode
1436  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1437  as follows:
1438  \code
1439  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1440  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1441  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1442  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1443  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1444  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1445  --selectId visit=903988 ccd=24
1446  \endcode
1447  This will process the HSC-I band data. The results are written in
1448  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1449  """
1450  ConfigClass = CompareWarpAssembleCoaddConfig
1451  _DefaultName = "compareWarpAssembleCoadd"
1452 
1453  def __init__(self, *args, **kwargs):
1454  """!
1455  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1456  """
1457  AssembleCoaddTask.__init__(self, *args, **kwargs)
1458  self.makeSubtask("assembleStaticSkyModel")
1459  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1460  self.makeSubtask("detect", schema=detectionSchema)
1461 
1462  def makeSupplementaryData(self, dataRef, selectDataList):
1463  """!
1464  \brief Make inputs specific to Subclass
1465 
1466  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1467  """
1468  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1469 
1470  if templateCoadd is None:
1471  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1472  self.assembleStaticSkyModel.warpType[1:])
1473  message = """No %(warpName)s warps were found to build the template coadd which is
1474  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1475  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1476  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1477 
1478  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1479  another algorithm like:
1480 
1481  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1482  config.assemble.retarget(SafeClipAssembleCoaddTask)
1483  """ % {"warpName": warpName}
1484  raise RuntimeError(message)
1485 
1486  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1487 
1488  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1489  supplementaryData, *args, **kwargs):
1490  """!
1491  \brief Assemble the coadd
1492 
1493  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1494  as the model of the static sky.
1495 
1496  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1497  new "CLIPPED" plane and updated "NO_DATA" plane.
1498  Then pass these alternative masks to the base class's assemble method.
1499 
1500  @param skyInfo: Patch geometry information
1501  @param tempExpRefList: List of data references to warps
1502  @param imageScalerList: List of image scalers
1503  @param weightList: List of weights
1504  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1505 
1506  return pipeBase.Struct with coaddExposure, nImage if requested
1507  """
1508  templateCoadd = supplementaryData.templateCoadd
1509  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1510  maskList = self.computeAltMaskList(tempExpRefList, spanSetMaskList)
1511  badMaskPlanes = self.config.badMaskPlanes[:]
1512  badMaskPlanes.append("CLIPPED")
1513  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1514 
1515  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1516  maskList, mask=badPixelMask)
1517 
1518  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1519  """!
1520  \brief Find artifacts
1521 
1522  Loop through warps twice. The first loop builds a map with the count of how many
1523  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1524  The second loop takes each difference image and filters the artifacts detected
1525  in each using count map to filter out variable sources and sources that are difficult to
1526  subtract cleanly.
1527 
1528  @param templateCoadd: Exposure to serve as model of static sky
1529  @param tempExpRefList: List of data references to warps
1530  @param imageScalerList: List of image scalers
1531  """
1532 
1533  self.log.debug("Generating Count Image, and mask lists.")
1534  coaddBBox = templateCoadd.getBBox()
1535  slateIm = afwImage.ImageU(coaddBBox)
1536  epochCountImage = afwImage.ImageU(coaddBBox)
1537  spanSetArtifactList = []
1538  spanSetNoDataMaskList = []
1539 
1540  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1541  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1542  if warpDiffExp is not None:
1543  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1544  fpSet.positive.merge(fpSet.negative)
1545  footprints = fpSet.positive
1546  slateIm.set(0)
1547  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1548  for spans in spanSetList:
1549  spans.setImage(slateIm, 1, doClip=True)
1550  epochCountImage += slateIm
1551 
1552  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1553  # undergo a second convolution. Pixels with data in the direct warp
1554  # but not in the PSF-matched warp will not have their artifacts detected.
1555  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1556  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1557  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1558  nansMask.setXY0(warpDiffExp.getXY0())
1559  else:
1560  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1561  # In this case, mask the whole epoch
1562  nansMask = afwImage.MaskX(coaddBBox, 1)
1563  spanSetList = []
1564 
1565  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1566 
1567  spanSetNoDataMaskList.append(spanSetNoDataMask)
1568  spanSetArtifactList.append(spanSetList)
1569 
1570  if lsstDebug.Info(__name__).saveCountIm:
1571  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1572  epochCountImage.writeFits(path)
1573 
1574  for i, spanSetList in enumerate(spanSetArtifactList):
1575  if spanSetList:
1576  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage)
1577  spanSetArtifactList[i] = filteredSpanSetList
1578 
1579  return pipeBase.Struct(artifacts=spanSetArtifactList,
1580  noData=spanSetNoDataMaskList)
1581 
1582  def computeAltMaskList(self, tempExpRefList, maskSpanSets):
1583  """!
1584  \brief Apply artifact span set lists to masks
1585 
1586  @param tempExpRefList: List of data references to warps
1587  @param maskSpanSets: Struct containing artifact and noData spanSet lists to apply
1588 
1589  return List of alternative masks
1590 
1591  Add artifact span set list as "CLIPPED" plane and NaNs to existing "NO_DATA" plane
1592  """
1593  spanSetMaskList = maskSpanSets.artifacts
1594  spanSetNoDataList = maskSpanSets.noData
1595  altMaskList = []
1596  for warpRef, artifacts, noData in zip(tempExpRefList, spanSetMaskList, spanSetNoDataList):
1597  warp = warpRef.get(self.getTempExpDatasetName(self.config.warpType), immediate=True)
1598  mask = warp.maskedImage.mask
1599  maskClipValue = mask.addMaskPlane("CLIPPED")
1600  noDataValue = mask.addMaskPlane("NO_DATA")
1601  for artifact in artifacts:
1602  artifact.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1603  for noDataRegion in noData:
1604  noDataRegion.clippedTo(mask.getBBox()).setMask(mask, 2**noDataValue)
1605  altMaskList.append(mask)
1606  if lsstDebug.Info(__name__).saveAltMask:
1607  mask.writeFits(self._dataRef2DebugPath("altMask", warpRef))
1608 
1609  return altMaskList
1610 
1611  def _filterArtifacts(self, spanSetList, epochCountImage):
1612  """!
1613  \brief Filter artifact candidates
1614 
1615  @param spanSetList: List of SpanSets representing artifact candidates
1616  @param epochCountImage: Image of accumulated number of warpDiff detections
1617 
1618  return List of SpanSets with artifacts
1619  """
1620 
1621  maskSpanSetList = []
1622  x0, y0 = epochCountImage.getXY0()
1623  for i, span in enumerate(spanSetList):
1624  y, x = span.indices()
1625  counts = epochCountImage.array[[y1 - y0 for y1 in y], [x1 - x0 for x1 in x]]
1626  nCountsBelowThreshold = numpy.count_nonzero((counts > 0) & (counts <= self.config.maxNumEpochs))
1627  percentBelowThreshold = nCountsBelowThreshold / len(counts)
1628  if percentBelowThreshold > self.config.spatialThreshold:
1629  maskSpanSetList.append(span)
1630  return maskSpanSetList
1631 
1632  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1633  """!
1634  \brief Fetch a warp from the butler and return a warpDiff
1635 
1636  @param warpRef: `Butler dataRef` for the warp
1637  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1638  @param templateCoadd: Exposure to be substracted from the scaled warp
1639 
1640  return Exposure of the image difference between the warp and template
1641  """
1642 
1643  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1644  warpName = self.getTempExpDatasetName('psfMatched')
1645  if not warpRef.datasetExists(warpName):
1646  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1647  return None
1648  warp = warpRef.get(warpName, immediate=True)
1649  # direct image scaler OK for PSF-matched Warp
1650  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1651  mi = warp.getMaskedImage()
1652  mi -= templateCoadd.getMaskedImage()
1653  return warp
1654 
1655  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1656  """!
1657  \brief Return a path to which to write debugging output
1658 
1659  @param prefix: string, prefix for filename
1660  @param warpRef: Butler dataRef
1661  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1662  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1663 
1664  Creates a hyphen-delimited string of dataId values for simple filenames.
1665  """
1666  if coaddLevel:
1667  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1668  else:
1669  keys = warpRef.dataId.keys()
1670  keyList = sorted(keys, reverse=True)
1671  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1672  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1673  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def _filterArtifacts(self, spanSetList, epochCountImage)
Filter artifact candidates.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def computeAltMaskList(self, tempExpRefList, maskSpanSets)
Apply artifact span set lists to masks.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue)
Find footprints from individual tempExp footprints for large footprints.
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.