lsst.pipe.tasks  14.0-44-gc76a4e8b+1
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import os
26 import numpy
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, scaleVariance
40 from .interpImage import InterpImageTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms import SourceDetectionTask
44 
45 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
46 
47 
48 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
49  """!
50 \anchor AssembleCoaddConfig_
51 
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53  """
54  warpType = pexConfig.Field(
55  doc="Warp name: one of 'direct' or 'psfMatched'",
56  dtype=str,
57  default="direct",
58  )
59  subregionSize = pexConfig.ListField(
60  dtype=int,
61  doc="Width, height of stack subregion size; "
62  "make small enough that a full stack of images will fit into memory at once.",
63  length=2,
64  default=(2000, 2000),
65  )
66  statistic = pexConfig.Field(
67  dtype=str,
68  doc="Main stacking statistic for aggregating over the epochs.",
69  default="MEANCLIP",
70  )
71  doSigmaClip = pexConfig.Field(
72  dtype=bool,
73  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
74  default=False,
75  )
76  sigmaClip = pexConfig.Field(
77  dtype=float,
78  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
79  default=3.0,
80  )
81  clipIter = pexConfig.Field(
82  dtype=int,
83  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
84  default=2,
85  )
86  calcErrorFromInputVariance = pexConfig.Field(
87  dtype=bool,
88  doc="Calculate coadd variance from input variance by stacking statistic."
89  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
90  default=True,
91  )
92  scaleZeroPoint = pexConfig.ConfigurableField(
93  target=ScaleZeroPointTask,
94  doc="Task to adjust the photometric zero point of the coadd temp exposures",
95  )
96  doInterp = pexConfig.Field(
97  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
98  dtype=bool,
99  default=True,
100  )
101  interpImage = pexConfig.ConfigurableField(
102  target=InterpImageTask,
103  doc="Task to interpolate (and extrapolate) over NaN pixels",
104  )
105  doWrite = pexConfig.Field(
106  doc="Persist coadd?",
107  dtype=bool,
108  default=True,
109  )
110  doNImage = pexConfig.Field(
111  doc="Create image of number of contributing exposures for each pixel",
112  dtype=bool,
113  default=False,
114  )
115  maskPropagationThresholds = pexConfig.DictField(
116  keytype=str,
117  itemtype=float,
118  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
119  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
120  "would have contributed exceeds this value."),
121  default={"SAT": 0.1},
122  )
123  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
124  doc="Mask planes to remove before coadding")
125  #
126  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
127  # To make this useful you *must* also configure the flags.pixel algorithm, for example
128  # by adding
129  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
130  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
131  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
132  #
133  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
134  doc="Set mask and flag bits for bright objects?")
135  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
136  doc="Name of mask bit used for bright objects")
137 
138  coaddPsf = pexConfig.ConfigField(
139  doc="Configuration for CoaddPsf",
140  dtype=measAlg.CoaddPsfConfig,
141  )
142  doAttachTransmissionCurve = pexConfig.Field(
143  dtype=bool, default=False, optional=False,
144  doc=("Attach a piecewise TransmissionCurve for the coadd? "
145  "(requires all input Exposures to have TransmissionCurves).")
146  )
147 
148  def setDefaults(self):
149  CoaddBaseTask.ConfigClass.setDefaults(self)
150  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
151 
152  def validate(self):
153  CoaddBaseTask.ConfigClass.validate(self)
154  if self.doPsfMatch:
155  # Backwards compatibility.
156  # Configs do not have loggers
157  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
158  self.warpType = 'psfMatched'
159  if self.doSigmaClip and self.statistic != "MEANCLIP":
160  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
161  self.statistic = "MEANCLIP"
162  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
163  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
164  "compute and set a non-zero coadd variance estimate." % (self.statistic))
165 
166  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
167  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
168  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
169  if str(k) not in unstackableStats]
170  raise ValueError("statistic %s is not allowed. Please choose one of %s."
171  % (self.statistic, stackableStats))
172 
173 
174 
181  """!
182 \anchor AssembleCoaddTask_
183 
184 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
185 
186 \section pipe_tasks_assembleCoadd_Contents Contents
187  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
188  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
189  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
190  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
191  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
192  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
193 
194 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
195 
196 \copybrief AssembleCoaddTask_
197 
198 We want to assemble a coadded image from a set of Warps (also called
199 coadded temporary exposures or coaddTempExps.
200 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
201 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
202 Warps that cover the specified patch (pointed at by dataRef).
203 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
204 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
205 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
206 config.makePsfMatched set which of the warp types will be coadded.
207 The coadd is computed as a mean with optional outlier rejection.
208 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
209 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
210 
211 AssembleCoaddTask uses several sub-tasks. These are
212 <DL>
213  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
214  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
215  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
216  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
217 </DL>
218 You can retarget these subtasks if you wish.
219 
220 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
221 \copydoc \_\_init\_\_
222 
223 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
224 \copydoc run
225 
226 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
227 See \ref AssembleCoaddConfig_
228 
229 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
230 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
231 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
232 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
233 the documetation for the subtasks for further information.
234 
235 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
236 
237 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
238 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
239 a data reference to the tract patch and filter to be coadded (specified using
240 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
241 Warps to attempt to coadd (specified using
242 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
243 that cover the specified tract and patch will be coadded. A list of the available optional
244 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
245 \code
246 assembleCoadd.py --help
247 \endcode
248 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
249 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
250 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
251 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
252 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
253 <DL>
254  <DT>processCcd</DT>
255  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
256  <DT>makeSkyMap</DT>
257  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
258  <DT>makeCoaddTempExp</DT>
259  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
260 </DL>
261 We can perform all of these steps by running
262 \code
263 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
264 \endcode
265 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
266 follows:
267 \code
268 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
269 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
270 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
271 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
272 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
273 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
274 --selectId visit=903988 ccd=24
275 \endcode
276 that will process the HSC-I band data. The results are written in
277 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
278 
279 You may also choose to run:
280 \code
281 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
282 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
283 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
284 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
285 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
286 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
287 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
288 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
289 \endcode
290 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
291 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
292 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
293  """
294  ConfigClass = AssembleCoaddConfig
295  _DefaultName = "assembleCoadd"
296 
297  def __init__(self, *args, **kwargs):
298  """!
299  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
300  & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
301  """
302  CoaddBaseTask.__init__(self, *args, **kwargs)
303  self.makeSubtask("interpImage")
304  self.makeSubtask("scaleZeroPoint")
305 
306  if self.config.doMaskBrightObjects:
307  mask = afwImage.Mask()
308  try:
309  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
310  except pexExceptions.LsstCppException:
311  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
312  mask.getMaskPlaneDict().keys())
313  del mask
314 
315  self.warpType = self.config.warpType
316 
317  @pipeBase.timeMethod
318  def run(self, dataRef, selectDataList=[]):
319  """!
320  \brief Assemble a coadd from a set of Warps
321 
322  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
323  match the photometric zeropoint to a reference Warp. Assemble the Warps using
324  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
325  exposure.
326 
327  \anchor runParams
328  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
329  (if config.autoReference=False). Used to access the following data products:
330  - [in] self.config.coaddName + "Coadd_skyMap"
331  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
332  - [out] self.config.coaddName + "Coadd"
333  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
334  selected from this list based on overlap with the patch defined by dataRef.
335 
336  \return a pipeBase.Struct with fields:
337  - coaddExposure: coadded exposure
338  - nImage: exposure count image
339  """
340  skyInfo = self.getSkyInfo(dataRef)
341  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
342  if len(calExpRefList) == 0:
343  self.log.warn("No exposures to coadd")
344  return
345  self.log.info("Coadding %d exposures", len(calExpRefList))
346 
347  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
348  inputData = self.prepareInputs(tempExpRefList)
349  self.log.info("Found %d %s", len(inputData.tempExpRefList),
350  self.getTempExpDatasetName(self.warpType))
351  if len(inputData.tempExpRefList) == 0:
352  self.log.warn("No coadd temporary exposures found")
353  return
354 
355  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
356 
357  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
358  inputData.weightList, supplementaryData=supplementaryData)
359 
360  if self.config.doInterp:
361  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
362  # The variance must be positive; work around for DM-3201.
363  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
364  with numpy.errstate(invalid="ignore"):
365  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
366 
367  if self.config.doMaskBrightObjects:
368  brightObjectMasks = self.readBrightObjectMasks(dataRef)
369  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
370 
371  if self.config.doWrite:
372  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
373  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
374  if self.config.doNImage and retStruct.nImage is not None:
375  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
376 
377  return retStruct
378 
379  def makeSupplementaryData(self, dataRef, selectDataList):
380  """!
381  \brief Make additional inputs to assemble() specific to subclasses.
382 
383  Available to be implemented by subclasses only if they need the
384  coadd dataRef for performing preliminary processing before
385  assembling the coadd.
386  """
387  pass
388 
389  def getTempExpRefList(self, patchRef, calExpRefList):
390  """!
391  \brief Generate list data references corresponding to warped exposures that lie within the
392  patch to be coadded.
393 
394  \param[in] patchRef: Data reference for patch
395  \param[in] calExpRefList: List of data references for input calexps
396  \return List of Warp/CoaddTempExp data references
397  """
398  butler = patchRef.getButler()
399  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
400  self.getTempExpDatasetName(self.warpType))
401  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
402  g, groupData.keys) for
403  g in groupData.groups.keys()]
404  return tempExpRefList
405 
406  def prepareInputs(self, refList):
407  """!
408  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
409  for the photometric zero point.
410 
411  Each Warp has its own photometric zeropoint and background variance. Before coadding these
412  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
413  weight for each Warp.
414 
415  \param[in] refList: List of data references to tempExp
416  \return Struct:
417  - tempExprefList: List of data references to tempExp
418  - weightList: List of weightings
419  - imageScalerList: List of image scalers
420  """
421  statsCtrl = afwMath.StatisticsControl()
422  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
423  statsCtrl.setNumIter(self.config.clipIter)
424  statsCtrl.setAndMask(self.getBadPixelMask())
425  statsCtrl.setNanSafe(True)
426  # compute tempExpRefList: a list of tempExpRef that actually exist
427  # and weightList: a list of the weight of the associated coadd tempExp
428  # and imageScalerList: a list of scale factors for the associated coadd tempExp
429  tempExpRefList = []
430  weightList = []
431  imageScalerList = []
432  tempExpName = self.getTempExpDatasetName(self.warpType)
433  for tempExpRef in refList:
434  if not tempExpRef.datasetExists(tempExpName):
435  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
436  continue
437 
438  tempExp = tempExpRef.get(tempExpName, immediate=True)
439  maskedImage = tempExp.getMaskedImage()
440  imageScaler = self.scaleZeroPoint.computeImageScaler(
441  exposure=tempExp,
442  dataRef=tempExpRef,
443  )
444  try:
445  imageScaler.scaleMaskedImage(maskedImage)
446  except Exception as e:
447  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
448  continue
449  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
450  afwMath.MEANCLIP, statsCtrl)
451  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
452  weight = 1.0 / float(meanVar)
453  if not numpy.isfinite(weight):
454  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
455  continue
456  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
457 
458  del maskedImage
459  del tempExp
460 
461  tempExpRefList.append(tempExpRef)
462  weightList.append(weight)
463  imageScalerList.append(imageScaler)
464 
465  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
466  imageScalerList=imageScalerList)
467 
468  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
469  altMaskList=None, mask=None, supplementaryData=None):
470  """!
471  \anchor AssembleCoaddTask.assemble_
472 
473  \brief Assemble a coadd from input warps
474 
475  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
476  large area), the assembly is performed over small areas on the image at a time in order to
477  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
478  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
479  statistic specified. Set the edge bits the coadd mask based on the weight map.
480 
481  \param[in] skyInfo: Patch geometry information, from getSkyInfo
482  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
483  \param[in] imageScalerList: List of image scalers
484  \param[in] weightList: List of weights
485  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
486  \param[in] mask: Mask to ignore when coadding
487  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
488  Only used by subclasses that implement makeSupplementaryData and override assemble.
489  \return pipeBase.Struct with coaddExposure, nImage if requested
490  """
491  tempExpName = self.getTempExpDatasetName(self.warpType)
492  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
493  if mask is None:
494  mask = self.getBadPixelMask()
495 
496  statsCtrl = afwMath.StatisticsControl()
497  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
498  statsCtrl.setNumIter(self.config.clipIter)
499  statsCtrl.setAndMask(mask)
500  statsCtrl.setNanSafe(True)
501  statsCtrl.setWeighted(True)
502  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
503  for plane, threshold in self.config.maskPropagationThresholds.items():
504  bit = afwImage.Mask.getMaskPlane(plane)
505  statsCtrl.setMaskPropagationThreshold(bit, threshold)
506 
507  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
508 
509  if altMaskList is None:
510  altMaskList = [None]*len(tempExpRefList)
511 
512  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
513  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
514  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
515  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
516  coaddMaskedImage = coaddExposure.getMaskedImage()
517  subregionSizeArr = self.config.subregionSize
518  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
519  # if nImage is requested, create a zero one which can be passed to assembleSubregion
520  if self.config.doNImage:
521  nImage = afwImage.ImageU(skyInfo.bbox)
522  else:
523  nImage = None
524  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
525  try:
526  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
527  weightList, altMaskList, statsFlags, statsCtrl,
528  nImage=nImage)
529  except Exception as e:
530  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
531 
532  self.setInexactPsf(coaddMaskedImage.getMask())
533  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
534  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
535  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
536  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
537 
538  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
539  """!
540  \brief Set the metadata for the coadd
541 
542  This basic implementation simply sets the filter from the
543  first input.
544 
545  \param[in] coaddExposure: The target image for the coadd
546  \param[in] tempExpRefList: List of data references to tempExp
547  \param[in] weightList: List of weights
548  """
549  assert len(tempExpRefList) == len(weightList), "Length mismatch"
550  tempExpName = self.getTempExpDatasetName(self.warpType)
551  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
552  # (and we need more than just the PropertySet that contains the header), which is not possible
553  # with the current butler (see #2777).
554  tempExpList = [tempExpRef.get(tempExpName + "_sub",
555  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
556  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
557  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
558 
559  coaddExposure.setFilter(tempExpList[0].getFilter())
560  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
561  coaddInputs.ccds.reserve(numCcds)
562  coaddInputs.visits.reserve(len(tempExpList))
563 
564  for tempExp, weight in zip(tempExpList, weightList):
565  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
566  coaddInputs.visits.sort()
567  if self.warpType == "psfMatched":
568  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
569  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
570  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
571  # having the maximum width (sufficient because square)
572  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
573  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
574  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
575  else:
576  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
577  self.config.coaddPsf.makeControl())
578  coaddExposure.setPsf(psf)
579  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
580  coaddExposure.getWcs())
581  coaddExposure.getInfo().setApCorrMap(apCorrMap)
582  if self.config.doAttachTransmissionCurve:
583  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
584  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
585 
586  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
587  altMaskList, statsFlags, statsCtrl, nImage=None):
588  """!
589  \brief Assemble the coadd for a sub-region.
590 
591  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
592  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
593  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
594  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
595  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
596  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
597 
598  \param[in] coaddExposure: The target image for the coadd
599  \param[in] bbox: Sub-region to coadd
600  \param[in] tempExpRefList: List of data reference to tempExp
601  \param[in] imageScalerList: List of image scalers
602  \param[in] weightList: List of weights
603  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
604  Each element is dict with keys = mask plane name to which to add the spans
605  \param[in] statsFlags: afwMath.Property object for statistic for coadd
606  \param[in] statsCtrl: Statistics control object for coadd
607  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
608  """
609  self.log.debug("Computing coadd over %s", bbox)
610  tempExpName = self.getTempExpDatasetName(self.warpType)
611  coaddExposure.mask.addMaskPlane("REJECTED")
612  coaddExposure.mask.addMaskPlane("CLIPPED")
613  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
614  # If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
615  # or CLIPPED, set it to REJECTED on the coadd.
616  # If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
617  # if a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
618  edge = afwImage.Mask.getPlaneBitMask("EDGE")
619  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
620  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
621  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
622  maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask("REJECTED")),
623  (edge, coaddExposure.mask.getPlaneBitMask("SENSOR_EDGE")),
624  (clipped, clipped)]
625  maskedImageList = []
626  if nImage is not None:
627  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
628  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
629  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
630  maskedImage = exposure.getMaskedImage()
631  mask = maskedImage.getMask()
632  if altMask is not None:
633  self.applyAltMaskPlanes(mask, altMask)
634  imageScaler.scaleMaskedImage(maskedImage)
635 
636  # Add 1 for each pixel which is not excluded by the exclude mask.
637  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
638  if nImage is not None:
639  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
640  if self.config.removeMaskPlanes:
641  mask = maskedImage.getMask()
642  for maskPlane in self.config.removeMaskPlanes:
643  try:
644  mask &= ~mask.getPlaneBitMask(maskPlane)
645  except Exception as e:
646  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
647 
648  maskedImageList.append(maskedImage)
649 
650  with self.timer("stack"):
651  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
652  clipped, # also set output to CLIPPED if sigma-clipped
653  maskMap)
654  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
655  if nImage is not None:
656  nImage.assign(subNImage, bbox)
657 
658  def applyAltMaskPlanes(self, mask, altMaskSpans):
659  """!
660  \brief Apply in place alt mask formatted as SpanSets to a mask
661 
662  @param mask: original mask
663  @param altMaskSpans: Dictionary containing spanSet lists to apply.
664  Each element contains the new mask plane name
665  (e.g. "CLIPPED and/or "NO_DATA") as the key,
666  and list of SpanSets to apply to the mask
667  """
668  for plane, spanSetList in altMaskSpans.items():
669  maskClipValue = mask.addMaskPlane(plane)
670  for spanSet in spanSetList:
671  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
672  return mask
673 
674  def readBrightObjectMasks(self, dataRef):
675  """Returns None on failure"""
676  try:
677  return dataRef.get("brightObjectMask", immediate=True)
678  except Exception as e:
679  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
680  return None
681 
682  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
683  """Set the bright object masks
684 
685  exposure: Exposure under consideration
686  dataId: Data identifier dict for patch
687  brightObjectMasks: afwTable of bright objects to mask
688  """
689  #
690  # Check the metadata specifying the tract/patch/filter
691  #
692  if brightObjectMasks is None:
693  self.log.warn("Unable to apply bright object mask: none supplied")
694  return
695  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
696  md = brightObjectMasks.table.getMetadata()
697  for k in dataId:
698  if not md.exists(k):
699  self.log.warn("Expected to see %s in metadata", k)
700  else:
701  if md.get(k) != dataId[k]:
702  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
703 
704  mask = exposure.getMaskedImage().getMask()
705  wcs = exposure.getWcs()
706  plateScale = wcs.pixelScale().asArcseconds()
707 
708  for rec in brightObjectMasks:
709  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
710  if rec["type"] == "box":
711  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
712  width = rec["width"].asArcseconds()/plateScale # convert to pixels
713  height = rec["height"].asArcseconds()/plateScale # convert to pixels
714 
715  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
716  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
717 
718  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
719  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
720  spans = afwGeom.SpanSet(bbox)
721  elif rec["type"] == "circle":
722  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
723  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
724  else:
725  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
726  continue
727  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
728 
729  def setInexactPsf(self, mask):
730  """Set INEXACT_PSF mask plane
731 
732  If any of the input images isn't represented in the coadd (due to
733  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
734  these pixels.
735 
736  Parameters
737  ----------
738  mask : `lsst.afw.image.Mask`
739  Coadded exposure's mask, modified in-place.
740  """
741  mask.addMaskPlane("INEXACT_PSF")
742  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
743  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
744  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
745  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
746  array = mask.getArray()
747  selected = array & (sensorEdge | clipped | rejected) > 0
748  array[selected] |= inexactPsf
749 
750  @classmethod
751  def _makeArgumentParser(cls):
752  """!
753  \brief Create an argument parser
754  """
755  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
756  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
757  cls.ConfigClass().warpType + "Warp",
758  help="data ID, e.g. --id tract=12345 patch=1,2",
759  ContainerClass=AssembleCoaddDataIdContainer)
760  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
761  ContainerClass=SelectDataIdContainer)
762  return parser
763 
764 
765 def _subBBoxIter(bbox, subregionSize):
766  """!
767  \brief Iterate over subregions of a bbox
768 
769  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
770  \param[in] subregionSize: size of sub-bboxes
771 
772  \return subBBox: next sub-bounding box of size subregionSize or smaller;
773  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
774  but it will never be empty
775  """
776  if bbox.isEmpty():
777  raise RuntimeError("bbox %s is empty" % (bbox,))
778  if subregionSize[0] < 1 or subregionSize[1] < 1:
779  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
780 
781  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
782  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
783  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
784  subBBox.clip(bbox)
785  if subBBox.isEmpty():
786  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
787  (bbox, subregionSize, colShift, rowShift))
788  yield subBBox
789 
790 
791 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
792  """!
793  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
794  """
795 
796  def makeDataRefList(self, namespace):
797  """!
798  \brief Make self.refList from self.idList.
799  """
800  datasetType = namespace.config.coaddName + "Coadd"
801  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
802 
803  for dataId in self.idList:
804  # tract and patch are required
805  for key in keysCoadd:
806  if key not in dataId:
807  raise RuntimeError("--id must include " + key)
808 
809  dataRef = namespace.butler.dataRef(
810  datasetType=datasetType,
811  dataId=dataId,
812  )
813  self.refList.append(dataRef)
814 
815 
816 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
817  """!
818  \brief Function to count the number of pixels with a specific mask in a footprint.
819 
820  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
821  have bitmask set but do not have ignoreMask set. Return the count.
822 
823  \param[in] mask: mask to define intersection region by.
824  \parma[in] footprint: footprint to define the intersection region by.
825  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
826  \param[in] ignoreMask: pixels to not consider.
827  \return count of number of pixels in footprint with specified mask.
828  """
829  bbox = footprint.getBBox()
830  bbox.clip(mask.getBBox(afwImage.PARENT))
831  fp = afwImage.Mask(bbox)
832  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
833  footprint.spans.setMask(fp, bitmask)
834  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
835  (subMask.getArray() & ignoreMask) == 0).sum()
836 
837 
839  """!
840 \anchor SafeClipAssembleCoaddConfig
841 
842 \brief Configuration parameters for the SafeClipAssembleCoaddTask
843  """
844  clipDetection = pexConfig.ConfigurableField(
845  target=SourceDetectionTask,
846  doc="Detect sources on difference between unclipped and clipped coadd")
847  minClipFootOverlap = pexConfig.Field(
848  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
849  dtype=float,
850  default=0.6
851  )
852  minClipFootOverlapSingle = pexConfig.Field(
853  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
854  "clipped when only one visit overlaps",
855  dtype=float,
856  default=0.5
857  )
858  minClipFootOverlapDouble = pexConfig.Field(
859  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
860  "clipped when two visits overlap",
861  dtype=float,
862  default=0.45
863  )
864  maxClipFootOverlapDouble = pexConfig.Field(
865  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
866  "considering two visits",
867  dtype=float,
868  default=0.15
869  )
870  minBigOverlap = pexConfig.Field(
871  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
872  "when labeling clipped footprints",
873  dtype=int,
874  default=100
875  )
876 
877  def setDefaults(self):
878  # The numeric values for these configuration parameters were empirically determined, future work
879  # may further refine them.
880  AssembleCoaddConfig.setDefaults(self)
881  self.clipDetection.doTempLocalBackground = False
882  self.clipDetection.reEstimateBackground = False
883  self.clipDetection.returnOriginalFootprints = False
884  self.clipDetection.thresholdPolarity = "both"
885  self.clipDetection.thresholdValue = 2
886  self.clipDetection.nSigmaToGrow = 2
887  self.clipDetection.minPixels = 4
888  self.clipDetection.isotropicGrow = True
889  self.clipDetection.thresholdType = "pixel_stdev"
890  self.sigmaClip = 1.5
891  self.clipIter = 3
892  self.statistic = "MEAN"
893 
894  def validate(self):
895  if self.doSigmaClip:
896  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
897  "Ignoring doSigmaClip.")
898  self.doSigmaClip = False
899  if self.statistic != "MEAN":
900  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
901  "(%s chosen). Please set statistic to MEAN."
902  % (self.statistic))
903  AssembleCoaddTask.ConfigClass.validate(self)
904 
905 
906 
912 
913 
915  """!
916  \anchor SafeClipAssembleCoaddTask_
917 
918  \brief Assemble a coadded image from a set of coadded temporary exposures,
919  being careful to clip & flag areas with potential artifacts.
920 
921  \section pipe_tasks_assembleCoadd_Contents Contents
922  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
923  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
924  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
925  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
926  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
927  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
928 
929  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
930 
931  \copybrief SafeClipAssembleCoaddTask
932 
933  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
934  SafeClipAssembleCoaddTask subtasks that task.
935  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
936  outliers).
937  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
938  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
939  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
940  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
941  that there is an outlier and ii. this outlier appears on only one or two images.
942  Such regions will not contribute to the final coadd.
943  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
944  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
945  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
946  surveys.
947 
948  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
949  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
950  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
951 
952  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
953  \copydoc \_\_init\_\_
954 
955  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
956  \copydoc run
957 
958  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
959  See \ref SafeClipAssembleCoaddConfig
960 
961  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
962  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
963  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
964  files.
965  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
966  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
967  for further information.
968 
969  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
970  SafeClipAssembleCoaddTask
971 
972  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
973  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
974  '--legacyCoadd'.
975  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
976  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
977  with a list of coaddTempExps to attempt to coadd (specified using
978  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
979  Only the coaddTempExps that cover the specified tract and patch will be coadded.
980  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
981  command line argument:
982  \code
983  assembleCoadd.py --help
984  \endcode
985  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
986  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
987  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
988  packages.
989  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
990  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
991  <DL>
992  <DT>processCcd</DT>
993  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
994  <DT>makeSkyMap</DT>
995  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
996  <DT>makeCoaddTempExp</DT>
997  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
998  </DL>
999  We can perform all of these steps by running
1000  \code
1001  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1002  \endcode
1003  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1004  as follows:
1005  \code
1006  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1007  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1008  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1009  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1010  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1011  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1012  --selectId visit=903988 ccd=24
1013  \endcode
1014  This will process the HSC-I band data. The results are written in
1015  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1016 
1017  You may also choose to run:
1018  \code
1019  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1020  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1021  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1022  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1023  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1024  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1025  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1026  --selectId visit=903346 ccd=12
1027  \endcode
1028  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1029  discussed in \ref pipeTasks_multiBand.
1030  """
1031  ConfigClass = SafeClipAssembleCoaddConfig
1032  _DefaultName = "safeClipAssembleCoadd"
1033 
1034  def __init__(self, *args, **kwargs):
1035  """!
1036  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1037  """
1038  AssembleCoaddTask.__init__(self, *args, **kwargs)
1039  schema = afwTable.SourceTable.makeMinimalSchema()
1040  self.makeSubtask("clipDetection", schema=schema)
1041 
1042  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1043  """!
1044  \brief Assemble the coadd for a region
1045 
1046  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1047  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1048  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1049  overlap between the clipped region and a source.
1050  This leaves us with a set of footprints from the difference image that have been identified as having
1051  occured on just one or two individual visits. However, these footprints were generated from a
1052  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1053  footprints acrosss the coadd difference in this process.
1054  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1055  these are big footprints.
1056  Combine the small and big clipped footprints and mark them on a new bad mask plane
1057  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1058  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1059  bad mask plane.
1060 
1061  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1062  parent task.
1063 
1064  @param skyInfo: Patch geometry information, from getSkyInfo
1065  @param tempExpRefList: List of data reference to tempExp
1066  @param imageScalerList: List of image scalers
1067  @param weightList: List of weights
1068  return pipeBase.Struct with coaddExposure, nImage
1069  """
1070  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1071  mask = exp.getMaskedImage().getMask()
1072  mask.addMaskPlane("CLIPPED")
1073 
1074  result = self.detectClip(exp, tempExpRefList)
1075 
1076  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1077 
1078  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1079  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1080  # Append big footprints from individual Warps to result.clipSpans
1081  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1082  result.detectionFootprints, maskClipValue, maskDetValue,
1083  exp.getBBox())
1084  # Create mask of the current clipped footprints
1085  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1086  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1087 
1088  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1089  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1090  maskClip |= maskClipBig
1091 
1092  # Assemble coadd from base class, but ignoring CLIPPED pixels
1093  badMaskPlanes = self.config.badMaskPlanes[:]
1094  badMaskPlanes.append("CLIPPED")
1095  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1096  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1097  result.clipSpans, mask=badPixelMask)
1098 
1099  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1100  """!
1101  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1102 
1103  Generate a difference image between clipped and unclipped coadds.
1104  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1105  Return the difference image.
1106 
1107  @param skyInfo: Patch geometry information, from getSkyInfo
1108  @param tempExpRefList: List of data reference to tempExp
1109  @param imageScalerList: List of image scalers
1110  @param weightList: List of weights
1111  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1112  """
1113  # Clone and upcast self.config because current self.config is frozen
1114  config = AssembleCoaddConfig()
1115  # getattr necessary because subtasks do not survive Config.toDict()
1116  configIntersection = {k: getattr(self.config, k)
1117  for k, v in self.config.toDict().items() if (k in config.keys())}
1118  config.update(**configIntersection)
1119 
1120  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1121  config.statistic = 'MEAN'
1122  task = AssembleCoaddTask(config=config)
1123  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1124 
1125  config.statistic = 'MEANCLIP'
1126  task = AssembleCoaddTask(config=config)
1127  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1128 
1129  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1130  coaddDiff -= coaddClip.getMaskedImage()
1131  exp = afwImage.ExposureF(coaddDiff)
1132  exp.setPsf(coaddMean.getPsf())
1133  return exp
1134 
1135  def detectClip(self, exp, tempExpRefList):
1136  """!
1137  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1138 
1139  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1140  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1141  fraction to thresholds set in the config.
1142  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1143  two).
1144  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1145  the coaddTempExp.
1146  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1147  with the clipped footprints and a list of new masks for the coaddTempExps.
1148 
1149  \param[in] exp: Exposure to run detection on
1150  \param[in] tempExpRefList: List of data reference to tempExp
1151  \return struct containing:
1152  - clipFootprints: list of clipped footprints
1153  - clipIndices: indices for each clippedFootprint in tempExpRefList
1154  - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new
1155  maskplane name ("CLIPPED")" as the key and list of SpanSets as value
1156  - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints
1157  """
1158  mask = exp.getMaskedImage().getMask()
1159  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1160  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1161  # Merge positive and negative together footprints together
1162  fpSet.positive.merge(fpSet.negative)
1163  footprints = fpSet.positive
1164  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1165  ignoreMask = self.getBadPixelMask()
1166 
1167  clipFootprints = []
1168  clipIndices = []
1169  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1170 
1171  # for use by detectClipBig
1172  visitDetectionFootprints = []
1173 
1174  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1175  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1176  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1177 
1178  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1179  for i, warpRef in enumerate(tempExpRefList):
1180  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1181  immediate=True).getMaskedImage().getMask()
1182  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1183  afwImage.PARENT, True)
1184  maskVisitDet &= maskDetValue
1185  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1186  visitDetectionFootprints.append(visitFootprints)
1187 
1188  for j, footprint in enumerate(footprints.getFootprints()):
1189  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1190  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1191 
1192  # build a list of clipped spans for each visit
1193  for j, footprint in enumerate(footprints.getFootprints()):
1194  nPixel = footprint.getArea()
1195  overlap = [] # hold the overlap with each visit
1196  indexList = [] # index of visit in global list
1197  for i in range(len(tempExpRefList)):
1198  ignore = ignoreArr[i, j]
1199  overlapDet = overlapDetArr[i, j]
1200  totPixel = nPixel - ignore
1201 
1202  # If we have more bad pixels than detection skip
1203  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1204  continue
1205  overlap.append(overlapDet/float(totPixel))
1206  indexList.append(i)
1207 
1208  overlap = numpy.array(overlap)
1209  if not len(overlap):
1210  continue
1211 
1212  keep = False # Should this footprint be marked as clipped?
1213  keepIndex = [] # Which tempExps does the clipped footprint belong to
1214 
1215  # If footprint only has one overlap use a lower threshold
1216  if len(overlap) == 1:
1217  if overlap[0] > self.config.minClipFootOverlapSingle:
1218  keep = True
1219  keepIndex = [0]
1220  else:
1221  # This is the general case where only visit should be clipped
1222  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1223  if len(clipIndex) == 1:
1224  keep = True
1225  keepIndex = [clipIndex[0]]
1226 
1227  # Test if there are clipped objects that overlap two different visits
1228  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1229  if len(clipIndex) == 2 and len(overlap) > 3:
1230  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1231  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1232  keep = True
1233  keepIndex = clipIndex
1234 
1235  if not keep:
1236  continue
1237 
1238  for index in keepIndex:
1239  globalIndex = indexList[index]
1240  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1241 
1242  clipIndices.append(numpy.array(indexList)[keepIndex])
1243  clipFootprints.append(footprint)
1244 
1245  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1246  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1247 
1248  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1249  maskClipValue, maskDetValue, coaddBBox):
1250  """!
1251  \brief Return individual warp footprints for large artifacts and append them to clipList in place
1252 
1253  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1254  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1255  significantly with each source in all the coaddTempExps.
1256  \param[in] clipList: List of alt mask SpanSets with clipping information. Modified.
1257  \param[in] clipFootprints: List of clipped footprints
1258  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1259  \param[in] maskClipValue: Mask value of clipped pixels
1260  \param[in] maskDetValue: Mask value of detected pixels
1261  \param[in] coaddBBox: BBox of the coadd and warps
1262  \return list of big footprints
1263  """
1264  bigFootprintsCoadd = []
1265  ignoreMask = self.getBadPixelMask()
1266  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1267  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1268  for footprint in visitFootprints.getFootprints():
1269  footprint.spans.setMask(maskVisitDet, maskDetValue)
1270 
1271  # build a mask of clipped footprints that are in this visit
1272  clippedFootprintsVisit = []
1273  for foot, clipIndex in zip(clipFootprints, clipIndices):
1274  if index not in clipIndex:
1275  continue
1276  clippedFootprintsVisit.append(foot)
1277  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1278  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1279 
1280  bigFootprintsVisit = []
1281  for foot in visitFootprints.getFootprints():
1282  if foot.getArea() < self.config.minBigOverlap:
1283  continue
1284  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1285  if nCount > self.config.minBigOverlap:
1286  bigFootprintsVisit.append(foot)
1287  bigFootprintsCoadd.append(foot)
1288 
1289  for footprint in bigFootprintsVisit:
1290  clippedSpans["CLIPPED"].append(footprint.spans)
1291 
1292  return bigFootprintsCoadd
1293 
1294 
1296  assembleStaticSkyModel = pexConfig.ConfigurableField(
1297  target=AssembleCoaddTask,
1298  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1299  " naive/first-iteration model of the static sky.",
1300  )
1301  detect = pexConfig.ConfigurableField(
1302  target=SourceDetectionTask,
1303  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1304  )
1305  maxNumEpochs = pexConfig.Field(
1306  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1307  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1308  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1309  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1310  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1311  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1312  "than transient and not masked.",
1313  dtype=int,
1314  default=2
1315  )
1316  maxFractionEpochsLow = pexConfig.RangeField(
1317  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1318  "Effective maxNumEpochs = "
1319  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1320  dtype=float,
1321  default=0.4,
1322  min=0., max=1.,
1323  )
1324  maxFractionEpochsHigh = pexConfig.RangeField(
1325  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1326  "Effective maxNumEpochs = "
1327  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1328  dtype=float,
1329  default=0.03,
1330  min=0., max=1.,
1331  )
1332  spatialThreshold = pexConfig.RangeField(
1333  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1334  "temporal criteria. If 0, clip all. If 1, clip none.",
1335  dtype=float,
1336  default=0.5,
1337  min=0., max=1.,
1338  inclusiveMin=True, inclusiveMax=True
1339  )
1340  doScaleWarpVariance = pexConfig.Field(
1341  doc="Rescale Warp variance plane using empirical noise?",
1342  dtype=bool,
1343  default=True,
1344  )
1345  maskScaleWarpVariance = pexConfig.ListField(
1346  dtype=str,
1347  default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
1348  doc="Mask planes for pixels to ignore when rescaling warp variance",
1349  )
1350 
1351  def setDefaults(self):
1352  AssembleCoaddConfig.setDefaults(self)
1353  self.statistic = 'MEAN'
1354  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1355  self.assembleStaticSkyModel.warpType = 'psfMatched'
1356  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1357  self.assembleStaticSkyModel.sigmaClip = 1.5
1358  self.assembleStaticSkyModel.clipIter = 3
1359  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1360  self.assembleStaticSkyModel.doWrite = False
1361  self.detect.doTempLocalBackground = False
1362  self.detect.reEstimateBackground = False
1363  self.detect.returnOriginalFootprints = False
1364  self.detect.thresholdPolarity = "both"
1365  self.detect.thresholdValue = 5
1366  self.detect.nSigmaToGrow = 2
1367  self.detect.minPixels = 4
1368  self.detect.isotropicGrow = True
1369  self.detect.thresholdType = "pixel_stdev"
1370 
1371 
1372 
1378 
1380  """!
1381  \anchor CompareWarpAssembleCoaddTask_
1382 
1383  \brief Assemble a compareWarp coadded image from a set of warps
1384  by masking artifacts detected by comparing PSF-matched warps
1385 
1386  \section pipe_tasks_assembleCoadd_Contents Contents
1387  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1388  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1389  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1390  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1391  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1392  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1393 
1394  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1395 
1396  \copybrief CompareWarpAssembleCoaddTask
1397 
1398  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1399  outliers).
1400  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1401  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1402  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1403  pixels in the individual warps suspected to contain an artifact.
1404  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1405  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1406  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1407  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1408  such as bright stars.
1409  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1410  The temporalThreshold is the maximum fraction of epochs that the deviation can
1411  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1412  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1413  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1414  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1415  These regions will not contribute to the final coadd.
1416  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1417  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1418  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1419  surveys.
1420 
1421  CompareWarpAssembleCoaddTask sub-classes
1422  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1423  as a subtask to generate the TemplateCoadd (the model of the static sky)
1424 
1425  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1426  \copydoc \_\_init\_\_
1427 
1428  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1429  \copydoc run
1430 
1431  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1432  See \ref CompareWarpAssembleCoaddConfig
1433 
1434  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1435  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1436  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1437  files.
1438 
1439  This task supports the following debug variables:
1440  <dl>
1441  <dt>`saveCountIm`
1442  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1443  <dt> `figPath`
1444  <dd> Path to save the debug fits images and figures
1445  </dl>
1446 
1447  For example, put something like:
1448  @code{.py}
1449  import lsstDebug
1450  def DebugInfo(name):
1451  di = lsstDebug.getInfo(name)
1452  if name == "lsst.pipe.tasks.assembleCoadd":
1453  di.saveCountIm = True
1454  di.figPath = "/desired/path/to/debugging/output/images"
1455  return di
1456  lsstDebug.Info = DebugInfo
1457  @endcode
1458  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1459  flag.
1460  Some subtasks may have their own debug variables; see individual Task
1461  documentation
1462 
1463  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1464  CompareWarpAssembleCoaddTask
1465 
1466  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1467  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1468  '--compareWarpCoadd'.
1469  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1470  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1471  with a list of coaddTempExps to attempt to coadd (specified using
1472  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1473  Only the warps that cover the specified tract and patch will be coadded.
1474  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1475  command line argument:
1476  \code
1477  assembleCoadd.py --help
1478  \endcode
1479  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1480  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1481  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1482  packages.
1483  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1484  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1485  <DL>
1486  <DT>processCcd</DT>
1487  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1488  <DT>makeSkyMap</DT>
1489  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1490  <DT>makeCoaddTempExp</DT>
1491  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1492  </DL>
1493  We can perform all of these steps by running
1494  \code
1495  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1496  \endcode
1497  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1498  as follows:
1499  \code
1500  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1501  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1502  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1503  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1504  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1505  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1506  --selectId visit=903988 ccd=24
1507  \endcode
1508  This will process the HSC-I band data. The results are written in
1509  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1510  """
1511  ConfigClass = CompareWarpAssembleCoaddConfig
1512  _DefaultName = "compareWarpAssembleCoadd"
1513 
1514  def __init__(self, *args, **kwargs):
1515  """!
1516  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1517  """
1518  AssembleCoaddTask.__init__(self, *args, **kwargs)
1519  self.makeSubtask("assembleStaticSkyModel")
1520  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1521  self.makeSubtask("detect", schema=detectionSchema)
1522 
1523  def makeSupplementaryData(self, dataRef, selectDataList):
1524  """!
1525  \brief Make inputs specific to Subclass
1526 
1527  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1528  """
1529  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1530 
1531  if templateCoadd is None:
1532  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1533  self.assembleStaticSkyModel.warpType[1:])
1534  message = """No %(warpName)s warps were found to build the template coadd which is
1535  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1536  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1537  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1538 
1539  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1540  another algorithm like:
1541 
1542  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1543  config.assemble.retarget(SafeClipAssembleCoaddTask)
1544  """ % {"warpName": warpName}
1545  raise RuntimeError(message)
1546 
1547  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1548 
1549  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1550  supplementaryData, *args, **kwargs):
1551  """!
1552  \brief Assemble the coadd
1553 
1554  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1555  as the model of the static sky.
1556 
1557  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1558  new "CLIPPED" plane and updated "NO_DATA" plane.
1559  Then pass these alternative masks to the base class's assemble method.
1560 
1561  @param skyInfo: Patch geometry information
1562  @param tempExpRefList: List of data references to warps
1563  @param imageScalerList: List of image scalers
1564  @param weightList: List of weights
1565  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1566 
1567  return pipeBase.Struct with coaddExposure, nImage if requested
1568  """
1569  templateCoadd = supplementaryData.templateCoadd
1570  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1571  badMaskPlanes = self.config.badMaskPlanes[:]
1572  badMaskPlanes.append("CLIPPED")
1573  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1574 
1575  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1576  spanSetMaskList, mask=badPixelMask)
1577 
1578  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1579  """!
1580  \brief Find artifacts
1581 
1582  Loop through warps twice. The first loop builds a map with the count of how many
1583  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1584  The second loop takes each difference image and filters the artifacts detected
1585  in each using count map to filter out variable sources and sources that are difficult to
1586  subtract cleanly.
1587 
1588  @param templateCoadd: Exposure to serve as model of static sky
1589  @param tempExpRefList: List of data references to warps
1590  @param imageScalerList: List of image scalers
1591  """
1592 
1593  self.log.debug("Generating Count Image, and mask lists.")
1594  coaddBBox = templateCoadd.getBBox()
1595  slateIm = afwImage.ImageU(coaddBBox)
1596  epochCountImage = afwImage.ImageU(coaddBBox)
1597  nImage = afwImage.ImageU(coaddBBox)
1598  spanSetArtifactList = []
1599  spanSetNoDataMaskList = []
1600 
1601  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1602  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1603  if warpDiffExp is not None:
1604  nImage.array += numpy.where(numpy.isnan(warpDiffExp.image.array),
1605  0, 1).astype(numpy.uint16)
1606  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1607  fpSet.positive.merge(fpSet.negative)
1608  footprints = fpSet.positive
1609  slateIm.set(0)
1610  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1611  for spans in spanSetList:
1612  spans.setImage(slateIm, 1, doClip=True)
1613  epochCountImage += slateIm
1614 
1615  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1616  # undergo a second convolution. Pixels with data in the direct warp
1617  # but not in the PSF-matched warp will not have their artifacts detected.
1618  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1619  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1620  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1621  nansMask.setXY0(warpDiffExp.getXY0())
1622  else:
1623  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1624  # In this case, mask the whole epoch
1625  nansMask = afwImage.MaskX(coaddBBox, 1)
1626  spanSetList = []
1627 
1628  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1629 
1630  spanSetNoDataMaskList.append(spanSetNoDataMask)
1631  spanSetArtifactList.append(spanSetList)
1632 
1633  if lsstDebug.Info(__name__).saveCountIm:
1634  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1635  epochCountImage.writeFits(path)
1636 
1637  for i, spanSetList in enumerate(spanSetArtifactList):
1638  if spanSetList:
1639  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage, nImage)
1640  spanSetArtifactList[i] = filteredSpanSetList
1641 
1642  altMasks = []
1643  for artifacts, noData in zip(spanSetArtifactList, spanSetNoDataMaskList):
1644  altMasks.append({'CLIPPED': artifacts,
1645  'NO_DATA': noData})
1646  return altMasks
1647 
1648  def _filterArtifacts(self, spanSetList, epochCountImage, nImage):
1649  """!
1650  \brief Filter artifact candidates
1651 
1652  @param spanSetList: List of SpanSets representing artifact candidates
1653  @param epochCountImage: Image of accumulated number of warpDiff detections
1654  @param nImage: Image of the accumulated number of total epochs contributing
1655 
1656  return List of SpanSets with artifacts
1657  """
1658 
1659  maskSpanSetList = []
1660  x0, y0 = epochCountImage.getXY0()
1661  for i, span in enumerate(spanSetList):
1662  y, x = span.indices()
1663  yIdxLocal = [y1 - y0 for y1 in y]
1664  xIdxLocal = [x1 - x0 for x1 in x]
1665  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1666  totalN = nImage.array[yIdxLocal, xIdxLocal]
1667 
1668  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1669  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1670  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1671  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1672  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1673  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1674  (outlierN <= effectiveMaxNumEpochs))
1675  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1676  if percentBelowThreshold > self.config.spatialThreshold:
1677  maskSpanSetList.append(span)
1678  return maskSpanSetList
1679 
1680  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1681  """!
1682  \brief Fetch a warp from the butler and return a warpDiff
1683 
1684  @param warpRef: `Butler dataRef` for the warp
1685  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1686  @param templateCoadd: Exposure to be substracted from the scaled warp
1687 
1688  return Exposure of the image difference between the warp and template
1689  """
1690 
1691  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1692  warpName = self.getTempExpDatasetName('psfMatched')
1693  if not warpRef.datasetExists(warpName):
1694  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1695  return None
1696  warp = warpRef.get(warpName, immediate=True)
1697  # direct image scaler OK for PSF-matched Warp
1698  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1699  mi = warp.getMaskedImage()
1700  if self.config.doScaleWarpVariance:
1701  scaleVariance(mi, self.config.maskScaleWarpVariance,
1702  log=self.log)
1703  mi -= templateCoadd.getMaskedImage()
1704  return warp
1705 
1706  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1707  """!
1708  \brief Return a path to which to write debugging output
1709 
1710  @param prefix: string, prefix for filename
1711  @param warpRef: Butler dataRef
1712  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1713  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1714 
1715  Creates a hyphen-delimited string of dataId values for simple filenames.
1716  """
1717  if coaddLevel:
1718  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1719  else:
1720  keys = warpRef.dataId.keys()
1721  keyList = sorted(keys, reverse=True)
1722  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1723  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1724  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def _filterArtifacts(self, spanSetList, epochCountImage, nImage)
Filter artifact candidates.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...