lsst.pipe.tasks  15.0-5-g389937dc+5
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import os
26 import numpy
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
40 from .interpImage import InterpImageTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from .scaleVariance import ScaleVarianceTask
44 from lsst.meas.algorithms import SourceDetectionTask
45 
46 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
47 
48 
49 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
50  """!
51 \anchor AssembleCoaddConfig_
52 
53 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
54  """
55  warpType = pexConfig.Field(
56  doc="Warp name: one of 'direct' or 'psfMatched'",
57  dtype=str,
58  default="direct",
59  )
60  subregionSize = pexConfig.ListField(
61  dtype=int,
62  doc="Width, height of stack subregion size; "
63  "make small enough that a full stack of images will fit into memory at once.",
64  length=2,
65  default=(2000, 2000),
66  )
67  statistic = pexConfig.Field(
68  dtype=str,
69  doc="Main stacking statistic for aggregating over the epochs.",
70  default="MEANCLIP",
71  )
72  doSigmaClip = pexConfig.Field(
73  dtype=bool,
74  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
75  default=False,
76  )
77  sigmaClip = pexConfig.Field(
78  dtype=float,
79  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
80  default=3.0,
81  )
82  clipIter = pexConfig.Field(
83  dtype=int,
84  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
85  default=2,
86  )
87  calcErrorFromInputVariance = pexConfig.Field(
88  dtype=bool,
89  doc="Calculate coadd variance from input variance by stacking statistic."
90  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
91  default=True,
92  )
93  scaleZeroPoint = pexConfig.ConfigurableField(
94  target=ScaleZeroPointTask,
95  doc="Task to adjust the photometric zero point of the coadd temp exposures",
96  )
97  doInterp = pexConfig.Field(
98  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
99  dtype=bool,
100  default=True,
101  )
102  interpImage = pexConfig.ConfigurableField(
103  target=InterpImageTask,
104  doc="Task to interpolate (and extrapolate) over NaN pixels",
105  )
106  doWrite = pexConfig.Field(
107  doc="Persist coadd?",
108  dtype=bool,
109  default=True,
110  )
111  doNImage = pexConfig.Field(
112  doc="Create image of number of contributing exposures for each pixel",
113  dtype=bool,
114  default=False,
115  )
116  doUsePsfMatchedPolygons = pexConfig.Field(
117  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
118  dtype=bool,
119  default=False,
120  )
121  maskPropagationThresholds = pexConfig.DictField(
122  keytype=str,
123  itemtype=float,
124  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126  "would have contributed exceeds this value."),
127  default={"SAT": 0.1},
128  )
129  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
130  doc="Mask planes to remove before coadding")
131  #
132  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
133  # To make this useful you *must* also configure the flags.pixel algorithm, for example
134  # by adding
135  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
136  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
137  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
138  #
139  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
140  doc="Set mask and flag bits for bright objects?")
141  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
142  doc="Name of mask bit used for bright objects")
143 
144  coaddPsf = pexConfig.ConfigField(
145  doc="Configuration for CoaddPsf",
146  dtype=measAlg.CoaddPsfConfig,
147  )
148  doAttachTransmissionCurve = pexConfig.Field(
149  dtype=bool, default=False, optional=False,
150  doc=("Attach a piecewise TransmissionCurve for the coadd? "
151  "(requires all input Exposures to have TransmissionCurves).")
152  )
153 
154  def setDefaults(self):
155  CoaddBaseTask.ConfigClass.setDefaults(self)
156  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
157 
158  def validate(self):
159  CoaddBaseTask.ConfigClass.validate(self)
160  if self.doPsfMatch:
161  # Backwards compatibility.
162  # Configs do not have loggers
163  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
164  self.warpType = 'psfMatched'
165  if self.doSigmaClip and self.statistic != "MEANCLIP":
166  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
167  self.statistic = "MEANCLIP"
168  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
169  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
170  "compute and set a non-zero coadd variance estimate." % (self.statistic))
171 
172  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
173  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
174  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
175  if str(k) not in unstackableStats]
176  raise ValueError("statistic %s is not allowed. Please choose one of %s."
177  % (self.statistic, stackableStats))
178 
179 
180 
187  """!
188 \anchor AssembleCoaddTask_
189 
190 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
191 
192 \section pipe_tasks_assembleCoadd_Contents Contents
193  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
194  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
195  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
196  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
197  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
198  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
199 
200 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
201 
202 \copybrief AssembleCoaddTask_
203 
204 We want to assemble a coadded image from a set of Warps (also called
205 coadded temporary exposures or coaddTempExps.
206 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
207 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
208 Warps that cover the specified patch (pointed at by dataRef).
209 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
210 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
211 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
212 config.makePsfMatched set which of the warp types will be coadded.
213 The coadd is computed as a mean with optional outlier rejection.
214 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
215 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
216 
217 AssembleCoaddTask uses several sub-tasks. These are
218 <DL>
219  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
220  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
221  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
222  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
223 </DL>
224 You can retarget these subtasks if you wish.
225 
226 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
227 \copydoc \_\_init\_\_
228 
229 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
230 \copydoc run
231 
232 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
233 See \ref AssembleCoaddConfig_
234 
235 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
236 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
237 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
238 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
239 the documetation for the subtasks for further information.
240 
241 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
242 
243 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
244 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
245 a data reference to the tract patch and filter to be coadded (specified using
246 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
247 Warps to attempt to coadd (specified using
248 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
249 that cover the specified tract and patch will be coadded. A list of the available optional
250 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
251 \code
252 assembleCoadd.py --help
253 \endcode
254 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
255 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
256 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
257 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
258 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
259 <DL>
260  <DT>processCcd</DT>
261  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
262  <DT>makeSkyMap</DT>
263  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
264  <DT>makeCoaddTempExp</DT>
265  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
266 </DL>
267 We can perform all of these steps by running
268 \code
269 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
270 \endcode
271 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
272 follows:
273 \code
274 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
275 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
276 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
277 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
278 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
279 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
280 --selectId visit=903988 ccd=24
281 \endcode
282 that will process the HSC-I band data. The results are written in
283 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
284 
285 You may also choose to run:
286 \code
287 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
288 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
289 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
290 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
291 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
292 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
293 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
294 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
295 \endcode
296 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
297 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
298 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
299  """
300  ConfigClass = AssembleCoaddConfig
301  _DefaultName = "assembleCoadd"
302 
303  def __init__(self, *args, **kwargs):
304  """!
305  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
306  & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
307  """
308  CoaddBaseTask.__init__(self, *args, **kwargs)
309  self.makeSubtask("interpImage")
310  self.makeSubtask("scaleZeroPoint")
311 
312  if self.config.doMaskBrightObjects:
313  mask = afwImage.Mask()
314  try:
315  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
316  except pexExceptions.LsstCppException:
317  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
318  mask.getMaskPlaneDict().keys())
319  del mask
320 
321  self.warpType = self.config.warpType
322 
323  @pipeBase.timeMethod
324  def run(self, dataRef, selectDataList=[]):
325  """!
326  \brief Assemble a coadd from a set of Warps
327 
328  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
329  match the photometric zeropoint to a reference Warp. Assemble the Warps using
330  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
331  exposure.
332 
333  \anchor runParams
334  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
335  (if config.autoReference=False). Used to access the following data products:
336  - [in] self.config.coaddName + "Coadd_skyMap"
337  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
338  - [out] self.config.coaddName + "Coadd"
339  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
340  selected from this list based on overlap with the patch defined by dataRef.
341 
342  \return a pipeBase.Struct with fields:
343  - coaddExposure: coadded exposure
344  - nImage: exposure count image
345  """
346  skyInfo = self.getSkyInfo(dataRef)
347  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
348  if len(calExpRefList) == 0:
349  self.log.warn("No exposures to coadd")
350  return
351  self.log.info("Coadding %d exposures", len(calExpRefList))
352 
353  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
354  inputData = self.prepareInputs(tempExpRefList)
355  self.log.info("Found %d %s", len(inputData.tempExpRefList),
356  self.getTempExpDatasetName(self.warpType))
357  if len(inputData.tempExpRefList) == 0:
358  self.log.warn("No coadd temporary exposures found")
359  return
360 
361  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
362 
363  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
364  inputData.weightList, supplementaryData=supplementaryData)
365 
366  if self.config.doInterp:
367  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
368  # The variance must be positive; work around for DM-3201.
369  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
370  with numpy.errstate(invalid="ignore"):
371  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
372 
373  if self.config.doMaskBrightObjects:
374  brightObjectMasks = self.readBrightObjectMasks(dataRef)
375  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
376 
377  if self.config.doWrite:
378  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
379  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
380  if self.config.doNImage and retStruct.nImage is not None:
381  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
382 
383  return retStruct
384 
385  def makeSupplementaryData(self, dataRef, selectDataList):
386  """!
387  \brief Make additional inputs to assemble() specific to subclasses.
388 
389  Available to be implemented by subclasses only if they need the
390  coadd dataRef for performing preliminary processing before
391  assembling the coadd.
392  """
393  pass
394 
395  def getTempExpRefList(self, patchRef, calExpRefList):
396  """!
397  \brief Generate list data references corresponding to warped exposures that lie within the
398  patch to be coadded.
399 
400  \param[in] patchRef: Data reference for patch
401  \param[in] calExpRefList: List of data references for input calexps
402  \return List of Warp/CoaddTempExp data references
403  """
404  butler = patchRef.getButler()
405  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
406  self.getTempExpDatasetName(self.warpType))
407  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
408  g, groupData.keys) for
409  g in groupData.groups.keys()]
410  return tempExpRefList
411 
412  def prepareInputs(self, refList):
413  """!
414  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
415  for the photometric zero point.
416 
417  Each Warp has its own photometric zeropoint and background variance. Before coadding these
418  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
419  weight for each Warp.
420 
421  \param[in] refList: List of data references to tempExp
422  \return Struct:
423  - tempExprefList: List of data references to tempExp
424  - weightList: List of weightings
425  - imageScalerList: List of image scalers
426  """
427  statsCtrl = afwMath.StatisticsControl()
428  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
429  statsCtrl.setNumIter(self.config.clipIter)
430  statsCtrl.setAndMask(self.getBadPixelMask())
431  statsCtrl.setNanSafe(True)
432  # compute tempExpRefList: a list of tempExpRef that actually exist
433  # and weightList: a list of the weight of the associated coadd tempExp
434  # and imageScalerList: a list of scale factors for the associated coadd tempExp
435  tempExpRefList = []
436  weightList = []
437  imageScalerList = []
438  tempExpName = self.getTempExpDatasetName(self.warpType)
439  for tempExpRef in refList:
440  if not tempExpRef.datasetExists(tempExpName):
441  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
442  continue
443 
444  tempExp = tempExpRef.get(tempExpName, immediate=True)
445  maskedImage = tempExp.getMaskedImage()
446  imageScaler = self.scaleZeroPoint.computeImageScaler(
447  exposure=tempExp,
448  dataRef=tempExpRef,
449  )
450  try:
451  imageScaler.scaleMaskedImage(maskedImage)
452  except Exception as e:
453  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
454  continue
455  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
456  afwMath.MEANCLIP, statsCtrl)
457  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
458  weight = 1.0 / float(meanVar)
459  if not numpy.isfinite(weight):
460  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
461  continue
462  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
463 
464  del maskedImage
465  del tempExp
466 
467  tempExpRefList.append(tempExpRef)
468  weightList.append(weight)
469  imageScalerList.append(imageScaler)
470 
471  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
472  imageScalerList=imageScalerList)
473 
474  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
475  altMaskList=None, mask=None, supplementaryData=None):
476  """!
477  \anchor AssembleCoaddTask.assemble_
478 
479  \brief Assemble a coadd from input warps
480 
481  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
482  large area), the assembly is performed over small areas on the image at a time in order to
483  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
484  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
485  statistic specified. Set the edge bits the coadd mask based on the weight map.
486 
487  \param[in] skyInfo: Patch geometry information, from getSkyInfo
488  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
489  \param[in] imageScalerList: List of image scalers
490  \param[in] weightList: List of weights
491  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
492  \param[in] mask: Mask to ignore when coadding
493  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
494  Only used by subclasses that implement makeSupplementaryData and override assemble.
495  \return pipeBase.Struct with coaddExposure, nImage if requested
496  """
497  tempExpName = self.getTempExpDatasetName(self.warpType)
498  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
499  if mask is None:
500  mask = self.getBadPixelMask()
501 
502  statsCtrl = afwMath.StatisticsControl()
503  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
504  statsCtrl.setNumIter(self.config.clipIter)
505  statsCtrl.setAndMask(mask)
506  statsCtrl.setNanSafe(True)
507  statsCtrl.setWeighted(True)
508  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
509  for plane, threshold in self.config.maskPropagationThresholds.items():
510  bit = afwImage.Mask.getMaskPlane(plane)
511  statsCtrl.setMaskPropagationThreshold(bit, threshold)
512 
513  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
514 
515  if altMaskList is None:
516  altMaskList = [None]*len(tempExpRefList)
517 
518  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
519  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
520  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
521  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
522  coaddMaskedImage = coaddExposure.getMaskedImage()
523  subregionSizeArr = self.config.subregionSize
524  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
525  # if nImage is requested, create a zero one which can be passed to assembleSubregion
526  if self.config.doNImage:
527  nImage = afwImage.ImageU(skyInfo.bbox)
528  else:
529  nImage = None
530  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
531  try:
532  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
533  weightList, altMaskList, statsFlags, statsCtrl,
534  nImage=nImage)
535  except Exception as e:
536  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
537 
538  self.setInexactPsf(coaddMaskedImage.getMask())
539  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
540  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
541  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
542  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
543 
544  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
545  """!
546  \brief Set the metadata for the coadd
547 
548  This basic implementation simply sets the filter from the
549  first input.
550 
551  \param[in] coaddExposure: The target image for the coadd
552  \param[in] tempExpRefList: List of data references to tempExp
553  \param[in] weightList: List of weights
554  """
555  assert len(tempExpRefList) == len(weightList), "Length mismatch"
556  tempExpName = self.getTempExpDatasetName(self.warpType)
557  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
558  # (and we need more than just the PropertySet that contains the header), which is not possible
559  # with the current butler (see #2777).
560  tempExpList = [tempExpRef.get(tempExpName + "_sub",
561  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
562  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
563  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
564 
565  coaddExposure.setFilter(tempExpList[0].getFilter())
566  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
567  coaddInputs.ccds.reserve(numCcds)
568  coaddInputs.visits.reserve(len(tempExpList))
569 
570  for tempExp, weight in zip(tempExpList, weightList):
571  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
572 
573  if self.config.doUsePsfMatchedPolygons:
574  self.shrinkValidPolygons(coaddInputs)
575 
576  coaddInputs.visits.sort()
577  if self.warpType == "psfMatched":
578  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
579  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
580  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
581  # having the maximum width (sufficient because square)
582  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
583  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
584  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
585  else:
586  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
587  self.config.coaddPsf.makeControl())
588  coaddExposure.setPsf(psf)
589  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
590  coaddExposure.getWcs())
591  coaddExposure.getInfo().setApCorrMap(apCorrMap)
592  if self.config.doAttachTransmissionCurve:
593  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
594  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
595 
596  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
597  altMaskList, statsFlags, statsCtrl, nImage=None):
598  """!
599  \brief Assemble the coadd for a sub-region.
600 
601  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
602  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
603  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
604  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
605  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
606  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
607 
608  \param[in] coaddExposure: The target image for the coadd
609  \param[in] bbox: Sub-region to coadd
610  \param[in] tempExpRefList: List of data reference to tempExp
611  \param[in] imageScalerList: List of image scalers
612  \param[in] weightList: List of weights
613  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
614  Each element is dict with keys = mask plane name to which to add the spans
615  \param[in] statsFlags: afwMath.Property object for statistic for coadd
616  \param[in] statsCtrl: Statistics control object for coadd
617  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
618  """
619  self.log.debug("Computing coadd over %s", bbox)
620  tempExpName = self.getTempExpDatasetName(self.warpType)
621  coaddExposure.mask.addMaskPlane("REJECTED")
622  coaddExposure.mask.addMaskPlane("CLIPPED")
623  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
624  # If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
625  # or CLIPPED, set it to REJECTED on the coadd.
626  # If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
627  # if a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
628  edge = afwImage.Mask.getPlaneBitMask("EDGE")
629  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
630  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
631  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
632  maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask("REJECTED")),
633  (edge, coaddExposure.mask.getPlaneBitMask("SENSOR_EDGE")),
634  (clipped, clipped)]
635  maskedImageList = []
636  if nImage is not None:
637  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
638  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
639  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
640  maskedImage = exposure.getMaskedImage()
641  mask = maskedImage.getMask()
642  if altMask is not None:
643  self.applyAltMaskPlanes(mask, altMask)
644  imageScaler.scaleMaskedImage(maskedImage)
645 
646  # Add 1 for each pixel which is not excluded by the exclude mask.
647  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
648  if nImage is not None:
649  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
650  if self.config.removeMaskPlanes:
651  mask = maskedImage.getMask()
652  for maskPlane in self.config.removeMaskPlanes:
653  try:
654  mask &= ~mask.getPlaneBitMask(maskPlane)
655  except Exception as e:
656  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
657 
658  maskedImageList.append(maskedImage)
659 
660  with self.timer("stack"):
661  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
662  clipped, # also set output to CLIPPED if sigma-clipped
663  maskMap)
664  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
665  if nImage is not None:
666  nImage.assign(subNImage, bbox)
667 
668  def applyAltMaskPlanes(self, mask, altMaskSpans):
669  """!
670  \brief Apply in place alt mask formatted as SpanSets to a mask
671 
672  @param mask: original mask
673  @param altMaskSpans: Dict containing spanSet lists to apply.
674  Each element contains the new mask plane name
675  (e.g. "CLIPPED and/or "NO_DATA") as the key,
676  and list of SpanSets to apply to the mask
677  """
678  if self.config.doUsePsfMatchedPolygons:
679  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
680  # Clear away any other masks outside the validPolygons. These pixels are no longer
681  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
682  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
683  # This mask-clearing step must only occur *before* applying the new masks below
684  for spanSet in altMaskSpans['NO_DATA']:
685  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
686 
687  for plane, spanSetList in altMaskSpans.items():
688  maskClipValue = mask.addMaskPlane(plane)
689  for spanSet in spanSetList:
690  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
691  return mask
692 
693  def shrinkValidPolygons(self, coaddInputs):
694  """!
695  \brief Shrink coaddInputs' ccds' ValidPolygons in place
696 
697  @param coaddInputs: original mask
698 
699  Either modify each ccd's validPolygon in place, or if CoaddInputs does not
700  have a validPolygon, create one from its bbox.
701  """
702  for ccd in coaddInputs.ccds:
703  polyOrig = ccd.getValidPolygon()
704  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
705  validPolyBBox.grow(-self.config.matchingKernelSize//2)
706  if polyOrig:
707  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
708  else:
709  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
710  ccd.setValidPolygon(validPolygon)
711 
712  def readBrightObjectMasks(self, dataRef):
713  """Returns None on failure"""
714  try:
715  return dataRef.get("brightObjectMask", immediate=True)
716  except Exception as e:
717  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
718  return None
719 
720  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
721  """Set the bright object masks
722 
723  exposure: Exposure under consideration
724  dataId: Data identifier dict for patch
725  brightObjectMasks: afwTable of bright objects to mask
726  """
727  #
728  # Check the metadata specifying the tract/patch/filter
729  #
730  if brightObjectMasks is None:
731  self.log.warn("Unable to apply bright object mask: none supplied")
732  return
733  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
734  md = brightObjectMasks.table.getMetadata()
735  for k in dataId:
736  if not md.exists(k):
737  self.log.warn("Expected to see %s in metadata", k)
738  else:
739  if md.get(k) != dataId[k]:
740  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
741 
742  mask = exposure.getMaskedImage().getMask()
743  wcs = exposure.getWcs()
744  plateScale = wcs.getPixelScale().asArcseconds()
745 
746  for rec in brightObjectMasks:
747  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
748  if rec["type"] == "box":
749  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
750  width = rec["width"].asArcseconds()/plateScale # convert to pixels
751  height = rec["height"].asArcseconds()/plateScale # convert to pixels
752 
753  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
754  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
755 
756  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
757  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
758  spans = afwGeom.SpanSet(bbox)
759  elif rec["type"] == "circle":
760  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
761  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
762  else:
763  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
764  continue
765  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
766 
767  def setInexactPsf(self, mask):
768  """Set INEXACT_PSF mask plane
769 
770  If any of the input images isn't represented in the coadd (due to
771  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
772  these pixels.
773 
774  Parameters
775  ----------
776  mask : `lsst.afw.image.Mask`
777  Coadded exposure's mask, modified in-place.
778  """
779  mask.addMaskPlane("INEXACT_PSF")
780  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
781  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
782  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
783  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
784  array = mask.getArray()
785  selected = array & (sensorEdge | clipped | rejected) > 0
786  array[selected] |= inexactPsf
787 
788  @classmethod
789  def _makeArgumentParser(cls):
790  """!
791  \brief Create an argument parser
792  """
793  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
794  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
795  cls.ConfigClass().warpType + "Warp",
796  help="data ID, e.g. --id tract=12345 patch=1,2",
797  ContainerClass=AssembleCoaddDataIdContainer)
798  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
799  ContainerClass=SelectDataIdContainer)
800  return parser
801 
802 
803 def _subBBoxIter(bbox, subregionSize):
804  """!
805  \brief Iterate over subregions of a bbox
806 
807  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
808  \param[in] subregionSize: size of sub-bboxes
809 
810  \return subBBox: next sub-bounding box of size subregionSize or smaller;
811  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
812  but it will never be empty
813  """
814  if bbox.isEmpty():
815  raise RuntimeError("bbox %s is empty" % (bbox,))
816  if subregionSize[0] < 1 or subregionSize[1] < 1:
817  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
818 
819  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
820  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
821  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
822  subBBox.clip(bbox)
823  if subBBox.isEmpty():
824  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
825  (bbox, subregionSize, colShift, rowShift))
826  yield subBBox
827 
828 
829 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
830  """!
831  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
832  """
833 
834  def makeDataRefList(self, namespace):
835  """!
836  \brief Make self.refList from self.idList.
837  """
838  datasetType = namespace.config.coaddName + "Coadd"
839  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
840 
841  for dataId in self.idList:
842  # tract and patch are required
843  for key in keysCoadd:
844  if key not in dataId:
845  raise RuntimeError("--id must include " + key)
846 
847  dataRef = namespace.butler.dataRef(
848  datasetType=datasetType,
849  dataId=dataId,
850  )
851  self.refList.append(dataRef)
852 
853 
854 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
855  """!
856  \brief Function to count the number of pixels with a specific mask in a footprint.
857 
858  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
859  have bitmask set but do not have ignoreMask set. Return the count.
860 
861  \param[in] mask: mask to define intersection region by.
862  \parma[in] footprint: footprint to define the intersection region by.
863  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
864  \param[in] ignoreMask: pixels to not consider.
865  \return count of number of pixels in footprint with specified mask.
866  """
867  bbox = footprint.getBBox()
868  bbox.clip(mask.getBBox(afwImage.PARENT))
869  fp = afwImage.Mask(bbox)
870  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
871  footprint.spans.setMask(fp, bitmask)
872  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
873  (subMask.getArray() & ignoreMask) == 0).sum()
874 
875 
877  """!
878 \anchor SafeClipAssembleCoaddConfig
879 
880 \brief Configuration parameters for the SafeClipAssembleCoaddTask
881  """
882  clipDetection = pexConfig.ConfigurableField(
883  target=SourceDetectionTask,
884  doc="Detect sources on difference between unclipped and clipped coadd")
885  minClipFootOverlap = pexConfig.Field(
886  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
887  dtype=float,
888  default=0.6
889  )
890  minClipFootOverlapSingle = pexConfig.Field(
891  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
892  "clipped when only one visit overlaps",
893  dtype=float,
894  default=0.5
895  )
896  minClipFootOverlapDouble = pexConfig.Field(
897  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
898  "clipped when two visits overlap",
899  dtype=float,
900  default=0.45
901  )
902  maxClipFootOverlapDouble = pexConfig.Field(
903  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
904  "considering two visits",
905  dtype=float,
906  default=0.15
907  )
908  minBigOverlap = pexConfig.Field(
909  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
910  "when labeling clipped footprints",
911  dtype=int,
912  default=100
913  )
914 
915  def setDefaults(self):
916  # The numeric values for these configuration parameters were empirically determined, future work
917  # may further refine them.
918  AssembleCoaddConfig.setDefaults(self)
919  self.clipDetection.doTempLocalBackground = False
920  self.clipDetection.reEstimateBackground = False
921  self.clipDetection.returnOriginalFootprints = False
922  self.clipDetection.thresholdPolarity = "both"
923  self.clipDetection.thresholdValue = 2
924  self.clipDetection.nSigmaToGrow = 2
925  self.clipDetection.minPixels = 4
926  self.clipDetection.isotropicGrow = True
927  self.clipDetection.thresholdType = "pixel_stdev"
928  self.sigmaClip = 1.5
929  self.clipIter = 3
930  self.statistic = "MEAN"
931 
932  def validate(self):
933  if self.doSigmaClip:
934  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
935  "Ignoring doSigmaClip.")
936  self.doSigmaClip = False
937  if self.statistic != "MEAN":
938  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
939  "(%s chosen). Please set statistic to MEAN."
940  % (self.statistic))
941  AssembleCoaddTask.ConfigClass.validate(self)
942 
943 
944 
950 
951 
953  """!
954  \anchor SafeClipAssembleCoaddTask_
955 
956  \brief Assemble a coadded image from a set of coadded temporary exposures,
957  being careful to clip & flag areas with potential artifacts.
958 
959  \section pipe_tasks_assembleCoadd_Contents Contents
960  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
961  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
962  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
963  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
964  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
965  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
966 
967  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
968 
969  \copybrief SafeClipAssembleCoaddTask
970 
971  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
972  SafeClipAssembleCoaddTask subtasks that task.
973  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
974  outliers).
975  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
976  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
977  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
978  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
979  that there is an outlier and ii. this outlier appears on only one or two images.
980  Such regions will not contribute to the final coadd.
981  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
982  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
983  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
984  surveys.
985 
986  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
987  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
988  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
989 
990  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
991  \copydoc \_\_init\_\_
992 
993  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
994  \copydoc run
995 
996  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
997  See \ref SafeClipAssembleCoaddConfig
998 
999  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1000  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1001  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1002  files.
1003  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1004  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1005  for further information.
1006 
1007  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
1008  SafeClipAssembleCoaddTask
1009 
1010  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1011  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1012  '--legacyCoadd'.
1013  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1014  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1015  with a list of coaddTempExps to attempt to coadd (specified using
1016  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1017  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1018  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1019  command line argument:
1020  \code
1021  assembleCoadd.py --help
1022  \endcode
1023  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1024  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1025  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1026  packages.
1027  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1028  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1029  <DL>
1030  <DT>processCcd</DT>
1031  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1032  <DT>makeSkyMap</DT>
1033  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1034  <DT>makeCoaddTempExp</DT>
1035  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1036  </DL>
1037  We can perform all of these steps by running
1038  \code
1039  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1040  \endcode
1041  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1042  as follows:
1043  \code
1044  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1045  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1046  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1047  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1048  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1049  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1050  --selectId visit=903988 ccd=24
1051  \endcode
1052  This will process the HSC-I band data. The results are written in
1053  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1054 
1055  You may also choose to run:
1056  \code
1057  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1058  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1059  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1060  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1061  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1062  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1063  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1064  --selectId visit=903346 ccd=12
1065  \endcode
1066  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1067  discussed in \ref pipeTasks_multiBand.
1068  """
1069  ConfigClass = SafeClipAssembleCoaddConfig
1070  _DefaultName = "safeClipAssembleCoadd"
1071 
1072  def __init__(self, *args, **kwargs):
1073  """!
1074  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1075  """
1076  AssembleCoaddTask.__init__(self, *args, **kwargs)
1077  schema = afwTable.SourceTable.makeMinimalSchema()
1078  self.makeSubtask("clipDetection", schema=schema)
1079 
1080  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1081  """!
1082  \brief Assemble the coadd for a region
1083 
1084  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1085  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1086  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1087  overlap between the clipped region and a source.
1088  This leaves us with a set of footprints from the difference image that have been identified as having
1089  occured on just one or two individual visits. However, these footprints were generated from a
1090  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1091  footprints acrosss the coadd difference in this process.
1092  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1093  these are big footprints.
1094  Combine the small and big clipped footprints and mark them on a new bad mask plane
1095  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1096  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1097  bad mask plane.
1098 
1099  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1100  parent task.
1101 
1102  @param skyInfo: Patch geometry information, from getSkyInfo
1103  @param tempExpRefList: List of data reference to tempExp
1104  @param imageScalerList: List of image scalers
1105  @param weightList: List of weights
1106  return pipeBase.Struct with coaddExposure, nImage
1107  """
1108  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1109  mask = exp.getMaskedImage().getMask()
1110  mask.addMaskPlane("CLIPPED")
1111 
1112  result = self.detectClip(exp, tempExpRefList)
1113 
1114  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1115 
1116  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1117  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1118  # Append big footprints from individual Warps to result.clipSpans
1119  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1120  result.detectionFootprints, maskClipValue, maskDetValue,
1121  exp.getBBox())
1122  # Create mask of the current clipped footprints
1123  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1124  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1125 
1126  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1127  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1128  maskClip |= maskClipBig
1129 
1130  # Assemble coadd from base class, but ignoring CLIPPED pixels
1131  badMaskPlanes = self.config.badMaskPlanes[:]
1132  badMaskPlanes.append("CLIPPED")
1133  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1134  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1135  result.clipSpans, mask=badPixelMask)
1136 
1137  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1138  """!
1139  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1140 
1141  Generate a difference image between clipped and unclipped coadds.
1142  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1143  Return the difference image.
1144 
1145  @param skyInfo: Patch geometry information, from getSkyInfo
1146  @param tempExpRefList: List of data reference to tempExp
1147  @param imageScalerList: List of image scalers
1148  @param weightList: List of weights
1149  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1150  """
1151  # Clone and upcast self.config because current self.config is frozen
1152  config = AssembleCoaddConfig()
1153  # getattr necessary because subtasks do not survive Config.toDict()
1154  configIntersection = {k: getattr(self.config, k)
1155  for k, v in self.config.toDict().items() if (k in config.keys())}
1156  config.update(**configIntersection)
1157 
1158  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1159  config.statistic = 'MEAN'
1160  task = AssembleCoaddTask(config=config)
1161  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1162 
1163  config.statistic = 'MEANCLIP'
1164  task = AssembleCoaddTask(config=config)
1165  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1166 
1167  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1168  coaddDiff -= coaddClip.getMaskedImage()
1169  exp = afwImage.ExposureF(coaddDiff)
1170  exp.setPsf(coaddMean.getPsf())
1171  return exp
1172 
1173  def detectClip(self, exp, tempExpRefList):
1174  """!
1175  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1176 
1177  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1178  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1179  fraction to thresholds set in the config.
1180  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1181  two).
1182  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1183  the coaddTempExp.
1184  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1185  with the clipped footprints and a list of new masks for the coaddTempExps.
1186 
1187  \param[in] exp: Exposure to run detection on
1188  \param[in] tempExpRefList: List of data reference to tempExp
1189  \return struct containing:
1190  - clipFootprints: list of clipped footprints
1191  - clipIndices: indices for each clippedFootprint in tempExpRefList
1192  - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new
1193  maskplane name ("CLIPPED")" as the key and list of SpanSets as value
1194  - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints
1195  """
1196  mask = exp.getMaskedImage().getMask()
1197  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1198  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1199  # Merge positive and negative together footprints together
1200  fpSet.positive.merge(fpSet.negative)
1201  footprints = fpSet.positive
1202  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1203  ignoreMask = self.getBadPixelMask()
1204 
1205  clipFootprints = []
1206  clipIndices = []
1207  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1208 
1209  # for use by detectClipBig
1210  visitDetectionFootprints = []
1211 
1212  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1213  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1214  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1215 
1216  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1217  for i, warpRef in enumerate(tempExpRefList):
1218  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1219  immediate=True).getMaskedImage().getMask()
1220  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1221  afwImage.PARENT, True)
1222  maskVisitDet &= maskDetValue
1223  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1224  visitDetectionFootprints.append(visitFootprints)
1225 
1226  for j, footprint in enumerate(footprints.getFootprints()):
1227  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1228  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1229 
1230  # build a list of clipped spans for each visit
1231  for j, footprint in enumerate(footprints.getFootprints()):
1232  nPixel = footprint.getArea()
1233  overlap = [] # hold the overlap with each visit
1234  indexList = [] # index of visit in global list
1235  for i in range(len(tempExpRefList)):
1236  ignore = ignoreArr[i, j]
1237  overlapDet = overlapDetArr[i, j]
1238  totPixel = nPixel - ignore
1239 
1240  # If we have more bad pixels than detection skip
1241  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1242  continue
1243  overlap.append(overlapDet/float(totPixel))
1244  indexList.append(i)
1245 
1246  overlap = numpy.array(overlap)
1247  if not len(overlap):
1248  continue
1249 
1250  keep = False # Should this footprint be marked as clipped?
1251  keepIndex = [] # Which tempExps does the clipped footprint belong to
1252 
1253  # If footprint only has one overlap use a lower threshold
1254  if len(overlap) == 1:
1255  if overlap[0] > self.config.minClipFootOverlapSingle:
1256  keep = True
1257  keepIndex = [0]
1258  else:
1259  # This is the general case where only visit should be clipped
1260  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1261  if len(clipIndex) == 1:
1262  keep = True
1263  keepIndex = [clipIndex[0]]
1264 
1265  # Test if there are clipped objects that overlap two different visits
1266  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1267  if len(clipIndex) == 2 and len(overlap) > 3:
1268  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1269  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1270  keep = True
1271  keepIndex = clipIndex
1272 
1273  if not keep:
1274  continue
1275 
1276  for index in keepIndex:
1277  globalIndex = indexList[index]
1278  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1279 
1280  clipIndices.append(numpy.array(indexList)[keepIndex])
1281  clipFootprints.append(footprint)
1282 
1283  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1284  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1285 
1286  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1287  maskClipValue, maskDetValue, coaddBBox):
1288  """!
1289  \brief Return individual warp footprints for large artifacts and append them to clipList in place
1290 
1291  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1292  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1293  significantly with each source in all the coaddTempExps.
1294  \param[in] clipList: List of alt mask SpanSets with clipping information. Modified.
1295  \param[in] clipFootprints: List of clipped footprints
1296  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1297  \param[in] maskClipValue: Mask value of clipped pixels
1298  \param[in] maskDetValue: Mask value of detected pixels
1299  \param[in] coaddBBox: BBox of the coadd and warps
1300  \return list of big footprints
1301  """
1302  bigFootprintsCoadd = []
1303  ignoreMask = self.getBadPixelMask()
1304  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1305  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1306  for footprint in visitFootprints.getFootprints():
1307  footprint.spans.setMask(maskVisitDet, maskDetValue)
1308 
1309  # build a mask of clipped footprints that are in this visit
1310  clippedFootprintsVisit = []
1311  for foot, clipIndex in zip(clipFootprints, clipIndices):
1312  if index not in clipIndex:
1313  continue
1314  clippedFootprintsVisit.append(foot)
1315  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1316  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1317 
1318  bigFootprintsVisit = []
1319  for foot in visitFootprints.getFootprints():
1320  if foot.getArea() < self.config.minBigOverlap:
1321  continue
1322  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1323  if nCount > self.config.minBigOverlap:
1324  bigFootprintsVisit.append(foot)
1325  bigFootprintsCoadd.append(foot)
1326 
1327  for footprint in bigFootprintsVisit:
1328  clippedSpans["CLIPPED"].append(footprint.spans)
1329 
1330  return bigFootprintsCoadd
1331 
1332 
1334  assembleStaticSkyModel = pexConfig.ConfigurableField(
1335  target=AssembleCoaddTask,
1336  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1337  " naive/first-iteration model of the static sky.",
1338  )
1339  detect = pexConfig.ConfigurableField(
1340  target=SourceDetectionTask,
1341  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1342  )
1343  detectTemplate = pexConfig.ConfigurableField(
1344  target=SourceDetectionTask,
1345  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1346  )
1347  maxNumEpochs = pexConfig.Field(
1348  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1349  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1350  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1351  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1352  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1353  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1354  "than transient and not masked.",
1355  dtype=int,
1356  default=2
1357  )
1358  maxFractionEpochsLow = pexConfig.RangeField(
1359  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1360  "Effective maxNumEpochs = "
1361  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1362  dtype=float,
1363  default=0.4,
1364  min=0., max=1.,
1365  )
1366  maxFractionEpochsHigh = pexConfig.RangeField(
1367  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1368  "Effective maxNumEpochs = "
1369  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1370  dtype=float,
1371  default=0.03,
1372  min=0., max=1.,
1373  )
1374  spatialThreshold = pexConfig.RangeField(
1375  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1376  "temporal criteria. If 0, clip all. If 1, clip none.",
1377  dtype=float,
1378  default=0.5,
1379  min=0., max=1.,
1380  inclusiveMin=True, inclusiveMax=True
1381  )
1382  doScaleWarpVariance = pexConfig.Field(
1383  doc="Rescale Warp variance plane using empirical noise?",
1384  dtype=bool,
1385  default=True,
1386  )
1387  scaleWarpVariance = pexConfig.ConfigurableField(
1388  target=ScaleVarianceTask,
1389  doc="Rescale variance on warps",
1390  )
1391  doPreserveContainedBySource = pexConfig.Field(
1392  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1393  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1394  dtype=bool,
1395  default=True,
1396  )
1397 
1398  def setDefaults(self):
1399  AssembleCoaddConfig.setDefaults(self)
1400  self.statistic = 'MEAN'
1402 
1403  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1404  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1405  if "EDGE" in self.badMaskPlanes:
1406  self.badMaskPlanes.remove('EDGE')
1407  self.removeMaskPlanes.append('EDGE')
1408  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1409  self.assembleStaticSkyModel.warpType = 'psfMatched'
1410  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1411  self.assembleStaticSkyModel.sigmaClip = 2.5
1412  self.assembleStaticSkyModel.clipIter = 3
1413  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1414  self.assembleStaticSkyModel.doWrite = False
1415  self.detect.doTempLocalBackground = False
1416  self.detect.reEstimateBackground = False
1417  self.detect.returnOriginalFootprints = False
1418  self.detect.thresholdPolarity = "both"
1419  self.detect.thresholdValue = 5
1420  self.detect.nSigmaToGrow = 2
1421  self.detect.minPixels = 4
1422  self.detect.isotropicGrow = True
1423  self.detect.thresholdType = "pixel_stdev"
1424  self.detectTemplate.nSigmaToGrow = 2.0
1425  self.detectTemplate.doTempLocalBackground = False
1426  self.detectTemplate.reEstimateBackground = False
1427  self.detectTemplate.returnOriginalFootprints = False
1428 
1429 
1430 
1436 
1438  """!
1439  \anchor CompareWarpAssembleCoaddTask_
1440 
1441  \brief Assemble a compareWarp coadded image from a set of warps
1442  by masking artifacts detected by comparing PSF-matched warps
1443 
1444  \section pipe_tasks_assembleCoadd_Contents Contents
1445  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1446  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1447  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1448  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1449  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1450  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1451 
1452  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1453 
1454  \copybrief CompareWarpAssembleCoaddTask
1455 
1456  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1457  outliers).
1458  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1459  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1460  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1461  pixels in the individual warps suspected to contain an artifact.
1462  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1463  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1464  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1465  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1466  such as bright stars.
1467  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1468  The temporalThreshold is the maximum fraction of epochs that the deviation can
1469  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1470  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1471  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1472  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1473  These regions will not contribute to the final coadd.
1474  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1475  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1476  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1477  surveys.
1478 
1479  CompareWarpAssembleCoaddTask sub-classes
1480  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1481  as a subtask to generate the TemplateCoadd (the model of the static sky)
1482 
1483  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1484  \copydoc \_\_init\_\_
1485 
1486  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1487  \copydoc run
1488 
1489  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1490  See \ref CompareWarpAssembleCoaddConfig
1491 
1492  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1493  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1494  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1495  files.
1496 
1497  This task supports the following debug variables:
1498  <dl>
1499  <dt>`saveCountIm`
1500  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1501  <dt> `figPath`
1502  <dd> Path to save the debug fits images and figures
1503  </dl>
1504 
1505  For example, put something like:
1506  @code{.py}
1507  import lsstDebug
1508  def DebugInfo(name):
1509  di = lsstDebug.getInfo(name)
1510  if name == "lsst.pipe.tasks.assembleCoadd":
1511  di.saveCountIm = True
1512  di.figPath = "/desired/path/to/debugging/output/images"
1513  return di
1514  lsstDebug.Info = DebugInfo
1515  @endcode
1516  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1517  flag.
1518  Some subtasks may have their own debug variables; see individual Task
1519  documentation
1520 
1521  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1522  CompareWarpAssembleCoaddTask
1523 
1524  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1525  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1526  '--compareWarpCoadd'.
1527  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1528  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1529  with a list of coaddTempExps to attempt to coadd (specified using
1530  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1531  Only the warps that cover the specified tract and patch will be coadded.
1532  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1533  command line argument:
1534  \code
1535  assembleCoadd.py --help
1536  \endcode
1537  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1538  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1539  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1540  packages.
1541  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1542  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1543  <DL>
1544  <DT>processCcd</DT>
1545  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1546  <DT>makeSkyMap</DT>
1547  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1548  <DT>makeCoaddTempExp</DT>
1549  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1550  </DL>
1551  We can perform all of these steps by running
1552  \code
1553  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1554  \endcode
1555  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1556  as follows:
1557  \code
1558  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1559  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1560  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1561  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1562  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1563  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1564  --selectId visit=903988 ccd=24
1565  \endcode
1566  This will process the HSC-I band data. The results are written in
1567  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1568  """
1569  ConfigClass = CompareWarpAssembleCoaddConfig
1570  _DefaultName = "compareWarpAssembleCoadd"
1571 
1572  def __init__(self, *args, **kwargs):
1573  """!
1574  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1575  """
1576  AssembleCoaddTask.__init__(self, *args, **kwargs)
1577  self.makeSubtask("assembleStaticSkyModel")
1578  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1579  self.makeSubtask("detect", schema=detectionSchema)
1580  if self.config.doPreserveContainedBySource:
1581  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1582  if self.config.doScaleWarpVariance:
1583  self.makeSubtask("scaleWarpVariance")
1584 
1585  def makeSupplementaryData(self, dataRef, selectDataList):
1586  """!
1587  \brief Make inputs specific to Subclass
1588 
1589  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1590  """
1591  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1592 
1593  if templateCoadd is None:
1594  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1595  self.assembleStaticSkyModel.warpType[1:])
1596  message = """No %(warpName)s warps were found to build the template coadd which is
1597  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1598  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1599  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1600 
1601  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1602  another algorithm like:
1603 
1604  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1605  config.assemble.retarget(SafeClipAssembleCoaddTask)
1606  """ % {"warpName": warpName}
1607  raise RuntimeError(message)
1608 
1609  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1610 
1611  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1612  supplementaryData, *args, **kwargs):
1613  """!
1614  \brief Assemble the coadd
1615 
1616  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1617  as the model of the static sky.
1618 
1619  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1620  new "CLIPPED" plane and updated "NO_DATA" plane.
1621  Then pass these alternative masks to the base class's assemble method.
1622 
1623  @param skyInfo: Patch geometry information
1624  @param tempExpRefList: List of data references to warps
1625  @param imageScalerList: List of image scalers
1626  @param weightList: List of weights
1627  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1628 
1629  return pipeBase.Struct with coaddExposure, nImage if requested
1630  """
1631  templateCoadd = supplementaryData.templateCoadd
1632  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1633  badMaskPlanes = self.config.badMaskPlanes[:]
1634  badMaskPlanes.append("CLIPPED")
1635  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1636 
1637  result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1638  spanSetMaskList, mask=badPixelMask)
1639 
1640  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
1641  # Psf-Matching moves the real edge inwards
1642  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1643  return result
1644 
1645  def applyAltEdgeMask(self, mask, altMaskList):
1646  """!
1647  \brief Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes
1648 
1649  @param mask: original mask
1650  @param altMaskList: List of Dicts containing spanSet lists.
1651  Each element contains the new mask plane name
1652  (e.g. "CLIPPED and/or "NO_DATA") as the key,
1653  and list of SpanSets to apply to the mask.
1654  """
1655  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1656  for visitMask in altMaskList:
1657  if "EDGE" in visitMask:
1658  for spanSet in visitMask['EDGE']:
1659  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1660 
1661  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1662  """!
1663  \brief Find artifacts
1664 
1665  Loop through warps twice. The first loop builds a map with the count of how many
1666  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1667  The second loop takes each difference image and filters the artifacts detected
1668  in each using count map to filter out variable sources and sources that are difficult to
1669  subtract cleanly.
1670 
1671  @param templateCoadd: Exposure to serve as model of static sky
1672  @param tempExpRefList: List of data references to warps
1673  @param imageScalerList: List of image scalers
1674  """
1675 
1676  self.log.debug("Generating Count Image, and mask lists.")
1677  coaddBBox = templateCoadd.getBBox()
1678  slateIm = afwImage.ImageU(coaddBBox)
1679  epochCountImage = afwImage.ImageU(coaddBBox)
1680  nImage = afwImage.ImageU(coaddBBox)
1681  spanSetArtifactList = []
1682  spanSetNoDataMaskList = []
1683  spanSetEdgeList = []
1684  badPixelMask = self.getBadPixelMask()
1685 
1686  # mask of the warp diffs should = that of only the warp
1687  templateCoadd.mask.clearAllMaskPlanes()
1688 
1689  if self.config.doPreserveContainedBySource:
1690  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1691  else:
1692  templateFootprints = None
1693 
1694  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1695  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1696  if warpDiffExp is not None:
1697  # This nImage only approximates the final nImage because it uses the PSF-matched mask
1698  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1699  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1700  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1701  fpSet.positive.merge(fpSet.negative)
1702  footprints = fpSet.positive
1703  slateIm.set(0)
1704  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1705  for spans in spanSetList:
1706  spans.setImage(slateIm, 1, doClip=True)
1707  epochCountImage += slateIm
1708 
1709  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1710  # undergo a second convolution. Pixels with data in the direct warp
1711  # but not in the PSF-matched warp will not have their artifacts detected.
1712  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1713  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1714  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1715  nansMask.setXY0(warpDiffExp.getXY0())
1716  edgeMask = warpDiffExp.mask
1717  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1718  edgeMask.getPlaneBitMask("EDGE")).split()
1719  else:
1720  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1721  # In this case, mask the whole epoch
1722  nansMask = afwImage.MaskX(coaddBBox, 1)
1723  spanSetList = []
1724  spanSetEdgeMask = []
1725 
1726  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1727 
1728  spanSetNoDataMaskList.append(spanSetNoDataMask)
1729  spanSetArtifactList.append(spanSetList)
1730  spanSetEdgeList.append(spanSetEdgeMask)
1731 
1732  if lsstDebug.Info(__name__).saveCountIm:
1733  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1734  epochCountImage.writeFits(path)
1735 
1736  for i, spanSetList in enumerate(spanSetArtifactList):
1737  if spanSetList:
1738  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage, nImage,
1739  templateFootprints)
1740  spanSetArtifactList[i] = filteredSpanSetList
1741 
1742  altMasks = []
1743  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1744  altMasks.append({'CLIPPED': artifacts,
1745  'NO_DATA': noData,
1746  'EDGE': edge})
1747  return altMasks
1748 
1749  def _filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1750  """!
1751  \brief Filter artifact candidates
1752 
1753  @param spanSetList: List of SpanSets representing artifact candidates
1754  @param epochCountImage: Image of accumulated number of warpDiff detections
1755  @param nImage: Image of the accumulated number of total epochs contributing
1756 
1757  return List of SpanSets with artifacts
1758  """
1759 
1760  maskSpanSetList = []
1761  x0, y0 = epochCountImage.getXY0()
1762  for i, span in enumerate(spanSetList):
1763  y, x = span.indices()
1764  yIdxLocal = [y1 - y0 for y1 in y]
1765  xIdxLocal = [x1 - x0 for x1 in x]
1766  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1767  totalN = nImage.array[yIdxLocal, xIdxLocal]
1768 
1769  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1770  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1771  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1772  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1773  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1774  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1775  (outlierN <= effectiveMaxNumEpochs))
1776  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1777  if percentBelowThreshold > self.config.spatialThreshold:
1778  maskSpanSetList.append(span)
1779 
1780  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
1781  # If a candidate is contained by a footprint on the template coadd, do not clip
1782  filteredMaskSpanSetList = []
1783  for span in maskSpanSetList:
1784  doKeep = True
1785  for footprint in footprintsToExclude.positive.getFootprints():
1786  if footprint.spans.contains(span):
1787  doKeep = False
1788  break
1789  if doKeep:
1790  filteredMaskSpanSetList.append(span)
1791  maskSpanSetList = filteredMaskSpanSetList
1792 
1793  return maskSpanSetList
1794 
1795  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1796  """!
1797  \brief Fetch a warp from the butler and return a warpDiff
1798 
1799  @param warpRef: `Butler dataRef` for the warp
1800  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1801  @param templateCoadd: Exposure to be substracted from the scaled warp
1802 
1803  return Exposure of the image difference between the warp and template
1804  """
1805 
1806  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1807  warpName = self.getTempExpDatasetName('psfMatched')
1808  if not warpRef.datasetExists(warpName):
1809  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1810  return None
1811  warp = warpRef.get(warpName, immediate=True)
1812  # direct image scaler OK for PSF-matched Warp
1813  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1814  mi = warp.getMaskedImage()
1815  if self.config.doScaleWarpVariance:
1816  try:
1817  self.scaleWarpVariance.run(mi)
1818  except Exception as exc:
1819  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
1820  mi -= templateCoadd.getMaskedImage()
1821  return warp
1822 
1823  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1824  """!
1825  \brief Return a path to which to write debugging output
1826 
1827  @param prefix: string, prefix for filename
1828  @param warpRef: Butler dataRef
1829  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1830  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1831 
1832  Creates a hyphen-delimited string of dataId values for simple filenames.
1833  """
1834  if coaddLevel:
1835  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1836  else:
1837  keys = warpRef.dataId.keys()
1838  keyList = sorted(keys, reverse=True)
1839  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1840  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1841  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
Shrink coaddInputs&#39; ccds&#39; ValidPolygons in place.
def _filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Filter artifact candidates.
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:174
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:97
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:130
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:189
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:224
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:110
def applyAltEdgeMask(self, mask, altMaskList)
Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...