lsst.pipe.tasks  15.0-12-g5312fa9a+2
assembleCoadd.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import lsst.pex.config as pexConfig
25 import lsst.pex.exceptions as pexExceptions
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.afw.table as afwTable
30 import lsst.afw.detection as afwDet
31 import lsst.coadd.utils as coaddUtils
32 import lsst.pipe.base as pipeBase
33 import lsst.meas.algorithms as measAlg
34 import lsst.log as log
35 import lsstDebug
36 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
37 from .interpImage import InterpImageTask
38 from .scaleZeroPoint import ScaleZeroPointTask
39 from .coaddHelpers import groupPatchExposures, getGroupDataRef
40 from .scaleVariance import ScaleVarianceTask
41 from lsst.meas.algorithms import SourceDetectionTask
42 
43 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
44 
45 
46 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
47  """!
48 @anchor AssembleCoaddConfig_
49 
50 @brief Configuration parameters for the @ref AssembleCoaddTask_ "AssembleCoaddTask"
51  """
52  warpType = pexConfig.Field(
53  doc="Warp name: one of 'direct' or 'psfMatched'",
54  dtype=str,
55  default="direct",
56  )
57  subregionSize = pexConfig.ListField(
58  dtype=int,
59  doc="Width, height of stack subregion size; "
60  "make small enough that a full stack of images will fit into memory at once.",
61  length=2,
62  default=(2000, 2000),
63  )
64  statistic = pexConfig.Field(
65  dtype=str,
66  doc="Main stacking statistic for aggregating over the epochs.",
67  default="MEANCLIP",
68  )
69  doSigmaClip = pexConfig.Field(
70  dtype=bool,
71  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
72  default=False,
73  )
74  sigmaClip = pexConfig.Field(
75  dtype=float,
76  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
77  default=3.0,
78  )
79  clipIter = pexConfig.Field(
80  dtype=int,
81  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
82  default=2,
83  )
84  calcErrorFromInputVariance = pexConfig.Field(
85  dtype=bool,
86  doc="Calculate coadd variance from input variance by stacking statistic."
87  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
88  default=True,
89  )
90  scaleZeroPoint = pexConfig.ConfigurableField(
91  target=ScaleZeroPointTask,
92  doc="Task to adjust the photometric zero point of the coadd temp exposures",
93  )
94  doInterp = pexConfig.Field(
95  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
96  dtype=bool,
97  default=True,
98  )
99  interpImage = pexConfig.ConfigurableField(
100  target=InterpImageTask,
101  doc="Task to interpolate (and extrapolate) over NaN pixels",
102  )
103  doWrite = pexConfig.Field(
104  doc="Persist coadd?",
105  dtype=bool,
106  default=True,
107  )
108  doNImage = pexConfig.Field(
109  doc="Create image of number of contributing exposures for each pixel",
110  dtype=bool,
111  default=False,
112  )
113  doUsePsfMatchedPolygons = pexConfig.Field(
114  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
115  dtype=bool,
116  default=False,
117  )
118  maskPropagationThresholds = pexConfig.DictField(
119  keytype=str,
120  itemtype=float,
121  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
122  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
123  "would have contributed exceeds this value."),
124  default={"SAT": 0.1},
125  )
126  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
127  doc="Mask planes to remove before coadding")
128  #
129  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
130  # To make this useful you *must* also configure the flags.pixel algorithm, for example
131  # by adding
132  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
133  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
134  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
135  #
136  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
137  doc="Set mask and flag bits for bright objects?")
138  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
139  doc="Name of mask bit used for bright objects")
140 
141  coaddPsf = pexConfig.ConfigField(
142  doc="Configuration for CoaddPsf",
143  dtype=measAlg.CoaddPsfConfig,
144  )
145  doAttachTransmissionCurve = pexConfig.Field(
146  dtype=bool, default=False, optional=False,
147  doc=("Attach a piecewise TransmissionCurve for the coadd? "
148  "(requires all input Exposures to have TransmissionCurves).")
149  )
150 
151  def setDefaults(self):
152  CoaddBaseTask.ConfigClass.setDefaults(self)
153  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
154 
155  def validate(self):
156  CoaddBaseTask.ConfigClass.validate(self)
157  if self.doPsfMatch:
158  # Backwards compatibility.
159  # Configs do not have loggers
160  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
161  self.warpType = 'psfMatched'
162  if self.doSigmaClip and self.statistic != "MEANCLIP":
163  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
164  self.statistic = "MEANCLIP"
165  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
166  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
167  "compute and set a non-zero coadd variance estimate." % (self.statistic))
168 
169  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
170  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
171  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
172  if str(k) not in unstackableStats]
173  raise ValueError("statistic %s is not allowed. Please choose one of %s."
174  % (self.statistic, stackableStats))
175 
176 
177 
184  """!
185 @anchor AssembleCoaddTask_
186 
187 @brief Assemble a coadded image from a set of warps (coadded temporary exposures).
188 
189 @section pipe_tasks_assembleCoadd_Contents Contents
190  - @ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
191  - @ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
192  - @ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
193  - @ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
194  - @ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
195  - @ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
196 
197 @section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
198 
199 @copybrief AssembleCoaddTask_
200 
201 We want to assemble a coadded image from a set of Warps (also called
202 coadded temporary exposures or coaddTempExps.
203 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
204 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
205 Warps that cover the specified patch (pointed at by dataRef).
206 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
207 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
208 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
209 config.makePsfMatched set which of the warp types will be coadded.
210 The coadd is computed as a mean with optional outlier rejection.
211 Criteria for outlier rejection are set in @ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
212 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
213 
214 AssembleCoaddTask uses several sub-tasks. These are
215 <DL>
216  <DT>@ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
217  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
218  <DT>@ref InterpImageTask_ "InterpImageTask"</DT>
219  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
220 </DL>
221 You can retarget these subtasks if you wish.
222 
223 @section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
224 @copydoc \_\_init\_\_
225 
226 @section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
227 @copydoc run
228 
229 @section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
230 See @ref AssembleCoaddConfig_
231 
232 @section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
233 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
234 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files.
235 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
236 the documetation for the subtasks for further information.
237 
238 @section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
239 
240 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
241 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
242 a data reference to the tract patch and filter to be coadded (specified using
243 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
244 Warps to attempt to coadd (specified using
245 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
246 that cover the specified tract and patch will be coadded. A list of the available optional
247 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
248 @code
249 assembleCoadd.py --help
250 @endcode
251 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
252 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
253 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
254 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
255 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
256 <DL>
257  <DT>processCcd</DT>
258  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
259  <DT>makeSkyMap</DT>
260  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
261  <DT>makeCoaddTempExp</DT>
262  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
263 </DL>
264 We can perform all of these steps by running
265 @code
266 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
267 @endcode
268 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
269 follows:
270 @code
271 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
272 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
273 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
274 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
275 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
276 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
277 --selectId visit=903988 ccd=24
278 @endcode
279 that will process the HSC-I band data. The results are written in
280 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
281 
282 You may also choose to run:
283 @code
284 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
285 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
286 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
287 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
288 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
289 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
290 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
291 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
292 @endcode
293 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
294 discussed in @ref pipeTasks_multiBand (but note that normally, one would use the
295 @ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
296  """
297  ConfigClass = AssembleCoaddConfig
298  _DefaultName = "assembleCoadd"
299 
300  def __init__(self, *args, **kwargs):
301  """!
302  @brief Initialize the task. Create the @ref InterpImageTask "interpImage",
303  & @ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
304  """
305  CoaddBaseTask.__init__(self, *args, **kwargs)
306  self.makeSubtask("interpImage")
307  self.makeSubtask("scaleZeroPoint")
308 
309  if self.config.doMaskBrightObjects:
310  mask = afwImage.Mask()
311  try:
312  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
313  except pexExceptions.LsstCppException:
314  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
315  mask.getMaskPlaneDict().keys())
316  del mask
317 
318  self.warpType = self.config.warpType
319 
320  @pipeBase.timeMethod
321  def run(self, dataRef, selectDataList=[]):
322  """!
323  @brief Assemble a coadd from a set of Warps
324 
325  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
326  match the photometric zeropoint to a reference Warp. Assemble the Warps using
327  @ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
328  exposure.
329 
330  @anchor runParams
331  @param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
332  (if config.autoReference=False). Used to access the following data products:
333  - [in] self.config.coaddName + "Coadd_skyMap"
334  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
335  - [out] self.config.coaddName + "Coadd"
336  @param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
337  selected from this list based on overlap with the patch defined by dataRef.
338 
339  @return a pipeBase.Struct with fields:
340  - coaddExposure: coadded exposure
341  - nImage: exposure count image
342  """
343  skyInfo = self.getSkyInfo(dataRef)
344  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
345  if len(calExpRefList) == 0:
346  self.log.warn("No exposures to coadd")
347  return
348  self.log.info("Coadding %d exposures", len(calExpRefList))
349 
350  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
351  inputData = self.prepareInputs(tempExpRefList)
352  self.log.info("Found %d %s", len(inputData.tempExpRefList),
353  self.getTempExpDatasetName(self.warpType))
354  if len(inputData.tempExpRefList) == 0:
355  self.log.warn("No coadd temporary exposures found")
356  return
357 
358  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
359 
360  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
361  inputData.weightList, supplementaryData=supplementaryData)
362 
363  if self.config.doInterp:
364  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
365  # The variance must be positive; work around for DM-3201.
366  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
367  with numpy.errstate(invalid="ignore"):
368  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
369 
370  if self.config.doMaskBrightObjects:
371  brightObjectMasks = self.readBrightObjectMasks(dataRef)
372  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
373 
374  if self.config.doWrite:
375  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
376  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
377  if self.config.doNImage and retStruct.nImage is not None:
378  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
379 
380  return retStruct
381 
382  def makeSupplementaryData(self, dataRef, selectDataList):
383  """!
384  @brief Make additional inputs to assemble() specific to subclasses.
385 
386  Available to be implemented by subclasses only if they need the
387  coadd dataRef for performing preliminary processing before
388  assembling the coadd.
389  """
390  pass
391 
392  def getTempExpRefList(self, patchRef, calExpRefList):
393  """!
394  @brief Generate list data references corresponding to warped exposures that lie within the
395  patch to be coadded.
396 
397  @param[in] patchRef: Data reference for patch
398  @param[in] calExpRefList: List of data references for input calexps
399  @return List of Warp/CoaddTempExp data references
400  """
401  butler = patchRef.getButler()
402  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
403  self.getTempExpDatasetName(self.warpType))
404  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
405  g, groupData.keys) for
406  g in groupData.groups.keys()]
407  return tempExpRefList
408 
409  def prepareInputs(self, refList):
410  """!
411  @brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
412  for the photometric zero point.
413 
414  Each Warp has its own photometric zeropoint and background variance. Before coadding these
415  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
416  weight for each Warp.
417 
418  @param[in] refList: List of data references to tempExp
419  @return Struct:
420  - tempExprefList: List of data references to tempExp
421  - weightList: List of weightings
422  - imageScalerList: List of image scalers
423  """
424  statsCtrl = afwMath.StatisticsControl()
425  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
426  statsCtrl.setNumIter(self.config.clipIter)
427  statsCtrl.setAndMask(self.getBadPixelMask())
428  statsCtrl.setNanSafe(True)
429  # compute tempExpRefList: a list of tempExpRef that actually exist
430  # and weightList: a list of the weight of the associated coadd tempExp
431  # and imageScalerList: a list of scale factors for the associated coadd tempExp
432  tempExpRefList = []
433  weightList = []
434  imageScalerList = []
435  tempExpName = self.getTempExpDatasetName(self.warpType)
436  for tempExpRef in refList:
437  if not tempExpRef.datasetExists(tempExpName):
438  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
439  continue
440 
441  tempExp = tempExpRef.get(tempExpName, immediate=True)
442  maskedImage = tempExp.getMaskedImage()
443  imageScaler = self.scaleZeroPoint.computeImageScaler(
444  exposure=tempExp,
445  dataRef=tempExpRef,
446  )
447  try:
448  imageScaler.scaleMaskedImage(maskedImage)
449  except Exception as e:
450  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
451  continue
452  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
453  afwMath.MEANCLIP, statsCtrl)
454  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
455  weight = 1.0 / float(meanVar)
456  if not numpy.isfinite(weight):
457  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
458  continue
459  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
460 
461  del maskedImage
462  del tempExp
463 
464  tempExpRefList.append(tempExpRef)
465  weightList.append(weight)
466  imageScalerList.append(imageScaler)
467 
468  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
469  imageScalerList=imageScalerList)
470 
471  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
472  altMaskList=None, mask=None, supplementaryData=None):
473  """!
474  @anchor AssembleCoaddTask.assemble_
475 
476  @brief Assemble a coadd from input warps
477 
478  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
479  large area), the assembly is performed over small areas on the image at a time in order to
480  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
481  @ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
482  statistic specified. Set the edge bits the coadd mask based on the weight map.
483 
484  @param[in] skyInfo: Patch geometry information, from getSkyInfo
485  @param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
486  @param[in] imageScalerList: List of image scalers
487  @param[in] weightList: List of weights
488  @param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
489  @param[in] mask: Mask to ignore when coadding
490  @param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
491  Only used by subclasses that implement makeSupplementaryData and override assemble.
492  @return pipeBase.Struct with coaddExposure, nImage if requested
493  """
494  tempExpName = self.getTempExpDatasetName(self.warpType)
495  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
496  if mask is None:
497  mask = self.getBadPixelMask()
498 
499  statsCtrl = afwMath.StatisticsControl()
500  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
501  statsCtrl.setNumIter(self.config.clipIter)
502  statsCtrl.setAndMask(mask)
503  statsCtrl.setNanSafe(True)
504  statsCtrl.setWeighted(True)
505  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
506  for plane, threshold in self.config.maskPropagationThresholds.items():
507  bit = afwImage.Mask.getMaskPlane(plane)
508  statsCtrl.setMaskPropagationThreshold(bit, threshold)
509 
510  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
511 
512  if altMaskList is None:
513  altMaskList = [None]*len(tempExpRefList)
514 
515  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
516  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
517  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
518  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
519  coaddMaskedImage = coaddExposure.getMaskedImage()
520  subregionSizeArr = self.config.subregionSize
521  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
522  # if nImage is requested, create a zero one which can be passed to assembleSubregion
523  if self.config.doNImage:
524  nImage = afwImage.ImageU(skyInfo.bbox)
525  else:
526  nImage = None
527  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
528  try:
529  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
530  weightList, altMaskList, statsFlags, statsCtrl,
531  nImage=nImage)
532  except Exception as e:
533  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
534 
535  self.setInexactPsf(coaddMaskedImage.getMask())
536  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
537  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
538  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
539  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
540 
541  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
542  """!
543  @brief Set the metadata for the coadd
544 
545  This basic implementation simply sets the filter from the
546  first input.
547 
548  @param[in] coaddExposure: The target image for the coadd
549  @param[in] tempExpRefList: List of data references to tempExp
550  @param[in] weightList: List of weights
551  """
552  assert len(tempExpRefList) == len(weightList), "Length mismatch"
553  tempExpName = self.getTempExpDatasetName(self.warpType)
554  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
555  # (and we need more than just the PropertySet that contains the header), which is not possible
556  # with the current butler (see #2777).
557  tempExpList = [tempExpRef.get(tempExpName + "_sub",
558  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
559  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
560  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
561 
562  coaddExposure.setFilter(tempExpList[0].getFilter())
563  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
564  coaddInputs.ccds.reserve(numCcds)
565  coaddInputs.visits.reserve(len(tempExpList))
566 
567  for tempExp, weight in zip(tempExpList, weightList):
568  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
569 
570  if self.config.doUsePsfMatchedPolygons:
571  self.shrinkValidPolygons(coaddInputs)
572 
573  coaddInputs.visits.sort()
574  if self.warpType == "psfMatched":
575  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
576  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
577  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
578  # having the maximum width (sufficient because square)
579  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
580  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
581  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
582  else:
583  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
584  self.config.coaddPsf.makeControl())
585  coaddExposure.setPsf(psf)
586  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
587  coaddExposure.getWcs())
588  coaddExposure.getInfo().setApCorrMap(apCorrMap)
589  if self.config.doAttachTransmissionCurve:
590  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
591  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
592 
593  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
594  altMaskList, statsFlags, statsCtrl, nImage=None):
595  """!
596  @brief Assemble the coadd for a sub-region.
597 
598  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
599  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
600  @ref afwMath.statisticsStack "statisticsStack" with the statistic specified
601  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
602  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
603  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
604 
605  @param[in] coaddExposure: The target image for the coadd
606  @param[in] bbox: Sub-region to coadd
607  @param[in] tempExpRefList: List of data reference to tempExp
608  @param[in] imageScalerList: List of image scalers
609  @param[in] weightList: List of weights
610  @param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
611  Each element is dict with keys = mask plane name to which to add the spans
612  @param[in] statsFlags: afwMath.Property object for statistic for coadd
613  @param[in] statsCtrl: Statistics control object for coadd
614  @param[in] nImage: optional ImageU keeps track of exposure count for each pixel
615  """
616  self.log.debug("Computing coadd over %s", bbox)
617  tempExpName = self.getTempExpDatasetName(self.warpType)
618  coaddExposure.mask.addMaskPlane("REJECTED")
619  coaddExposure.mask.addMaskPlane("CLIPPED")
620  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
621  # If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
622  # or CLIPPED, set it to REJECTED on the coadd.
623  # If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
624  # if a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
625  edge = afwImage.Mask.getPlaneBitMask("EDGE")
626  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
627  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
628  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
629  maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask("REJECTED")),
630  (edge, coaddExposure.mask.getPlaneBitMask("SENSOR_EDGE")),
631  (clipped, clipped)]
632  maskedImageList = []
633  if nImage is not None:
634  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
635  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
636  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
637  maskedImage = exposure.getMaskedImage()
638  mask = maskedImage.getMask()
639  if altMask is not None:
640  self.applyAltMaskPlanes(mask, altMask)
641  imageScaler.scaleMaskedImage(maskedImage)
642 
643  # Add 1 for each pixel which is not excluded by the exclude mask.
644  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
645  if nImage is not None:
646  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
647  if self.config.removeMaskPlanes:
648  mask = maskedImage.getMask()
649  for maskPlane in self.config.removeMaskPlanes:
650  try:
651  mask &= ~mask.getPlaneBitMask(maskPlane)
652  except Exception as e:
653  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
654 
655  maskedImageList.append(maskedImage)
656 
657  with self.timer("stack"):
658  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
659  clipped, # also set output to CLIPPED if sigma-clipped
660  maskMap)
661  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
662  if nImage is not None:
663  nImage.assign(subNImage, bbox)
664 
665  def applyAltMaskPlanes(self, mask, altMaskSpans):
666  """!
667  @brief Apply in place alt mask formatted as SpanSets to a mask
668 
669  @param mask: original mask
670  @param altMaskSpans: Dict containing spanSet lists to apply.
671  Each element contains the new mask plane name
672  (e.g. "CLIPPED and/or "NO_DATA") as the key,
673  and list of SpanSets to apply to the mask
674  """
675  if self.config.doUsePsfMatchedPolygons:
676  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
677  # Clear away any other masks outside the validPolygons. These pixels are no longer
678  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
679  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
680  # This mask-clearing step must only occur *before* applying the new masks below
681  for spanSet in altMaskSpans['NO_DATA']:
682  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
683 
684  for plane, spanSetList in altMaskSpans.items():
685  maskClipValue = mask.addMaskPlane(plane)
686  for spanSet in spanSetList:
687  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
688  return mask
689 
690  def shrinkValidPolygons(self, coaddInputs):
691  """!
692  @brief Shrink coaddInputs' ccds' ValidPolygons in place
693 
694  @param coaddInputs: original mask
695 
696  Either modify each ccd's validPolygon in place, or if CoaddInputs does not
697  have a validPolygon, create one from its bbox.
698  """
699  for ccd in coaddInputs.ccds:
700  polyOrig = ccd.getValidPolygon()
701  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
702  validPolyBBox.grow(-self.config.matchingKernelSize//2)
703  if polyOrig:
704  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
705  else:
706  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
707  ccd.setValidPolygon(validPolygon)
708 
709  def readBrightObjectMasks(self, dataRef):
710  """Returns None on failure"""
711  try:
712  return dataRef.get("brightObjectMask", immediate=True)
713  except Exception as e:
714  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
715  return None
716 
717  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
718  """Set the bright object masks
719 
720  exposure: Exposure under consideration
721  dataId: Data identifier dict for patch
722  brightObjectMasks: afwTable of bright objects to mask
723  """
724  #
725  # Check the metadata specifying the tract/patch/filter
726  #
727  if brightObjectMasks is None:
728  self.log.warn("Unable to apply bright object mask: none supplied")
729  return
730  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
731  md = brightObjectMasks.table.getMetadata()
732  for k in dataId:
733  if not md.exists(k):
734  self.log.warn("Expected to see %s in metadata", k)
735  else:
736  if md.get(k) != dataId[k]:
737  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
738 
739  mask = exposure.getMaskedImage().getMask()
740  wcs = exposure.getWcs()
741  plateScale = wcs.getPixelScale().asArcseconds()
742 
743  for rec in brightObjectMasks:
744  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
745  if rec["type"] == "box":
746  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
747  width = rec["width"].asArcseconds()/plateScale # convert to pixels
748  height = rec["height"].asArcseconds()/plateScale # convert to pixels
749 
750  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
751  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
752 
753  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
754  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
755  spans = afwGeom.SpanSet(bbox)
756  elif rec["type"] == "circle":
757  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
758  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
759  else:
760  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
761  continue
762  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
763 
764  def setInexactPsf(self, mask):
765  """Set INEXACT_PSF mask plane
766 
767  If any of the input images isn't represented in the coadd (due to
768  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
769  these pixels.
770 
771  Parameters
772  ----------
773  mask : `lsst.afw.image.Mask`
774  Coadded exposure's mask, modified in-place.
775  """
776  mask.addMaskPlane("INEXACT_PSF")
777  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
778  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
779  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
780  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
781  array = mask.getArray()
782  selected = array & (sensorEdge | clipped | rejected) > 0
783  array[selected] |= inexactPsf
784 
785  @classmethod
786  def _makeArgumentParser(cls):
787  """!
788  @brief Create an argument parser
789  """
790  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
791  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
792  cls.ConfigClass().warpType + "Warp",
793  help="data ID, e.g. --id tract=12345 patch=1,2",
794  ContainerClass=AssembleCoaddDataIdContainer)
795  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
796  ContainerClass=SelectDataIdContainer)
797  return parser
798 
799 
800 def _subBBoxIter(bbox, subregionSize):
801  """!
802  @brief Iterate over subregions of a bbox
803 
804  @param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
805  @param[in] subregionSize: size of sub-bboxes
806 
807  @return subBBox: next sub-bounding box of size subregionSize or smaller;
808  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
809  but it will never be empty
810  """
811  if bbox.isEmpty():
812  raise RuntimeError("bbox %s is empty" % (bbox,))
813  if subregionSize[0] < 1 or subregionSize[1] < 1:
814  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
815 
816  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
817  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
818  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
819  subBBox.clip(bbox)
820  if subBBox.isEmpty():
821  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
822  (bbox, subregionSize, colShift, rowShift))
823  yield subBBox
824 
825 
826 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
827  """!
828  @brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
829  """
830 
831  def makeDataRefList(self, namespace):
832  """!
833  @brief Make self.refList from self.idList.
834  """
835  datasetType = namespace.config.coaddName + "Coadd"
836  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
837 
838  for dataId in self.idList:
839  # tract and patch are required
840  for key in keysCoadd:
841  if key not in dataId:
842  raise RuntimeError("--id must include " + key)
843 
844  dataRef = namespace.butler.dataRef(
845  datasetType=datasetType,
846  dataId=dataId,
847  )
848  self.refList.append(dataRef)
849 
850 
851 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
852  """!
853  @brief Function to count the number of pixels with a specific mask in a footprint.
854 
855  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
856  have bitmask set but do not have ignoreMask set. Return the count.
857 
858  @param[in] mask: mask to define intersection region by.
859  @parma[in] footprint: footprint to define the intersection region by.
860  @param[in] bitmask: specific mask that we wish to count the number of occurances of.
861  @param[in] ignoreMask: pixels to not consider.
862  @return count of number of pixels in footprint with specified mask.
863  """
864  bbox = footprint.getBBox()
865  bbox.clip(mask.getBBox(afwImage.PARENT))
866  fp = afwImage.Mask(bbox)
867  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
868  footprint.spans.setMask(fp, bitmask)
869  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
870  (subMask.getArray() & ignoreMask) == 0).sum()
871 
872 
874  """!
875 @anchor SafeClipAssembleCoaddConfig
876 
877 @brief Configuration parameters for the SafeClipAssembleCoaddTask
878  """
879  clipDetection = pexConfig.ConfigurableField(
880  target=SourceDetectionTask,
881  doc="Detect sources on difference between unclipped and clipped coadd")
882  minClipFootOverlap = pexConfig.Field(
883  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
884  dtype=float,
885  default=0.6
886  )
887  minClipFootOverlapSingle = pexConfig.Field(
888  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
889  "clipped when only one visit overlaps",
890  dtype=float,
891  default=0.5
892  )
893  minClipFootOverlapDouble = pexConfig.Field(
894  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
895  "clipped when two visits overlap",
896  dtype=float,
897  default=0.45
898  )
899  maxClipFootOverlapDouble = pexConfig.Field(
900  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
901  "considering two visits",
902  dtype=float,
903  default=0.15
904  )
905  minBigOverlap = pexConfig.Field(
906  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
907  "when labeling clipped footprints",
908  dtype=int,
909  default=100
910  )
911 
912  def setDefaults(self):
913  # The numeric values for these configuration parameters were empirically determined, future work
914  # may further refine them.
915  AssembleCoaddConfig.setDefaults(self)
916  self.clipDetection.doTempLocalBackground = False
917  self.clipDetection.reEstimateBackground = False
918  self.clipDetection.returnOriginalFootprints = False
919  self.clipDetection.thresholdPolarity = "both"
920  self.clipDetection.thresholdValue = 2
921  self.clipDetection.nSigmaToGrow = 2
922  self.clipDetection.minPixels = 4
923  self.clipDetection.isotropicGrow = True
924  self.clipDetection.thresholdType = "pixel_stdev"
925  self.sigmaClip = 1.5
926  self.clipIter = 3
927  self.statistic = "MEAN"
928 
929  def validate(self):
930  if self.doSigmaClip:
931  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
932  "Ignoring doSigmaClip.")
933  self.doSigmaClip = False
934  if self.statistic != "MEAN":
935  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
936  "(%s chosen). Please set statistic to MEAN."
937  % (self.statistic))
938  AssembleCoaddTask.ConfigClass.validate(self)
939 
940 
941 
947 
948 
950  """!
951  @anchor SafeClipAssembleCoaddTask_
952 
953  @brief Assemble a coadded image from a set of coadded temporary exposures,
954  being careful to clip & flag areas with potential artifacts.
955 
956  @section pipe_tasks_assembleCoadd_Contents Contents
957  - @ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
958  - @ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
959  - @ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
960  - @ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
961  - @ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
962  - @ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
963 
964  @section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
965 
966  @copybrief SafeClipAssembleCoaddTask
967 
968  Read the documentation for @ref AssembleCoaddTask_ "AssembleCoaddTask" first since
969  SafeClipAssembleCoaddTask subtasks that task.
970  In @ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
971  outliers).
972  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
973  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
974  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
975  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
976  that there is an outlier and ii. this outlier appears on only one or two images.
977  Such regions will not contribute to the final coadd.
978  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
979  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
980  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
981  surveys.
982 
983  SafeClipAssembleCoaddTask uses a @ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
984  @ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
985  @ref SourceDetectionTask_ "clipDetection" subtask if you wish.
986 
987  @section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
988  @copydoc \_\_init\_\_
989 
990  @section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
991  @copydoc run
992 
993  @section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
994  See @ref SafeClipAssembleCoaddConfig
995 
996  @section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
997  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
998  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
999  files.
1000  SafeClipAssembleCoaddTask has no debug variables of its own. The @ref SourceDetectionTask_ "clipDetection"
1001  subtasks may support debug variables. See the documetation for @ref SourceDetectionTask_ "clipDetection"
1002  for further information.
1003 
1004  @section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
1005  SafeClipAssembleCoaddTask
1006 
1007  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1008  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1009  '--legacyCoadd'.
1010  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1011  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1012  with a list of coaddTempExps to attempt to coadd (specified using
1013  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1014  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1015  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1016  command line argument:
1017  @code
1018  assembleCoadd.py --help
1019  @endcode
1020  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1021  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1022  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1023  packages.
1024  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1025  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1026  <DL>
1027  <DT>processCcd</DT>
1028  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1029  <DT>makeSkyMap</DT>
1030  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1031  <DT>makeCoaddTempExp</DT>
1032  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1033  </DL>
1034  We can perform all of these steps by running
1035  @code
1036  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1037  @endcode
1038  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1039  as follows:
1040  @code
1041  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1042  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1043  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1044  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1045  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1046  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1047  --selectId visit=903988 ccd=24
1048  @endcode
1049  This will process the HSC-I band data. The results are written in
1050  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1051 
1052  You may also choose to run:
1053  @code
1054  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1055  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1056  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1057  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1058  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1059  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1060  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1061  --selectId visit=903346 ccd=12
1062  @endcode
1063  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1064  discussed in @ref pipeTasks_multiBand.
1065  """
1066  ConfigClass = SafeClipAssembleCoaddConfig
1067  _DefaultName = "safeClipAssembleCoadd"
1068 
1069  def __init__(self, *args, **kwargs):
1070  """!
1071  @brief Initialize the task and make the @ref SourceDetectionTask_ "clipDetection" subtask.
1072  """
1073  AssembleCoaddTask.__init__(self, *args, **kwargs)
1074  schema = afwTable.SourceTable.makeMinimalSchema()
1075  self.makeSubtask("clipDetection", schema=schema)
1076 
1077  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1078  """!
1079  @brief Assemble the coadd for a region
1080 
1081  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1082  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1083  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1084  overlap between the clipped region and a source.
1085  This leaves us with a set of footprints from the difference image that have been identified as having
1086  occured on just one or two individual visits. However, these footprints were generated from a
1087  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1088  footprints acrosss the coadd difference in this process.
1089  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1090  these are big footprints.
1091  Combine the small and big clipped footprints and mark them on a new bad mask plane
1092  Generate the coadd using @ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1093  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1094  bad mask plane.
1095 
1096  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1097  parent task.
1098 
1099  @param skyInfo: Patch geometry information, from getSkyInfo
1100  @param tempExpRefList: List of data reference to tempExp
1101  @param imageScalerList: List of image scalers
1102  @param weightList: List of weights
1103  return pipeBase.Struct with coaddExposure, nImage
1104  """
1105  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1106  mask = exp.getMaskedImage().getMask()
1107  mask.addMaskPlane("CLIPPED")
1108 
1109  result = self.detectClip(exp, tempExpRefList)
1110 
1111  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1112 
1113  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1114  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1115  # Append big footprints from individual Warps to result.clipSpans
1116  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1117  result.detectionFootprints, maskClipValue, maskDetValue,
1118  exp.getBBox())
1119  # Create mask of the current clipped footprints
1120  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1121  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1122 
1123  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1124  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1125  maskClip |= maskClipBig
1126 
1127  # Assemble coadd from base class, but ignoring CLIPPED pixels
1128  badMaskPlanes = self.config.badMaskPlanes[:]
1129  badMaskPlanes.append("CLIPPED")
1130  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1131  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1132  result.clipSpans, mask=badPixelMask)
1133 
1134  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1135  """!
1136  @brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1137 
1138  Generate a difference image between clipped and unclipped coadds.
1139  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1140  Return the difference image.
1141 
1142  @param skyInfo: Patch geometry information, from getSkyInfo
1143  @param tempExpRefList: List of data reference to tempExp
1144  @param imageScalerList: List of image scalers
1145  @param weightList: List of weights
1146  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1147  """
1148  # Clone and upcast self.config because current self.config is frozen
1149  config = AssembleCoaddConfig()
1150  # getattr necessary because subtasks do not survive Config.toDict()
1151  configIntersection = {k: getattr(self.config, k)
1152  for k, v in self.config.toDict().items() if (k in config.keys())}
1153  config.update(**configIntersection)
1154 
1155  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1156  config.statistic = 'MEAN'
1157  task = AssembleCoaddTask(config=config)
1158  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1159 
1160  config.statistic = 'MEANCLIP'
1161  task = AssembleCoaddTask(config=config)
1162  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1163 
1164  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1165  coaddDiff -= coaddClip.getMaskedImage()
1166  exp = afwImage.ExposureF(coaddDiff)
1167  exp.setPsf(coaddMean.getPsf())
1168  return exp
1169 
1170  def detectClip(self, exp, tempExpRefList):
1171  """!
1172  @brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1173 
1174  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1175  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1176  fraction to thresholds set in the config.
1177  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1178  two).
1179  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1180  the coaddTempExp.
1181  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1182  with the clipped footprints and a list of new masks for the coaddTempExps.
1183 
1184  @param[in] exp: Exposure to run detection on
1185  @param[in] tempExpRefList: List of data reference to tempExp
1186  @return struct containing:
1187  - clipFootprints: list of clipped footprints
1188  - clipIndices: indices for each clippedFootprint in tempExpRefList
1189  - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new
1190  maskplane name ("CLIPPED")" as the key and list of SpanSets as value
1191  - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints
1192  """
1193  mask = exp.getMaskedImage().getMask()
1194  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1195  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1196  # Merge positive and negative together footprints together
1197  fpSet.positive.merge(fpSet.negative)
1198  footprints = fpSet.positive
1199  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1200  ignoreMask = self.getBadPixelMask()
1201 
1202  clipFootprints = []
1203  clipIndices = []
1204  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1205 
1206  # for use by detectClipBig
1207  visitDetectionFootprints = []
1208 
1209  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1210  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1211  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1212 
1213  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1214  for i, warpRef in enumerate(tempExpRefList):
1215  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1216  immediate=True).getMaskedImage().getMask()
1217  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1218  afwImage.PARENT, True)
1219  maskVisitDet &= maskDetValue
1220  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1221  visitDetectionFootprints.append(visitFootprints)
1222 
1223  for j, footprint in enumerate(footprints.getFootprints()):
1224  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1225  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1226 
1227  # build a list of clipped spans for each visit
1228  for j, footprint in enumerate(footprints.getFootprints()):
1229  nPixel = footprint.getArea()
1230  overlap = [] # hold the overlap with each visit
1231  indexList = [] # index of visit in global list
1232  for i in range(len(tempExpRefList)):
1233  ignore = ignoreArr[i, j]
1234  overlapDet = overlapDetArr[i, j]
1235  totPixel = nPixel - ignore
1236 
1237  # If we have more bad pixels than detection skip
1238  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1239  continue
1240  overlap.append(overlapDet/float(totPixel))
1241  indexList.append(i)
1242 
1243  overlap = numpy.array(overlap)
1244  if not len(overlap):
1245  continue
1246 
1247  keep = False # Should this footprint be marked as clipped?
1248  keepIndex = [] # Which tempExps does the clipped footprint belong to
1249 
1250  # If footprint only has one overlap use a lower threshold
1251  if len(overlap) == 1:
1252  if overlap[0] > self.config.minClipFootOverlapSingle:
1253  keep = True
1254  keepIndex = [0]
1255  else:
1256  # This is the general case where only visit should be clipped
1257  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1258  if len(clipIndex) == 1:
1259  keep = True
1260  keepIndex = [clipIndex[0]]
1261 
1262  # Test if there are clipped objects that overlap two different visits
1263  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1264  if len(clipIndex) == 2 and len(overlap) > 3:
1265  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1266  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1267  keep = True
1268  keepIndex = clipIndex
1269 
1270  if not keep:
1271  continue
1272 
1273  for index in keepIndex:
1274  globalIndex = indexList[index]
1275  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1276 
1277  clipIndices.append(numpy.array(indexList)[keepIndex])
1278  clipFootprints.append(footprint)
1279 
1280  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1281  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1282 
1283  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1284  maskClipValue, maskDetValue, coaddBBox):
1285  """!
1286  @brief Return individual warp footprints for large artifacts and append them to clipList in place
1287 
1288  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1289  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1290  significantly with each source in all the coaddTempExps.
1291  @param[in] clipList: List of alt mask SpanSets with clipping information. Modified.
1292  @param[in] clipFootprints: List of clipped footprints
1293  @param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1294  @param[in] maskClipValue: Mask value of clipped pixels
1295  @param[in] maskDetValue: Mask value of detected pixels
1296  @param[in] coaddBBox: BBox of the coadd and warps
1297  @return list of big footprints
1298  """
1299  bigFootprintsCoadd = []
1300  ignoreMask = self.getBadPixelMask()
1301  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1302  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1303  for footprint in visitFootprints.getFootprints():
1304  footprint.spans.setMask(maskVisitDet, maskDetValue)
1305 
1306  # build a mask of clipped footprints that are in this visit
1307  clippedFootprintsVisit = []
1308  for foot, clipIndex in zip(clipFootprints, clipIndices):
1309  if index not in clipIndex:
1310  continue
1311  clippedFootprintsVisit.append(foot)
1312  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1313  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1314 
1315  bigFootprintsVisit = []
1316  for foot in visitFootprints.getFootprints():
1317  if foot.getArea() < self.config.minBigOverlap:
1318  continue
1319  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1320  if nCount > self.config.minBigOverlap:
1321  bigFootprintsVisit.append(foot)
1322  bigFootprintsCoadd.append(foot)
1323 
1324  for footprint in bigFootprintsVisit:
1325  clippedSpans["CLIPPED"].append(footprint.spans)
1326 
1327  return bigFootprintsCoadd
1328 
1329 
1331  assembleStaticSkyModel = pexConfig.ConfigurableField(
1332  target=AssembleCoaddTask,
1333  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1334  " naive/first-iteration model of the static sky.",
1335  )
1336  detect = pexConfig.ConfigurableField(
1337  target=SourceDetectionTask,
1338  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1339  )
1340  detectTemplate = pexConfig.ConfigurableField(
1341  target=SourceDetectionTask,
1342  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1343  )
1344  maxNumEpochs = pexConfig.Field(
1345  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1346  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1347  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1348  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1349  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1350  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1351  "than transient and not masked.",
1352  dtype=int,
1353  default=2
1354  )
1355  maxFractionEpochsLow = pexConfig.RangeField(
1356  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1357  "Effective maxNumEpochs = "
1358  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1359  dtype=float,
1360  default=0.4,
1361  min=0., max=1.,
1362  )
1363  maxFractionEpochsHigh = pexConfig.RangeField(
1364  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1365  "Effective maxNumEpochs = "
1366  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1367  dtype=float,
1368  default=0.03,
1369  min=0., max=1.,
1370  )
1371  spatialThreshold = pexConfig.RangeField(
1372  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1373  "temporal criteria. If 0, clip all. If 1, clip none.",
1374  dtype=float,
1375  default=0.5,
1376  min=0., max=1.,
1377  inclusiveMin=True, inclusiveMax=True
1378  )
1379  doScaleWarpVariance = pexConfig.Field(
1380  doc="Rescale Warp variance plane using empirical noise?",
1381  dtype=bool,
1382  default=True,
1383  )
1384  scaleWarpVariance = pexConfig.ConfigurableField(
1385  target=ScaleVarianceTask,
1386  doc="Rescale variance on warps",
1387  )
1388  doPreserveContainedBySource = pexConfig.Field(
1389  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1390  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1391  dtype=bool,
1392  default=True,
1393  )
1394  doPrefilterArtifacts = pexConfig.Field(
1395  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1396  "because they will be excluded anyway. This prevents them from contributing "
1397  "to the outlier epoch count image and potentially being labeled as persistant."
1398  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1399  dtype=bool,
1400  default=True
1401  )
1402  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1403  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1404  dtype=str,
1405  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1406  )
1407  prefilterArtifactsRatio = pexConfig.Field(
1408  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1409  dtype=float,
1410  default=0.05
1411  )
1412 
1413  def setDefaults(self):
1414  AssembleCoaddConfig.setDefaults(self)
1415  self.statistic = 'MEAN'
1417 
1418  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1419  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1420  if "EDGE" in self.badMaskPlanes:
1421  self.badMaskPlanes.remove('EDGE')
1422  self.removeMaskPlanes.append('EDGE')
1423  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1424  self.assembleStaticSkyModel.warpType = 'psfMatched'
1425  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1426  self.assembleStaticSkyModel.sigmaClip = 2.5
1427  self.assembleStaticSkyModel.clipIter = 3
1428  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1429  self.assembleStaticSkyModel.doWrite = False
1430  self.detect.doTempLocalBackground = False
1431  self.detect.reEstimateBackground = False
1432  self.detect.returnOriginalFootprints = False
1433  self.detect.thresholdPolarity = "both"
1434  self.detect.thresholdValue = 5
1435  self.detect.nSigmaToGrow = 2
1436  self.detect.minPixels = 4
1437  self.detect.isotropicGrow = True
1438  self.detect.thresholdType = "pixel_stdev"
1439  self.detectTemplate.nSigmaToGrow = 2.0
1440  self.detectTemplate.doTempLocalBackground = False
1441  self.detectTemplate.reEstimateBackground = False
1442  self.detectTemplate.returnOriginalFootprints = False
1443 
1444 
1445 
1451 
1453  """!
1454  @anchor CompareWarpAssembleCoaddTask_
1455 
1456  @brief Assemble a compareWarp coadded image from a set of warps
1457  by masking artifacts detected by comparing PSF-matched warps
1458 
1459  @section pipe_tasks_assembleCoadd_Contents Contents
1460  - @ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1461  - @ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1462  - @ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1463  - @ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1464  - @ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1465  - @ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1466 
1467  @section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1468 
1469  @copybrief CompareWarpAssembleCoaddTask
1470 
1471  In @ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1472  outliers).
1473  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1474  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1475  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1476  pixels in the individual warps suspected to contain an artifact.
1477  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1478  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1479  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1480  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1481  such as bright stars.
1482  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1483  The temporalThreshold is the maximum fraction of epochs that the deviation can
1484  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1485  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1486  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1487  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1488  These regions will not contribute to the final coadd.
1489  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1490  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1491  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1492  surveys.
1493 
1494  CompareWarpAssembleCoaddTask sub-classes
1495  @ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates @ref AssembleCoaddTask_ "AssembleCoaddTask"
1496  as a subtask to generate the TemplateCoadd (the model of the static sky)
1497 
1498  @section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1499  @copydoc \_\_init\_\_
1500 
1501  @section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1502  @copydoc run
1503 
1504  @section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1505  See @ref CompareWarpAssembleCoaddConfig
1506 
1507  @section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1508  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
1509  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
1510  files.
1511 
1512  This task supports the following debug variables:
1513  <dl>
1514  <dt>`saveCountIm`
1515  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1516  <dt> `figPath`
1517  <dd> Path to save the debug fits images and figures
1518  </dl>
1519 
1520  For example, put something like:
1521  @code{.py}
1522  import lsstDebug
1523  def DebugInfo(name):
1524  di = lsstDebug.getInfo(name)
1525  if name == "lsst.pipe.tasks.assembleCoadd":
1526  di.saveCountIm = True
1527  di.figPath = "/desired/path/to/debugging/output/images"
1528  return di
1529  lsstDebug.Info = DebugInfo
1530  @endcode
1531  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1532  flag.
1533  Some subtasks may have their own debug variables; see individual Task
1534  documentation
1535 
1536  @section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1537  CompareWarpAssembleCoaddTask
1538 
1539  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1540  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1541  '--compareWarpCoadd'.
1542  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1543  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1544  with a list of coaddTempExps to attempt to coadd (specified using
1545  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1546  Only the warps that cover the specified tract and patch will be coadded.
1547  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1548  command line argument:
1549  @code
1550  assembleCoadd.py --help
1551  @endcode
1552  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1553  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1554  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1555  packages.
1556  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1557  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1558  <DL>
1559  <DT>processCcd</DT>
1560  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1561  <DT>makeSkyMap</DT>
1562  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1563  <DT>makeCoaddTempExp</DT>
1564  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1565  </DL>
1566  We can perform all of these steps by running
1567  @code
1568  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1569  @endcode
1570  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1571  as follows:
1572  @code
1573  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1574  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1575  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1576  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1577  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1578  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1579  --selectId visit=903988 ccd=24
1580  @endcode
1581  This will process the HSC-I band data. The results are written in
1582  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1583  """
1584  ConfigClass = CompareWarpAssembleCoaddConfig
1585  _DefaultName = "compareWarpAssembleCoadd"
1586 
1587  def __init__(self, *args, **kwargs):
1588  """!
1589  @brief Initialize the task and make the @ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1590  """
1591  AssembleCoaddTask.__init__(self, *args, **kwargs)
1592  self.makeSubtask("assembleStaticSkyModel")
1593  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1594  self.makeSubtask("detect", schema=detectionSchema)
1595  if self.config.doPreserveContainedBySource:
1596  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1597  if self.config.doScaleWarpVariance:
1598  self.makeSubtask("scaleWarpVariance")
1599 
1600  def makeSupplementaryData(self, dataRef, selectDataList):
1601  """!
1602  @brief Make inputs specific to Subclass
1603 
1604  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1605  """
1606  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1607 
1608  if templateCoadd is None:
1609  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1610  self.assembleStaticSkyModel.warpType[1:])
1611  message = """No %(warpName)s warps were found to build the template coadd which is
1612  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1613  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1614  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1615 
1616  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1617  another algorithm like:
1618 
1619  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1620  config.assemble.retarget(SafeClipAssembleCoaddTask)
1621  """ % {"warpName": warpName}
1622  raise RuntimeError(message)
1623 
1624  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1625 
1626  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1627  supplementaryData, *args, **kwargs):
1628  """!
1629  @brief Assemble the coadd
1630 
1631  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1632  as the model of the static sky.
1633 
1634  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1635  new "CLIPPED" plane and updated "NO_DATA" plane.
1636  Then pass these alternative masks to the base class's assemble method.
1637 
1638  @param skyInfo: Patch geometry information
1639  @param tempExpRefList: List of data references to warps
1640  @param imageScalerList: List of image scalers
1641  @param weightList: List of weights
1642  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1643 
1644  return pipeBase.Struct with coaddExposure, nImage if requested
1645  """
1646  templateCoadd = supplementaryData.templateCoadd
1647  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1648  badMaskPlanes = self.config.badMaskPlanes[:]
1649  badMaskPlanes.append("CLIPPED")
1650  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1651 
1652  result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1653  spanSetMaskList, mask=badPixelMask)
1654 
1655  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
1656  # Psf-Matching moves the real edge inwards
1657  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1658  return result
1659 
1660  def applyAltEdgeMask(self, mask, altMaskList):
1661  """!
1662  @brief Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes
1663 
1664  @param mask: original mask
1665  @param altMaskList: List of Dicts containing spanSet lists.
1666  Each element contains the new mask plane name
1667  (e.g. "CLIPPED and/or "NO_DATA") as the key,
1668  and list of SpanSets to apply to the mask.
1669  """
1670  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1671  for visitMask in altMaskList:
1672  if "EDGE" in visitMask:
1673  for spanSet in visitMask['EDGE']:
1674  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1675 
1676  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1677  """!
1678  @brief Find artifacts
1679 
1680  Loop through warps twice. The first loop builds a map with the count of how many
1681  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1682  The second loop takes each difference image and filters the artifacts detected
1683  in each using count map to filter out variable sources and sources that are difficult to
1684  subtract cleanly.
1685 
1686  @param templateCoadd: Exposure to serve as model of static sky
1687  @param tempExpRefList: List of data references to warps
1688  @param imageScalerList: List of image scalers
1689  """
1690 
1691  self.log.debug("Generating Count Image, and mask lists.")
1692  coaddBBox = templateCoadd.getBBox()
1693  slateIm = afwImage.ImageU(coaddBBox)
1694  epochCountImage = afwImage.ImageU(coaddBBox)
1695  nImage = afwImage.ImageU(coaddBBox)
1696  spanSetArtifactList = []
1697  spanSetNoDataMaskList = []
1698  spanSetEdgeList = []
1699  badPixelMask = self.getBadPixelMask()
1700 
1701  # mask of the warp diffs should = that of only the warp
1702  templateCoadd.mask.clearAllMaskPlanes()
1703 
1704  if self.config.doPreserveContainedBySource:
1705  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1706  else:
1707  templateFootprints = None
1708 
1709  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1710  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1711  if warpDiffExp is not None:
1712  # This nImage only approximates the final nImage because it uses the PSF-matched mask
1713  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1714  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1715  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1716  fpSet.positive.merge(fpSet.negative)
1717  footprints = fpSet.positive
1718  slateIm.set(0)
1719  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1720 
1721  # Remove artifacts due to defects before they contribute to the epochCountImage
1722  if self.config.doPrefilterArtifacts:
1723  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
1724  for spans in spanSetList:
1725  spans.setImage(slateIm, 1, doClip=True)
1726  epochCountImage += slateIm
1727 
1728  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1729  # undergo a second convolution. Pixels with data in the direct warp
1730  # but not in the PSF-matched warp will not have their artifacts detected.
1731  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1732  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1733  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1734  nansMask.setXY0(warpDiffExp.getXY0())
1735  edgeMask = warpDiffExp.mask
1736  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1737  edgeMask.getPlaneBitMask("EDGE")).split()
1738  else:
1739  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1740  # In this case, mask the whole epoch
1741  nansMask = afwImage.MaskX(coaddBBox, 1)
1742  spanSetList = []
1743  spanSetEdgeMask = []
1744 
1745  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1746 
1747  spanSetNoDataMaskList.append(spanSetNoDataMask)
1748  spanSetArtifactList.append(spanSetList)
1749  spanSetEdgeList.append(spanSetEdgeMask)
1750 
1751  if lsstDebug.Info(__name__).saveCountIm:
1752  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1753  epochCountImage.writeFits(path)
1754 
1755  for i, spanSetList in enumerate(spanSetArtifactList):
1756  if spanSetList:
1757  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
1758  templateFootprints)
1759  spanSetArtifactList[i] = filteredSpanSetList
1760 
1761  altMasks = []
1762  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1763  altMasks.append({'CLIPPED': artifacts,
1764  'NO_DATA': noData,
1765  'EDGE': edge})
1766  return altMasks
1767 
1768  def prefilterArtifacts(self, spanSetList, exp):
1769  """!
1770  @brief Remove artifact candidates covered by bad mask plane
1771 
1772  Any future editing of the candidate list that does not depend on
1773  temporal information should go in this method.
1774 
1775  @param spanSetList: List of SpanSets representing artifact candidates
1776  @param exp: Exposure containing mask planes used to prefilter
1777 
1778  return List of SpanSets with artifacts
1779  """
1780  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1781  goodArr = (exp.mask.array & badPixelMask) == 0
1782  returnSpanSetList = []
1783  bbox = exp.getBBox()
1784  x0, y0 = exp.getXY0()
1785  for i, span in enumerate(spanSetList):
1786  y, x = span.clippedTo(bbox).indices()
1787  yIndexLocal = numpy.array(y) - y0
1788  xIndexLocal = numpy.array(x) - x0
1789  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1790  if goodRatio > self.config.prefilterArtifactsRatio:
1791  returnSpanSetList.append(span)
1792  return returnSpanSetList
1793 
1794  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1795  """!
1796  @brief Filter artifact candidates
1797 
1798  @param spanSetList: List of SpanSets representing artifact candidates
1799  @param epochCountImage: Image of accumulated number of warpDiff detections
1800  @param nImage: Image of the accumulated number of total epochs contributing
1801 
1802  return List of SpanSets with artifacts
1803  """
1804 
1805  maskSpanSetList = []
1806  x0, y0 = epochCountImage.getXY0()
1807  for i, span in enumerate(spanSetList):
1808  y, x = span.indices()
1809  yIdxLocal = [y1 - y0 for y1 in y]
1810  xIdxLocal = [x1 - x0 for x1 in x]
1811  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1812  totalN = nImage.array[yIdxLocal, xIdxLocal]
1813 
1814  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1815  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1816  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1817  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1818  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1819  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1820  (outlierN <= effectiveMaxNumEpochs))
1821  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1822  if percentBelowThreshold > self.config.spatialThreshold:
1823  maskSpanSetList.append(span)
1824 
1825  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
1826  # If a candidate is contained by a footprint on the template coadd, do not clip
1827  filteredMaskSpanSetList = []
1828  for span in maskSpanSetList:
1829  doKeep = True
1830  for footprint in footprintsToExclude.positive.getFootprints():
1831  if footprint.spans.contains(span):
1832  doKeep = False
1833  break
1834  if doKeep:
1835  filteredMaskSpanSetList.append(span)
1836  maskSpanSetList = filteredMaskSpanSetList
1837 
1838  return maskSpanSetList
1839 
1840  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1841  """!
1842  @brief Fetch a warp from the butler and return a warpDiff
1843 
1844  @param warpRef: `Butler dataRef` for the warp
1845  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1846  @param templateCoadd: Exposure to be substracted from the scaled warp
1847 
1848  return Exposure of the image difference between the warp and template
1849  """
1850 
1851  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1852  warpName = self.getTempExpDatasetName('psfMatched')
1853  if not warpRef.datasetExists(warpName):
1854  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1855  return None
1856  warp = warpRef.get(warpName, immediate=True)
1857  # direct image scaler OK for PSF-matched Warp
1858  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1859  mi = warp.getMaskedImage()
1860  if self.config.doScaleWarpVariance:
1861  try:
1862  self.scaleWarpVariance.run(mi)
1863  except Exception as exc:
1864  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
1865  mi -= templateCoadd.getMaskedImage()
1866  return warp
1867 
1868  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1869  """!
1870  @brief Return a path to which to write debugging output
1871 
1872  @param prefix: string, prefix for filename
1873  @param warpRef: Butler dataRef
1874  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1875  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1876 
1877  Creates a hyphen-delimited string of dataId values for simple filenames.
1878  """
1879  if coaddLevel:
1880  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1881  else:
1882  keys = warpRef.dataId.keys()
1883  keyList = sorted(keys, reverse=True)
1884  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1885  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1886  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
Shrink coaddInputs&#39; ccds&#39; ValidPolygons in place.
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:171
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:94
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:127
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:221
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Filter artifact candidates.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:107
def applyAltEdgeMask(self, mask, altMaskList)
Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def prefilterArtifacts(self, spanSetList, exp)
Remove artifact candidates covered by bad mask plane.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...