lsst.pipe.tasks  14.0-41-g36ac3fe5+1
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import os
26 import numpy
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, scaleVariance
40 from .interpImage import InterpImageTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms import SourceDetectionTask
44 
45 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
46 
47 
48 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
49  """!
50 \anchor AssembleCoaddConfig_
51 
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53  """
54  warpType = pexConfig.Field(
55  doc="Warp name: one of 'direct' or 'psfMatched'",
56  dtype=str,
57  default="direct",
58  )
59  subregionSize = pexConfig.ListField(
60  dtype=int,
61  doc="Width, height of stack subregion size; "
62  "make small enough that a full stack of images will fit into memory at once.",
63  length=2,
64  default=(2000, 2000),
65  )
66  statistic = pexConfig.Field(
67  dtype=str,
68  doc="Main stacking statistic for aggregating over the epochs.",
69  default="MEANCLIP",
70  )
71  doSigmaClip = pexConfig.Field(
72  dtype=bool,
73  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
74  default=False,
75  )
76  sigmaClip = pexConfig.Field(
77  dtype=float,
78  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
79  default=3.0,
80  )
81  clipIter = pexConfig.Field(
82  dtype=int,
83  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
84  default=2,
85  )
86  calcErrorFromInputVariance = pexConfig.Field(
87  dtype=bool,
88  doc="Calculate coadd variance from input variance by stacking statistic."
89  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
90  default=True,
91  )
92  scaleZeroPoint = pexConfig.ConfigurableField(
93  target=ScaleZeroPointTask,
94  doc="Task to adjust the photometric zero point of the coadd temp exposures",
95  )
96  doInterp = pexConfig.Field(
97  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
98  dtype=bool,
99  default=True,
100  )
101  interpImage = pexConfig.ConfigurableField(
102  target=InterpImageTask,
103  doc="Task to interpolate (and extrapolate) over NaN pixels",
104  )
105  doWrite = pexConfig.Field(
106  doc="Persist coadd?",
107  dtype=bool,
108  default=True,
109  )
110  doNImage = pexConfig.Field(
111  doc="Create image of number of contributing exposures for each pixel",
112  dtype=bool,
113  default=False,
114  )
115  maskPropagationThresholds = pexConfig.DictField(
116  keytype=str,
117  itemtype=float,
118  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
119  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
120  "would have contributed exceeds this value."),
121  default={"SAT": 0.1},
122  )
123  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
124  doc="Mask planes to remove before coadding")
125  #
126  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
127  # To make this useful you *must* also configure the flags.pixel algorithm, for example
128  # by adding
129  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
130  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
131  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
132  #
133  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
134  doc="Set mask and flag bits for bright objects?")
135  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
136  doc="Name of mask bit used for bright objects")
137 
138  coaddPsf = pexConfig.ConfigField(
139  doc="Configuration for CoaddPsf",
140  dtype=measAlg.CoaddPsfConfig,
141  )
142 
143  def setDefaults(self):
144  CoaddBaseTask.ConfigClass.setDefaults(self)
145  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
146 
147  def validate(self):
148  CoaddBaseTask.ConfigClass.validate(self)
149  if self.doPsfMatch:
150  # Backwards compatibility.
151  # Configs do not have loggers
152  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
153  self.warpType = 'psfMatched'
154  if self.doSigmaClip and self.statistic != "MEANCLIP":
155  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
156  self.statistic = "MEANCLIP"
157  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
158  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
159  "compute and set a non-zero coadd variance estimate." % (self.statistic))
160 
161  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
162  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
163  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
164  if str(k) not in unstackableStats]
165  raise ValueError("statistic %s is not allowed. Please choose one of %s."
166  % (self.statistic, stackableStats))
167 
168 
169 
176  """!
177 \anchor AssembleCoaddTask_
178 
179 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
180 
181 \section pipe_tasks_assembleCoadd_Contents Contents
182  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
183  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
184  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
185  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
186  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
187  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
188 
189 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
190 
191 \copybrief AssembleCoaddTask_
192 
193 We want to assemble a coadded image from a set of Warps (also called
194 coadded temporary exposures or coaddTempExps.
195 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
196 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
197 Warps that cover the specified patch (pointed at by dataRef).
198 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
199 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
200 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
201 config.makePsfMatched set which of the warp types will be coadded.
202 The coadd is computed as a mean with optional outlier rejection.
203 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
204 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
205 
206 AssembleCoaddTask uses several sub-tasks. These are
207 <DL>
208  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
209  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
210  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
211  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
212 </DL>
213 You can retarget these subtasks if you wish.
214 
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
216 \copydoc \_\_init\_\_
217 
218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
219 \copydoc run
220 
221 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
222 See \ref AssembleCoaddConfig_
223 
224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
225 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
226 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
227 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
228 the documetation for the subtasks for further information.
229 
230 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
231 
232 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
233 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
234 a data reference to the tract patch and filter to be coadded (specified using
235 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
236 Warps to attempt to coadd (specified using
237 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
238 that cover the specified tract and patch will be coadded. A list of the available optional
239 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
240 \code
241 assembleCoadd.py --help
242 \endcode
243 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
244 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
245 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
246 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
247 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
248 <DL>
249  <DT>processCcd</DT>
250  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
251  <DT>makeSkyMap</DT>
252  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
253  <DT>makeCoaddTempExp</DT>
254  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
255 </DL>
256 We can perform all of these steps by running
257 \code
258 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
259 \endcode
260 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
261 follows:
262 \code
263 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
264 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
265 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
266 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
267 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
268 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
269 --selectId visit=903988 ccd=24
270 \endcode
271 that will process the HSC-I band data. The results are written in
272 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
273 
274 You may also choose to run:
275 \code
276 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
277 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
278 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
279 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
280 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
281 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
282 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
283 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
284 \endcode
285 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
286 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
287 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
288  """
289  ConfigClass = AssembleCoaddConfig
290  _DefaultName = "assembleCoadd"
291 
292  def __init__(self, *args, **kwargs):
293  """!
294  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
295  & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
296  """
297  CoaddBaseTask.__init__(self, *args, **kwargs)
298  self.makeSubtask("interpImage")
299  self.makeSubtask("scaleZeroPoint")
300 
301  if self.config.doMaskBrightObjects:
302  mask = afwImage.Mask()
303  try:
304  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
305  except pexExceptions.LsstCppException:
306  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
307  mask.getMaskPlaneDict().keys())
308  del mask
309 
310  self.warpType = self.config.warpType
311 
312  @pipeBase.timeMethod
313  def run(self, dataRef, selectDataList=[]):
314  """!
315  \brief Assemble a coadd from a set of Warps
316 
317  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
318  match the photometric zeropoint to a reference Warp. Assemble the Warps using
319  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
320  exposure.
321 
322  \anchor runParams
323  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
324  (if config.autoReference=False). Used to access the following data products:
325  - [in] self.config.coaddName + "Coadd_skyMap"
326  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
327  - [out] self.config.coaddName + "Coadd"
328  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
329  selected from this list based on overlap with the patch defined by dataRef.
330 
331  \return a pipeBase.Struct with fields:
332  - coaddExposure: coadded exposure
333  - nImage: exposure count image
334  """
335  skyInfo = self.getSkyInfo(dataRef)
336  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
337  if len(calExpRefList) == 0:
338  self.log.warn("No exposures to coadd")
339  return
340  self.log.info("Coadding %d exposures", len(calExpRefList))
341 
342  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
343  inputData = self.prepareInputs(tempExpRefList)
344  self.log.info("Found %d %s", len(inputData.tempExpRefList),
345  self.getTempExpDatasetName(self.warpType))
346  if len(inputData.tempExpRefList) == 0:
347  self.log.warn("No coadd temporary exposures found")
348  return
349 
350  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
351 
352  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
353  inputData.weightList, supplementaryData=supplementaryData)
354 
355  if self.config.doInterp:
356  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
357  # The variance must be positive; work around for DM-3201.
358  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
359  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
360 
361  if self.config.doMaskBrightObjects:
362  brightObjectMasks = self.readBrightObjectMasks(dataRef)
363  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
364 
365  if self.config.doWrite:
366  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
367  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
368  if self.config.doNImage and retStruct.nImage is not None:
369  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
370 
371  return retStruct
372 
373  def makeSupplementaryData(self, dataRef, selectDataList):
374  """!
375  \brief Make additional inputs to assemble() specific to subclasses.
376 
377  Available to be implemented by subclasses only if they need the
378  coadd dataRef for performing preliminary processing before
379  assembling the coadd.
380  """
381  pass
382 
383  def getTempExpRefList(self, patchRef, calExpRefList):
384  """!
385  \brief Generate list data references corresponding to warped exposures that lie within the
386  patch to be coadded.
387 
388  \param[in] patchRef: Data reference for patch
389  \param[in] calExpRefList: List of data references for input calexps
390  \return List of Warp/CoaddTempExp data references
391  """
392  butler = patchRef.getButler()
393  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
394  self.getTempExpDatasetName(self.warpType))
395  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
396  g, groupData.keys) for
397  g in groupData.groups.keys()]
398  return tempExpRefList
399 
400  def prepareInputs(self, refList):
401  """!
402  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
403  for the photometric zero point.
404 
405  Each Warp has its own photometric zeropoint and background variance. Before coadding these
406  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
407  weight for each Warp.
408 
409  \param[in] refList: List of data references to tempExp
410  \return Struct:
411  - tempExprefList: List of data references to tempExp
412  - weightList: List of weightings
413  - imageScalerList: List of image scalers
414  """
415  statsCtrl = afwMath.StatisticsControl()
416  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
417  statsCtrl.setNumIter(self.config.clipIter)
418  statsCtrl.setAndMask(self.getBadPixelMask())
419  statsCtrl.setNanSafe(True)
420  # compute tempExpRefList: a list of tempExpRef that actually exist
421  # and weightList: a list of the weight of the associated coadd tempExp
422  # and imageScalerList: a list of scale factors for the associated coadd tempExp
423  tempExpRefList = []
424  weightList = []
425  imageScalerList = []
426  tempExpName = self.getTempExpDatasetName(self.warpType)
427  for tempExpRef in refList:
428  if not tempExpRef.datasetExists(tempExpName):
429  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
430  continue
431 
432  tempExp = tempExpRef.get(tempExpName, immediate=True)
433  maskedImage = tempExp.getMaskedImage()
434  imageScaler = self.scaleZeroPoint.computeImageScaler(
435  exposure=tempExp,
436  dataRef=tempExpRef,
437  )
438  try:
439  imageScaler.scaleMaskedImage(maskedImage)
440  except Exception as e:
441  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
442  continue
443  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
444  afwMath.MEANCLIP, statsCtrl)
445  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
446  weight = 1.0 / float(meanVar)
447  if not numpy.isfinite(weight):
448  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
449  continue
450  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
451 
452  del maskedImage
453  del tempExp
454 
455  tempExpRefList.append(tempExpRef)
456  weightList.append(weight)
457  imageScalerList.append(imageScaler)
458 
459  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
460  imageScalerList=imageScalerList)
461 
462  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
463  altMaskList=None, mask=None, supplementaryData=None):
464  """!
465  \anchor AssembleCoaddTask.assemble_
466 
467  \brief Assemble a coadd from input warps
468 
469  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
470  large area), the assembly is performed over small areas on the image at a time in order to
471  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
472  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
473  statistic specified. Set the edge bits the coadd mask based on the weight map.
474 
475  \param[in] skyInfo: Patch geometry information, from getSkyInfo
476  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
477  \param[in] imageScalerList: List of image scalers
478  \param[in] weightList: List of weights
479  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
480  \param[in] mask: Mask to ignore when coadding
481  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
482  Only used by subclasses that implement makeSupplementaryData and override assemble.
483  \return pipeBase.Struct with coaddExposure, nImage if requested
484  """
485  tempExpName = self.getTempExpDatasetName(self.warpType)
486  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
487  if mask is None:
488  mask = self.getBadPixelMask()
489 
490  statsCtrl = afwMath.StatisticsControl()
491  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
492  statsCtrl.setNumIter(self.config.clipIter)
493  statsCtrl.setAndMask(mask)
494  statsCtrl.setNanSafe(True)
495  statsCtrl.setWeighted(True)
496  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
497  for plane, threshold in self.config.maskPropagationThresholds.items():
498  bit = afwImage.Mask.getMaskPlane(plane)
499  statsCtrl.setMaskPropagationThreshold(bit, threshold)
500 
501  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
502 
503  if altMaskList is None:
504  altMaskList = [None]*len(tempExpRefList)
505 
506  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
507  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
508  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
509  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
510  coaddMaskedImage = coaddExposure.getMaskedImage()
511  subregionSizeArr = self.config.subregionSize
512  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
513  # if nImage is requested, create a zero one which can be passed to assembleSubregion
514  if self.config.doNImage:
515  nImage = afwImage.ImageU(skyInfo.bbox)
516  else:
517  nImage = None
518  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
519  try:
520  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
521  weightList, altMaskList, statsFlags, statsCtrl,
522  nImage=nImage)
523  except Exception as e:
524  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
525 
526  self.setInexactPsf(coaddMaskedImage.getMask())
527  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
528  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
529  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
530  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
531 
532  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
533  """!
534  \brief Set the metadata for the coadd
535 
536  This basic implementation simply sets the filter from the
537  first input.
538 
539  \param[in] coaddExposure: The target image for the coadd
540  \param[in] tempExpRefList: List of data references to tempExp
541  \param[in] weightList: List of weights
542  """
543  assert len(tempExpRefList) == len(weightList), "Length mismatch"
544  tempExpName = self.getTempExpDatasetName(self.warpType)
545  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
546  # (and we need more than just the PropertySet that contains the header), which is not possible
547  # with the current butler (see #2777).
548  tempExpList = [tempExpRef.get(tempExpName + "_sub",
549  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
550  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
551  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
552 
553  coaddExposure.setFilter(tempExpList[0].getFilter())
554  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
555  coaddInputs.ccds.reserve(numCcds)
556  coaddInputs.visits.reserve(len(tempExpList))
557 
558  for tempExp, weight in zip(tempExpList, weightList):
559  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
560  coaddInputs.visits.sort()
561  if self.warpType == "psfMatched":
562  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
563  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
564  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
565  # having the maximum width (sufficient because square)
566  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
567  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
568  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
569  else:
570  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
571  self.config.coaddPsf.makeControl())
572  coaddExposure.setPsf(psf)
573  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
574  coaddExposure.getWcs())
575  coaddExposure.getInfo().setApCorrMap(apCorrMap)
576 
577  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
578  altMaskList, statsFlags, statsCtrl, nImage=None):
579  """!
580  \brief Assemble the coadd for a sub-region.
581 
582  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
583  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
584  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
585  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
586  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
587  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
588 
589  \param[in] coaddExposure: The target image for the coadd
590  \param[in] bbox: Sub-region to coadd
591  \param[in] tempExpRefList: List of data reference to tempExp
592  \param[in] imageScalerList: List of image scalers
593  \param[in] weightList: List of weights
594  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
595  Each element is dict with keys = mask plane name to which to add the spans
596  \param[in] statsFlags: afwMath.Property object for statistic for coadd
597  \param[in] statsCtrl: Statistics control object for coadd
598  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
599  """
600  self.log.debug("Computing coadd over %s", bbox)
601  tempExpName = self.getTempExpDatasetName(self.warpType)
602  coaddExposure.mask.addMaskPlane("REJECTED")
603  coaddExposure.mask.addMaskPlane("CLIPPED")
604  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
605  # If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
606  # or CLIPPED, set it to REJECTED on the coadd.
607  # If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
608  # if a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
609  edge = afwImage.Mask.getPlaneBitMask("EDGE")
610  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
611  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
612  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
613  maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask("REJECTED")),
614  (edge, coaddExposure.mask.getPlaneBitMask("SENSOR_EDGE")),
615  (clipped, clipped)]
616  maskedImageList = []
617  if nImage is not None:
618  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
619  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
620  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
621  maskedImage = exposure.getMaskedImage()
622  mask = maskedImage.getMask()
623  if altMask is not None:
624  self.applyAltMaskPlanes(mask, altMask)
625  imageScaler.scaleMaskedImage(maskedImage)
626 
627  # Add 1 for each pixel which is not excluded by the exclude mask.
628  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
629  if nImage is not None:
630  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
631  if self.config.removeMaskPlanes:
632  mask = maskedImage.getMask()
633  for maskPlane in self.config.removeMaskPlanes:
634  try:
635  mask &= ~mask.getPlaneBitMask(maskPlane)
636  except Exception as e:
637  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
638 
639  maskedImageList.append(maskedImage)
640 
641  with self.timer("stack"):
642  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
643  clipped, # also set output to CLIPPED if sigma-clipped
644  maskMap)
645  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
646  if nImage is not None:
647  nImage.assign(subNImage, bbox)
648 
649  def applyAltMaskPlanes(self, mask, altMaskSpans):
650  """!
651  \brief Apply in place alt mask formatted as SpanSets to a mask
652 
653  @param mask: original mask
654  @param altMaskSpans: Dictionary containing spanSet lists to apply.
655  Each element contains the new mask plane name
656  (e.g. "CLIPPED and/or "NO_DATA") as the key,
657  and list of SpanSets to apply to the mask
658  """
659  for plane, spanSetList in altMaskSpans.items():
660  maskClipValue = mask.addMaskPlane(plane)
661  for spanSet in spanSetList:
662  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
663  return mask
664 
665  def readBrightObjectMasks(self, dataRef):
666  """Returns None on failure"""
667  try:
668  return dataRef.get("brightObjectMask", immediate=True)
669  except Exception as e:
670  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
671  return None
672 
673  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
674  """Set the bright object masks
675 
676  exposure: Exposure under consideration
677  dataId: Data identifier dict for patch
678  brightObjectMasks: afwTable of bright objects to mask
679  """
680  #
681  # Check the metadata specifying the tract/patch/filter
682  #
683  if brightObjectMasks is None:
684  self.log.warn("Unable to apply bright object mask: none supplied")
685  return
686  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
687  md = brightObjectMasks.table.getMetadata()
688  for k in dataId:
689  if not md.exists(k):
690  self.log.warn("Expected to see %s in metadata", k)
691  else:
692  if md.get(k) != dataId[k]:
693  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
694 
695  mask = exposure.getMaskedImage().getMask()
696  wcs = exposure.getWcs()
697  plateScale = wcs.pixelScale().asArcseconds()
698 
699  for rec in brightObjectMasks:
700  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
701  if rec["type"] == "box":
702  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
703  width = rec["width"].asArcseconds()/plateScale # convert to pixels
704  height = rec["height"].asArcseconds()/plateScale # convert to pixels
705 
706  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
707  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
708 
709  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
710  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
711  spans = afwGeom.SpanSet(bbox)
712  elif rec["type"] == "circle":
713  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
714  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
715  else:
716  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
717  continue
718  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
719 
720  def setInexactPsf(self, mask):
721  """Set INEXACT_PSF mask plane
722 
723  If any of the input images isn't represented in the coadd (due to
724  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
725  these pixels.
726 
727  Parameters
728  ----------
729  mask : `lsst.afw.image.Mask`
730  Coadded exposure's mask, modified in-place.
731  """
732  mask.addMaskPlane("INEXACT_PSF")
733  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
734  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
735  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
736  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
737  array = mask.getArray()
738  selected = array & (sensorEdge | clipped | rejected) > 0
739  array[selected] |= inexactPsf
740 
741  @classmethod
742  def _makeArgumentParser(cls):
743  """!
744  \brief Create an argument parser
745  """
746  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
747  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
748  cls.ConfigClass().warpType + "Warp",
749  help="data ID, e.g. --id tract=12345 patch=1,2",
750  ContainerClass=AssembleCoaddDataIdContainer)
751  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
752  ContainerClass=SelectDataIdContainer)
753  return parser
754 
755 
756 def _subBBoxIter(bbox, subregionSize):
757  """!
758  \brief Iterate over subregions of a bbox
759 
760  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
761  \param[in] subregionSize: size of sub-bboxes
762 
763  \return subBBox: next sub-bounding box of size subregionSize or smaller;
764  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
765  but it will never be empty
766  """
767  if bbox.isEmpty():
768  raise RuntimeError("bbox %s is empty" % (bbox,))
769  if subregionSize[0] < 1 or subregionSize[1] < 1:
770  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
771 
772  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
773  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
774  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
775  subBBox.clip(bbox)
776  if subBBox.isEmpty():
777  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
778  (bbox, subregionSize, colShift, rowShift))
779  yield subBBox
780 
781 
782 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
783  """!
784  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
785  """
786 
787  def makeDataRefList(self, namespace):
788  """!
789  \brief Make self.refList from self.idList.
790  """
791  datasetType = namespace.config.coaddName + "Coadd"
792  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
793 
794  for dataId in self.idList:
795  # tract and patch are required
796  for key in keysCoadd:
797  if key not in dataId:
798  raise RuntimeError("--id must include " + key)
799 
800  dataRef = namespace.butler.dataRef(
801  datasetType=datasetType,
802  dataId=dataId,
803  )
804  self.refList.append(dataRef)
805 
806 
807 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
808  """!
809  \brief Function to count the number of pixels with a specific mask in a footprint.
810 
811  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
812  have bitmask set but do not have ignoreMask set. Return the count.
813 
814  \param[in] mask: mask to define intersection region by.
815  \parma[in] footprint: footprint to define the intersection region by.
816  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
817  \param[in] ignoreMask: pixels to not consider.
818  \return count of number of pixels in footprint with specified mask.
819  """
820  bbox = footprint.getBBox()
821  bbox.clip(mask.getBBox(afwImage.PARENT))
822  fp = afwImage.Mask(bbox)
823  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
824  footprint.spans.setMask(fp, bitmask)
825  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
826  (subMask.getArray() & ignoreMask) == 0).sum()
827 
828 
830  """!
831 \anchor SafeClipAssembleCoaddConfig
832 
833 \brief Configuration parameters for the SafeClipAssembleCoaddTask
834  """
835  clipDetection = pexConfig.ConfigurableField(
836  target=SourceDetectionTask,
837  doc="Detect sources on difference between unclipped and clipped coadd")
838  minClipFootOverlap = pexConfig.Field(
839  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
840  dtype=float,
841  default=0.6
842  )
843  minClipFootOverlapSingle = pexConfig.Field(
844  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
845  "clipped when only one visit overlaps",
846  dtype=float,
847  default=0.5
848  )
849  minClipFootOverlapDouble = pexConfig.Field(
850  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
851  "clipped when two visits overlap",
852  dtype=float,
853  default=0.45
854  )
855  maxClipFootOverlapDouble = pexConfig.Field(
856  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
857  "considering two visits",
858  dtype=float,
859  default=0.15
860  )
861  minBigOverlap = pexConfig.Field(
862  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
863  "when labeling clipped footprints",
864  dtype=int,
865  default=100
866  )
867 
868  def setDefaults(self):
869  # The numeric values for these configuration parameters were empirically determined, future work
870  # may further refine them.
871  AssembleCoaddConfig.setDefaults(self)
872  self.clipDetection.doTempLocalBackground = False
873  self.clipDetection.reEstimateBackground = False
874  self.clipDetection.returnOriginalFootprints = False
875  self.clipDetection.thresholdPolarity = "both"
876  self.clipDetection.thresholdValue = 2
877  self.clipDetection.nSigmaToGrow = 2
878  self.clipDetection.minPixels = 4
879  self.clipDetection.isotropicGrow = True
880  self.clipDetection.thresholdType = "pixel_stdev"
881  self.sigmaClip = 1.5
882  self.clipIter = 3
883  self.statistic = "MEAN"
884 
885  def validate(self):
886  if self.doSigmaClip:
887  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
888  "Ignoring doSigmaClip.")
889  self.doSigmaClip = False
890  if self.statistic != "MEAN":
891  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
892  "(%s chosen). Please set statistic to MEAN."
893  % (self.statistic))
894  AssembleCoaddTask.ConfigClass.validate(self)
895 
896 
897 
903 
904 
906  """!
907  \anchor SafeClipAssembleCoaddTask_
908 
909  \brief Assemble a coadded image from a set of coadded temporary exposures,
910  being careful to clip & flag areas with potential artifacts.
911 
912  \section pipe_tasks_assembleCoadd_Contents Contents
913  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
914  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
915  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
916  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
917  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
918  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
919 
920  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
921 
922  \copybrief SafeClipAssembleCoaddTask
923 
924  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
925  SafeClipAssembleCoaddTask subtasks that task.
926  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
927  outliers).
928  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
929  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
930  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
931  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
932  that there is an outlier and ii. this outlier appears on only one or two images.
933  Such regions will not contribute to the final coadd.
934  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
935  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
936  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
937  surveys.
938 
939  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
940  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
941  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
942 
943  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
944  \copydoc \_\_init\_\_
945 
946  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
947  \copydoc run
948 
949  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
950  See \ref SafeClipAssembleCoaddConfig
951 
952  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
953  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
954  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
955  files.
956  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
957  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
958  for further information.
959 
960  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
961  SafeClipAssembleCoaddTask
962 
963  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
964  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
965  '--legacyCoadd'.
966  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
967  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
968  with a list of coaddTempExps to attempt to coadd (specified using
969  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
970  Only the coaddTempExps that cover the specified tract and patch will be coadded.
971  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
972  command line argument:
973  \code
974  assembleCoadd.py --help
975  \endcode
976  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
977  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
978  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
979  packages.
980  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
981  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
982  <DL>
983  <DT>processCcd</DT>
984  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
985  <DT>makeSkyMap</DT>
986  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
987  <DT>makeCoaddTempExp</DT>
988  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
989  </DL>
990  We can perform all of these steps by running
991  \code
992  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
993  \endcode
994  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
995  as follows:
996  \code
997  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
998  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
999  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1000  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1001  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1002  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1003  --selectId visit=903988 ccd=24
1004  \endcode
1005  This will process the HSC-I band data. The results are written in
1006  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1007 
1008  You may also choose to run:
1009  \code
1010  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1011  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1012  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1013  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1014  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1015  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1016  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1017  --selectId visit=903346 ccd=12
1018  \endcode
1019  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1020  discussed in \ref pipeTasks_multiBand.
1021  """
1022  ConfigClass = SafeClipAssembleCoaddConfig
1023  _DefaultName = "safeClipAssembleCoadd"
1024 
1025  def __init__(self, *args, **kwargs):
1026  """!
1027  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1028  """
1029  AssembleCoaddTask.__init__(self, *args, **kwargs)
1030  schema = afwTable.SourceTable.makeMinimalSchema()
1031  self.makeSubtask("clipDetection", schema=schema)
1032 
1033  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1034  """!
1035  \brief Assemble the coadd for a region
1036 
1037  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1038  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1039  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1040  overlap between the clipped region and a source.
1041  This leaves us with a set of footprints from the difference image that have been identified as having
1042  occured on just one or two individual visits. However, these footprints were generated from a
1043  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1044  footprints acrosss the coadd difference in this process.
1045  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1046  these are big footprints.
1047  Combine the small and big clipped footprints and mark them on a new bad mask plane
1048  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1049  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1050  bad mask plane.
1051 
1052  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1053  parent task.
1054 
1055  @param skyInfo: Patch geometry information, from getSkyInfo
1056  @param tempExpRefList: List of data reference to tempExp
1057  @param imageScalerList: List of image scalers
1058  @param weightList: List of weights
1059  return pipeBase.Struct with coaddExposure, nImage
1060  """
1061  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1062  mask = exp.getMaskedImage().getMask()
1063  mask.addMaskPlane("CLIPPED")
1064 
1065  result = self.detectClip(exp, tempExpRefList)
1066 
1067  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1068 
1069  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1070  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1071  # Append big footprints from individual Warps to result.clipSpans
1072  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1073  result.detectionFootprints, maskClipValue, maskDetValue,
1074  exp.getBBox())
1075  # Create mask of the current clipped footprints
1076  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1077  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1078 
1079  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1080  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1081  maskClip |= maskClipBig
1082 
1083  # Assemble coadd from base class, but ignoring CLIPPED pixels
1084  badMaskPlanes = self.config.badMaskPlanes[:]
1085  badMaskPlanes.append("CLIPPED")
1086  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1087  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1088  result.clipSpans, mask=badPixelMask)
1089 
1090  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1091  """!
1092  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1093 
1094  Generate a difference image between clipped and unclipped coadds.
1095  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1096  Return the difference image.
1097 
1098  @param skyInfo: Patch geometry information, from getSkyInfo
1099  @param tempExpRefList: List of data reference to tempExp
1100  @param imageScalerList: List of image scalers
1101  @param weightList: List of weights
1102  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1103  """
1104  # Clone and upcast self.config because current self.config is frozen
1105  config = AssembleCoaddConfig()
1106  # getattr necessary because subtasks do not survive Config.toDict()
1107  configIntersection = {k: getattr(self.config, k)
1108  for k, v in self.config.toDict().items() if (k in config.keys())}
1109  config.update(**configIntersection)
1110 
1111  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1112  config.statistic = 'MEAN'
1113  task = AssembleCoaddTask(config=config)
1114  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1115 
1116  config.statistic = 'MEANCLIP'
1117  task = AssembleCoaddTask(config=config)
1118  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1119 
1120  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1121  coaddDiff -= coaddClip.getMaskedImage()
1122  exp = afwImage.ExposureF(coaddDiff)
1123  exp.setPsf(coaddMean.getPsf())
1124  return exp
1125 
1126  def detectClip(self, exp, tempExpRefList):
1127  """!
1128  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1129 
1130  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1131  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1132  fraction to thresholds set in the config.
1133  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1134  two).
1135  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1136  the coaddTempExp.
1137  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1138  with the clipped footprints and a list of new masks for the coaddTempExps.
1139 
1140  \param[in] exp: Exposure to run detection on
1141  \param[in] tempExpRefList: List of data reference to tempExp
1142  \return struct containing:
1143  - clipFootprints: list of clipped footprints
1144  - clipIndices: indices for each clippedFootprint in tempExpRefList
1145  - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new
1146  maskplane name ("CLIPPED")" as the key and list of SpanSets as value
1147  - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints
1148  """
1149  mask = exp.getMaskedImage().getMask()
1150  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1151  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1152  # Merge positive and negative together footprints together
1153  fpSet.positive.merge(fpSet.negative)
1154  footprints = fpSet.positive
1155  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1156  ignoreMask = self.getBadPixelMask()
1157 
1158  clipFootprints = []
1159  clipIndices = []
1160  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1161 
1162  # for use by detectClipBig
1163  visitDetectionFootprints = []
1164 
1165  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1166  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1167  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1168 
1169  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1170  for i, warpRef in enumerate(tempExpRefList):
1171  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1172  immediate=True).getMaskedImage().getMask()
1173  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1174  afwImage.PARENT, True)
1175  maskVisitDet &= maskDetValue
1176  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1177  visitDetectionFootprints.append(visitFootprints)
1178 
1179  for j, footprint in enumerate(footprints.getFootprints()):
1180  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1181  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1182 
1183  # build a list of clipped spans for each visit
1184  for j, footprint in enumerate(footprints.getFootprints()):
1185  nPixel = footprint.getArea()
1186  overlap = [] # hold the overlap with each visit
1187  indexList = [] # index of visit in global list
1188  for i in range(len(tempExpRefList)):
1189  ignore = ignoreArr[i, j]
1190  overlapDet = overlapDetArr[i, j]
1191  totPixel = nPixel - ignore
1192 
1193  # If we have more bad pixels than detection skip
1194  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1195  continue
1196  overlap.append(overlapDet/float(totPixel))
1197  indexList.append(i)
1198 
1199  overlap = numpy.array(overlap)
1200  if not len(overlap):
1201  continue
1202 
1203  keep = False # Should this footprint be marked as clipped?
1204  keepIndex = [] # Which tempExps does the clipped footprint belong to
1205 
1206  # If footprint only has one overlap use a lower threshold
1207  if len(overlap) == 1:
1208  if overlap[0] > self.config.minClipFootOverlapSingle:
1209  keep = True
1210  keepIndex = [0]
1211  else:
1212  # This is the general case where only visit should be clipped
1213  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1214  if len(clipIndex) == 1:
1215  keep = True
1216  keepIndex = [clipIndex[0]]
1217 
1218  # Test if there are clipped objects that overlap two different visits
1219  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1220  if len(clipIndex) == 2 and len(overlap) > 3:
1221  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1222  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1223  keep = True
1224  keepIndex = clipIndex
1225 
1226  if not keep:
1227  continue
1228 
1229  for index in keepIndex:
1230  globalIndex = indexList[index]
1231  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1232 
1233  clipIndices.append(numpy.array(indexList)[keepIndex])
1234  clipFootprints.append(footprint)
1235 
1236  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1237  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1238 
1239  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1240  maskClipValue, maskDetValue, coaddBBox):
1241  """!
1242  \brief Return individual warp footprints for large artifacts and append them to clipList in place
1243 
1244  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1245  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1246  significantly with each source in all the coaddTempExps.
1247  \param[in] clipList: List of alt mask SpanSets with clipping information. Modified.
1248  \param[in] clipFootprints: List of clipped footprints
1249  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1250  \param[in] maskClipValue: Mask value of clipped pixels
1251  \param[in] maskDetValue: Mask value of detected pixels
1252  \param[in] coaddBBox: BBox of the coadd and warps
1253  \return list of big footprints
1254  """
1255  bigFootprintsCoadd = []
1256  ignoreMask = self.getBadPixelMask()
1257  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1258  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1259  for footprint in visitFootprints.getFootprints():
1260  footprint.spans.setMask(maskVisitDet, maskDetValue)
1261 
1262  # build a mask of clipped footprints that are in this visit
1263  clippedFootprintsVisit = []
1264  for foot, clipIndex in zip(clipFootprints, clipIndices):
1265  if index not in clipIndex:
1266  continue
1267  clippedFootprintsVisit.append(foot)
1268  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1269  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1270 
1271  bigFootprintsVisit = []
1272  for foot in visitFootprints.getFootprints():
1273  if foot.getArea() < self.config.minBigOverlap:
1274  continue
1275  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1276  if nCount > self.config.minBigOverlap:
1277  bigFootprintsVisit.append(foot)
1278  bigFootprintsCoadd.append(foot)
1279 
1280  for footprint in bigFootprintsVisit:
1281  clippedSpans["CLIPPED"].append(footprint.spans)
1282 
1283  return bigFootprintsCoadd
1284 
1285 
1287  assembleStaticSkyModel = pexConfig.ConfigurableField(
1288  target=AssembleCoaddTask,
1289  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1290  " naive/first-iteration model of the static sky.",
1291  )
1292  detect = pexConfig.ConfigurableField(
1293  target=SourceDetectionTask,
1294  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1295  )
1296  maxNumEpochs = pexConfig.Field(
1297  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1298  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1299  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1300  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1301  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1302  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1303  "than transient and not masked.",
1304  dtype=int,
1305  default=2
1306  )
1307  maxFractionEpochsLow = pexConfig.RangeField(
1308  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1309  "Effective maxNumEpochs = "
1310  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1311  dtype=float,
1312  default=0.4,
1313  min=0., max=1.,
1314  )
1315  maxFractionEpochsHigh = pexConfig.RangeField(
1316  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1317  "Effective maxNumEpochs = "
1318  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1319  dtype=float,
1320  default=0.03,
1321  min=0., max=1.,
1322  )
1323  spatialThreshold = pexConfig.RangeField(
1324  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1325  "temporal criteria. If 0, clip all. If 1, clip none.",
1326  dtype=float,
1327  default=0.5,
1328  min=0., max=1.,
1329  inclusiveMin=True, inclusiveMax=True
1330  )
1331  doScaleWarpVariance = pexConfig.Field(
1332  doc="Rescale Warp variance plane using empirical noise?",
1333  dtype=bool,
1334  default=True,
1335  )
1336  maskScaleWarpVariance = pexConfig.ListField(
1337  dtype=str,
1338  default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
1339  doc="Mask planes for pixels to ignore when rescaling warp variance",
1340  )
1341 
1342  def setDefaults(self):
1343  AssembleCoaddConfig.setDefaults(self)
1344  self.statistic = 'MEAN'
1345  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1346  self.assembleStaticSkyModel.warpType = 'psfMatched'
1347  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1348  self.assembleStaticSkyModel.sigmaClip = 1.5
1349  self.assembleStaticSkyModel.clipIter = 3
1350  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1351  self.assembleStaticSkyModel.doWrite = False
1352  self.detect.doTempLocalBackground = False
1353  self.detect.reEstimateBackground = False
1354  self.detect.returnOriginalFootprints = False
1355  self.detect.thresholdPolarity = "both"
1356  self.detect.thresholdValue = 5
1357  self.detect.nSigmaToGrow = 2
1358  self.detect.minPixels = 4
1359  self.detect.isotropicGrow = True
1360  self.detect.thresholdType = "pixel_stdev"
1361 
1362 
1363 
1369 
1371  """!
1372  \anchor CompareWarpAssembleCoaddTask_
1373 
1374  \brief Assemble a compareWarp coadded image from a set of warps
1375  by masking artifacts detected by comparing PSF-matched warps
1376 
1377  \section pipe_tasks_assembleCoadd_Contents Contents
1378  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1379  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1380  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1381  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1382  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1383  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1384 
1385  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1386 
1387  \copybrief CompareWarpAssembleCoaddTask
1388 
1389  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1390  outliers).
1391  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1392  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1393  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1394  pixels in the individual warps suspected to contain an artifact.
1395  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1396  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1397  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1398  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1399  such as bright stars.
1400  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1401  The temporalThreshold is the maximum fraction of epochs that the deviation can
1402  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1403  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1404  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1405  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1406  These regions will not contribute to the final coadd.
1407  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1408  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1409  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1410  surveys.
1411 
1412  CompareWarpAssembleCoaddTask sub-classes
1413  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1414  as a subtask to generate the TemplateCoadd (the model of the static sky)
1415 
1416  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1417  \copydoc \_\_init\_\_
1418 
1419  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1420  \copydoc run
1421 
1422  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1423  See \ref CompareWarpAssembleCoaddConfig
1424 
1425  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1426  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1427  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1428  files.
1429 
1430  This task supports the following debug variables:
1431  <dl>
1432  <dt>`saveCountIm`
1433  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1434  <dt> `figPath`
1435  <dd> Path to save the debug fits images and figures
1436  </dl>
1437 
1438  For example, put something like:
1439  @code{.py}
1440  import lsstDebug
1441  def DebugInfo(name):
1442  di = lsstDebug.getInfo(name)
1443  if name == "lsst.pipe.tasks.assembleCoadd":
1444  di.saveCountIm = True
1445  di.figPath = "/desired/path/to/debugging/output/images"
1446  return di
1447  lsstDebug.Info = DebugInfo
1448  @endcode
1449  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1450  flag.
1451  Some subtasks may have their own debug variables; see individual Task
1452  documentation
1453 
1454  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1455  CompareWarpAssembleCoaddTask
1456 
1457  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1458  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1459  '--compareWarpCoadd'.
1460  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1461  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1462  with a list of coaddTempExps to attempt to coadd (specified using
1463  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1464  Only the warps that cover the specified tract and patch will be coadded.
1465  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1466  command line argument:
1467  \code
1468  assembleCoadd.py --help
1469  \endcode
1470  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1471  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1472  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1473  packages.
1474  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1475  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1476  <DL>
1477  <DT>processCcd</DT>
1478  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1479  <DT>makeSkyMap</DT>
1480  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1481  <DT>makeCoaddTempExp</DT>
1482  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1483  </DL>
1484  We can perform all of these steps by running
1485  \code
1486  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1487  \endcode
1488  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1489  as follows:
1490  \code
1491  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1492  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1493  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1494  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1495  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1496  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1497  --selectId visit=903988 ccd=24
1498  \endcode
1499  This will process the HSC-I band data. The results are written in
1500  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1501  """
1502  ConfigClass = CompareWarpAssembleCoaddConfig
1503  _DefaultName = "compareWarpAssembleCoadd"
1504 
1505  def __init__(self, *args, **kwargs):
1506  """!
1507  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1508  """
1509  AssembleCoaddTask.__init__(self, *args, **kwargs)
1510  self.makeSubtask("assembleStaticSkyModel")
1511  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1512  self.makeSubtask("detect", schema=detectionSchema)
1513 
1514  def makeSupplementaryData(self, dataRef, selectDataList):
1515  """!
1516  \brief Make inputs specific to Subclass
1517 
1518  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1519  """
1520  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1521 
1522  if templateCoadd is None:
1523  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1524  self.assembleStaticSkyModel.warpType[1:])
1525  message = """No %(warpName)s warps were found to build the template coadd which is
1526  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1527  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1528  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1529 
1530  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1531  another algorithm like:
1532 
1533  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1534  config.assemble.retarget(SafeClipAssembleCoaddTask)
1535  """ % {"warpName": warpName}
1536  raise RuntimeError(message)
1537 
1538  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1539 
1540  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1541  supplementaryData, *args, **kwargs):
1542  """!
1543  \brief Assemble the coadd
1544 
1545  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1546  as the model of the static sky.
1547 
1548  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1549  new "CLIPPED" plane and updated "NO_DATA" plane.
1550  Then pass these alternative masks to the base class's assemble method.
1551 
1552  @param skyInfo: Patch geometry information
1553  @param tempExpRefList: List of data references to warps
1554  @param imageScalerList: List of image scalers
1555  @param weightList: List of weights
1556  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1557 
1558  return pipeBase.Struct with coaddExposure, nImage if requested
1559  """
1560  templateCoadd = supplementaryData.templateCoadd
1561  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1562  badMaskPlanes = self.config.badMaskPlanes[:]
1563  badMaskPlanes.append("CLIPPED")
1564  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1565 
1566  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1567  spanSetMaskList, mask=badPixelMask)
1568 
1569  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1570  """!
1571  \brief Find artifacts
1572 
1573  Loop through warps twice. The first loop builds a map with the count of how many
1574  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1575  The second loop takes each difference image and filters the artifacts detected
1576  in each using count map to filter out variable sources and sources that are difficult to
1577  subtract cleanly.
1578 
1579  @param templateCoadd: Exposure to serve as model of static sky
1580  @param tempExpRefList: List of data references to warps
1581  @param imageScalerList: List of image scalers
1582  """
1583 
1584  self.log.debug("Generating Count Image, and mask lists.")
1585  coaddBBox = templateCoadd.getBBox()
1586  slateIm = afwImage.ImageU(coaddBBox)
1587  epochCountImage = afwImage.ImageU(coaddBBox)
1588  nImage = afwImage.ImageU(coaddBBox)
1589  spanSetArtifactList = []
1590  spanSetNoDataMaskList = []
1591 
1592  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1593  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1594  if warpDiffExp is not None:
1595  nImage.array += numpy.where(numpy.isnan(warpDiffExp.image.array),
1596  0, 1).astype(numpy.uint16)
1597  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1598  fpSet.positive.merge(fpSet.negative)
1599  footprints = fpSet.positive
1600  slateIm.set(0)
1601  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1602  for spans in spanSetList:
1603  spans.setImage(slateIm, 1, doClip=True)
1604  epochCountImage += slateIm
1605 
1606  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1607  # undergo a second convolution. Pixels with data in the direct warp
1608  # but not in the PSF-matched warp will not have their artifacts detected.
1609  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1610  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1611  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1612  nansMask.setXY0(warpDiffExp.getXY0())
1613  else:
1614  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1615  # In this case, mask the whole epoch
1616  nansMask = afwImage.MaskX(coaddBBox, 1)
1617  spanSetList = []
1618 
1619  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1620 
1621  spanSetNoDataMaskList.append(spanSetNoDataMask)
1622  spanSetArtifactList.append(spanSetList)
1623 
1624  if lsstDebug.Info(__name__).saveCountIm:
1625  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1626  epochCountImage.writeFits(path)
1627 
1628  for i, spanSetList in enumerate(spanSetArtifactList):
1629  if spanSetList:
1630  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage, nImage)
1631  spanSetArtifactList[i] = filteredSpanSetList
1632 
1633  altMasks = []
1634  for artifacts, noData in zip(spanSetArtifactList, spanSetNoDataMaskList):
1635  altMasks.append({'CLIPPED': artifacts,
1636  'NO_DATA': noData})
1637  return altMasks
1638 
1639  def _filterArtifacts(self, spanSetList, epochCountImage, nImage):
1640  """!
1641  \brief Filter artifact candidates
1642 
1643  @param spanSetList: List of SpanSets representing artifact candidates
1644  @param epochCountImage: Image of accumulated number of warpDiff detections
1645  @param nImage: Image of the accumulated number of total epochs contributing
1646 
1647  return List of SpanSets with artifacts
1648  """
1649 
1650  maskSpanSetList = []
1651  x0, y0 = epochCountImage.getXY0()
1652  for i, span in enumerate(spanSetList):
1653  y, x = span.indices()
1654  yIdxLocal = [y1 - y0 for y1 in y]
1655  xIdxLocal = [x1 - x0 for x1 in x]
1656  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1657  totalN = nImage.array[yIdxLocal, xIdxLocal]
1658 
1659  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1660  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1661  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1662  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1663  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1664  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1665  (outlierN <= effectiveMaxNumEpochs))
1666  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1667  if percentBelowThreshold > self.config.spatialThreshold:
1668  maskSpanSetList.append(span)
1669  return maskSpanSetList
1670 
1671  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1672  """!
1673  \brief Fetch a warp from the butler and return a warpDiff
1674 
1675  @param warpRef: `Butler dataRef` for the warp
1676  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1677  @param templateCoadd: Exposure to be substracted from the scaled warp
1678 
1679  return Exposure of the image difference between the warp and template
1680  """
1681 
1682  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1683  warpName = self.getTempExpDatasetName('psfMatched')
1684  if not warpRef.datasetExists(warpName):
1685  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1686  return None
1687  warp = warpRef.get(warpName, immediate=True)
1688  # direct image scaler OK for PSF-matched Warp
1689  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1690  mi = warp.getMaskedImage()
1691  if self.config.doScaleWarpVariance:
1692  scaleVariance(mi, self.config.maskScaleWarpVariance,
1693  log=self.log)
1694  mi -= templateCoadd.getMaskedImage()
1695  return warp
1696 
1697  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1698  """!
1699  \brief Return a path to which to write debugging output
1700 
1701  @param prefix: string, prefix for filename
1702  @param warpRef: Butler dataRef
1703  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1704  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1705 
1706  Creates a hyphen-delimited string of dataId values for simple filenames.
1707  """
1708  if coaddLevel:
1709  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1710  else:
1711  keys = warpRef.dataId.keys()
1712  keyList = sorted(keys, reverse=True)
1713  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1714  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1715  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def _filterArtifacts(self, spanSetList, epochCountImage, nImage)
Filter artifact candidates.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...