lsst.pipe.tasks  14.0-40-g03723daf
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import os
26 import numpy
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, scaleVariance
40 from .interpImage import InterpImageTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms import SourceDetectionTask
44 
45 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
46 
47 
48 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
49  """!
50 \anchor AssembleCoaddConfig_
51 
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53  """
54  warpType = pexConfig.Field(
55  doc="Warp name: one of 'direct' or 'psfMatched'",
56  dtype=str,
57  default="direct",
58  )
59  subregionSize = pexConfig.ListField(
60  dtype=int,
61  doc="Width, height of stack subregion size; "
62  "make small enough that a full stack of images will fit into memory at once.",
63  length=2,
64  default=(2000, 2000),
65  )
66  statistic = pexConfig.Field(
67  dtype=str,
68  doc="Main stacking statistic for aggregating over the epochs.",
69  default="MEANCLIP",
70  )
71  doSigmaClip = pexConfig.Field(
72  dtype=bool,
73  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
74  default=False,
75  )
76  sigmaClip = pexConfig.Field(
77  dtype=float,
78  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
79  default=3.0,
80  )
81  clipIter = pexConfig.Field(
82  dtype=int,
83  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
84  default=2,
85  )
86  calcErrorFromInputVariance = pexConfig.Field(
87  dtype=bool,
88  doc="Calculate coadd variance from input variance by stacking statistic."
89  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
90  default=True,
91  )
92  scaleZeroPoint = pexConfig.ConfigurableField(
93  target=ScaleZeroPointTask,
94  doc="Task to adjust the photometric zero point of the coadd temp exposures",
95  )
96  doInterp = pexConfig.Field(
97  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
98  dtype=bool,
99  default=True,
100  )
101  interpImage = pexConfig.ConfigurableField(
102  target=InterpImageTask,
103  doc="Task to interpolate (and extrapolate) over NaN pixels",
104  )
105  doWrite = pexConfig.Field(
106  doc="Persist coadd?",
107  dtype=bool,
108  default=True,
109  )
110  doNImage = pexConfig.Field(
111  doc="Create image of number of contributing exposures for each pixel",
112  dtype=bool,
113  default=False,
114  )
115  maskPropagationThresholds = pexConfig.DictField(
116  keytype=str,
117  itemtype=float,
118  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
119  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
120  "would have contributed exceeds this value."),
121  default={"SAT": 0.1},
122  )
123  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
124  doc="Mask planes to remove before coadding")
125  #
126  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
127  # To make this useful you *must* also configure the flags.pixel algorithm, for example
128  # by adding
129  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
130  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
131  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
132  #
133  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
134  doc="Set mask and flag bits for bright objects?")
135  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
136  doc="Name of mask bit used for bright objects")
137 
138  coaddPsf = pexConfig.ConfigField(
139  doc="Configuration for CoaddPsf",
140  dtype=measAlg.CoaddPsfConfig,
141  )
142 
143  def setDefaults(self):
144  CoaddBaseTask.ConfigClass.setDefaults(self)
145  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
146 
147  def validate(self):
148  CoaddBaseTask.ConfigClass.validate(self)
149  if self.doPsfMatch:
150  # Backwards compatibility.
151  # Configs do not have loggers
152  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
153  self.warpType = 'psfMatched'
154  if self.doSigmaClip and self.statistic != "MEANCLIP":
155  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
156  self.statistic = "MEANCLIP"
157  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
158  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
159  "compute and set a non-zero coadd variance estimate." % (self.statistic))
160 
161  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
162  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
163  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
164  if str(k) not in unstackableStats]
165  raise ValueError("statistic %s is not allowed. Please choose one of %s."
166  % (self.statistic, stackableStats))
167 
168 
169 
176  """!
177 \anchor AssembleCoaddTask_
178 
179 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
180 
181 \section pipe_tasks_assembleCoadd_Contents Contents
182  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
183  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
184  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
185  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
186  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
187  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
188 
189 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
190 
191 \copybrief AssembleCoaddTask_
192 
193 We want to assemble a coadded image from a set of Warps (also called
194 coadded temporary exposures or coaddTempExps.
195 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
196 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
197 Warps that cover the specified patch (pointed at by dataRef).
198 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
199 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
200 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
201 config.makePsfMatched set which of the warp types will be coadded.
202 The coadd is computed as a mean with optional outlier rejection.
203 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
204 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
205 
206 AssembleCoaddTask uses several sub-tasks. These are
207 <DL>
208  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
209  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
210  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
211  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
212 </DL>
213 You can retarget these subtasks if you wish.
214 
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
216 \copydoc \_\_init\_\_
217 
218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
219 \copydoc run
220 
221 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
222 See \ref AssembleCoaddConfig_
223 
224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
225 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
226 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
227 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
228 the documetation for the subtasks for further information.
229 
230 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
231 
232 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
233 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
234 a data reference to the tract patch and filter to be coadded (specified using
235 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
236 Warps to attempt to coadd (specified using
237 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
238 that cover the specified tract and patch will be coadded. A list of the available optional
239 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
240 \code
241 assembleCoadd.py --help
242 \endcode
243 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
244 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
245 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
246 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
247 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
248 <DL>
249  <DT>processCcd</DT>
250  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
251  <DT>makeSkyMap</DT>
252  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
253  <DT>makeCoaddTempExp</DT>
254  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
255 </DL>
256 We can perform all of these steps by running
257 \code
258 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
259 \endcode
260 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
261 follows:
262 \code
263 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
264 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
265 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
266 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
267 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
268 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
269 --selectId visit=903988 ccd=24
270 \endcode
271 that will process the HSC-I band data. The results are written in
272 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
273 
274 You may also choose to run:
275 \code
276 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
277 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
278 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
279 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
280 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
281 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
282 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
283 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
284 \endcode
285 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
286 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
287 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
288  """
289  ConfigClass = AssembleCoaddConfig
290  _DefaultName = "assembleCoadd"
291 
292  def __init__(self, *args, **kwargs):
293  """!
294  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
295  & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
296  """
297  CoaddBaseTask.__init__(self, *args, **kwargs)
298  self.makeSubtask("interpImage")
299  self.makeSubtask("scaleZeroPoint")
300 
301  if self.config.doMaskBrightObjects:
302  mask = afwImage.Mask()
303  try:
304  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
305  except pexExceptions.LsstCppException:
306  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
307  mask.getMaskPlaneDict().keys())
308  del mask
309 
310  self.warpType = self.config.warpType
311 
312  @pipeBase.timeMethod
313  def run(self, dataRef, selectDataList=[]):
314  """!
315  \brief Assemble a coadd from a set of Warps
316 
317  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
318  match the photometric zeropoint to a reference Warp. Assemble the Warps using
319  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
320  exposure.
321 
322  \anchor runParams
323  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
324  (if config.autoReference=False). Used to access the following data products:
325  - [in] self.config.coaddName + "Coadd_skyMap"
326  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
327  - [out] self.config.coaddName + "Coadd"
328  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
329  selected from this list based on overlap with the patch defined by dataRef.
330 
331  \return a pipeBase.Struct with fields:
332  - coaddExposure: coadded exposure
333  - nImage: exposure count image
334  """
335  skyInfo = self.getSkyInfo(dataRef)
336  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
337  if len(calExpRefList) == 0:
338  self.log.warn("No exposures to coadd")
339  return
340  self.log.info("Coadding %d exposures", len(calExpRefList))
341 
342  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
343  inputData = self.prepareInputs(tempExpRefList)
344  self.log.info("Found %d %s", len(inputData.tempExpRefList),
345  self.getTempExpDatasetName(self.warpType))
346  if len(inputData.tempExpRefList) == 0:
347  self.log.warn("No coadd temporary exposures found")
348  return
349 
350  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
351 
352  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
353  inputData.weightList, supplementaryData=supplementaryData)
354 
355  if self.config.doInterp:
356  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
357  # The variance must be positive; work around for DM-3201.
358  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
359  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
360 
361  if self.config.doMaskBrightObjects:
362  brightObjectMasks = self.readBrightObjectMasks(dataRef)
363  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
364 
365  if self.config.doWrite:
366  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
367  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
368  if self.config.doNImage and retStruct.nImage is not None:
369  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
370 
371  return retStruct
372 
373  def makeSupplementaryData(self, dataRef, selectDataList):
374  """!
375  \brief Make additional inputs to assemble() specific to subclasses.
376 
377  Available to be implemented by subclasses only if they need the
378  coadd dataRef for performing preliminary processing before
379  assembling the coadd.
380  """
381  pass
382 
383  def getTempExpRefList(self, patchRef, calExpRefList):
384  """!
385  \brief Generate list data references corresponding to warped exposures that lie within the
386  patch to be coadded.
387 
388  \param[in] patchRef: Data reference for patch
389  \param[in] calExpRefList: List of data references for input calexps
390  \return List of Warp/CoaddTempExp data references
391  """
392  butler = patchRef.getButler()
393  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
394  self.getTempExpDatasetName(self.warpType))
395  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
396  g, groupData.keys) for
397  g in groupData.groups.keys()]
398  return tempExpRefList
399 
400  def prepareInputs(self, refList):
401  """!
402  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
403  for the photometric zero point.
404 
405  Each Warp has its own photometric zeropoint and background variance. Before coadding these
406  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
407  weight for each Warp.
408 
409  \param[in] refList: List of data references to tempExp
410  \return Struct:
411  - tempExprefList: List of data references to tempExp
412  - weightList: List of weightings
413  - imageScalerList: List of image scalers
414  """
415  statsCtrl = afwMath.StatisticsControl()
416  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
417  statsCtrl.setNumIter(self.config.clipIter)
418  statsCtrl.setAndMask(self.getBadPixelMask())
419  statsCtrl.setNanSafe(True)
420  # compute tempExpRefList: a list of tempExpRef that actually exist
421  # and weightList: a list of the weight of the associated coadd tempExp
422  # and imageScalerList: a list of scale factors for the associated coadd tempExp
423  tempExpRefList = []
424  weightList = []
425  imageScalerList = []
426  tempExpName = self.getTempExpDatasetName(self.warpType)
427  for tempExpRef in refList:
428  if not tempExpRef.datasetExists(tempExpName):
429  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
430  continue
431 
432  tempExp = tempExpRef.get(tempExpName, immediate=True)
433  maskedImage = tempExp.getMaskedImage()
434  imageScaler = self.scaleZeroPoint.computeImageScaler(
435  exposure=tempExp,
436  dataRef=tempExpRef,
437  )
438  try:
439  imageScaler.scaleMaskedImage(maskedImage)
440  except Exception as e:
441  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
442  continue
443  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
444  afwMath.MEANCLIP, statsCtrl)
445  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
446  weight = 1.0 / float(meanVar)
447  if not numpy.isfinite(weight):
448  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
449  continue
450  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
451 
452  del maskedImage
453  del tempExp
454 
455  tempExpRefList.append(tempExpRef)
456  weightList.append(weight)
457  imageScalerList.append(imageScaler)
458 
459  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
460  imageScalerList=imageScalerList)
461 
462  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
463  altMaskList=None, mask=None, supplementaryData=None):
464  """!
465  \anchor AssembleCoaddTask.assemble_
466 
467  \brief Assemble a coadd from input warps
468 
469  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
470  large area), the assembly is performed over small areas on the image at a time in order to
471  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
472  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
473  statistic specified. Set the edge bits the coadd mask based on the weight map.
474 
475  \param[in] skyInfo: Patch geometry information, from getSkyInfo
476  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
477  \param[in] imageScalerList: List of image scalers
478  \param[in] weightList: List of weights
479  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
480  \param[in] mask: Mask to ignore when coadding
481  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
482  Only used by subclasses that implement makeSupplementaryData and override assemble.
483  \return pipeBase.Struct with coaddExposure, nImage if requested
484  """
485  tempExpName = self.getTempExpDatasetName(self.warpType)
486  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
487  if mask is None:
488  mask = self.getBadPixelMask()
489 
490  statsCtrl = afwMath.StatisticsControl()
491  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
492  statsCtrl.setNumIter(self.config.clipIter)
493  statsCtrl.setAndMask(mask)
494  statsCtrl.setNanSafe(True)
495  statsCtrl.setWeighted(True)
496  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
497  for plane, threshold in self.config.maskPropagationThresholds.items():
498  bit = afwImage.Mask.getMaskPlane(plane)
499  statsCtrl.setMaskPropagationThreshold(bit, threshold)
500 
501  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
502 
503  if altMaskList is None:
504  altMaskList = [None]*len(tempExpRefList)
505 
506  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
507  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
508  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
509  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
510  coaddMaskedImage = coaddExposure.getMaskedImage()
511  subregionSizeArr = self.config.subregionSize
512  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
513  # if nImage is requested, create a zero one which can be passed to assembleSubregion
514  if self.config.doNImage:
515  nImage = afwImage.ImageU(skyInfo.bbox)
516  else:
517  nImage = None
518  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
519  try:
520  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
521  weightList, altMaskList, statsFlags, statsCtrl,
522  nImage=nImage)
523  except Exception as e:
524  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
525 
526  self.setEdge(coaddMaskedImage.getMask())
527  self.setInexactPsf(coaddMaskedImage.getMask())
528  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
529  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
530  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
531  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
532 
533  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
534  """!
535  \brief Set the metadata for the coadd
536 
537  This basic implementation simply sets the filter from the
538  first input.
539 
540  \param[in] coaddExposure: The target image for the coadd
541  \param[in] tempExpRefList: List of data references to tempExp
542  \param[in] weightList: List of weights
543  """
544  assert len(tempExpRefList) == len(weightList), "Length mismatch"
545  tempExpName = self.getTempExpDatasetName(self.warpType)
546  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
547  # (and we need more than just the PropertySet that contains the header), which is not possible
548  # with the current butler (see #2777).
549  tempExpList = [tempExpRef.get(tempExpName + "_sub",
550  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
551  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
552  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
553 
554  coaddExposure.setFilter(tempExpList[0].getFilter())
555  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
556  coaddInputs.ccds.reserve(numCcds)
557  coaddInputs.visits.reserve(len(tempExpList))
558 
559  for tempExp, weight in zip(tempExpList, weightList):
560  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
561  coaddInputs.visits.sort()
562  if self.warpType == "psfMatched":
563  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
564  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
565  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
566  # having the maximum width (sufficient because square)
567  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
568  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
569  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
570  else:
571  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
572  self.config.coaddPsf.makeControl())
573  coaddExposure.setPsf(psf)
574  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
575  coaddExposure.getWcs())
576  coaddExposure.getInfo().setApCorrMap(apCorrMap)
577 
578  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
579  altMaskList, statsFlags, statsCtrl, nImage=None):
580  """!
581  \brief Assemble the coadd for a sub-region.
582 
583  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
584  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
585  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
586  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
587  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
588  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
589 
590  \param[in] coaddExposure: The target image for the coadd
591  \param[in] bbox: Sub-region to coadd
592  \param[in] tempExpRefList: List of data reference to tempExp
593  \param[in] imageScalerList: List of image scalers
594  \param[in] weightList: List of weights
595  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
596  Each element is dict with keys = mask plane name to which to add the spans
597  \param[in] statsFlags: afwMath.Property object for statistic for coadd
598  \param[in] statsCtrl: Statistics control object for coadd
599  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
600  """
601  self.log.debug("Computing coadd over %s", bbox)
602  tempExpName = self.getTempExpDatasetName(self.warpType)
603  coaddMaskedImage = coaddExposure.getMaskedImage()
604  coaddMaskedImage.getMask().addMaskPlane("CLIPPED")
605  maskedImageList = []
606  if nImage is not None:
607  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
608  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
609  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
610  maskedImage = exposure.getMaskedImage()
611  mask = maskedImage.getMask()
612  if altMask is not None:
613  self.applyAltMaskPlanes(mask, altMask)
614  imageScaler.scaleMaskedImage(maskedImage)
615 
616  # Add 1 for each pixel which is not excluded by the exclude mask.
617  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
618  if nImage is not None:
619  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
620  if self.config.removeMaskPlanes:
621  mask = maskedImage.getMask()
622  for maskPlane in self.config.removeMaskPlanes:
623  try:
624  mask &= ~mask.getPlaneBitMask(maskPlane)
625  except Exception as e:
626  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
627 
628  maskedImageList.append(maskedImage)
629 
630  with self.timer("stack"):
631  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
632  coaddMaskedImage.getMask().getPlaneBitMask("CLIPPED"),
633  coaddMaskedImage.getMask().getPlaneBitMask("NO_DATA"))
634  coaddMaskedImage.assign(coaddSubregion, bbox)
635  if nImage is not None:
636  nImage.assign(subNImage, bbox)
637 
638  def applyAltMaskPlanes(self, mask, altMaskSpans):
639  """!
640  \brief Apply in place alt mask formatted as SpanSets to a mask
641 
642  @param mask: original mask
643  @param altMaskSpans: Dictionary containing spanSet lists to apply.
644  Each element contains the new mask plane name
645  (e.g. "CLIPPED and/or "NO_DATA") as the key,
646  and list of SpanSets to apply to the mask
647  """
648  for plane, spanSetList in altMaskSpans.items():
649  maskClipValue = mask.addMaskPlane(plane)
650  for spanSet in spanSetList:
651  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
652  return mask
653 
654  def readBrightObjectMasks(self, dataRef):
655  """Returns None on failure"""
656  try:
657  return dataRef.get("brightObjectMask", immediate=True)
658  except Exception as e:
659  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
660  return None
661 
662  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
663  """Set the bright object masks
664 
665  exposure: Exposure under consideration
666  dataId: Data identifier dict for patch
667  brightObjectMasks: afwTable of bright objects to mask
668  """
669  #
670  # Check the metadata specifying the tract/patch/filter
671  #
672  if brightObjectMasks is None:
673  self.log.warn("Unable to apply bright object mask: none supplied")
674  return
675  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
676  md = brightObjectMasks.table.getMetadata()
677  for k in dataId:
678  if not md.exists(k):
679  self.log.warn("Expected to see %s in metadata", k)
680  else:
681  if md.get(k) != dataId[k]:
682  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
683 
684  mask = exposure.getMaskedImage().getMask()
685  wcs = exposure.getWcs()
686  plateScale = wcs.pixelScale().asArcseconds()
687 
688  for rec in brightObjectMasks:
689  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
690  if rec["type"] == "box":
691  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
692  width = rec["width"].asArcseconds()/plateScale # convert to pixels
693  height = rec["height"].asArcseconds()/plateScale # convert to pixels
694 
695  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
696  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
697 
698  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
699  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
700  spans = afwGeom.SpanSet(bbox)
701  elif rec["type"] == "circle":
702  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
703  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
704  else:
705  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
706  continue
707  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
708 
709  def setEdge(self, mask):
710  """Set EDGE bits as SENSOR_EDGE
711 
712  The EDGE pixels from the individual CCDs have printed through into the
713  coadd, but EDGE means "we couldn't search for sources in this area
714  because we couldn't convolve by the PSF near the edge of the image",
715  so this mask plane needs to be converted to something else if we want
716  to keep them. We do want to be able to identify pixels near the edge
717  of the detector because they will have an inexact `CoaddPsf`. We
718  rename EDGE pixels as SENSOR_EDGE.
719 
720  Parameters
721  ----------
722  mask : `lsst.afw.image.Mask`
723  Coadded exposure's mask, modified in-place.
724  """
725  mask.addMaskPlane("SENSOR_EDGE")
726  edge = mask.getPlaneBitMask("EDGE")
727  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE")
728  array = mask.getArray()
729  selected = (array & edge > 0)
730  array[selected] |= sensorEdge
731  array[selected] &= ~edge
732 
733  def setInexactPsf(self, mask):
734  """Set INEXACT_PSF mask plane
735 
736  If any of the input images isn't represented in the coadd (due to
737  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
738  these pixels.
739 
740  Parameters
741  ----------
742  mask : `lsst.afw.image.Mask`
743  Coadded exposure's mask, modified in-place.
744  """
745  mask.addMaskPlane("INEXACT_PSF")
746  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
747  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip gaps
748  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
749  array = mask.getArray()
750  selected = array & (sensorEdge | clipped) > 0
751  array[selected] |= inexactPsf
752 
753  @classmethod
754  def _makeArgumentParser(cls):
755  """!
756  \brief Create an argument parser
757  """
758  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
759  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
760  cls.ConfigClass().warpType + "Warp",
761  help="data ID, e.g. --id tract=12345 patch=1,2",
762  ContainerClass=AssembleCoaddDataIdContainer)
763  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
764  ContainerClass=SelectDataIdContainer)
765  return parser
766 
767 
768 def _subBBoxIter(bbox, subregionSize):
769  """!
770  \brief Iterate over subregions of a bbox
771 
772  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
773  \param[in] subregionSize: size of sub-bboxes
774 
775  \return subBBox: next sub-bounding box of size subregionSize or smaller;
776  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
777  but it will never be empty
778  """
779  if bbox.isEmpty():
780  raise RuntimeError("bbox %s is empty" % (bbox,))
781  if subregionSize[0] < 1 or subregionSize[1] < 1:
782  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
783 
784  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
785  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
786  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
787  subBBox.clip(bbox)
788  if subBBox.isEmpty():
789  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
790  (bbox, subregionSize, colShift, rowShift))
791  yield subBBox
792 
793 
794 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
795  """!
796  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
797  """
798 
799  def makeDataRefList(self, namespace):
800  """!
801  \brief Make self.refList from self.idList.
802  """
803  datasetType = namespace.config.coaddName + "Coadd"
804  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
805 
806  for dataId in self.idList:
807  # tract and patch are required
808  for key in keysCoadd:
809  if key not in dataId:
810  raise RuntimeError("--id must include " + key)
811 
812  dataRef = namespace.butler.dataRef(
813  datasetType=datasetType,
814  dataId=dataId,
815  )
816  self.refList.append(dataRef)
817 
818 
819 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
820  """!
821  \brief Function to count the number of pixels with a specific mask in a footprint.
822 
823  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
824  have bitmask set but do not have ignoreMask set. Return the count.
825 
826  \param[in] mask: mask to define intersection region by.
827  \parma[in] footprint: footprint to define the intersection region by.
828  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
829  \param[in] ignoreMask: pixels to not consider.
830  \return count of number of pixels in footprint with specified mask.
831  """
832  bbox = footprint.getBBox()
833  bbox.clip(mask.getBBox(afwImage.PARENT))
834  fp = afwImage.Mask(bbox)
835  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
836  footprint.spans.setMask(fp, bitmask)
837  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
838  (subMask.getArray() & ignoreMask) == 0).sum()
839 
840 
842  """!
843 \anchor SafeClipAssembleCoaddConfig
844 
845 \brief Configuration parameters for the SafeClipAssembleCoaddTask
846  """
847  clipDetection = pexConfig.ConfigurableField(
848  target=SourceDetectionTask,
849  doc="Detect sources on difference between unclipped and clipped coadd")
850  minClipFootOverlap = pexConfig.Field(
851  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
852  dtype=float,
853  default=0.6
854  )
855  minClipFootOverlapSingle = pexConfig.Field(
856  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
857  "clipped when only one visit overlaps",
858  dtype=float,
859  default=0.5
860  )
861  minClipFootOverlapDouble = pexConfig.Field(
862  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
863  "clipped when two visits overlap",
864  dtype=float,
865  default=0.45
866  )
867  maxClipFootOverlapDouble = pexConfig.Field(
868  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
869  "considering two visits",
870  dtype=float,
871  default=0.15
872  )
873  minBigOverlap = pexConfig.Field(
874  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
875  "when labeling clipped footprints",
876  dtype=int,
877  default=100
878  )
879 
880  def setDefaults(self):
881  # The numeric values for these configuration parameters were empirically determined, future work
882  # may further refine them.
883  AssembleCoaddConfig.setDefaults(self)
884  self.clipDetection.doTempLocalBackground = False
885  self.clipDetection.reEstimateBackground = False
886  self.clipDetection.returnOriginalFootprints = False
887  self.clipDetection.thresholdPolarity = "both"
888  self.clipDetection.thresholdValue = 2
889  self.clipDetection.nSigmaToGrow = 2
890  self.clipDetection.minPixels = 4
891  self.clipDetection.isotropicGrow = True
892  self.clipDetection.thresholdType = "pixel_stdev"
893  self.sigmaClip = 1.5
894  self.clipIter = 3
895  self.statistic = "MEAN"
896 
897  def validate(self):
898  if self.doSigmaClip:
899  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
900  "Ignoring doSigmaClip.")
901  self.doSigmaClip = False
902  if self.statistic != "MEAN":
903  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
904  "(%s chosen). Please set statistic to MEAN."
905  % (self.statistic))
906  AssembleCoaddTask.ConfigClass.validate(self)
907 
908 
909 
915 
916 
918  """!
919  \anchor SafeClipAssembleCoaddTask_
920 
921  \brief Assemble a coadded image from a set of coadded temporary exposures,
922  being careful to clip & flag areas with potential artifacts.
923 
924  \section pipe_tasks_assembleCoadd_Contents Contents
925  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
926  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
927  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
928  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
929  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
930  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
931 
932  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
933 
934  \copybrief SafeClipAssembleCoaddTask
935 
936  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
937  SafeClipAssembleCoaddTask subtasks that task.
938  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
939  outliers).
940  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
941  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
942  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
943  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
944  that there is an outlier and ii. this outlier appears on only one or two images.
945  Such regions will not contribute to the final coadd.
946  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
947  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
948  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
949  surveys.
950 
951  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
952  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
953  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
954 
955  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
956  \copydoc \_\_init\_\_
957 
958  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
959  \copydoc run
960 
961  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
962  See \ref SafeClipAssembleCoaddConfig
963 
964  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
965  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
966  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
967  files.
968  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
969  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
970  for further information.
971 
972  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
973  SafeClipAssembleCoaddTask
974 
975  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
976  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
977  '--legacyCoadd'.
978  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
979  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
980  with a list of coaddTempExps to attempt to coadd (specified using
981  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
982  Only the coaddTempExps that cover the specified tract and patch will be coadded.
983  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
984  command line argument:
985  \code
986  assembleCoadd.py --help
987  \endcode
988  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
989  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
990  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
991  packages.
992  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
993  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
994  <DL>
995  <DT>processCcd</DT>
996  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
997  <DT>makeSkyMap</DT>
998  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
999  <DT>makeCoaddTempExp</DT>
1000  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1001  </DL>
1002  We can perform all of these steps by running
1003  \code
1004  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1005  \endcode
1006  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1007  as follows:
1008  \code
1009  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1010  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1011  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1012  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1013  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1014  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1015  --selectId visit=903988 ccd=24
1016  \endcode
1017  This will process the HSC-I band data. The results are written in
1018  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1019 
1020  You may also choose to run:
1021  \code
1022  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1023  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1024  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1025  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1026  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1027  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1028  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1029  --selectId visit=903346 ccd=12
1030  \endcode
1031  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1032  discussed in \ref pipeTasks_multiBand.
1033  """
1034  ConfigClass = SafeClipAssembleCoaddConfig
1035  _DefaultName = "safeClipAssembleCoadd"
1036 
1037  def __init__(self, *args, **kwargs):
1038  """!
1039  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1040  """
1041  AssembleCoaddTask.__init__(self, *args, **kwargs)
1042  schema = afwTable.SourceTable.makeMinimalSchema()
1043  self.makeSubtask("clipDetection", schema=schema)
1044 
1045  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1046  """!
1047  \brief Assemble the coadd for a region
1048 
1049  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1050  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1051  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1052  overlap between the clipped region and a source.
1053  This leaves us with a set of footprints from the difference image that have been identified as having
1054  occured on just one or two individual visits. However, these footprints were generated from a
1055  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1056  footprints acrosss the coadd difference in this process.
1057  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1058  these are big footprints.
1059  Combine the small and big clipped footprints and mark them on a new bad mask plane
1060  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1061  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1062  bad mask plane.
1063 
1064  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1065  parent task.
1066 
1067  @param skyInfo: Patch geometry information, from getSkyInfo
1068  @param tempExpRefList: List of data reference to tempExp
1069  @param imageScalerList: List of image scalers
1070  @param weightList: List of weights
1071  return pipeBase.Struct with coaddExposure, nImage
1072  """
1073  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1074  mask = exp.getMaskedImage().getMask()
1075  mask.addMaskPlane("CLIPPED")
1076 
1077  result = self.detectClip(exp, tempExpRefList)
1078 
1079  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1080 
1081  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1082  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1083  # Append big footprints from individual Warps to result.clipSpans
1084  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1085  result.detectionFootprints, maskClipValue, maskDetValue,
1086  exp.getBBox())
1087  # Create mask of the current clipped footprints
1088  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1089  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1090 
1091  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1092  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1093  maskClip |= maskClipBig
1094 
1095  # Assemble coadd from base class, but ignoring CLIPPED pixels
1096  badMaskPlanes = self.config.badMaskPlanes[:]
1097  badMaskPlanes.append("CLIPPED")
1098  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1099  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1100  result.clipSpans, mask=badPixelMask)
1101 
1102  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1103  """!
1104  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1105 
1106  Generate a difference image between clipped and unclipped coadds.
1107  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1108  Return the difference image.
1109 
1110  @param skyInfo: Patch geometry information, from getSkyInfo
1111  @param tempExpRefList: List of data reference to tempExp
1112  @param imageScalerList: List of image scalers
1113  @param weightList: List of weights
1114  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1115  """
1116  # Clone and upcast self.config because current self.config is frozen
1117  config = AssembleCoaddConfig()
1118  # getattr necessary because subtasks do not survive Config.toDict()
1119  configIntersection = {k: getattr(self.config, k)
1120  for k, v in self.config.toDict().items() if (k in config.keys())}
1121  config.update(**configIntersection)
1122 
1123  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1124  config.statistic = 'MEAN'
1125  task = AssembleCoaddTask(config=config)
1126  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1127 
1128  config.statistic = 'MEANCLIP'
1129  task = AssembleCoaddTask(config=config)
1130  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1131 
1132  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1133  coaddDiff -= coaddClip.getMaskedImage()
1134  exp = afwImage.ExposureF(coaddDiff)
1135  exp.setPsf(coaddMean.getPsf())
1136  return exp
1137 
1138  def detectClip(self, exp, tempExpRefList):
1139  """!
1140  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1141 
1142  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1143  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1144  fraction to thresholds set in the config.
1145  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1146  two).
1147  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1148  the coaddTempExp.
1149  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1150  with the clipped footprints and a list of new masks for the coaddTempExps.
1151 
1152  \param[in] exp: Exposure to run detection on
1153  \param[in] tempExpRefList: List of data reference to tempExp
1154  \return struct containing:
1155  - clipFootprints: list of clipped footprints
1156  - clipIndices: indices for each clippedFootprint in tempExpRefList
1157  - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new
1158  maskplane name ("CLIPPED")" as the key and list of SpanSets as value
1159  - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints
1160  """
1161  mask = exp.getMaskedImage().getMask()
1162  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1163  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1164  # Merge positive and negative together footprints together
1165  fpSet.positive.merge(fpSet.negative)
1166  footprints = fpSet.positive
1167  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1168  ignoreMask = self.getBadPixelMask()
1169 
1170  clipFootprints = []
1171  clipIndices = []
1172  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1173 
1174  # for use by detectClipBig
1175  visitDetectionFootprints = []
1176 
1177  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1178  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1179  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1180 
1181  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1182  for i, warpRef in enumerate(tempExpRefList):
1183  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1184  immediate=True).getMaskedImage().getMask()
1185  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1186  afwImage.PARENT, True)
1187  maskVisitDet &= maskDetValue
1188  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1189  visitDetectionFootprints.append(visitFootprints)
1190 
1191  for j, footprint in enumerate(footprints.getFootprints()):
1192  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1193  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1194 
1195  # build a list of clipped spans for each visit
1196  for j, footprint in enumerate(footprints.getFootprints()):
1197  nPixel = footprint.getArea()
1198  overlap = [] # hold the overlap with each visit
1199  indexList = [] # index of visit in global list
1200  for i in range(len(tempExpRefList)):
1201  ignore = ignoreArr[i, j]
1202  overlapDet = overlapDetArr[i, j]
1203  totPixel = nPixel - ignore
1204 
1205  # If we have more bad pixels than detection skip
1206  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1207  continue
1208  overlap.append(overlapDet/float(totPixel))
1209  indexList.append(i)
1210 
1211  overlap = numpy.array(overlap)
1212  if not len(overlap):
1213  continue
1214 
1215  keep = False # Should this footprint be marked as clipped?
1216  keepIndex = [] # Which tempExps does the clipped footprint belong to
1217 
1218  # If footprint only has one overlap use a lower threshold
1219  if len(overlap) == 1:
1220  if overlap[0] > self.config.minClipFootOverlapSingle:
1221  keep = True
1222  keepIndex = [0]
1223  else:
1224  # This is the general case where only visit should be clipped
1225  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1226  if len(clipIndex) == 1:
1227  keep = True
1228  keepIndex = [clipIndex[0]]
1229 
1230  # Test if there are clipped objects that overlap two different visits
1231  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1232  if len(clipIndex) == 2 and len(overlap) > 3:
1233  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1234  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1235  keep = True
1236  keepIndex = clipIndex
1237 
1238  if not keep:
1239  continue
1240 
1241  for index in keepIndex:
1242  globalIndex = indexList[index]
1243  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1244 
1245  clipIndices.append(numpy.array(indexList)[keepIndex])
1246  clipFootprints.append(footprint)
1247 
1248  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1249  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1250 
1251  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1252  maskClipValue, maskDetValue, coaddBBox):
1253  """!
1254  \brief Return individual warp footprints for large artifacts and append them to clipList in place
1255 
1256  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1257  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1258  significantly with each source in all the coaddTempExps.
1259  \param[in] clipList: List of alt mask SpanSets with clipping information. Modified.
1260  \param[in] clipFootprints: List of clipped footprints
1261  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1262  \param[in] maskClipValue: Mask value of clipped pixels
1263  \param[in] maskDetValue: Mask value of detected pixels
1264  \param[in] coaddBBox: BBox of the coadd and warps
1265  \return list of big footprints
1266  """
1267  bigFootprintsCoadd = []
1268  ignoreMask = self.getBadPixelMask()
1269  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1270  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1271  for footprint in visitFootprints.getFootprints():
1272  footprint.spans.setMask(maskVisitDet, maskDetValue)
1273 
1274  # build a mask of clipped footprints that are in this visit
1275  clippedFootprintsVisit = []
1276  for foot, clipIndex in zip(clipFootprints, clipIndices):
1277  if index not in clipIndex:
1278  continue
1279  clippedFootprintsVisit.append(foot)
1280  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1281  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1282 
1283  bigFootprintsVisit = []
1284  for foot in visitFootprints.getFootprints():
1285  if foot.getArea() < self.config.minBigOverlap:
1286  continue
1287  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1288  if nCount > self.config.minBigOverlap:
1289  bigFootprintsVisit.append(foot)
1290  bigFootprintsCoadd.append(foot)
1291 
1292  for footprint in bigFootprintsVisit:
1293  clippedSpans["CLIPPED"].append(footprint.spans)
1294 
1295  return bigFootprintsCoadd
1296 
1297 
1299  assembleStaticSkyModel = pexConfig.ConfigurableField(
1300  target=AssembleCoaddTask,
1301  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1302  " naive/first-iteration model of the static sky.",
1303  )
1304  detect = pexConfig.ConfigurableField(
1305  target=SourceDetectionTask,
1306  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1307  )
1308  maxNumEpochs = pexConfig.Field(
1309  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1310  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1311  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1312  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1313  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1314  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1315  "than transient and not masked.",
1316  dtype=int,
1317  default=2
1318  )
1319  maxFractionEpochsLow = pexConfig.RangeField(
1320  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1321  "Effective maxNumEpochs = "
1322  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1323  dtype=float,
1324  default=0.4,
1325  min=0., max=1.,
1326  )
1327  maxFractionEpochsHigh = pexConfig.RangeField(
1328  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1329  "Effective maxNumEpochs = "
1330  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1331  dtype=float,
1332  default=0.03,
1333  min=0., max=1.,
1334  )
1335  spatialThreshold = pexConfig.RangeField(
1336  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1337  "temporal criteria. If 0, clip all. If 1, clip none.",
1338  dtype=float,
1339  default=0.5,
1340  min=0., max=1.,
1341  inclusiveMin=True, inclusiveMax=True
1342  )
1343  doScaleWarpVariance = pexConfig.Field(
1344  doc="Rescale Warp variance plane using empirical noise?",
1345  dtype=bool,
1346  default=True,
1347  )
1348  maskScaleWarpVariance = pexConfig.ListField(
1349  dtype=str,
1350  default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
1351  doc="Mask planes for pixels to ignore when rescaling warp variance",
1352  )
1353 
1354  def setDefaults(self):
1355  AssembleCoaddConfig.setDefaults(self)
1356  self.statistic = 'MEAN'
1357  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1358  self.assembleStaticSkyModel.warpType = 'psfMatched'
1359  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1360  self.assembleStaticSkyModel.sigmaClip = 1.5
1361  self.assembleStaticSkyModel.clipIter = 3
1362  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1363  self.assembleStaticSkyModel.doWrite = False
1364  self.detect.doTempLocalBackground = False
1365  self.detect.reEstimateBackground = False
1366  self.detect.returnOriginalFootprints = False
1367  self.detect.thresholdPolarity = "both"
1368  self.detect.thresholdValue = 5
1369  self.detect.nSigmaToGrow = 2
1370  self.detect.minPixels = 4
1371  self.detect.isotropicGrow = True
1372  self.detect.thresholdType = "pixel_stdev"
1373 
1374 
1375 
1381 
1383  """!
1384  \anchor CompareWarpAssembleCoaddTask_
1385 
1386  \brief Assemble a compareWarp coadded image from a set of warps
1387  by masking artifacts detected by comparing PSF-matched warps
1388 
1389  \section pipe_tasks_assembleCoadd_Contents Contents
1390  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1391  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1392  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1393  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1394  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1395  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1396 
1397  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1398 
1399  \copybrief CompareWarpAssembleCoaddTask
1400 
1401  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1402  outliers).
1403  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1404  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1405  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1406  pixels in the individual warps suspected to contain an artifact.
1407  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1408  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1409  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1410  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1411  such as bright stars.
1412  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1413  The temporalThreshold is the maximum fraction of epochs that the deviation can
1414  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1415  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1416  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1417  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1418  These regions will not contribute to the final coadd.
1419  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1420  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1421  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1422  surveys.
1423 
1424  CompareWarpAssembleCoaddTask sub-classes
1425  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1426  as a subtask to generate the TemplateCoadd (the model of the static sky)
1427 
1428  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1429  \copydoc \_\_init\_\_
1430 
1431  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1432  \copydoc run
1433 
1434  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1435  See \ref CompareWarpAssembleCoaddConfig
1436 
1437  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1438  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1439  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1440  files.
1441 
1442  This task supports the following debug variables:
1443  <dl>
1444  <dt>`saveCountIm`
1445  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1446  <dt> `figPath`
1447  <dd> Path to save the debug fits images and figures
1448  </dl>
1449 
1450  For example, put something like:
1451  @code{.py}
1452  import lsstDebug
1453  def DebugInfo(name):
1454  di = lsstDebug.getInfo(name)
1455  if name == "lsst.pipe.tasks.assembleCoadd":
1456  di.saveCountIm = True
1457  di.figPath = "/desired/path/to/debugging/output/images"
1458  return di
1459  lsstDebug.Info = DebugInfo
1460  @endcode
1461  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1462  flag.
1463  Some subtasks may have their own debug variables; see individual Task
1464  documentation
1465 
1466  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1467  CompareWarpAssembleCoaddTask
1468 
1469  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1470  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1471  '--compareWarpCoadd'.
1472  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1473  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1474  with a list of coaddTempExps to attempt to coadd (specified using
1475  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1476  Only the warps that cover the specified tract and patch will be coadded.
1477  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1478  command line argument:
1479  \code
1480  assembleCoadd.py --help
1481  \endcode
1482  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1483  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1484  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1485  packages.
1486  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1487  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1488  <DL>
1489  <DT>processCcd</DT>
1490  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1491  <DT>makeSkyMap</DT>
1492  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1493  <DT>makeCoaddTempExp</DT>
1494  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1495  </DL>
1496  We can perform all of these steps by running
1497  \code
1498  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1499  \endcode
1500  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1501  as follows:
1502  \code
1503  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1504  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1505  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1506  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1507  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1508  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1509  --selectId visit=903988 ccd=24
1510  \endcode
1511  This will process the HSC-I band data. The results are written in
1512  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1513  """
1514  ConfigClass = CompareWarpAssembleCoaddConfig
1515  _DefaultName = "compareWarpAssembleCoadd"
1516 
1517  def __init__(self, *args, **kwargs):
1518  """!
1519  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1520  """
1521  AssembleCoaddTask.__init__(self, *args, **kwargs)
1522  self.makeSubtask("assembleStaticSkyModel")
1523  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1524  self.makeSubtask("detect", schema=detectionSchema)
1525 
1526  def makeSupplementaryData(self, dataRef, selectDataList):
1527  """!
1528  \brief Make inputs specific to Subclass
1529 
1530  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1531  """
1532  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1533 
1534  if templateCoadd is None:
1535  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1536  self.assembleStaticSkyModel.warpType[1:])
1537  message = """No %(warpName)s warps were found to build the template coadd which is
1538  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1539  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1540  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1541 
1542  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1543  another algorithm like:
1544 
1545  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1546  config.assemble.retarget(SafeClipAssembleCoaddTask)
1547  """ % {"warpName": warpName}
1548  raise RuntimeError(message)
1549 
1550  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1551 
1552  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1553  supplementaryData, *args, **kwargs):
1554  """!
1555  \brief Assemble the coadd
1556 
1557  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1558  as the model of the static sky.
1559 
1560  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1561  new "CLIPPED" plane and updated "NO_DATA" plane.
1562  Then pass these alternative masks to the base class's assemble method.
1563 
1564  @param skyInfo: Patch geometry information
1565  @param tempExpRefList: List of data references to warps
1566  @param imageScalerList: List of image scalers
1567  @param weightList: List of weights
1568  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1569 
1570  return pipeBase.Struct with coaddExposure, nImage if requested
1571  """
1572  templateCoadd = supplementaryData.templateCoadd
1573  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1574  badMaskPlanes = self.config.badMaskPlanes[:]
1575  badMaskPlanes.append("CLIPPED")
1576  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1577 
1578  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1579  spanSetMaskList, mask=badPixelMask)
1580 
1581  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1582  """!
1583  \brief Find artifacts
1584 
1585  Loop through warps twice. The first loop builds a map with the count of how many
1586  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1587  The second loop takes each difference image and filters the artifacts detected
1588  in each using count map to filter out variable sources and sources that are difficult to
1589  subtract cleanly.
1590 
1591  @param templateCoadd: Exposure to serve as model of static sky
1592  @param tempExpRefList: List of data references to warps
1593  @param imageScalerList: List of image scalers
1594  """
1595 
1596  self.log.debug("Generating Count Image, and mask lists.")
1597  coaddBBox = templateCoadd.getBBox()
1598  slateIm = afwImage.ImageU(coaddBBox)
1599  epochCountImage = afwImage.ImageU(coaddBBox)
1600  nImage = afwImage.ImageU(coaddBBox)
1601  spanSetArtifactList = []
1602  spanSetNoDataMaskList = []
1603 
1604  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1605  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1606  if warpDiffExp is not None:
1607  nImage.array += numpy.where(numpy.isnan(warpDiffExp.image.array),
1608  0, 1).astype(numpy.uint16)
1609  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1610  fpSet.positive.merge(fpSet.negative)
1611  footprints = fpSet.positive
1612  slateIm.set(0)
1613  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1614  for spans in spanSetList:
1615  spans.setImage(slateIm, 1, doClip=True)
1616  epochCountImage += slateIm
1617 
1618  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1619  # undergo a second convolution. Pixels with data in the direct warp
1620  # but not in the PSF-matched warp will not have their artifacts detected.
1621  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1622  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1623  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1624  nansMask.setXY0(warpDiffExp.getXY0())
1625  else:
1626  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1627  # In this case, mask the whole epoch
1628  nansMask = afwImage.MaskX(coaddBBox, 1)
1629  spanSetList = []
1630 
1631  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1632 
1633  spanSetNoDataMaskList.append(spanSetNoDataMask)
1634  spanSetArtifactList.append(spanSetList)
1635 
1636  if lsstDebug.Info(__name__).saveCountIm:
1637  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1638  epochCountImage.writeFits(path)
1639 
1640  for i, spanSetList in enumerate(spanSetArtifactList):
1641  if spanSetList:
1642  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage, nImage)
1643  spanSetArtifactList[i] = filteredSpanSetList
1644 
1645  altMasks = []
1646  for artifacts, noData in zip(spanSetArtifactList, spanSetNoDataMaskList):
1647  altMasks.append({'CLIPPED': artifacts,
1648  'NO_DATA': noData})
1649  return altMasks
1650 
1651  def _filterArtifacts(self, spanSetList, epochCountImage, nImage):
1652  """!
1653  \brief Filter artifact candidates
1654 
1655  @param spanSetList: List of SpanSets representing artifact candidates
1656  @param epochCountImage: Image of accumulated number of warpDiff detections
1657  @param nImage: Image of the accumulated number of total epochs contributing
1658 
1659  return List of SpanSets with artifacts
1660  """
1661 
1662  maskSpanSetList = []
1663  x0, y0 = epochCountImage.getXY0()
1664  for i, span in enumerate(spanSetList):
1665  y, x = span.indices()
1666  yIdxLocal = [y1 - y0 for y1 in y]
1667  xIdxLocal = [x1 - x0 for x1 in x]
1668  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1669  totalN = nImage.array[yIdxLocal, xIdxLocal]
1670 
1671  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1672  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1673  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1674  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1675  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1676  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1677  (outlierN <= effectiveMaxNumEpochs))
1678  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1679  if percentBelowThreshold > self.config.spatialThreshold:
1680  maskSpanSetList.append(span)
1681  return maskSpanSetList
1682 
1683  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1684  """!
1685  \brief Fetch a warp from the butler and return a warpDiff
1686 
1687  @param warpRef: `Butler dataRef` for the warp
1688  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1689  @param templateCoadd: Exposure to be substracted from the scaled warp
1690 
1691  return Exposure of the image difference between the warp and template
1692  """
1693 
1694  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1695  warpName = self.getTempExpDatasetName('psfMatched')
1696  if not warpRef.datasetExists(warpName):
1697  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1698  return None
1699  warp = warpRef.get(warpName, immediate=True)
1700  # direct image scaler OK for PSF-matched Warp
1701  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1702  mi = warp.getMaskedImage()
1703  if self.config.doScaleWarpVariance:
1704  scaleVariance(mi, self.config.maskScaleWarpVariance,
1705  log=self.log)
1706  mi -= templateCoadd.getMaskedImage()
1707  return warp
1708 
1709  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1710  """!
1711  \brief Return a path to which to write debugging output
1712 
1713  @param prefix: string, prefix for filename
1714  @param warpRef: Butler dataRef
1715  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1716  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1717 
1718  Creates a hyphen-delimited string of dataId values for simple filenames.
1719  """
1720  if coaddLevel:
1721  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1722  else:
1723  keys = warpRef.dataId.keys()
1724  keyList = sorted(keys, reverse=True)
1725  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1726  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1727  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def _filterArtifacts(self, spanSetList, epochCountImage, nImage)
Filter artifact candidates.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...