lsst.pipe.tasks  14.0-39-g03bf09b5
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import os
26 import numpy
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, scaleVariance
40 from .interpImage import InterpImageTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms import SourceDetectionTask
44 
45 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
46 
47 
48 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
49  """!
50 \anchor AssembleCoaddConfig_
51 
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53  """
54  warpType = pexConfig.Field(
55  doc="Warp name: one of 'direct' or 'psfMatched'",
56  dtype=str,
57  default="direct",
58  )
59  subregionSize = pexConfig.ListField(
60  dtype=int,
61  doc="Width, height of stack subregion size; "
62  "make small enough that a full stack of images will fit into memory at once.",
63  length=2,
64  default=(2000, 2000),
65  )
66  statistic = pexConfig.Field(
67  dtype=str,
68  doc="Main stacking statistic for aggregating over the epochs.",
69  default="MEANCLIP",
70  )
71  doSigmaClip = pexConfig.Field(
72  dtype=bool,
73  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
74  default=False,
75  )
76  sigmaClip = pexConfig.Field(
77  dtype=float,
78  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
79  default=3.0,
80  )
81  clipIter = pexConfig.Field(
82  dtype=int,
83  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
84  default=2,
85  )
86  calcErrorFromInputVariance = pexConfig.Field(
87  dtype=bool,
88  doc="Calculate coadd variance from input variance by stacking statistic."
89  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
90  default=True,
91  )
92  scaleZeroPoint = pexConfig.ConfigurableField(
93  target=ScaleZeroPointTask,
94  doc="Task to adjust the photometric zero point of the coadd temp exposures",
95  )
96  doInterp = pexConfig.Field(
97  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
98  dtype=bool,
99  default=True,
100  )
101  interpImage = pexConfig.ConfigurableField(
102  target=InterpImageTask,
103  doc="Task to interpolate (and extrapolate) over NaN pixels",
104  )
105  doWrite = pexConfig.Field(
106  doc="Persist coadd?",
107  dtype=bool,
108  default=True,
109  )
110  doNImage = pexConfig.Field(
111  doc="Create image of number of contributing exposures for each pixel",
112  dtype=bool,
113  default=False,
114  )
115  maskPropagationThresholds = pexConfig.DictField(
116  keytype=str,
117  itemtype=float,
118  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
119  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
120  "would have contributed exceeds this value."),
121  default={"SAT": 0.1},
122  )
123  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
124  doc="Mask planes to remove before coadding")
125  #
126  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
127  # To make this useful you *must* also configure the flags.pixel algorithm, for example
128  # by adding
129  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
130  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
131  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
132  #
133  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
134  doc="Set mask and flag bits for bright objects?")
135  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
136  doc="Name of mask bit used for bright objects")
137 
138  coaddPsf = pexConfig.ConfigField(
139  doc="Configuration for CoaddPsf",
140  dtype=measAlg.CoaddPsfConfig,
141  )
142 
143  def setDefaults(self):
144  CoaddBaseTask.ConfigClass.setDefaults(self)
145  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
146 
147  def validate(self):
148  CoaddBaseTask.ConfigClass.validate(self)
149  if self.doPsfMatch:
150  # Backwards compatibility.
151  # Configs do not have loggers
152  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
153  self.warpType = 'psfMatched'
154  if self.doSigmaClip and self.statistic != "MEANCLIP":
155  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
156  self.statistic = "MEANCLIP"
157  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
158  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
159  "compute and set a non-zero coadd variance estimate." % (self.statistic))
160 
161  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
162  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
163  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
164  if str(k) not in unstackableStats]
165  raise ValueError("statistic %s is not allowed. Please choose one of %s."
166  % (self.statistic, stackableStats))
167 
168 
169 
176  """!
177 \anchor AssembleCoaddTask_
178 
179 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
180 
181 \section pipe_tasks_assembleCoadd_Contents Contents
182  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
183  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
184  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
185  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
186  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
187  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
188 
189 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
190 
191 \copybrief AssembleCoaddTask_
192 
193 We want to assemble a coadded image from a set of Warps (also called
194 coadded temporary exposures or coaddTempExps.
195 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
196 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
197 Warps that cover the specified patch (pointed at by dataRef).
198 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
199 Therefore, we must scale each Warp to set it to a common photometric zeropoint.
200 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
201 config.makePsfMatched set which of the warp types will be coadded.
202 The coadd is computed as a mean with optional outlier rejection.
203 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
204 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
205 
206 AssembleCoaddTask uses several sub-tasks. These are
207 <DL>
208  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
209  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
210  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
211  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
212 </DL>
213 You can retarget these subtasks if you wish.
214 
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
216 \copydoc \_\_init\_\_
217 
218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
219 \copydoc run
220 
221 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
222 See \ref AssembleCoaddConfig_
223 
224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
225 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
226 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
227 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
228 the documetation for the subtasks for further information.
229 
230 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
231 
232 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
233 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
234 a data reference to the tract patch and filter to be coadded (specified using
235 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
236 Warps to attempt to coadd (specified using
237 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
238 that cover the specified tract and patch will be coadded. A list of the available optional
239 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
240 \code
241 assembleCoadd.py --help
242 \endcode
243 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
244 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
245 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
246 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
247 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
248 <DL>
249  <DT>processCcd</DT>
250  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
251  <DT>makeSkyMap</DT>
252  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
253  <DT>makeCoaddTempExp</DT>
254  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
255 </DL>
256 We can perform all of these steps by running
257 \code
258 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
259 \endcode
260 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
261 follows:
262 \code
263 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
264 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
265 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
266 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
267 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
268 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
269 --selectId visit=903988 ccd=24
270 \endcode
271 that will process the HSC-I band data. The results are written in
272 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
273 
274 You may also choose to run:
275 \code
276 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
277 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
278 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
279 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
280 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
281 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
282 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
283 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
284 \endcode
285 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
286 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
287 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
288  """
289  ConfigClass = AssembleCoaddConfig
290  _DefaultName = "assembleCoadd"
291 
292  def __init__(self, *args, **kwargs):
293  """!
294  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
295  & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
296  """
297  CoaddBaseTask.__init__(self, *args, **kwargs)
298  self.makeSubtask("interpImage")
299  self.makeSubtask("scaleZeroPoint")
300 
301  if self.config.doMaskBrightObjects:
302  mask = afwImage.Mask()
303  try:
304  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
305  except pexExceptions.LsstCppException:
306  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
307  mask.getMaskPlaneDict().keys())
308  del mask
309 
310  self.warpType = self.config.warpType
311 
312  @pipeBase.timeMethod
313  def run(self, dataRef, selectDataList=[]):
314  """!
315  \brief Assemble a coadd from a set of Warps
316 
317  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
318  match the photometric zeropoint to a reference Warp. Assemble the Warps using
319  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
320  exposure.
321 
322  \anchor runParams
323  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
324  (if config.autoReference=False). Used to access the following data products:
325  - [in] self.config.coaddName + "Coadd_skyMap"
326  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
327  - [out] self.config.coaddName + "Coadd"
328  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
329  selected from this list based on overlap with the patch defined by dataRef.
330 
331  \return a pipeBase.Struct with fields:
332  - coaddExposure: coadded exposure
333  - nImage: exposure count image
334  """
335  skyInfo = self.getSkyInfo(dataRef)
336  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
337  if len(calExpRefList) == 0:
338  self.log.warn("No exposures to coadd")
339  return
340  self.log.info("Coadding %d exposures", len(calExpRefList))
341 
342  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
343  inputData = self.prepareInputs(tempExpRefList)
344  self.log.info("Found %d %s", len(inputData.tempExpRefList),
345  self.getTempExpDatasetName(self.warpType))
346  if len(inputData.tempExpRefList) == 0:
347  self.log.warn("No coadd temporary exposures found")
348  return
349 
350  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
351 
352  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
353  inputData.weightList, supplementaryData=supplementaryData)
354 
355  if self.config.doInterp:
356  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
357  # The variance must be positive; work around for DM-3201.
358  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
359  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
360 
361  if self.config.doMaskBrightObjects:
362  brightObjectMasks = self.readBrightObjectMasks(dataRef)
363  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
364 
365  if self.config.doWrite:
366  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
367  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
368  if self.config.doNImage and retStruct.nImage is not None:
369  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
370 
371  return retStruct
372 
373  def makeSupplementaryData(self, dataRef, selectDataList):
374  """!
375  \brief Make additional inputs to assemble() specific to subclasses.
376 
377  Available to be implemented by subclasses only if they need the
378  coadd dataRef for performing preliminary processing before
379  assembling the coadd.
380  """
381  pass
382 
383  def getTempExpRefList(self, patchRef, calExpRefList):
384  """!
385  \brief Generate list data references corresponding to warped exposures that lie within the
386  patch to be coadded.
387 
388  \param[in] patchRef: Data reference for patch
389  \param[in] calExpRefList: List of data references for input calexps
390  \return List of Warp/CoaddTempExp data references
391  """
392  butler = patchRef.getButler()
393  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
394  self.getTempExpDatasetName(self.warpType))
395  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
396  g, groupData.keys) for
397  g in groupData.groups.keys()]
398  return tempExpRefList
399 
400  def prepareInputs(self, refList):
401  """!
402  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
403  for the photometric zero point.
404 
405  Each Warp has its own photometric zeropoint and background variance. Before coadding these
406  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
407  weight for each Warp.
408 
409  \param[in] refList: List of data references to tempExp
410  \return Struct:
411  - tempExprefList: List of data references to tempExp
412  - weightList: List of weightings
413  - imageScalerList: List of image scalers
414  """
415  statsCtrl = afwMath.StatisticsControl()
416  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
417  statsCtrl.setNumIter(self.config.clipIter)
418  statsCtrl.setAndMask(self.getBadPixelMask())
419  statsCtrl.setNanSafe(True)
420  # compute tempExpRefList: a list of tempExpRef that actually exist
421  # and weightList: a list of the weight of the associated coadd tempExp
422  # and imageScalerList: a list of scale factors for the associated coadd tempExp
423  tempExpRefList = []
424  weightList = []
425  imageScalerList = []
426  tempExpName = self.getTempExpDatasetName(self.warpType)
427  for tempExpRef in refList:
428  if not tempExpRef.datasetExists(tempExpName):
429  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
430  continue
431 
432  tempExp = tempExpRef.get(tempExpName, immediate=True)
433  maskedImage = tempExp.getMaskedImage()
434  imageScaler = self.scaleZeroPoint.computeImageScaler(
435  exposure=tempExp,
436  dataRef=tempExpRef,
437  )
438  try:
439  imageScaler.scaleMaskedImage(maskedImage)
440  except Exception as e:
441  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
442  continue
443  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
444  afwMath.MEANCLIP, statsCtrl)
445  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
446  weight = 1.0 / float(meanVar)
447  if not numpy.isfinite(weight):
448  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
449  continue
450  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
451 
452  del maskedImage
453  del tempExp
454 
455  tempExpRefList.append(tempExpRef)
456  weightList.append(weight)
457  imageScalerList.append(imageScaler)
458 
459  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
460  imageScalerList=imageScalerList)
461 
462  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
463  altMaskList=None, mask=None, supplementaryData=None):
464  """!
465  \anchor AssembleCoaddTask.assemble_
466 
467  \brief Assemble a coadd from input warps
468 
469  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
470  large area), the assembly is performed over small areas on the image at a time in order to
471  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
472  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
473  statistic specified. Set the edge bits the coadd mask based on the weight map.
474 
475  \param[in] skyInfo: Patch geometry information, from getSkyInfo
476  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
477  \param[in] imageScalerList: List of image scalers
478  \param[in] weightList: List of weights
479  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
480  \param[in] mask: Mask to ignore when coadding
481  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
482  Only used by subclasses that implement makeSupplementaryData and override assemble.
483  \return pipeBase.Struct with coaddExposure, nImage if requested
484  """
485  tempExpName = self.getTempExpDatasetName(self.warpType)
486  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
487  if mask is None:
488  mask = self.getBadPixelMask()
489 
490  statsCtrl = afwMath.StatisticsControl()
491  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
492  statsCtrl.setNumIter(self.config.clipIter)
493  statsCtrl.setAndMask(mask)
494  statsCtrl.setNanSafe(True)
495  statsCtrl.setWeighted(True)
496  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
497  for plane, threshold in self.config.maskPropagationThresholds.items():
498  bit = afwImage.Mask.getMaskPlane(plane)
499  statsCtrl.setMaskPropagationThreshold(bit, threshold)
500 
501  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
502 
503  if altMaskList is None:
504  altMaskList = [None]*len(tempExpRefList)
505 
506  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
507  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
508  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
509  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
510  coaddMaskedImage = coaddExposure.getMaskedImage()
511  subregionSizeArr = self.config.subregionSize
512  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
513  # if nImage is requested, create a zero one which can be passed to assembleSubregion
514  if self.config.doNImage:
515  nImage = afwImage.ImageU(skyInfo.bbox)
516  else:
517  nImage = None
518  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
519  try:
520  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
521  weightList, altMaskList, statsFlags, statsCtrl,
522  nImage=nImage)
523  except Exception as e:
524  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
525 
526  self.setEdge(coaddMaskedImage.getMask())
527  self.setInexactPsf(coaddMaskedImage.getMask())
528  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
529  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
530  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
531  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
532 
533  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
534  """!
535  \brief Set the metadata for the coadd
536 
537  This basic implementation simply sets the filter from the
538  first input.
539 
540  \param[in] coaddExposure: The target image for the coadd
541  \param[in] tempExpRefList: List of data references to tempExp
542  \param[in] weightList: List of weights
543  """
544  assert len(tempExpRefList) == len(weightList), "Length mismatch"
545  tempExpName = self.getTempExpDatasetName(self.warpType)
546  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
547  # (and we need more than just the PropertySet that contains the header), which is not possible
548  # with the current butler (see #2777).
549  tempExpList = [tempExpRef.get(tempExpName + "_sub",
550  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
551  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
552  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
553 
554  coaddExposure.setFilter(tempExpList[0].getFilter())
555  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
556  coaddInputs.ccds.reserve(numCcds)
557  coaddInputs.visits.reserve(len(tempExpList))
558 
559  for tempExp, weight in zip(tempExpList, weightList):
560  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
561  coaddInputs.visits.sort()
562  if self.warpType == "psfMatched":
563  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
564  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
565  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
566  # having the maximum width (sufficient because square)
567  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
568  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
569  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
570  else:
571  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
572  self.config.coaddPsf.makeControl())
573  coaddExposure.setPsf(psf)
574  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
575  coaddExposure.getWcs())
576  coaddExposure.getInfo().setApCorrMap(apCorrMap)
577 
578  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
579  altMaskList, statsFlags, statsCtrl, nImage=None):
580  """!
581  \brief Assemble the coadd for a sub-region.
582 
583  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask
584  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
585  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
586  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
587  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
588  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
589 
590  \param[in] coaddExposure: The target image for the coadd
591  \param[in] bbox: Sub-region to coadd
592  \param[in] tempExpRefList: List of data reference to tempExp
593  \param[in] imageScalerList: List of image scalers
594  \param[in] weightList: List of weights
595  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
596  Each element is dict with keys = mask plane name to which to add the spans
597  \param[in] statsFlags: afwMath.Property object for statistic for coadd
598  \param[in] statsCtrl: Statistics control object for coadd
599  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
600  """
601  self.log.debug("Computing coadd over %s", bbox)
602  tempExpName = self.getTempExpDatasetName(self.warpType)
603  coaddMaskedImage = coaddExposure.getMaskedImage()
604  coaddMaskedImage.getMask().addMaskPlane("CLIPPED")
605  maskedImageList = []
606  if nImage is not None:
607  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
608  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
609  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
610  maskedImage = exposure.getMaskedImage()
611  mask = maskedImage.getMask()
612  if altMask is not None:
613  self.applyAltMaskPlanes(mask, altMask)
614  imageScaler.scaleMaskedImage(maskedImage)
615 
616  # Add 1 for each pixel which is not excluded by the exclude mask.
617  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
618  if nImage is not None:
619  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
620  if self.config.removeMaskPlanes:
621  mask = maskedImage.getMask()
622  for maskPlane in self.config.removeMaskPlanes:
623  try:
624  mask &= ~mask.getPlaneBitMask(maskPlane)
625  except Exception as e:
626  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
627 
628  maskedImageList.append(maskedImage)
629 
630  with self.timer("stack"):
631  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
632  coaddMaskedImage.getMask().getPlaneBitMask("CLIPPED"),
633  coaddMaskedImage.getMask().getPlaneBitMask("NO_DATA"))
634  coaddMaskedImage.assign(coaddSubregion, bbox)
635  if nImage is not None:
636  nImage.assign(subNImage, bbox)
637 
638  def applyAltMaskPlanes(self, mask, altMaskSpans):
639  """!
640  \brief Apply in place alt mask formatted as SpanSets to a mask
641 
642  @param mask: original mask
643  @param altMaskSpans: Dictionary containing spanSet lists to apply.
644  Each element contains the new mask plane name
645  (e.g. "CLIPPED and/or "NO_DATA") as the key,
646  and list of SpanSets to apply to the mask
647  """
648  for plane, spanSetList in altMaskSpans.items():
649  maskClipValue = mask.addMaskPlane(plane)
650  for spanSet in spanSetList:
651  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
652  return mask
653 
654  def readBrightObjectMasks(self, dataRef):
655  """Returns None on failure"""
656  try:
657  return dataRef.get("brightObjectMask", immediate=True)
658  except Exception as e:
659  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
660  return None
661 
662  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
663  """Set the bright object masks
664 
665  exposure: Exposure under consideration
666  dataId: Data identifier dict for patch
667  brightObjectMasks: afwTable of bright objects to mask
668  """
669  #
670  # Check the metadata specifying the tract/patch/filter
671  #
672  if brightObjectMasks is None:
673  self.log.warn("Unable to apply bright object mask: none supplied")
674  return
675  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
676  md = brightObjectMasks.table.getMetadata()
677  for k in dataId:
678  if not md.exists(k):
679  self.log.warn("Expected to see %s in metadata", k)
680  else:
681  if md.get(k) != dataId[k]:
682  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
683 
684  mask = exposure.getMaskedImage().getMask()
685  wcs = exposure.getWcs()
686  plateScale = wcs.pixelScale().asArcseconds()
687 
688  for rec in brightObjectMasks:
689  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
690  if rec["type"] == "box":
691  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
692  width = rec["width"].asArcseconds()/plateScale # convert to pixels
693  height = rec["height"].asArcseconds()/plateScale # convert to pixels
694 
695  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
696  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
697 
698  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
699  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
700  spans = afwGeom.SpanSet(bbox)
701  elif rec["type"] == "circle":
702  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
703  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
704  else:
705  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
706  continue
707  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
708 
709  def setEdge(self, mask):
710  """Set EDGE bits as SENSOR_EDGE
711 
712  The EDGE pixels from the individual CCDs have printed through into the
713  coadd, but EDGE means "we couldn't search for sources in this area
714  because we couldn't convolve by the PSF near the edge of the image",
715  so this mask plane needs to be converted to something else if we want
716  to keep them. We do want to be able to identify pixels near the edge
717  of the detector because they will have an inexact `CoaddPsf`. We
718  rename EDGE pixels as SENSOR_EDGE.
719 
720  Parameters
721  ----------
722  mask : `lsst.afw.image.Mask`
723  Coadded exposure's mask, modified in-place.
724  """
725  mask.addMaskPlane("SENSOR_EDGE")
726  edge = mask.getPlaneBitMask("EDGE")
727  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE")
728  array = mask.getArray()
729  selected = (array & edge > 0)
730  array[selected] |= sensorEdge
731  array[selected] &= ~edge
732 
733  def setInexactPsf(self, mask):
734  """Set INEXACT_PSF mask plane
735 
736  If any of the input images isn't represented in the coadd (due to
737  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
738  these pixels.
739 
740  Parameters
741  ----------
742  mask : `lsst.afw.image.Mask`
743  Coadded exposure's mask, modified in-place.
744  """
745  mask.addMaskPlane("INEXACT_PSF")
746  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
747  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip gaps
748  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
749  array = mask.getArray()
750  selected = array & (sensorEdge | clipped) > 0
751  array[selected] |= inexactPsf
752 
753  @classmethod
754  def _makeArgumentParser(cls):
755  """!
756  \brief Create an argument parser
757  """
758  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
759  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
760  cls.ConfigClass().warpType + "Warp",
761  help="data ID, e.g. --id tract=12345 patch=1,2",
762  ContainerClass=AssembleCoaddDataIdContainer)
763  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
764  ContainerClass=SelectDataIdContainer)
765  return parser
766 
767 
768 def _subBBoxIter(bbox, subregionSize):
769  """!
770  \brief Iterate over subregions of a bbox
771 
772  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
773  \param[in] subregionSize: size of sub-bboxes
774 
775  \return subBBox: next sub-bounding box of size subregionSize or smaller;
776  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
777  but it will never be empty
778  """
779  if bbox.isEmpty():
780  raise RuntimeError("bbox %s is empty" % (bbox,))
781  if subregionSize[0] < 1 or subregionSize[1] < 1:
782  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
783 
784  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
785  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
786  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
787  subBBox.clip(bbox)
788  if subBBox.isEmpty():
789  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
790  (bbox, subregionSize, colShift, rowShift))
791  yield subBBox
792 
793 
794 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
795  """!
796  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
797  """
798 
799  def makeDataRefList(self, namespace):
800  """!
801  \brief Make self.refList from self.idList.
802  """
803  datasetType = namespace.config.coaddName + "Coadd"
804  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
805 
806  for dataId in self.idList:
807  # tract and patch are required
808  for key in keysCoadd:
809  if key not in dataId:
810  raise RuntimeError("--id must include " + key)
811 
812  dataRef = namespace.butler.dataRef(
813  datasetType=datasetType,
814  dataId=dataId,
815  )
816  self.refList.append(dataRef)
817 
818 
819 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
820  """!
821  \brief Function to count the number of pixels with a specific mask in a footprint.
822 
823  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
824  have bitmask set but do not have ignoreMask set. Return the count.
825 
826  \param[in] mask: mask to define intersection region by.
827  \parma[in] footprint: footprint to define the intersection region by.
828  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
829  \param[in] ignoreMask: pixels to not consider.
830  \return count of number of pixels in footprint with specified mask.
831  """
832  bbox = footprint.getBBox()
833  bbox.clip(mask.getBBox(afwImage.PARENT))
834  fp = afwImage.Mask(bbox)
835  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
836  footprint.spans.setMask(fp, bitmask)
837  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
838  (subMask.getArray() & ignoreMask) == 0).sum()
839 
840 
842  """!
843 \anchor SafeClipAssembleCoaddConfig
844 
845 \brief Configuration parameters for the SafeClipAssembleCoaddTask
846  """
847  clipDetection = pexConfig.ConfigurableField(
848  target=SourceDetectionTask,
849  doc="Detect sources on difference between unclipped and clipped coadd")
850  minClipFootOverlap = pexConfig.Field(
851  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
852  dtype=float,
853  default=0.6
854  )
855  minClipFootOverlapSingle = pexConfig.Field(
856  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
857  "clipped when only one visit overlaps",
858  dtype=float,
859  default=0.5
860  )
861  minClipFootOverlapDouble = pexConfig.Field(
862  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
863  "clipped when two visits overlap",
864  dtype=float,
865  default=0.45
866  )
867  maxClipFootOverlapDouble = pexConfig.Field(
868  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
869  "considering two visits",
870  dtype=float,
871  default=0.15
872  )
873  minBigOverlap = pexConfig.Field(
874  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
875  "when labeling clipped footprints",
876  dtype=int,
877  default=100
878  )
879 
880  def setDefaults(self):
881  # The numeric values for these configuration parameters were empirically determined, future work
882  # may further refine them.
883  AssembleCoaddConfig.setDefaults(self)
884  self.clipDetection.doTempLocalBackground = False
885  self.clipDetection.reEstimateBackground = False
886  self.clipDetection.returnOriginalFootprints = False
887  self.clipDetection.thresholdPolarity = "both"
888  self.clipDetection.thresholdValue = 2
889  self.clipDetection.nSigmaToGrow = 2
890  self.clipDetection.minPixels = 4
891  self.clipDetection.isotropicGrow = True
892  self.clipDetection.thresholdType = "pixel_stdev"
893  self.sigmaClip = 1.5
894  self.clipIter = 3
895  self.statistic = "MEAN"
896 
897  def validate(self):
898  if self.doSigmaClip:
899  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
900  "Ignoring doSigmaClip.")
901  self.doSigmaClip = False
902  if self.statistic != "MEAN":
903  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
904  "(%s chosen). Please set statistic to MEAN."
905  % (self.statistic))
906  AssembleCoaddTask.ConfigClass.validate(self)
907 
908 
909 
915 
916 
918  """!
919  \anchor SafeClipAssembleCoaddTask_
920 
921  \brief Assemble a coadded image from a set of coadded temporary exposures,
922  being careful to clip & flag areas with potential artifacts.
923 
924  \section pipe_tasks_assembleCoadd_Contents Contents
925  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
926  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
927  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
928  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
929  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
930  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
931 
932  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
933 
934  \copybrief SafeClipAssembleCoaddTask
935 
936  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
937  SafeClipAssembleCoaddTask subtasks that task.
938  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
939  outliers).
940  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
941  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
942  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
943  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
944  that there is an outlier and ii. this outlier appears on only one or two images.
945  Such regions will not contribute to the final coadd.
946  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
947  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
948  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
949  surveys.
950 
951  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
952  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
953  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
954 
955  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
956  \copydoc \_\_init\_\_
957 
958  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
959  \copydoc run
960 
961  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
962  See \ref SafeClipAssembleCoaddConfig
963 
964  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
965  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
966  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
967  files.
968  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
969  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
970  for further information.
971 
972  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
973  SafeClipAssembleCoaddTask
974 
975  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
976  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
977  '--legacyCoadd'.
978  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
979  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
980  with a list of coaddTempExps to attempt to coadd (specified using
981  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
982  Only the coaddTempExps that cover the specified tract and patch will be coadded.
983  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
984  command line argument:
985  \code
986  assembleCoadd.py --help
987  \endcode
988  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
989  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
990  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
991  packages.
992  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
993  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
994  <DL>
995  <DT>processCcd</DT>
996  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
997  <DT>makeSkyMap</DT>
998  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
999  <DT>makeCoaddTempExp</DT>
1000  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1001  </DL>
1002  We can perform all of these steps by running
1003  \code
1004  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1005  \endcode
1006  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1007  as follows:
1008  \code
1009  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1010  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1011  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1012  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1013  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1014  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1015  --selectId visit=903988 ccd=24
1016  \endcode
1017  This will process the HSC-I band data. The results are written in
1018  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1019 
1020  You may also choose to run:
1021  \code
1022  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1023  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1024  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1025  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1026  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1027  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1028  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1029  --selectId visit=903346 ccd=12
1030  \endcode
1031  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1032  discussed in \ref pipeTasks_multiBand.
1033  """
1034  ConfigClass = SafeClipAssembleCoaddConfig
1035  _DefaultName = "safeClipAssembleCoadd"
1036 
1037  def __init__(self, *args, **kwargs):
1038  """!
1039  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1040  """
1041  AssembleCoaddTask.__init__(self, *args, **kwargs)
1042  schema = afwTable.SourceTable.makeMinimalSchema()
1043  self.makeSubtask("clipDetection", schema=schema)
1044 
1045  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1046  """!
1047  \brief Assemble the coadd for a region
1048 
1049  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1050  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1051  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1052  overlap between the clipped region and a source.
1053  This leaves us with a set of footprints from the difference image that have been identified as having
1054  occured on just one or two individual visits. However, these footprints were generated from a
1055  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1056  footprints acrosss the coadd difference in this process.
1057  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1058  these are big footprints.
1059  Combine the small and big clipped footprints and mark them on a new bad mask plane
1060  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1061  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1062  bad mask plane.
1063 
1064  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1065  parent task.
1066 
1067  @param skyInfo: Patch geometry information, from getSkyInfo
1068  @param tempExpRefList: List of data reference to tempExp
1069  @param imageScalerList: List of image scalers
1070  @param weightList: List of weights
1071  return pipeBase.Struct with coaddExposure, nImage
1072  """
1073  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1074  mask = exp.getMaskedImage().getMask()
1075  mask.addMaskPlane("CLIPPED")
1076 
1077  result = self.detectClip(exp, tempExpRefList)
1078 
1079  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1080 
1081  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1082  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1083  # Append big footprints from individual Warps to result.clipSpans
1084  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1085  result.detectionFootprints, maskClipValue, maskDetValue,
1086  exp.getBBox())
1087  # Create mask of the current clipped footprints
1088  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1089  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1090 
1091  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1092  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1093  maskClip |= maskClipBig
1094 
1095  # Assemble coadd from base class, but ignoring CLIPPED pixels
1096  badMaskPlanes = self.config.badMaskPlanes[:]
1097  badMaskPlanes.append("CLIPPED")
1098  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1099  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1100  result.clipSpans, mask=badPixelMask)
1101 
1102  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1103  """!
1104  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1105 
1106  Generate a difference image between clipped and unclipped coadds.
1107  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1108  Return the difference image.
1109 
1110  @param skyInfo: Patch geometry information, from getSkyInfo
1111  @param tempExpRefList: List of data reference to tempExp
1112  @param imageScalerList: List of image scalers
1113  @param weightList: List of weights
1114  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1115  """
1116  # Clone and upcast self.config because current self.config is frozen
1117  config = AssembleCoaddConfig()
1118  # getattr necessary because subtasks do not survive Config.toDict()
1119  configIntersection = {k: getattr(self.config, k)
1120  for k, v in self.config.toDict().items() if (k in config.keys())}
1121  config.update(**configIntersection)
1122 
1123  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1124  config.statistic = 'MEAN'
1125  task = AssembleCoaddTask(config=config)
1126  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1127 
1128  config.statistic = 'MEANCLIP'
1129  task = AssembleCoaddTask(config=config)
1130  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1131 
1132  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1133  coaddDiff -= coaddClip.getMaskedImage()
1134  exp = afwImage.ExposureF(coaddDiff)
1135  exp.setPsf(coaddMean.getPsf())
1136  return exp
1137 
1138  def detectClip(self, exp, tempExpRefList):
1139  """!
1140  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1141 
1142  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1143  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1144  fraction to thresholds set in the config.
1145  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1146  two).
1147  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1148  the coaddTempExp.
1149  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1150  with the clipped footprints and a list of new masks for the coaddTempExps.
1151 
1152  \param[in] exp: Exposure to run detection on
1153  \param[in] tempExpRefList: List of data reference to tempExp
1154  \return struct containing:
1155  - clipFootprints: list of clipped footprints
1156  - clipIndices: indices for each clippedFootprint in tempExpRefList
1157  - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new
1158  maskplane name ("CLIPPED")" as the key and list of SpanSets as value
1159  - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints
1160  """
1161  mask = exp.getMaskedImage().getMask()
1162  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1163  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1164  # Merge positive and negative together footprints together
1165  fpSet.positive.merge(fpSet.negative)
1166  footprints = fpSet.positive
1167  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1168  ignoreMask = self.getBadPixelMask()
1169 
1170  clipFootprints = []
1171  clipIndices = []
1172  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1173 
1174  # for use by detectClipBig
1175  visitDetectionFootprints = []
1176 
1177  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1178  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1179  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1180 
1181  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1182  for i, warpRef in enumerate(tempExpRefList):
1183  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1184  immediate=True).getMaskedImage().getMask()
1185  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1186  afwImage.PARENT, True)
1187  maskVisitDet &= maskDetValue
1188  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1189  visitDetectionFootprints.append(visitFootprints)
1190 
1191  for j, footprint in enumerate(footprints.getFootprints()):
1192  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1193  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1194 
1195  # build a list of clipped spans for each visit
1196  for j, footprint in enumerate(footprints.getFootprints()):
1197  nPixel = footprint.getArea()
1198  overlap = [] # hold the overlap with each visit
1199  indexList = [] # index of visit in global list
1200  for i in range(len(tempExpRefList)):
1201  ignore = ignoreArr[i, j]
1202  overlapDet = overlapDetArr[i, j]
1203  totPixel = nPixel - ignore
1204 
1205  # If we have more bad pixels than detection skip
1206  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1207  continue
1208  overlap.append(overlapDet/float(totPixel))
1209  indexList.append(i)
1210 
1211  overlap = numpy.array(overlap)
1212  if not len(overlap):
1213  continue
1214 
1215  keep = False # Should this footprint be marked as clipped?
1216  keepIndex = [] # Which tempExps does the clipped footprint belong to
1217 
1218  # If footprint only has one overlap use a lower threshold
1219  if len(overlap) == 1:
1220  if overlap[0] > self.config.minClipFootOverlapSingle:
1221  keep = True
1222  keepIndex = [0]
1223  else:
1224  # This is the general case where only visit should be clipped
1225  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1226  if len(clipIndex) == 1:
1227  keep = True
1228  keepIndex = [clipIndex[0]]
1229 
1230  # Test if there are clipped objects that overlap two different visits
1231  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1232  if len(clipIndex) == 2 and len(overlap) > 3:
1233  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1234  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1235  keep = True
1236  keepIndex = clipIndex
1237 
1238  if not keep:
1239  continue
1240 
1241  for index in keepIndex:
1242  globalIndex = indexList[index]
1243  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1244 
1245  clipIndices.append(numpy.array(indexList)[keepIndex])
1246  clipFootprints.append(footprint)
1247 
1248  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1249  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1250 
1251  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1252  maskClipValue, maskDetValue, coaddBBox):
1253  """!
1254  \brief Return individual warp footprints for large artifacts and append them to clipList in place
1255 
1256  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1257  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1258  significantly with each source in all the coaddTempExps.
1259  \param[in] clipList: List of alt mask SpanSets with clipping information. Modified.
1260  \param[in] clipFootprints: List of clipped footprints
1261  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1262  \param[in] maskClipValue: Mask value of clipped pixels
1263  \param[in] maskDetValue: Mask value of detected pixels
1264  \param[in] coaddBBox: BBox of the coadd and warps
1265  \return list of big footprints
1266  """
1267  bigFootprintsCoadd = []
1268  ignoreMask = self.getBadPixelMask()
1269  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1270  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1271  for footprint in visitFootprints.getFootprints():
1272  footprint.spans.setMask(maskVisitDet, maskDetValue)
1273 
1274  # build a mask of clipped footprints that are in this visit
1275  clippedFootprintsVisit = []
1276  for foot, clipIndex in zip(clipFootprints, clipIndices):
1277  if index not in clipIndex:
1278  continue
1279  clippedFootprintsVisit.append(foot)
1280  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1281  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1282 
1283  bigFootprintsVisit = []
1284  for foot in visitFootprints.getFootprints():
1285  if foot.getArea() < self.config.minBigOverlap:
1286  continue
1287  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1288  if nCount > self.config.minBigOverlap:
1289  bigFootprintsVisit.append(foot)
1290  bigFootprintsCoadd.append(foot)
1291 
1292  for footprint in bigFootprintsVisit:
1293  clippedSpans["CLIPPED"].append(footprint.spans)
1294 
1295  return bigFootprintsCoadd
1296 
1297 
1299  assembleStaticSkyModel = pexConfig.ConfigurableField(
1300  target=AssembleCoaddTask,
1301  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1302  " naive/first-iteration model of the static sky.",
1303  )
1304  detect = pexConfig.ConfigurableField(
1305  target=SourceDetectionTask,
1306  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1307  )
1308  maxNumEpochs = pexConfig.Field(
1309  doc="Maximum number of epochs/visits in which an artifact candidate can appear and still be masked. "
1310  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1311  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1312  "than maxNumEpochs, the artifact candidate is persistant rather than transient and not masked.",
1313  dtype=int,
1314  default=2
1315  )
1316  maxFractionEpochs = pexConfig.RangeField(
1317  doc="Fraction of local number of epochs (N) to use as maxNumEpochs. "
1318  "Effective maxNumEpochs is the lesser of floor(maxNumEpochsMinFraction*N) and maxNumEpochs. ",
1319  dtype=float,
1320  default=0.4,
1321  min=0., max=1.,
1322  )
1323  spatialThreshold = pexConfig.RangeField(
1324  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1325  "temporal criteria. If 0, clip all. If 1, clip none.",
1326  dtype=float,
1327  default=0.5,
1328  min=0., max=1.,
1329  inclusiveMin=True, inclusiveMax=True
1330  )
1331  doScaleWarpVariance = pexConfig.Field(
1332  doc="Rescale Warp variance plane using empirical noise?",
1333  dtype=bool,
1334  default=True,
1335  )
1336  maskScaleWarpVariance = pexConfig.ListField(
1337  dtype=str,
1338  default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
1339  doc="Mask planes for pixels to ignore when rescaling warp variance",
1340  )
1341 
1342  def setDefaults(self):
1343  AssembleCoaddConfig.setDefaults(self)
1344  self.statistic = 'MEAN'
1345  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1346  self.assembleStaticSkyModel.warpType = 'psfMatched'
1347  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1348  self.assembleStaticSkyModel.sigmaClip = 1.5
1349  self.assembleStaticSkyModel.clipIter = 3
1350  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1351  self.assembleStaticSkyModel.doWrite = False
1352  self.detect.doTempLocalBackground = False
1353  self.detect.reEstimateBackground = False
1354  self.detect.returnOriginalFootprints = False
1355  self.detect.thresholdPolarity = "both"
1356  self.detect.thresholdValue = 5
1357  self.detect.nSigmaToGrow = 2
1358  self.detect.minPixels = 4
1359  self.detect.isotropicGrow = True
1360  self.detect.thresholdType = "pixel_stdev"
1361 
1362 
1363 
1369 
1371  """!
1372  \anchor CompareWarpAssembleCoaddTask_
1373 
1374  \brief Assemble a compareWarp coadded image from a set of warps
1375  by masking artifacts detected by comparing PSF-matched warps
1376 
1377  \section pipe_tasks_assembleCoadd_Contents Contents
1378  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1379  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1380  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1381  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1382  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1383  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1384 
1385  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1386 
1387  \copybrief CompareWarpAssembleCoaddTask
1388 
1389  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1390  outliers).
1391  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1392  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1393  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1394  pixels in the individual warps suspected to contain an artifact.
1395  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1396  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1397  template coadd by more than config.detect.threshold sigma, is an artifact candidate.
1398  The candidates are then filtered to remove variable sources and sources that are difficult to subtract
1399  such as bright stars.
1400  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1401  The temporalThreshold is the maximum fraction of epochs that the deviation can
1402  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1403  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1404  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1405  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask.
1406  These regions will not contribute to the final coadd.
1407  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1408  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1409  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1410  surveys.
1411 
1412  CompareWarpAssembleCoaddTask sub-classes
1413  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1414  as a subtask to generate the TemplateCoadd (the model of the static sky)
1415 
1416  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1417  \copydoc \_\_init\_\_
1418 
1419  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1420  \copydoc run
1421 
1422  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1423  See \ref CompareWarpAssembleCoaddConfig
1424 
1425  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1426  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1427  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1428  files.
1429 
1430  This task supports the following debug variables:
1431  <dl>
1432  <dt>`saveCountIm`
1433  <dd> If True then save the Epoch Count Image as a fits file in the `figPath`
1434  <dt> `figPath`
1435  <dd> Path to save the debug fits images and figures
1436  </dl>
1437 
1438  For example, put something like:
1439  @code{.py}
1440  import lsstDebug
1441  def DebugInfo(name):
1442  di = lsstDebug.getInfo(name)
1443  if name == "lsst.pipe.tasks.assembleCoadd":
1444  di.saveCountIm = True
1445  di.figPath = "/desired/path/to/debugging/output/images"
1446  return di
1447  lsstDebug.Info = DebugInfo
1448  @endcode
1449  into your `debug.py` file and run `assemebleCoadd.py` with the `--debug`
1450  flag.
1451  Some subtasks may have their own debug variables; see individual Task
1452  documentation
1453 
1454  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1455  CompareWarpAssembleCoaddTask
1456 
1457  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1458  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1459  '--compareWarpCoadd'.
1460  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1461  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1462  with a list of coaddTempExps to attempt to coadd (specified using
1463  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1464  Only the warps that cover the specified tract and patch will be coadded.
1465  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1466  command line argument:
1467  \code
1468  assembleCoadd.py --help
1469  \endcode
1470  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1471  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1472  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1473  packages.
1474  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1475  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1476  <DL>
1477  <DT>processCcd</DT>
1478  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1479  <DT>makeSkyMap</DT>
1480  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1481  <DT>makeCoaddTempExp</DT>
1482  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1483  </DL>
1484  We can perform all of these steps by running
1485  \code
1486  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1487  \endcode
1488  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1489  as follows:
1490  \code
1491  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1492  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1493  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1494  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1495  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1496  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1497  --selectId visit=903988 ccd=24
1498  \endcode
1499  This will process the HSC-I band data. The results are written in
1500  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1501  """
1502  ConfigClass = CompareWarpAssembleCoaddConfig
1503  _DefaultName = "compareWarpAssembleCoadd"
1504 
1505  def __init__(self, *args, **kwargs):
1506  """!
1507  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1508  """
1509  AssembleCoaddTask.__init__(self, *args, **kwargs)
1510  self.makeSubtask("assembleStaticSkyModel")
1511  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1512  self.makeSubtask("detect", schema=detectionSchema)
1513 
1514  def makeSupplementaryData(self, dataRef, selectDataList):
1515  """!
1516  \brief Make inputs specific to Subclass
1517 
1518  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1519  """
1520  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1521 
1522  if templateCoadd is None:
1523  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1524  self.assembleStaticSkyModel.warpType[1:])
1525  message = """No %(warpName)s warps were found to build the template coadd which is
1526  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1527  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1528  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1529 
1530  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1531  another algorithm like:
1532 
1533  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1534  config.assemble.retarget(SafeClipAssembleCoaddTask)
1535  """ % {"warpName": warpName}
1536  raise RuntimeError(message)
1537 
1538  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1539 
1540  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1541  supplementaryData, *args, **kwargs):
1542  """!
1543  \brief Assemble the coadd
1544 
1545  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1546  as the model of the static sky.
1547 
1548  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1549  new "CLIPPED" plane and updated "NO_DATA" plane.
1550  Then pass these alternative masks to the base class's assemble method.
1551 
1552  @param skyInfo: Patch geometry information
1553  @param tempExpRefList: List of data references to warps
1554  @param imageScalerList: List of image scalers
1555  @param weightList: List of weights
1556  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1557 
1558  return pipeBase.Struct with coaddExposure, nImage if requested
1559  """
1560  templateCoadd = supplementaryData.templateCoadd
1561  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1562  badMaskPlanes = self.config.badMaskPlanes[:]
1563  badMaskPlanes.append("CLIPPED")
1564  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1565 
1566  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1567  spanSetMaskList, mask=badPixelMask)
1568 
1569  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1570  """!
1571  \brief Find artifacts
1572 
1573  Loop through warps twice. The first loop builds a map with the count of how many
1574  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1575  The second loop takes each difference image and filters the artifacts detected
1576  in each using count map to filter out variable sources and sources that are difficult to
1577  subtract cleanly.
1578 
1579  @param templateCoadd: Exposure to serve as model of static sky
1580  @param tempExpRefList: List of data references to warps
1581  @param imageScalerList: List of image scalers
1582  """
1583 
1584  self.log.debug("Generating Count Image, and mask lists.")
1585  coaddBBox = templateCoadd.getBBox()
1586  slateIm = afwImage.ImageU(coaddBBox)
1587  epochCountImage = afwImage.ImageU(coaddBBox)
1588  nImage = afwImage.ImageU(coaddBBox)
1589  spanSetArtifactList = []
1590  spanSetNoDataMaskList = []
1591 
1592  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1593  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1594  if warpDiffExp is not None:
1595  nImage.array += numpy.where(numpy.isnan(warpDiffExp.image.array),
1596  0, 1).astype(numpy.uint16)
1597  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1598  fpSet.positive.merge(fpSet.negative)
1599  footprints = fpSet.positive
1600  slateIm.set(0)
1601  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1602  for spans in spanSetList:
1603  spans.setImage(slateIm, 1, doClip=True)
1604  epochCountImage += slateIm
1605 
1606  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1607  # undergo a second convolution. Pixels with data in the direct warp
1608  # but not in the PSF-matched warp will not have their artifacts detected.
1609  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1610  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1611  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1612  nansMask.setXY0(warpDiffExp.getXY0())
1613  else:
1614  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1615  # In this case, mask the whole epoch
1616  nansMask = afwImage.MaskX(coaddBBox, 1)
1617  spanSetList = []
1618 
1619  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1620 
1621  spanSetNoDataMaskList.append(spanSetNoDataMask)
1622  spanSetArtifactList.append(spanSetList)
1623 
1624  if lsstDebug.Info(__name__).saveCountIm:
1625  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1626  epochCountImage.writeFits(path)
1627 
1628  for i, spanSetList in enumerate(spanSetArtifactList):
1629  if spanSetList:
1630  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage, nImage)
1631  spanSetArtifactList[i] = filteredSpanSetList
1632 
1633  altMasks = []
1634  for artifacts, noData in zip(spanSetArtifactList, spanSetNoDataMaskList):
1635  altMasks.append({'CLIPPED': artifacts,
1636  'NO_DATA': noData})
1637  return altMasks
1638 
1639  def _filterArtifacts(self, spanSetList, epochCountImage, nImage):
1640  """!
1641  \brief Filter artifact candidates
1642 
1643  @param spanSetList: List of SpanSets representing artifact candidates
1644  @param epochCountImage: Image of accumulated number of warpDiff detections
1645  @param nImage: Image of the accumulated number of total epochs contributing
1646 
1647  return List of SpanSets with artifacts
1648  """
1649 
1650  maskSpanSetList = []
1651  x0, y0 = epochCountImage.getXY0()
1652  for i, span in enumerate(spanSetList):
1653  y, x = span.indices()
1654  yIdxLocal = [y1 - y0 for y1 in y]
1655  xIdxLocal = [x1 - x0 for x1 in x]
1656  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1657  totalN = nImage.array[yIdxLocal, xIdxLocal]
1658  effectiveMaxNumEpochs = min(self.config.maxNumEpochs,
1659  int(self.config.maxFractionEpochs * numpy.mean(totalN)))
1660  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1661  (outlierN <= effectiveMaxNumEpochs))
1662  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1663  if percentBelowThreshold > self.config.spatialThreshold:
1664  maskSpanSetList.append(span)
1665  return maskSpanSetList
1666 
1667  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1668  """!
1669  \brief Fetch a warp from the butler and return a warpDiff
1670 
1671  @param warpRef: `Butler dataRef` for the warp
1672  @param imageScaler: `scaleZeroPoint.ImageScaler` object
1673  @param templateCoadd: Exposure to be substracted from the scaled warp
1674 
1675  return Exposure of the image difference between the warp and template
1676  """
1677 
1678  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1679  warpName = self.getTempExpDatasetName('psfMatched')
1680  if not warpRef.datasetExists(warpName):
1681  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1682  return None
1683  warp = warpRef.get(warpName, immediate=True)
1684  # direct image scaler OK for PSF-matched Warp
1685  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1686  mi = warp.getMaskedImage()
1687  if self.config.doScaleWarpVariance:
1688  scaleVariance(mi, self.config.maskScaleWarpVariance,
1689  log=self.log)
1690  mi -= templateCoadd.getMaskedImage()
1691  return warp
1692 
1693  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1694  """!
1695  \brief Return a path to which to write debugging output
1696 
1697  @param prefix: string, prefix for filename
1698  @param warpRef: Butler dataRef
1699  @param coaddLevel: bool, optional. If True, include only coadd-level keys
1700  (e.g. 'tract', 'patch', 'filter', but no 'visit')
1701 
1702  Creates a hyphen-delimited string of dataId values for simple filenames.
1703  """
1704  if coaddLevel:
1705  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
1706  else:
1707  keys = warpRef.dataId.keys()
1708  keyList = sorted(keys, reverse=True)
1709  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
1710  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
1711  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def _filterArtifacts(self, spanSetList, epochCountImage, nImage)
Filter artifact candidates.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...