lsst.pipe.tasks  13.0-64-gd461ca2f+3
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import numpy
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 import lsst.afw.detection as afwDet
33 import lsst.coadd.utils as coaddUtils
34 import lsst.pipe.base as pipeBase
35 import lsst.meas.algorithms as measAlg
36 import lsst.log as log
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage import InterpImageTask
39 from .matchBackgrounds import MatchBackgroundsTask
40 from .scaleZeroPoint import ScaleZeroPointTask
41 from .coaddHelpers import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms import SourceDetectionTask
43 
44 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """!
49 \anchor AssembleCoaddConfig_
50 
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
52  """
53  warpType = pexConfig.Field(
54  doc="Warp name: one of 'direct' or 'psfMatched'",
55  dtype=str,
56  default="direct",
57  )
58  subregionSize = pexConfig.ListField(
59  dtype=int,
60  doc="Width, height of stack subregion size; "
61  "make small enough that a full stack of images will fit into memory at once.",
62  length=2,
63  default=(2000, 2000),
64  )
65  statistic = pexConfig.Field(
66  dtype=str,
67  doc="Main stacking statistic for aggregating over the epochs.",
68  default="MEANCLIP",
69  )
70  doSigmaClip = pexConfig.Field(
71  dtype=bool,
72  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
73  default=False,
74  )
75  sigmaClip = pexConfig.Field(
76  dtype=float,
77  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
78  default=3.0,
79  )
80  clipIter = pexConfig.Field(
81  dtype=int,
82  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
83  default=2,
84  )
85  scaleZeroPoint = pexConfig.ConfigurableField(
86  target=ScaleZeroPointTask,
87  doc="Task to adjust the photometric zero point of the coadd temp exposures",
88  )
89  doInterp = pexConfig.Field(
90  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
91  dtype=bool,
92  default=True,
93  )
94  interpImage = pexConfig.ConfigurableField(
95  target=InterpImageTask,
96  doc="Task to interpolate (and extrapolate) over NaN pixels",
97  )
98  matchBackgrounds = pexConfig.ConfigurableField(
99  target=MatchBackgroundsTask,
100  doc="Task to match backgrounds",
101  )
102  maxMatchResidualRatio = pexConfig.Field(
103  doc="Maximum ratio of the mean squared error of the background matching model to the variance "
104  "of the difference in backgrounds",
105  dtype=float,
106  default=1.1
107  )
108  maxMatchResidualRMS = pexConfig.Field(
109  doc="Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
110  dtype=float,
111  default=1.0
112  )
113  doWrite = pexConfig.Field(
114  doc="Persist coadd?",
115  dtype=bool,
116  default=True,
117  )
118  doNImage = pexConfig.Field(
119  doc="Create image of number of contributing exposures for each pixel",
120  dtype=bool,
121  default=False,
122  )
123  doMatchBackgrounds = pexConfig.Field(
124  doc="Match backgrounds of coadd temp exposures before coadding them? "
125  "If False, the coadd temp expsosures must already have been background subtracted or matched",
126  dtype=bool,
127  default=False,
128  )
129  autoReference = pexConfig.Field(
130  doc="Automatically select the coadd temp exposure to use as a reference for background matching? "
131  "Ignored if doMatchBackgrounds false. "
132  "If False you must specify the reference temp exposure as the data Id",
133  dtype=bool,
134  default=True,
135  )
136  maskPropagationThresholds = pexConfig.DictField(
137  keytype=str,
138  itemtype=float,
139  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
140  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
141  "would have contributed exceeds this value."),
142  default={"SAT": 0.1},
143  )
144  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["CROSSTALK", "NOT_DEBLENDED"],
145  doc="Mask planes to remove before coadding")
146  #
147  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
148  # To make this useful you *must* also configure the flags.pixel algorithm, for example
149  # by adding
150  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
151  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
152  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
153  #
154  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
155  doc="Set mask and flag bits for bright objects?")
156  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
157  doc="Name of mask bit used for bright objects")
158 
159  coaddPsf = pexConfig.ConfigField(
160  doc="Configuration for CoaddPsf",
161  dtype=measAlg.CoaddPsfConfig,
162  )
163 
164  def setDefaults(self):
165  CoaddBaseTask.ConfigClass.setDefaults(self)
166  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
167 
168  def validate(self):
169  CoaddBaseTask.ConfigClass.validate(self)
170  if self.doPsfMatch:
171  # Backwards compatibility.
172  # Configs do not have loggers
173  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
174  self.warpType = 'psfMatched'
175  if self.doSigmaClip and self.statistic != "MEANCLIP":
176  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
177  self.statistic = "MEANCLIP"
178  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
179  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
180  "compute and set a non-zero coadd variance estimate." % (self.statistic))
181 
182  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
183  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
184  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
185  if str(k) not in unstackableStats]
186  raise ValueError("statistic %s is not allowed. Please choose one of %s."
187  % (self.statistic, stackableStats))
188 # \addtogroup LSST_task_documentation
189 # \{
190 # \page AssembleCoaddTask
191 # \ref AssembleCoaddTask_ "AssembleCoaddTask"
192 # \copybrief AssembleCoaddTask
193 # \}
194 
196  """!
197 \anchor AssembleCoaddTask_
198 
199 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
200 
201 \section pipe_tasks_assembleCoadd_Contents Contents
202  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
203  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
204  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
205  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
206  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
207  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
208 
209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
210 
211 \copybrief AssembleCoaddTask_
212 
213 We want to assemble a coadded image from a set of Warps (also called
214 coadded temporary exposures or coaddTempExps.
215 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
216 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
217 Warps that cover the specified patch (pointed at by dataRef).
218 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
219 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
220 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
221 When background matching is enabled, the task may be configured to automatically select a reference exposure
222 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
223 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
224 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
225 config.makePsfMatched set which of the warp types will be coadded.
226 The coadd is computed as a mean with optional outlier rejection.
227 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
228 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
229 
230 AssembleCoaddTask uses several sub-tasks. These are
231 <DL>
232  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
233  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
234  <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
235  <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
236  not provided).</DD>
237  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
238  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
239 </DL>
240 You can retarget these subtasks if you wish.
241 
242 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
243 \copydoc \_\_init\_\_
244 
245 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
246 \copydoc run
247 
248 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
249 See \ref AssembleCoaddConfig_
250 
251 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
252 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
253 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
254 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
255 the documetation for the subtasks for further information.
256 
257 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
258 
259 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
260 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
261 a data reference to the tract patch and filter to be coadded (specified using
262 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
263 Warps to attempt to coadd (specified using
264 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
265 that cover the specified tract and patch will be coadded. A list of the available optional
266 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
267 \code
268 assembleCoadd.py --help
269 \endcode
270 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
271 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
272 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
273 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
274 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
275 <DL>
276  <DT>processCcd</DT>
277  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
278  <DT>makeSkyMap</DT>
279  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
280  <DT>makeCoaddTempExp</DT>
281  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
282 </DL>
283 We can perform all of these steps by running
284 \code
285 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
286 \endcode
287 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
288 follows:
289 \code
290 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
291 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
292 You may also choose to run:
293 \code
294 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
295 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
296 \endcode
297 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
298 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
299 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
300  """
301  ConfigClass = AssembleCoaddConfig
302  _DefaultName = "assembleCoadd"
303 
304  def __init__(self, *args, **kwargs):
305  """!
306  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
307  \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
308  """
309  CoaddBaseTask.__init__(self, *args, **kwargs)
310  self.makeSubtask("interpImage")
311  self.makeSubtask("matchBackgrounds")
312  self.makeSubtask("scaleZeroPoint")
313 
314  if self.config.doMaskBrightObjects:
315  mask = afwImage.Mask()
316  try:
317  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
318  except pexExceptions.LsstCppException:
319  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
320  mask.getMaskPlaneDict().keys())
321  del mask
322 
323  self.warpType = self.config.warpType
324 
325 
326  @pipeBase.timeMethod
327  def run(self, dataRef, selectDataList=[]):
328  """!
329  \brief Assemble a coadd from a set of Warps
330 
331  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
332  match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
333  Warps if the background has not already been removed. Assemble the Warps using
334  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
335  exposure.
336 
337  \anchor runParams
338  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
339  (if config.autoReference=False). Used to access the following data products:
340  - [in] self.config.coaddName + "Coadd_skyMap"
341  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
342  - [out] self.config.coaddName + "Coadd"
343  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
344  selected from this list based on overlap with the patch defined by dataRef.
345 
346  \return a pipeBase.Struct with fields:
347  - coaddExposure: coadded exposure
348  - nImage: exposure count image
349  """
350  skyInfo = self.getSkyInfo(dataRef)
351  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
352  if len(calExpRefList) == 0:
353  self.log.warn("No exposures to coadd")
354  return
355  self.log.info("Coadding %d exposures", len(calExpRefList))
356 
357  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
358  inputData = self.prepareInputs(tempExpRefList)
359  self.log.info("Found %d %s", len(inputData.tempExpRefList),
360  self.getTempExpDatasetName(self.warpType))
361  if len(inputData.tempExpRefList) == 0:
362  self.log.warn("No coadd temporary exposures found")
363  return
364  if self.config.doMatchBackgrounds:
365  refImageScaler = self.getBackgroundReferenceScaler(dataRef)
366  inputData = self.backgroundMatching(inputData, dataRef, refImageScaler)
367  if len(inputData.tempExpRefList) == 0:
368  self.log.warn("No valid background models")
369  return
370 
371  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
372  # nImage is created if it is requested by self.config. Otherwise, None.
373  retStruct = self.assemble(
374  skyInfo, inputData.tempExpRefList, inputData.imageScalerList, inputData.weightList,
375  inputData.backgroundInfoList if self.config.doMatchBackgrounds else None
376  )
377  if self.config.doMatchBackgrounds:
378  self.addBackgroundMatchingMetadata(retStruct.coaddExposure, inputData.tempExpRefList,
379  inputData.backgroundInfoList)
380 
381  if self.config.doInterp:
382  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
383  # The variance must be positive; work around for DM-3201.
384  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
385  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
386 
387  if self.config.doMaskBrightObjects:
388  brightObjectMasks = self.readBrightObjectMasks(dataRef)
389  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
390 
391  if self.config.doWrite:
392  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
393  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
394  if retStruct.nImage is not None:
395  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType)+'_nImage')
396 
397  return retStruct
398 
399  def makeSupplementaryData(self, dataRef, selectDataList):
400  """!
401  \brief Make additional inputs to assemble() specific to subclasses.
402 
403  Available to be implemented by subclasses only if they need the
404  coadd dataRef for performing preliminary processing before
405  assembling the coadd.
406  """
407  pass
408 
409  def getTempExpRefList(self, patchRef, calExpRefList):
410  """!
411  \brief Generate list data references corresponding to warped exposures that lie within the
412  patch to be coadded.
413 
414  \param[in] patchRef: Data reference for patch
415  \param[in] calExpRefList: List of data references for input calexps
416  \return List of Warp/CoaddTempExp data references
417  """
418  butler = patchRef.getButler()
419  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
420  self.getTempExpDatasetName(self.warpType))
421  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
422  g, groupData.keys) for
423  g in groupData.groups.keys()]
424  return tempExpRefList
425 
426  def getBackgroundReferenceScaler(self, dataRef):
427  """!
428  \brief Construct an image scaler for the background reference frame
429 
430  Each Warp has a different background level. A reference background level must be chosen before
431  coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
432  this routine is a no-op and None is returned. Otherwise, use the
433  \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
434  reference image and return it.
435 
436  \param[in] dataRef: Data reference for the background reference frame, or None
437  \return image scaler, or None
438  """
439  if self.config.autoReference:
440  return None
441 
442  # We've been given the data reference
443  dataset = self.getTempExpDatasetName(self.warpType)
444  if not dataRef.datasetExists(dataset):
445  raise RuntimeError("Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
446 
447  refExposure = dataRef.get(self.getTempExpDatasetName(self.warpType), immediate=True)
448  refImageScaler = self.scaleZeroPoint.computeImageScaler(
449  exposure=refExposure,
450  dataRef=dataRef,
451  )
452  return refImageScaler
453 
454  def prepareInputs(self, refList):
455  """!
456  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
457  for the photometric zero point.
458 
459  Each Warp has its own photometric zeropoint and background variance. Before coadding these
460  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
461  weight for each Warp.
462 
463  \param[in] refList: List of data references to tempExp
464  \return Struct:
465  - tempExprefList: List of data references to tempExp
466  - weightList: List of weightings
467  - imageScalerList: List of image scalers
468  """
469  statsCtrl = afwMath.StatisticsControl()
470  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
471  statsCtrl.setNumIter(self.config.clipIter)
472  statsCtrl.setAndMask(self.getBadPixelMask())
473  statsCtrl.setNanSafe(True)
474  # compute tempExpRefList: a list of tempExpRef that actually exist
475  # and weightList: a list of the weight of the associated coadd tempExp
476  # and imageScalerList: a list of scale factors for the associated coadd tempExp
477  tempExpRefList = []
478  weightList = []
479  imageScalerList = []
480  tempExpName = self.getTempExpDatasetName(self.warpType)
481  for tempExpRef in refList:
482  if not tempExpRef.datasetExists(tempExpName):
483  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
484  continue
485 
486  tempExp = tempExpRef.get(tempExpName, immediate=True)
487  maskedImage = tempExp.getMaskedImage()
488  imageScaler = self.scaleZeroPoint.computeImageScaler(
489  exposure=tempExp,
490  dataRef=tempExpRef,
491  )
492  try:
493  imageScaler.scaleMaskedImage(maskedImage)
494  except Exception as e:
495  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
496  continue
497  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
498  afwMath.MEANCLIP, statsCtrl)
499  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
500  weight = 1.0 / float(meanVar)
501  if not numpy.isfinite(weight):
502  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
503  continue
504  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
505 
506  del maskedImage
507  del tempExp
508 
509  tempExpRefList.append(tempExpRef)
510  weightList.append(weight)
511  imageScalerList.append(imageScaler)
512 
513  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
514  imageScalerList=imageScalerList)
515 
516  def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None):
517  """!
518  \brief Perform background matching on the prepared inputs
519 
520  Each Warp has a different background level that must be normalized to a reference level
521  before coaddition. If no reference is provided, the background matcher selects one. If the background
522  matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
523  consistent with the scaled background.
524 
525  \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
526  \param[in] refExpDataRef: Data reference for background reference Warp, or None
527  \param[in] refImageScaler: Image scaler for background reference Warp, or None
528  \return Struct:
529  - tempExprefList: List of data references to warped exposures (coaddTempExps)
530  - weightList: List of weightings
531  - imageScalerList: List of image scalers
532  - backgroundInfoList: result from background matching
533  """
534  try:
535  backgroundInfoList = self.matchBackgrounds.run(
536  expRefList=inputData.tempExpRefList,
537  imageScalerList=inputData.imageScalerList,
538  refExpDataRef=refExpDataRef if not self.config.autoReference else None,
539  refImageScaler=refImageScaler,
540  expDatasetType=self.getTempExpDatasetName(self.warpType),
541  ).backgroundInfoList
542  except Exception as e:
543  self.log.fatal("Cannot match backgrounds: %s", e)
544  raise pipeBase.TaskError("Background matching failed.")
545 
546  newWeightList = []
547  newTempExpRefList = []
548  newBackgroundStructList = []
549  newScaleList = []
550  # the number of good backgrounds may be < than len(tempExpList)
551  # sync these up and correct the weights
552  for tempExpRef, bgInfo, scaler, weight in zip(inputData.tempExpRefList, backgroundInfoList,
553  inputData.imageScalerList, inputData.weightList):
554  if not bgInfo.isReference:
555  # skip exposure if it has no backgroundModel
556  # or if fit was bad
557  if (bgInfo.backgroundModel is None):
558  self.log.info("No background offset model available for %s: skipping", tempExpRef.dataId)
559  continue
560  try:
561  varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
562  except Exception as e:
563  self.log.info("MSE/Var ratio not calculable (%s) for %s: skipping",
564  e, tempExpRef.dataId)
565  continue
566  if not numpy.isfinite(varianceRatio):
567  self.log.info("MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
568  bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
569  continue
570  elif (varianceRatio > self.config.maxMatchResidualRatio):
571  self.log.info("Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
572  varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
573  continue
574  elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
575  self.log.info("Bad fit. RMS %.2f > %.2f for %s: skipping",
576  bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
577  continue
578  newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
579  newTempExpRefList.append(tempExpRef)
580  newBackgroundStructList.append(bgInfo)
581  newScaleList.append(scaler)
582 
583  return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
584  imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
585 
586  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
587  altMaskList=None, mask=None, supplementaryData=None):
588  """!
589  \anchor AssembleCoaddTask.assemble_
590 
591  \brief Assemble a coadd from input warps
592 
593  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
594  large area), the assembly is performed over small areas on the image at a time in order to
595  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
596  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
597  statistic specified. Set the edge bits the coadd mask based on the weight map.
598 
599  \param[in] skyInfo: Patch geometry information, from getSkyInfo
600  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
601  \param[in] imageScalerList: List of image scalers
602  \param[in] weightList: List of weights
603  \param[in] bgInfoList: List of background data from background matching, or None
604  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
605  \param[in] mask: Mask to ignore when coadding
606  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
607  Only used by subclasses that implement makeSupplementaryData and override assemble.
608  \return pipeBase.Struct with coaddExposure, nImage if requested
609  """
610  tempExpName = self.getTempExpDatasetName(self.warpType)
611  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
612  if mask is None:
613  mask = self.getBadPixelMask()
614 
615  statsCtrl = afwMath.StatisticsControl()
616  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
617  statsCtrl.setNumIter(self.config.clipIter)
618  statsCtrl.setAndMask(mask)
619  statsCtrl.setNanSafe(True)
620  statsCtrl.setWeighted(True)
621  statsCtrl.setCalcErrorFromInputVariance(True)
622  for plane, threshold in self.config.maskPropagationThresholds.items():
623  bit = afwImage.Mask.getMaskPlane(plane)
624  statsCtrl.setMaskPropagationThreshold(bit, threshold)
625 
626  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
627 
628  if bgInfoList is None:
629  bgInfoList = [None]*len(tempExpRefList)
630 
631  if altMaskList is None:
632  altMaskList = [None]*len(tempExpRefList)
633 
634  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
635  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
636  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
637  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
638  coaddMaskedImage = coaddExposure.getMaskedImage()
639  subregionSizeArr = self.config.subregionSize
640  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
641  # if nImage is requested, create a zero one which can be passed to assembleSubregion
642  if self.config.doNImage:
643  nImage = afwImage.ImageU(skyInfo.bbox)
644  else:
645  nImage = None
646  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
647  try:
648  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
649  weightList, bgInfoList, altMaskList, statsFlags, statsCtrl,
650  nImage=nImage)
651  except Exception as e:
652  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
653 
654  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
655  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
656 
657  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
658  """!
659  \brief Set the metadata for the coadd
660 
661  This basic implementation simply sets the filter from the
662  first input.
663 
664  \param[in] coaddExposure: The target image for the coadd
665  \param[in] tempExpRefList: List of data references to tempExp
666  \param[in] weightList: List of weights
667  """
668  assert len(tempExpRefList) == len(weightList), "Length mismatch"
669  tempExpName = self.getTempExpDatasetName(self.warpType)
670  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
671  # (and we need more than just the PropertySet that contains the header), which is not possible
672  # with the current butler (see #2777).
673  tempExpList = [tempExpRef.get(tempExpName + "_sub",
674  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
675  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
676  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
677 
678  coaddExposure.setFilter(tempExpList[0].getFilter())
679  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
680  coaddInputs.ccds.reserve(numCcds)
681  coaddInputs.visits.reserve(len(tempExpList))
682 
683  for tempExp, weight in zip(tempExpList, weightList):
684  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
685  coaddInputs.visits.sort()
686  if self.warpType == "psfMatched":
687  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
688  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
689  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
690  # having the maximum width (sufficient because square)
691  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
692  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
693  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
694  else:
695  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
696  self.config.coaddPsf.makeControl())
697  coaddExposure.setPsf(psf)
698  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
699  coaddExposure.getWcs())
700  coaddExposure.getInfo().setApCorrMap(apCorrMap)
701 
702  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
703  bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None):
704  """!
705  \brief Assemble the coadd for a sub-region.
706 
707  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
708  matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
709  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
710  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
711  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
712  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
713  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
714 
715  \param[in] coaddExposure: The target image for the coadd
716  \param[in] bbox: Sub-region to coadd
717  \param[in] tempExpRefList: List of data reference to tempExp
718  \param[in] imageScalerList: List of image scalers
719  \param[in] weightList: List of weights
720  \param[in] bgInfoList: List of background data from background matching
721  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
722  \param[in] statsFlags: afwMath.Property object for statistic for coadd
723  \param[in] statsCtrl: Statistics control object for coadd
724  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
725  """
726  self.log.debug("Computing coadd over %s", bbox)
727  tempExpName = self.getTempExpDatasetName(self.warpType)
728  coaddMaskedImage = coaddExposure.getMaskedImage()
729  maskedImageList = []
730  if nImage is not None:
731  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
732  for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
733  altMaskList):
734  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
735  maskedImage = exposure.getMaskedImage()
736  if altMask:
737  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
738  maskedImage.getMask().swap(altMaskSub)
739  imageScaler.scaleMaskedImage(maskedImage)
740 
741  if self.config.doMatchBackgrounds and not bgInfo.isReference:
742  backgroundModel = bgInfo.backgroundModel
743  backgroundImage = backgroundModel.getImage() if \
744  self.matchBackgrounds.config.usePolynomial else \
745  backgroundModel.getImageF()
746  backgroundImage.setXY0(coaddMaskedImage.getXY0())
747  maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
748  var = maskedImage.getVariance()
749  var += (bgInfo.fitRMS)**2
750  # Add 1 for each pixel which is not excluded by the exclude mask.
751  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
752  if nImage is not None:
753  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
754  if self.config.removeMaskPlanes:
755  mask = maskedImage.getMask()
756  for maskPlane in self.config.removeMaskPlanes:
757  try:
758  mask &= ~mask.getPlaneBitMask(maskPlane)
759  except Exception as e:
760  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.message)
761 
762  maskedImageList.append(maskedImage)
763 
764  with self.timer("stack"):
765  coaddSubregion = afwMath.statisticsStack(
766  maskedImageList, statsFlags, statsCtrl, weightList)
767  coaddMaskedImage.assign(coaddSubregion, bbox)
768  if nImage is not None:
769  nImage.assign(subNImage, bbox)
770 
771  def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList):
772  """!
773  \brief Add metadata from the background matching to the coadd
774 
775  \param[in] coaddExposure: Coadd
776  \param[in] tempExpRefList: List of data references for temp exps to go into coadd
777  \param[in] backgroundInfoList: List of background info, results from background matching
778  """
779  self.log.info("Adding exposure information to metadata")
780  metadata = coaddExposure.getMetadata()
781  metadata.addString("CTExp_SDQA1_DESCRIPTION",
782  "Background matching: Ratio of matchedMSE / diffImVar")
783  for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
784  tempExpStr = '&'.join('%s=%s' % (k, v) for k, v in tempExpRef.dataId.items())
785  if backgroundInfo.isReference:
786  metadata.addString("ReferenceExp_ID", tempExpStr)
787  else:
788  metadata.addString("CTExp_ID_%d" % (ind), tempExpStr)
789  metadata.addDouble("CTExp_SDQA1_%d" % (ind),
790  backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
791  metadata.addDouble("CTExp_SDQA2_%d" % (ind),
792  backgroundInfo.fitRMS)
793 
794  def readBrightObjectMasks(self, dataRef):
795  """Returns None on failure"""
796  try:
797  return dataRef.get("brightObjectMask", immediate=True)
798  except Exception as e:
799  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
800  return None
801 
802  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
803  """Set the bright object masks
804 
805  exposure: Exposure under consideration
806  dataId: Data identifier dict for patch
807  brightObjectMasks: afwTable of bright objects to mask
808  """
809  #
810  # Check the metadata specifying the tract/patch/filter
811  #
812  if brightObjectMasks is None:
813  self.log.warn("Unable to apply bright object mask: none supplied")
814  return
815  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
816  md = brightObjectMasks.table.getMetadata()
817  for k in dataId:
818  if not md.exists(k):
819  self.log.warn("Expected to see %s in metadata", k)
820  else:
821  if md.get(k) != dataId[k]:
822  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
823 
824  mask = exposure.getMaskedImage().getMask()
825  wcs = exposure.getWcs()
826  plateScale = wcs.pixelScale().asArcseconds()
827 
828  for rec in brightObjectMasks:
829  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
830  if rec["type"] == "box":
831  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
832  width = rec["width"].asArcseconds()/plateScale # convert to pixels
833  height = rec["height"].asArcseconds()/plateScale # convert to pixels
834 
835  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
836  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
837 
838  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
839  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
840  spans = afwGeom.SpanSet(bbox)
841  elif rec["type"] == "circle":
842  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
843  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
844  else:
845  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
846  continue
847  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
848 
849  @classmethod
850  def _makeArgumentParser(cls):
851  """!
852  \brief Create an argument parser
853  """
854  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
855  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
856  cls.ConfigClass().warpType + "Warp",
857  help="data ID, e.g. --id tract=12345 patch=1,2",
858  ContainerClass=AssembleCoaddDataIdContainer)
859  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
860  ContainerClass=SelectDataIdContainer)
861  return parser
862 
863 
864 def _subBBoxIter(bbox, subregionSize):
865  """!
866  \brief Iterate over subregions of a bbox
867 
868  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
869  \param[in] subregionSize: size of sub-bboxes
870 
871  \return subBBox: next sub-bounding box of size subregionSize or smaller;
872  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
873  but it will never be empty
874  """
875  if bbox.isEmpty():
876  raise RuntimeError("bbox %s is empty" % (bbox,))
877  if subregionSize[0] < 1 or subregionSize[1] < 1:
878  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
879 
880  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
881  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
882  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
883  subBBox.clip(bbox)
884  if subBBox.isEmpty():
885  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
886  (bbox, subregionSize, colShift, rowShift))
887  yield subBBox
888 
889 
890 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
891  """!
892  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
893  """
894 
895  def makeDataRefList(self, namespace):
896  """!
897  \brief Make self.refList from self.idList.
898 
899  Interpret the config.doMatchBackgrounds, config.autoReference,
900  and whether a visit/run supplied.
901  If a visit/run is supplied, config.autoReference is automatically set to False.
902  if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
903 
904  """
905  keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd",
906  level=self.level)
907  keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
908  "Coadd_directWarp", level=self.level)
909 
910  if namespace.config.doMatchBackgrounds:
911  if namespace.config.autoReference: # matcher will pick it's own reference image
912  datasetType = namespace.config.coaddName + "Coadd"
913  validKeys = keysCoadd
914  else:
915  datasetType = namespace.config.coaddName + "Coadd_directWarp"
916  validKeys = keysCoaddTempExp
917  else: # bkg subtracted coadd
918  datasetType = namespace.config.coaddName + "Coadd"
919  validKeys = keysCoadd
920 
921  for dataId in self.idList:
922  # tract and patch are required
923  for key in validKeys:
924  if key not in dataId:
925  raise RuntimeError("--id must include " + key)
926 
927  for key in dataId: # check if users supplied visit/run
928  if (key not in keysCoadd) and (key in keysCoaddTempExp): # user supplied a visit/run
929  if namespace.config.autoReference:
930  # user probably meant: autoReference = False
931  namespace.config.autoReference = False
932  datasetType = namespace.config.coaddName + "Coadd_directWarp"
933  print("Switching config.autoReference to False; applies only to background Matching.")
934  break
935 
936  dataRef = namespace.butler.dataRef(
937  datasetType=datasetType,
938  dataId=dataId,
939  )
940  self.refList.append(dataRef)
941 
942 
943 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
944  """!
945  \brief Function to count the number of pixels with a specific mask in a footprint.
946 
947  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
948  have bitmask set but do not have ignoreMask set. Return the count.
949 
950  \param[in] mask: mask to define intersection region by.
951  \parma[in] footprint: footprint to define the intersection region by.
952  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
953  \param[in] ignoreMask: pixels to not consider.
954  \return count of number of pixels in footprint with specified mask.
955  """
956  bbox = footprint.getBBox()
957  bbox.clip(mask.getBBox(afwImage.PARENT))
958  fp = afwImage.Mask(bbox)
959  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
960  footprint.spans.setMask(fp, bitmask)
961  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
962  (subMask.getArray() & ignoreMask) == 0).sum()
963 
964 
966  """!
967 \anchor SafeClipAssembleCoaddConfig
968 
969 \brief Configuration parameters for the SafeClipAssembleCoaddTask
970  """
971  clipDetection = pexConfig.ConfigurableField(
972  target=SourceDetectionTask,
973  doc="Detect sources on difference between unclipped and clipped coadd")
974  minClipFootOverlap = pexConfig.Field(
975  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
976  dtype=float,
977  default=0.6
978  )
979  minClipFootOverlapSingle = pexConfig.Field(
980  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
981  "clipped when only one visit overlaps",
982  dtype=float,
983  default=0.5
984  )
985  minClipFootOverlapDouble = pexConfig.Field(
986  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
987  "clipped when two visits overlap",
988  dtype=float,
989  default=0.45
990  )
991  maxClipFootOverlapDouble = pexConfig.Field(
992  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
993  "considering two visits",
994  dtype=float,
995  default=0.15
996  )
997  minBigOverlap = pexConfig.Field(
998  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
999  "when labeling clipped footprints",
1000  dtype=int,
1001  default=100
1002  )
1003 
1004  def setDefaults(self):
1005  # The numeric values for these configuration parameters were empirically determined, future work
1006  # may further refine them.
1007  AssembleCoaddConfig.setDefaults(self)
1008  self.clipDetection.doTempLocalBackground = False
1009  self.clipDetection.reEstimateBackground = False
1010  self.clipDetection.returnOriginalFootprints = False
1011  self.clipDetection.thresholdPolarity = "both"
1012  self.clipDetection.thresholdValue = 2
1013  self.clipDetection.nSigmaToGrow = 2
1014  self.clipDetection.minPixels = 4
1015  self.clipDetection.isotropicGrow = True
1016  self.clipDetection.thresholdType = "pixel_stdev"
1017  self.sigmaClip = 1.5
1018  self.clipIter = 3
1019  self.statistic = "MEAN"
1020 
1021  def validate(self):
1022  if self.doSigmaClip:
1023  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1024  "Ignoring doSigmaClip.")
1025  self.doSigmaClip = False
1026  if self.statistic != "MEAN":
1027  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1028  "(%s chosen). Please set statistic to MEAN."
1029  % (self.statistic))
1030  AssembleCoaddTask.ConfigClass.validate(self)
1031 
1032 
1033 
1039 
1040 
1042  """!
1043  \anchor SafeClipAssembleCoaddTask_
1044 
1045  \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
1046  with potential artifacts.
1047 
1048  \section pipe_tasks_assembleCoadd_Contents Contents
1049  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
1050  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
1051  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
1052  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
1053  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
1054  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
1055 
1056  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
1057 
1058  \copybrief SafeClipAssembleCoaddTask
1059 
1060  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
1061  SafeClipAssembleCoaddTask subtasks that task.
1062  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1063  outliers).
1064  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1065  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1066  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
1067  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
1068  that there is an outlier and ii. this outlier appears on only one or two images.
1069  Such regions will not contribute to the final coadd.
1070  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1071  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1072  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1073  surveys.
1074 
1075  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1076  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1077  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1078 
1079  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1080  \copydoc \_\_init\_\_
1081 
1082  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1083  \copydoc run
1084 
1085  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1086  See \ref SafeClipAssembleCoaddConfig
1087 
1088  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1089  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1090  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1091  files.
1092  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1093  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1094  for further information.
1095 
1096  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1097 
1098  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1099  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1100  '--legacyCoadd'.
1101  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1102  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1103  with a list of coaddTempExps to attempt to coadd (specified using
1104  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1105  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1106  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1107  command line argument:
1108  \code
1109  assembleCoadd.py --help
1110  \endcode
1111  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1112  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1113  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1114  packages.
1115  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1116  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1117  <DL>
1118  <DT>processCcd</DT>
1119  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1120  <DT>makeSkyMap</DT>
1121  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1122  <DT>makeCoaddTempExp</DT>
1123  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1124  </DL>
1125  We can perform all of these steps by running
1126  \code
1127  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1128  \endcode
1129  This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1130  as follows:
1131  \code
1132  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1133  \endcode
1134  This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1135  You may also choose to run:
1136  \code
1137  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1138  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1139  \endcode
1140  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1141  discussed in \ref pipeTasks_multiBand.
1142  """
1143  ConfigClass = SafeClipAssembleCoaddConfig
1144  _DefaultName = "safeClipAssembleCoadd"
1145 
1146  def __init__(self, *args, **kwargs):
1147  """!
1148  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1149  """
1150  AssembleCoaddTask.__init__(self, *args, **kwargs)
1151  schema = afwTable.SourceTable.makeMinimalSchema()
1152  self.makeSubtask("clipDetection", schema=schema)
1153 
1154  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1155  *args, **kwargs):
1156  """!
1157  \brief Assemble the coadd for a region
1158 
1159  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1160  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1161  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1162  overlap between the clipped region and a source.
1163  This leaves us with a set of footprints from the difference image that have been identified as having
1164  occured on just one or two individual visits. However, these footprints were generated from a
1165  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1166  footprints acrosss the coadd difference in this process.
1167  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1168  these are big footprints.
1169  Combine the small and big clipped footprints and mark them on a new bad mask plane
1170  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1171  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1172  bad mask plane.
1173 
1174  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1175  parent task.
1176 
1177  @param skyInfo: Patch geometry information, from getSkyInfo
1178  @param tempExpRefList: List of data reference to tempExp
1179  @param imageScalerList: List of image scalers
1180  @param weightList: List of weights
1181  @param bgModelList: List of background models from background matching
1182  return pipeBase.Struct with coaddExposure, nImage
1183  """
1184  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1185  mask = exp.getMaskedImage().getMask()
1186  mask.addMaskPlane("CLIPPED")
1187 
1188  result = self.detectClip(exp, tempExpRefList)
1189 
1190  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1191 
1192  # Go to individual visits for big footprints
1193  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1194  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1195  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1196  maskClipValue, maskDetValue)
1197 
1198  # Create mask of the current clipped footprints
1199  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1200  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1201 
1202  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1203  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1204  maskClip |= maskClipBig
1205 
1206  # Assemble coadd from base class, but ignoring CLIPPED pixels
1207  badMaskPlanes = self.config.badMaskPlanes[:]
1208  badMaskPlanes.append("CLIPPED")
1209  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1210  retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1211  bgModelList, result.tempExpClipList, mask=badPixelMask)
1212 
1213  # Set the coadd CLIPPED mask from the footprints since currently pixels that are masked
1214  # do not get propagated
1215  maskExp = retStruct.coaddExposure.getMaskedImage().getMask()
1216  maskExp |= maskClip
1217 
1218  return retStruct
1219 
1220  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):
1221  """!
1222  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1223 
1224  Generate a difference image between clipped and unclipped coadds.
1225  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1226  Return the difference image.
1227 
1228  @param skyInfo: Patch geometry information, from getSkyInfo
1229  @param tempExpRefList: List of data reference to tempExp
1230  @param imageScalerList: List of image scalers
1231  @param weightList: List of weights
1232  @param bgModelList: List of background models from background matching
1233  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1234  """
1235  # Clone and upcast self.config because current self.config is frozen
1236  config = AssembleCoaddConfig()
1237  # getattr necessary because subtasks do not survive Config.toDict()
1238  configIntersection = {k: getattr(self.config, k)
1239  for k, v in self.config.toDict().items() if (k in config.keys())}
1240  config.update(**configIntersection)
1241 
1242  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1243  config.statistic = 'MEAN'
1244  task = AssembleCoaddTask(config=config)
1245  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1246  bgModelList).coaddExposure
1247 
1248  config.statistic = 'MEANCLIP'
1249  task = AssembleCoaddTask(config=config)
1250  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1251  bgModelList).coaddExposure
1252 
1253  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1254  coaddDiff -= coaddClip.getMaskedImage()
1255  exp = afwImage.ExposureF(coaddDiff)
1256  exp.setPsf(coaddMean.getPsf())
1257  return exp
1258 
1259  def detectClip(self, exp, tempExpRefList):
1260  """!
1261  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1262 
1263  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1264  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1265  fraction to thresholds set in the config.
1266  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1267  two).
1268  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1269  the coaddTempExp.
1270  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1271  with the clipped footprints and a list of new masks for the coaddTempExps.
1272 
1273  \param[in] exp: Exposure to run detection on
1274  \param[in] tempExpRefList: List of data reference to tempExp
1275  \return struct containing:
1276  - clippedFootprints: list of clipped footprints
1277  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1278  - tempExpClipList: list of new masks for tempExp
1279  """
1280  mask = exp.getMaskedImage().getMask()
1281  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1282  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1283  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1284  # Merge positive and negative together footprints together
1285  fpSet.positive.merge(fpSet.negative)
1286  footprints = fpSet.positive
1287  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1288  ignoreMask = self.getBadPixelMask()
1289 
1290  clipFootprints = []
1291  clipIndices = []
1292 
1293  # build a list with a mask for each visit which can be modified with clipping information
1294  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.warpType),
1295  immediate=True).getMaskedImage().getMask() for
1296  tmpExpRef in tempExpRefList]
1297 
1298  for footprint in footprints.getFootprints():
1299  nPixel = footprint.getArea()
1300  overlap = [] # hold the overlap with each visit
1301  maskList = [] # which visit mask match
1302  indexList = [] # index of visit in global list
1303  for i, tmpExpMask in enumerate(tempExpClipList):
1304  # Determine the overlap with the footprint
1305  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1306  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1307  totPixel = nPixel - ignore
1308 
1309  # If we have more bad pixels than detection skip
1310  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1311  continue
1312  overlap.append(overlapDet/float(totPixel))
1313  maskList.append(tmpExpMask)
1314  indexList.append(i)
1315 
1316  overlap = numpy.array(overlap)
1317  if not len(overlap):
1318  continue
1319 
1320  keep = False # Should this footprint be marked as clipped?
1321  keepIndex = [] # Which tempExps does the clipped footprint belong to
1322 
1323  # If footprint only has one overlap use a lower threshold
1324  if len(overlap) == 1:
1325  if overlap[0] > self.config.minClipFootOverlapSingle:
1326  keep = True
1327  keepIndex = [0]
1328  else:
1329  # This is the general case where only visit should be clipped
1330  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1331  if len(clipIndex) == 1:
1332  keep = True
1333  keepIndex = [clipIndex[0]]
1334 
1335  # Test if there are clipped objects that overlap two different visits
1336  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1337  if len(clipIndex) == 2 and len(overlap) > 3:
1338  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1339  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1340  keep = True
1341  keepIndex = clipIndex
1342 
1343  if not keep:
1344  continue
1345 
1346  for index in keepIndex:
1347  footprint.spans.setMask(maskList[index], maskClipValue)
1348 
1349  clipIndices.append(numpy.array(indexList)[keepIndex])
1350  clipFootprints.append(footprint)
1351 
1352  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1353  tempExpClipList=tempExpClipList)
1354 
1355  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1356  """!
1357  \brief Find footprints from individual tempExp footprints for large footprints.
1358 
1359  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1360  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1361  significantly with each source in all the coaddTempExps.
1362 
1363  \param[in] tempExpClipList: List of tempExp masks with clipping information
1364  \param[in] clipFootprints: List of clipped footprints
1365  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1366  \param[in] maskClipValue: Mask value of clipped pixels
1367  \param[in] maskClipValue: Mask value of detected pixels
1368  \return list of big footprints
1369  """
1370  bigFootprintsCoadd = []
1371  ignoreMask = self.getBadPixelMask()
1372  for index, tmpExpMask in enumerate(tempExpClipList):
1373 
1374  # Create list of footprints from the DETECTED pixels
1375  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1376  afwImage.PARENT, True)
1377  maskVisitDet &= maskDetValue
1378  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1379 
1380  # build a mask of clipped footprints that are in this visit
1381  clippedFootprintsVisit = []
1382  for foot, clipIndex in zip(clipFootprints, clipIndices):
1383  if index not in clipIndex:
1384  continue
1385  clippedFootprintsVisit.append(foot)
1386  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1387  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1388 
1389  bigFootprintsVisit = []
1390  for foot in visitFootprints.getFootprints():
1391  if foot.getArea() < self.config.minBigOverlap:
1392  continue
1393  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1394  if nCount > self.config.minBigOverlap:
1395  bigFootprintsVisit.append(foot)
1396  bigFootprintsCoadd.append(foot)
1397 
1398  # Update single visit masks
1399  maskVisitClip.clearAllMaskPlanes()
1400  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1401  tmpExpMask |= maskVisitClip
1402 
1403  return bigFootprintsCoadd
1404 
1405 
1407  assembleStaticSkyModel = pexConfig.ConfigurableField(
1408  target=AssembleCoaddTask,
1409  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1410  " naive/first-iteration model of the static sky.",
1411  )
1412  chiThreshold = pexConfig.RangeField(
1413  doc="Detection threshold (sigma) for artifacts in warp diff "
1414  "(chi-image of PSF-Matched warp minus the model of the static sky)",
1415  dtype=float,
1416  default=5,
1417  min=0,
1418  )
1419  minPixels = pexConfig.Field(
1420  doc="Minimum number of pixels in a region (in a warp diff chi-image) "
1421  "above chiThreshold to trigger masking. "
1422  "Detected artifacts with fewer than the specified number of pixels will be ignored.",
1423  dtype=int,
1424  default=1
1425  )
1426  doMaskNegative = pexConfig.Field(
1427  doc="Also mask outlier regions of flux less than the static sky model?",
1428  dtype=bool,
1429  default=True
1430  )
1431  temporalThreshold = pexConfig.Field(
1432  doc="Unitless fraction of number of epochs to classify as an artifact/outlier source versus"
1433  " a source that is intrinsically variable or difficult to subtract cleanly. "
1434  "If outlier region in warp-diff Chi-image is mostly (defined by spatialThreshold) "
1435  "an outlier in less than temporalThreshold * number of epochs, then mask. "
1436  "Otherwise, do not mask.",
1437  dtype=float,
1438  default=0.075
1439  )
1440  spatialThreshold = pexConfig.Field(
1441  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1442  "temporal criteria",
1443  dtype=float,
1444  default=0.5
1445  )
1446  growMaskBy = pexConfig.Field(
1447  doc="Number of pixels by which to grow the masks on artifacts.",
1448  dtype=int,
1449  default=5
1450  )
1451 
1452  def setDefaults(self):
1453  AssembleCoaddConfig.setDefaults(self)
1454  self.assembleStaticSkyModel.warpType = 'psfMatched'
1455  self.assembleStaticSkyModel.statistic = 'MEDIAN'
1456  self.assembleStaticSkyModel.doWrite = False
1457 
1458 
1460  """!
1461  \anchor CompareWarpAssembleCoaddTask_
1462 
1463  \brief Assemble a compareWarp coadded image from a set of warps
1464  by masking artifacts detected by comparing PSF-matched warps
1465 
1466  \section pipe_tasks_assembleCoadd_Contents Contents
1467  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1468  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1469  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1470  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1471  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1472  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1473 
1474  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1475 
1476  \copybrief CompareWarpAssembleCoaddTask
1477 
1478  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1479  outliers).
1480  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1481  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1482  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1483  pixels in the individual warps suspected to contain an artifact.
1484  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1485  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1486  median coadd by more than config.chiThreshold sigma, is an artifact candidate. The candidates are then
1487  filtered to remove variable sources and sources that are difficult to subtract such as bright stars.
1488  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1489  The temporalThreshold is the maximum fraction of epochs that the deviation can
1490  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1491  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1492  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1493  that deviate in only a few epochs, it is grown by growMaskBy and bits set in the 'CLIPPED' plane.
1494  These regions will not contribute to the final coadd.
1495  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1496  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1497  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1498  surveys.
1499 
1500  CompareWarpAssembleCoaddTask sub-classes
1501  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1502  as a subtask to generate the TemplateCoadd (the model of the static sky)
1503 
1504  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1505  \copydoc \_\_init\_\_
1506 
1507  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1508  \copydoc run
1509 
1510  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1511  See \ref CompareWarpAssembleCoaddConfig
1512 
1513  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1514  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1515  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1516  files.
1517  CompareWarpAssembleCoaddTask has no debug variables of its own.
1518 
1519  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using CompareWarpAssembleCoaddTask
1520 
1521  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1522  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1523  '--compareWarpCoadd'.
1524  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1525  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1526  with a list of coaddTempExps to attempt to coadd (specified using
1527  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1528  Only the warps that cover the specified tract and patch will be coadded.
1529  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1530  command line argument:
1531  \code
1532  assembleCoadd.py --help
1533  \endcode
1534  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing, we
1535  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1536  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1537  packages.
1538  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1539  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1540  <DL>
1541  <DT>processCcd</DT>
1542  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1543  <DT>makeSkyMap</DT>
1544  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1545  <DT>makeCoaddTempExp</DT>
1546  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1547  </DL>
1548  We can perform all of these steps by running
1549  \code
1550  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1551  \endcode
1552  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1553  as follows:
1554  \code
1555  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1556  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1557  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1558  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1559  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1560  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1561  --selectId visit=903988 ccd=24
1562  \endcode
1563  This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1564  """
1565  ConfigClass = CompareWarpAssembleCoaddConfig
1566  _DefaultName = "compareWarpAssembleCoadd"
1567 
1568  def __init__(self, *args, **kwargs):
1569  """!
1570  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1571  """
1572  AssembleCoaddTask.__init__(self, *args, **kwargs)
1573  self.makeSubtask("assembleStaticSkyModel")
1574 
1575  def makeSupplementaryData(self, dataRef, selectDataList):
1576  """!
1577  \brief Make inputs specific to Subclass
1578 
1579  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1580  """
1581  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList).coaddExposure
1582  return pipeBase.Struct(templateCoadd=templateCoadd)
1583 
1584  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1585  supplementaryData, *args, **kwargs):
1586  """!
1587  \brief Assemble the coadd
1588 
1589  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1590  as the model of the static sky.
1591 
1592  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1593  new "CLIPPED" plane and updated "NO_DATA" plane.
1594  Then pass these alternative masks to the base class's assemble method.
1595 
1596  @param skyInfo: Patch geometry information
1597  @param tempExpRefList: List of data references to warps
1598  @param imageScalerList: List of image scalers
1599  @param weightList: List of weights
1600  @param bgModelList: List of background models from background matching
1601  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1602 
1603  return coadd exposure
1604  """
1605  templateCoadd = supplementaryData.templateCoadd
1606  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1607  maskList = self.computeAltMaskList(tempExpRefList, spanSetMaskList)
1608  badMaskPlanes = self.config.badMaskPlanes[:]
1609  badMaskPlanes.append("CLIPPED")
1610  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1611 
1612  coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1613  bgModelList, maskList, mask=badPixelMask)
1614  return coaddExp
1615 
1616  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1617  """!
1618  \brief Find artifacts
1619 
1620  Loop through warps twice. The first loop builds a map with the count of how many
1621  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1622  The second loop takes each difference image and filters the artifacts detected
1623  in each using count map to filter out variable sources and sources that are difficult to
1624  subtract cleanly.
1625 
1626  @param templateCoadd: Exposure to serve as model of static sky
1627  @param tempExpRefList: List of data references to warps
1628  @param imageScalerList: List of image scalers
1629  """
1630 
1631  self.log.debug("Generating Count Image. First loop through warps")
1632  epochCountImage = afwImage.ImageU(templateCoadd.getBBox())
1633  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1634  mi = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1635  chiIm = self._makeChiIm(mi)
1636  chiOneMap = self._snrToBinaryIm(chiIm)
1637  epochCountImage += chiOneMap
1638 
1639  self.log.debug("Generating Mask List. Second loop through warps")
1640  spanSetArtifactList = []
1641  spanSetNoDataMaskList = []
1642 
1643  maxNumEpochs = int(max(1, self.config.temporalThreshold*len(tempExpRefList)))
1644  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1645  mi = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1646  chiIm = self._makeChiIm(mi)
1647  chiOneMap = self._snrToBinaryArr(chiIm.array)
1648  outliers = afwImage.makeMaskFromArray(chiOneMap.astype(afwImage.MaskPixel))
1649  outliers.setXY0(mi.getXY0())
1650  spanSetList = afwGeom.SpanSet.fromMask(outliers).split()
1651 
1652  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1653  # undergo a second convolution. Pixels with data in the direct warp
1654  # but not in the PSF-matched warp will not have their artifacts detected.
1655  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1656  nans = numpy.where(numpy.isnan(chiIm.array), 1, 0)
1657  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1658  nansMask.setXY0(chiIm.getXY0())
1659  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1660 
1661  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage,
1662  maxNumEpochs=maxNumEpochs)
1663  dilatedSpanSetList = [s.dilated(self.config.growMaskBy, afwGeom.Stencil.CIRCLE)
1664  for s in filteredSpanSetList]
1665  spanSetArtifactList.append(dilatedSpanSetList)
1666  spanSetNoDataMaskList.append(spanSetNoDataMask)
1667  return pipeBase.Struct(artifacts=spanSetArtifactList,
1668  noData=spanSetNoDataMaskList)
1669 
1670  def computeAltMaskList(self, tempExpRefList, maskSpanSets):
1671  """!
1672  \brief Apply artifact span set lists to masks
1673 
1674  @param tempExpRefList: List of data references to warps
1675  @param maskSpanSets: Struct containing artifact and noData spanSet lists to apply
1676 
1677  return List of alternative masks
1678 
1679  Add artifact span set list as "CLIPPED" plane and NaNs to existing "NO_DATA" plane
1680  """
1681  spanSetMaskList = maskSpanSets.artifacts
1682  spanSetNoDataList = maskSpanSets.noData
1683  altMaskList = []
1684  for warpRef, artifacts, noData in zip(tempExpRefList, spanSetMaskList, spanSetNoDataList):
1685  warp = warpRef.get(self.getTempExpDatasetName(self.config.warpType), immediate=True)
1686  mask = warp.maskedImage.mask
1687  maskClipValue = mask.addMaskPlane("CLIPPED")
1688  noDataValue = mask.addMaskPlane("NO_DATA")
1689  for artifact in artifacts:
1690  artifact.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1691  for noDataRegion in noData:
1692  noDataRegion.clippedTo(mask.getBBox()).setMask(mask, 2**noDataValue)
1693  altMaskList.append(mask)
1694  return altMaskList
1695 
1696  def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None, minPixels=None):
1697  if minPixels is None:
1698  minPixels = self.config.minPixels
1699  maskSpanSetList = []
1700  x0, y0 = epochCountImage.getXY0()
1701  for i, span in enumerate(spanSetList):
1702  y, x = span.indices()
1703  if len(y) < minPixels:
1704  continue
1705  counts = epochCountImage.array[[y1 - y0 for y1 in y], [x1 - x0 for x1 in x]]
1706  idx = numpy.where((counts > 0) & (counts <= maxNumEpochs))
1707  percentBelowThreshold = len(idx[0]) / len(counts)
1708  if percentBelowThreshold > self.config.spatialThreshold:
1709  maskSpanSetList.append(span)
1710  return maskSpanSetList
1711 
1712  def _snrToBinaryArr(self, arrIn):
1713  with numpy.errstate(invalid='ignore'):
1714  arr = numpy.where(arrIn >= self.config.chiThreshold, 1, 0)
1715  if self.config.doMaskNegative:
1716  arr[numpy.where(arrIn <= -self.config.chiThreshold)] = 1
1717  return arr
1718 
1719  def _snrToBinaryIm(self, im):
1720  arr = self._snrToBinaryArr(im.array)
1721  return afwImage.ImageU(arr.astype(numpy.uint16), xy0=im.getXY0())
1722 
1723  def _makeChiIm(self, maskedImage):
1724  chiIm = maskedImage.image.Factory(1./numpy.sqrt(maskedImage.variance.array))
1725  chiIm *= maskedImage.image
1726  chiIm.setXY0(maskedImage.getXY0())
1727  return chiIm
1728 
1729  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1730  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1731  warp = warpRef.get(self.getTempExpDatasetName('psfMatched'), immediate=True)
1732  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1733  mi = warp.getMaskedImage()
1734  mi -= templateCoadd.getMaskedImage()
1735  return mi
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None)
Perform background matching on the prepared inputs.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, args, kwargs)
Assemble the coadd for a region.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def computeAltMaskList(self, tempExpRefList, maskSpanSets)
Apply artifact span set lists to masks.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue)
Find footprints from individual tempExp footprints for large footprints.
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None, minPixels=None)
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def getBackgroundReferenceScaler(self, dataRef)
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList)
Add metadata from the background matching to the coadd.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, supplementaryData, args, kwargs)
Assemble the coadd.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63