lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import numpy
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 import lsst.afw.detection as afwDet
33 import lsst.coadd.utils as coaddUtils
34 import lsst.pipe.base as pipeBase
35 import lsst.meas.algorithms as measAlg
36 import lsst.log as log
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage import InterpImageTask
39 from .matchBackgrounds import MatchBackgroundsTask
40 from .scaleZeroPoint import ScaleZeroPointTask
41 from .coaddHelpers import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms import SourceDetectionTask
43 
44 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask", "CompareWarpAssembleCoaddTask"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """!
49 \anchor AssembleCoaddConfig_
50 
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
52  """
53  warpType = pexConfig.Field(
54  doc="Warp name: one of 'direct' or 'psfMatched'",
55  dtype=str,
56  default="direct",
57  )
58  subregionSize = pexConfig.ListField(
59  dtype=int,
60  doc="Width, height of stack subregion size; "
61  "make small enough that a full stack of images will fit into memory at once.",
62  length=2,
63  default=(2000, 2000),
64  )
65  statistic = pexConfig.Field(
66  dtype=str,
67  doc="Main stacking statistic for aggregating over the epochs.",
68  default="MEANCLIP",
69  )
70  doSigmaClip = pexConfig.Field(
71  dtype=bool,
72  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
73  default=False,
74  )
75  sigmaClip = pexConfig.Field(
76  dtype=float,
77  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
78  default=3.0,
79  )
80  clipIter = pexConfig.Field(
81  dtype=int,
82  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
83  default=2,
84  )
85  scaleZeroPoint = pexConfig.ConfigurableField(
86  target=ScaleZeroPointTask,
87  doc="Task to adjust the photometric zero point of the coadd temp exposures",
88  )
89  doInterp = pexConfig.Field(
90  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
91  dtype=bool,
92  default=True,
93  )
94  interpImage = pexConfig.ConfigurableField(
95  target=InterpImageTask,
96  doc="Task to interpolate (and extrapolate) over NaN pixels",
97  )
98  matchBackgrounds = pexConfig.ConfigurableField(
99  target=MatchBackgroundsTask,
100  doc="Task to match backgrounds",
101  )
102  maxMatchResidualRatio = pexConfig.Field(
103  doc="Maximum ratio of the mean squared error of the background matching model to the variance "
104  "of the difference in backgrounds",
105  dtype=float,
106  default=1.1
107  )
108  maxMatchResidualRMS = pexConfig.Field(
109  doc="Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
110  dtype=float,
111  default=1.0
112  )
113  doWrite = pexConfig.Field(
114  doc="Persist coadd?",
115  dtype=bool,
116  default=True,
117  )
118  doNImage = pexConfig.Field(
119  doc="Create image of number of contributing exposures for each pixel",
120  dtype=bool,
121  default=False,
122  )
123  doMatchBackgrounds = pexConfig.Field(
124  doc="Match backgrounds of coadd temp exposures before coadding them? "
125  "If False, the coadd temp expsosures must already have been background subtracted or matched",
126  dtype=bool,
127  default=False,
128  )
129  autoReference = pexConfig.Field(
130  doc="Automatically select the coadd temp exposure to use as a reference for background matching? "
131  "Ignored if doMatchBackgrounds false. "
132  "If False you must specify the reference temp exposure as the data Id",
133  dtype=bool,
134  default=True,
135  )
136  maskPropagationThresholds = pexConfig.DictField(
137  keytype=str,
138  itemtype=float,
139  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
140  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
141  "would have contributed exceeds this value."),
142  default={"SAT": 0.1},
143  )
144  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["CROSSTALK", "NOT_DEBLENDED"],
145  doc="Mask planes to remove before coadding")
146  #
147  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
148  # To make this useful you *must* also configure the flags.pixel algorithm, for example
149  # by adding
150  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
151  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
152  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
153  #
154  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
155  doc="Set mask and flag bits for bright objects?")
156  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
157  doc="Name of mask bit used for bright objects")
158 
159  coaddPsf = pexConfig.ConfigField(
160  doc="Configuration for CoaddPsf",
161  dtype=measAlg.CoaddPsfConfig,
162  )
163 
164  def setDefaults(self):
165  CoaddBaseTask.ConfigClass.setDefaults(self)
166  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
167 
168  def validate(self):
169  CoaddBaseTask.ConfigClass.validate(self)
170  if self.doPsfMatch:
171  # Backwards compatibility.
172  # Configs do not have loggers
173  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
174  self.warpType = 'psfMatched'
175  if self.doSigmaClip and self.statistic != "MEANCLIP":
176  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
177  self.statistic = "MEANCLIP"
178  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
179  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
180  "compute and set a non-zero coadd variance estimate." % (self.statistic))
181 
182  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
183  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
184  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
185  if str(k) not in unstackableStats]
186  raise ValueError("statistic %s is not allowed. Please choose one of %s."
187  % (self.statistic, stackableStats))
188 
189 
190 
197  """!
198 \anchor AssembleCoaddTask_
199 
200 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
201 
202 \section pipe_tasks_assembleCoadd_Contents Contents
203  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
204  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
205  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
206  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
207  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
208  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
209 
210 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
211 
212 \copybrief AssembleCoaddTask_
213 
214 We want to assemble a coadded image from a set of Warps (also called
215 coadded temporary exposures or coaddTempExps.
216 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
217 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
218 Warps that cover the specified patch (pointed at by dataRef).
219 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
220 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
221 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
222 When background matching is enabled, the task may be configured to automatically select a reference exposure
223 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
224 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
225 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
226 config.makePsfMatched set which of the warp types will be coadded.
227 The coadd is computed as a mean with optional outlier rejection.
228 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
229 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
230 
231 AssembleCoaddTask uses several sub-tasks. These are
232 <DL>
233  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
234  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
235  <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
236  <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
237  not provided).</DD>
238  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
239  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
240 </DL>
241 You can retarget these subtasks if you wish.
242 
243 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
244 \copydoc \_\_init\_\_
245 
246 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
247 \copydoc run
248 
249 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
250 See \ref AssembleCoaddConfig_
251 
252 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
253 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
254 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
255 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
256 the documetation for the subtasks for further information.
257 
258 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
259 
260 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
261 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
262 a data reference to the tract patch and filter to be coadded (specified using
263 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
264 Warps to attempt to coadd (specified using
265 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
266 that cover the specified tract and patch will be coadded. A list of the available optional
267 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
268 \code
269 assembleCoadd.py --help
270 \endcode
271 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
272 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
273 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
274 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
275 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
276 <DL>
277  <DT>processCcd</DT>
278  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
279  <DT>makeSkyMap</DT>
280  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
281  <DT>makeCoaddTempExp</DT>
282  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
283 </DL>
284 We can perform all of these steps by running
285 \code
286 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
287 \endcode
288 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
289 follows:
290 \code
291 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
292 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
293 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
294 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
295 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
296 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
297 --selectId visit=903988 ccd=24
298 \endcode
299 that will process the HSC-I band data. The results are written in
300 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
301 
302 You may also choose to run:
303 \code
304 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
305 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
306 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
307 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
308 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
309 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
310 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
311 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
312 \endcode
313 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
314 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
315 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
316  """
317  ConfigClass = AssembleCoaddConfig
318  _DefaultName = "assembleCoadd"
319 
320  def __init__(self, *args, **kwargs):
321  """!
322  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
323  \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
324  """
325  CoaddBaseTask.__init__(self, *args, **kwargs)
326  self.makeSubtask("interpImage")
327  self.makeSubtask("matchBackgrounds")
328  self.makeSubtask("scaleZeroPoint")
329 
330  if self.config.doMaskBrightObjects:
331  mask = afwImage.Mask()
332  try:
333  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
334  except pexExceptions.LsstCppException:
335  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
336  mask.getMaskPlaneDict().keys())
337  del mask
338 
339  self.warpType = self.config.warpType
340 
341  @pipeBase.timeMethod
342  def run(self, dataRef, selectDataList=[]):
343  """!
344  \brief Assemble a coadd from a set of Warps
345 
346  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
347  match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
348  Warps if the background has not already been removed. Assemble the Warps using
349  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
350  exposure.
351 
352  \anchor runParams
353  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
354  (if config.autoReference=False). Used to access the following data products:
355  - [in] self.config.coaddName + "Coadd_skyMap"
356  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
357  - [out] self.config.coaddName + "Coadd"
358  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
359  selected from this list based on overlap with the patch defined by dataRef.
360 
361  \return a pipeBase.Struct with fields:
362  - coaddExposure: coadded exposure
363  - nImage: exposure count image
364  """
365  skyInfo = self.getSkyInfo(dataRef)
366  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
367  if len(calExpRefList) == 0:
368  self.log.warn("No exposures to coadd")
369  return
370  self.log.info("Coadding %d exposures", len(calExpRefList))
371 
372  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
373  inputData = self.prepareInputs(tempExpRefList)
374  self.log.info("Found %d %s", len(inputData.tempExpRefList),
375  self.getTempExpDatasetName(self.warpType))
376  if len(inputData.tempExpRefList) == 0:
377  self.log.warn("No coadd temporary exposures found")
378  return
379  if self.config.doMatchBackgrounds:
380  refImageScaler = self.getBackgroundReferenceScaler(dataRef)
381  inputData = self.backgroundMatching(inputData, dataRef, refImageScaler)
382  if len(inputData.tempExpRefList) == 0:
383  self.log.warn("No valid background models")
384  return
385 
386  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
387 
388  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
389  inputData.weightList,
390  inputData.backgroundInfoList if self.config.doMatchBackgrounds else None,
391  supplementaryData=supplementaryData)
392 
393  if self.config.doMatchBackgrounds:
394  self.addBackgroundMatchingMetadata(retStruct.coaddExposure, inputData.tempExpRefList,
395  inputData.backgroundInfoList)
396 
397  if self.config.doInterp:
398  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
399  # The variance must be positive; work around for DM-3201.
400  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
401  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
402 
403  if self.config.doMaskBrightObjects:
404  brightObjectMasks = self.readBrightObjectMasks(dataRef)
405  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
406 
407  if self.config.doWrite:
408  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
409  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
410  if retStruct.nImage is not None:
411  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
412 
413  return retStruct
414 
415  def makeSupplementaryData(self, dataRef, selectDataList):
416  """!
417  \brief Make additional inputs to assemble() specific to subclasses.
418 
419  Available to be implemented by subclasses only if they need the
420  coadd dataRef for performing preliminary processing before
421  assembling the coadd.
422  """
423  pass
424 
425  def getTempExpRefList(self, patchRef, calExpRefList):
426  """!
427  \brief Generate list data references corresponding to warped exposures that lie within the
428  patch to be coadded.
429 
430  \param[in] patchRef: Data reference for patch
431  \param[in] calExpRefList: List of data references for input calexps
432  \return List of Warp/CoaddTempExp data references
433  """
434  butler = patchRef.getButler()
435  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
436  self.getTempExpDatasetName(self.warpType))
437  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
438  g, groupData.keys) for
439  g in groupData.groups.keys()]
440  return tempExpRefList
441 
442  def getBackgroundReferenceScaler(self, dataRef):
443  """!
444  \brief Construct an image scaler for the background reference frame
445 
446  Each Warp has a different background level. A reference background level must be chosen before
447  coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
448  this routine is a no-op and None is returned. Otherwise, use the
449  \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
450  reference image and return it.
451 
452  \param[in] dataRef: Data reference for the background reference frame, or None
453  \return image scaler, or None
454  """
455  if self.config.autoReference:
456  return None
457 
458  # We've been given the data reference
459  dataset = self.getTempExpDatasetName(self.warpType)
460  if not dataRef.datasetExists(dataset):
461  raise RuntimeError("Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
462 
463  refExposure = dataRef.get(self.getTempExpDatasetName(self.warpType), immediate=True)
464  refImageScaler = self.scaleZeroPoint.computeImageScaler(
465  exposure=refExposure,
466  dataRef=dataRef,
467  )
468  return refImageScaler
469 
470  def prepareInputs(self, refList):
471  """!
472  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
473  for the photometric zero point.
474 
475  Each Warp has its own photometric zeropoint and background variance. Before coadding these
476  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
477  weight for each Warp.
478 
479  \param[in] refList: List of data references to tempExp
480  \return Struct:
481  - tempExprefList: List of data references to tempExp
482  - weightList: List of weightings
483  - imageScalerList: List of image scalers
484  """
485  statsCtrl = afwMath.StatisticsControl()
486  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
487  statsCtrl.setNumIter(self.config.clipIter)
488  statsCtrl.setAndMask(self.getBadPixelMask())
489  statsCtrl.setNanSafe(True)
490  # compute tempExpRefList: a list of tempExpRef that actually exist
491  # and weightList: a list of the weight of the associated coadd tempExp
492  # and imageScalerList: a list of scale factors for the associated coadd tempExp
493  tempExpRefList = []
494  weightList = []
495  imageScalerList = []
496  tempExpName = self.getTempExpDatasetName(self.warpType)
497  for tempExpRef in refList:
498  if not tempExpRef.datasetExists(tempExpName):
499  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
500  continue
501 
502  tempExp = tempExpRef.get(tempExpName, immediate=True)
503  maskedImage = tempExp.getMaskedImage()
504  imageScaler = self.scaleZeroPoint.computeImageScaler(
505  exposure=tempExp,
506  dataRef=tempExpRef,
507  )
508  try:
509  imageScaler.scaleMaskedImage(maskedImage)
510  except Exception as e:
511  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
512  continue
513  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
514  afwMath.MEANCLIP, statsCtrl)
515  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
516  weight = 1.0 / float(meanVar)
517  if not numpy.isfinite(weight):
518  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
519  continue
520  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
521 
522  del maskedImage
523  del tempExp
524 
525  tempExpRefList.append(tempExpRef)
526  weightList.append(weight)
527  imageScalerList.append(imageScaler)
528 
529  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
530  imageScalerList=imageScalerList)
531 
532  def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None):
533  """!
534  \brief Perform background matching on the prepared inputs
535 
536  Each Warp has a different background level that must be normalized to a reference level
537  before coaddition. If no reference is provided, the background matcher selects one. If the background
538  matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
539  consistent with the scaled background.
540 
541  \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
542  \param[in] refExpDataRef: Data reference for background reference Warp, or None
543  \param[in] refImageScaler: Image scaler for background reference Warp, or None
544  \return Struct:
545  - tempExprefList: List of data references to warped exposures (coaddTempExps)
546  - weightList: List of weightings
547  - imageScalerList: List of image scalers
548  - backgroundInfoList: result from background matching
549  """
550  try:
551  backgroundInfoList = self.matchBackgrounds.run(
552  expRefList=inputData.tempExpRefList,
553  imageScalerList=inputData.imageScalerList,
554  refExpDataRef=refExpDataRef if not self.config.autoReference else None,
555  refImageScaler=refImageScaler,
556  expDatasetType=self.getTempExpDatasetName(self.warpType),
557  ).backgroundInfoList
558  except Exception as e:
559  self.log.fatal("Cannot match backgrounds: %s", e)
560  raise pipeBase.TaskError("Background matching failed.")
561 
562  newWeightList = []
563  newTempExpRefList = []
564  newBackgroundStructList = []
565  newScaleList = []
566  # the number of good backgrounds may be < than len(tempExpList)
567  # sync these up and correct the weights
568  for tempExpRef, bgInfo, scaler, weight in zip(inputData.tempExpRefList, backgroundInfoList,
569  inputData.imageScalerList, inputData.weightList):
570  if not bgInfo.isReference:
571  # skip exposure if it has no backgroundModel
572  # or if fit was bad
573  if (bgInfo.backgroundModel is None):
574  self.log.info("No background offset model available for %s: skipping", tempExpRef.dataId)
575  continue
576  try:
577  varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
578  except Exception as e:
579  self.log.info("MSE/Var ratio not calculable (%s) for %s: skipping",
580  e, tempExpRef.dataId)
581  continue
582  if not numpy.isfinite(varianceRatio):
583  self.log.info("MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
584  bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
585  continue
586  elif (varianceRatio > self.config.maxMatchResidualRatio):
587  self.log.info("Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
588  varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
589  continue
590  elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
591  self.log.info("Bad fit. RMS %.2f > %.2f for %s: skipping",
592  bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
593  continue
594  newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
595  newTempExpRefList.append(tempExpRef)
596  newBackgroundStructList.append(bgInfo)
597  newScaleList.append(scaler)
598 
599  return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
600  imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
601 
602  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
603  altMaskList=None, mask=None, supplementaryData=None):
604  """!
605  \anchor AssembleCoaddTask.assemble_
606 
607  \brief Assemble a coadd from input warps
608 
609  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
610  large area), the assembly is performed over small areas on the image at a time in order to
611  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
612  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
613  statistic specified. Set the edge bits the coadd mask based on the weight map.
614 
615  \param[in] skyInfo: Patch geometry information, from getSkyInfo
616  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
617  \param[in] imageScalerList: List of image scalers
618  \param[in] weightList: List of weights
619  \param[in] bgInfoList: List of background data from background matching, or None
620  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
621  \param[in] mask: Mask to ignore when coadding
622  \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd.
623  Only used by subclasses that implement makeSupplementaryData and override assemble.
624  \return pipeBase.Struct with coaddExposure, nImage if requested
625  """
626  tempExpName = self.getTempExpDatasetName(self.warpType)
627  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
628  if mask is None:
629  mask = self.getBadPixelMask()
630 
631  statsCtrl = afwMath.StatisticsControl()
632  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
633  statsCtrl.setNumIter(self.config.clipIter)
634  statsCtrl.setAndMask(mask)
635  statsCtrl.setNanSafe(True)
636  statsCtrl.setWeighted(True)
637  statsCtrl.setCalcErrorFromInputVariance(True)
638  for plane, threshold in self.config.maskPropagationThresholds.items():
639  bit = afwImage.Mask.getMaskPlane(plane)
640  statsCtrl.setMaskPropagationThreshold(bit, threshold)
641 
642  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
643 
644  if bgInfoList is None:
645  bgInfoList = [None]*len(tempExpRefList)
646 
647  if altMaskList is None:
648  altMaskList = [None]*len(tempExpRefList)
649 
650  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
651  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
652  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
653  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
654  coaddMaskedImage = coaddExposure.getMaskedImage()
655  subregionSizeArr = self.config.subregionSize
656  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
657  # if nImage is requested, create a zero one which can be passed to assembleSubregion
658  if self.config.doNImage:
659  nImage = afwImage.ImageU(skyInfo.bbox)
660  else:
661  nImage = None
662  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
663  try:
664  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
665  weightList, bgInfoList, altMaskList, statsFlags, statsCtrl,
666  nImage=nImage)
667  except Exception as e:
668  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
669 
670  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
671  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
672 
673  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
674  """!
675  \brief Set the metadata for the coadd
676 
677  This basic implementation simply sets the filter from the
678  first input.
679 
680  \param[in] coaddExposure: The target image for the coadd
681  \param[in] tempExpRefList: List of data references to tempExp
682  \param[in] weightList: List of weights
683  """
684  assert len(tempExpRefList) == len(weightList), "Length mismatch"
685  tempExpName = self.getTempExpDatasetName(self.warpType)
686  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
687  # (and we need more than just the PropertySet that contains the header), which is not possible
688  # with the current butler (see #2777).
689  tempExpList = [tempExpRef.get(tempExpName + "_sub",
690  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
691  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
692  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
693 
694  coaddExposure.setFilter(tempExpList[0].getFilter())
695  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
696  coaddInputs.ccds.reserve(numCcds)
697  coaddInputs.visits.reserve(len(tempExpList))
698 
699  for tempExp, weight in zip(tempExpList, weightList):
700  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
701  coaddInputs.visits.sort()
702  if self.warpType == "psfMatched":
703  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
704  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
705  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
706  # having the maximum width (sufficient because square)
707  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
708  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
709  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
710  else:
711  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
712  self.config.coaddPsf.makeControl())
713  coaddExposure.setPsf(psf)
714  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
715  coaddExposure.getWcs())
716  coaddExposure.getInfo().setApCorrMap(apCorrMap)
717 
718  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
719  bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None):
720  """!
721  \brief Assemble the coadd for a sub-region.
722 
723  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
724  matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
725  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
726  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
727  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
728  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
729  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
730 
731  \param[in] coaddExposure: The target image for the coadd
732  \param[in] bbox: Sub-region to coadd
733  \param[in] tempExpRefList: List of data reference to tempExp
734  \param[in] imageScalerList: List of image scalers
735  \param[in] weightList: List of weights
736  \param[in] bgInfoList: List of background data from background matching
737  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
738  \param[in] statsFlags: afwMath.Property object for statistic for coadd
739  \param[in] statsCtrl: Statistics control object for coadd
740  \param[in] nImage: optional ImageU keeps track of exposure count for each pixel
741  """
742  self.log.debug("Computing coadd over %s", bbox)
743  tempExpName = self.getTempExpDatasetName(self.warpType)
744  coaddMaskedImage = coaddExposure.getMaskedImage()
745  maskedImageList = []
746  if nImage is not None:
747  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
748  for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
749  altMaskList):
750  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
751  maskedImage = exposure.getMaskedImage()
752  if altMask:
753  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
754  maskedImage.getMask().swap(altMaskSub)
755  imageScaler.scaleMaskedImage(maskedImage)
756 
757  if self.config.doMatchBackgrounds and not bgInfo.isReference:
758  backgroundModel = bgInfo.backgroundModel
759  backgroundImage = backgroundModel.getImage() if \
760  self.matchBackgrounds.config.usePolynomial else \
761  backgroundModel.getImageF()
762  backgroundImage.setXY0(coaddMaskedImage.getXY0())
763  maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
764  var = maskedImage.getVariance()
765  var += (bgInfo.fitRMS)**2
766  # Add 1 for each pixel which is not excluded by the exclude mask.
767  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
768  if nImage is not None:
769  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
770  if self.config.removeMaskPlanes:
771  mask = maskedImage.getMask()
772  for maskPlane in self.config.removeMaskPlanes:
773  try:
774  mask &= ~mask.getPlaneBitMask(maskPlane)
775  except Exception as e:
776  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.message)
777 
778  maskedImageList.append(maskedImage)
779 
780  with self.timer("stack"):
781  coaddSubregion = afwMath.statisticsStack(
782  maskedImageList, statsFlags, statsCtrl, weightList)
783  coaddMaskedImage.assign(coaddSubregion, bbox)
784  if nImage is not None:
785  nImage.assign(subNImage, bbox)
786 
787  def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList):
788  """!
789  \brief Add metadata from the background matching to the coadd
790 
791  \param[in] coaddExposure: Coadd
792  \param[in] tempExpRefList: List of data references for temp exps to go into coadd
793  \param[in] backgroundInfoList: List of background info, results from background matching
794  """
795  self.log.info("Adding exposure information to metadata")
796  metadata = coaddExposure.getMetadata()
797  metadata.addString("CTExp_SDQA1_DESCRIPTION",
798  "Background matching: Ratio of matchedMSE / diffImVar")
799  for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
800  tempExpStr = '&'.join('%s=%s' % (k, v) for k, v in tempExpRef.dataId.items())
801  if backgroundInfo.isReference:
802  metadata.addString("ReferenceExp_ID", tempExpStr)
803  else:
804  metadata.addString("CTExp_ID_%d" % (ind), tempExpStr)
805  metadata.addDouble("CTExp_SDQA1_%d" % (ind),
806  backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
807  metadata.addDouble("CTExp_SDQA2_%d" % (ind),
808  backgroundInfo.fitRMS)
809 
810  def readBrightObjectMasks(self, dataRef):
811  """Returns None on failure"""
812  try:
813  return dataRef.get("brightObjectMask", immediate=True)
814  except Exception as e:
815  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
816  return None
817 
818  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
819  """Set the bright object masks
820 
821  exposure: Exposure under consideration
822  dataId: Data identifier dict for patch
823  brightObjectMasks: afwTable of bright objects to mask
824  """
825  #
826  # Check the metadata specifying the tract/patch/filter
827  #
828  if brightObjectMasks is None:
829  self.log.warn("Unable to apply bright object mask: none supplied")
830  return
831  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
832  md = brightObjectMasks.table.getMetadata()
833  for k in dataId:
834  if not md.exists(k):
835  self.log.warn("Expected to see %s in metadata", k)
836  else:
837  if md.get(k) != dataId[k]:
838  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
839 
840  mask = exposure.getMaskedImage().getMask()
841  wcs = exposure.getWcs()
842  plateScale = wcs.pixelScale().asArcseconds()
843 
844  for rec in brightObjectMasks:
845  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
846  if rec["type"] == "box":
847  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
848  width = rec["width"].asArcseconds()/plateScale # convert to pixels
849  height = rec["height"].asArcseconds()/plateScale # convert to pixels
850 
851  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
852  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
853 
854  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
855  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
856  spans = afwGeom.SpanSet(bbox)
857  elif rec["type"] == "circle":
858  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
859  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
860  else:
861  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
862  continue
863  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
864 
865  @classmethod
866  def _makeArgumentParser(cls):
867  """!
868  \brief Create an argument parser
869  """
870  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
871  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
872  cls.ConfigClass().warpType + "Warp",
873  help="data ID, e.g. --id tract=12345 patch=1,2",
874  ContainerClass=AssembleCoaddDataIdContainer)
875  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
876  ContainerClass=SelectDataIdContainer)
877  return parser
878 
879 
880 def _subBBoxIter(bbox, subregionSize):
881  """!
882  \brief Iterate over subregions of a bbox
883 
884  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
885  \param[in] subregionSize: size of sub-bboxes
886 
887  \return subBBox: next sub-bounding box of size subregionSize or smaller;
888  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
889  but it will never be empty
890  """
891  if bbox.isEmpty():
892  raise RuntimeError("bbox %s is empty" % (bbox,))
893  if subregionSize[0] < 1 or subregionSize[1] < 1:
894  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
895 
896  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
897  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
898  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
899  subBBox.clip(bbox)
900  if subBBox.isEmpty():
901  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
902  (bbox, subregionSize, colShift, rowShift))
903  yield subBBox
904 
905 
906 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
907  """!
908  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
909  """
910 
911  def makeDataRefList(self, namespace):
912  """!
913  \brief Make self.refList from self.idList.
914 
915  Interpret the config.doMatchBackgrounds, config.autoReference,
916  and whether a visit/run supplied.
917  If a visit/run is supplied, config.autoReference is automatically set to False.
918  if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
919 
920  """
921  keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd",
922  level=self.level)
923  keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
924  "Coadd_directWarp", level=self.level)
925 
926  if namespace.config.doMatchBackgrounds:
927  if namespace.config.autoReference: # matcher will pick it's own reference image
928  datasetType = namespace.config.coaddName + "Coadd"
929  validKeys = keysCoadd
930  else:
931  datasetType = namespace.config.coaddName + "Coadd_directWarp"
932  validKeys = keysCoaddTempExp
933  else: # bkg subtracted coadd
934  datasetType = namespace.config.coaddName + "Coadd"
935  validKeys = keysCoadd
936 
937  for dataId in self.idList:
938  # tract and patch are required
939  for key in validKeys:
940  if key not in dataId:
941  raise RuntimeError("--id must include " + key)
942 
943  for key in dataId: # check if users supplied visit/run
944  if (key not in keysCoadd) and (key in keysCoaddTempExp): # user supplied a visit/run
945  if namespace.config.autoReference:
946  # user probably meant: autoReference = False
947  namespace.config.autoReference = False
948  datasetType = namespace.config.coaddName + "Coadd_directWarp"
949  print("Switching config.autoReference to False; applies only to background Matching.")
950  break
951 
952  dataRef = namespace.butler.dataRef(
953  datasetType=datasetType,
954  dataId=dataId,
955  )
956  self.refList.append(dataRef)
957 
958 
959 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
960  """!
961  \brief Function to count the number of pixels with a specific mask in a footprint.
962 
963  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
964  have bitmask set but do not have ignoreMask set. Return the count.
965 
966  \param[in] mask: mask to define intersection region by.
967  \parma[in] footprint: footprint to define the intersection region by.
968  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
969  \param[in] ignoreMask: pixels to not consider.
970  \return count of number of pixels in footprint with specified mask.
971  """
972  bbox = footprint.getBBox()
973  bbox.clip(mask.getBBox(afwImage.PARENT))
974  fp = afwImage.Mask(bbox)
975  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
976  footprint.spans.setMask(fp, bitmask)
977  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
978  (subMask.getArray() & ignoreMask) == 0).sum()
979 
980 
982  """!
983 \anchor SafeClipAssembleCoaddConfig
984 
985 \brief Configuration parameters for the SafeClipAssembleCoaddTask
986  """
987  clipDetection = pexConfig.ConfigurableField(
988  target=SourceDetectionTask,
989  doc="Detect sources on difference between unclipped and clipped coadd")
990  minClipFootOverlap = pexConfig.Field(
991  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
992  dtype=float,
993  default=0.6
994  )
995  minClipFootOverlapSingle = pexConfig.Field(
996  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
997  "clipped when only one visit overlaps",
998  dtype=float,
999  default=0.5
1000  )
1001  minClipFootOverlapDouble = pexConfig.Field(
1002  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1003  "clipped when two visits overlap",
1004  dtype=float,
1005  default=0.45
1006  )
1007  maxClipFootOverlapDouble = pexConfig.Field(
1008  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1009  "considering two visits",
1010  dtype=float,
1011  default=0.15
1012  )
1013  minBigOverlap = pexConfig.Field(
1014  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1015  "when labeling clipped footprints",
1016  dtype=int,
1017  default=100
1018  )
1019 
1020  def setDefaults(self):
1021  # The numeric values for these configuration parameters were empirically determined, future work
1022  # may further refine them.
1023  AssembleCoaddConfig.setDefaults(self)
1024  self.clipDetection.doTempLocalBackground = False
1025  self.clipDetection.reEstimateBackground = False
1026  self.clipDetection.returnOriginalFootprints = False
1027  self.clipDetection.thresholdPolarity = "both"
1028  self.clipDetection.thresholdValue = 2
1029  self.clipDetection.nSigmaToGrow = 2
1030  self.clipDetection.minPixels = 4
1031  self.clipDetection.isotropicGrow = True
1032  self.clipDetection.thresholdType = "pixel_stdev"
1033  self.sigmaClip = 1.5
1034  self.clipIter = 3
1035  self.statistic = "MEAN"
1036 
1037  def validate(self):
1038  if self.doSigmaClip:
1039  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1040  "Ignoring doSigmaClip.")
1041  self.doSigmaClip = False
1042  if self.statistic != "MEAN":
1043  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1044  "(%s chosen). Please set statistic to MEAN."
1045  % (self.statistic))
1046  AssembleCoaddTask.ConfigClass.validate(self)
1047 
1048 
1049 
1055 
1056 
1058  """!
1059  \anchor SafeClipAssembleCoaddTask_
1060 
1061  \brief Assemble a coadded image from a set of coadded temporary exposures,
1062  being careful to clip & flag areas with potential artifacts.
1063 
1064  \section pipe_tasks_assembleCoadd_Contents Contents
1065  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
1066  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
1067  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
1068  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
1069  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
1070  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
1071 
1072  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
1073 
1074  \copybrief SafeClipAssembleCoaddTask
1075 
1076  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
1077  SafeClipAssembleCoaddTask subtasks that task.
1078  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1079  outliers).
1080  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1081  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1082  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
1083  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
1084  that there is an outlier and ii. this outlier appears on only one or two images.
1085  Such regions will not contribute to the final coadd.
1086  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1087  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1088  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1089  surveys.
1090 
1091  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1092  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1093  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1094 
1095  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1096  \copydoc \_\_init\_\_
1097 
1098  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1099  \copydoc run
1100 
1101  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1102  See \ref SafeClipAssembleCoaddConfig
1103 
1104  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1105  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1106  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1107  files.
1108  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1109  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1110  for further information.
1111 
1112  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using
1113  SafeClipAssembleCoaddTask
1114 
1115  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1116  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1117  '--legacyCoadd'.
1118  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1119  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1120  with a list of coaddTempExps to attempt to coadd (specified using
1121  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1122  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1123  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1124  command line argument:
1125  \code
1126  assembleCoadd.py --help
1127  \endcode
1128  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1129  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1130  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1131  packages.
1132  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1133  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1134  <DL>
1135  <DT>processCcd</DT>
1136  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1137  <DT>makeSkyMap</DT>
1138  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1139  <DT>makeCoaddTempExp</DT>
1140  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1141  </DL>
1142  We can perform all of these steps by running
1143  \code
1144  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1145  \endcode
1146  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1147  as follows:
1148  \code
1149  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1150  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1151  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1152  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1153  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1154  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1155  --selectId visit=903988 ccd=24
1156  \endcode
1157  This will process the HSC-I band data. The results are written in
1158  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1159 
1160  You may also choose to run:
1161  \code
1162  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1163  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1164  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1165  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1166  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1167  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1168  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1169  --selectId visit=903346 ccd=12
1170  \endcode
1171  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1172  discussed in \ref pipeTasks_multiBand.
1173  """
1174  ConfigClass = SafeClipAssembleCoaddConfig
1175  _DefaultName = "safeClipAssembleCoadd"
1176 
1177  def __init__(self, *args, **kwargs):
1178  """!
1179  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1180  """
1181  AssembleCoaddTask.__init__(self, *args, **kwargs)
1182  schema = afwTable.SourceTable.makeMinimalSchema()
1183  self.makeSubtask("clipDetection", schema=schema)
1184 
1185  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1186  *args, **kwargs):
1187  """!
1188  \brief Assemble the coadd for a region
1189 
1190  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1191  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1192  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1193  overlap between the clipped region and a source.
1194  This leaves us with a set of footprints from the difference image that have been identified as having
1195  occured on just one or two individual visits. However, these footprints were generated from a
1196  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1197  footprints acrosss the coadd difference in this process.
1198  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1199  these are big footprints.
1200  Combine the small and big clipped footprints and mark them on a new bad mask plane
1201  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1202  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1203  bad mask plane.
1204 
1205  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1206  parent task.
1207 
1208  @param skyInfo: Patch geometry information, from getSkyInfo
1209  @param tempExpRefList: List of data reference to tempExp
1210  @param imageScalerList: List of image scalers
1211  @param weightList: List of weights
1212  @param bgModelList: List of background models from background matching
1213  return pipeBase.Struct with coaddExposure, nImage
1214  """
1215  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1216  mask = exp.getMaskedImage().getMask()
1217  mask.addMaskPlane("CLIPPED")
1218 
1219  result = self.detectClip(exp, tempExpRefList)
1220 
1221  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1222 
1223  # Go to individual visits for big footprints
1224  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1225  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1226  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1227  maskClipValue, maskDetValue)
1228 
1229  # Create mask of the current clipped footprints
1230  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1231  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1232 
1233  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1234  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1235  maskClip |= maskClipBig
1236 
1237  # Assemble coadd from base class, but ignoring CLIPPED pixels
1238  badMaskPlanes = self.config.badMaskPlanes[:]
1239  badMaskPlanes.append("CLIPPED")
1240  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1241  retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1242  bgModelList, result.tempExpClipList, mask=badPixelMask)
1243 
1244  # Set the coadd CLIPPED mask from the footprints since currently pixels that are masked
1245  # do not get propagated
1246  maskExp = retStruct.coaddExposure.getMaskedImage().getMask()
1247  maskExp |= maskClip
1248 
1249  return retStruct
1250 
1251  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):
1252  """!
1253  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1254 
1255  Generate a difference image between clipped and unclipped coadds.
1256  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1257  Return the difference image.
1258 
1259  @param skyInfo: Patch geometry information, from getSkyInfo
1260  @param tempExpRefList: List of data reference to tempExp
1261  @param imageScalerList: List of image scalers
1262  @param weightList: List of weights
1263  @param bgModelList: List of background models from background matching
1264  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1265  """
1266  # Clone and upcast self.config because current self.config is frozen
1267  config = AssembleCoaddConfig()
1268  # getattr necessary because subtasks do not survive Config.toDict()
1269  configIntersection = {k: getattr(self.config, k)
1270  for k, v in self.config.toDict().items() if (k in config.keys())}
1271  config.update(**configIntersection)
1272 
1273  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1274  config.statistic = 'MEAN'
1275  task = AssembleCoaddTask(config=config)
1276  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1277  bgModelList).coaddExposure
1278 
1279  config.statistic = 'MEANCLIP'
1280  task = AssembleCoaddTask(config=config)
1281  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1282  bgModelList).coaddExposure
1283 
1284  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1285  coaddDiff -= coaddClip.getMaskedImage()
1286  exp = afwImage.ExposureF(coaddDiff)
1287  exp.setPsf(coaddMean.getPsf())
1288  return exp
1289 
1290  def detectClip(self, exp, tempExpRefList):
1291  """!
1292  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1293 
1294  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1295  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1296  fraction to thresholds set in the config.
1297  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1298  two).
1299  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1300  the coaddTempExp.
1301  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1302  with the clipped footprints and a list of new masks for the coaddTempExps.
1303 
1304  \param[in] exp: Exposure to run detection on
1305  \param[in] tempExpRefList: List of data reference to tempExp
1306  \return struct containing:
1307  - clippedFootprints: list of clipped footprints
1308  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1309  - tempExpClipList: list of new masks for tempExp
1310  """
1311  mask = exp.getMaskedImage().getMask()
1312  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1313  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1314  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1315  # Merge positive and negative together footprints together
1316  fpSet.positive.merge(fpSet.negative)
1317  footprints = fpSet.positive
1318  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1319  ignoreMask = self.getBadPixelMask()
1320 
1321  clipFootprints = []
1322  clipIndices = []
1323 
1324  # build a list with a mask for each visit which can be modified with clipping information
1325  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.warpType),
1326  immediate=True).getMaskedImage().getMask() for
1327  tmpExpRef in tempExpRefList]
1328 
1329  for footprint in footprints.getFootprints():
1330  nPixel = footprint.getArea()
1331  overlap = [] # hold the overlap with each visit
1332  maskList = [] # which visit mask match
1333  indexList = [] # index of visit in global list
1334  for i, tmpExpMask in enumerate(tempExpClipList):
1335  # Determine the overlap with the footprint
1336  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1337  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1338  totPixel = nPixel - ignore
1339 
1340  # If we have more bad pixels than detection skip
1341  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1342  continue
1343  overlap.append(overlapDet/float(totPixel))
1344  maskList.append(tmpExpMask)
1345  indexList.append(i)
1346 
1347  overlap = numpy.array(overlap)
1348  if not len(overlap):
1349  continue
1350 
1351  keep = False # Should this footprint be marked as clipped?
1352  keepIndex = [] # Which tempExps does the clipped footprint belong to
1353 
1354  # If footprint only has one overlap use a lower threshold
1355  if len(overlap) == 1:
1356  if overlap[0] > self.config.minClipFootOverlapSingle:
1357  keep = True
1358  keepIndex = [0]
1359  else:
1360  # This is the general case where only visit should be clipped
1361  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1362  if len(clipIndex) == 1:
1363  keep = True
1364  keepIndex = [clipIndex[0]]
1365 
1366  # Test if there are clipped objects that overlap two different visits
1367  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1368  if len(clipIndex) == 2 and len(overlap) > 3:
1369  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1370  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1371  keep = True
1372  keepIndex = clipIndex
1373 
1374  if not keep:
1375  continue
1376 
1377  for index in keepIndex:
1378  footprint.spans.setMask(maskList[index], maskClipValue)
1379 
1380  clipIndices.append(numpy.array(indexList)[keepIndex])
1381  clipFootprints.append(footprint)
1382 
1383  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1384  tempExpClipList=tempExpClipList)
1385 
1386  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1387  """!
1388  \brief Find footprints from individual tempExp footprints for large footprints.
1389 
1390  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1391  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1392  significantly with each source in all the coaddTempExps.
1393 
1394  \param[in] tempExpClipList: List of tempExp masks with clipping information
1395  \param[in] clipFootprints: List of clipped footprints
1396  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1397  \param[in] maskClipValue: Mask value of clipped pixels
1398  \param[in] maskClipValue: Mask value of detected pixels
1399  \return list of big footprints
1400  """
1401  bigFootprintsCoadd = []
1402  ignoreMask = self.getBadPixelMask()
1403  for index, tmpExpMask in enumerate(tempExpClipList):
1404 
1405  # Create list of footprints from the DETECTED pixels
1406  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1407  afwImage.PARENT, True)
1408  maskVisitDet &= maskDetValue
1409  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1410 
1411  # build a mask of clipped footprints that are in this visit
1412  clippedFootprintsVisit = []
1413  for foot, clipIndex in zip(clipFootprints, clipIndices):
1414  if index not in clipIndex:
1415  continue
1416  clippedFootprintsVisit.append(foot)
1417  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1418  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1419 
1420  bigFootprintsVisit = []
1421  for foot in visitFootprints.getFootprints():
1422  if foot.getArea() < self.config.minBigOverlap:
1423  continue
1424  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1425  if nCount > self.config.minBigOverlap:
1426  bigFootprintsVisit.append(foot)
1427  bigFootprintsCoadd.append(foot)
1428 
1429  # Update single visit masks
1430  maskVisitClip.clearAllMaskPlanes()
1431  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1432  tmpExpMask |= maskVisitClip
1433 
1434  return bigFootprintsCoadd
1435 
1436 
1438  assembleStaticSkyModel = pexConfig.ConfigurableField(
1439  target=AssembleCoaddTask,
1440  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1441  " naive/first-iteration model of the static sky.",
1442  )
1443  chiThreshold = pexConfig.RangeField(
1444  doc="Detection threshold (sigma) for artifacts in warp diff "
1445  "(chi-image of PSF-Matched warp minus the model of the static sky)",
1446  dtype=float,
1447  default=5,
1448  min=0,
1449  )
1450  minPixels = pexConfig.Field(
1451  doc="Minimum number of pixels in a region (in a warp diff chi-image) "
1452  "above chiThreshold to trigger masking. "
1453  "Detected artifacts with fewer than the specified number of pixels will be ignored.",
1454  dtype=int,
1455  default=1
1456  )
1457  doMaskNegative = pexConfig.Field(
1458  doc="Also mask outlier regions of flux less than the static sky model?",
1459  dtype=bool,
1460  default=True
1461  )
1462  temporalThreshold = pexConfig.Field(
1463  doc="Unitless fraction of number of epochs to classify as an artifact/outlier source versus"
1464  " a source that is intrinsically variable or difficult to subtract cleanly. "
1465  "If outlier region in warp-diff Chi-image is mostly (defined by spatialThreshold) "
1466  "an outlier in less than temporalThreshold * number of epochs, then mask. "
1467  "Otherwise, do not mask.",
1468  dtype=float,
1469  default=0.075
1470  )
1471  spatialThreshold = pexConfig.Field(
1472  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1473  "temporal criteria",
1474  dtype=float,
1475  default=0.5
1476  )
1477  growMaskBy = pexConfig.Field(
1478  doc="Number of pixels by which to grow the masks on artifacts.",
1479  dtype=int,
1480  default=5
1481  )
1482 
1483  def setDefaults(self):
1484  AssembleCoaddConfig.setDefaults(self)
1485  self.assembleStaticSkyModel.warpType = 'psfMatched'
1486  self.assembleStaticSkyModel.statistic = 'MEDIAN'
1487  self.assembleStaticSkyModel.doWrite = False
1488  self.statistic = 'MEAN'
1489 
1490 
1491 
1497 
1499  """!
1500  \anchor CompareWarpAssembleCoaddTask_
1501 
1502  \brief Assemble a compareWarp coadded image from a set of warps
1503  by masking artifacts detected by comparing PSF-matched warps
1504 
1505  \section pipe_tasks_assembleCoadd_Contents Contents
1506  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose
1507  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize
1508  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run
1509  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config
1510  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug
1511  - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example
1512 
1513  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description
1514 
1515  \copybrief CompareWarpAssembleCoaddTask
1516 
1517  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1518  outliers).
1519  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1520  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1521  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks
1522  pixels in the individual warps suspected to contain an artifact.
1523  We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd
1524  which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched
1525  median coadd by more than config.chiThreshold sigma, is an artifact candidate. The candidates are then
1526  filtered to remove variable sources and sources that are difficult to subtract such as bright stars.
1527  This filter is configured using the config parameters temporalThreshold and spatialThreshold.
1528  The temporalThreshold is the maximum fraction of epochs that the deviation can
1529  appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in
1530  the footprint of the deviation that appear in other epochs (where other epochs is defined by the
1531  temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels
1532  that deviate in only a few epochs, it is grown by growMaskBy and bits set in the 'CLIPPED' plane.
1533  These regions will not contribute to the final coadd.
1534  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1535  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1536  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1537  surveys.
1538 
1539  CompareWarpAssembleCoaddTask sub-classes
1540  \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask"
1541  as a subtask to generate the TemplateCoadd (the model of the static sky)
1542 
1543  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization
1544  \copydoc \_\_init\_\_
1545 
1546  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task
1547  \copydoc run
1548 
1549  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters
1550  See \ref CompareWarpAssembleCoaddConfig
1551 
1552  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables
1553  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1554  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1555  files.
1556  CompareWarpAssembleCoaddTask has no debug variables of its own.
1557 
1558  \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using
1559  CompareWarpAssembleCoaddTask
1560 
1561  CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image.
1562  The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag
1563  '--compareWarpCoadd'.
1564  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1565  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1566  with a list of coaddTempExps to attempt to coadd (specified using
1567  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1568  Only the warps that cover the specified tract and patch will be coadded.
1569  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1570  command line argument:
1571  \code
1572  assembleCoadd.py --help
1573  \endcode
1574  To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing,
1575  we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package.
1576  To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1577  packages.
1578  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1579  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1580  <DL>
1581  <DT>processCcd</DT>
1582  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1583  <DT>makeSkyMap</DT>
1584  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1585  <DT>makeCoaddTempExp</DT>
1586  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1587  </DL>
1588  We can perform all of these steps by running
1589  \code
1590  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1591  \endcode
1592  This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py
1593  as follows:
1594  \code
1595  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1596  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1597  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1598  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1599  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1600  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1601  --selectId visit=903988 ccd=24
1602  \endcode
1603  This will process the HSC-I band data. The results are written in
1604  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
1605  """
1606  ConfigClass = CompareWarpAssembleCoaddConfig
1607  _DefaultName = "compareWarpAssembleCoadd"
1608 
1609  def __init__(self, *args, **kwargs):
1610  """!
1611  \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask.
1612  """
1613  AssembleCoaddTask.__init__(self, *args, **kwargs)
1614  self.makeSubtask("assembleStaticSkyModel")
1615 
1616  def makeSupplementaryData(self, dataRef, selectDataList):
1617  """!
1618  \brief Make inputs specific to Subclass
1619 
1620  Generate a templateCoadd to use as a native model of static sky to subtract from warps.
1621  """
1622  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList).coaddExposure
1623  return pipeBase.Struct(templateCoadd=templateCoadd)
1624 
1625  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1626  supplementaryData, *args, **kwargs):
1627  """!
1628  \brief Assemble the coadd
1629 
1630  Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves
1631  as the model of the static sky.
1632 
1633  Find artifacts and apply them to the warps' masks creating a list of alternative masks with a
1634  new "CLIPPED" plane and updated "NO_DATA" plane.
1635  Then pass these alternative masks to the base class's assemble method.
1636 
1637  @param skyInfo: Patch geometry information
1638  @param tempExpRefList: List of data references to warps
1639  @param imageScalerList: List of image scalers
1640  @param weightList: List of weights
1641  @param bgModelList: List of background models from background matching
1642  @param supplementaryData: PipeBase.Struct containing a templateCoadd
1643 
1644  return pipeBase.Struct with coaddExposure, nImage if requested
1645  """
1646  templateCoadd = supplementaryData.templateCoadd
1647  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1648  maskList = self.computeAltMaskList(tempExpRefList, spanSetMaskList)
1649  badMaskPlanes = self.config.badMaskPlanes[:]
1650  badMaskPlanes.append("CLIPPED")
1651  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1652 
1653  retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1654  bgModelList, maskList, mask=badPixelMask)
1655  return retStruct
1656 
1657  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1658  """!
1659  \brief Find artifacts
1660 
1661  Loop through warps twice. The first loop builds a map with the count of how many
1662  epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma.
1663  The second loop takes each difference image and filters the artifacts detected
1664  in each using count map to filter out variable sources and sources that are difficult to
1665  subtract cleanly.
1666 
1667  @param templateCoadd: Exposure to serve as model of static sky
1668  @param tempExpRefList: List of data references to warps
1669  @param imageScalerList: List of image scalers
1670  """
1671 
1672  self.log.debug("Generating Count Image. First loop through warps")
1673  epochCountImage = afwImage.ImageU(templateCoadd.getBBox())
1674  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1675  mi = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1676  if mi is None:
1677  continue
1678  chiIm = self._makeChiIm(mi)
1679  chiOneMap = self._snrToBinaryIm(chiIm)
1680  epochCountImage += chiOneMap
1681 
1682  self.log.debug("Generating Mask List. Second loop through warps")
1683  spanSetArtifactList = []
1684  spanSetNoDataMaskList = []
1685 
1686  maxNumEpochs = int(max(1, self.config.temporalThreshold*len(tempExpRefList)))
1687  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1688  mi = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1689  if mi is not None:
1690  chiIm = self._makeChiIm(mi)
1691  chiOneMap = self._snrToBinaryArr(chiIm.array)
1692  outliers = afwImage.makeMaskFromArray(chiOneMap.astype(afwImage.MaskPixel))
1693  outliers.setXY0(mi.getXY0())
1694  spanSetList = afwGeom.SpanSet.fromMask(outliers).split()
1695  else:
1696  chiIm = afwImage.ImageF(templateCoadd.getBBox(), numpy.nan)
1697  spanSetList = []
1698 
1699  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1700  # undergo a second convolution. Pixels with data in the direct warp
1701  # but not in the PSF-matched warp will not have their artifacts detected.
1702  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1703  nans = numpy.where(numpy.isnan(chiIm.array), 1, 0)
1704  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1705  nansMask.setXY0(chiIm.getXY0())
1706  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1707 
1708  filteredSpanSetList = self._filterArtifacts(spanSetList, epochCountImage,
1709  maxNumEpochs=maxNumEpochs)
1710  dilatedSpanSetList = [s.dilated(self.config.growMaskBy, afwGeom.Stencil.CIRCLE)
1711  for s in filteredSpanSetList]
1712  spanSetArtifactList.append(dilatedSpanSetList)
1713  spanSetNoDataMaskList.append(spanSetNoDataMask)
1714  return pipeBase.Struct(artifacts=spanSetArtifactList,
1715  noData=spanSetNoDataMaskList)
1716 
1717  def computeAltMaskList(self, tempExpRefList, maskSpanSets):
1718  """!
1719  \brief Apply artifact span set lists to masks
1720 
1721  @param tempExpRefList: List of data references to warps
1722  @param maskSpanSets: Struct containing artifact and noData spanSet lists to apply
1723 
1724  return List of alternative masks
1725 
1726  Add artifact span set list as "CLIPPED" plane and NaNs to existing "NO_DATA" plane
1727  """
1728  spanSetMaskList = maskSpanSets.artifacts
1729  spanSetNoDataList = maskSpanSets.noData
1730  altMaskList = []
1731  for warpRef, artifacts, noData in zip(tempExpRefList, spanSetMaskList, spanSetNoDataList):
1732  warp = warpRef.get(self.getTempExpDatasetName(self.config.warpType), immediate=True)
1733  mask = warp.maskedImage.mask
1734  maskClipValue = mask.addMaskPlane("CLIPPED")
1735  noDataValue = mask.addMaskPlane("NO_DATA")
1736  for artifact in artifacts:
1737  artifact.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1738  for noDataRegion in noData:
1739  noDataRegion.clippedTo(mask.getBBox()).setMask(mask, 2**noDataValue)
1740  altMaskList.append(mask)
1741  return altMaskList
1742 
1743  def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None, minPixels=None):
1744  if minPixels is None:
1745  minPixels = self.config.minPixels
1746  maskSpanSetList = []
1747  x0, y0 = epochCountImage.getXY0()
1748  for i, span in enumerate(spanSetList):
1749  y, x = span.indices()
1750  if len(y) < minPixels:
1751  continue
1752  counts = epochCountImage.array[[y1 - y0 for y1 in y], [x1 - x0 for x1 in x]]
1753  idx = numpy.where((counts > 0) & (counts <= maxNumEpochs))
1754  percentBelowThreshold = len(idx[0]) / len(counts)
1755  if percentBelowThreshold > self.config.spatialThreshold:
1756  maskSpanSetList.append(span)
1757  return maskSpanSetList
1758 
1759  def _snrToBinaryArr(self, arrIn):
1760  with numpy.errstate(invalid='ignore'):
1761  arr = numpy.where(arrIn >= self.config.chiThreshold, 1, 0)
1762  if self.config.doMaskNegative:
1763  arr[numpy.where(arrIn <= -self.config.chiThreshold)] = 1
1764  return arr
1765 
1766  def _snrToBinaryIm(self, im):
1767  arr = self._snrToBinaryArr(im.array)
1768  return afwImage.ImageU(arr.astype(numpy.uint16), xy0=im.getXY0())
1769 
1770  def _makeChiIm(self, maskedImage):
1771  chiIm = maskedImage.image.Factory(1./numpy.sqrt(maskedImage.variance.array))
1772  chiIm *= maskedImage.image
1773  chiIm.setXY0(maskedImage.getXY0())
1774  return chiIm
1775 
1776  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1777  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
1778  warpName = self.getTempExpDatasetName('psfMatched')
1779  if not warpRef.datasetExists(warpName):
1780  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
1781  return None
1782  warp = warpRef.get(self.getTempExpDatasetName('psfMatched'), immediate=True)
1783  # direct image scaler OK for PSF-matched Warp
1784  imageScaler.scaleMaskedImage(warp.getMaskedImage())
1785  mi = warp.getMaskedImage()
1786  mi -= templateCoadd.getMaskedImage()
1787  return mi
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:167
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
Definition: coaddBase.py:90
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None)
Perform background matching on the prepared inputs.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, args, kwargs)
Assemble the coadd for a region.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:123
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:182
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:217
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def computeAltMaskList(self, tempExpRefList, maskSpanSets)
Apply artifact span set lists to masks.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:103
def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue)
Find footprints from individual tempExp footprints for large footprints.
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None, minPixels=None)
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def getBackgroundReferenceScaler(self, dataRef)
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList)
Add metadata from the background matching to the coadd.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, supplementaryData, args, kwargs)
Assemble the coadd.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:63