Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from .healSparseMapping import HealSparseInputMapTask 

48from lsst.meas.algorithms import SourceDetectionTask 

49from lsst.daf.butler import DeferredDatasetHandle 

50 

51__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

52 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

53 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

54 

55 

56class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

57 dimensions=("tract", "patch", "band", "skymap"), 

58 defaultTemplates={"inputCoaddName": "deep", 

59 "outputCoaddName": "deep", 

60 "warpType": "direct", 

61 "warpTypeSuffix": "", 

62 "fakesType": ""}): 

63 inputWarps = pipeBase.connectionTypes.Input( 

64 doc=("Input list of warps to be assemebled i.e. stacked." 

65 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

66 name="{inputCoaddName}Coadd_{warpType}Warp", 

67 storageClass="ExposureF", 

68 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

69 deferLoad=True, 

70 multiple=True 

71 ) 

72 skyMap = pipeBase.connectionTypes.Input( 

73 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

75 storageClass="SkyMap", 

76 dimensions=("skymap", ), 

77 ) 

78 selectedVisits = pipeBase.connectionTypes.Input( 

79 doc="Selected visits to be coadded.", 

80 name="{outputCoaddName}Visits", 

81 storageClass="StructuredDataDict", 

82 dimensions=("instrument", "tract", "patch", "skymap", "band") 

83 ) 

84 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

85 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

86 " BRIGHT_OBJECT."), 

87 name="brightObjectMask", 

88 storageClass="ObjectMaskCatalog", 

89 dimensions=("tract", "patch", "skymap", "band"), 

90 ) 

91 coaddExposure = pipeBase.connectionTypes.Output( 

92 doc="Output coadded exposure, produced by stacking input warps", 

93 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

94 storageClass="ExposureF", 

95 dimensions=("tract", "patch", "skymap", "band"), 

96 ) 

97 nImage = pipeBase.connectionTypes.Output( 

98 doc="Output image of number of input images per pixel", 

99 name="{outputCoaddName}Coadd_nImage", 

100 storageClass="ImageU", 

101 dimensions=("tract", "patch", "skymap", "band"), 

102 ) 

103 inputMap = pipeBase.connectionTypes.Output( 

104 doc="Output healsparse map of input images", 

105 name="{outputCoaddName}Coadd_inputMap", 

106 storageClass="HealSparseMap", 

107 dimensions=("tract", "patch", "skymap", "band"), 

108 ) 

109 

110 def __init__(self, *, config=None): 

111 super().__init__(config=config) 

112 

113 # Override the connection's name template with config to replicate Gen2 behavior 

114 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

115 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

116 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

117 templateValues['warpType'] = config.warpType 

118 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

119 if config.hasFakes: 

120 templateValues['fakesType'] = "_fakes" 

121 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

122 for name in self.allConnections} 

123 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

124 # End code to remove after deprecation 

125 

126 if not config.doMaskBrightObjects: 

127 self.prerequisiteInputs.remove("brightObjectMask") 

128 

129 if not config.doSelectVisits: 

130 self.inputs.remove("selectedVisits") 

131 

132 if not config.doNImage: 

133 self.outputs.remove("nImage") 

134 

135 if not self.config.doInputMap: 

136 self.outputs.remove("inputMap") 

137 

138 

139class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

140 pipelineConnections=AssembleCoaddConnections): 

141 """Configuration parameters for the `AssembleCoaddTask`. 

142 

143 Notes 

144 ----- 

145 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

146 only set the bitplane config.brightObjectMaskName. To make this useful you 

147 *must* also configure the flags.pixel algorithm, for example by adding 

148 

149 .. code-block:: none 

150 

151 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

152 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

153 

154 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

155 """ 

156 warpType = pexConfig.Field( 

157 doc="Warp name: one of 'direct' or 'psfMatched'", 

158 dtype=str, 

159 default="direct", 

160 ) 

161 subregionSize = pexConfig.ListField( 

162 dtype=int, 

163 doc="Width, height of stack subregion size; " 

164 "make small enough that a full stack of images will fit into memory at once.", 

165 length=2, 

166 default=(2000, 2000), 

167 ) 

168 statistic = pexConfig.Field( 

169 dtype=str, 

170 doc="Main stacking statistic for aggregating over the epochs.", 

171 default="MEANCLIP", 

172 ) 

173 doSigmaClip = pexConfig.Field( 

174 dtype=bool, 

175 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

176 default=False, 

177 ) 

178 sigmaClip = pexConfig.Field( 

179 dtype=float, 

180 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

181 default=3.0, 

182 ) 

183 clipIter = pexConfig.Field( 

184 dtype=int, 

185 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

186 default=2, 

187 ) 

188 calcErrorFromInputVariance = pexConfig.Field( 

189 dtype=bool, 

190 doc="Calculate coadd variance from input variance by stacking statistic." 

191 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

192 default=True, 

193 ) 

194 scaleZeroPoint = pexConfig.ConfigurableField( 

195 target=ScaleZeroPointTask, 

196 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

197 ) 

198 doInterp = pexConfig.Field( 

199 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

200 dtype=bool, 

201 default=True, 

202 ) 

203 interpImage = pexConfig.ConfigurableField( 

204 target=InterpImageTask, 

205 doc="Task to interpolate (and extrapolate) over NaN pixels", 

206 ) 

207 doWrite = pexConfig.Field( 

208 doc="Persist coadd?", 

209 dtype=bool, 

210 default=True, 

211 ) 

212 doNImage = pexConfig.Field( 

213 doc="Create image of number of contributing exposures for each pixel", 

214 dtype=bool, 

215 default=False, 

216 ) 

217 doUsePsfMatchedPolygons = pexConfig.Field( 

218 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

219 dtype=bool, 

220 default=False, 

221 ) 

222 maskPropagationThresholds = pexConfig.DictField( 

223 keytype=str, 

224 itemtype=float, 

225 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

226 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

227 "would have contributed exceeds this value."), 

228 default={"SAT": 0.1}, 

229 ) 

230 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

231 doc="Mask planes to remove before coadding") 

232 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

233 doc="Set mask and flag bits for bright objects?") 

234 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

235 doc="Name of mask bit used for bright objects") 

236 coaddPsf = pexConfig.ConfigField( 

237 doc="Configuration for CoaddPsf", 

238 dtype=measAlg.CoaddPsfConfig, 

239 ) 

240 doAttachTransmissionCurve = pexConfig.Field( 

241 dtype=bool, default=False, optional=False, 

242 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

243 "(requires all input Exposures to have TransmissionCurves).") 

244 ) 

245 hasFakes = pexConfig.Field( 

246 dtype=bool, 

247 default=False, 

248 doc="Should be set to True if fake sources have been inserted into the input data." 

249 ) 

250 doSelectVisits = pexConfig.Field( 

251 doc="Coadd only visits selected by a SelectVisitsTask", 

252 dtype=bool, 

253 default=False, 

254 ) 

255 doInputMap = pexConfig.Field( 

256 doc="Create a bitwise map of coadd inputs", 

257 dtype=bool, 

258 default=False, 

259 ) 

260 inputMapper = pexConfig.ConfigurableField( 

261 doc="Input map creation subtask.", 

262 target=HealSparseInputMapTask, 

263 ) 

264 

265 def setDefaults(self): 

266 super().setDefaults() 

267 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

268 

269 def validate(self): 

270 super().validate() 

271 if self.doPsfMatch: 

272 # Backwards compatibility. 

273 # Configs do not have loggers 

274 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

275 self.warpType = 'psfMatched' 

276 if self.doSigmaClip and self.statistic != "MEANCLIP": 

277 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

278 self.statistic = "MEANCLIP" 

279 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

280 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

281 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

282 

283 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

284 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

285 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

286 if str(k) not in unstackableStats] 

287 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

288 % (self.statistic, stackableStats)) 

289 

290 

291class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

292 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

293 

294 We want to assemble a coadded image from a set of Warps (also called 

295 coadded temporary exposures or ``coaddTempExps``). 

296 Each input Warp covers a patch on the sky and corresponds to a single 

297 run/visit/exposure of the covered patch. We provide the task with a list 

298 of Warps (``selectDataList``) from which it selects Warps that cover the 

299 specified patch (pointed at by ``dataRef``). 

300 Each Warp that goes into a coadd will typically have an independent 

301 photometric zero-point. Therefore, we must scale each Warp to set it to 

302 a common photometric zeropoint. WarpType may be one of 'direct' or 

303 'psfMatched', and the boolean configs `config.makeDirect` and 

304 `config.makePsfMatched` set which of the warp types will be coadded. 

305 The coadd is computed as a mean with optional outlier rejection. 

306 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

307 Finally, Warps can have bad 'NaN' pixels which received no input from the 

308 source calExps. We interpolate over these bad (NaN) pixels. 

309 

310 `AssembleCoaddTask` uses several sub-tasks. These are 

311 

312 - `ScaleZeroPointTask` 

313 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

314 - `InterpImageTask` 

315 - interpolate across bad pixels (NaN) in the final coadd 

316 

317 You can retarget these subtasks if you wish. 

318 

319 Notes 

320 ----- 

321 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

322 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

323 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

324 no debug variables of its own. Some of the subtasks may support debug 

325 variables. See the documentation for the subtasks for further information. 

326 

327 Examples 

328 -------- 

329 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

330 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

331 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

332 inputs: a data reference to the tract patch and filter to be coadded, and 

333 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

334 ``--selectId``, respectively: 

335 

336 .. code-block:: none 

337 

338 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

339 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

340 

341 Only the Warps that cover the specified tract and patch will be coadded. 

342 A list of the available optional arguments can be obtained by calling 

343 ``assembleCoadd.py`` with the ``--help`` command line argument: 

344 

345 .. code-block:: none 

346 

347 assembleCoadd.py --help 

348 

349 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

350 multi-band processing, we will generate the HSC-I & -R band coadds from 

351 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

352 assuming that the lsst stack has been already set up, we must set up the 

353 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

354 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

355 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

356 coadds, we must first 

357 

358 - processCcd 

359 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

360 - makeSkyMap 

361 - create a skymap that covers the area of the sky present in the raw exposures 

362 - makeCoaddTempExp 

363 - warp the individual calibrated exposures to the tangent plane of the coadd 

364 

365 We can perform all of these steps by running 

366 

367 .. code-block:: none 

368 

369 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

370 

371 This will produce warped exposures for each visit. To coadd the warped 

372 data, we call assembleCoadd.py as follows: 

373 

374 .. code-block:: none 

375 

376 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

377 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

378 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

379 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

380 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

381 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

382 --selectId visit=903988 ccd=24 

383 

384 that will process the HSC-I band data. The results are written in 

385 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

386 

387 You may also choose to run: 

388 

389 .. code-block:: none 

390 

391 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

392 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

393 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

394 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

395 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

396 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

397 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

398 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

399 

400 to generate the coadd for the HSC-R band if you are interested in 

401 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

402 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

403 rather than `AssembleCoaddTask` to make the coadd. 

404 """ 

405 ConfigClass = AssembleCoaddConfig 

406 _DefaultName = "assembleCoadd" 

407 

408 def __init__(self, *args, **kwargs): 

409 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

410 if args: 

411 argNames = ["config", "name", "parentTask", "log"] 

412 kwargs.update({k: v for k, v in zip(argNames, args)}) 

413 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

414 "PipelineTask will not take positional args" % argNames, FutureWarning) 

415 

416 super().__init__(**kwargs) 

417 self.makeSubtask("interpImage") 

418 self.makeSubtask("scaleZeroPoint") 

419 

420 if self.config.doMaskBrightObjects: 

421 mask = afwImage.Mask() 

422 try: 

423 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

424 except pexExceptions.LsstCppException: 

425 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

426 mask.getMaskPlaneDict().keys()) 

427 del mask 

428 

429 if self.config.doInputMap: 

430 self.makeSubtask("inputMapper") 

431 

432 self.warpType = self.config.warpType 

433 

434 @utils.inheritDoc(pipeBase.PipelineTask) 

435 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

436 # Docstring to be formatted with info from PipelineTask.runQuantum 

437 """ 

438 Notes 

439 ----- 

440 Assemble a coadd from a set of Warps. 

441 

442 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

443 Analogous to `runDataRef`, it prepares all the data products to be 

444 passed to `run`, and processes the results before returning a struct 

445 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

446 Therefore, its inputs are accessed subregion by subregion 

447 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

448 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

449 correspond to an update in `runDataRef` while both entry points 

450 are used. 

451 """ 

452 inputData = butlerQC.get(inputRefs) 

453 

454 # Construct skyInfo expected by run 

455 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

456 skyMap = inputData["skyMap"] 

457 outputDataId = butlerQC.quantum.dataId 

458 

459 inputData['skyInfo'] = makeSkyInfo(skyMap, 

460 tractId=outputDataId['tract'], 

461 patchId=outputDataId['patch']) 

462 

463 if self.config.doSelectVisits: 

464 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

465 else: 

466 warpRefList = inputData['inputWarps'] 

467 

468 # Perform same middle steps as `runDataRef` does 

469 inputs = self.prepareInputs(warpRefList) 

470 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

471 self.getTempExpDatasetName(self.warpType)) 

472 if len(inputs.tempExpRefList) == 0: 

473 self.log.warn("No coadd temporary exposures found") 

474 return 

475 

476 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

477 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

478 inputs.weightList, supplementaryData=supplementaryData) 

479 

480 inputData.setdefault('brightObjectMask', None) 

481 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

482 

483 if self.config.doWrite: 

484 butlerQC.put(retStruct, outputRefs) 

485 return retStruct 

486 

487 @pipeBase.timeMethod 

488 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

489 """Assemble a coadd from a set of Warps. 

490 

491 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

492 Compute weights to be applied to each Warp and 

493 find scalings to match the photometric zeropoint to a reference Warp. 

494 Assemble the Warps using `run`. Interpolate over NaNs and 

495 optionally write the coadd to disk. Return the coadded exposure. 

496 

497 Parameters 

498 ---------- 

499 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

500 Data reference defining the patch for coaddition and the 

501 reference Warp (if ``config.autoReference=False``). 

502 Used to access the following data products: 

503 - ``self.config.coaddName + "Coadd_skyMap"`` 

504 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

505 - ``self.config.coaddName + "Coadd"`` 

506 selectDataList : `list` 

507 List of data references to Calexps. Data to be coadded will be 

508 selected from this list based on overlap with the patch defined 

509 by dataRef, grouped by visit, and converted to a list of data 

510 references to warps. 

511 warpRefList : `list` 

512 List of data references to Warps to be coadded. 

513 Note: `warpRefList` is just the new name for `tempExpRefList`. 

514 

515 Returns 

516 ------- 

517 retStruct : `lsst.pipe.base.Struct` 

518 Result struct with components: 

519 

520 - ``coaddExposure``: coadded exposure (``Exposure``). 

521 - ``nImage``: exposure count image (``Image``). 

522 """ 

523 if selectDataList and warpRefList: 

524 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

525 "and which to use is ambiguous. Please pass only one.") 

526 

527 skyInfo = self.getSkyInfo(dataRef) 

528 if warpRefList is None: 

529 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

530 if len(calExpRefList) == 0: 

531 self.log.warn("No exposures to coadd") 

532 return 

533 self.log.info("Coadding %d exposures", len(calExpRefList)) 

534 

535 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

536 

537 inputData = self.prepareInputs(warpRefList) 

538 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

539 self.getTempExpDatasetName(self.warpType)) 

540 if len(inputData.tempExpRefList) == 0: 

541 self.log.warn("No coadd temporary exposures found") 

542 return 

543 

544 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

545 

546 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

547 inputData.weightList, supplementaryData=supplementaryData) 

548 

549 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

550 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

551 

552 if self.config.doWrite: 

553 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 

554 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

555 else: 

556 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

557 self.log.info("Persisting %s" % coaddDatasetName) 

558 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

559 if self.config.doNImage and retStruct.nImage is not None: 

560 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

561 

562 return retStruct 

563 

564 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

565 """Interpolate over missing data and mask bright stars. 

566 

567 Parameters 

568 ---------- 

569 coaddExposure : `lsst.afw.image.Exposure` 

570 The coadded exposure to process. 

571 dataRef : `lsst.daf.persistence.ButlerDataRef` 

572 Butler data reference for supplementary data. 

573 """ 

574 if self.config.doInterp: 

575 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

576 # The variance must be positive; work around for DM-3201. 

577 varArray = coaddExposure.variance.array 

578 with numpy.errstate(invalid="ignore"): 

579 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

580 

581 if self.config.doMaskBrightObjects: 

582 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

583 

584 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

585 """Make additional inputs to run() specific to subclasses (Gen2) 

586 

587 Duplicates interface of `runDataRef` method 

588 Available to be implemented by subclasses only if they need the 

589 coadd dataRef for performing preliminary processing before 

590 assembling the coadd. 

591 

592 Parameters 

593 ---------- 

594 dataRef : `lsst.daf.persistence.ButlerDataRef` 

595 Butler data reference for supplementary data. 

596 selectDataList : `list` (optional) 

597 Optional List of data references to Calexps. 

598 warpRefList : `list` (optional) 

599 Optional List of data references to Warps. 

600 """ 

601 return pipeBase.Struct() 

602 

603 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

604 """Make additional inputs to run() specific to subclasses (Gen3) 

605 

606 Duplicates interface of `runQuantum` method. 

607 Available to be implemented by subclasses only if they need the 

608 coadd dataRef for performing preliminary processing before 

609 assembling the coadd. 

610 

611 Parameters 

612 ---------- 

613 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

614 Gen3 Butler object for fetching additional data products before 

615 running the Task specialized for quantum being processed 

616 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

617 Attributes are the names of the connections describing input dataset types. 

618 Values are DatasetRefs that task consumes for corresponding dataset type. 

619 DataIds are guaranteed to match data objects in ``inputData``. 

620 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

621 Attributes are the names of the connections describing output dataset types. 

622 Values are DatasetRefs that task is to produce 

623 for corresponding dataset type. 

624 """ 

625 return pipeBase.Struct() 

626 

627 def getTempExpRefList(self, patchRef, calExpRefList): 

628 """Generate list data references corresponding to warped exposures 

629 that lie within the patch to be coadded. 

630 

631 Parameters 

632 ---------- 

633 patchRef : `dataRef` 

634 Data reference for patch. 

635 calExpRefList : `list` 

636 List of data references for input calexps. 

637 

638 Returns 

639 ------- 

640 tempExpRefList : `list` 

641 List of Warp/CoaddTempExp data references. 

642 """ 

643 butler = patchRef.getButler() 

644 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

645 self.getTempExpDatasetName(self.warpType)) 

646 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

647 g, groupData.keys) for 

648 g in groupData.groups.keys()] 

649 return tempExpRefList 

650 

651 def prepareInputs(self, refList): 

652 """Prepare the input warps for coaddition by measuring the weight for 

653 each warp and the scaling for the photometric zero point. 

654 

655 Each Warp has its own photometric zeropoint and background variance. 

656 Before coadding these Warps together, compute a scale factor to 

657 normalize the photometric zeropoint and compute the weight for each Warp. 

658 

659 Parameters 

660 ---------- 

661 refList : `list` 

662 List of data references to tempExp 

663 

664 Returns 

665 ------- 

666 result : `lsst.pipe.base.Struct` 

667 Result struct with components: 

668 

669 - ``tempExprefList``: `list` of data references to tempExp. 

670 - ``weightList``: `list` of weightings. 

671 - ``imageScalerList``: `list` of image scalers. 

672 """ 

673 statsCtrl = afwMath.StatisticsControl() 

674 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

675 statsCtrl.setNumIter(self.config.clipIter) 

676 statsCtrl.setAndMask(self.getBadPixelMask()) 

677 statsCtrl.setNanSafe(True) 

678 # compute tempExpRefList: a list of tempExpRef that actually exist 

679 # and weightList: a list of the weight of the associated coadd tempExp 

680 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

681 tempExpRefList = [] 

682 weightList = [] 

683 imageScalerList = [] 

684 tempExpName = self.getTempExpDatasetName(self.warpType) 

685 for tempExpRef in refList: 

686 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

687 # therefore have no datasetExists() method 

688 if not isinstance(tempExpRef, DeferredDatasetHandle): 

689 if not tempExpRef.datasetExists(tempExpName): 

690 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

691 continue 

692 

693 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

694 # Ignore any input warp that is empty of data 

695 if numpy.isnan(tempExp.image.array).all(): 

696 continue 

697 maskedImage = tempExp.getMaskedImage() 

698 imageScaler = self.scaleZeroPoint.computeImageScaler( 

699 exposure=tempExp, 

700 dataRef=tempExpRef, 

701 ) 

702 try: 

703 imageScaler.scaleMaskedImage(maskedImage) 

704 except Exception as e: 

705 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

706 continue 

707 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

708 afwMath.MEANCLIP, statsCtrl) 

709 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

710 weight = 1.0 / float(meanVar) 

711 if not numpy.isfinite(weight): 

712 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

713 continue 

714 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

715 

716 del maskedImage 

717 del tempExp 

718 

719 tempExpRefList.append(tempExpRef) 

720 weightList.append(weight) 

721 imageScalerList.append(imageScaler) 

722 

723 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

724 imageScalerList=imageScalerList) 

725 

726 def prepareStats(self, mask=None): 

727 """Prepare the statistics for coadding images. 

728 

729 Parameters 

730 ---------- 

731 mask : `int`, optional 

732 Bit mask value to exclude from coaddition. 

733 

734 Returns 

735 ------- 

736 stats : `lsst.pipe.base.Struct` 

737 Statistics structure with the following fields: 

738 

739 - ``statsCtrl``: Statistics control object for coadd 

740 (`lsst.afw.math.StatisticsControl`) 

741 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

742 """ 

743 if mask is None: 

744 mask = self.getBadPixelMask() 

745 statsCtrl = afwMath.StatisticsControl() 

746 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

747 statsCtrl.setNumIter(self.config.clipIter) 

748 statsCtrl.setAndMask(mask) 

749 statsCtrl.setNanSafe(True) 

750 statsCtrl.setWeighted(True) 

751 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

752 for plane, threshold in self.config.maskPropagationThresholds.items(): 

753 bit = afwImage.Mask.getMaskPlane(plane) 

754 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

755 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

756 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

757 

758 @pipeBase.timeMethod 

759 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

760 altMaskList=None, mask=None, supplementaryData=None): 

761 """Assemble a coadd from input warps 

762 

763 Assemble the coadd using the provided list of coaddTempExps. Since 

764 the full coadd covers a patch (a large area), the assembly is 

765 performed over small areas on the image at a time in order to 

766 conserve memory usage. Iterate over subregions within the outer 

767 bbox of the patch using `assembleSubregion` to stack the corresponding 

768 subregions from the coaddTempExps with the statistic specified. 

769 Set the edge bits the coadd mask based on the weight map. 

770 

771 Parameters 

772 ---------- 

773 skyInfo : `lsst.pipe.base.Struct` 

774 Struct with geometric information about the patch. 

775 tempExpRefList : `list` 

776 List of data references to Warps (previously called CoaddTempExps). 

777 imageScalerList : `list` 

778 List of image scalers. 

779 weightList : `list` 

780 List of weights 

781 altMaskList : `list`, optional 

782 List of alternate masks to use rather than those stored with 

783 tempExp. 

784 mask : `int`, optional 

785 Bit mask value to exclude from coaddition. 

786 supplementaryData : lsst.pipe.base.Struct, optional 

787 Struct with additional data products needed to assemble coadd. 

788 Only used by subclasses that implement `makeSupplementaryData` 

789 and override `run`. 

790 

791 Returns 

792 ------- 

793 result : `lsst.pipe.base.Struct` 

794 Result struct with components: 

795 

796 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

797 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

798 - ``inputMap``: bit-wise map of inputs, if requested. 

799 - ``warpRefList``: input list of refs to the warps ( 

800 ``lsst.daf.butler.DeferredDatasetHandle`` or 

801 ``lsst.daf.persistence.ButlerDataRef``) 

802 (unmodified) 

803 - ``imageScalerList``: input list of image scalers (unmodified) 

804 - ``weightList``: input list of weights (unmodified) 

805 """ 

806 tempExpName = self.getTempExpDatasetName(self.warpType) 

807 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

808 stats = self.prepareStats(mask=mask) 

809 

810 if altMaskList is None: 

811 altMaskList = [None]*len(tempExpRefList) 

812 

813 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

814 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

815 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

816 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

817 coaddMaskedImage = coaddExposure.getMaskedImage() 

818 subregionSizeArr = self.config.subregionSize 

819 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

820 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

821 if self.config.doNImage: 

822 nImage = afwImage.ImageU(skyInfo.bbox) 

823 else: 

824 nImage = None 

825 # If inputMap is requested, create the initial version that can be masked in 

826 # assembleSubregion. 

827 if self.config.doInputMap: 

828 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

829 skyInfo.wcs, 

830 coaddExposure.getInfo().getCoaddInputs().ccds) 

831 

832 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

833 try: 

834 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

835 weightList, altMaskList, stats.flags, stats.ctrl, 

836 nImage=nImage) 

837 except Exception as e: 

838 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

839 

840 # If inputMap is requested, we must finalize the map after the accumulation. 

841 if self.config.doInputMap: 

842 self.inputMapper.finalize_ccd_input_map_mask() 

843 inputMap = self.inputMapper.ccd_input_map 

844 else: 

845 inputMap = None 

846 

847 self.setInexactPsf(coaddMaskedImage.getMask()) 

848 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

849 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

850 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

851 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

852 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

853 weightList=weightList, inputMap=inputMap) 

854 

855 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

856 """Set the metadata for the coadd. 

857 

858 This basic implementation sets the filter from the first input. 

859 

860 Parameters 

861 ---------- 

862 coaddExposure : `lsst.afw.image.Exposure` 

863 The target exposure for the coadd. 

864 tempExpRefList : `list` 

865 List of data references to tempExp. 

866 weightList : `list` 

867 List of weights. 

868 """ 

869 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

870 tempExpName = self.getTempExpDatasetName(self.warpType) 

871 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

872 # (and we need more than just the PropertySet that contains the header), which is not possible 

873 # with the current butler (see #2777). 

874 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

875 

876 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 

877 # Gen 3 API 

878 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

879 else: 

880 # Gen 2 API. Delete this when Gen 2 retired 

881 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

882 for tempExpRef in tempExpRefList] 

883 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

884 

885 # Set the coadd FilterLabel to the band of the first input exposure: 

886 # Coadds are calibrated, so the physical label is now meaningless. 

887 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

888 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

889 coaddInputs.ccds.reserve(numCcds) 

890 coaddInputs.visits.reserve(len(tempExpList)) 

891 

892 for tempExp, weight in zip(tempExpList, weightList): 

893 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

894 

895 if self.config.doUsePsfMatchedPolygons: 

896 self.shrinkValidPolygons(coaddInputs) 

897 

898 coaddInputs.visits.sort() 

899 if self.warpType == "psfMatched": 

900 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

901 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

902 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

903 # having the maximum width (sufficient because square) 

904 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

905 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

906 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

907 else: 

908 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

909 self.config.coaddPsf.makeControl()) 

910 coaddExposure.setPsf(psf) 

911 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

912 coaddExposure.getWcs()) 

913 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

914 if self.config.doAttachTransmissionCurve: 

915 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

916 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

917 

918 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

919 altMaskList, statsFlags, statsCtrl, nImage=None): 

920 """Assemble the coadd for a sub-region. 

921 

922 For each coaddTempExp, check for (and swap in) an alternative mask 

923 if one is passed. Remove mask planes listed in 

924 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

925 `lsst.afw.math.statisticsStack` with the statistic specified by 

926 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

927 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

928 an N-sigma clipped mean where N and iterations are specified by 

929 statsCtrl. Assign the stacked subregion back to the coadd. 

930 

931 Parameters 

932 ---------- 

933 coaddExposure : `lsst.afw.image.Exposure` 

934 The target exposure for the coadd. 

935 bbox : `lsst.geom.Box` 

936 Sub-region to coadd. 

937 tempExpRefList : `list` 

938 List of data reference to tempExp. 

939 imageScalerList : `list` 

940 List of image scalers. 

941 weightList : `list` 

942 List of weights. 

943 altMaskList : `list` 

944 List of alternate masks to use rather than those stored with 

945 tempExp, or None. Each element is dict with keys = mask plane 

946 name to which to add the spans. 

947 statsFlags : `lsst.afw.math.Property` 

948 Property object for statistic for coadd. 

949 statsCtrl : `lsst.afw.math.StatisticsControl` 

950 Statistics control object for coadd. 

951 nImage : `lsst.afw.image.ImageU`, optional 

952 Keeps track of exposure count for each pixel. 

953 """ 

954 self.log.debug("Computing coadd over %s", bbox) 

955 tempExpName = self.getTempExpDatasetName(self.warpType) 

956 coaddExposure.mask.addMaskPlane("REJECTED") 

957 coaddExposure.mask.addMaskPlane("CLIPPED") 

958 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

959 maskMap = self.setRejectedMaskMapping(statsCtrl) 

960 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

961 maskedImageList = [] 

962 if nImage is not None: 

963 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

964 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

965 

966 if isinstance(tempExpRef, DeferredDatasetHandle): 

967 # Gen 3 API 

968 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

969 else: 

970 # Gen 2 API. Delete this when Gen 2 retired 

971 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

972 

973 maskedImage = exposure.getMaskedImage() 

974 mask = maskedImage.getMask() 

975 if altMask is not None: 

976 self.applyAltMaskPlanes(mask, altMask) 

977 imageScaler.scaleMaskedImage(maskedImage) 

978 

979 # Add 1 for each pixel which is not excluded by the exclude mask. 

980 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

981 if nImage is not None: 

982 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

983 if self.config.removeMaskPlanes: 

984 self.removeMaskPlanes(maskedImage) 

985 maskedImageList.append(maskedImage) 

986 

987 if self.config.doInputMap: 

988 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

989 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

990 

991 with self.timer("stack"): 

992 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

993 clipped, # also set output to CLIPPED if sigma-clipped 

994 maskMap) 

995 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

996 if nImage is not None: 

997 nImage.assign(subNImage, bbox) 

998 

999 def removeMaskPlanes(self, maskedImage): 

1000 """Unset the mask of an image for mask planes specified in the config. 

1001 

1002 Parameters 

1003 ---------- 

1004 maskedImage : `lsst.afw.image.MaskedImage` 

1005 The masked image to be modified. 

1006 """ 

1007 mask = maskedImage.getMask() 

1008 for maskPlane in self.config.removeMaskPlanes: 

1009 try: 

1010 mask &= ~mask.getPlaneBitMask(maskPlane) 

1011 except pexExceptions.InvalidParameterError: 

1012 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1013 maskPlane) 

1014 

1015 @staticmethod 

1016 def setRejectedMaskMapping(statsCtrl): 

1017 """Map certain mask planes of the warps to new planes for the coadd. 

1018 

1019 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1020 or CLIPPED, set it to REJECTED on the coadd. 

1021 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1022 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1023 

1024 Parameters 

1025 ---------- 

1026 statsCtrl : `lsst.afw.math.StatisticsControl` 

1027 Statistics control object for coadd 

1028 

1029 Returns 

1030 ------- 

1031 maskMap : `list` of `tuple` of `int` 

1032 A list of mappings of mask planes of the warped exposures to 

1033 mask planes of the coadd. 

1034 """ 

1035 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1036 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1037 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1038 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1039 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1040 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1041 (clipped, clipped)] 

1042 return maskMap 

1043 

1044 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1045 """Apply in place alt mask formatted as SpanSets to a mask. 

1046 

1047 Parameters 

1048 ---------- 

1049 mask : `lsst.afw.image.Mask` 

1050 Original mask. 

1051 altMaskSpans : `dict` 

1052 SpanSet lists to apply. Each element contains the new mask 

1053 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1054 and list of SpanSets to apply to the mask. 

1055 

1056 Returns 

1057 ------- 

1058 mask : `lsst.afw.image.Mask` 

1059 Updated mask. 

1060 """ 

1061 if self.config.doUsePsfMatchedPolygons: 

1062 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

1063 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1064 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1065 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1066 # This mask-clearing step must only occur *before* applying the new masks below 

1067 for spanSet in altMaskSpans['NO_DATA']: 

1068 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1069 

1070 for plane, spanSetList in altMaskSpans.items(): 

1071 maskClipValue = mask.addMaskPlane(plane) 

1072 for spanSet in spanSetList: 

1073 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1074 return mask 

1075 

1076 def shrinkValidPolygons(self, coaddInputs): 

1077 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1078 

1079 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1080 does not have a validPolygon, create one from its bbox. 

1081 

1082 Parameters 

1083 ---------- 

1084 coaddInputs : `lsst.afw.image.coaddInputs` 

1085 Original mask. 

1086 

1087 """ 

1088 for ccd in coaddInputs.ccds: 

1089 polyOrig = ccd.getValidPolygon() 

1090 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1091 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1092 if polyOrig: 

1093 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1094 else: 

1095 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1096 ccd.setValidPolygon(validPolygon) 

1097 

1098 def readBrightObjectMasks(self, dataRef): 

1099 """Retrieve the bright object masks. 

1100 

1101 Returns None on failure. 

1102 

1103 Parameters 

1104 ---------- 

1105 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1106 A Butler dataRef. 

1107 

1108 Returns 

1109 ------- 

1110 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1111 Bright object mask from the Butler object, or None if it cannot 

1112 be retrieved. 

1113 """ 

1114 try: 

1115 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1116 except Exception as e: 

1117 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1118 return None 

1119 

1120 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1121 """Set the bright object masks. 

1122 

1123 Parameters 

1124 ---------- 

1125 exposure : `lsst.afw.image.Exposure` 

1126 Exposure under consideration. 

1127 dataId : `lsst.daf.persistence.dataId` 

1128 Data identifier dict for patch. 

1129 brightObjectMasks : `lsst.afw.table` 

1130 Table of bright objects to mask. 

1131 """ 

1132 

1133 if brightObjectMasks is None: 

1134 self.log.warn("Unable to apply bright object mask: none supplied") 

1135 return 

1136 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1137 mask = exposure.getMaskedImage().getMask() 

1138 wcs = exposure.getWcs() 

1139 plateScale = wcs.getPixelScale().asArcseconds() 

1140 

1141 for rec in brightObjectMasks: 

1142 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1143 if rec["type"] == "box": 

1144 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1145 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1146 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1147 

1148 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1149 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1150 

1151 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1152 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1153 spans = afwGeom.SpanSet(bbox) 

1154 elif rec["type"] == "circle": 

1155 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1156 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1157 else: 

1158 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1159 continue 

1160 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1161 

1162 def setInexactPsf(self, mask): 

1163 """Set INEXACT_PSF mask plane. 

1164 

1165 If any of the input images isn't represented in the coadd (due to 

1166 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1167 these pixels. 

1168 

1169 Parameters 

1170 ---------- 

1171 mask : `lsst.afw.image.Mask` 

1172 Coadded exposure's mask, modified in-place. 

1173 """ 

1174 mask.addMaskPlane("INEXACT_PSF") 

1175 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1176 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1177 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1178 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1179 array = mask.getArray() 

1180 selected = array & (sensorEdge | clipped | rejected) > 0 

1181 array[selected] |= inexactPsf 

1182 

1183 @classmethod 

1184 def _makeArgumentParser(cls): 

1185 """Create an argument parser. 

1186 """ 

1187 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1188 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1189 + cls.ConfigClass().warpType + "Warp", 

1190 help="data ID, e.g. --id tract=12345 patch=1,2", 

1191 ContainerClass=AssembleCoaddDataIdContainer) 

1192 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1193 ContainerClass=SelectDataIdContainer) 

1194 return parser 

1195 

1196 @staticmethod 

1197 def _subBBoxIter(bbox, subregionSize): 

1198 """Iterate over subregions of a bbox. 

1199 

1200 Parameters 

1201 ---------- 

1202 bbox : `lsst.geom.Box2I` 

1203 Bounding box over which to iterate. 

1204 subregionSize: `lsst.geom.Extent2I` 

1205 Size of sub-bboxes. 

1206 

1207 Yields 

1208 ------ 

1209 subBBox : `lsst.geom.Box2I` 

1210 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1211 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1212 the edges of ``bbox``, but it will never be empty. 

1213 """ 

1214 if bbox.isEmpty(): 

1215 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1216 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1217 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1218 

1219 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1220 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1221 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1222 subBBox.clip(bbox) 

1223 if subBBox.isEmpty(): 

1224 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1225 "colShift=%s, rowShift=%s" % 

1226 (bbox, subregionSize, colShift, rowShift)) 

1227 yield subBBox 

1228 

1229 def filterWarps(self, inputs, goodVisits): 

1230 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1231 

1232 Parameters 

1233 ---------- 

1234 inputs : list 

1235 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1236 goodVisit : `dict` 

1237 Dictionary with good visitIds as the keys. Value ignored. 

1238 

1239 Returns: 

1240 -------- 

1241 filteredInputs : `list` 

1242 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1243 """ 

1244 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1245 filteredInputs = [] 

1246 for visit in goodVisits.keys(): 

1247 filteredInputs.append(inputWarpDict[visit]) 

1248 return filteredInputs 

1249 

1250 

1251class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1252 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1253 """ 

1254 

1255 def makeDataRefList(self, namespace): 

1256 """Make self.refList from self.idList. 

1257 

1258 Parameters 

1259 ---------- 

1260 namespace 

1261 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1262 """ 

1263 datasetType = namespace.config.coaddName + "Coadd" 

1264 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1265 

1266 for dataId in self.idList: 

1267 # tract and patch are required 

1268 for key in keysCoadd: 

1269 if key not in dataId: 

1270 raise RuntimeError("--id must include " + key) 

1271 

1272 dataRef = namespace.butler.dataRef( 

1273 datasetType=datasetType, 

1274 dataId=dataId, 

1275 ) 

1276 self.refList.append(dataRef) 

1277 

1278 

1279def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1280 """Function to count the number of pixels with a specific mask in a 

1281 footprint. 

1282 

1283 Find the intersection of mask & footprint. Count all pixels in the mask 

1284 that are in the intersection that have bitmask set but do not have 

1285 ignoreMask set. Return the count. 

1286 

1287 Parameters 

1288 ---------- 

1289 mask : `lsst.afw.image.Mask` 

1290 Mask to define intersection region by. 

1291 footprint : `lsst.afw.detection.Footprint` 

1292 Footprint to define the intersection region by. 

1293 bitmask 

1294 Specific mask that we wish to count the number of occurances of. 

1295 ignoreMask 

1296 Pixels to not consider. 

1297 

1298 Returns 

1299 ------- 

1300 result : `int` 

1301 Count of number of pixels in footprint with specified mask. 

1302 """ 

1303 bbox = footprint.getBBox() 

1304 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1305 fp = afwImage.Mask(bbox) 

1306 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1307 footprint.spans.setMask(fp, bitmask) 

1308 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1309 (subMask.getArray() & ignoreMask) == 0).sum() 

1310 

1311 

1312class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1313 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1314 """ 

1315 clipDetection = pexConfig.ConfigurableField( 

1316 target=SourceDetectionTask, 

1317 doc="Detect sources on difference between unclipped and clipped coadd") 

1318 minClipFootOverlap = pexConfig.Field( 

1319 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1320 dtype=float, 

1321 default=0.6 

1322 ) 

1323 minClipFootOverlapSingle = pexConfig.Field( 

1324 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1325 "clipped when only one visit overlaps", 

1326 dtype=float, 

1327 default=0.5 

1328 ) 

1329 minClipFootOverlapDouble = pexConfig.Field( 

1330 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1331 "clipped when two visits overlap", 

1332 dtype=float, 

1333 default=0.45 

1334 ) 

1335 maxClipFootOverlapDouble = pexConfig.Field( 

1336 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1337 "considering two visits", 

1338 dtype=float, 

1339 default=0.15 

1340 ) 

1341 minBigOverlap = pexConfig.Field( 

1342 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1343 "when labeling clipped footprints", 

1344 dtype=int, 

1345 default=100 

1346 ) 

1347 

1348 def setDefaults(self): 

1349 """Set default values for clipDetection. 

1350 

1351 Notes 

1352 ----- 

1353 The numeric values for these configuration parameters were 

1354 empirically determined, future work may further refine them. 

1355 """ 

1356 AssembleCoaddConfig.setDefaults(self) 

1357 self.clipDetection.doTempLocalBackground = False 

1358 self.clipDetection.reEstimateBackground = False 

1359 self.clipDetection.returnOriginalFootprints = False 

1360 self.clipDetection.thresholdPolarity = "both" 

1361 self.clipDetection.thresholdValue = 2 

1362 self.clipDetection.nSigmaToGrow = 2 

1363 self.clipDetection.minPixels = 4 

1364 self.clipDetection.isotropicGrow = True 

1365 self.clipDetection.thresholdType = "pixel_stdev" 

1366 self.sigmaClip = 1.5 

1367 self.clipIter = 3 

1368 self.statistic = "MEAN" 

1369 

1370 def validate(self): 

1371 if self.doSigmaClip: 

1372 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1373 "Ignoring doSigmaClip.") 

1374 self.doSigmaClip = False 

1375 if self.statistic != "MEAN": 

1376 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1377 "(%s chosen). Please set statistic to MEAN." 

1378 % (self.statistic)) 

1379 AssembleCoaddTask.ConfigClass.validate(self) 

1380 

1381 

1382class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1383 """Assemble a coadded image from a set of coadded temporary exposures, 

1384 being careful to clip & flag areas with potential artifacts. 

1385 

1386 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1387 we clip outliers). The problem with doing this is that when computing the 

1388 coadd PSF at a given location, individual visit PSFs from visits with 

1389 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1390 In this task, we correct for this behavior by creating a new 

1391 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1392 coaddTempExps and the final coadd where 

1393 

1394 i. difference imaging suggests that there is an outlier and 

1395 ii. this outlier appears on only one or two images. 

1396 

1397 Such regions will not contribute to the final coadd. Furthermore, any 

1398 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1399 Note that the algorithm implemented by this task is preliminary and works 

1400 correctly for HSC data. Parameter modifications and or considerable 

1401 redesigning of the algorithm is likley required for other surveys. 

1402 

1403 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1404 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1405 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1406 if you wish. 

1407 

1408 Notes 

1409 ----- 

1410 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1411 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1412 see `baseDebug` for more about ``debug.py`` files. 

1413 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1414 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1415 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1416 for further information. 

1417 

1418 Examples 

1419 -------- 

1420 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1421 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1422 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1423 

1424 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1425 and filter to be coadded (specified using 

1426 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1427 along with a list of coaddTempExps to attempt to coadd (specified using 

1428 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1429 Only the coaddTempExps that cover the specified tract and patch will be 

1430 coadded. A list of the available optional arguments can be obtained by 

1431 calling assembleCoadd.py with the --help command line argument: 

1432 

1433 .. code-block:: none 

1434 

1435 assembleCoadd.py --help 

1436 

1437 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1438 context of multi-band processing, we will generate the HSC-I & -R band 

1439 coadds from HSC engineering test data provided in the ci_hsc package. 

1440 To begin, assuming that the lsst stack has been already set up, we must 

1441 set up the obs_subaru and ci_hsc packages. This defines the environment 

1442 variable $CI_HSC_DIR and points at the location of the package. The raw 

1443 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1444 the coadds, we must first 

1445 

1446 - ``processCcd`` 

1447 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1448 - ``makeSkyMap`` 

1449 create a skymap that covers the area of the sky present in the raw exposures 

1450 - ``makeCoaddTempExp`` 

1451 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1452 

1453 We can perform all of these steps by running 

1454 

1455 .. code-block:: none 

1456 

1457 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1458 

1459 This will produce warped coaddTempExps for each visit. To coadd the 

1460 warped data, we call ``assembleCoadd.py`` as follows: 

1461 

1462 .. code-block:: none 

1463 

1464 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1465 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1466 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1467 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1468 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1469 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1470 --selectId visit=903988 ccd=24 

1471 

1472 This will process the HSC-I band data. The results are written in 

1473 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1474 

1475 You may also choose to run: 

1476 

1477 .. code-block:: none 

1478 

1479 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1480 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1481 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1482 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1483 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1484 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1485 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1486 --selectId visit=903346 ccd=12 

1487 

1488 to generate the coadd for the HSC-R band if you are interested in following 

1489 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1490 """ 

1491 ConfigClass = SafeClipAssembleCoaddConfig 

1492 _DefaultName = "safeClipAssembleCoadd" 

1493 

1494 def __init__(self, *args, **kwargs): 

1495 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1496 schema = afwTable.SourceTable.makeMinimalSchema() 

1497 self.makeSubtask("clipDetection", schema=schema) 

1498 

1499 @utils.inheritDoc(AssembleCoaddTask) 

1500 @pipeBase.timeMethod 

1501 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1502 """Assemble the coadd for a region. 

1503 

1504 Compute the difference of coadds created with and without outlier 

1505 rejection to identify coadd pixels that have outlier values in some 

1506 individual visits. 

1507 Detect clipped regions on the difference image and mark these regions 

1508 on the one or two individual coaddTempExps where they occur if there 

1509 is significant overlap between the clipped region and a source. This 

1510 leaves us with a set of footprints from the difference image that have 

1511 been identified as having occured on just one or two individual visits. 

1512 However, these footprints were generated from a difference image. It 

1513 is conceivable for a large diffuse source to have become broken up 

1514 into multiple footprints acrosss the coadd difference in this process. 

1515 Determine the clipped region from all overlapping footprints from the 

1516 detected sources in each visit - these are big footprints. 

1517 Combine the small and big clipped footprints and mark them on a new 

1518 bad mask plane. 

1519 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1520 removal. Clipped footprints will no longer make it into the coadd 

1521 because they are marked in the new bad mask plane. 

1522 

1523 Notes 

1524 ----- 

1525 args and kwargs are passed but ignored in order to match the call 

1526 signature expected by the parent task. 

1527 """ 

1528 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1529 mask = exp.getMaskedImage().getMask() 

1530 mask.addMaskPlane("CLIPPED") 

1531 

1532 result = self.detectClip(exp, tempExpRefList) 

1533 

1534 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1535 

1536 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1537 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1538 # Append big footprints from individual Warps to result.clipSpans 

1539 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1540 result.detectionFootprints, maskClipValue, maskDetValue, 

1541 exp.getBBox()) 

1542 # Create mask of the current clipped footprints 

1543 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1544 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1545 

1546 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1547 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1548 maskClip |= maskClipBig 

1549 

1550 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1551 badMaskPlanes = self.config.badMaskPlanes[:] 

1552 badMaskPlanes.append("CLIPPED") 

1553 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1554 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1555 result.clipSpans, mask=badPixelMask) 

1556 

1557 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1558 """Return an exposure that contains the difference between unclipped 

1559 and clipped coadds. 

1560 

1561 Generate a difference image between clipped and unclipped coadds. 

1562 Compute the difference image by subtracting an outlier-clipped coadd 

1563 from an outlier-unclipped coadd. Return the difference image. 

1564 

1565 Parameters 

1566 ---------- 

1567 skyInfo : `lsst.pipe.base.Struct` 

1568 Patch geometry information, from getSkyInfo 

1569 tempExpRefList : `list` 

1570 List of data reference to tempExp 

1571 imageScalerList : `list` 

1572 List of image scalers 

1573 weightList : `list` 

1574 List of weights 

1575 

1576 Returns 

1577 ------- 

1578 exp : `lsst.afw.image.Exposure` 

1579 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1580 """ 

1581 config = AssembleCoaddConfig() 

1582 # getattr necessary because subtasks do not survive Config.toDict() 

1583 # exclude connections because the class of self.config.connections is not 

1584 # the same as AssembleCoaddConfig.connections, and the connections are not 

1585 # needed to run this task anyway. 

1586 configIntersection = {k: getattr(self.config, k) 

1587 for k, v in self.config.toDict().items() 

1588 if (k in config.keys() and k != "connections")} 

1589 configIntersection['doInputMap'] = False 

1590 configIntersection['doNImage'] = False 

1591 config.update(**configIntersection) 

1592 

1593 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1594 config.statistic = 'MEAN' 

1595 task = AssembleCoaddTask(config=config) 

1596 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1597 

1598 config.statistic = 'MEANCLIP' 

1599 task = AssembleCoaddTask(config=config) 

1600 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1601 

1602 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1603 coaddDiff -= coaddClip.getMaskedImage() 

1604 exp = afwImage.ExposureF(coaddDiff) 

1605 exp.setPsf(coaddMean.getPsf()) 

1606 return exp 

1607 

1608 def detectClip(self, exp, tempExpRefList): 

1609 """Detect clipped regions on an exposure and set the mask on the 

1610 individual tempExp masks. 

1611 

1612 Detect footprints in the difference image after smoothing the 

1613 difference image with a Gaussian kernal. Identify footprints that 

1614 overlap with one or two input ``coaddTempExps`` by comparing the 

1615 computed overlap fraction to thresholds set in the config. A different 

1616 threshold is applied depending on the number of overlapping visits 

1617 (restricted to one or two). If the overlap exceeds the thresholds, 

1618 the footprint is considered "CLIPPED" and is marked as such on the 

1619 coaddTempExp. Return a struct with the clipped footprints, the indices 

1620 of the ``coaddTempExps`` that end up overlapping with the clipped 

1621 footprints, and a list of new masks for the ``coaddTempExps``. 

1622 

1623 Parameters 

1624 ---------- 

1625 exp : `lsst.afw.image.Exposure` 

1626 Exposure to run detection on. 

1627 tempExpRefList : `list` 

1628 List of data reference to tempExp. 

1629 

1630 Returns 

1631 ------- 

1632 result : `lsst.pipe.base.Struct` 

1633 Result struct with components: 

1634 

1635 - ``clipFootprints``: list of clipped footprints. 

1636 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1637 ``tempExpRefList``. 

1638 - ``clipSpans``: List of dictionaries containing spanSet lists 

1639 to clip. Each element contains the new maskplane name 

1640 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1641 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1642 compressed into footprints. 

1643 """ 

1644 mask = exp.getMaskedImage().getMask() 

1645 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1646 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1647 # Merge positive and negative together footprints together 

1648 fpSet.positive.merge(fpSet.negative) 

1649 footprints = fpSet.positive 

1650 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1651 ignoreMask = self.getBadPixelMask() 

1652 

1653 clipFootprints = [] 

1654 clipIndices = [] 

1655 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1656 

1657 # for use by detectClipBig 

1658 visitDetectionFootprints = [] 

1659 

1660 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1661 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1662 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1663 

1664 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1665 for i, warpRef in enumerate(tempExpRefList): 

1666 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1667 immediate=True).getMaskedImage().getMask() 

1668 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1669 afwImage.PARENT, True) 

1670 maskVisitDet &= maskDetValue 

1671 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1672 visitDetectionFootprints.append(visitFootprints) 

1673 

1674 for j, footprint in enumerate(footprints.getFootprints()): 

1675 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1676 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1677 

1678 # build a list of clipped spans for each visit 

1679 for j, footprint in enumerate(footprints.getFootprints()): 

1680 nPixel = footprint.getArea() 

1681 overlap = [] # hold the overlap with each visit 

1682 indexList = [] # index of visit in global list 

1683 for i in range(len(tempExpRefList)): 

1684 ignore = ignoreArr[i, j] 

1685 overlapDet = overlapDetArr[i, j] 

1686 totPixel = nPixel - ignore 

1687 

1688 # If we have more bad pixels than detection skip 

1689 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 

1690 continue 

1691 overlap.append(overlapDet/float(totPixel)) 

1692 indexList.append(i) 

1693 

1694 overlap = numpy.array(overlap) 

1695 if not len(overlap): 

1696 continue 

1697 

1698 keep = False # Should this footprint be marked as clipped? 

1699 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1700 

1701 # If footprint only has one overlap use a lower threshold 

1702 if len(overlap) == 1: 

1703 if overlap[0] > self.config.minClipFootOverlapSingle: 

1704 keep = True 

1705 keepIndex = [0] 

1706 else: 

1707 # This is the general case where only visit should be clipped 

1708 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1709 if len(clipIndex) == 1: 

1710 keep = True 

1711 keepIndex = [clipIndex[0]] 

1712 

1713 # Test if there are clipped objects that overlap two different visits 

1714 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1715 if len(clipIndex) == 2 and len(overlap) > 3: 

1716 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1717 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1718 keep = True 

1719 keepIndex = clipIndex 

1720 

1721 if not keep: 

1722 continue 

1723 

1724 for index in keepIndex: 

1725 globalIndex = indexList[index] 

1726 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1727 

1728 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1729 clipFootprints.append(footprint) 

1730 

1731 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1732 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1733 

1734 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1735 maskClipValue, maskDetValue, coaddBBox): 

1736 """Return individual warp footprints for large artifacts and append 

1737 them to ``clipList`` in place. 

1738 

1739 Identify big footprints composed of many sources in the coadd 

1740 difference that may have originated in a large diffuse source in the 

1741 coadd. We do this by indentifying all clipped footprints that overlap 

1742 significantly with each source in all the coaddTempExps. 

1743 

1744 Parameters 

1745 ---------- 

1746 clipList : `list` 

1747 List of alt mask SpanSets with clipping information. Modified. 

1748 clipFootprints : `list` 

1749 List of clipped footprints. 

1750 clipIndices : `list` 

1751 List of which entries in tempExpClipList each footprint belongs to. 

1752 maskClipValue 

1753 Mask value of clipped pixels. 

1754 maskDetValue 

1755 Mask value of detected pixels. 

1756 coaddBBox : `lsst.geom.Box` 

1757 BBox of the coadd and warps. 

1758 

1759 Returns 

1760 ------- 

1761 bigFootprintsCoadd : `list` 

1762 List of big footprints 

1763 """ 

1764 bigFootprintsCoadd = [] 

1765 ignoreMask = self.getBadPixelMask() 

1766 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1767 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1768 for footprint in visitFootprints.getFootprints(): 

1769 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1770 

1771 # build a mask of clipped footprints that are in this visit 

1772 clippedFootprintsVisit = [] 

1773 for foot, clipIndex in zip(clipFootprints, clipIndices): 

1774 if index not in clipIndex: 

1775 continue 

1776 clippedFootprintsVisit.append(foot) 

1777 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1778 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1779 

1780 bigFootprintsVisit = [] 

1781 for foot in visitFootprints.getFootprints(): 

1782 if foot.getArea() < self.config.minBigOverlap: 

1783 continue 

1784 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1785 if nCount > self.config.minBigOverlap: 

1786 bigFootprintsVisit.append(foot) 

1787 bigFootprintsCoadd.append(foot) 

1788 

1789 for footprint in bigFootprintsVisit: 

1790 clippedSpans["CLIPPED"].append(footprint.spans) 

1791 

1792 return bigFootprintsCoadd 

1793 

1794 

1795class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1796 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1797 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1798 "Only PSF-Matched Warps make sense for image subtraction. " 

1799 "Therefore, they must be an additional declared input."), 

1800 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1801 storageClass="ExposureF", 

1802 dimensions=("tract", "patch", "skymap", "visit"), 

1803 deferLoad=True, 

1804 multiple=True 

1805 ) 

1806 templateCoadd = pipeBase.connectionTypes.Output( 

1807 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1808 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1809 name="{fakesType}{outputCoaddName}CoaddPsfMatched", 

1810 storageClass="ExposureF", 

1811 dimensions=("tract", "patch", "skymap", "band"), 

1812 ) 

1813 

1814 def __init__(self, *, config=None): 

1815 super().__init__(config=config) 

1816 if not config.assembleStaticSkyModel.doWrite: 

1817 self.outputs.remove("templateCoadd") 

1818 config.validate() 

1819 

1820 

1821class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1822 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1823 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1824 target=AssembleCoaddTask, 

1825 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1826 " naive/first-iteration model of the static sky.", 

1827 ) 

1828 detect = pexConfig.ConfigurableField( 

1829 target=SourceDetectionTask, 

1830 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1831 ) 

1832 detectTemplate = pexConfig.ConfigurableField( 

1833 target=SourceDetectionTask, 

1834 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1835 ) 

1836 maskStreaks = pexConfig.ConfigurableField( 

1837 target=MaskStreaksTask, 

1838 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1839 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1840 "streakMaskName" 

1841 ) 

1842 streakMaskName = pexConfig.Field( 

1843 dtype=str, 

1844 default="STREAK", 

1845 doc="Name of mask bit used for streaks" 

1846 ) 

1847 maxNumEpochs = pexConfig.Field( 

1848 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1849 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1850 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1851 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1852 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1853 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1854 "than transient and not masked.", 

1855 dtype=int, 

1856 default=2 

1857 ) 

1858 maxFractionEpochsLow = pexConfig.RangeField( 

1859 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1860 "Effective maxNumEpochs = " 

1861 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1862 dtype=float, 

1863 default=0.4, 

1864 min=0., max=1., 

1865 ) 

1866 maxFractionEpochsHigh = pexConfig.RangeField( 

1867 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1868 "Effective maxNumEpochs = " 

1869 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1870 dtype=float, 

1871 default=0.03, 

1872 min=0., max=1., 

1873 ) 

1874 spatialThreshold = pexConfig.RangeField( 

1875 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1876 "temporal criteria. If 0, clip all. If 1, clip none.", 

1877 dtype=float, 

1878 default=0.5, 

1879 min=0., max=1., 

1880 inclusiveMin=True, inclusiveMax=True 

1881 ) 

1882 doScaleWarpVariance = pexConfig.Field( 

1883 doc="Rescale Warp variance plane using empirical noise?", 

1884 dtype=bool, 

1885 default=True, 

1886 ) 

1887 scaleWarpVariance = pexConfig.ConfigurableField( 

1888 target=ScaleVarianceTask, 

1889 doc="Rescale variance on warps", 

1890 ) 

1891 doPreserveContainedBySource = pexConfig.Field( 

1892 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1893 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1894 dtype=bool, 

1895 default=True, 

1896 ) 

1897 doPrefilterArtifacts = pexConfig.Field( 

1898 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1899 "because they will be excluded anyway. This prevents them from contributing " 

1900 "to the outlier epoch count image and potentially being labeled as persistant." 

1901 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1902 dtype=bool, 

1903 default=True 

1904 ) 

1905 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1906 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1907 dtype=str, 

1908 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1909 ) 

1910 prefilterArtifactsRatio = pexConfig.Field( 

1911 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1912 dtype=float, 

1913 default=0.05 

1914 ) 

1915 doFilterMorphological = pexConfig.Field( 

1916 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1917 "be streaks.", 

1918 dtype=bool, 

1919 default=False 

1920 ) 

1921 

1922 def setDefaults(self): 

1923 AssembleCoaddConfig.setDefaults(self) 

1924 self.statistic = 'MEAN' 

1925 self.doUsePsfMatchedPolygons = True 

1926 

1927 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1928 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1929 if "EDGE" in self.badMaskPlanes: 

1930 self.badMaskPlanes.remove('EDGE') 

1931 self.removeMaskPlanes.append('EDGE') 

1932 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1933 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1934 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1935 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1936 self.assembleStaticSkyModel.sigmaClip = 2.5 

1937 self.assembleStaticSkyModel.clipIter = 3 

1938 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1939 self.assembleStaticSkyModel.doWrite = False 

1940 self.detect.doTempLocalBackground = False 

1941 self.detect.reEstimateBackground = False 

1942 self.detect.returnOriginalFootprints = False 

1943 self.detect.thresholdPolarity = "both" 

1944 self.detect.thresholdValue = 5 

1945 self.detect.minPixels = 4 

1946 self.detect.isotropicGrow = True 

1947 self.detect.thresholdType = "pixel_stdev" 

1948 self.detect.nSigmaToGrow = 0.4 

1949 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1950 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1951 self.detectTemplate.nSigmaToGrow = 2.4 

1952 self.detectTemplate.doTempLocalBackground = False 

1953 self.detectTemplate.reEstimateBackground = False 

1954 self.detectTemplate.returnOriginalFootprints = False 

1955 

1956 def validate(self): 

1957 super().validate() 

1958 if self.assembleStaticSkyModel.doNImage: 

1959 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1960 "Please set assembleStaticSkyModel.doNImage=False") 

1961 

1962 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1963 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1964 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1965 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1966 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1967 

1968 

1969class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1970 """Assemble a compareWarp coadded image from a set of warps 

1971 by masking artifacts detected by comparing PSF-matched warps. 

1972 

1973 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1974 we clip outliers). The problem with doing this is that when computing the 

1975 coadd PSF at a given location, individual visit PSFs from visits with 

1976 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1977 In this task, we correct for this behavior by creating a new badMaskPlane 

1978 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1979 an artifact. We populate this plane on the input warps by comparing 

1980 PSF-matched warps with a PSF-matched median coadd which serves as a 

1981 model of the static sky. Any group of pixels that deviates from the 

1982 PSF-matched template coadd by more than config.detect.threshold sigma, 

1983 is an artifact candidate. The candidates are then filtered to remove 

1984 variable sources and sources that are difficult to subtract such as 

1985 bright stars. This filter is configured using the config parameters 

1986 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1987 the maximum fraction of epochs that the deviation can appear in and still 

1988 be considered an artifact. The spatialThreshold is the maximum fraction of 

1989 pixels in the footprint of the deviation that appear in other epochs 

1990 (where other epochs is defined by the temporalThreshold). If the deviant 

1991 region meets this criteria of having a significant percentage of pixels 

1992 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1993 set in the mask. These regions will not contribute to the final coadd. 

1994 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1995 of clipped regions. Note that the algorithm implemented by this task is 

1996 preliminary and works correctly for HSC data. Parameter modifications and 

1997 or considerable redesigning of the algorithm is likley required for other 

1998 surveys. 

1999 

2000 ``CompareWarpAssembleCoaddTask`` sub-classes 

2001 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

2002 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2003 

2004 Notes 

2005 ----- 

2006 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2007 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2008 ``baseDebug`` for more about ``debug.py`` files. 

2009 

2010 This task supports the following debug variables: 

2011 

2012 - ``saveCountIm`` 

2013 If True then save the Epoch Count Image as a fits file in the `figPath` 

2014 - ``figPath`` 

2015 Path to save the debug fits images and figures 

2016 

2017 For example, put something like: 

2018 

2019 .. code-block:: python 

2020 

2021 import lsstDebug 

2022 def DebugInfo(name): 

2023 di = lsstDebug.getInfo(name) 

2024 if name == "lsst.pipe.tasks.assembleCoadd": 

2025 di.saveCountIm = True 

2026 di.figPath = "/desired/path/to/debugging/output/images" 

2027 return di 

2028 lsstDebug.Info = DebugInfo 

2029 

2030 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2031 ``--debug`` flag. Some subtasks may have their own debug variables; 

2032 see individual Task documentation. 

2033 

2034 Examples 

2035 -------- 

2036 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2037 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2038 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2039 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2040 and filter to be coadded (specified using 

2041 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2042 along with a list of coaddTempExps to attempt to coadd (specified using 

2043 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2044 Only the warps that cover the specified tract and patch will be coadded. 

2045 A list of the available optional arguments can be obtained by calling 

2046 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2047 

2048 .. code-block:: none 

2049 

2050 assembleCoadd.py --help 

2051 

2052 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2053 context of multi-band processing, we will generate the HSC-I & -R band 

2054 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2055 To begin, assuming that the lsst stack has been already set up, we must 

2056 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2057 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2058 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2059 directory. To begin assembling the coadds, we must first 

2060 

2061 - processCcd 

2062 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2063 - makeSkyMap 

2064 create a skymap that covers the area of the sky present in the raw exposures 

2065 - makeCoaddTempExp 

2066 warp the individual calibrated exposures to the tangent plane of the coadd 

2067 

2068 We can perform all of these steps by running 

2069 

2070 .. code-block:: none 

2071 

2072 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2073 

2074 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2075 warped data, we call ``assembleCoadd.py`` as follows: 

2076 

2077 .. code-block:: none 

2078 

2079 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2080 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2081 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2082 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2083 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2084 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2085 --selectId visit=903988 ccd=24 

2086 

2087 This will process the HSC-I band data. The results are written in 

2088 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2089 """ 

2090 ConfigClass = CompareWarpAssembleCoaddConfig 

2091 _DefaultName = "compareWarpAssembleCoadd" 

2092 

2093 def __init__(self, *args, **kwargs): 

2094 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2095 self.makeSubtask("assembleStaticSkyModel") 

2096 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2097 self.makeSubtask("detect", schema=detectionSchema) 

2098 if self.config.doPreserveContainedBySource: 

2099 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2100 if self.config.doScaleWarpVariance: 

2101 self.makeSubtask("scaleWarpVariance") 

2102 if self.config.doFilterMorphological: 

2103 self.makeSubtask("maskStreaks") 

2104 

2105 @utils.inheritDoc(AssembleCoaddTask) 

2106 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2107 """ 

2108 Generate a templateCoadd to use as a naive model of static sky to 

2109 subtract from PSF-Matched warps. 

2110 

2111 Returns 

2112 ------- 

2113 result : `lsst.pipe.base.Struct` 

2114 Result struct with components: 

2115 

2116 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2117 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2118 """ 

2119 # Ensure that psfMatchedWarps are used as input warps for template generation 

2120 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2121 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2122 

2123 # Because subtasks don't have connections we have to make one. 

2124 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2125 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2126 if self.config.assembleStaticSkyModel.doWrite: 

2127 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2128 # Remove template coadd from both subtask's and main tasks outputs, 

2129 # because it is handled by the subtask as `coaddExposure` 

2130 del outputRefs.templateCoadd 

2131 del staticSkyModelOutputRefs.templateCoadd 

2132 

2133 # A PSF-Matched nImage does not exist as a dataset type 

2134 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2135 del staticSkyModelOutputRefs.nImage 

2136 

2137 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2138 staticSkyModelOutputRefs) 

2139 if templateCoadd is None: 

2140 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2141 

2142 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2143 nImage=templateCoadd.nImage, 

2144 warpRefList=templateCoadd.warpRefList, 

2145 imageScalerList=templateCoadd.imageScalerList, 

2146 weightList=templateCoadd.weightList) 

2147 

2148 @utils.inheritDoc(AssembleCoaddTask) 

2149 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2150 """ 

2151 Generate a templateCoadd to use as a naive model of static sky to 

2152 subtract from PSF-Matched warps. 

2153 

2154 Returns 

2155 ------- 

2156 result : `lsst.pipe.base.Struct` 

2157 Result struct with components: 

2158 

2159 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2160 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2161 """ 

2162 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2163 if templateCoadd is None: 

2164 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2165 

2166 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2167 nImage=templateCoadd.nImage, 

2168 warpRefList=templateCoadd.warpRefList, 

2169 imageScalerList=templateCoadd.imageScalerList, 

2170 weightList=templateCoadd.weightList) 

2171 

2172 def _noTemplateMessage(self, warpType): 

2173 warpName = (warpType[0].upper() + warpType[1:]) 

2174 message = """No %(warpName)s warps were found to build the template coadd which is 

2175 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2176 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2177 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2178 

2179 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2180 another algorithm like: 

2181 

2182 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2183 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2184 """ % {"warpName": warpName} 

2185 return message 

2186 

2187 @utils.inheritDoc(AssembleCoaddTask) 

2188 @pipeBase.timeMethod 

2189 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2190 supplementaryData, *args, **kwargs): 

2191 """Assemble the coadd. 

2192 

2193 Find artifacts and apply them to the warps' masks creating a list of 

2194 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2195 plane. Then pass these alternative masks to the base class's `run` 

2196 method. 

2197 

2198 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2199 that must contain a ``templateCoadd`` that serves as the 

2200 model of the static sky. 

2201 """ 

2202 

2203 # Check and match the order of the supplementaryData 

2204 # (PSF-matched) inputs to the order of the direct inputs, 

2205 # so that the artifact mask is applied to the right warp 

2206 dataIds = [ref.dataId for ref in tempExpRefList] 

2207 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2208 

2209 if dataIds != psfMatchedDataIds: 

2210 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2211 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2212 psfMatchedDataIds, dataIds) 

2213 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2214 psfMatchedDataIds, dataIds) 

2215 

2216 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2217 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2218 supplementaryData.warpRefList, 

2219 supplementaryData.imageScalerList) 

2220 

2221 badMaskPlanes = self.config.badMaskPlanes[:] 

2222 badMaskPlanes.append("CLIPPED") 

2223 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2224 

2225 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2226 spanSetMaskList, mask=badPixelMask) 

2227 

2228 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2229 # Psf-Matching moves the real edge inwards 

2230 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2231 return result 

2232 

2233 def applyAltEdgeMask(self, mask, altMaskList): 

2234 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2235 

2236 Parameters 

2237 ---------- 

2238 mask : `lsst.afw.image.Mask` 

2239 Original mask. 

2240 altMaskList : `list` 

2241 List of Dicts containing ``spanSet`` lists. 

2242 Each element contains the new mask plane name (e.g. "CLIPPED 

2243 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2244 the mask. 

2245 """ 

2246 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2247 for visitMask in altMaskList: 

2248 if "EDGE" in visitMask: 

2249 for spanSet in visitMask['EDGE']: 

2250 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2251 

2252 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2253 """Find artifacts. 

2254 

2255 Loop through warps twice. The first loop builds a map with the count 

2256 of how many epochs each pixel deviates from the templateCoadd by more 

2257 than ``config.chiThreshold`` sigma. The second loop takes each 

2258 difference image and filters the artifacts detected in each using 

2259 count map to filter out variable sources and sources that are 

2260 difficult to subtract cleanly. 

2261 

2262 Parameters 

2263 ---------- 

2264 templateCoadd : `lsst.afw.image.Exposure` 

2265 Exposure to serve as model of static sky. 

2266 tempExpRefList : `list` 

2267 List of data references to warps. 

2268 imageScalerList : `list` 

2269 List of image scalers. 

2270 

2271 Returns 

2272 ------- 

2273 altMasks : `list` 

2274 List of dicts containing information about CLIPPED 

2275 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2276 """ 

2277 

2278 self.log.debug("Generating Count Image, and mask lists.") 

2279 coaddBBox = templateCoadd.getBBox() 

2280 slateIm = afwImage.ImageU(coaddBBox) 

2281 epochCountImage = afwImage.ImageU(coaddBBox) 

2282 nImage = afwImage.ImageU(coaddBBox) 

2283 spanSetArtifactList = [] 

2284 spanSetNoDataMaskList = [] 

2285 spanSetEdgeList = [] 

2286 spanSetBadMorphoList = [] 

2287 badPixelMask = self.getBadPixelMask() 

2288 

2289 # mask of the warp diffs should = that of only the warp 

2290 templateCoadd.mask.clearAllMaskPlanes() 

2291 

2292 if self.config.doPreserveContainedBySource: 

2293 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2294 else: 

2295 templateFootprints = None 

2296 

2297 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2298 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2299 if warpDiffExp is not None: 

2300 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2301 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2302 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2303 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2304 fpSet.positive.merge(fpSet.negative) 

2305 footprints = fpSet.positive 

2306 slateIm.set(0) 

2307 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2308 

2309 # Remove artifacts due to defects before they contribute to the epochCountImage 

2310 if self.config.doPrefilterArtifacts: 

2311 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2312 

2313 # Clear mask before adding prefiltered spanSets 

2314 self.detect.clearMask(warpDiffExp.mask) 

2315 for spans in spanSetList: 

2316 spans.setImage(slateIm, 1, doClip=True) 

2317 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2318 epochCountImage += slateIm 

2319 

2320 if self.config.doFilterMorphological: 

2321 maskName = self.config.streakMaskName 

2322 _ = self.maskStreaks.run(warpDiffExp) 

2323 streakMask = warpDiffExp.mask 

2324 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2325 streakMask.getPlaneBitMask(maskName)).split() 

2326 

2327 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2328 # undergo a second convolution. Pixels with data in the direct warp 

2329 # but not in the PSF-matched warp will not have their artifacts detected. 

2330 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2331 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2332 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2333 nansMask.setXY0(warpDiffExp.getXY0()) 

2334 edgeMask = warpDiffExp.mask 

2335 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2336 edgeMask.getPlaneBitMask("EDGE")).split() 

2337 else: 

2338 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2339 # In this case, mask the whole epoch 

2340 nansMask = afwImage.MaskX(coaddBBox, 1) 

2341 spanSetList = [] 

2342 spanSetEdgeMask = [] 

2343 spanSetStreak = [] 

2344 

2345 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2346 

2347 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2348 spanSetArtifactList.append(spanSetList) 

2349 spanSetEdgeList.append(spanSetEdgeMask) 

2350 if self.config.doFilterMorphological: 

2351 spanSetBadMorphoList.append(spanSetStreak) 

2352 

2353 if lsstDebug.Info(__name__).saveCountIm: 

2354 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2355 epochCountImage.writeFits(path) 

2356 

2357 for i, spanSetList in enumerate(spanSetArtifactList): 

2358 if spanSetList: 

2359 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2360 templateFootprints) 

2361 spanSetArtifactList[i] = filteredSpanSetList 

2362 if self.config.doFilterMorphological: 

2363 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2364 

2365 altMasks = [] 

2366 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2367 altMasks.append({'CLIPPED': artifacts, 

2368 'NO_DATA': noData, 

2369 'EDGE': edge}) 

2370 return altMasks 

2371 

2372 def prefilterArtifacts(self, spanSetList, exp): 

2373 """Remove artifact candidates covered by bad mask plane. 

2374 

2375 Any future editing of the candidate list that does not depend on 

2376 temporal information should go in this method. 

2377 

2378 Parameters 

2379 ---------- 

2380 spanSetList : `list` 

2381 List of SpanSets representing artifact candidates. 

2382 exp : `lsst.afw.image.Exposure` 

2383 Exposure containing mask planes used to prefilter. 

2384 

2385 Returns 

2386 ------- 

2387 returnSpanSetList : `list` 

2388 List of SpanSets with artifacts. 

2389 """ 

2390 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2391 goodArr = (exp.mask.array & badPixelMask) == 0 

2392 returnSpanSetList = [] 

2393 bbox = exp.getBBox() 

2394 x0, y0 = exp.getXY0() 

2395 for i, span in enumerate(spanSetList): 

2396 y, x = span.clippedTo(bbox).indices() 

2397 yIndexLocal = numpy.array(y) - y0 

2398 xIndexLocal = numpy.array(x) - x0 

2399 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2400 if goodRatio > self.config.prefilterArtifactsRatio: 

2401 returnSpanSetList.append(span) 

2402 return returnSpanSetList 

2403 

2404 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2405 """Filter artifact candidates. 

2406 

2407 Parameters 

2408 ---------- 

2409 spanSetList : `list` 

2410 List of SpanSets representing artifact candidates. 

2411 epochCountImage : `lsst.afw.image.Image` 

2412 Image of accumulated number of warpDiff detections. 

2413 nImage : `lsst.afw.image.Image` 

2414 Image of the accumulated number of total epochs contributing. 

2415 

2416 Returns 

2417 ------- 

2418 maskSpanSetList : `list` 

2419 List of SpanSets with artifacts. 

2420 """ 

2421 

2422 maskSpanSetList = [] 

2423 x0, y0 = epochCountImage.getXY0() 

2424 for i, span in enumerate(spanSetList): 

2425 y, x = span.indices() 

2426 yIdxLocal = [y1 - y0 for y1 in y] 

2427 xIdxLocal = [x1 - x0 for x1 in x] 

2428 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2429 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2430 

2431 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2432 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2433 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2434 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2435 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2436 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2437 & (outlierN <= effectiveMaxNumEpochs)) 

2438 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2439 if percentBelowThreshold > self.config.spatialThreshold: 

2440 maskSpanSetList.append(span) 

2441 

2442 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

2443 # If a candidate is contained by a footprint on the template coadd, do not clip 

2444 filteredMaskSpanSetList = [] 

2445 for span in maskSpanSetList: 

2446 doKeep = True 

2447 for footprint in footprintsToExclude.positive.getFootprints(): 

2448 if footprint.spans.contains(span): 

2449 doKeep = False 

2450 break 

2451 if doKeep: 

2452 filteredMaskSpanSetList.append(span) 

2453 maskSpanSetList = filteredMaskSpanSetList 

2454 

2455 return maskSpanSetList 

2456 

2457 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2458 """Fetch a warp from the butler and return a warpDiff. 

2459 

2460 Parameters 

2461 ---------- 

2462 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2463 Butler dataRef for the warp. 

2464 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2465 An image scaler object. 

2466 templateCoadd : `lsst.afw.image.Exposure` 

2467 Exposure to be substracted from the scaled warp. 

2468 

2469 Returns 

2470 ------- 

2471 warp : `lsst.afw.image.Exposure` 

2472 Exposure of the image difference between the warp and template. 

2473 """ 

2474 

2475 # If the PSF-Matched warp did not exist for this direct warp 

2476 # None is holding its place to maintain order in Gen 3 

2477 if warpRef is None: 

2478 return None 

2479 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2480 warpName = self.getTempExpDatasetName('psfMatched') 

2481 if not isinstance(warpRef, DeferredDatasetHandle): 

2482 if not warpRef.datasetExists(warpName): 

2483 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2484 return None 

2485 warp = warpRef.get(datasetType=warpName, immediate=True) 

2486 # direct image scaler OK for PSF-matched Warp 

2487 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2488 mi = warp.getMaskedImage() 

2489 if self.config.doScaleWarpVariance: 

2490 try: 

2491 self.scaleWarpVariance.run(mi) 

2492 except Exception as exc: 

2493 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2494 mi -= templateCoadd.getMaskedImage() 

2495 return warp 

2496 

2497 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2498 """Return a path to which to write debugging output. 

2499 

2500 Creates a hyphen-delimited string of dataId values for simple filenames. 

2501 

2502 Parameters 

2503 ---------- 

2504 prefix : `str` 

2505 Prefix for filename. 

2506 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2507 Butler dataRef to make the path from. 

2508 coaddLevel : `bool`, optional. 

2509 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2510 'filter', but no 'visit'). 

2511 

2512 Returns 

2513 ------- 

2514 result : `str` 

2515 Path for debugging output. 

2516 """ 

2517 if coaddLevel: 

2518 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2519 else: 

2520 keys = warpRef.dataId.keys() 

2521 keyList = sorted(keys, reverse=True) 

2522 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2523 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2524 return os.path.join(directory, filename)