Coverage for python/lsst/drp/tasks/assemble_coadd.py: 16%

633 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 12:15 +0000

1# This file is part of drp_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

23 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

24 

25import copy 

26import numpy 

27import warnings 

28import logging 

29import lsst.pex.config as pexConfig 

30import lsst.pex.exceptions as pexExceptions 

31import lsst.geom as geom 

32import lsst.afw.geom as afwGeom 

33import lsst.afw.image as afwImage 

34import lsst.afw.math as afwMath 

35import lsst.afw.table as afwTable 

36import lsst.coadd.utils as coaddUtils 

37import lsst.pipe.base as pipeBase 

38import lsst.meas.algorithms as measAlg 

39import lsstDebug 

40import lsst.utils as utils 

41from lsst.skymap import BaseSkyMap 

42from lsst.pipe.tasks.coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList, subBBoxIter 

43from lsst.pipe.tasks.interpImage import InterpImageTask 

44from lsst.pipe.tasks.scaleZeroPoint import ScaleZeroPointTask 

45from lsst.pipe.tasks.maskStreaks import MaskStreaksTask 

46from lsst.pipe.tasks.healSparseMapping import HealSparseInputMapTask 

47from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask 

48from lsst.utils.timer import timeMethod 

49from deprecated.sphinx import deprecated 

50 

51log = logging.getLogger(__name__) 

52 

53 

54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

55 dimensions=("tract", "patch", "band", "skymap"), 

56 defaultTemplates={"inputCoaddName": "deep", 

57 "outputCoaddName": "deep", 

58 "warpType": "direct", 

59 "warpTypeSuffix": ""}): 

60 

61 inputWarps = pipeBase.connectionTypes.Input( 

62 doc=("Input list of warps to be assemebled i.e. stacked." 

63 "WarpType (e.g. direct, psfMatched) is controlled by the " 

64 "warpType config parameter"), 

65 name="{inputCoaddName}Coadd_{warpType}Warp", 

66 storageClass="ExposureF", 

67 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

68 deferLoad=True, 

69 multiple=True 

70 ) 

71 skyMap = pipeBase.connectionTypes.Input( 

72 doc="Input definition of geometry/bbox and projection/wcs for coadded " 

73 "exposures", 

74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

75 storageClass="SkyMap", 

76 dimensions=("skymap", ), 

77 ) 

78 selectedVisits = pipeBase.connectionTypes.Input( 

79 doc="Selected visits to be coadded.", 

80 name="{outputCoaddName}Visits", 

81 storageClass="StructuredDataDict", 

82 dimensions=("instrument", "tract", "patch", "skymap", "band") 

83 ) 

84 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

85 doc=("Input Bright Object Mask mask produced with external catalogs " 

86 "to be applied to the mask plane BRIGHT_OBJECT." 

87 ), 

88 name="brightObjectMask", 

89 storageClass="ObjectMaskCatalog", 

90 dimensions=("tract", "patch", "skymap", "band"), 

91 minimum=0, 

92 ) 

93 coaddExposure = pipeBase.connectionTypes.Output( 

94 doc="Output coadded exposure, produced by stacking input warps", 

95 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

96 storageClass="ExposureF", 

97 dimensions=("tract", "patch", "skymap", "band"), 

98 ) 

99 nImage = pipeBase.connectionTypes.Output( 

100 doc="Output image of number of input images per pixel", 

101 name="{outputCoaddName}Coadd_nImage", 

102 storageClass="ImageU", 

103 dimensions=("tract", "patch", "skymap", "band"), 

104 ) 

105 inputMap = pipeBase.connectionTypes.Output( 

106 doc="Output healsparse map of input images", 

107 name="{outputCoaddName}Coadd_inputMap", 

108 storageClass="HealSparseMap", 

109 dimensions=("tract", "patch", "skymap", "band"), 

110 ) 

111 

112 def __init__(self, *, config=None): 

113 super().__init__(config=config) 

114 

115 if not config.doMaskBrightObjects: 

116 self.prerequisiteInputs.remove("brightObjectMask") 

117 

118 if not config.doSelectVisits: 

119 self.inputs.remove("selectedVisits") 

120 

121 if not config.doNImage: 

122 self.outputs.remove("nImage") 

123 

124 if not self.config.doInputMap: 

125 self.outputs.remove("inputMap") 

126 

127 

128class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

129 pipelineConnections=AssembleCoaddConnections): 

130 warpType = pexConfig.Field( 

131 doc="Warp name: one of 'direct' or 'psfMatched'", 

132 dtype=str, 

133 default="direct", 

134 ) 

135 subregionSize = pexConfig.ListField( 

136 dtype=int, 

137 doc="Width, height of stack subregion size; " 

138 "make small enough that a full stack of images will fit into memory " 

139 " at once.", 

140 length=2, 

141 default=(2000, 2000), 

142 ) 

143 statistic = pexConfig.Field( 

144 dtype=str, 

145 doc="Main stacking statistic for aggregating over the epochs.", 

146 default="MEANCLIP", 

147 ) 

148 doOnlineForMean = pexConfig.Field( 

149 dtype=bool, 

150 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

151 default=False, 

152 ) 

153 doSigmaClip = pexConfig.Field( 

154 dtype=bool, 

155 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic?", 

156 deprecated=True, 

157 default=False, 

158 ) 

159 sigmaClip = pexConfig.Field( 

160 dtype=float, 

161 doc="Sigma for outlier rejection; ignored if non-clipping statistic " 

162 "selected.", 

163 default=3.0, 

164 ) 

165 clipIter = pexConfig.Field( 

166 dtype=int, 

167 doc="Number of iterations of outlier rejection; ignored if " 

168 "non-clipping statistic selected.", 

169 default=2, 

170 ) 

171 calcErrorFromInputVariance = pexConfig.Field( 

172 dtype=bool, 

173 doc="Calculate coadd variance from input variance by stacking " 

174 "statistic. Passed to " 

175 "StatisticsControl.setCalcErrorFromInputVariance()", 

176 default=True, 

177 ) 

178 scaleZeroPoint = pexConfig.ConfigurableField( 

179 target=ScaleZeroPointTask, 

180 doc="Task to adjust the photometric zero point of the coadd temp " 

181 "exposures", 

182 ) 

183 doInterp = pexConfig.Field( 

184 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but " 

185 "the results are ugly.", 

186 dtype=bool, 

187 default=True, 

188 ) 

189 interpImage = pexConfig.ConfigurableField( 

190 target=InterpImageTask, 

191 doc="Task to interpolate (and extrapolate) over NaN pixels", 

192 ) 

193 doWrite = pexConfig.Field( 

194 doc="Persist coadd?", 

195 dtype=bool, 

196 default=True, 

197 ) 

198 doNImage = pexConfig.Field( 

199 doc="Create image of number of contributing exposures for each pixel", 

200 dtype=bool, 

201 default=False, 

202 ) 

203 doUsePsfMatchedPolygons = pexConfig.Field( 

204 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set " 

205 "to True by CompareWarp only.", 

206 dtype=bool, 

207 default=False, 

208 ) 

209 maskPropagationThresholds = pexConfig.DictField( 

210 keytype=str, 

211 itemtype=float, 

212 doc=("Threshold (in fractional weight) of rejection at which we " 

213 "propagate a mask plane to the coadd; that is, we set the mask " 

214 "bit on the coadd if the fraction the rejected frames " 

215 "would have contributed exceeds this value."), 

216 default={"SAT": 0.1}, 

217 ) 

218 removeMaskPlanes = pexConfig.ListField( 

219 dtype=str, 

220 default=["NOT_DEBLENDED"], 

221 doc="Mask planes to remove before coadding", 

222 ) 

223 doMaskBrightObjects = pexConfig.Field( 

224 dtype=bool, 

225 default=False, 

226 doc="Set mask and flag bits for bright objects?", 

227 ) 

228 brightObjectMaskName = pexConfig.Field( 

229 dtype=str, 

230 default="BRIGHT_OBJECT", 

231 doc="Name of mask bit used for bright objects", 

232 ) 

233 coaddPsf = pexConfig.ConfigField( 

234 doc="Configuration for CoaddPsf", 

235 dtype=measAlg.CoaddPsfConfig, 

236 ) 

237 doAttachTransmissionCurve = pexConfig.Field( 

238 dtype=bool, 

239 default=False, 

240 optional=False, 

241 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

242 "(requires all input Exposures to have TransmissionCurves).") 

243 ) 

244 hasFakes = pexConfig.Field( 

245 dtype=bool, 

246 default=False, 

247 doc="Should be set to True if fake sources have been inserted into the input data." 

248 ) 

249 doSelectVisits = pexConfig.Field( 

250 doc="Coadd only visits selected by a SelectVisitsTask", 

251 dtype=bool, 

252 default=False, 

253 ) 

254 doInputMap = pexConfig.Field( 

255 doc="Create a bitwise map of coadd inputs", 

256 dtype=bool, 

257 default=False, 

258 ) 

259 inputMapper = pexConfig.ConfigurableField( 

260 doc="Input map creation subtask.", 

261 target=HealSparseInputMapTask, 

262 ) 

263 

264 def setDefaults(self): 

265 super().setDefaults() 

266 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

267 

268 def validate(self): 

269 super().validate() 

270 if self.doPsfMatch: # TODO: Remove this in DM-39841 

271 # Backwards compatibility. 

272 # Configs do not have loggers 

273 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

274 self.warpType = 'psfMatched' 

275 if self.doSigmaClip and self.statistic != "MEANCLIP": 

276 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

277 self.statistic = "MEANCLIP" 

278 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

279 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

280 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

281 

282 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

283 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

284 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

285 if str(k) not in unstackableStats] 

286 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

287 % (self.statistic, stackableStats)) 

288 

289 

290class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

291 """Assemble a coadded image from a set of warps. 

292 

293 Each Warp that goes into a coadd will typically have an independent 

294 photometric zero-point. Therefore, we must scale each Warp to set it to 

295 a common photometric zeropoint. WarpType may be one of 'direct' or 

296 'psfMatched', and the boolean configs `config.makeDirect` and 

297 `config.makePsfMatched` set which of the warp types will be coadded. 

298 The coadd is computed as a mean with optional outlier rejection. 

299 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

300 Finally, Warps can have bad 'NaN' pixels which received no input from the 

301 source calExps. We interpolate over these bad (NaN) pixels. 

302 

303 `AssembleCoaddTask` uses several sub-tasks. These are 

304 

305 - `~lsst.pipe.tasks.ScaleZeroPointTask` 

306 - create and use an ``imageScaler`` object to scale the photometric 

307 zeropoint for each Warp 

308 - `~lsst.pipe.tasks.InterpImageTask` 

309 - interpolate across bad pixels (NaN) in the final coadd 

310 

311 You can retarget these subtasks if you wish. 

312 

313 Raises 

314 ------ 

315 RuntimeError 

316 Raised if unable to define mask plane for bright objects. 

317 

318 Notes 

319 ----- 

320 Debugging: 

321 `AssembleCoaddTask` has no debug variables of its own. Some of the 

322 subtasks may support `~lsst.base.lsstDebug` variables. See the 

323 documentation for the subtasks for further information. 

324 """ 

325 

326 ConfigClass = AssembleCoaddConfig 

327 _DefaultName = "assembleCoadd" 

328 

329 def __init__(self, *args, **kwargs): 

330 # TODO: DM-17415 better way to handle previously allowed passed args 

331 # e.g.`AssembleCoaddTask(config)` 

332 if args: 

333 argNames = ["config", "name", "parentTask", "log"] 

334 kwargs.update({k: v for k, v in zip(argNames, args)}) 

335 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

336 "PipelineTask will not take positional args" % argNames, FutureWarning, 

337 stacklevel=2) 

338 

339 super().__init__(**kwargs) 

340 self.makeSubtask("interpImage") 

341 self.makeSubtask("scaleZeroPoint") 

342 

343 if self.config.doMaskBrightObjects: 

344 mask = afwImage.Mask() 

345 try: 

346 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

347 except pexExceptions.LsstCppException: 

348 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

349 mask.getMaskPlaneDict().keys()) 

350 del mask 

351 

352 if self.config.doInputMap: 

353 self.makeSubtask("inputMapper") 

354 

355 self.warpType = self.config.warpType 

356 

357 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

358 inputData = butlerQC.get(inputRefs) 

359 

360 # Construct skyInfo expected by run 

361 # Do not remove skyMap from inputData in case _makeSupplementaryData 

362 # needs it 

363 skyMap = inputData["skyMap"] 

364 outputDataId = butlerQC.quantum.dataId 

365 

366 inputData['skyInfo'] = makeSkyInfo(skyMap, 

367 tractId=outputDataId['tract'], 

368 patchId=outputDataId['patch']) 

369 

370 if self.config.doSelectVisits: 

371 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

372 else: 

373 warpRefList = inputData['inputWarps'] 

374 

375 inputs = self.prepareInputs(warpRefList) 

376 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

377 self.getTempExpDatasetName(self.warpType)) 

378 if len(inputs.tempExpRefList) == 0: 

379 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

380 

381 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

382 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

383 inputs.weightList, supplementaryData=supplementaryData) 

384 

385 inputData.setdefault('brightObjectMask', None) 

386 if self.config.doMaskBrightObjects and inputData["brightObjectMask"] is None: 

387 log.warning("doMaskBrightObjects is set to True, but brightObjectMask not loaded") 

388 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

389 

390 if self.config.doWrite: 

391 butlerQC.put(retStruct, outputRefs) 

392 return retStruct 

393 

394 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

395 """Interpolate over missing data and mask bright stars. 

396 

397 Parameters 

398 ---------- 

399 coaddExposure : `lsst.afw.image.Exposure` 

400 The coadded exposure to process. 

401 brightObjectMasks : `lsst.afw.table` or `None`, optional 

402 Table of bright objects to mask. 

403 dataId : `lsst.daf.butler.DataId` or `None`, optional 

404 Data identification. 

405 """ 

406 if self.config.doInterp: 

407 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

408 # The variance must be positive; work around for DM-3201. 

409 varArray = coaddExposure.variance.array 

410 with numpy.errstate(invalid="ignore"): 

411 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

412 

413 if self.config.doMaskBrightObjects: 

414 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

415 

416 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

417 """Make additional inputs to run() specific to subclasses (Gen3). 

418 

419 Duplicates interface of `runQuantum` method. 

420 Available to be implemented by subclasses only if they need the 

421 coadd dataRef for performing preliminary processing before 

422 assembling the coadd. 

423 

424 Parameters 

425 ---------- 

426 butlerQC : `~lsst.pipe.base.ButlerQuantumContext` 

427 Gen3 Butler object for fetching additional data products before 

428 running the Task specialized for quantum being processed. 

429 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

430 Attributes are the names of the connections describing input 

431 dataset types. Values are DatasetRefs that task consumes for the 

432 corresponding dataset type. DataIds are guaranteed to match data 

433 objects in ``inputData``. 

434 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

435 Attributes are the names of the connections describing output 

436 dataset types. Values are DatasetRefs that task is to produce 

437 for the corresponding dataset type. 

438 """ 

439 return pipeBase.Struct() 

440 

441 @deprecated( 

442 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData", 

443 version="v25.0", 

444 category=FutureWarning 

445 ) 

446 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

447 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

448 

449 def prepareInputs(self, refList): 

450 """Prepare the input warps for coaddition by measuring the weight for 

451 each warp and the scaling for the photometric zero point. 

452 

453 Each Warp has its own photometric zeropoint and background variance. 

454 Before coadding these Warps together, compute a scale factor to 

455 normalize the photometric zeropoint and compute the weight for each 

456 Warp. 

457 

458 Parameters 

459 ---------- 

460 refList : `list` 

461 List of data references to tempExp. 

462 

463 Returns 

464 ------- 

465 result : `~lsst.pipe.base.Struct` 

466 Results as a struct with attributes: 

467 

468 ``tempExprefList`` 

469 `list` of data references to tempExp. 

470 ``weightList`` 

471 `list` of weightings. 

472 ``imageScalerList`` 

473 `list` of image scalers. 

474 """ 

475 statsCtrl = afwMath.StatisticsControl() 

476 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

477 statsCtrl.setNumIter(self.config.clipIter) 

478 statsCtrl.setAndMask(self.getBadPixelMask()) 

479 statsCtrl.setNanSafe(True) 

480 # compute tempExpRefList: a list of tempExpRef that actually exist 

481 # and weightList: a list of the weight of the associated coadd tempExp 

482 # and imageScalerList: a list of scale factors for the associated coadd 

483 # tempExp. 

484 tempExpRefList = [] 

485 weightList = [] 

486 imageScalerList = [] 

487 tempExpName = self.getTempExpDatasetName(self.warpType) 

488 for tempExpRef in refList: 

489 tempExp = tempExpRef.get() 

490 # Ignore any input warp that is empty of data 

491 if numpy.isnan(tempExp.image.array).all(): 

492 continue 

493 maskedImage = tempExp.getMaskedImage() 

494 imageScaler = self.scaleZeroPoint.computeImageScaler( 

495 exposure=tempExp, 

496 dataRef=tempExpRef, # FIXME 

497 ) 

498 try: 

499 imageScaler.scaleMaskedImage(maskedImage) 

500 except Exception as e: 

501 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

502 continue 

503 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

504 afwMath.MEANCLIP, statsCtrl) 

505 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

506 weight = 1.0 / float(meanVar) 

507 if not numpy.isfinite(weight): 

508 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

509 continue 

510 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

511 

512 del maskedImage 

513 del tempExp 

514 

515 tempExpRefList.append(tempExpRef) 

516 weightList.append(weight) 

517 imageScalerList.append(imageScaler) 

518 

519 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

520 imageScalerList=imageScalerList) 

521 

522 def prepareStats(self, mask=None): 

523 """Prepare the statistics for coadding images. 

524 

525 Parameters 

526 ---------- 

527 mask : `int`, optional 

528 Bit mask value to exclude from coaddition. 

529 

530 Returns 

531 ------- 

532 stats : `~lsst.pipe.base.Struct` 

533 Statistics as a struct with attributes: 

534 

535 ``statsCtrl`` 

536 Statistics control object for coadd 

537 (`~lsst.afw.math.StatisticsControl`). 

538 ``statsFlags`` 

539 Statistic for coadd (`~lsst.afw.math.Property`). 

540 """ 

541 if mask is None: 

542 mask = self.getBadPixelMask() 

543 statsCtrl = afwMath.StatisticsControl() 

544 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

545 statsCtrl.setNumIter(self.config.clipIter) 

546 statsCtrl.setAndMask(mask) 

547 statsCtrl.setNanSafe(True) 

548 statsCtrl.setWeighted(True) 

549 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

550 for plane, threshold in self.config.maskPropagationThresholds.items(): 

551 bit = afwImage.Mask.getMaskPlane(plane) 

552 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

553 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

554 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

555 

556 @timeMethod 

557 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

558 altMaskList=None, mask=None, supplementaryData=None): 

559 """Assemble a coadd from input warps. 

560 

561 Assemble the coadd using the provided list of coaddTempExps. Since 

562 the full coadd covers a patch (a large area), the assembly is 

563 performed over small areas on the image at a time in order to 

564 conserve memory usage. Iterate over subregions within the outer 

565 bbox of the patch using `assembleSubregion` to stack the corresponding 

566 subregions from the coaddTempExps with the statistic specified. 

567 Set the edge bits the coadd mask based on the weight map. 

568 

569 Parameters 

570 ---------- 

571 skyInfo : `~lsst.pipe.base.Struct` 

572 Struct with geometric information about the patch. 

573 tempExpRefList : `list` 

574 List of data references to Warps (previously called CoaddTempExps). 

575 imageScalerList : `list` 

576 List of image scalers. 

577 weightList : `list` 

578 List of weights. 

579 altMaskList : `list`, optional 

580 List of alternate masks to use rather than those stored with 

581 tempExp. 

582 mask : `int`, optional 

583 Bit mask value to exclude from coaddition. 

584 supplementaryData : `~lsst.pipe.base.Struct`, optional 

585 Struct with additional data products needed to assemble coadd. 

586 Only used by subclasses that implement ``_makeSupplementaryData`` 

587 and override `run`. 

588 

589 Returns 

590 ------- 

591 result : `~lsst.pipe.base.Struct` 

592 Results as a struct with attributes: 

593 

594 ``coaddExposure`` 

595 Coadded exposure (`~lsst.afw.image.Exposure`). 

596 ``nImage`` 

597 Exposure count image (`~lsst.afw.image.Image`), if requested. 

598 ``inputMap`` 

599 Bit-wise map of inputs, if requested. 

600 ``warpRefList`` 

601 Input list of refs to the warps 

602 (`~lsst.daf.butler.DeferredDatasetHandle`) (unmodified). 

603 ``imageScalerList`` 

604 Input list of image scalers (`list`) (unmodified). 

605 ``weightList`` 

606 Input list of weights (`list`) (unmodified). 

607 

608 Raises 

609 ------ 

610 lsst.pipe.base.NoWorkFound 

611 Raised if no data references are provided. 

612 """ 

613 tempExpName = self.getTempExpDatasetName(self.warpType) 

614 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

615 if not tempExpRefList: 

616 raise pipeBase.NoWorkFound("No exposures provided for co-addition.") 

617 

618 stats = self.prepareStats(mask=mask) 

619 

620 if altMaskList is None: 

621 altMaskList = [None]*len(tempExpRefList) 

622 

623 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

624 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

625 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

626 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

627 coaddMaskedImage = coaddExposure.getMaskedImage() 

628 subregionSizeArr = self.config.subregionSize 

629 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

630 # if nImage is requested, create a zero one which can be passed to 

631 # assembleSubregion. 

632 if self.config.doNImage: 

633 nImage = afwImage.ImageU(skyInfo.bbox) 

634 else: 

635 nImage = None 

636 # If inputMap is requested, create the initial version that can be 

637 # masked in assembleSubregion. 

638 if self.config.doInputMap: 

639 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

640 skyInfo.wcs, 

641 coaddExposure.getInfo().getCoaddInputs().ccds) 

642 

643 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 

644 try: 

645 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

646 weightList, altMaskList, stats.ctrl, 

647 nImage=nImage) 

648 except Exception as e: 

649 self.log.exception("Cannot compute online coadd %s", e) 

650 raise 

651 else: 

652 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize): 

653 try: 

654 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

655 weightList, altMaskList, stats.flags, stats.ctrl, 

656 nImage=nImage) 

657 except Exception as e: 

658 self.log.exception("Cannot compute coadd %s: %s", subBBox, e) 

659 raise 

660 

661 # If inputMap is requested, we must finalize the map after the 

662 # accumulation. 

663 if self.config.doInputMap: 

664 self.inputMapper.finalize_ccd_input_map_mask() 

665 inputMap = self.inputMapper.ccd_input_map 

666 else: 

667 inputMap = None 

668 

669 self.setInexactPsf(coaddMaskedImage.getMask()) 

670 # Despite the name, the following doesn't really deal with "EDGE" 

671 # pixels: it identifies pixels that didn't receive any unmasked inputs 

672 # (as occurs around the edge of the field). 

673 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

674 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

675 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

676 weightList=weightList, inputMap=inputMap) 

677 

678 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

679 """Set the metadata for the coadd. 

680 

681 This basic implementation sets the filter from the first input. 

682 

683 Parameters 

684 ---------- 

685 coaddExposure : `lsst.afw.image.Exposure` 

686 The target exposure for the coadd. 

687 tempExpRefList : `list` 

688 List of data references to tempExp. 

689 weightList : `list` 

690 List of weights. 

691 

692 Raises 

693 ------ 

694 AssertionError 

695 Raised if there is a length mismatch. 

696 """ 

697 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

698 

699 # We load a single pixel of each coaddTempExp, because we just want to 

700 # get at the metadata (and we need more than just the PropertySet that 

701 # contains the header), which is not possible with the current butler 

702 # (see #2777). 

703 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

704 

705 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

706 

707 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

708 

709 # Set the coadd FilterLabel to the band of the first input exposure: 

710 # Coadds are calibrated, so the physical label is now meaningless. 

711 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel)) 

712 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

713 coaddInputs.ccds.reserve(numCcds) 

714 coaddInputs.visits.reserve(len(tempExpList)) 

715 

716 for tempExp, weight in zip(tempExpList, weightList): 

717 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

718 

719 if self.config.doUsePsfMatchedPolygons: 

720 self.shrinkValidPolygons(coaddInputs) 

721 

722 coaddInputs.visits.sort() 

723 coaddInputs.ccds.sort() 

724 if self.warpType == "psfMatched": 

725 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was 

726 # dynamically defined by ModelPsfMatchTask as the square box 

727 # bounding its spatially-variable, pre-matched WarpedPsf. 

728 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

729 # having the maximum width (sufficient because square) 

730 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

731 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

732 for modelPsf in modelPsfList] 

733 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

734 else: 

735 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

736 self.config.coaddPsf.makeControl()) 

737 coaddExposure.setPsf(psf) 

738 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

739 coaddExposure.getWcs()) 

740 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

741 if self.config.doAttachTransmissionCurve: 

742 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

743 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

744 

745 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

746 altMaskList, statsFlags, statsCtrl, nImage=None): 

747 """Assemble the coadd for a sub-region. 

748 

749 For each coaddTempExp, check for (and swap in) an alternative mask 

750 if one is passed. Remove mask planes listed in 

751 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

752 `lsst.afw.math.statisticsStack` with the statistic specified by 

753 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN 

754 for a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection 

755 using an N-sigma clipped mean where N and iterations are specified by 

756 statsCtrl. Assign the stacked subregion back to the coadd. 

757 

758 Parameters 

759 ---------- 

760 coaddExposure : `lsst.afw.image.Exposure` 

761 The target exposure for the coadd. 

762 bbox : `lsst.geom.Box` 

763 Sub-region to coadd. 

764 tempExpRefList : `list` 

765 List of data reference to tempExp. 

766 imageScalerList : `list` 

767 List of image scalers. 

768 weightList : `list` 

769 List of weights. 

770 altMaskList : `list` 

771 List of alternate masks to use rather than those stored with 

772 tempExp, or None. Each element is dict with keys = mask plane 

773 name to which to add the spans. 

774 statsFlags : `lsst.afw.math.Property` 

775 Property object for statistic for coadd. 

776 statsCtrl : `lsst.afw.math.StatisticsControl` 

777 Statistics control object for coadd. 

778 nImage : `lsst.afw.image.ImageU`, optional 

779 Keeps track of exposure count for each pixel. 

780 """ 

781 self.log.debug("Computing coadd over %s", bbox) 

782 

783 coaddExposure.mask.addMaskPlane("REJECTED") 

784 coaddExposure.mask.addMaskPlane("CLIPPED") 

785 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

786 maskMap = self.setRejectedMaskMapping(statsCtrl) 

787 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

788 maskedImageList = [] 

789 if nImage is not None: 

790 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

791 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

792 

793 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

794 

795 maskedImage = exposure.getMaskedImage() 

796 mask = maskedImage.getMask() 

797 if altMask is not None: 

798 self.applyAltMaskPlanes(mask, altMask) 

799 imageScaler.scaleMaskedImage(maskedImage) 

800 

801 # Add 1 for each pixel which is not excluded by the exclude mask. 

802 # In legacyCoadd, pixels may also be excluded by 

803 # afwMath.statisticsStack. 

804 if nImage is not None: 

805 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

806 if self.config.removeMaskPlanes: 

807 self.removeMaskPlanes(maskedImage) 

808 maskedImageList.append(maskedImage) 

809 

810 if self.config.doInputMap: 

811 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

812 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

813 

814 with self.timer("stack"): 

815 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

816 clipped, # also set output to CLIPPED if sigma-clipped 

817 maskMap) 

818 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

819 if nImage is not None: 

820 nImage.assign(subNImage, bbox) 

821 

822 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

823 altMaskList, statsCtrl, nImage=None): 

824 """Assemble the coadd using the "online" method. 

825 

826 This method takes a running sum of images and weights to save memory. 

827 It only works for MEAN statistics. 

828 

829 Parameters 

830 ---------- 

831 coaddExposure : `lsst.afw.image.Exposure` 

832 The target exposure for the coadd. 

833 tempExpRefList : `list` 

834 List of data reference to tempExp. 

835 imageScalerList : `list` 

836 List of image scalers. 

837 weightList : `list` 

838 List of weights. 

839 altMaskList : `list` 

840 List of alternate masks to use rather than those stored with 

841 tempExp, or None. Each element is dict with keys = mask plane 

842 name to which to add the spans. 

843 statsCtrl : `lsst.afw.math.StatisticsControl` 

844 Statistics control object for coadd. 

845 nImage : `lsst.afw.image.ImageU`, optional 

846 Keeps track of exposure count for each pixel. 

847 """ 

848 self.log.debug("Computing online coadd.") 

849 

850 coaddExposure.mask.addMaskPlane("REJECTED") 

851 coaddExposure.mask.addMaskPlane("CLIPPED") 

852 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

853 maskMap = self.setRejectedMaskMapping(statsCtrl) 

854 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

855 

856 bbox = coaddExposure.maskedImage.getBBox() 

857 

858 stacker = AccumulatorMeanStack( 

859 coaddExposure.image.array.shape, 

860 statsCtrl.getAndMask(), 

861 mask_threshold_dict=thresholdDict, 

862 mask_map=maskMap, 

863 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

864 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

865 compute_n_image=(nImage is not None) 

866 ) 

867 

868 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

869 imageScalerList, 

870 altMaskList, 

871 weightList): 

872 exposure = tempExpRef.get() 

873 maskedImage = exposure.getMaskedImage() 

874 mask = maskedImage.getMask() 

875 if altMask is not None: 

876 self.applyAltMaskPlanes(mask, altMask) 

877 imageScaler.scaleMaskedImage(maskedImage) 

878 if self.config.removeMaskPlanes: 

879 self.removeMaskPlanes(maskedImage) 

880 

881 stacker.add_masked_image(maskedImage, weight=weight) 

882 

883 if self.config.doInputMap: 

884 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

885 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

886 

887 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

888 

889 if nImage is not None: 

890 nImage.array[:, :] = stacker.n_image 

891 

892 def removeMaskPlanes(self, maskedImage): 

893 """Unset the mask of an image for mask planes specified in the config. 

894 

895 Parameters 

896 ---------- 

897 maskedImage : `lsst.afw.image.MaskedImage` 

898 The masked image to be modified. 

899 

900 Raises 

901 ------ 

902 InvalidParameterError 

903 Raised if no mask plane with that name was found. 

904 """ 

905 mask = maskedImage.getMask() 

906 for maskPlane in self.config.removeMaskPlanes: 

907 try: 

908 mask &= ~mask.getPlaneBitMask(maskPlane) 

909 except pexExceptions.InvalidParameterError: 

910 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

911 maskPlane) 

912 

913 @staticmethod 

914 def setRejectedMaskMapping(statsCtrl): 

915 """Map certain mask planes of the warps to new planes for the coadd. 

916 

917 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

918 or CLIPPED, set it to REJECTED on the coadd. 

919 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

920 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

921 

922 Parameters 

923 ---------- 

924 statsCtrl : `lsst.afw.math.StatisticsControl` 

925 Statistics control object for coadd. 

926 

927 Returns 

928 ------- 

929 maskMap : `list` of `tuple` of `int` 

930 A list of mappings of mask planes of the warped exposures to 

931 mask planes of the coadd. 

932 """ 

933 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

934 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

935 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

936 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

937 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

938 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

939 (clipped, clipped)] 

940 return maskMap 

941 

942 def applyAltMaskPlanes(self, mask, altMaskSpans): 

943 """Apply in place alt mask formatted as SpanSets to a mask. 

944 

945 Parameters 

946 ---------- 

947 mask : `lsst.afw.image.Mask` 

948 Original mask. 

949 altMaskSpans : `dict` 

950 SpanSet lists to apply. Each element contains the new mask 

951 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

952 and list of SpanSets to apply to the mask. 

953 

954 Returns 

955 ------- 

956 mask : `lsst.afw.image.Mask` 

957 Updated mask. 

958 """ 

959 if self.config.doUsePsfMatchedPolygons: 

960 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

961 # Clear away any other masks outside the validPolygons. These 

962 # pixels are no longer contributing to inexact PSFs, and will 

963 # still be rejected because of NO_DATA. 

964 # self.config.doUsePsfMatchedPolygons should be True only in 

965 # CompareWarpAssemble. This mask-clearing step must only occur 

966 # *before* applying the new masks below. 

967 for spanSet in altMaskSpans['NO_DATA']: 

968 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

969 

970 for plane, spanSetList in altMaskSpans.items(): 

971 maskClipValue = mask.addMaskPlane(plane) 

972 for spanSet in spanSetList: 

973 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

974 return mask 

975 

976 def shrinkValidPolygons(self, coaddInputs): 

977 """Shrink coaddInputs' ccds' ValidPolygons in place. 

978 

979 Either modify each ccd's validPolygon in place, or if CoaddInputs 

980 does not have a validPolygon, create one from its bbox. 

981 

982 Parameters 

983 ---------- 

984 coaddInputs : `lsst.afw.image.coaddInputs` 

985 Original mask. 

986 """ 

987 for ccd in coaddInputs.ccds: 

988 polyOrig = ccd.getValidPolygon() 

989 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

990 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

991 if polyOrig: 

992 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

993 else: 

994 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

995 ccd.setValidPolygon(validPolygon) 

996 

997 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

998 """Set the bright object masks. 

999 

1000 Parameters 

1001 ---------- 

1002 exposure : `lsst.afw.image.Exposure` 

1003 Exposure under consideration. 

1004 brightObjectMasks : `lsst.afw.table` 

1005 Table of bright objects to mask. 

1006 dataId : `lsst.daf.butler.DataId`, optional 

1007 Data identifier dict for patch. 

1008 """ 

1009 if brightObjectMasks is None: 

1010 self.log.warning("Unable to apply bright object mask: none supplied") 

1011 return 

1012 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1013 mask = exposure.getMaskedImage().getMask() 

1014 wcs = exposure.getWcs() 

1015 plateScale = wcs.getPixelScale().asArcseconds() 

1016 

1017 for rec in brightObjectMasks: 

1018 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1019 if rec["type"] == "box": 

1020 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1021 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1022 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1023 

1024 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1025 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1026 

1027 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1028 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1029 spans = afwGeom.SpanSet(bbox) 

1030 elif rec["type"] == "circle": 

1031 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1032 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1033 else: 

1034 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1035 continue 

1036 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1037 

1038 def setInexactPsf(self, mask): 

1039 """Set INEXACT_PSF mask plane. 

1040 

1041 If any of the input images isn't represented in the coadd (due to 

1042 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1043 these pixels. 

1044 

1045 Parameters 

1046 ---------- 

1047 mask : `lsst.afw.image.Mask` 

1048 Coadded exposure's mask, modified in-place. 

1049 """ 

1050 mask.addMaskPlane("INEXACT_PSF") 

1051 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1052 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1053 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1054 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1055 array = mask.getArray() 

1056 selected = array & (sensorEdge | clipped | rejected) > 0 

1057 array[selected] |= inexactPsf 

1058 

1059 def filterWarps(self, inputs, goodVisits): 

1060 """Return list of only inputRefs with visitId in goodVisits ordered by 

1061 goodVisit. 

1062 

1063 Parameters 

1064 ---------- 

1065 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef` 

1066 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId 

1067 containing visit. 

1068 goodVisit : `dict` 

1069 Dictionary with good visitIds as the keys. Value ignored. 

1070 

1071 Returns 

1072 ------- 

1073 filteredInputs : `list` \ 

1074 [`~lsst.pipe.base.connections.DeferredDatasetRef`] 

1075 Filtered and sorted list of inputRefs with visitId in goodVisits 

1076 ordered by goodVisit. 

1077 """ 

1078 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1079 filteredInputs = [] 

1080 for visit in goodVisits.keys(): 

1081 if visit in inputWarpDict: 

1082 filteredInputs.append(inputWarpDict[visit]) 

1083 return filteredInputs 

1084 

1085 

1086def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1087 """Function to count the number of pixels with a specific mask in a 

1088 footprint. 

1089 

1090 Find the intersection of mask & footprint. Count all pixels in the mask 

1091 that are in the intersection that have bitmask set but do not have 

1092 ignoreMask set. Return the count. 

1093 

1094 Parameters 

1095 ---------- 

1096 mask : `lsst.afw.image.Mask` 

1097 Mask to define intersection region by. 

1098 footprint : `lsst.afw.detection.Footprint` 

1099 Footprint to define the intersection region by. 

1100 bitmask : `Unknown` 

1101 Specific mask that we wish to count the number of occurances of. 

1102 ignoreMask : `Unknown` 

1103 Pixels to not consider. 

1104 

1105 Returns 

1106 ------- 

1107 result : `int` 

1108 Number of pixels in footprint with specified mask. 

1109 """ 

1110 bbox = footprint.getBBox() 

1111 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1112 fp = afwImage.Mask(bbox) 

1113 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1114 footprint.spans.setMask(fp, bitmask) 

1115 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1116 (subMask.getArray() & ignoreMask) == 0).sum() 

1117 

1118 

1119class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1120 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1121 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1122 "Only PSF-Matched Warps make sense for image subtraction. " 

1123 "Therefore, they must be an additional declared input."), 

1124 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1125 storageClass="ExposureF", 

1126 dimensions=("tract", "patch", "skymap", "visit"), 

1127 deferLoad=True, 

1128 multiple=True 

1129 ) 

1130 templateCoadd = pipeBase.connectionTypes.Output( 

1131 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1132 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1133 name="{outputCoaddName}CoaddPsfMatched", 

1134 storageClass="ExposureF", 

1135 dimensions=("tract", "patch", "skymap", "band"), 

1136 ) 

1137 

1138 def __init__(self, *, config=None): 

1139 super().__init__(config=config) 

1140 if not config.assembleStaticSkyModel.doWrite: 

1141 self.outputs.remove("templateCoadd") 

1142 config.validate() 

1143 

1144 

1145class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1146 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1147 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1148 target=AssembleCoaddTask, 

1149 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as " 

1150 "a naive/first-iteration model of the static sky.", 

1151 ) 

1152 detect = pexConfig.ConfigurableField( 

1153 target=SourceDetectionTask, 

1154 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1155 ) 

1156 detectTemplate = pexConfig.ConfigurableField( 

1157 target=SourceDetectionTask, 

1158 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1159 ) 

1160 maskStreaks = pexConfig.ConfigurableField( 

1161 target=MaskStreaksTask, 

1162 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1163 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1164 "streakMaskName" 

1165 ) 

1166 streakMaskName = pexConfig.Field( 

1167 dtype=str, 

1168 default="STREAK", 

1169 doc="Name of mask bit used for streaks" 

1170 ) 

1171 maxNumEpochs = pexConfig.Field( 

1172 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1173 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1174 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1175 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1176 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1177 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1178 "than transient and not masked.", 

1179 dtype=int, 

1180 default=2 

1181 ) 

1182 maxFractionEpochsLow = pexConfig.RangeField( 

1183 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1184 "Effective maxNumEpochs = " 

1185 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1186 dtype=float, 

1187 default=0.4, 

1188 min=0., max=1., 

1189 ) 

1190 maxFractionEpochsHigh = pexConfig.RangeField( 

1191 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1192 "Effective maxNumEpochs = " 

1193 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1194 dtype=float, 

1195 default=0.03, 

1196 min=0., max=1., 

1197 ) 

1198 spatialThreshold = pexConfig.RangeField( 

1199 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1200 "temporal criteria. If 0, clip all. If 1, clip none.", 

1201 dtype=float, 

1202 default=0.5, 

1203 min=0., max=1., 

1204 inclusiveMin=True, inclusiveMax=True 

1205 ) 

1206 doScaleWarpVariance = pexConfig.Field( 

1207 doc="Rescale Warp variance plane using empirical noise?", 

1208 dtype=bool, 

1209 default=True, 

1210 ) 

1211 scaleWarpVariance = pexConfig.ConfigurableField( 

1212 target=ScaleVarianceTask, 

1213 doc="Rescale variance on warps", 

1214 ) 

1215 doPreserveContainedBySource = pexConfig.Field( 

1216 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1217 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1218 dtype=bool, 

1219 default=True, 

1220 ) 

1221 doPrefilterArtifacts = pexConfig.Field( 

1222 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1223 "because they will be excluded anyway. This prevents them from contributing " 

1224 "to the outlier epoch count image and potentially being labeled as persistant." 

1225 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1226 dtype=bool, 

1227 default=True 

1228 ) 

1229 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1230 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1231 dtype=str, 

1232 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1233 ) 

1234 prefilterArtifactsRatio = pexConfig.Field( 

1235 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1236 dtype=float, 

1237 default=0.05 

1238 ) 

1239 doFilterMorphological = pexConfig.Field( 

1240 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1241 "be streaks.", 

1242 dtype=bool, 

1243 default=False 

1244 ) 

1245 growStreakFp = pexConfig.Field( 

1246 doc="Grow streak footprints by this number multiplied by the PSF width", 

1247 dtype=float, 

1248 default=5 

1249 ) 

1250 

1251 def setDefaults(self): 

1252 AssembleCoaddConfig.setDefaults(self) 

1253 self.statistic = 'MEAN' 

1254 self.doUsePsfMatchedPolygons = True 

1255 

1256 # Real EDGE removed by psfMatched NO_DATA border half the width of the 

1257 # matching kernel. CompareWarp applies psfMatched EDGE pixels to 

1258 # directWarps before assembling. 

1259 if "EDGE" in self.badMaskPlanes: 

1260 self.badMaskPlanes.remove('EDGE') 

1261 self.removeMaskPlanes.append('EDGE') 

1262 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1263 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1264 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1265 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1266 self.assembleStaticSkyModel.sigmaClip = 2.5 

1267 self.assembleStaticSkyModel.clipIter = 3 

1268 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1269 self.assembleStaticSkyModel.doWrite = False 

1270 self.detect.doTempLocalBackground = False 

1271 self.detect.reEstimateBackground = False 

1272 self.detect.returnOriginalFootprints = False 

1273 self.detect.thresholdPolarity = "both" 

1274 self.detect.thresholdValue = 5 

1275 self.detect.minPixels = 4 

1276 self.detect.isotropicGrow = True 

1277 self.detect.thresholdType = "pixel_stdev" 

1278 self.detect.nSigmaToGrow = 0.4 

1279 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1280 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1281 self.detectTemplate.nSigmaToGrow = 2.4 

1282 self.detectTemplate.doTempLocalBackground = False 

1283 self.detectTemplate.reEstimateBackground = False 

1284 self.detectTemplate.returnOriginalFootprints = False 

1285 

1286 def validate(self): 

1287 super().validate() 

1288 if self.assembleStaticSkyModel.doNImage: 

1289 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1290 "Please set assembleStaticSkyModel.doNImage=False") 

1291 

1292 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1293 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1294 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1295 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1296 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1297 

1298 

1299class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1300 """Assemble a compareWarp coadded image from a set of warps 

1301 by masking artifacts detected by comparing PSF-matched warps. 

1302 

1303 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1304 we clip outliers). The problem with doing this is that when computing the 

1305 coadd PSF at a given location, individual visit PSFs from visits with 

1306 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1307 In this task, we correct for this behavior by creating a new badMaskPlane 

1308 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1309 an artifact. We populate this plane on the input warps by comparing 

1310 PSF-matched warps with a PSF-matched median coadd which serves as a 

1311 model of the static sky. Any group of pixels that deviates from the 

1312 PSF-matched template coadd by more than config.detect.threshold sigma, 

1313 is an artifact candidate. The candidates are then filtered to remove 

1314 variable sources and sources that are difficult to subtract such as 

1315 bright stars. This filter is configured using the config parameters 

1316 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1317 the maximum fraction of epochs that the deviation can appear in and still 

1318 be considered an artifact. The spatialThreshold is the maximum fraction of 

1319 pixels in the footprint of the deviation that appear in other epochs 

1320 (where other epochs is defined by the temporalThreshold). If the deviant 

1321 region meets this criteria of having a significant percentage of pixels 

1322 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1323 set in the mask. These regions will not contribute to the final coadd. 

1324 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1325 of clipped regions. Note that the algorithm implemented by this task is 

1326 preliminary and works correctly for HSC data. Parameter modifications and 

1327 or considerable redesigning of the algorithm is likley required for other 

1328 surveys. 

1329 

1330 ``CompareWarpAssembleCoaddTask`` sub-classes 

1331 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1332 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1333 

1334 Notes 

1335 ----- 

1336 Debugging: 

1337 This task supports the following debug variables: 

1338 - ``saveCountIm`` 

1339 If True then save the Epoch Count Image as a fits file in the `figPath` 

1340 - ``figPath`` 

1341 Path to save the debug fits images and figures 

1342 """ 

1343 

1344 ConfigClass = CompareWarpAssembleCoaddConfig 

1345 _DefaultName = "compareWarpAssembleCoadd" 

1346 

1347 def __init__(self, *args, **kwargs): 

1348 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1349 self.makeSubtask("assembleStaticSkyModel") 

1350 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

1351 self.makeSubtask("detect", schema=detectionSchema) 

1352 if self.config.doPreserveContainedBySource: 

1353 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

1354 if self.config.doScaleWarpVariance: 

1355 self.makeSubtask("scaleWarpVariance") 

1356 if self.config.doFilterMorphological: 

1357 self.makeSubtask("maskStreaks") 

1358 

1359 @utils.inheritDoc(AssembleCoaddTask) 

1360 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

1361 """Generate a templateCoadd to use as a naive model of static sky to 

1362 subtract from PSF-Matched warps. 

1363 

1364 Returns 

1365 ------- 

1366 result : `~lsst.pipe.base.Struct` 

1367 Results as a struct with attributes: 

1368 

1369 ``templateCoadd`` 

1370 Coadded exposure (`lsst.afw.image.Exposure`). 

1371 ``nImage`` 

1372 Keeps track of exposure count for each pixel 

1373 (`lsst.afw.image.ImageU`). 

1374 

1375 Raises 

1376 ------ 

1377 RuntimeError 

1378 Raised if ``templateCoadd`` is `None`. 

1379 """ 

1380 # Ensure that psfMatchedWarps are used as input warps for template 

1381 # generation. 

1382 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

1383 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

1384 

1385 # Because subtasks don't have connections we have to make one. 

1386 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

1387 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

1388 if self.config.assembleStaticSkyModel.doWrite: 

1389 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

1390 # Remove template coadd from both subtask's and main tasks outputs, 

1391 # because it is handled by the subtask as `coaddExposure` 

1392 del outputRefs.templateCoadd 

1393 del staticSkyModelOutputRefs.templateCoadd 

1394 

1395 # A PSF-Matched nImage does not exist as a dataset type 

1396 if 'nImage' in staticSkyModelOutputRefs.keys(): 

1397 del staticSkyModelOutputRefs.nImage 

1398 

1399 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

1400 staticSkyModelOutputRefs) 

1401 if templateCoadd is None: 

1402 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

1403 

1404 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

1405 nImage=templateCoadd.nImage, 

1406 warpRefList=templateCoadd.warpRefList, 

1407 imageScalerList=templateCoadd.imageScalerList, 

1408 weightList=templateCoadd.weightList) 

1409 

1410 def _noTemplateMessage(self, warpType): 

1411 warpName = (warpType[0].upper() + warpType[1:]) 

1412 message = """No %(warpName)s warps were found to build the template coadd which is 

1413 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

1414 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

1415 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

1416 

1417 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

1418 another algorithm like: 

1419 

1420 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

1421 config.assemble.retarget(SafeClipAssembleCoaddTask) 

1422 """ % {"warpName": warpName} 

1423 return message 

1424 

1425 @utils.inheritDoc(AssembleCoaddTask) 

1426 @timeMethod 

1427 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1428 supplementaryData): 

1429 """Notes 

1430 ----- 

1431 Assemble the coadd. 

1432 

1433 Find artifacts and apply them to the warps' masks creating a list of 

1434 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

1435 plane. Then pass these alternative masks to the base class's ``run`` 

1436 method. 

1437 """ 

1438 # Check and match the order of the supplementaryData 

1439 # (PSF-matched) inputs to the order of the direct inputs, 

1440 # so that the artifact mask is applied to the right warp 

1441 dataIds = [ref.dataId for ref in tempExpRefList] 

1442 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

1443 

1444 if dataIds != psfMatchedDataIds: 

1445 self.log.info("Reordering and or/padding PSF-matched visit input list") 

1446 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

1447 psfMatchedDataIds, dataIds) 

1448 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

1449 psfMatchedDataIds, dataIds) 

1450 

1451 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find 

1452 # artifacts. 

1453 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

1454 supplementaryData.warpRefList, 

1455 supplementaryData.imageScalerList) 

1456 

1457 badMaskPlanes = self.config.badMaskPlanes[:] 

1458 badMaskPlanes.append("CLIPPED") 

1459 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1460 

1461 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1462 spanSetMaskList, mask=badPixelMask) 

1463 

1464 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and 

1465 # INEXACT_PSF. Psf-Matching moves the real edge inwards. 

1466 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

1467 return result 

1468 

1469 def applyAltEdgeMask(self, mask, altMaskList): 

1470 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

1471 

1472 Parameters 

1473 ---------- 

1474 mask : `lsst.afw.image.Mask` 

1475 Original mask. 

1476 altMaskList : `list` of `dict` 

1477 List of Dicts containing ``spanSet`` lists. 

1478 Each element contains the new mask plane name (e.g. "CLIPPED 

1479 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

1480 the mask. 

1481 """ 

1482 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

1483 for visitMask in altMaskList: 

1484 if "EDGE" in visitMask: 

1485 for spanSet in visitMask['EDGE']: 

1486 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

1487 

1488 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

1489 """Find artifacts. 

1490 

1491 Loop through warps twice. The first loop builds a map with the count 

1492 of how many epochs each pixel deviates from the templateCoadd by more 

1493 than ``config.chiThreshold`` sigma. The second loop takes each 

1494 difference image and filters the artifacts detected in each using 

1495 count map to filter out variable sources and sources that are 

1496 difficult to subtract cleanly. 

1497 

1498 Parameters 

1499 ---------- 

1500 templateCoadd : `lsst.afw.image.Exposure` 

1501 Exposure to serve as model of static sky. 

1502 tempExpRefList : `list` 

1503 List of data references to warps. 

1504 imageScalerList : `list` 

1505 List of image scalers. 

1506 

1507 Returns 

1508 ------- 

1509 altMasks : `list` of `dict` 

1510 List of dicts containing information about CLIPPED 

1511 (i.e., artifacts), NO_DATA, and EDGE pixels. 

1512 """ 

1513 self.log.debug("Generating Count Image, and mask lists.") 

1514 coaddBBox = templateCoadd.getBBox() 

1515 slateIm = afwImage.ImageU(coaddBBox) 

1516 epochCountImage = afwImage.ImageU(coaddBBox) 

1517 nImage = afwImage.ImageU(coaddBBox) 

1518 spanSetArtifactList = [] 

1519 spanSetNoDataMaskList = [] 

1520 spanSetEdgeList = [] 

1521 spanSetBadMorphoList = [] 

1522 badPixelMask = self.getBadPixelMask() 

1523 

1524 # mask of the warp diffs should = that of only the warp 

1525 templateCoadd.mask.clearAllMaskPlanes() 

1526 

1527 if self.config.doPreserveContainedBySource: 

1528 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

1529 else: 

1530 templateFootprints = None 

1531 

1532 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

1533 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

1534 if warpDiffExp is not None: 

1535 # This nImage only approximates the final nImage because it 

1536 # uses the PSF-matched mask. 

1537 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

1538 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

1539 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

1540 fpSet.positive.merge(fpSet.negative) 

1541 footprints = fpSet.positive 

1542 slateIm.set(0) 

1543 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

1544 

1545 # Remove artifacts due to defects before they contribute to 

1546 # the epochCountImage. 

1547 if self.config.doPrefilterArtifacts: 

1548 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

1549 

1550 # Clear mask before adding prefiltered spanSets 

1551 self.detect.clearMask(warpDiffExp.mask) 

1552 for spans in spanSetList: 

1553 spans.setImage(slateIm, 1, doClip=True) 

1554 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

1555 epochCountImage += slateIm 

1556 

1557 if self.config.doFilterMorphological: 

1558 maskName = self.config.streakMaskName 

1559 _ = self.maskStreaks.run(warpDiffExp) 

1560 streakMask = warpDiffExp.mask 

1561 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

1562 streakMask.getPlaneBitMask(maskName)).split() 

1563 # Pad the streaks to account for low-surface brightness 

1564 # wings. 

1565 psf = warpDiffExp.getPsf() 

1566 for s, sset in enumerate(spanSetStreak): 

1567 psfShape = psf.computeShape(sset.computeCentroid()) 

1568 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius() 

1569 sset_dilated = sset.dilated(int(dilation)) 

1570 spanSetStreak[s] = sset_dilated 

1571 

1572 # PSF-Matched warps have less available area (~the matching 

1573 # kernel) because the calexps undergo a second convolution. 

1574 # Pixels with data in the direct warp but not in the 

1575 # PSF-matched warp will not have their artifacts detected. 

1576 # NaNs from the PSF-matched warp therefore must be masked in 

1577 # the direct warp. 

1578 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

1579 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

1580 nansMask.setXY0(warpDiffExp.getXY0()) 

1581 edgeMask = warpDiffExp.mask 

1582 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

1583 edgeMask.getPlaneBitMask("EDGE")).split() 

1584 else: 

1585 # If the directWarp has <1% coverage, the psfMatchedWarp can 

1586 # have 0% and not exist. In this case, mask the whole epoch. 

1587 nansMask = afwImage.MaskX(coaddBBox, 1) 

1588 spanSetList = [] 

1589 spanSetEdgeMask = [] 

1590 spanSetStreak = [] 

1591 

1592 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

1593 

1594 spanSetNoDataMaskList.append(spanSetNoDataMask) 

1595 spanSetArtifactList.append(spanSetList) 

1596 spanSetEdgeList.append(spanSetEdgeMask) 

1597 if self.config.doFilterMorphological: 

1598 spanSetBadMorphoList.append(spanSetStreak) 

1599 

1600 if lsstDebug.Info(__name__).saveCountIm: 

1601 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

1602 epochCountImage.writeFits(path) 

1603 

1604 for i, spanSetList in enumerate(spanSetArtifactList): 

1605 if spanSetList: 

1606 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

1607 templateFootprints) 

1608 spanSetArtifactList[i] = filteredSpanSetList 

1609 if self.config.doFilterMorphological: 

1610 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

1611 

1612 altMasks = [] 

1613 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

1614 altMasks.append({'CLIPPED': artifacts, 

1615 'NO_DATA': noData, 

1616 'EDGE': edge}) 

1617 return altMasks 

1618 

1619 def prefilterArtifacts(self, spanSetList, exp): 

1620 """Remove artifact candidates covered by bad mask plane. 

1621 

1622 Any future editing of the candidate list that does not depend on 

1623 temporal information should go in this method. 

1624 

1625 Parameters 

1626 ---------- 

1627 spanSetList : `list` [`lsst.afw.geom.SpanSet`] 

1628 List of SpanSets representing artifact candidates. 

1629 exp : `lsst.afw.image.Exposure` 

1630 Exposure containing mask planes used to prefilter. 

1631 

1632 Returns 

1633 ------- 

1634 returnSpanSetList : `list` [`lsst.afw.geom.SpanSet`] 

1635 List of SpanSets with artifacts. 

1636 """ 

1637 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

1638 goodArr = (exp.mask.array & badPixelMask) == 0 

1639 returnSpanSetList = [] 

1640 bbox = exp.getBBox() 

1641 x0, y0 = exp.getXY0() 

1642 for i, span in enumerate(spanSetList): 

1643 y, x = span.clippedTo(bbox).indices() 

1644 yIndexLocal = numpy.array(y) - y0 

1645 xIndexLocal = numpy.array(x) - x0 

1646 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

1647 if goodRatio > self.config.prefilterArtifactsRatio: 

1648 returnSpanSetList.append(span) 

1649 return returnSpanSetList 

1650 

1651 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

1652 """Filter artifact candidates. 

1653 

1654 Parameters 

1655 ---------- 

1656 spanSetList : `list` [`lsst.afw.geom.SpanSet`] 

1657 List of SpanSets representing artifact candidates. 

1658 epochCountImage : `lsst.afw.image.Image` 

1659 Image of accumulated number of warpDiff detections. 

1660 nImage : `lsst.afw.image.ImageU` 

1661 Image of the accumulated number of total epochs contributing. 

1662 

1663 Returns 

1664 ------- 

1665 maskSpanSetList : `list` [`lsst.afw.geom.SpanSet`] 

1666 List of SpanSets with artifacts. 

1667 """ 

1668 maskSpanSetList = [] 

1669 x0, y0 = epochCountImage.getXY0() 

1670 for i, span in enumerate(spanSetList): 

1671 y, x = span.indices() 

1672 yIdxLocal = [y1 - y0 for y1 in y] 

1673 xIdxLocal = [x1 - x0 for x1 in x] 

1674 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

1675 totalN = nImage.array[yIdxLocal, xIdxLocal] 

1676 

1677 # effectiveMaxNumEpochs is broken line (fraction of N) with 

1678 # characteristic config.maxNumEpochs. 

1679 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

1680 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

1681 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

1682 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

1683 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

1684 & (outlierN <= effectiveMaxNumEpochs)) 

1685 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

1686 if percentBelowThreshold > self.config.spatialThreshold: 

1687 maskSpanSetList.append(span) 

1688 

1689 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

1690 # If a candidate is contained by a footprint on the template coadd, 

1691 # do not clip. 

1692 filteredMaskSpanSetList = [] 

1693 for span in maskSpanSetList: 

1694 doKeep = True 

1695 for footprint in footprintsToExclude.positive.getFootprints(): 

1696 if footprint.spans.contains(span): 

1697 doKeep = False 

1698 break 

1699 if doKeep: 

1700 filteredMaskSpanSetList.append(span) 

1701 maskSpanSetList = filteredMaskSpanSetList 

1702 

1703 return maskSpanSetList 

1704 

1705 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

1706 """Fetch a warp from the butler and return a warpDiff. 

1707 

1708 Parameters 

1709 ---------- 

1710 warpRef : `lsst.daf.butler.DeferredDatasetHandle` 

1711 Handle for the warp. 

1712 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

1713 An image scaler object. 

1714 templateCoadd : `lsst.afw.image.Exposure` 

1715 Exposure to be substracted from the scaled warp. 

1716 

1717 Returns 

1718 ------- 

1719 warp : `lsst.afw.image.Exposure` 

1720 Exposure of the image difference between the warp and template. 

1721 """ 

1722 # If the PSF-Matched warp did not exist for this direct warp 

1723 # None is holding its place to maintain order in Gen 3 

1724 if warpRef is None: 

1725 return None 

1726 

1727 warp = warpRef.get() 

1728 # direct image scaler OK for PSF-matched Warp 

1729 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

1730 mi = warp.getMaskedImage() 

1731 if self.config.doScaleWarpVariance: 

1732 try: 

1733 self.scaleWarpVariance.run(mi) 

1734 except Exception as exc: 

1735 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

1736 mi -= templateCoadd.getMaskedImage() 

1737 return warp