Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 16%

643 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-12 01:27 -0700

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import copy 

23import numpy 

24import warnings 

25import logging 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.coadd.utils as coaddUtils 

34import lsst.pipe.base as pipeBase 

35import lsst.meas.algorithms as measAlg 

36import lsstDebug 

37import lsst.utils as utils 

38from lsst.skymap import BaseSkyMap 

39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

40from .interpImage import InterpImageTask 

41from .scaleZeroPoint import ScaleZeroPointTask 

42from .maskStreaks import MaskStreaksTask 

43from .healSparseMapping import HealSparseInputMapTask 

44from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask 

45from lsst.utils.timer import timeMethod 

46from deprecated.sphinx import deprecated 

47 

48__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

49 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

50 

51log = logging.getLogger(__name__) 

52 

53 

54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

55 dimensions=("tract", "patch", "band", "skymap"), 

56 defaultTemplates={"inputCoaddName": "deep", 

57 "outputCoaddName": "deep", 

58 "warpType": "direct", 

59 "warpTypeSuffix": ""}): 

60 

61 inputWarps = pipeBase.connectionTypes.Input( 

62 doc=("Input list of warps to be assemebled i.e. stacked." 

63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

64 name="{inputCoaddName}Coadd_{warpType}Warp", 

65 storageClass="ExposureF", 

66 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

67 deferLoad=True, 

68 multiple=True 

69 ) 

70 skyMap = pipeBase.connectionTypes.Input( 

71 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

73 storageClass="SkyMap", 

74 dimensions=("skymap", ), 

75 ) 

76 selectedVisits = pipeBase.connectionTypes.Input( 

77 doc="Selected visits to be coadded.", 

78 name="{outputCoaddName}Visits", 

79 storageClass="StructuredDataDict", 

80 dimensions=("instrument", "tract", "patch", "skymap", "band") 

81 ) 

82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

83 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

84 " BRIGHT_OBJECT."), 

85 name="brightObjectMask", 

86 storageClass="ObjectMaskCatalog", 

87 dimensions=("tract", "patch", "skymap", "band"), 

88 ) 

89 coaddExposure = pipeBase.connectionTypes.Output( 

90 doc="Output coadded exposure, produced by stacking input warps", 

91 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

92 storageClass="ExposureF", 

93 dimensions=("tract", "patch", "skymap", "band"), 

94 ) 

95 nImage = pipeBase.connectionTypes.Output( 

96 doc="Output image of number of input images per pixel", 

97 name="{outputCoaddName}Coadd_nImage", 

98 storageClass="ImageU", 

99 dimensions=("tract", "patch", "skymap", "band"), 

100 ) 

101 inputMap = pipeBase.connectionTypes.Output( 

102 doc="Output healsparse map of input images", 

103 name="{outputCoaddName}Coadd_inputMap", 

104 storageClass="HealSparseMap", 

105 dimensions=("tract", "patch", "skymap", "band"), 

106 ) 

107 

108 def __init__(self, *, config=None): 

109 super().__init__(config=config) 

110 

111 if not config.doMaskBrightObjects: 

112 self.prerequisiteInputs.remove("brightObjectMask") 

113 

114 if not config.doSelectVisits: 

115 self.inputs.remove("selectedVisits") 

116 

117 if not config.doNImage: 

118 self.outputs.remove("nImage") 

119 

120 if not self.config.doInputMap: 

121 self.outputs.remove("inputMap") 

122 

123 

124class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

125 pipelineConnections=AssembleCoaddConnections): 

126 """Configuration parameters for the `AssembleCoaddTask`. 

127 

128 Notes 

129 ----- 

130 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

131 only set the bitplane config.brightObjectMaskName. To make this useful you 

132 *must* also configure the flags.pixel algorithm, for example by adding 

133 

134 .. code-block:: none 

135 

136 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

137 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

138 

139 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

140 """ 

141 warpType = pexConfig.Field( 

142 doc="Warp name: one of 'direct' or 'psfMatched'", 

143 dtype=str, 

144 default="direct", 

145 ) 

146 subregionSize = pexConfig.ListField( 

147 dtype=int, 

148 doc="Width, height of stack subregion size; " 

149 "make small enough that a full stack of images will fit into memory at once.", 

150 length=2, 

151 default=(2000, 2000), 

152 ) 

153 statistic = pexConfig.Field( 

154 dtype=str, 

155 doc="Main stacking statistic for aggregating over the epochs.", 

156 default="MEANCLIP", 

157 ) 

158 doOnlineForMean = pexConfig.Field( 

159 dtype=bool, 

160 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

161 default=False, 

162 ) 

163 doSigmaClip = pexConfig.Field( 

164 dtype=bool, 

165 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

166 default=False, 

167 ) 

168 sigmaClip = pexConfig.Field( 

169 dtype=float, 

170 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

171 default=3.0, 

172 ) 

173 clipIter = pexConfig.Field( 

174 dtype=int, 

175 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

176 default=2, 

177 ) 

178 calcErrorFromInputVariance = pexConfig.Field( 

179 dtype=bool, 

180 doc="Calculate coadd variance from input variance by stacking statistic." 

181 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

182 default=True, 

183 ) 

184 scaleZeroPoint = pexConfig.ConfigurableField( 

185 target=ScaleZeroPointTask, 

186 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

187 ) 

188 doInterp = pexConfig.Field( 

189 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

190 dtype=bool, 

191 default=True, 

192 ) 

193 interpImage = pexConfig.ConfigurableField( 

194 target=InterpImageTask, 

195 doc="Task to interpolate (and extrapolate) over NaN pixels", 

196 ) 

197 doWrite = pexConfig.Field( 

198 doc="Persist coadd?", 

199 dtype=bool, 

200 default=True, 

201 ) 

202 doNImage = pexConfig.Field( 

203 doc="Create image of number of contributing exposures for each pixel", 

204 dtype=bool, 

205 default=False, 

206 ) 

207 doUsePsfMatchedPolygons = pexConfig.Field( 

208 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

209 dtype=bool, 

210 default=False, 

211 ) 

212 maskPropagationThresholds = pexConfig.DictField( 

213 keytype=str, 

214 itemtype=float, 

215 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

216 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

217 "would have contributed exceeds this value."), 

218 default={"SAT": 0.1}, 

219 ) 

220 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

221 doc="Mask planes to remove before coadding") 

222 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

223 doc="Set mask and flag bits for bright objects?") 

224 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

225 doc="Name of mask bit used for bright objects") 

226 coaddPsf = pexConfig.ConfigField( 

227 doc="Configuration for CoaddPsf", 

228 dtype=measAlg.CoaddPsfConfig, 

229 ) 

230 doAttachTransmissionCurve = pexConfig.Field( 

231 dtype=bool, default=False, optional=False, 

232 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

233 "(requires all input Exposures to have TransmissionCurves).") 

234 ) 

235 hasFakes = pexConfig.Field( 

236 dtype=bool, 

237 default=False, 

238 doc="Should be set to True if fake sources have been inserted into the input data." 

239 ) 

240 doSelectVisits = pexConfig.Field( 

241 doc="Coadd only visits selected by a SelectVisitsTask", 

242 dtype=bool, 

243 default=False, 

244 ) 

245 doInputMap = pexConfig.Field( 

246 doc="Create a bitwise map of coadd inputs", 

247 dtype=bool, 

248 default=False, 

249 ) 

250 inputMapper = pexConfig.ConfigurableField( 

251 doc="Input map creation subtask.", 

252 target=HealSparseInputMapTask, 

253 ) 

254 

255 def setDefaults(self): 

256 super().setDefaults() 

257 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

258 

259 def validate(self): 

260 super().validate() 

261 if self.doPsfMatch: 

262 # Backwards compatibility. 

263 # Configs do not have loggers 

264 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

265 self.warpType = 'psfMatched' 

266 if self.doSigmaClip and self.statistic != "MEANCLIP": 

267 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

268 self.statistic = "MEANCLIP" 

269 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

270 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

271 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

272 

273 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

274 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

275 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

276 if str(k) not in unstackableStats] 

277 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

278 % (self.statistic, stackableStats)) 

279 

280 

281class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

282 """Assemble a coadded image from a set of warps. 

283 

284 Each Warp that goes into a coadd will typically have an independent 

285 photometric zero-point. Therefore, we must scale each Warp to set it to 

286 a common photometric zeropoint. WarpType may be one of 'direct' or 

287 'psfMatched', and the boolean configs `config.makeDirect` and 

288 `config.makePsfMatched` set which of the warp types will be coadded. 

289 The coadd is computed as a mean with optional outlier rejection. 

290 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

291 Finally, Warps can have bad 'NaN' pixels which received no input from the 

292 source calExps. We interpolate over these bad (NaN) pixels. 

293 """ 

294 ConfigClass = AssembleCoaddConfig 

295 _DefaultName = "assembleCoadd" 

296 

297 def __init__(self, *args, **kwargs): 

298 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

299 if args: 

300 argNames = ["config", "name", "parentTask", "log"] 

301 kwargs.update({k: v for k, v in zip(argNames, args)}) 

302 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

303 "PipelineTask will not take positional args" % argNames, FutureWarning) 

304 

305 super().__init__(**kwargs) 

306 self.makeSubtask("interpImage") 

307 self.makeSubtask("scaleZeroPoint") 

308 

309 if self.config.doMaskBrightObjects: 

310 mask = afwImage.Mask() 

311 try: 

312 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

313 except pexExceptions.LsstCppException: 

314 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

315 mask.getMaskPlaneDict().keys()) 

316 del mask 

317 

318 if self.config.doInputMap: 

319 self.makeSubtask("inputMapper") 

320 

321 self.warpType = self.config.warpType 

322 

323 @utils.inheritDoc(pipeBase.PipelineTask) 

324 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

325 inputData = butlerQC.get(inputRefs) 

326 

327 # Construct skyInfo expected by run 

328 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it 

329 skyMap = inputData["skyMap"] 

330 outputDataId = butlerQC.quantum.dataId 

331 

332 inputData['skyInfo'] = makeSkyInfo(skyMap, 

333 tractId=outputDataId['tract'], 

334 patchId=outputDataId['patch']) 

335 

336 if self.config.doSelectVisits: 

337 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

338 else: 

339 warpRefList = inputData['inputWarps'] 

340 

341 inputs = self.prepareInputs(warpRefList) 

342 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

343 self.getTempExpDatasetName(self.warpType)) 

344 if len(inputs.tempExpRefList) == 0: 

345 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

346 

347 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

348 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

349 inputs.weightList, supplementaryData=supplementaryData) 

350 

351 inputData.setdefault('brightObjectMask', None) 

352 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

353 

354 if self.config.doWrite: 

355 butlerQC.put(retStruct, outputRefs) 

356 return retStruct 

357 

358 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

359 """Interpolate over missing data and mask bright stars. 

360 

361 Parameters 

362 ---------- 

363 coaddExposure : `lsst.afw.image.Exposure` 

364 The coadded exposure to process. 

365 brightObjectMasks : `lsst.afw.table`, optional 

366 Table of bright objects to mask. 

367 dataId : `lsst.daf.butler.DataId`, optional 

368 Data identification. 

369 """ 

370 if self.config.doInterp: 

371 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

372 # The variance must be positive; work around for DM-3201. 

373 varArray = coaddExposure.variance.array 

374 with numpy.errstate(invalid="ignore"): 

375 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

376 

377 if self.config.doMaskBrightObjects: 

378 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

379 

380 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

381 """Make additional inputs to run() specific to subclasses (Gen3) 

382 

383 Duplicates interface of `runQuantum` method. 

384 Available to be implemented by subclasses only if they need the 

385 coadd dataRef for performing preliminary processing before 

386 assembling the coadd. 

387 

388 Parameters 

389 ---------- 

390 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

391 Gen3 Butler object for fetching additional data products before 

392 running the Task specialized for quantum being processed 

393 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

394 Attributes are the names of the connections describing input dataset types. 

395 Values are DatasetRefs that task consumes for corresponding dataset type. 

396 DataIds are guaranteed to match data objects in ``inputData``. 

397 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

398 Attributes are the names of the connections describing output dataset types. 

399 Values are DatasetRefs that task is to produce 

400 for corresponding dataset type. 

401 """ 

402 return pipeBase.Struct() 

403 

404 @deprecated( 

405 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData", 

406 version="v25.0", 

407 category=FutureWarning 

408 ) 

409 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

410 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

411 

412 def prepareInputs(self, refList): 

413 """Prepare the input warps for coaddition by measuring the weight for 

414 each warp and the scaling for the photometric zero point. 

415 

416 Each Warp has its own photometric zeropoint and background variance. 

417 Before coadding these Warps together, compute a scale factor to 

418 normalize the photometric zeropoint and compute the weight for each Warp. 

419 

420 Parameters 

421 ---------- 

422 refList : `list` 

423 List of data references to tempExp 

424 

425 Returns 

426 ------- 

427 result : `lsst.pipe.base.Struct` 

428 Result struct with components: 

429 

430 - ``tempExprefList``: `list` of data references to tempExp. 

431 - ``weightList``: `list` of weightings. 

432 - ``imageScalerList``: `list` of image scalers. 

433 """ 

434 statsCtrl = afwMath.StatisticsControl() 

435 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

436 statsCtrl.setNumIter(self.config.clipIter) 

437 statsCtrl.setAndMask(self.getBadPixelMask()) 

438 statsCtrl.setNanSafe(True) 

439 # compute tempExpRefList: a list of tempExpRef that actually exist 

440 # and weightList: a list of the weight of the associated coadd tempExp 

441 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

442 tempExpRefList = [] 

443 weightList = [] 

444 imageScalerList = [] 

445 tempExpName = self.getTempExpDatasetName(self.warpType) 

446 for tempExpRef in refList: 

447 tempExp = tempExpRef.get() 

448 # Ignore any input warp that is empty of data 

449 if numpy.isnan(tempExp.image.array).all(): 

450 continue 

451 maskedImage = tempExp.getMaskedImage() 

452 imageScaler = self.scaleZeroPoint.computeImageScaler( 

453 exposure=tempExp, 

454 dataRef=tempExpRef, # FIXME 

455 ) 

456 try: 

457 imageScaler.scaleMaskedImage(maskedImage) 

458 except Exception as e: 

459 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

460 continue 

461 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

462 afwMath.MEANCLIP, statsCtrl) 

463 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

464 weight = 1.0 / float(meanVar) 

465 if not numpy.isfinite(weight): 

466 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

467 continue 

468 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

469 

470 del maskedImage 

471 del tempExp 

472 

473 tempExpRefList.append(tempExpRef) 

474 weightList.append(weight) 

475 imageScalerList.append(imageScaler) 

476 

477 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

478 imageScalerList=imageScalerList) 

479 

480 def prepareStats(self, mask=None): 

481 """Prepare the statistics for coadding images. 

482 

483 Parameters 

484 ---------- 

485 mask : `int`, optional 

486 Bit mask value to exclude from coaddition. 

487 

488 Returns 

489 ------- 

490 stats : `lsst.pipe.base.Struct` 

491 Statistics structure with the following fields: 

492 

493 - ``statsCtrl``: Statistics control object for coadd 

494 (`lsst.afw.math.StatisticsControl`) 

495 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

496 """ 

497 if mask is None: 

498 mask = self.getBadPixelMask() 

499 statsCtrl = afwMath.StatisticsControl() 

500 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

501 statsCtrl.setNumIter(self.config.clipIter) 

502 statsCtrl.setAndMask(mask) 

503 statsCtrl.setNanSafe(True) 

504 statsCtrl.setWeighted(True) 

505 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

506 for plane, threshold in self.config.maskPropagationThresholds.items(): 

507 bit = afwImage.Mask.getMaskPlane(plane) 

508 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

509 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

510 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

511 

512 @timeMethod 

513 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

514 altMaskList=None, mask=None, supplementaryData=None): 

515 """Assemble a coadd from input warps 

516 

517 Assemble the coadd using the provided list of coaddTempExps. Since 

518 the full coadd covers a patch (a large area), the assembly is 

519 performed over small areas on the image at a time in order to 

520 conserve memory usage. Iterate over subregions within the outer 

521 bbox of the patch using `assembleSubregion` to stack the corresponding 

522 subregions from the coaddTempExps with the statistic specified. 

523 Set the edge bits the coadd mask based on the weight map. 

524 

525 Parameters 

526 ---------- 

527 skyInfo : `lsst.pipe.base.Struct` 

528 Struct with geometric information about the patch. 

529 tempExpRefList : `list` 

530 List of data references to Warps (previously called CoaddTempExps). 

531 imageScalerList : `list` 

532 List of image scalers. 

533 weightList : `list` 

534 List of weights 

535 altMaskList : `list`, optional 

536 List of alternate masks to use rather than those stored with 

537 tempExp. 

538 mask : `int`, optional 

539 Bit mask value to exclude from coaddition. 

540 supplementaryData : lsst.pipe.base.Struct, optional 

541 Struct with additional data products needed to assemble coadd. 

542 Only used by subclasses that implement `_makeSupplementaryData` 

543 and override `run`. 

544 

545 Returns 

546 ------- 

547 result : `lsst.pipe.base.Struct` 

548 Result struct with components: 

549 

550 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

551 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

552 - ``inputMap``: bit-wise map of inputs, if requested. 

553 - ``warpRefList``: input list of refs to the warps ( 

554 ``lsst.daf.butler.DeferredDatasetHandle``) 

555 (unmodified) 

556 - ``imageScalerList``: input list of image scalers (unmodified) 

557 - ``weightList``: input list of weights (unmodified) 

558 """ 

559 tempExpName = self.getTempExpDatasetName(self.warpType) 

560 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

561 stats = self.prepareStats(mask=mask) 

562 

563 if altMaskList is None: 

564 altMaskList = [None]*len(tempExpRefList) 

565 

566 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

567 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

568 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

569 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

570 coaddMaskedImage = coaddExposure.getMaskedImage() 

571 subregionSizeArr = self.config.subregionSize 

572 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

573 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

574 if self.config.doNImage: 

575 nImage = afwImage.ImageU(skyInfo.bbox) 

576 else: 

577 nImage = None 

578 # If inputMap is requested, create the initial version that can be masked in 

579 # assembleSubregion. 

580 if self.config.doInputMap: 

581 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

582 skyInfo.wcs, 

583 coaddExposure.getInfo().getCoaddInputs().ccds) 

584 

585 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 

586 try: 

587 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

588 weightList, altMaskList, stats.ctrl, 

589 nImage=nImage) 

590 except Exception as e: 

591 self.log.exception("Cannot compute online coadd %s", e) 

592 raise 

593 else: 

594 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

595 try: 

596 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

597 weightList, altMaskList, stats.flags, stats.ctrl, 

598 nImage=nImage) 

599 except Exception as e: 

600 self.log.exception("Cannot compute coadd %s: %s", subBBox, e) 

601 raise 

602 

603 # If inputMap is requested, we must finalize the map after the accumulation. 

604 if self.config.doInputMap: 

605 self.inputMapper.finalize_ccd_input_map_mask() 

606 inputMap = self.inputMapper.ccd_input_map 

607 else: 

608 inputMap = None 

609 

610 self.setInexactPsf(coaddMaskedImage.getMask()) 

611 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

612 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

613 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

614 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

615 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

616 weightList=weightList, inputMap=inputMap) 

617 

618 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

619 """Set the metadata for the coadd. 

620 

621 This basic implementation sets the filter from the first input. 

622 

623 Parameters 

624 ---------- 

625 coaddExposure : `lsst.afw.image.Exposure` 

626 The target exposure for the coadd. 

627 tempExpRefList : `list` 

628 List of data references to tempExp. 

629 weightList : `list` 

630 List of weights. 

631 """ 

632 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

633 

634 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

635 # (and we need more than just the PropertySet that contains the header), which is not possible 

636 # with the current butler (see #2777). 

637 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

638 

639 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

640 

641 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

642 

643 # Set the coadd FilterLabel to the band of the first input exposure: 

644 # Coadds are calibrated, so the physical label is now meaningless. 

645 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel)) 

646 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

647 coaddInputs.ccds.reserve(numCcds) 

648 coaddInputs.visits.reserve(len(tempExpList)) 

649 

650 for tempExp, weight in zip(tempExpList, weightList): 

651 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

652 

653 if self.config.doUsePsfMatchedPolygons: 

654 self.shrinkValidPolygons(coaddInputs) 

655 

656 coaddInputs.visits.sort() 

657 coaddInputs.ccds.sort() 

658 if self.warpType == "psfMatched": 

659 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

660 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

661 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

662 # having the maximum width (sufficient because square) 

663 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

664 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

665 for modelPsf in modelPsfList] 

666 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

667 else: 

668 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

669 self.config.coaddPsf.makeControl()) 

670 coaddExposure.setPsf(psf) 

671 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

672 coaddExposure.getWcs()) 

673 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

674 if self.config.doAttachTransmissionCurve: 

675 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

676 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

677 

678 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

679 altMaskList, statsFlags, statsCtrl, nImage=None): 

680 """Assemble the coadd for a sub-region. 

681 

682 For each coaddTempExp, check for (and swap in) an alternative mask 

683 if one is passed. Remove mask planes listed in 

684 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

685 `lsst.afw.math.statisticsStack` with the statistic specified by 

686 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

687 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

688 an N-sigma clipped mean where N and iterations are specified by 

689 statsCtrl. Assign the stacked subregion back to the coadd. 

690 

691 Parameters 

692 ---------- 

693 coaddExposure : `lsst.afw.image.Exposure` 

694 The target exposure for the coadd. 

695 bbox : `lsst.geom.Box` 

696 Sub-region to coadd. 

697 tempExpRefList : `list` 

698 List of data reference to tempExp. 

699 imageScalerList : `list` 

700 List of image scalers. 

701 weightList : `list` 

702 List of weights. 

703 altMaskList : `list` 

704 List of alternate masks to use rather than those stored with 

705 tempExp, or None. Each element is dict with keys = mask plane 

706 name to which to add the spans. 

707 statsFlags : `lsst.afw.math.Property` 

708 Property object for statistic for coadd. 

709 statsCtrl : `lsst.afw.math.StatisticsControl` 

710 Statistics control object for coadd. 

711 nImage : `lsst.afw.image.ImageU`, optional 

712 Keeps track of exposure count for each pixel. 

713 """ 

714 self.log.debug("Computing coadd over %s", bbox) 

715 

716 coaddExposure.mask.addMaskPlane("REJECTED") 

717 coaddExposure.mask.addMaskPlane("CLIPPED") 

718 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

719 maskMap = self.setRejectedMaskMapping(statsCtrl) 

720 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

721 maskedImageList = [] 

722 if nImage is not None: 

723 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

724 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

725 

726 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

727 

728 maskedImage = exposure.getMaskedImage() 

729 mask = maskedImage.getMask() 

730 if altMask is not None: 

731 self.applyAltMaskPlanes(mask, altMask) 

732 imageScaler.scaleMaskedImage(maskedImage) 

733 

734 # Add 1 for each pixel which is not excluded by the exclude mask. 

735 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

736 if nImage is not None: 

737 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

738 if self.config.removeMaskPlanes: 

739 self.removeMaskPlanes(maskedImage) 

740 maskedImageList.append(maskedImage) 

741 

742 if self.config.doInputMap: 

743 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

744 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

745 

746 with self.timer("stack"): 

747 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

748 clipped, # also set output to CLIPPED if sigma-clipped 

749 maskMap) 

750 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

751 if nImage is not None: 

752 nImage.assign(subNImage, bbox) 

753 

754 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

755 altMaskList, statsCtrl, nImage=None): 

756 """Assemble the coadd using the "online" method. 

757 

758 This method takes a running sum of images and weights to save memory. 

759 It only works for MEAN statistics. 

760 

761 Parameters 

762 ---------- 

763 coaddExposure : `lsst.afw.image.Exposure` 

764 The target exposure for the coadd. 

765 tempExpRefList : `list` 

766 List of data reference to tempExp. 

767 imageScalerList : `list` 

768 List of image scalers. 

769 weightList : `list` 

770 List of weights. 

771 altMaskList : `list` 

772 List of alternate masks to use rather than those stored with 

773 tempExp, or None. Each element is dict with keys = mask plane 

774 name to which to add the spans. 

775 statsCtrl : `lsst.afw.math.StatisticsControl` 

776 Statistics control object for coadd 

777 nImage : `lsst.afw.image.ImageU`, optional 

778 Keeps track of exposure count for each pixel. 

779 """ 

780 self.log.debug("Computing online coadd.") 

781 

782 coaddExposure.mask.addMaskPlane("REJECTED") 

783 coaddExposure.mask.addMaskPlane("CLIPPED") 

784 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

785 maskMap = self.setRejectedMaskMapping(statsCtrl) 

786 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

787 

788 bbox = coaddExposure.maskedImage.getBBox() 

789 

790 stacker = AccumulatorMeanStack( 

791 coaddExposure.image.array.shape, 

792 statsCtrl.getAndMask(), 

793 mask_threshold_dict=thresholdDict, 

794 mask_map=maskMap, 

795 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

796 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

797 compute_n_image=(nImage is not None) 

798 ) 

799 

800 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

801 imageScalerList, 

802 altMaskList, 

803 weightList): 

804 exposure = tempExpRef.get() 

805 maskedImage = exposure.getMaskedImage() 

806 mask = maskedImage.getMask() 

807 if altMask is not None: 

808 self.applyAltMaskPlanes(mask, altMask) 

809 imageScaler.scaleMaskedImage(maskedImage) 

810 if self.config.removeMaskPlanes: 

811 self.removeMaskPlanes(maskedImage) 

812 

813 stacker.add_masked_image(maskedImage, weight=weight) 

814 

815 if self.config.doInputMap: 

816 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

817 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

818 

819 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

820 

821 if nImage is not None: 

822 nImage.array[:, :] = stacker.n_image 

823 

824 def removeMaskPlanes(self, maskedImage): 

825 """Unset the mask of an image for mask planes specified in the config. 

826 

827 Parameters 

828 ---------- 

829 maskedImage : `lsst.afw.image.MaskedImage` 

830 The masked image to be modified. 

831 """ 

832 mask = maskedImage.getMask() 

833 for maskPlane in self.config.removeMaskPlanes: 

834 try: 

835 mask &= ~mask.getPlaneBitMask(maskPlane) 

836 except pexExceptions.InvalidParameterError: 

837 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

838 maskPlane) 

839 

840 @staticmethod 

841 def setRejectedMaskMapping(statsCtrl): 

842 """Map certain mask planes of the warps to new planes for the coadd. 

843 

844 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

845 or CLIPPED, set it to REJECTED on the coadd. 

846 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

847 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

848 

849 Parameters 

850 ---------- 

851 statsCtrl : `lsst.afw.math.StatisticsControl` 

852 Statistics control object for coadd 

853 

854 Returns 

855 ------- 

856 maskMap : `list` of `tuple` of `int` 

857 A list of mappings of mask planes of the warped exposures to 

858 mask planes of the coadd. 

859 """ 

860 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

861 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

862 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

863 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

864 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

865 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

866 (clipped, clipped)] 

867 return maskMap 

868 

869 def applyAltMaskPlanes(self, mask, altMaskSpans): 

870 """Apply in place alt mask formatted as SpanSets to a mask. 

871 

872 Parameters 

873 ---------- 

874 mask : `lsst.afw.image.Mask` 

875 Original mask. 

876 altMaskSpans : `dict` 

877 SpanSet lists to apply. Each element contains the new mask 

878 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

879 and list of SpanSets to apply to the mask. 

880 

881 Returns 

882 ------- 

883 mask : `lsst.afw.image.Mask` 

884 Updated mask. 

885 """ 

886 if self.config.doUsePsfMatchedPolygons: 

887 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

888 # Clear away any other masks outside the validPolygons. These pixels are no longer 

889 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

890 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

891 # This mask-clearing step must only occur *before* applying the new masks below 

892 for spanSet in altMaskSpans['NO_DATA']: 

893 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

894 

895 for plane, spanSetList in altMaskSpans.items(): 

896 maskClipValue = mask.addMaskPlane(plane) 

897 for spanSet in spanSetList: 

898 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

899 return mask 

900 

901 def shrinkValidPolygons(self, coaddInputs): 

902 """Shrink coaddInputs' ccds' ValidPolygons in place. 

903 

904 Either modify each ccd's validPolygon in place, or if CoaddInputs 

905 does not have a validPolygon, create one from its bbox. 

906 

907 Parameters 

908 ---------- 

909 coaddInputs : `lsst.afw.image.coaddInputs` 

910 Original mask. 

911 

912 """ 

913 for ccd in coaddInputs.ccds: 

914 polyOrig = ccd.getValidPolygon() 

915 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

916 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

917 if polyOrig: 

918 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

919 else: 

920 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

921 ccd.setValidPolygon(validPolygon) 

922 

923 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

924 """Set the bright object masks. 

925 

926 Parameters 

927 ---------- 

928 exposure : `lsst.afw.image.Exposure` 

929 Exposure under consideration. 

930 brightObjectMasks : `lsst.afw.table` 

931 Table of bright objects to mask. 

932 dataId : `lsst.daf.butler.DataId`, optional 

933 Data identifier dict for patch. 

934 """ 

935 

936 if brightObjectMasks is None: 

937 self.log.warning("Unable to apply bright object mask: none supplied") 

938 return 

939 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

940 mask = exposure.getMaskedImage().getMask() 

941 wcs = exposure.getWcs() 

942 plateScale = wcs.getPixelScale().asArcseconds() 

943 

944 for rec in brightObjectMasks: 

945 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

946 if rec["type"] == "box": 

947 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

948 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

949 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

950 

951 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

952 bbox = geom.Box2I(center - halfSize, center + halfSize) 

953 

954 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

955 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

956 spans = afwGeom.SpanSet(bbox) 

957 elif rec["type"] == "circle": 

958 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

959 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

960 else: 

961 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

962 continue 

963 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

964 

965 def setInexactPsf(self, mask): 

966 """Set INEXACT_PSF mask plane. 

967 

968 If any of the input images isn't represented in the coadd (due to 

969 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

970 these pixels. 

971 

972 Parameters 

973 ---------- 

974 mask : `lsst.afw.image.Mask` 

975 Coadded exposure's mask, modified in-place. 

976 """ 

977 mask.addMaskPlane("INEXACT_PSF") 

978 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

979 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

980 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

981 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

982 array = mask.getArray() 

983 selected = array & (sensorEdge | clipped | rejected) > 0 

984 array[selected] |= inexactPsf 

985 

986 @staticmethod 

987 def _subBBoxIter(bbox, subregionSize): 

988 """Iterate over subregions of a bbox. 

989 

990 Parameters 

991 ---------- 

992 bbox : `lsst.geom.Box2I` 

993 Bounding box over which to iterate. 

994 subregionSize: `lsst.geom.Extent2I` 

995 Size of sub-bboxes. 

996 

997 Yields 

998 ------ 

999 subBBox : `lsst.geom.Box2I` 

1000 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1001 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1002 the edges of ``bbox``, but it will never be empty. 

1003 """ 

1004 if bbox.isEmpty(): 

1005 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1006 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1007 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1008 

1009 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1010 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1011 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1012 subBBox.clip(bbox) 

1013 if subBBox.isEmpty(): 

1014 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1015 "colShift=%s, rowShift=%s" % 

1016 (bbox, subregionSize, colShift, rowShift)) 

1017 yield subBBox 

1018 

1019 def filterWarps(self, inputs, goodVisits): 

1020 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1021 

1022 Parameters 

1023 ---------- 

1024 inputs : list 

1025 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1026 goodVisit : `dict` 

1027 Dictionary with good visitIds as the keys. Value ignored. 

1028 

1029 Returns 

1030 ------- 

1031 filteredInputs : `list` 

1032 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1033 """ 

1034 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1035 filteredInputs = [] 

1036 for visit in goodVisits.keys(): 

1037 if visit in inputWarpDict: 

1038 filteredInputs.append(inputWarpDict[visit]) 

1039 return filteredInputs 

1040 

1041 

1042def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1043 """Function to count the number of pixels with a specific mask in a 

1044 footprint. 

1045 

1046 Find the intersection of mask & footprint. Count all pixels in the mask 

1047 that are in the intersection that have bitmask set but do not have 

1048 ignoreMask set. Return the count. 

1049 

1050 Parameters 

1051 ---------- 

1052 mask : `lsst.afw.image.Mask` 

1053 Mask to define intersection region by. 

1054 footprint : `lsst.afw.detection.Footprint` 

1055 Footprint to define the intersection region by. 

1056 bitmask 

1057 Specific mask that we wish to count the number of occurances of. 

1058 ignoreMask 

1059 Pixels to not consider. 

1060 

1061 Returns 

1062 ------- 

1063 result : `int` 

1064 Count of number of pixels in footprint with specified mask. 

1065 """ 

1066 bbox = footprint.getBBox() 

1067 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1068 fp = afwImage.Mask(bbox) 

1069 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1070 footprint.spans.setMask(fp, bitmask) 

1071 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1072 (subMask.getArray() & ignoreMask) == 0).sum() 

1073 

1074 

1075class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1076 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1077 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1078 "Only PSF-Matched Warps make sense for image subtraction. " 

1079 "Therefore, they must be an additional declared input."), 

1080 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1081 storageClass="ExposureF", 

1082 dimensions=("tract", "patch", "skymap", "visit"), 

1083 deferLoad=True, 

1084 multiple=True 

1085 ) 

1086 templateCoadd = pipeBase.connectionTypes.Output( 

1087 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1088 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1089 name="{outputCoaddName}CoaddPsfMatched", 

1090 storageClass="ExposureF", 

1091 dimensions=("tract", "patch", "skymap", "band"), 

1092 ) 

1093 

1094 def __init__(self, *, config=None): 

1095 super().__init__(config=config) 

1096 if not config.assembleStaticSkyModel.doWrite: 

1097 self.outputs.remove("templateCoadd") 

1098 config.validate() 

1099 

1100 

1101class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1102 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1103 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1104 target=AssembleCoaddTask, 

1105 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1106 " naive/first-iteration model of the static sky.", 

1107 ) 

1108 detect = pexConfig.ConfigurableField( 

1109 target=SourceDetectionTask, 

1110 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1111 ) 

1112 detectTemplate = pexConfig.ConfigurableField( 

1113 target=SourceDetectionTask, 

1114 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1115 ) 

1116 maskStreaks = pexConfig.ConfigurableField( 

1117 target=MaskStreaksTask, 

1118 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1119 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1120 "streakMaskName" 

1121 ) 

1122 streakMaskName = pexConfig.Field( 

1123 dtype=str, 

1124 default="STREAK", 

1125 doc="Name of mask bit used for streaks" 

1126 ) 

1127 maxNumEpochs = pexConfig.Field( 

1128 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1129 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1130 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1131 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1132 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1133 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1134 "than transient and not masked.", 

1135 dtype=int, 

1136 default=2 

1137 ) 

1138 maxFractionEpochsLow = pexConfig.RangeField( 

1139 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1140 "Effective maxNumEpochs = " 

1141 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1142 dtype=float, 

1143 default=0.4, 

1144 min=0., max=1., 

1145 ) 

1146 maxFractionEpochsHigh = pexConfig.RangeField( 

1147 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1148 "Effective maxNumEpochs = " 

1149 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1150 dtype=float, 

1151 default=0.03, 

1152 min=0., max=1., 

1153 ) 

1154 spatialThreshold = pexConfig.RangeField( 

1155 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1156 "temporal criteria. If 0, clip all. If 1, clip none.", 

1157 dtype=float, 

1158 default=0.5, 

1159 min=0., max=1., 

1160 inclusiveMin=True, inclusiveMax=True 

1161 ) 

1162 doScaleWarpVariance = pexConfig.Field( 

1163 doc="Rescale Warp variance plane using empirical noise?", 

1164 dtype=bool, 

1165 default=True, 

1166 ) 

1167 scaleWarpVariance = pexConfig.ConfigurableField( 

1168 target=ScaleVarianceTask, 

1169 doc="Rescale variance on warps", 

1170 ) 

1171 doPreserveContainedBySource = pexConfig.Field( 

1172 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1173 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1174 dtype=bool, 

1175 default=True, 

1176 ) 

1177 doPrefilterArtifacts = pexConfig.Field( 

1178 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1179 "because they will be excluded anyway. This prevents them from contributing " 

1180 "to the outlier epoch count image and potentially being labeled as persistant." 

1181 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1182 dtype=bool, 

1183 default=True 

1184 ) 

1185 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1186 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1187 dtype=str, 

1188 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1189 ) 

1190 prefilterArtifactsRatio = pexConfig.Field( 

1191 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1192 dtype=float, 

1193 default=0.05 

1194 ) 

1195 doFilterMorphological = pexConfig.Field( 

1196 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1197 "be streaks.", 

1198 dtype=bool, 

1199 default=False 

1200 ) 

1201 growStreakFp = pexConfig.Field( 

1202 doc="Grow streak footprints by this number multiplied by the PSF width", 

1203 dtype=float, 

1204 default=5 

1205 ) 

1206 

1207 def setDefaults(self): 

1208 AssembleCoaddConfig.setDefaults(self) 

1209 self.statistic = 'MEAN' 

1210 self.doUsePsfMatchedPolygons = True 

1211 

1212 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1213 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1214 if "EDGE" in self.badMaskPlanes: 

1215 self.badMaskPlanes.remove('EDGE') 

1216 self.removeMaskPlanes.append('EDGE') 

1217 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1218 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1219 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1220 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1221 self.assembleStaticSkyModel.sigmaClip = 2.5 

1222 self.assembleStaticSkyModel.clipIter = 3 

1223 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1224 self.assembleStaticSkyModel.doWrite = False 

1225 self.detect.doTempLocalBackground = False 

1226 self.detect.reEstimateBackground = False 

1227 self.detect.returnOriginalFootprints = False 

1228 self.detect.thresholdPolarity = "both" 

1229 self.detect.thresholdValue = 5 

1230 self.detect.minPixels = 4 

1231 self.detect.isotropicGrow = True 

1232 self.detect.thresholdType = "pixel_stdev" 

1233 self.detect.nSigmaToGrow = 0.4 

1234 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1235 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1236 self.detectTemplate.nSigmaToGrow = 2.4 

1237 self.detectTemplate.doTempLocalBackground = False 

1238 self.detectTemplate.reEstimateBackground = False 

1239 self.detectTemplate.returnOriginalFootprints = False 

1240 

1241 def validate(self): 

1242 super().validate() 

1243 if self.assembleStaticSkyModel.doNImage: 

1244 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1245 "Please set assembleStaticSkyModel.doNImage=False") 

1246 

1247 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1248 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1249 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1250 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1251 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1252 

1253 

1254class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1255 """Assemble a compareWarp coadded image from a set of warps 

1256 by masking artifacts detected by comparing PSF-matched warps. 

1257 

1258 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1259 we clip outliers). The problem with doing this is that when computing the 

1260 coadd PSF at a given location, individual visit PSFs from visits with 

1261 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1262 In this task, we correct for this behavior by creating a new badMaskPlane 

1263 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1264 an artifact. We populate this plane on the input warps by comparing 

1265 PSF-matched warps with a PSF-matched median coadd which serves as a 

1266 model of the static sky. Any group of pixels that deviates from the 

1267 PSF-matched template coadd by more than config.detect.threshold sigma, 

1268 is an artifact candidate. The candidates are then filtered to remove 

1269 variable sources and sources that are difficult to subtract such as 

1270 bright stars. This filter is configured using the config parameters 

1271 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1272 the maximum fraction of epochs that the deviation can appear in and still 

1273 be considered an artifact. The spatialThreshold is the maximum fraction of 

1274 pixels in the footprint of the deviation that appear in other epochs 

1275 (where other epochs is defined by the temporalThreshold). If the deviant 

1276 region meets this criteria of having a significant percentage of pixels 

1277 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1278 set in the mask. These regions will not contribute to the final coadd. 

1279 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1280 of clipped regions. Note that the algorithm implemented by this task is 

1281 preliminary and works correctly for HSC data. Parameter modifications and 

1282 or considerable redesigning of the algorithm is likley required for other 

1283 surveys. 

1284 

1285 ``CompareWarpAssembleCoaddTask`` sub-classes 

1286 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1287 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1288 """ 

1289 ConfigClass = CompareWarpAssembleCoaddConfig 

1290 _DefaultName = "compareWarpAssembleCoadd" 

1291 

1292 def __init__(self, *args, **kwargs): 

1293 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1294 self.makeSubtask("assembleStaticSkyModel") 

1295 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

1296 self.makeSubtask("detect", schema=detectionSchema) 

1297 if self.config.doPreserveContainedBySource: 

1298 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

1299 if self.config.doScaleWarpVariance: 

1300 self.makeSubtask("scaleWarpVariance") 

1301 if self.config.doFilterMorphological: 

1302 self.makeSubtask("maskStreaks") 

1303 

1304 @utils.inheritDoc(AssembleCoaddTask) 

1305 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

1306 """ 

1307 Generate a templateCoadd to use as a naive model of static sky to 

1308 subtract from PSF-Matched warps. 

1309 

1310 Returns 

1311 ------- 

1312 result : `lsst.pipe.base.Struct` 

1313 Result struct with components: 

1314 

1315 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

1316 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

1317 """ 

1318 # Ensure that psfMatchedWarps are used as input warps for template generation 

1319 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

1320 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

1321 

1322 # Because subtasks don't have connections we have to make one. 

1323 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

1324 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

1325 if self.config.assembleStaticSkyModel.doWrite: 

1326 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

1327 # Remove template coadd from both subtask's and main tasks outputs, 

1328 # because it is handled by the subtask as `coaddExposure` 

1329 del outputRefs.templateCoadd 

1330 del staticSkyModelOutputRefs.templateCoadd 

1331 

1332 # A PSF-Matched nImage does not exist as a dataset type 

1333 if 'nImage' in staticSkyModelOutputRefs.keys(): 

1334 del staticSkyModelOutputRefs.nImage 

1335 

1336 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

1337 staticSkyModelOutputRefs) 

1338 if templateCoadd is None: 

1339 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

1340 

1341 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

1342 nImage=templateCoadd.nImage, 

1343 warpRefList=templateCoadd.warpRefList, 

1344 imageScalerList=templateCoadd.imageScalerList, 

1345 weightList=templateCoadd.weightList) 

1346 

1347 def _noTemplateMessage(self, warpType): 

1348 warpName = (warpType[0].upper() + warpType[1:]) 

1349 message = """No %(warpName)s warps were found to build the template coadd which is 

1350 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

1351 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

1352 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

1353 

1354 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

1355 another algorithm like: 

1356 

1357 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

1358 config.assemble.retarget(SafeClipAssembleCoaddTask) 

1359 """ % {"warpName": warpName} 

1360 return message 

1361 

1362 @utils.inheritDoc(AssembleCoaddTask) 

1363 @timeMethod 

1364 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1365 supplementaryData, *args, **kwargs): 

1366 """Assemble the coadd. 

1367 

1368 Find artifacts and apply them to the warps' masks creating a list of 

1369 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

1370 plane. Then pass these alternative masks to the base class's `run` 

1371 method. 

1372 

1373 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

1374 that must contain a ``templateCoadd`` that serves as the 

1375 model of the static sky. 

1376 """ 

1377 

1378 # Check and match the order of the supplementaryData 

1379 # (PSF-matched) inputs to the order of the direct inputs, 

1380 # so that the artifact mask is applied to the right warp 

1381 dataIds = [ref.dataId for ref in tempExpRefList] 

1382 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

1383 

1384 if dataIds != psfMatchedDataIds: 

1385 self.log.info("Reordering and or/padding PSF-matched visit input list") 

1386 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

1387 psfMatchedDataIds, dataIds) 

1388 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

1389 psfMatchedDataIds, dataIds) 

1390 

1391 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

1392 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

1393 supplementaryData.warpRefList, 

1394 supplementaryData.imageScalerList) 

1395 

1396 badMaskPlanes = self.config.badMaskPlanes[:] 

1397 badMaskPlanes.append("CLIPPED") 

1398 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1399 

1400 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1401 spanSetMaskList, mask=badPixelMask) 

1402 

1403 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

1404 # Psf-Matching moves the real edge inwards 

1405 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

1406 return result 

1407 

1408 def applyAltEdgeMask(self, mask, altMaskList): 

1409 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

1410 

1411 Parameters 

1412 ---------- 

1413 mask : `lsst.afw.image.Mask` 

1414 Original mask. 

1415 altMaskList : `list` 

1416 List of Dicts containing ``spanSet`` lists. 

1417 Each element contains the new mask plane name (e.g. "CLIPPED 

1418 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

1419 the mask. 

1420 """ 

1421 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

1422 for visitMask in altMaskList: 

1423 if "EDGE" in visitMask: 

1424 for spanSet in visitMask['EDGE']: 

1425 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

1426 

1427 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

1428 """Find artifacts. 

1429 

1430 Loop through warps twice. The first loop builds a map with the count 

1431 of how many epochs each pixel deviates from the templateCoadd by more 

1432 than ``config.chiThreshold`` sigma. The second loop takes each 

1433 difference image and filters the artifacts detected in each using 

1434 count map to filter out variable sources and sources that are 

1435 difficult to subtract cleanly. 

1436 

1437 Parameters 

1438 ---------- 

1439 templateCoadd : `lsst.afw.image.Exposure` 

1440 Exposure to serve as model of static sky. 

1441 tempExpRefList : `list` 

1442 List of data references to warps. 

1443 imageScalerList : `list` 

1444 List of image scalers. 

1445 

1446 Returns 

1447 ------- 

1448 altMasks : `list` 

1449 List of dicts containing information about CLIPPED 

1450 (i.e., artifacts), NO_DATA, and EDGE pixels. 

1451 """ 

1452 

1453 self.log.debug("Generating Count Image, and mask lists.") 

1454 coaddBBox = templateCoadd.getBBox() 

1455 slateIm = afwImage.ImageU(coaddBBox) 

1456 epochCountImage = afwImage.ImageU(coaddBBox) 

1457 nImage = afwImage.ImageU(coaddBBox) 

1458 spanSetArtifactList = [] 

1459 spanSetNoDataMaskList = [] 

1460 spanSetEdgeList = [] 

1461 spanSetBadMorphoList = [] 

1462 badPixelMask = self.getBadPixelMask() 

1463 

1464 # mask of the warp diffs should = that of only the warp 

1465 templateCoadd.mask.clearAllMaskPlanes() 

1466 

1467 if self.config.doPreserveContainedBySource: 

1468 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

1469 else: 

1470 templateFootprints = None 

1471 

1472 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

1473 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

1474 if warpDiffExp is not None: 

1475 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

1476 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

1477 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

1478 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

1479 fpSet.positive.merge(fpSet.negative) 

1480 footprints = fpSet.positive 

1481 slateIm.set(0) 

1482 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

1483 

1484 # Remove artifacts due to defects before they contribute to the epochCountImage 

1485 if self.config.doPrefilterArtifacts: 

1486 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

1487 

1488 # Clear mask before adding prefiltered spanSets 

1489 self.detect.clearMask(warpDiffExp.mask) 

1490 for spans in spanSetList: 

1491 spans.setImage(slateIm, 1, doClip=True) 

1492 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

1493 epochCountImage += slateIm 

1494 

1495 if self.config.doFilterMorphological: 

1496 maskName = self.config.streakMaskName 

1497 _ = self.maskStreaks.run(warpDiffExp) 

1498 streakMask = warpDiffExp.mask 

1499 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

1500 streakMask.getPlaneBitMask(maskName)).split() 

1501 # Pad the streaks to account for low-surface brightness wings 

1502 psf = warpDiffExp.getPsf() 

1503 for s, sset in enumerate(spanSetStreak): 

1504 psfShape = psf.computeShape(sset.computeCentroid()) 

1505 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius() 

1506 sset_dilated = sset.dilated(int(dilation)) 

1507 spanSetStreak[s] = sset_dilated 

1508 

1509 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

1510 # undergo a second convolution. Pixels with data in the direct warp 

1511 # but not in the PSF-matched warp will not have their artifacts detected. 

1512 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

1513 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

1514 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

1515 nansMask.setXY0(warpDiffExp.getXY0()) 

1516 edgeMask = warpDiffExp.mask 

1517 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

1518 edgeMask.getPlaneBitMask("EDGE")).split() 

1519 else: 

1520 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

1521 # In this case, mask the whole epoch 

1522 nansMask = afwImage.MaskX(coaddBBox, 1) 

1523 spanSetList = [] 

1524 spanSetEdgeMask = [] 

1525 spanSetStreak = [] 

1526 

1527 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

1528 

1529 spanSetNoDataMaskList.append(spanSetNoDataMask) 

1530 spanSetArtifactList.append(spanSetList) 

1531 spanSetEdgeList.append(spanSetEdgeMask) 

1532 if self.config.doFilterMorphological: 

1533 spanSetBadMorphoList.append(spanSetStreak) 

1534 

1535 if lsstDebug.Info(__name__).saveCountIm: 

1536 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

1537 epochCountImage.writeFits(path) 

1538 

1539 for i, spanSetList in enumerate(spanSetArtifactList): 

1540 if spanSetList: 

1541 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

1542 templateFootprints) 

1543 spanSetArtifactList[i] = filteredSpanSetList 

1544 if self.config.doFilterMorphological: 

1545 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

1546 

1547 altMasks = [] 

1548 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

1549 altMasks.append({'CLIPPED': artifacts, 

1550 'NO_DATA': noData, 

1551 'EDGE': edge}) 

1552 return altMasks 

1553 

1554 def prefilterArtifacts(self, spanSetList, exp): 

1555 """Remove artifact candidates covered by bad mask plane. 

1556 

1557 Any future editing of the candidate list that does not depend on 

1558 temporal information should go in this method. 

1559 

1560 Parameters 

1561 ---------- 

1562 spanSetList : `list` 

1563 List of SpanSets representing artifact candidates. 

1564 exp : `lsst.afw.image.Exposure` 

1565 Exposure containing mask planes used to prefilter. 

1566 

1567 Returns 

1568 ------- 

1569 returnSpanSetList : `list` 

1570 List of SpanSets with artifacts. 

1571 """ 

1572 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

1573 goodArr = (exp.mask.array & badPixelMask) == 0 

1574 returnSpanSetList = [] 

1575 bbox = exp.getBBox() 

1576 x0, y0 = exp.getXY0() 

1577 for i, span in enumerate(spanSetList): 

1578 y, x = span.clippedTo(bbox).indices() 

1579 yIndexLocal = numpy.array(y) - y0 

1580 xIndexLocal = numpy.array(x) - x0 

1581 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

1582 if goodRatio > self.config.prefilterArtifactsRatio: 

1583 returnSpanSetList.append(span) 

1584 return returnSpanSetList 

1585 

1586 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

1587 """Filter artifact candidates. 

1588 

1589 Parameters 

1590 ---------- 

1591 spanSetList : `list` 

1592 List of SpanSets representing artifact candidates. 

1593 epochCountImage : `lsst.afw.image.Image` 

1594 Image of accumulated number of warpDiff detections. 

1595 nImage : `lsst.afw.image.Image` 

1596 Image of the accumulated number of total epochs contributing. 

1597 

1598 Returns 

1599 ------- 

1600 maskSpanSetList : `list` 

1601 List of SpanSets with artifacts. 

1602 """ 

1603 

1604 maskSpanSetList = [] 

1605 x0, y0 = epochCountImage.getXY0() 

1606 for i, span in enumerate(spanSetList): 

1607 y, x = span.indices() 

1608 yIdxLocal = [y1 - y0 for y1 in y] 

1609 xIdxLocal = [x1 - x0 for x1 in x] 

1610 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

1611 totalN = nImage.array[yIdxLocal, xIdxLocal] 

1612 

1613 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

1614 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

1615 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

1616 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

1617 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

1618 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

1619 & (outlierN <= effectiveMaxNumEpochs)) 

1620 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

1621 if percentBelowThreshold > self.config.spatialThreshold: 

1622 maskSpanSetList.append(span) 

1623 

1624 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

1625 # If a candidate is contained by a footprint on the template coadd, do not clip 

1626 filteredMaskSpanSetList = [] 

1627 for span in maskSpanSetList: 

1628 doKeep = True 

1629 for footprint in footprintsToExclude.positive.getFootprints(): 

1630 if footprint.spans.contains(span): 

1631 doKeep = False 

1632 break 

1633 if doKeep: 

1634 filteredMaskSpanSetList.append(span) 

1635 maskSpanSetList = filteredMaskSpanSetList 

1636 

1637 return maskSpanSetList 

1638 

1639 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

1640 """Fetch a warp from the butler and return a warpDiff. 

1641 Parameters 

1642 ---------- 

1643 warpRef : `lsst.daf.butler.DeferredDatasetHandle` 

1644 Handle for the warp. 

1645 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

1646 An image scaler object. 

1647 templateCoadd : `lsst.afw.image.Exposure` 

1648 Exposure to be substracted from the scaled warp. 

1649 Returns 

1650 ------- 

1651 warp : `lsst.afw.image.Exposure` 

1652 Exposure of the image difference between the warp and template. 

1653 """ 

1654 

1655 # If the PSF-Matched warp did not exist for this direct warp 

1656 # None is holding its place to maintain order in Gen 3 

1657 if warpRef is None: 

1658 return None 

1659 

1660 warp = warpRef.get() 

1661 # direct image scaler OK for PSF-matched Warp 

1662 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

1663 mi = warp.getMaskedImage() 

1664 if self.config.doScaleWarpVariance: 

1665 try: 

1666 self.scaleWarpVariance.run(mi) 

1667 except Exception as exc: 

1668 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

1669 mi -= templateCoadd.getMaskedImage() 

1670 return warp