Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 16%

646 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-17 10:06 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

23 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

24 

25import copy 

26import numpy 

27import warnings 

28import logging 

29import lsst.pex.config as pexConfig 

30import lsst.pex.exceptions as pexExceptions 

31import lsst.geom as geom 

32import lsst.afw.geom as afwGeom 

33import lsst.afw.image as afwImage 

34import lsst.afw.math as afwMath 

35import lsst.afw.table as afwTable 

36import lsst.coadd.utils as coaddUtils 

37import lsst.pipe.base as pipeBase 

38import lsst.meas.algorithms as measAlg 

39import lsstDebug 

40import lsst.utils as utils 

41from lsst.skymap import BaseSkyMap 

42from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

43from .interpImage import InterpImageTask 

44from .scaleZeroPoint import ScaleZeroPointTask 

45from .maskStreaks import MaskStreaksTask 

46from .healSparseMapping import HealSparseInputMapTask 

47from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask 

48from lsst.utils.timer import timeMethod 

49from deprecated.sphinx import deprecated 

50 

51log = logging.getLogger(__name__) 

52 

53 

54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

55 dimensions=("tract", "patch", "band", "skymap"), 

56 defaultTemplates={"inputCoaddName": "deep", 

57 "outputCoaddName": "deep", 

58 "warpType": "direct", 

59 "warpTypeSuffix": ""}): 

60 

61 inputWarps = pipeBase.connectionTypes.Input( 

62 doc=("Input list of warps to be assemebled i.e. stacked." 

63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

64 name="{inputCoaddName}Coadd_{warpType}Warp", 

65 storageClass="ExposureF", 

66 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

67 deferLoad=True, 

68 multiple=True 

69 ) 

70 skyMap = pipeBase.connectionTypes.Input( 

71 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

73 storageClass="SkyMap", 

74 dimensions=("skymap", ), 

75 ) 

76 selectedVisits = pipeBase.connectionTypes.Input( 

77 doc="Selected visits to be coadded.", 

78 name="{outputCoaddName}Visits", 

79 storageClass="StructuredDataDict", 

80 dimensions=("instrument", "tract", "patch", "skymap", "band") 

81 ) 

82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

83 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

84 " BRIGHT_OBJECT."), 

85 name="brightObjectMask", 

86 storageClass="ObjectMaskCatalog", 

87 dimensions=("tract", "patch", "skymap", "band"), 

88 minimum=0, 

89 ) 

90 coaddExposure = pipeBase.connectionTypes.Output( 

91 doc="Output coadded exposure, produced by stacking input warps", 

92 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

93 storageClass="ExposureF", 

94 dimensions=("tract", "patch", "skymap", "band"), 

95 ) 

96 nImage = pipeBase.connectionTypes.Output( 

97 doc="Output image of number of input images per pixel", 

98 name="{outputCoaddName}Coadd_nImage", 

99 storageClass="ImageU", 

100 dimensions=("tract", "patch", "skymap", "band"), 

101 ) 

102 inputMap = pipeBase.connectionTypes.Output( 

103 doc="Output healsparse map of input images", 

104 name="{outputCoaddName}Coadd_inputMap", 

105 storageClass="HealSparseMap", 

106 dimensions=("tract", "patch", "skymap", "band"), 

107 ) 

108 

109 def __init__(self, *, config=None): 

110 super().__init__(config=config) 

111 

112 if not config.doMaskBrightObjects: 

113 self.prerequisiteInputs.remove("brightObjectMask") 

114 

115 if not config.doSelectVisits: 

116 self.inputs.remove("selectedVisits") 

117 

118 if not config.doNImage: 

119 self.outputs.remove("nImage") 

120 

121 if not self.config.doInputMap: 

122 self.outputs.remove("inputMap") 

123 

124 

125class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

126 pipelineConnections=AssembleCoaddConnections): 

127 warpType = pexConfig.Field( 

128 doc="Warp name: one of 'direct' or 'psfMatched'", 

129 dtype=str, 

130 default="direct", 

131 ) 

132 subregionSize = pexConfig.ListField( 

133 dtype=int, 

134 doc="Width, height of stack subregion size; " 

135 "make small enough that a full stack of images will fit into memory at once.", 

136 length=2, 

137 default=(2000, 2000), 

138 ) 

139 statistic = pexConfig.Field( 

140 dtype=str, 

141 doc="Main stacking statistic for aggregating over the epochs.", 

142 default="MEANCLIP", 

143 ) 

144 doOnlineForMean = pexConfig.Field( 

145 dtype=bool, 

146 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

147 default=False, 

148 ) 

149 doSigmaClip = pexConfig.Field( 

150 dtype=bool, 

151 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

152 default=False, 

153 ) 

154 sigmaClip = pexConfig.Field( 

155 dtype=float, 

156 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

157 default=3.0, 

158 ) 

159 clipIter = pexConfig.Field( 

160 dtype=int, 

161 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

162 default=2, 

163 ) 

164 calcErrorFromInputVariance = pexConfig.Field( 

165 dtype=bool, 

166 doc="Calculate coadd variance from input variance by stacking statistic." 

167 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

168 default=True, 

169 ) 

170 scaleZeroPoint = pexConfig.ConfigurableField( 

171 target=ScaleZeroPointTask, 

172 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

173 ) 

174 doInterp = pexConfig.Field( 

175 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

176 dtype=bool, 

177 default=True, 

178 ) 

179 interpImage = pexConfig.ConfigurableField( 

180 target=InterpImageTask, 

181 doc="Task to interpolate (and extrapolate) over NaN pixels", 

182 ) 

183 doWrite = pexConfig.Field( 

184 doc="Persist coadd?", 

185 dtype=bool, 

186 default=True, 

187 ) 

188 doNImage = pexConfig.Field( 

189 doc="Create image of number of contributing exposures for each pixel", 

190 dtype=bool, 

191 default=False, 

192 ) 

193 doUsePsfMatchedPolygons = pexConfig.Field( 

194 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

195 dtype=bool, 

196 default=False, 

197 ) 

198 maskPropagationThresholds = pexConfig.DictField( 

199 keytype=str, 

200 itemtype=float, 

201 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

202 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

203 "would have contributed exceeds this value."), 

204 default={"SAT": 0.1}, 

205 ) 

206 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

207 doc="Mask planes to remove before coadding") 

208 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

209 doc="Set mask and flag bits for bright objects?") 

210 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

211 doc="Name of mask bit used for bright objects") 

212 coaddPsf = pexConfig.ConfigField( 

213 doc="Configuration for CoaddPsf", 

214 dtype=measAlg.CoaddPsfConfig, 

215 ) 

216 doAttachTransmissionCurve = pexConfig.Field( 

217 dtype=bool, default=False, optional=False, 

218 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

219 "(requires all input Exposures to have TransmissionCurves).") 

220 ) 

221 hasFakes = pexConfig.Field( 

222 dtype=bool, 

223 default=False, 

224 doc="Should be set to True if fake sources have been inserted into the input data." 

225 ) 

226 doSelectVisits = pexConfig.Field( 

227 doc="Coadd only visits selected by a SelectVisitsTask", 

228 dtype=bool, 

229 default=False, 

230 ) 

231 doInputMap = pexConfig.Field( 

232 doc="Create a bitwise map of coadd inputs", 

233 dtype=bool, 

234 default=False, 

235 ) 

236 inputMapper = pexConfig.ConfigurableField( 

237 doc="Input map creation subtask.", 

238 target=HealSparseInputMapTask, 

239 ) 

240 

241 def setDefaults(self): 

242 super().setDefaults() 

243 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

244 

245 def validate(self): 

246 super().validate() 

247 if self.doPsfMatch: # TODO: Remove this in DM-39841 

248 # Backwards compatibility. 

249 # Configs do not have loggers 

250 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

251 self.warpType = 'psfMatched' 

252 if self.doSigmaClip and self.statistic != "MEANCLIP": 

253 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

254 self.statistic = "MEANCLIP" 

255 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

256 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

257 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

258 

259 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

260 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

261 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

262 if str(k) not in unstackableStats] 

263 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

264 % (self.statistic, stackableStats)) 

265 

266 

267class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

268 """Assemble a coadded image from a set of warps. 

269 

270 Each Warp that goes into a coadd will typically have an independent 

271 photometric zero-point. Therefore, we must scale each Warp to set it to 

272 a common photometric zeropoint. WarpType may be one of 'direct' or 

273 'psfMatched', and the boolean configs `config.makeDirect` and 

274 `config.makePsfMatched` set which of the warp types will be coadded. 

275 The coadd is computed as a mean with optional outlier rejection. 

276 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

277 Finally, Warps can have bad 'NaN' pixels which received no input from the 

278 source calExps. We interpolate over these bad (NaN) pixels. 

279 

280 `AssembleCoaddTask` uses several sub-tasks. These are 

281 

282 - `~lsst.pipe.tasks.ScaleZeroPointTask` 

283 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

284 - `~lsst.pipe.tasks.InterpImageTask` 

285 - interpolate across bad pixels (NaN) in the final coadd 

286 

287 You can retarget these subtasks if you wish. 

288 

289 Raises 

290 ------ 

291 RuntimeError 

292 Raised if unable to define mask plane for bright objects. 

293 

294 Notes 

295 ----- 

296 Debugging: 

297 `AssembleCoaddTask` has no debug variables of its own. Some of the 

298 subtasks may support `~lsst.base.lsstDebug` variables. See the 

299 documentation for the subtasks for further information. 

300 

301 Examples 

302 -------- 

303 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

304 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

305 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

306 inputs: a data reference to the tract patch and filter to be coadded, and 

307 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

308 ``--selectId``, respectively: 

309 

310 .. code-block:: none 

311 

312 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

313 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

314 

315 Only the Warps that cover the specified tract and patch will be coadded. 

316 A list of the available optional arguments can be obtained by calling 

317 ``assembleCoadd.py`` with the ``--help`` command line argument: 

318 

319 .. code-block:: none 

320 

321 assembleCoadd.py --help 

322 

323 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

324 multi-band processing, we will generate the HSC-I & -R band coadds from 

325 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

326 assuming that the lsst stack has been already set up, we must set up the 

327 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

328 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

329 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

330 coadds, we must first run: 

331 

332 - processCcd 

333 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

334 - makeSkyMap 

335 - create a skymap that covers the area of the sky present in the raw exposures 

336 - makeCoaddTempExp 

337 - warp the individual calibrated exposures to the tangent plane of the coadd 

338 

339 We can perform all of these steps by running 

340 

341 .. code-block:: none 

342 

343 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

344 

345 This will produce warped exposures for each visit. To coadd the warped 

346 data, we call assembleCoadd.py as follows: 

347 

348 .. code-block:: none 

349 

350 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

351 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

352 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

353 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

354 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

355 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

356 --selectId visit=903988 ccd=24 

357 

358 that will process the HSC-I band data. The results are written in 

359 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

360 

361 You may also choose to run: 

362 

363 .. code-block:: none 

364 

365 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

366 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

367 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

368 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

369 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

370 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

371 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

372 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

373 

374 to generate the coadd for the HSC-R band if you are interested in 

375 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

376 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

377 rather than `AssembleCoaddTask` to make the coadd. 

378 """ 

379 

380 ConfigClass = AssembleCoaddConfig 

381 _DefaultName = "assembleCoadd" 

382 

383 def __init__(self, *args, **kwargs): 

384 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

385 if args: 

386 argNames = ["config", "name", "parentTask", "log"] 

387 kwargs.update({k: v for k, v in zip(argNames, args)}) 

388 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

389 "PipelineTask will not take positional args" % argNames, FutureWarning, 

390 stacklevel=2) 

391 

392 super().__init__(**kwargs) 

393 self.makeSubtask("interpImage") 

394 self.makeSubtask("scaleZeroPoint") 

395 

396 if self.config.doMaskBrightObjects: 

397 mask = afwImage.Mask() 

398 try: 

399 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

400 except pexExceptions.LsstCppException: 

401 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

402 mask.getMaskPlaneDict().keys()) 

403 del mask 

404 

405 if self.config.doInputMap: 

406 self.makeSubtask("inputMapper") 

407 

408 self.warpType = self.config.warpType 

409 

410 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

411 inputData = butlerQC.get(inputRefs) 

412 

413 # Construct skyInfo expected by run 

414 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it 

415 skyMap = inputData["skyMap"] 

416 outputDataId = butlerQC.quantum.dataId 

417 

418 inputData['skyInfo'] = makeSkyInfo(skyMap, 

419 tractId=outputDataId['tract'], 

420 patchId=outputDataId['patch']) 

421 

422 if self.config.doSelectVisits: 

423 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

424 else: 

425 warpRefList = inputData['inputWarps'] 

426 

427 inputs = self.prepareInputs(warpRefList) 

428 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

429 self.getTempExpDatasetName(self.warpType)) 

430 if len(inputs.tempExpRefList) == 0: 

431 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

432 

433 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

434 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

435 inputs.weightList, supplementaryData=supplementaryData) 

436 

437 inputData.setdefault('brightObjectMask', None) 

438 if self.config.doMaskBrightObjects and inputData["brightObjectMask"] is None: 

439 log.warning("doMaskBrightObjects is set to True, but brightObjectMask not loaded") 

440 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

441 

442 if self.config.doWrite: 

443 butlerQC.put(retStruct, outputRefs) 

444 return retStruct 

445 

446 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

447 """Interpolate over missing data and mask bright stars. 

448 

449 Parameters 

450 ---------- 

451 coaddExposure : `lsst.afw.image.Exposure` 

452 The coadded exposure to process. 

453 brightObjectMasks : `lsst.afw.table` or `None`, optional 

454 Table of bright objects to mask. 

455 dataId : `lsst.daf.butler.DataId` or `None`, optional 

456 Data identification. 

457 """ 

458 if self.config.doInterp: 

459 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

460 # The variance must be positive; work around for DM-3201. 

461 varArray = coaddExposure.variance.array 

462 with numpy.errstate(invalid="ignore"): 

463 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

464 

465 if self.config.doMaskBrightObjects: 

466 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

467 

468 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

469 """Make additional inputs to run() specific to subclasses (Gen3). 

470 

471 Duplicates interface of `runQuantum` method. 

472 Available to be implemented by subclasses only if they need the 

473 coadd dataRef for performing preliminary processing before 

474 assembling the coadd. 

475 

476 Parameters 

477 ---------- 

478 butlerQC : `~lsst.pipe.base.ButlerQuantumContext` 

479 Gen3 Butler object for fetching additional data products before 

480 running the Task specialized for quantum being processed. 

481 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

482 Attributes are the names of the connections describing input dataset types. 

483 Values are DatasetRefs that task consumes for corresponding dataset type. 

484 DataIds are guaranteed to match data objects in ``inputData``. 

485 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

486 Attributes are the names of the connections describing output dataset types. 

487 Values are DatasetRefs that task is to produce 

488 for corresponding dataset type. 

489 """ 

490 return pipeBase.Struct() 

491 

492 @deprecated( 

493 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData", 

494 version="v25.0", 

495 category=FutureWarning 

496 ) 

497 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

498 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

499 

500 def prepareInputs(self, refList): 

501 """Prepare the input warps for coaddition by measuring the weight for 

502 each warp and the scaling for the photometric zero point. 

503 

504 Each Warp has its own photometric zeropoint and background variance. 

505 Before coadding these Warps together, compute a scale factor to 

506 normalize the photometric zeropoint and compute the weight for each Warp. 

507 

508 Parameters 

509 ---------- 

510 refList : `list` 

511 List of data references to tempExp. 

512 

513 Returns 

514 ------- 

515 result : `~lsst.pipe.base.Struct` 

516 Results as a struct with attributes: 

517 

518 ``tempExprefList`` 

519 `list` of data references to tempExp. 

520 ``weightList`` 

521 `list` of weightings. 

522 ``imageScalerList`` 

523 `list` of image scalers. 

524 """ 

525 statsCtrl = afwMath.StatisticsControl() 

526 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

527 statsCtrl.setNumIter(self.config.clipIter) 

528 statsCtrl.setAndMask(self.getBadPixelMask()) 

529 statsCtrl.setNanSafe(True) 

530 # compute tempExpRefList: a list of tempExpRef that actually exist 

531 # and weightList: a list of the weight of the associated coadd tempExp 

532 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

533 tempExpRefList = [] 

534 weightList = [] 

535 imageScalerList = [] 

536 tempExpName = self.getTempExpDatasetName(self.warpType) 

537 for tempExpRef in refList: 

538 tempExp = tempExpRef.get() 

539 # Ignore any input warp that is empty of data 

540 if numpy.isnan(tempExp.image.array).all(): 

541 continue 

542 maskedImage = tempExp.getMaskedImage() 

543 imageScaler = self.scaleZeroPoint.computeImageScaler( 

544 exposure=tempExp, 

545 dataRef=tempExpRef, # FIXME 

546 ) 

547 try: 

548 imageScaler.scaleMaskedImage(maskedImage) 

549 except Exception as e: 

550 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

551 continue 

552 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

553 afwMath.MEANCLIP, statsCtrl) 

554 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

555 weight = 1.0 / float(meanVar) 

556 if not numpy.isfinite(weight): 

557 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

558 continue 

559 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

560 

561 del maskedImage 

562 del tempExp 

563 

564 tempExpRefList.append(tempExpRef) 

565 weightList.append(weight) 

566 imageScalerList.append(imageScaler) 

567 

568 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

569 imageScalerList=imageScalerList) 

570 

571 def prepareStats(self, mask=None): 

572 """Prepare the statistics for coadding images. 

573 

574 Parameters 

575 ---------- 

576 mask : `int`, optional 

577 Bit mask value to exclude from coaddition. 

578 

579 Returns 

580 ------- 

581 stats : `~lsst.pipe.base.Struct` 

582 Statistics as a struct with attributes: 

583 

584 ``statsCtrl`` 

585 Statistics control object for coadd (`~lsst.afw.math.StatisticsControl`). 

586 ``statsFlags`` 

587 Statistic for coadd (`~lsst.afw.math.Property`). 

588 """ 

589 if mask is None: 

590 mask = self.getBadPixelMask() 

591 statsCtrl = afwMath.StatisticsControl() 

592 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

593 statsCtrl.setNumIter(self.config.clipIter) 

594 statsCtrl.setAndMask(mask) 

595 statsCtrl.setNanSafe(True) 

596 statsCtrl.setWeighted(True) 

597 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

598 for plane, threshold in self.config.maskPropagationThresholds.items(): 

599 bit = afwImage.Mask.getMaskPlane(plane) 

600 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

601 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

602 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

603 

604 @timeMethod 

605 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

606 altMaskList=None, mask=None, supplementaryData=None): 

607 """Assemble a coadd from input warps. 

608 

609 Assemble the coadd using the provided list of coaddTempExps. Since 

610 the full coadd covers a patch (a large area), the assembly is 

611 performed over small areas on the image at a time in order to 

612 conserve memory usage. Iterate over subregions within the outer 

613 bbox of the patch using `assembleSubregion` to stack the corresponding 

614 subregions from the coaddTempExps with the statistic specified. 

615 Set the edge bits the coadd mask based on the weight map. 

616 

617 Parameters 

618 ---------- 

619 skyInfo : `~lsst.pipe.base.Struct` 

620 Struct with geometric information about the patch. 

621 tempExpRefList : `list` 

622 List of data references to Warps (previously called CoaddTempExps). 

623 imageScalerList : `list` 

624 List of image scalers. 

625 weightList : `list` 

626 List of weights. 

627 altMaskList : `list`, optional 

628 List of alternate masks to use rather than those stored with 

629 tempExp. 

630 mask : `int`, optional 

631 Bit mask value to exclude from coaddition. 

632 supplementaryData : `~lsst.pipe.base.Struct`, optional 

633 Struct with additional data products needed to assemble coadd. 

634 Only used by subclasses that implement ``_makeSupplementaryData`` 

635 and override `run`. 

636 

637 Returns 

638 ------- 

639 result : `~lsst.pipe.base.Struct` 

640 Results as a struct with attributes: 

641 

642 ``coaddExposure`` 

643 Coadded exposure (``lsst.afw.image.Exposure``). 

644 ``nImage`` 

645 Exposure count image (``lsst.afw.image.Image``), if requested. 

646 ``inputMap`` 

647 Bit-wise map of inputs, if requested. 

648 ``warpRefList`` 

649 Input list of refs to the warps (``lsst.daf.butler.DeferredDatasetHandle``) 

650 (unmodified). 

651 ``imageScalerList`` 

652 Input list of image scalers (`list`) (unmodified). 

653 ``weightList`` 

654 Input list of weights (`list`) (unmodified). 

655 

656 Raises 

657 ------ 

658 lsst.pipe.base.NoWorkFound 

659 Raised if no data references are provided. 

660 """ 

661 tempExpName = self.getTempExpDatasetName(self.warpType) 

662 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

663 if not tempExpRefList: 

664 raise pipeBase.NoWorkFound("No exposures provided for co-addition.") 

665 

666 stats = self.prepareStats(mask=mask) 

667 

668 if altMaskList is None: 

669 altMaskList = [None]*len(tempExpRefList) 

670 

671 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

672 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

673 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

674 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

675 coaddMaskedImage = coaddExposure.getMaskedImage() 

676 subregionSizeArr = self.config.subregionSize 

677 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

678 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

679 if self.config.doNImage: 

680 nImage = afwImage.ImageU(skyInfo.bbox) 

681 else: 

682 nImage = None 

683 # If inputMap is requested, create the initial version that can be masked in 

684 # assembleSubregion. 

685 if self.config.doInputMap: 

686 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

687 skyInfo.wcs, 

688 coaddExposure.getInfo().getCoaddInputs().ccds) 

689 

690 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 

691 try: 

692 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

693 weightList, altMaskList, stats.ctrl, 

694 nImage=nImage) 

695 except Exception as e: 

696 self.log.exception("Cannot compute online coadd %s", e) 

697 raise 

698 else: 

699 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

700 try: 

701 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

702 weightList, altMaskList, stats.flags, stats.ctrl, 

703 nImage=nImage) 

704 except Exception as e: 

705 self.log.exception("Cannot compute coadd %s: %s", subBBox, e) 

706 raise 

707 

708 # If inputMap is requested, we must finalize the map after the accumulation. 

709 if self.config.doInputMap: 

710 self.inputMapper.finalize_ccd_input_map_mask() 

711 inputMap = self.inputMapper.ccd_input_map 

712 else: 

713 inputMap = None 

714 

715 self.setInexactPsf(coaddMaskedImage.getMask()) 

716 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

717 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

718 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

719 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

720 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

721 weightList=weightList, inputMap=inputMap) 

722 

723 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

724 """Set the metadata for the coadd. 

725 

726 This basic implementation sets the filter from the first input. 

727 

728 Parameters 

729 ---------- 

730 coaddExposure : `lsst.afw.image.Exposure` 

731 The target exposure for the coadd. 

732 tempExpRefList : `list` 

733 List of data references to tempExp. 

734 weightList : `list` 

735 List of weights. 

736 

737 Raises 

738 ------ 

739 AssertionError 

740 Raised if there is a length mismatch. 

741 """ 

742 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

743 

744 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

745 # (and we need more than just the PropertySet that contains the header), which is not possible 

746 # with the current butler (see #2777). 

747 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

748 

749 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

750 

751 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

752 

753 # Set the coadd FilterLabel to the band of the first input exposure: 

754 # Coadds are calibrated, so the physical label is now meaningless. 

755 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel)) 

756 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

757 coaddInputs.ccds.reserve(numCcds) 

758 coaddInputs.visits.reserve(len(tempExpList)) 

759 

760 for tempExp, weight in zip(tempExpList, weightList): 

761 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

762 

763 if self.config.doUsePsfMatchedPolygons: 

764 self.shrinkValidPolygons(coaddInputs) 

765 

766 coaddInputs.visits.sort() 

767 coaddInputs.ccds.sort() 

768 if self.warpType == "psfMatched": 

769 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

770 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

771 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

772 # having the maximum width (sufficient because square) 

773 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

774 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

775 for modelPsf in modelPsfList] 

776 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

777 else: 

778 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

779 self.config.coaddPsf.makeControl()) 

780 coaddExposure.setPsf(psf) 

781 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

782 coaddExposure.getWcs()) 

783 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

784 if self.config.doAttachTransmissionCurve: 

785 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

786 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

787 

788 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

789 altMaskList, statsFlags, statsCtrl, nImage=None): 

790 """Assemble the coadd for a sub-region. 

791 

792 For each coaddTempExp, check for (and swap in) an alternative mask 

793 if one is passed. Remove mask planes listed in 

794 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

795 `lsst.afw.math.statisticsStack` with the statistic specified by 

796 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

797 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

798 an N-sigma clipped mean where N and iterations are specified by 

799 statsCtrl. Assign the stacked subregion back to the coadd. 

800 

801 Parameters 

802 ---------- 

803 coaddExposure : `lsst.afw.image.Exposure` 

804 The target exposure for the coadd. 

805 bbox : `lsst.geom.Box` 

806 Sub-region to coadd. 

807 tempExpRefList : `list` 

808 List of data reference to tempExp. 

809 imageScalerList : `list` 

810 List of image scalers. 

811 weightList : `list` 

812 List of weights. 

813 altMaskList : `list` 

814 List of alternate masks to use rather than those stored with 

815 tempExp, or None. Each element is dict with keys = mask plane 

816 name to which to add the spans. 

817 statsFlags : `lsst.afw.math.Property` 

818 Property object for statistic for coadd. 

819 statsCtrl : `lsst.afw.math.StatisticsControl` 

820 Statistics control object for coadd. 

821 nImage : `lsst.afw.image.ImageU`, optional 

822 Keeps track of exposure count for each pixel. 

823 """ 

824 self.log.debug("Computing coadd over %s", bbox) 

825 

826 coaddExposure.mask.addMaskPlane("REJECTED") 

827 coaddExposure.mask.addMaskPlane("CLIPPED") 

828 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

829 maskMap = self.setRejectedMaskMapping(statsCtrl) 

830 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

831 maskedImageList = [] 

832 if nImage is not None: 

833 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

834 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

835 

836 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

837 

838 maskedImage = exposure.getMaskedImage() 

839 mask = maskedImage.getMask() 

840 if altMask is not None: 

841 self.applyAltMaskPlanes(mask, altMask) 

842 imageScaler.scaleMaskedImage(maskedImage) 

843 

844 # Add 1 for each pixel which is not excluded by the exclude mask. 

845 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

846 if nImage is not None: 

847 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

848 if self.config.removeMaskPlanes: 

849 self.removeMaskPlanes(maskedImage) 

850 maskedImageList.append(maskedImage) 

851 

852 if self.config.doInputMap: 

853 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

854 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

855 

856 with self.timer("stack"): 

857 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

858 clipped, # also set output to CLIPPED if sigma-clipped 

859 maskMap) 

860 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

861 if nImage is not None: 

862 nImage.assign(subNImage, bbox) 

863 

864 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

865 altMaskList, statsCtrl, nImage=None): 

866 """Assemble the coadd using the "online" method. 

867 

868 This method takes a running sum of images and weights to save memory. 

869 It only works for MEAN statistics. 

870 

871 Parameters 

872 ---------- 

873 coaddExposure : `lsst.afw.image.Exposure` 

874 The target exposure for the coadd. 

875 tempExpRefList : `list` 

876 List of data reference to tempExp. 

877 imageScalerList : `list` 

878 List of image scalers. 

879 weightList : `list` 

880 List of weights. 

881 altMaskList : `list` 

882 List of alternate masks to use rather than those stored with 

883 tempExp, or None. Each element is dict with keys = mask plane 

884 name to which to add the spans. 

885 statsCtrl : `lsst.afw.math.StatisticsControl` 

886 Statistics control object for coadd. 

887 nImage : `lsst.afw.image.ImageU`, optional 

888 Keeps track of exposure count for each pixel. 

889 """ 

890 self.log.debug("Computing online coadd.") 

891 

892 coaddExposure.mask.addMaskPlane("REJECTED") 

893 coaddExposure.mask.addMaskPlane("CLIPPED") 

894 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

895 maskMap = self.setRejectedMaskMapping(statsCtrl) 

896 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

897 

898 bbox = coaddExposure.maskedImage.getBBox() 

899 

900 stacker = AccumulatorMeanStack( 

901 coaddExposure.image.array.shape, 

902 statsCtrl.getAndMask(), 

903 mask_threshold_dict=thresholdDict, 

904 mask_map=maskMap, 

905 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

906 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

907 compute_n_image=(nImage is not None) 

908 ) 

909 

910 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

911 imageScalerList, 

912 altMaskList, 

913 weightList): 

914 exposure = tempExpRef.get() 

915 maskedImage = exposure.getMaskedImage() 

916 mask = maskedImage.getMask() 

917 if altMask is not None: 

918 self.applyAltMaskPlanes(mask, altMask) 

919 imageScaler.scaleMaskedImage(maskedImage) 

920 if self.config.removeMaskPlanes: 

921 self.removeMaskPlanes(maskedImage) 

922 

923 stacker.add_masked_image(maskedImage, weight=weight) 

924 

925 if self.config.doInputMap: 

926 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

927 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

928 

929 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

930 

931 if nImage is not None: 

932 nImage.array[:, :] = stacker.n_image 

933 

934 def removeMaskPlanes(self, maskedImage): 

935 """Unset the mask of an image for mask planes specified in the config. 

936 

937 Parameters 

938 ---------- 

939 maskedImage : `lsst.afw.image.MaskedImage` 

940 The masked image to be modified. 

941 

942 Raises 

943 ------ 

944 InvalidParameterError 

945 Raised if no mask plane with that name was found. 

946 """ 

947 mask = maskedImage.getMask() 

948 for maskPlane in self.config.removeMaskPlanes: 

949 try: 

950 mask &= ~mask.getPlaneBitMask(maskPlane) 

951 except pexExceptions.InvalidParameterError: 

952 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

953 maskPlane) 

954 

955 @staticmethod 

956 def setRejectedMaskMapping(statsCtrl): 

957 """Map certain mask planes of the warps to new planes for the coadd. 

958 

959 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

960 or CLIPPED, set it to REJECTED on the coadd. 

961 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

962 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

963 

964 Parameters 

965 ---------- 

966 statsCtrl : `lsst.afw.math.StatisticsControl` 

967 Statistics control object for coadd. 

968 

969 Returns 

970 ------- 

971 maskMap : `list` of `tuple` of `int` 

972 A list of mappings of mask planes of the warped exposures to 

973 mask planes of the coadd. 

974 """ 

975 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

976 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

977 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

978 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

979 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

980 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

981 (clipped, clipped)] 

982 return maskMap 

983 

984 def applyAltMaskPlanes(self, mask, altMaskSpans): 

985 """Apply in place alt mask formatted as SpanSets to a mask. 

986 

987 Parameters 

988 ---------- 

989 mask : `lsst.afw.image.Mask` 

990 Original mask. 

991 altMaskSpans : `dict` 

992 SpanSet lists to apply. Each element contains the new mask 

993 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

994 and list of SpanSets to apply to the mask. 

995 

996 Returns 

997 ------- 

998 mask : `lsst.afw.image.Mask` 

999 Updated mask. 

1000 """ 

1001 if self.config.doUsePsfMatchedPolygons: 

1002 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

1003 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1004 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1005 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1006 # This mask-clearing step must only occur *before* applying the new masks below 

1007 for spanSet in altMaskSpans['NO_DATA']: 

1008 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1009 

1010 for plane, spanSetList in altMaskSpans.items(): 

1011 maskClipValue = mask.addMaskPlane(plane) 

1012 for spanSet in spanSetList: 

1013 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1014 return mask 

1015 

1016 def shrinkValidPolygons(self, coaddInputs): 

1017 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1018 

1019 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1020 does not have a validPolygon, create one from its bbox. 

1021 

1022 Parameters 

1023 ---------- 

1024 coaddInputs : `lsst.afw.image.coaddInputs` 

1025 Original mask. 

1026 """ 

1027 for ccd in coaddInputs.ccds: 

1028 polyOrig = ccd.getValidPolygon() 

1029 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1030 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1031 if polyOrig: 

1032 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1033 else: 

1034 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1035 ccd.setValidPolygon(validPolygon) 

1036 

1037 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1038 """Set the bright object masks. 

1039 

1040 Parameters 

1041 ---------- 

1042 exposure : `lsst.afw.image.Exposure` 

1043 Exposure under consideration. 

1044 brightObjectMasks : `lsst.afw.table` 

1045 Table of bright objects to mask. 

1046 dataId : `lsst.daf.butler.DataId`, optional 

1047 Data identifier dict for patch. 

1048 """ 

1049 if brightObjectMasks is None: 

1050 self.log.warning("Unable to apply bright object mask: none supplied") 

1051 return 

1052 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1053 mask = exposure.getMaskedImage().getMask() 

1054 wcs = exposure.getWcs() 

1055 plateScale = wcs.getPixelScale().asArcseconds() 

1056 

1057 for rec in brightObjectMasks: 

1058 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1059 if rec["type"] == "box": 

1060 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1061 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1062 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1063 

1064 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1065 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1066 

1067 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1068 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1069 spans = afwGeom.SpanSet(bbox) 

1070 elif rec["type"] == "circle": 

1071 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1072 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1073 else: 

1074 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1075 continue 

1076 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1077 

1078 def setInexactPsf(self, mask): 

1079 """Set INEXACT_PSF mask plane. 

1080 

1081 If any of the input images isn't represented in the coadd (due to 

1082 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1083 these pixels. 

1084 

1085 Parameters 

1086 ---------- 

1087 mask : `lsst.afw.image.Mask` 

1088 Coadded exposure's mask, modified in-place. 

1089 """ 

1090 mask.addMaskPlane("INEXACT_PSF") 

1091 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1092 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1093 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1094 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1095 array = mask.getArray() 

1096 selected = array & (sensorEdge | clipped | rejected) > 0 

1097 array[selected] |= inexactPsf 

1098 

1099 @staticmethod 

1100 def _subBBoxIter(bbox, subregionSize): 

1101 """Iterate over subregions of a bbox. 

1102 

1103 Parameters 

1104 ---------- 

1105 bbox : `lsst.geom.Box2I` 

1106 Bounding box over which to iterate. 

1107 subregionSize : `lsst.geom.Extent2I` 

1108 Size of sub-bboxes. 

1109 

1110 Yields 

1111 ------ 

1112 subBBox : `lsst.geom.Box2I` 

1113 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1114 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1115 the edges of ``bbox``, but it will never be empty. 

1116 

1117 Raises 

1118 ------ 

1119 RuntimeError 

1120 Raised if any of the following occur: 

1121 - The given bbox is empty. 

1122 - The subregionSize is 0. 

1123 """ 

1124 if bbox.isEmpty(): 

1125 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1126 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1127 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1128 

1129 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1130 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1131 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1132 subBBox.clip(bbox) 

1133 if subBBox.isEmpty(): 

1134 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1135 "colShift=%s, rowShift=%s" % 

1136 (bbox, subregionSize, colShift, rowShift)) 

1137 yield subBBox 

1138 

1139 def filterWarps(self, inputs, goodVisits): 

1140 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit. 

1141 

1142 Parameters 

1143 ---------- 

1144 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef` 

1145 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit. 

1146 goodVisit : `dict` 

1147 Dictionary with good visitIds as the keys. Value ignored. 

1148 

1149 Returns 

1150 ------- 

1151 filteredInputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef` 

1152 Filtered and sorted list of inputRefs with visitId in goodVisits ordered by goodVisit. 

1153 """ 

1154 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1155 filteredInputs = [] 

1156 for visit in goodVisits.keys(): 

1157 if visit in inputWarpDict: 

1158 filteredInputs.append(inputWarpDict[visit]) 

1159 return filteredInputs 

1160 

1161 

1162def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1163 """Function to count the number of pixels with a specific mask in a 

1164 footprint. 

1165 

1166 Find the intersection of mask & footprint. Count all pixels in the mask 

1167 that are in the intersection that have bitmask set but do not have 

1168 ignoreMask set. Return the count. 

1169 

1170 Parameters 

1171 ---------- 

1172 mask : `lsst.afw.image.Mask` 

1173 Mask to define intersection region by. 

1174 footprint : `lsst.afw.detection.Footprint` 

1175 Footprint to define the intersection region by. 

1176 bitmask : `Unknown` 

1177 Specific mask that we wish to count the number of occurances of. 

1178 ignoreMask : `Unknown` 

1179 Pixels to not consider. 

1180 

1181 Returns 

1182 ------- 

1183 result : `int` 

1184 Number of pixels in footprint with specified mask. 

1185 """ 

1186 bbox = footprint.getBBox() 

1187 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1188 fp = afwImage.Mask(bbox) 

1189 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1190 footprint.spans.setMask(fp, bitmask) 

1191 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1192 (subMask.getArray() & ignoreMask) == 0).sum() 

1193 

1194 

1195class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1196 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1197 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1198 "Only PSF-Matched Warps make sense for image subtraction. " 

1199 "Therefore, they must be an additional declared input."), 

1200 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1201 storageClass="ExposureF", 

1202 dimensions=("tract", "patch", "skymap", "visit"), 

1203 deferLoad=True, 

1204 multiple=True 

1205 ) 

1206 templateCoadd = pipeBase.connectionTypes.Output( 

1207 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1208 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1209 name="{outputCoaddName}CoaddPsfMatched", 

1210 storageClass="ExposureF", 

1211 dimensions=("tract", "patch", "skymap", "band"), 

1212 ) 

1213 

1214 def __init__(self, *, config=None): 

1215 super().__init__(config=config) 

1216 if not config.assembleStaticSkyModel.doWrite: 

1217 self.outputs.remove("templateCoadd") 

1218 config.validate() 

1219 

1220 

1221class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1222 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1223 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1224 target=AssembleCoaddTask, 

1225 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1226 " naive/first-iteration model of the static sky.", 

1227 ) 

1228 detect = pexConfig.ConfigurableField( 

1229 target=SourceDetectionTask, 

1230 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1231 ) 

1232 detectTemplate = pexConfig.ConfigurableField( 

1233 target=SourceDetectionTask, 

1234 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1235 ) 

1236 maskStreaks = pexConfig.ConfigurableField( 

1237 target=MaskStreaksTask, 

1238 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1239 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1240 "streakMaskName" 

1241 ) 

1242 streakMaskName = pexConfig.Field( 

1243 dtype=str, 

1244 default="STREAK", 

1245 doc="Name of mask bit used for streaks" 

1246 ) 

1247 maxNumEpochs = pexConfig.Field( 

1248 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1249 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1250 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1251 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1252 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1253 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1254 "than transient and not masked.", 

1255 dtype=int, 

1256 default=2 

1257 ) 

1258 maxFractionEpochsLow = pexConfig.RangeField( 

1259 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1260 "Effective maxNumEpochs = " 

1261 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1262 dtype=float, 

1263 default=0.4, 

1264 min=0., max=1., 

1265 ) 

1266 maxFractionEpochsHigh = pexConfig.RangeField( 

1267 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1268 "Effective maxNumEpochs = " 

1269 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1270 dtype=float, 

1271 default=0.03, 

1272 min=0., max=1., 

1273 ) 

1274 spatialThreshold = pexConfig.RangeField( 

1275 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1276 "temporal criteria. If 0, clip all. If 1, clip none.", 

1277 dtype=float, 

1278 default=0.5, 

1279 min=0., max=1., 

1280 inclusiveMin=True, inclusiveMax=True 

1281 ) 

1282 doScaleWarpVariance = pexConfig.Field( 

1283 doc="Rescale Warp variance plane using empirical noise?", 

1284 dtype=bool, 

1285 default=True, 

1286 ) 

1287 scaleWarpVariance = pexConfig.ConfigurableField( 

1288 target=ScaleVarianceTask, 

1289 doc="Rescale variance on warps", 

1290 ) 

1291 doPreserveContainedBySource = pexConfig.Field( 

1292 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1293 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1294 dtype=bool, 

1295 default=True, 

1296 ) 

1297 doPrefilterArtifacts = pexConfig.Field( 

1298 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1299 "because they will be excluded anyway. This prevents them from contributing " 

1300 "to the outlier epoch count image and potentially being labeled as persistant." 

1301 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1302 dtype=bool, 

1303 default=True 

1304 ) 

1305 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1306 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1307 dtype=str, 

1308 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1309 ) 

1310 prefilterArtifactsRatio = pexConfig.Field( 

1311 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1312 dtype=float, 

1313 default=0.05 

1314 ) 

1315 doFilterMorphological = pexConfig.Field( 

1316 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1317 "be streaks.", 

1318 dtype=bool, 

1319 default=False 

1320 ) 

1321 growStreakFp = pexConfig.Field( 

1322 doc="Grow streak footprints by this number multiplied by the PSF width", 

1323 dtype=float, 

1324 default=5 

1325 ) 

1326 

1327 def setDefaults(self): 

1328 AssembleCoaddConfig.setDefaults(self) 

1329 self.statistic = 'MEAN' 

1330 self.doUsePsfMatchedPolygons = True 

1331 

1332 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1333 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1334 if "EDGE" in self.badMaskPlanes: 

1335 self.badMaskPlanes.remove('EDGE') 

1336 self.removeMaskPlanes.append('EDGE') 

1337 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1338 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1339 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1340 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1341 self.assembleStaticSkyModel.sigmaClip = 2.5 

1342 self.assembleStaticSkyModel.clipIter = 3 

1343 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1344 self.assembleStaticSkyModel.doWrite = False 

1345 self.detect.doTempLocalBackground = False 

1346 self.detect.reEstimateBackground = False 

1347 self.detect.returnOriginalFootprints = False 

1348 self.detect.thresholdPolarity = "both" 

1349 self.detect.thresholdValue = 5 

1350 self.detect.minPixels = 4 

1351 self.detect.isotropicGrow = True 

1352 self.detect.thresholdType = "pixel_stdev" 

1353 self.detect.nSigmaToGrow = 0.4 

1354 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1355 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1356 self.detectTemplate.nSigmaToGrow = 2.4 

1357 self.detectTemplate.doTempLocalBackground = False 

1358 self.detectTemplate.reEstimateBackground = False 

1359 self.detectTemplate.returnOriginalFootprints = False 

1360 

1361 def validate(self): 

1362 super().validate() 

1363 if self.assembleStaticSkyModel.doNImage: 

1364 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1365 "Please set assembleStaticSkyModel.doNImage=False") 

1366 

1367 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1368 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1369 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1370 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1371 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1372 

1373 

1374class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1375 """Assemble a compareWarp coadded image from a set of warps 

1376 by masking artifacts detected by comparing PSF-matched warps. 

1377 

1378 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1379 we clip outliers). The problem with doing this is that when computing the 

1380 coadd PSF at a given location, individual visit PSFs from visits with 

1381 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1382 In this task, we correct for this behavior by creating a new badMaskPlane 

1383 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1384 an artifact. We populate this plane on the input warps by comparing 

1385 PSF-matched warps with a PSF-matched median coadd which serves as a 

1386 model of the static sky. Any group of pixels that deviates from the 

1387 PSF-matched template coadd by more than config.detect.threshold sigma, 

1388 is an artifact candidate. The candidates are then filtered to remove 

1389 variable sources and sources that are difficult to subtract such as 

1390 bright stars. This filter is configured using the config parameters 

1391 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1392 the maximum fraction of epochs that the deviation can appear in and still 

1393 be considered an artifact. The spatialThreshold is the maximum fraction of 

1394 pixels in the footprint of the deviation that appear in other epochs 

1395 (where other epochs is defined by the temporalThreshold). If the deviant 

1396 region meets this criteria of having a significant percentage of pixels 

1397 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1398 set in the mask. These regions will not contribute to the final coadd. 

1399 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1400 of clipped regions. Note that the algorithm implemented by this task is 

1401 preliminary and works correctly for HSC data. Parameter modifications and 

1402 or considerable redesigning of the algorithm is likley required for other 

1403 surveys. 

1404 

1405 ``CompareWarpAssembleCoaddTask`` sub-classes 

1406 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1407 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1408 

1409 Notes 

1410 ----- 

1411 Debugging: 

1412 This task supports the following debug variables: 

1413 - ``saveCountIm`` 

1414 If True then save the Epoch Count Image as a fits file in the `figPath` 

1415 - ``figPath`` 

1416 Path to save the debug fits images and figures 

1417 """ 

1418 

1419 ConfigClass = CompareWarpAssembleCoaddConfig 

1420 _DefaultName = "compareWarpAssembleCoadd" 

1421 

1422 def __init__(self, *args, **kwargs): 

1423 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1424 self.makeSubtask("assembleStaticSkyModel") 

1425 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

1426 self.makeSubtask("detect", schema=detectionSchema) 

1427 if self.config.doPreserveContainedBySource: 

1428 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

1429 if self.config.doScaleWarpVariance: 

1430 self.makeSubtask("scaleWarpVariance") 

1431 if self.config.doFilterMorphological: 

1432 self.makeSubtask("maskStreaks") 

1433 

1434 @utils.inheritDoc(AssembleCoaddTask) 

1435 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

1436 """Generate a templateCoadd to use as a naive model of static sky to 

1437 subtract from PSF-Matched warps. 

1438 

1439 Returns 

1440 ------- 

1441 result : `~lsst.pipe.base.Struct` 

1442 Results as a struct with attributes: 

1443 

1444 ``templateCoadd`` 

1445 Coadded exposure (`lsst.afw.image.Exposure`). 

1446 ``nImage`` 

1447 Keeps track of exposure count for each pixel (`lsst.afw.image.ImageU`). 

1448 

1449 Raises 

1450 ------ 

1451 RuntimeError 

1452 Raised if ``templateCoadd`` is `None`. 

1453 """ 

1454 # Ensure that psfMatchedWarps are used as input warps for template generation 

1455 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

1456 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

1457 

1458 # Because subtasks don't have connections we have to make one. 

1459 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

1460 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

1461 if self.config.assembleStaticSkyModel.doWrite: 

1462 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

1463 # Remove template coadd from both subtask's and main tasks outputs, 

1464 # because it is handled by the subtask as `coaddExposure` 

1465 del outputRefs.templateCoadd 

1466 del staticSkyModelOutputRefs.templateCoadd 

1467 

1468 # A PSF-Matched nImage does not exist as a dataset type 

1469 if 'nImage' in staticSkyModelOutputRefs.keys(): 

1470 del staticSkyModelOutputRefs.nImage 

1471 

1472 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

1473 staticSkyModelOutputRefs) 

1474 if templateCoadd is None: 

1475 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

1476 

1477 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

1478 nImage=templateCoadd.nImage, 

1479 warpRefList=templateCoadd.warpRefList, 

1480 imageScalerList=templateCoadd.imageScalerList, 

1481 weightList=templateCoadd.weightList) 

1482 

1483 def _noTemplateMessage(self, warpType): 

1484 warpName = (warpType[0].upper() + warpType[1:]) 

1485 message = """No %(warpName)s warps were found to build the template coadd which is 

1486 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

1487 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

1488 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

1489 

1490 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

1491 another algorithm like: 

1492 

1493 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

1494 config.assemble.retarget(SafeClipAssembleCoaddTask) 

1495 """ % {"warpName": warpName} 

1496 return message 

1497 

1498 @utils.inheritDoc(AssembleCoaddTask) 

1499 @timeMethod 

1500 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1501 supplementaryData): 

1502 """Notes 

1503 ----- 

1504 Assemble the coadd. 

1505 

1506 Find artifacts and apply them to the warps' masks creating a list of 

1507 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

1508 plane. Then pass these alternative masks to the base class's ``run`` 

1509 method. 

1510 """ 

1511 # Check and match the order of the supplementaryData 

1512 # (PSF-matched) inputs to the order of the direct inputs, 

1513 # so that the artifact mask is applied to the right warp 

1514 dataIds = [ref.dataId for ref in tempExpRefList] 

1515 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

1516 

1517 if dataIds != psfMatchedDataIds: 

1518 self.log.info("Reordering and or/padding PSF-matched visit input list") 

1519 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

1520 psfMatchedDataIds, dataIds) 

1521 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

1522 psfMatchedDataIds, dataIds) 

1523 

1524 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

1525 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

1526 supplementaryData.warpRefList, 

1527 supplementaryData.imageScalerList) 

1528 

1529 badMaskPlanes = self.config.badMaskPlanes[:] 

1530 badMaskPlanes.append("CLIPPED") 

1531 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1532 

1533 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1534 spanSetMaskList, mask=badPixelMask) 

1535 

1536 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

1537 # Psf-Matching moves the real edge inwards 

1538 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

1539 return result 

1540 

1541 def applyAltEdgeMask(self, mask, altMaskList): 

1542 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

1543 

1544 Parameters 

1545 ---------- 

1546 mask : `lsst.afw.image.Mask` 

1547 Original mask. 

1548 altMaskList : `list` of `dict` 

1549 List of Dicts containing ``spanSet`` lists. 

1550 Each element contains the new mask plane name (e.g. "CLIPPED 

1551 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

1552 the mask. 

1553 """ 

1554 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

1555 for visitMask in altMaskList: 

1556 if "EDGE" in visitMask: 

1557 for spanSet in visitMask['EDGE']: 

1558 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

1559 

1560 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

1561 """Find artifacts. 

1562 

1563 Loop through warps twice. The first loop builds a map with the count 

1564 of how many epochs each pixel deviates from the templateCoadd by more 

1565 than ``config.chiThreshold`` sigma. The second loop takes each 

1566 difference image and filters the artifacts detected in each using 

1567 count map to filter out variable sources and sources that are 

1568 difficult to subtract cleanly. 

1569 

1570 Parameters 

1571 ---------- 

1572 templateCoadd : `lsst.afw.image.Exposure` 

1573 Exposure to serve as model of static sky. 

1574 tempExpRefList : `list` 

1575 List of data references to warps. 

1576 imageScalerList : `list` 

1577 List of image scalers. 

1578 

1579 Returns 

1580 ------- 

1581 altMasks : `list` of `dict` 

1582 List of dicts containing information about CLIPPED 

1583 (i.e., artifacts), NO_DATA, and EDGE pixels. 

1584 """ 

1585 self.log.debug("Generating Count Image, and mask lists.") 

1586 coaddBBox = templateCoadd.getBBox() 

1587 slateIm = afwImage.ImageU(coaddBBox) 

1588 epochCountImage = afwImage.ImageU(coaddBBox) 

1589 nImage = afwImage.ImageU(coaddBBox) 

1590 spanSetArtifactList = [] 

1591 spanSetNoDataMaskList = [] 

1592 spanSetEdgeList = [] 

1593 spanSetBadMorphoList = [] 

1594 badPixelMask = self.getBadPixelMask() 

1595 

1596 # mask of the warp diffs should = that of only the warp 

1597 templateCoadd.mask.clearAllMaskPlanes() 

1598 

1599 if self.config.doPreserveContainedBySource: 

1600 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

1601 else: 

1602 templateFootprints = None 

1603 

1604 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

1605 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

1606 if warpDiffExp is not None: 

1607 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

1608 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

1609 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

1610 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

1611 fpSet.positive.merge(fpSet.negative) 

1612 footprints = fpSet.positive 

1613 slateIm.set(0) 

1614 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

1615 

1616 # Remove artifacts due to defects before they contribute to the epochCountImage 

1617 if self.config.doPrefilterArtifacts: 

1618 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

1619 

1620 # Clear mask before adding prefiltered spanSets 

1621 self.detect.clearMask(warpDiffExp.mask) 

1622 for spans in spanSetList: 

1623 spans.setImage(slateIm, 1, doClip=True) 

1624 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

1625 epochCountImage += slateIm 

1626 

1627 if self.config.doFilterMorphological: 

1628 maskName = self.config.streakMaskName 

1629 _ = self.maskStreaks.run(warpDiffExp) 

1630 streakMask = warpDiffExp.mask 

1631 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

1632 streakMask.getPlaneBitMask(maskName)).split() 

1633 # Pad the streaks to account for low-surface brightness wings 

1634 psf = warpDiffExp.getPsf() 

1635 for s, sset in enumerate(spanSetStreak): 

1636 psfShape = psf.computeShape(sset.computeCentroid()) 

1637 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius() 

1638 sset_dilated = sset.dilated(int(dilation)) 

1639 spanSetStreak[s] = sset_dilated 

1640 

1641 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

1642 # undergo a second convolution. Pixels with data in the direct warp 

1643 # but not in the PSF-matched warp will not have their artifacts detected. 

1644 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

1645 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

1646 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

1647 nansMask.setXY0(warpDiffExp.getXY0()) 

1648 edgeMask = warpDiffExp.mask 

1649 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

1650 edgeMask.getPlaneBitMask("EDGE")).split() 

1651 else: 

1652 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

1653 # In this case, mask the whole epoch 

1654 nansMask = afwImage.MaskX(coaddBBox, 1) 

1655 spanSetList = [] 

1656 spanSetEdgeMask = [] 

1657 spanSetStreak = [] 

1658 

1659 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

1660 

1661 spanSetNoDataMaskList.append(spanSetNoDataMask) 

1662 spanSetArtifactList.append(spanSetList) 

1663 spanSetEdgeList.append(spanSetEdgeMask) 

1664 if self.config.doFilterMorphological: 

1665 spanSetBadMorphoList.append(spanSetStreak) 

1666 

1667 if lsstDebug.Info(__name__).saveCountIm: 

1668 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

1669 epochCountImage.writeFits(path) 

1670 

1671 for i, spanSetList in enumerate(spanSetArtifactList): 

1672 if spanSetList: 

1673 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

1674 templateFootprints) 

1675 spanSetArtifactList[i] = filteredSpanSetList 

1676 if self.config.doFilterMorphological: 

1677 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

1678 

1679 altMasks = [] 

1680 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

1681 altMasks.append({'CLIPPED': artifacts, 

1682 'NO_DATA': noData, 

1683 'EDGE': edge}) 

1684 return altMasks 

1685 

1686 def prefilterArtifacts(self, spanSetList, exp): 

1687 """Remove artifact candidates covered by bad mask plane. 

1688 

1689 Any future editing of the candidate list that does not depend on 

1690 temporal information should go in this method. 

1691 

1692 Parameters 

1693 ---------- 

1694 spanSetList : `list` of `lsst.afw.geom.SpanSet` 

1695 List of SpanSets representing artifact candidates. 

1696 exp : `lsst.afw.image.Exposure` 

1697 Exposure containing mask planes used to prefilter. 

1698 

1699 Returns 

1700 ------- 

1701 returnSpanSetList : `list` of `lsst.afw.geom.SpanSet` 

1702 List of SpanSets with artifacts. 

1703 """ 

1704 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

1705 goodArr = (exp.mask.array & badPixelMask) == 0 

1706 returnSpanSetList = [] 

1707 bbox = exp.getBBox() 

1708 x0, y0 = exp.getXY0() 

1709 for i, span in enumerate(spanSetList): 

1710 y, x = span.clippedTo(bbox).indices() 

1711 yIndexLocal = numpy.array(y) - y0 

1712 xIndexLocal = numpy.array(x) - x0 

1713 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

1714 if goodRatio > self.config.prefilterArtifactsRatio: 

1715 returnSpanSetList.append(span) 

1716 return returnSpanSetList 

1717 

1718 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

1719 """Filter artifact candidates. 

1720 

1721 Parameters 

1722 ---------- 

1723 spanSetList : `list` of `lsst.afw.geom.SpanSet` 

1724 List of SpanSets representing artifact candidates. 

1725 epochCountImage : `lsst.afw.image.Image` 

1726 Image of accumulated number of warpDiff detections. 

1727 nImage : `lsst.afw.image.ImageU` 

1728 Image of the accumulated number of total epochs contributing. 

1729 

1730 Returns 

1731 ------- 

1732 maskSpanSetList : `list` 

1733 List of SpanSets with artifacts. 

1734 """ 

1735 maskSpanSetList = [] 

1736 x0, y0 = epochCountImage.getXY0() 

1737 for i, span in enumerate(spanSetList): 

1738 y, x = span.indices() 

1739 yIdxLocal = [y1 - y0 for y1 in y] 

1740 xIdxLocal = [x1 - x0 for x1 in x] 

1741 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

1742 totalN = nImage.array[yIdxLocal, xIdxLocal] 

1743 

1744 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

1745 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

1746 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

1747 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

1748 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

1749 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

1750 & (outlierN <= effectiveMaxNumEpochs)) 

1751 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

1752 if percentBelowThreshold > self.config.spatialThreshold: 

1753 maskSpanSetList.append(span) 

1754 

1755 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

1756 # If a candidate is contained by a footprint on the template coadd, do not clip 

1757 filteredMaskSpanSetList = [] 

1758 for span in maskSpanSetList: 

1759 doKeep = True 

1760 for footprint in footprintsToExclude.positive.getFootprints(): 

1761 if footprint.spans.contains(span): 

1762 doKeep = False 

1763 break 

1764 if doKeep: 

1765 filteredMaskSpanSetList.append(span) 

1766 maskSpanSetList = filteredMaskSpanSetList 

1767 

1768 return maskSpanSetList 

1769 

1770 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

1771 """Fetch a warp from the butler and return a warpDiff. 

1772 

1773 Parameters 

1774 ---------- 

1775 warpRef : `lsst.daf.butler.DeferredDatasetHandle` 

1776 Handle for the warp. 

1777 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

1778 An image scaler object. 

1779 templateCoadd : `lsst.afw.image.Exposure` 

1780 Exposure to be substracted from the scaled warp. 

1781 

1782 Returns 

1783 ------- 

1784 warp : `lsst.afw.image.Exposure` 

1785 Exposure of the image difference between the warp and template. 

1786 """ 

1787 # If the PSF-Matched warp did not exist for this direct warp 

1788 # None is holding its place to maintain order in Gen 3 

1789 if warpRef is None: 

1790 return None 

1791 

1792 warp = warpRef.get() 

1793 # direct image scaler OK for PSF-matched Warp 

1794 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

1795 mi = warp.getMaskedImage() 

1796 if self.config.doScaleWarpVariance: 

1797 try: 

1798 self.scaleWarpVariance.run(mi) 

1799 except Exception as exc: 

1800 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

1801 mi -= templateCoadd.getMaskedImage() 

1802 return warp