Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 15%

643 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-04 02:22 -0800

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

23 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

24 

25import copy 

26import numpy 

27import warnings 

28import logging 

29import lsst.pex.config as pexConfig 

30import lsst.pex.exceptions as pexExceptions 

31import lsst.geom as geom 

32import lsst.afw.geom as afwGeom 

33import lsst.afw.image as afwImage 

34import lsst.afw.math as afwMath 

35import lsst.afw.table as afwTable 

36import lsst.coadd.utils as coaddUtils 

37import lsst.pipe.base as pipeBase 

38import lsst.meas.algorithms as measAlg 

39import lsstDebug 

40import lsst.utils as utils 

41from lsst.skymap import BaseSkyMap 

42from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList 

43from .interpImage import InterpImageTask 

44from .scaleZeroPoint import ScaleZeroPointTask 

45from .maskStreaks import MaskStreaksTask 

46from .healSparseMapping import HealSparseInputMapTask 

47from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask 

48from lsst.utils.timer import timeMethod 

49from deprecated.sphinx import deprecated 

50 

51log = logging.getLogger(__name__) 

52 

53 

54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

55 dimensions=("tract", "patch", "band", "skymap"), 

56 defaultTemplates={"inputCoaddName": "deep", 

57 "outputCoaddName": "deep", 

58 "warpType": "direct", 

59 "warpTypeSuffix": ""}): 

60 

61 inputWarps = pipeBase.connectionTypes.Input( 

62 doc=("Input list of warps to be assemebled i.e. stacked." 

63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

64 name="{inputCoaddName}Coadd_{warpType}Warp", 

65 storageClass="ExposureF", 

66 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

67 deferLoad=True, 

68 multiple=True 

69 ) 

70 skyMap = pipeBase.connectionTypes.Input( 

71 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

73 storageClass="SkyMap", 

74 dimensions=("skymap", ), 

75 ) 

76 selectedVisits = pipeBase.connectionTypes.Input( 

77 doc="Selected visits to be coadded.", 

78 name="{outputCoaddName}Visits", 

79 storageClass="StructuredDataDict", 

80 dimensions=("instrument", "tract", "patch", "skymap", "band") 

81 ) 

82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

83 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

84 " BRIGHT_OBJECT."), 

85 name="brightObjectMask", 

86 storageClass="ObjectMaskCatalog", 

87 dimensions=("tract", "patch", "skymap", "band"), 

88 ) 

89 coaddExposure = pipeBase.connectionTypes.Output( 

90 doc="Output coadded exposure, produced by stacking input warps", 

91 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

92 storageClass="ExposureF", 

93 dimensions=("tract", "patch", "skymap", "band"), 

94 ) 

95 nImage = pipeBase.connectionTypes.Output( 

96 doc="Output image of number of input images per pixel", 

97 name="{outputCoaddName}Coadd_nImage", 

98 storageClass="ImageU", 

99 dimensions=("tract", "patch", "skymap", "band"), 

100 ) 

101 inputMap = pipeBase.connectionTypes.Output( 

102 doc="Output healsparse map of input images", 

103 name="{outputCoaddName}Coadd_inputMap", 

104 storageClass="HealSparseMap", 

105 dimensions=("tract", "patch", "skymap", "band"), 

106 ) 

107 

108 def __init__(self, *, config=None): 

109 super().__init__(config=config) 

110 

111 if not config.doMaskBrightObjects: 

112 self.prerequisiteInputs.remove("brightObjectMask") 

113 

114 if not config.doSelectVisits: 

115 self.inputs.remove("selectedVisits") 

116 

117 if not config.doNImage: 

118 self.outputs.remove("nImage") 

119 

120 if not self.config.doInputMap: 

121 self.outputs.remove("inputMap") 

122 

123 

124class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

125 pipelineConnections=AssembleCoaddConnections): 

126 warpType = pexConfig.Field( 

127 doc="Warp name: one of 'direct' or 'psfMatched'", 

128 dtype=str, 

129 default="direct", 

130 ) 

131 subregionSize = pexConfig.ListField( 

132 dtype=int, 

133 doc="Width, height of stack subregion size; " 

134 "make small enough that a full stack of images will fit into memory at once.", 

135 length=2, 

136 default=(2000, 2000), 

137 ) 

138 statistic = pexConfig.Field( 

139 dtype=str, 

140 doc="Main stacking statistic for aggregating over the epochs.", 

141 default="MEANCLIP", 

142 ) 

143 doOnlineForMean = pexConfig.Field( 

144 dtype=bool, 

145 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

146 default=False, 

147 ) 

148 doSigmaClip = pexConfig.Field( 

149 dtype=bool, 

150 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

151 default=False, 

152 ) 

153 sigmaClip = pexConfig.Field( 

154 dtype=float, 

155 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

156 default=3.0, 

157 ) 

158 clipIter = pexConfig.Field( 

159 dtype=int, 

160 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

161 default=2, 

162 ) 

163 calcErrorFromInputVariance = pexConfig.Field( 

164 dtype=bool, 

165 doc="Calculate coadd variance from input variance by stacking statistic." 

166 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

167 default=True, 

168 ) 

169 scaleZeroPoint = pexConfig.ConfigurableField( 

170 target=ScaleZeroPointTask, 

171 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

172 ) 

173 doInterp = pexConfig.Field( 

174 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

175 dtype=bool, 

176 default=True, 

177 ) 

178 interpImage = pexConfig.ConfigurableField( 

179 target=InterpImageTask, 

180 doc="Task to interpolate (and extrapolate) over NaN pixels", 

181 ) 

182 doWrite = pexConfig.Field( 

183 doc="Persist coadd?", 

184 dtype=bool, 

185 default=True, 

186 ) 

187 doNImage = pexConfig.Field( 

188 doc="Create image of number of contributing exposures for each pixel", 

189 dtype=bool, 

190 default=False, 

191 ) 

192 doUsePsfMatchedPolygons = pexConfig.Field( 

193 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

194 dtype=bool, 

195 default=False, 

196 ) 

197 maskPropagationThresholds = pexConfig.DictField( 

198 keytype=str, 

199 itemtype=float, 

200 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

201 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

202 "would have contributed exceeds this value."), 

203 default={"SAT": 0.1}, 

204 ) 

205 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

206 doc="Mask planes to remove before coadding") 

207 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

208 doc="Set mask and flag bits for bright objects?") 

209 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

210 doc="Name of mask bit used for bright objects") 

211 coaddPsf = pexConfig.ConfigField( 

212 doc="Configuration for CoaddPsf", 

213 dtype=measAlg.CoaddPsfConfig, 

214 ) 

215 doAttachTransmissionCurve = pexConfig.Field( 

216 dtype=bool, default=False, optional=False, 

217 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

218 "(requires all input Exposures to have TransmissionCurves).") 

219 ) 

220 hasFakes = pexConfig.Field( 

221 dtype=bool, 

222 default=False, 

223 doc="Should be set to True if fake sources have been inserted into the input data." 

224 ) 

225 doSelectVisits = pexConfig.Field( 

226 doc="Coadd only visits selected by a SelectVisitsTask", 

227 dtype=bool, 

228 default=False, 

229 ) 

230 doInputMap = pexConfig.Field( 

231 doc="Create a bitwise map of coadd inputs", 

232 dtype=bool, 

233 default=False, 

234 ) 

235 inputMapper = pexConfig.ConfigurableField( 

236 doc="Input map creation subtask.", 

237 target=HealSparseInputMapTask, 

238 ) 

239 

240 def setDefaults(self): 

241 super().setDefaults() 

242 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

243 

244 def validate(self): 

245 super().validate() 

246 if self.doPsfMatch: 

247 # Backwards compatibility. 

248 # Configs do not have loggers 

249 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

250 self.warpType = 'psfMatched' 

251 if self.doSigmaClip and self.statistic != "MEANCLIP": 

252 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

253 self.statistic = "MEANCLIP" 

254 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

255 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

256 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

257 

258 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

259 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

260 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

261 if str(k) not in unstackableStats] 

262 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

263 % (self.statistic, stackableStats)) 

264 

265 

266class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

267 """Assemble a coadded image from a set of warps. 

268 

269 Each Warp that goes into a coadd will typically have an independent 

270 photometric zero-point. Therefore, we must scale each Warp to set it to 

271 a common photometric zeropoint. WarpType may be one of 'direct' or 

272 'psfMatched', and the boolean configs `config.makeDirect` and 

273 `config.makePsfMatched` set which of the warp types will be coadded. 

274 The coadd is computed as a mean with optional outlier rejection. 

275 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

276 Finally, Warps can have bad 'NaN' pixels which received no input from the 

277 source calExps. We interpolate over these bad (NaN) pixels. 

278 

279 `AssembleCoaddTask` uses several sub-tasks. These are 

280 

281 - `~lsst.pipe.tasks.ScaleZeroPointTask` 

282 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

283 - `~lsst.pipe.tasks.InterpImageTask` 

284 - interpolate across bad pixels (NaN) in the final coadd 

285 

286 You can retarget these subtasks if you wish. 

287 

288 Raises 

289 ------ 

290 RuntimeError 

291 Raised if unable to define mask plane for bright objects. 

292 

293 Notes 

294 ----- 

295 Debugging: 

296 `AssembleCoaddTask` has no debug variables of its own. Some of the 

297 subtasks may support `~lsst.base.lsstDebug` variables. See the 

298 documentation for the subtasks for further information. 

299 

300 Examples 

301 -------- 

302 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

303 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

304 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

305 inputs: a data reference to the tract patch and filter to be coadded, and 

306 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

307 ``--selectId``, respectively: 

308 

309 .. code-block:: none 

310 

311 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

312 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

313 

314 Only the Warps that cover the specified tract and patch will be coadded. 

315 A list of the available optional arguments can be obtained by calling 

316 ``assembleCoadd.py`` with the ``--help`` command line argument: 

317 

318 .. code-block:: none 

319 

320 assembleCoadd.py --help 

321 

322 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

323 multi-band processing, we will generate the HSC-I & -R band coadds from 

324 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

325 assuming that the lsst stack has been already set up, we must set up the 

326 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

327 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

328 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

329 coadds, we must first run: 

330 

331 - processCcd 

332 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

333 - makeSkyMap 

334 - create a skymap that covers the area of the sky present in the raw exposures 

335 - makeCoaddTempExp 

336 - warp the individual calibrated exposures to the tangent plane of the coadd 

337 

338 We can perform all of these steps by running 

339 

340 .. code-block:: none 

341 

342 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

343 

344 This will produce warped exposures for each visit. To coadd the warped 

345 data, we call assembleCoadd.py as follows: 

346 

347 .. code-block:: none 

348 

349 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

350 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

351 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

352 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

353 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

354 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

355 --selectId visit=903988 ccd=24 

356 

357 that will process the HSC-I band data. The results are written in 

358 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

359 

360 You may also choose to run: 

361 

362 .. code-block:: none 

363 

364 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

365 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

366 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

367 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

368 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

369 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

370 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

371 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

372 

373 to generate the coadd for the HSC-R band if you are interested in 

374 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

375 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

376 rather than `AssembleCoaddTask` to make the coadd. 

377 """ 

378 

379 ConfigClass = AssembleCoaddConfig 

380 _DefaultName = "assembleCoadd" 

381 

382 def __init__(self, *args, **kwargs): 

383 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

384 if args: 

385 argNames = ["config", "name", "parentTask", "log"] 

386 kwargs.update({k: v for k, v in zip(argNames, args)}) 

387 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

388 "PipelineTask will not take positional args" % argNames, FutureWarning) 

389 

390 super().__init__(**kwargs) 

391 self.makeSubtask("interpImage") 

392 self.makeSubtask("scaleZeroPoint") 

393 

394 if self.config.doMaskBrightObjects: 

395 mask = afwImage.Mask() 

396 try: 

397 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

398 except pexExceptions.LsstCppException: 

399 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

400 mask.getMaskPlaneDict().keys()) 

401 del mask 

402 

403 if self.config.doInputMap: 

404 self.makeSubtask("inputMapper") 

405 

406 self.warpType = self.config.warpType 

407 

408 @utils.inheritDoc(pipeBase.PipelineTask) 

409 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

410 inputData = butlerQC.get(inputRefs) 

411 

412 # Construct skyInfo expected by run 

413 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it 

414 skyMap = inputData["skyMap"] 

415 outputDataId = butlerQC.quantum.dataId 

416 

417 inputData['skyInfo'] = makeSkyInfo(skyMap, 

418 tractId=outputDataId['tract'], 

419 patchId=outputDataId['patch']) 

420 

421 if self.config.doSelectVisits: 

422 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

423 else: 

424 warpRefList = inputData['inputWarps'] 

425 

426 inputs = self.prepareInputs(warpRefList) 

427 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

428 self.getTempExpDatasetName(self.warpType)) 

429 if len(inputs.tempExpRefList) == 0: 

430 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

431 

432 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

433 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

434 inputs.weightList, supplementaryData=supplementaryData) 

435 

436 inputData.setdefault('brightObjectMask', None) 

437 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

438 

439 if self.config.doWrite: 

440 butlerQC.put(retStruct, outputRefs) 

441 return retStruct 

442 

443 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

444 """Interpolate over missing data and mask bright stars. 

445 

446 Parameters 

447 ---------- 

448 coaddExposure : `lsst.afw.image.Exposure` 

449 The coadded exposure to process. 

450 brightObjectMasks : `lsst.afw.table` or `None`, optional 

451 Table of bright objects to mask. 

452 dataId : `lsst.daf.butler.DataId` or `None`, optional 

453 Data identification. 

454 """ 

455 if self.config.doInterp: 

456 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

457 # The variance must be positive; work around for DM-3201. 

458 varArray = coaddExposure.variance.array 

459 with numpy.errstate(invalid="ignore"): 

460 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

461 

462 if self.config.doMaskBrightObjects: 

463 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

464 

465 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

466 """Make additional inputs to run() specific to subclasses (Gen3). 

467 

468 Duplicates interface of `runQuantum` method. 

469 Available to be implemented by subclasses only if they need the 

470 coadd dataRef for performing preliminary processing before 

471 assembling the coadd. 

472 

473 Parameters 

474 ---------- 

475 butlerQC : `~lsst.pipe.base.ButlerQuantumContext` 

476 Gen3 Butler object for fetching additional data products before 

477 running the Task specialized for quantum being processed. 

478 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

479 Attributes are the names of the connections describing input dataset types. 

480 Values are DatasetRefs that task consumes for corresponding dataset type. 

481 DataIds are guaranteed to match data objects in ``inputData``. 

482 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

483 Attributes are the names of the connections describing output dataset types. 

484 Values are DatasetRefs that task is to produce 

485 for corresponding dataset type. 

486 """ 

487 return pipeBase.Struct() 

488 

489 @deprecated( 

490 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData", 

491 version="v25.0", 

492 category=FutureWarning 

493 ) 

494 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

495 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs) 

496 

497 def prepareInputs(self, refList): 

498 """Prepare the input warps for coaddition by measuring the weight for 

499 each warp and the scaling for the photometric zero point. 

500 

501 Each Warp has its own photometric zeropoint and background variance. 

502 Before coadding these Warps together, compute a scale factor to 

503 normalize the photometric zeropoint and compute the weight for each Warp. 

504 

505 Parameters 

506 ---------- 

507 refList : `list` 

508 List of data references to tempExp. 

509 

510 Returns 

511 ------- 

512 result : `~lsst.pipe.base.Struct` 

513 Results as a struct with attributes: 

514 

515 ``tempExprefList`` 

516 `list` of data references to tempExp. 

517 ``weightList`` 

518 `list` of weightings. 

519 ``imageScalerList`` 

520 `list` of image scalers. 

521 """ 

522 statsCtrl = afwMath.StatisticsControl() 

523 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

524 statsCtrl.setNumIter(self.config.clipIter) 

525 statsCtrl.setAndMask(self.getBadPixelMask()) 

526 statsCtrl.setNanSafe(True) 

527 # compute tempExpRefList: a list of tempExpRef that actually exist 

528 # and weightList: a list of the weight of the associated coadd tempExp 

529 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

530 tempExpRefList = [] 

531 weightList = [] 

532 imageScalerList = [] 

533 tempExpName = self.getTempExpDatasetName(self.warpType) 

534 for tempExpRef in refList: 

535 tempExp = tempExpRef.get() 

536 # Ignore any input warp that is empty of data 

537 if numpy.isnan(tempExp.image.array).all(): 

538 continue 

539 maskedImage = tempExp.getMaskedImage() 

540 imageScaler = self.scaleZeroPoint.computeImageScaler( 

541 exposure=tempExp, 

542 dataRef=tempExpRef, # FIXME 

543 ) 

544 try: 

545 imageScaler.scaleMaskedImage(maskedImage) 

546 except Exception as e: 

547 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

548 continue 

549 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

550 afwMath.MEANCLIP, statsCtrl) 

551 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

552 weight = 1.0 / float(meanVar) 

553 if not numpy.isfinite(weight): 

554 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

555 continue 

556 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

557 

558 del maskedImage 

559 del tempExp 

560 

561 tempExpRefList.append(tempExpRef) 

562 weightList.append(weight) 

563 imageScalerList.append(imageScaler) 

564 

565 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

566 imageScalerList=imageScalerList) 

567 

568 def prepareStats(self, mask=None): 

569 """Prepare the statistics for coadding images. 

570 

571 Parameters 

572 ---------- 

573 mask : `int`, optional 

574 Bit mask value to exclude from coaddition. 

575 

576 Returns 

577 ------- 

578 stats : `~lsst.pipe.base.Struct` 

579 Statistics as a struct with attributes: 

580 

581 ``statsCtrl`` 

582 Statistics control object for coadd (`~lsst.afw.math.StatisticsControl`). 

583 ``statsFlags`` 

584 Statistic for coadd (`~lsst.afw.math.Property`). 

585 """ 

586 if mask is None: 

587 mask = self.getBadPixelMask() 

588 statsCtrl = afwMath.StatisticsControl() 

589 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

590 statsCtrl.setNumIter(self.config.clipIter) 

591 statsCtrl.setAndMask(mask) 

592 statsCtrl.setNanSafe(True) 

593 statsCtrl.setWeighted(True) 

594 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

595 for plane, threshold in self.config.maskPropagationThresholds.items(): 

596 bit = afwImage.Mask.getMaskPlane(plane) 

597 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

598 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

599 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

600 

601 @timeMethod 

602 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

603 altMaskList=None, mask=None, supplementaryData=None): 

604 """Assemble a coadd from input warps. 

605 

606 Assemble the coadd using the provided list of coaddTempExps. Since 

607 the full coadd covers a patch (a large area), the assembly is 

608 performed over small areas on the image at a time in order to 

609 conserve memory usage. Iterate over subregions within the outer 

610 bbox of the patch using `assembleSubregion` to stack the corresponding 

611 subregions from the coaddTempExps with the statistic specified. 

612 Set the edge bits the coadd mask based on the weight map. 

613 

614 Parameters 

615 ---------- 

616 skyInfo : `~lsst.pipe.base.Struct` 

617 Struct with geometric information about the patch. 

618 tempExpRefList : `list` 

619 List of data references to Warps (previously called CoaddTempExps). 

620 imageScalerList : `list` 

621 List of image scalers. 

622 weightList : `list` 

623 List of weights. 

624 altMaskList : `list`, optional 

625 List of alternate masks to use rather than those stored with 

626 tempExp. 

627 mask : `int`, optional 

628 Bit mask value to exclude from coaddition. 

629 supplementaryData : `~lsst.pipe.base.Struct`, optional 

630 Struct with additional data products needed to assemble coadd. 

631 Only used by subclasses that implement ``_makeSupplementaryData`` 

632 and override `run`. 

633 

634 Returns 

635 ------- 

636 result : `~lsst.pipe.base.Struct` 

637 Results as a struct with attributes: 

638 

639 ``coaddExposure`` 

640 Coadded exposure (``lsst.afw.image.Exposure``). 

641 ``nImage`` 

642 Exposure count image (``lsst.afw.image.Image``), if requested. 

643 ``inputMap`` 

644 Bit-wise map of inputs, if requested. 

645 ``warpRefList`` 

646 Input list of refs to the warps (``lsst.daf.butler.DeferredDatasetHandle``) 

647 (unmodified). 

648 ``imageScalerList`` 

649 Input list of image scalers (`list`) (unmodified). 

650 ``weightList`` 

651 Input list of weights (`list`) (unmodified). 

652 """ 

653 tempExpName = self.getTempExpDatasetName(self.warpType) 

654 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

655 stats = self.prepareStats(mask=mask) 

656 

657 if altMaskList is None: 

658 altMaskList = [None]*len(tempExpRefList) 

659 

660 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

661 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

662 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

663 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

664 coaddMaskedImage = coaddExposure.getMaskedImage() 

665 subregionSizeArr = self.config.subregionSize 

666 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

667 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

668 if self.config.doNImage: 

669 nImage = afwImage.ImageU(skyInfo.bbox) 

670 else: 

671 nImage = None 

672 # If inputMap is requested, create the initial version that can be masked in 

673 # assembleSubregion. 

674 if self.config.doInputMap: 

675 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

676 skyInfo.wcs, 

677 coaddExposure.getInfo().getCoaddInputs().ccds) 

678 

679 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 

680 try: 

681 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

682 weightList, altMaskList, stats.ctrl, 

683 nImage=nImage) 

684 except Exception as e: 

685 self.log.exception("Cannot compute online coadd %s", e) 

686 raise 

687 else: 

688 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

689 try: 

690 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

691 weightList, altMaskList, stats.flags, stats.ctrl, 

692 nImage=nImage) 

693 except Exception as e: 

694 self.log.exception("Cannot compute coadd %s: %s", subBBox, e) 

695 raise 

696 

697 # If inputMap is requested, we must finalize the map after the accumulation. 

698 if self.config.doInputMap: 

699 self.inputMapper.finalize_ccd_input_map_mask() 

700 inputMap = self.inputMapper.ccd_input_map 

701 else: 

702 inputMap = None 

703 

704 self.setInexactPsf(coaddMaskedImage.getMask()) 

705 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

706 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

707 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

708 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

709 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

710 weightList=weightList, inputMap=inputMap) 

711 

712 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

713 """Set the metadata for the coadd. 

714 

715 This basic implementation sets the filter from the first input. 

716 

717 Parameters 

718 ---------- 

719 coaddExposure : `lsst.afw.image.Exposure` 

720 The target exposure for the coadd. 

721 tempExpRefList : `list` 

722 List of data references to tempExp. 

723 weightList : `list` 

724 List of weights. 

725 

726 Raises 

727 ------ 

728 AssertionError 

729 Raised if there is a length mismatch. 

730 """ 

731 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

732 

733 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

734 # (and we need more than just the PropertySet that contains the header), which is not possible 

735 # with the current butler (see #2777). 

736 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

737 

738 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

739 

740 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

741 

742 # Set the coadd FilterLabel to the band of the first input exposure: 

743 # Coadds are calibrated, so the physical label is now meaningless. 

744 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel)) 

745 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

746 coaddInputs.ccds.reserve(numCcds) 

747 coaddInputs.visits.reserve(len(tempExpList)) 

748 

749 for tempExp, weight in zip(tempExpList, weightList): 

750 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

751 

752 if self.config.doUsePsfMatchedPolygons: 

753 self.shrinkValidPolygons(coaddInputs) 

754 

755 coaddInputs.visits.sort() 

756 coaddInputs.ccds.sort() 

757 if self.warpType == "psfMatched": 

758 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

759 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

760 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

761 # having the maximum width (sufficient because square) 

762 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

763 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

764 for modelPsf in modelPsfList] 

765 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

766 else: 

767 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

768 self.config.coaddPsf.makeControl()) 

769 coaddExposure.setPsf(psf) 

770 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

771 coaddExposure.getWcs()) 

772 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

773 if self.config.doAttachTransmissionCurve: 

774 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

775 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

776 

777 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

778 altMaskList, statsFlags, statsCtrl, nImage=None): 

779 """Assemble the coadd for a sub-region. 

780 

781 For each coaddTempExp, check for (and swap in) an alternative mask 

782 if one is passed. Remove mask planes listed in 

783 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

784 `lsst.afw.math.statisticsStack` with the statistic specified by 

785 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

786 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

787 an N-sigma clipped mean where N and iterations are specified by 

788 statsCtrl. Assign the stacked subregion back to the coadd. 

789 

790 Parameters 

791 ---------- 

792 coaddExposure : `lsst.afw.image.Exposure` 

793 The target exposure for the coadd. 

794 bbox : `lsst.geom.Box` 

795 Sub-region to coadd. 

796 tempExpRefList : `list` 

797 List of data reference to tempExp. 

798 imageScalerList : `list` 

799 List of image scalers. 

800 weightList : `list` 

801 List of weights. 

802 altMaskList : `list` 

803 List of alternate masks to use rather than those stored with 

804 tempExp, or None. Each element is dict with keys = mask plane 

805 name to which to add the spans. 

806 statsFlags : `lsst.afw.math.Property` 

807 Property object for statistic for coadd. 

808 statsCtrl : `lsst.afw.math.StatisticsControl` 

809 Statistics control object for coadd. 

810 nImage : `lsst.afw.image.ImageU`, optional 

811 Keeps track of exposure count for each pixel. 

812 """ 

813 self.log.debug("Computing coadd over %s", bbox) 

814 

815 coaddExposure.mask.addMaskPlane("REJECTED") 

816 coaddExposure.mask.addMaskPlane("CLIPPED") 

817 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

818 maskMap = self.setRejectedMaskMapping(statsCtrl) 

819 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

820 maskedImageList = [] 

821 if nImage is not None: 

822 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

823 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

824 

825 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

826 

827 maskedImage = exposure.getMaskedImage() 

828 mask = maskedImage.getMask() 

829 if altMask is not None: 

830 self.applyAltMaskPlanes(mask, altMask) 

831 imageScaler.scaleMaskedImage(maskedImage) 

832 

833 # Add 1 for each pixel which is not excluded by the exclude mask. 

834 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

835 if nImage is not None: 

836 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

837 if self.config.removeMaskPlanes: 

838 self.removeMaskPlanes(maskedImage) 

839 maskedImageList.append(maskedImage) 

840 

841 if self.config.doInputMap: 

842 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

843 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

844 

845 with self.timer("stack"): 

846 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

847 clipped, # also set output to CLIPPED if sigma-clipped 

848 maskMap) 

849 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

850 if nImage is not None: 

851 nImage.assign(subNImage, bbox) 

852 

853 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

854 altMaskList, statsCtrl, nImage=None): 

855 """Assemble the coadd using the "online" method. 

856 

857 This method takes a running sum of images and weights to save memory. 

858 It only works for MEAN statistics. 

859 

860 Parameters 

861 ---------- 

862 coaddExposure : `lsst.afw.image.Exposure` 

863 The target exposure for the coadd. 

864 tempExpRefList : `list` 

865 List of data reference to tempExp. 

866 imageScalerList : `list` 

867 List of image scalers. 

868 weightList : `list` 

869 List of weights. 

870 altMaskList : `list` 

871 List of alternate masks to use rather than those stored with 

872 tempExp, or None. Each element is dict with keys = mask plane 

873 name to which to add the spans. 

874 statsCtrl : `lsst.afw.math.StatisticsControl` 

875 Statistics control object for coadd. 

876 nImage : `lsst.afw.image.ImageU`, optional 

877 Keeps track of exposure count for each pixel. 

878 """ 

879 self.log.debug("Computing online coadd.") 

880 

881 coaddExposure.mask.addMaskPlane("REJECTED") 

882 coaddExposure.mask.addMaskPlane("CLIPPED") 

883 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

884 maskMap = self.setRejectedMaskMapping(statsCtrl) 

885 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

886 

887 bbox = coaddExposure.maskedImage.getBBox() 

888 

889 stacker = AccumulatorMeanStack( 

890 coaddExposure.image.array.shape, 

891 statsCtrl.getAndMask(), 

892 mask_threshold_dict=thresholdDict, 

893 mask_map=maskMap, 

894 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

895 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

896 compute_n_image=(nImage is not None) 

897 ) 

898 

899 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

900 imageScalerList, 

901 altMaskList, 

902 weightList): 

903 exposure = tempExpRef.get() 

904 maskedImage = exposure.getMaskedImage() 

905 mask = maskedImage.getMask() 

906 if altMask is not None: 

907 self.applyAltMaskPlanes(mask, altMask) 

908 imageScaler.scaleMaskedImage(maskedImage) 

909 if self.config.removeMaskPlanes: 

910 self.removeMaskPlanes(maskedImage) 

911 

912 stacker.add_masked_image(maskedImage, weight=weight) 

913 

914 if self.config.doInputMap: 

915 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

916 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

917 

918 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

919 

920 if nImage is not None: 

921 nImage.array[:, :] = stacker.n_image 

922 

923 def removeMaskPlanes(self, maskedImage): 

924 """Unset the mask of an image for mask planes specified in the config. 

925 

926 Parameters 

927 ---------- 

928 maskedImage : `lsst.afw.image.MaskedImage` 

929 The masked image to be modified. 

930 

931 Raises 

932 ------ 

933 InvalidParameterError 

934 Raised if no mask plane with that name was found. 

935 """ 

936 mask = maskedImage.getMask() 

937 for maskPlane in self.config.removeMaskPlanes: 

938 try: 

939 mask &= ~mask.getPlaneBitMask(maskPlane) 

940 except pexExceptions.InvalidParameterError: 

941 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

942 maskPlane) 

943 

944 @staticmethod 

945 def setRejectedMaskMapping(statsCtrl): 

946 """Map certain mask planes of the warps to new planes for the coadd. 

947 

948 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

949 or CLIPPED, set it to REJECTED on the coadd. 

950 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

951 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

952 

953 Parameters 

954 ---------- 

955 statsCtrl : `lsst.afw.math.StatisticsControl` 

956 Statistics control object for coadd. 

957 

958 Returns 

959 ------- 

960 maskMap : `list` of `tuple` of `int` 

961 A list of mappings of mask planes of the warped exposures to 

962 mask planes of the coadd. 

963 """ 

964 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

965 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

966 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

967 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

968 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

969 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

970 (clipped, clipped)] 

971 return maskMap 

972 

973 def applyAltMaskPlanes(self, mask, altMaskSpans): 

974 """Apply in place alt mask formatted as SpanSets to a mask. 

975 

976 Parameters 

977 ---------- 

978 mask : `lsst.afw.image.Mask` 

979 Original mask. 

980 altMaskSpans : `dict` 

981 SpanSet lists to apply. Each element contains the new mask 

982 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

983 and list of SpanSets to apply to the mask. 

984 

985 Returns 

986 ------- 

987 mask : `lsst.afw.image.Mask` 

988 Updated mask. 

989 """ 

990 if self.config.doUsePsfMatchedPolygons: 

991 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

992 # Clear away any other masks outside the validPolygons. These pixels are no longer 

993 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

994 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

995 # This mask-clearing step must only occur *before* applying the new masks below 

996 for spanSet in altMaskSpans['NO_DATA']: 

997 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

998 

999 for plane, spanSetList in altMaskSpans.items(): 

1000 maskClipValue = mask.addMaskPlane(plane) 

1001 for spanSet in spanSetList: 

1002 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1003 return mask 

1004 

1005 def shrinkValidPolygons(self, coaddInputs): 

1006 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1007 

1008 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1009 does not have a validPolygon, create one from its bbox. 

1010 

1011 Parameters 

1012 ---------- 

1013 coaddInputs : `lsst.afw.image.coaddInputs` 

1014 Original mask. 

1015 """ 

1016 for ccd in coaddInputs.ccds: 

1017 polyOrig = ccd.getValidPolygon() 

1018 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1019 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1020 if polyOrig: 

1021 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1022 else: 

1023 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1024 ccd.setValidPolygon(validPolygon) 

1025 

1026 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1027 """Set the bright object masks. 

1028 

1029 Parameters 

1030 ---------- 

1031 exposure : `lsst.afw.image.Exposure` 

1032 Exposure under consideration. 

1033 brightObjectMasks : `lsst.afw.table` 

1034 Table of bright objects to mask. 

1035 dataId : `lsst.daf.butler.DataId`, optional 

1036 Data identifier dict for patch. 

1037 """ 

1038 if brightObjectMasks is None: 

1039 self.log.warning("Unable to apply bright object mask: none supplied") 

1040 return 

1041 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1042 mask = exposure.getMaskedImage().getMask() 

1043 wcs = exposure.getWcs() 

1044 plateScale = wcs.getPixelScale().asArcseconds() 

1045 

1046 for rec in brightObjectMasks: 

1047 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1048 if rec["type"] == "box": 

1049 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1050 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1051 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1052 

1053 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1054 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1055 

1056 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1057 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1058 spans = afwGeom.SpanSet(bbox) 

1059 elif rec["type"] == "circle": 

1060 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1061 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1062 else: 

1063 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1064 continue 

1065 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1066 

1067 def setInexactPsf(self, mask): 

1068 """Set INEXACT_PSF mask plane. 

1069 

1070 If any of the input images isn't represented in the coadd (due to 

1071 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1072 these pixels. 

1073 

1074 Parameters 

1075 ---------- 

1076 mask : `lsst.afw.image.Mask` 

1077 Coadded exposure's mask, modified in-place. 

1078 """ 

1079 mask.addMaskPlane("INEXACT_PSF") 

1080 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1081 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1082 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1083 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1084 array = mask.getArray() 

1085 selected = array & (sensorEdge | clipped | rejected) > 0 

1086 array[selected] |= inexactPsf 

1087 

1088 @staticmethod 

1089 def _subBBoxIter(bbox, subregionSize): 

1090 """Iterate over subregions of a bbox. 

1091 

1092 Parameters 

1093 ---------- 

1094 bbox : `lsst.geom.Box2I` 

1095 Bounding box over which to iterate. 

1096 subregionSize : `lsst.geom.Extent2I` 

1097 Size of sub-bboxes. 

1098 

1099 Yields 

1100 ------ 

1101 subBBox : `lsst.geom.Box2I` 

1102 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1103 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1104 the edges of ``bbox``, but it will never be empty. 

1105 

1106 Raises 

1107 ------ 

1108 RuntimeError 

1109 Raised if any of the following occur: 

1110 - The given bbox is empty. 

1111 - The subregionSize is 0. 

1112 """ 

1113 if bbox.isEmpty(): 

1114 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1115 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1116 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1117 

1118 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1119 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1120 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1121 subBBox.clip(bbox) 

1122 if subBBox.isEmpty(): 

1123 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1124 "colShift=%s, rowShift=%s" % 

1125 (bbox, subregionSize, colShift, rowShift)) 

1126 yield subBBox 

1127 

1128 def filterWarps(self, inputs, goodVisits): 

1129 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit. 

1130 

1131 Parameters 

1132 ---------- 

1133 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef` 

1134 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit. 

1135 goodVisit : `dict` 

1136 Dictionary with good visitIds as the keys. Value ignored. 

1137 

1138 Returns 

1139 ------- 

1140 filteredInputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef` 

1141 Filtered and sorted list of inputRefs with visitId in goodVisits ordered by goodVisit. 

1142 """ 

1143 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1144 filteredInputs = [] 

1145 for visit in goodVisits.keys(): 

1146 if visit in inputWarpDict: 

1147 filteredInputs.append(inputWarpDict[visit]) 

1148 return filteredInputs 

1149 

1150 

1151def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1152 """Function to count the number of pixels with a specific mask in a 

1153 footprint. 

1154 

1155 Find the intersection of mask & footprint. Count all pixels in the mask 

1156 that are in the intersection that have bitmask set but do not have 

1157 ignoreMask set. Return the count. 

1158 

1159 Parameters 

1160 ---------- 

1161 mask : `lsst.afw.image.Mask` 

1162 Mask to define intersection region by. 

1163 footprint : `lsst.afw.detection.Footprint` 

1164 Footprint to define the intersection region by. 

1165 bitmask : `Unknown` 

1166 Specific mask that we wish to count the number of occurances of. 

1167 ignoreMask : `Unknown` 

1168 Pixels to not consider. 

1169 

1170 Returns 

1171 ------- 

1172 result : `int` 

1173 Number of pixels in footprint with specified mask. 

1174 """ 

1175 bbox = footprint.getBBox() 

1176 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1177 fp = afwImage.Mask(bbox) 

1178 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1179 footprint.spans.setMask(fp, bitmask) 

1180 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1181 (subMask.getArray() & ignoreMask) == 0).sum() 

1182 

1183 

1184class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1185 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1186 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1187 "Only PSF-Matched Warps make sense for image subtraction. " 

1188 "Therefore, they must be an additional declared input."), 

1189 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1190 storageClass="ExposureF", 

1191 dimensions=("tract", "patch", "skymap", "visit"), 

1192 deferLoad=True, 

1193 multiple=True 

1194 ) 

1195 templateCoadd = pipeBase.connectionTypes.Output( 

1196 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1197 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1198 name="{outputCoaddName}CoaddPsfMatched", 

1199 storageClass="ExposureF", 

1200 dimensions=("tract", "patch", "skymap", "band"), 

1201 ) 

1202 

1203 def __init__(self, *, config=None): 

1204 super().__init__(config=config) 

1205 if not config.assembleStaticSkyModel.doWrite: 

1206 self.outputs.remove("templateCoadd") 

1207 config.validate() 

1208 

1209 

1210class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1211 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1212 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1213 target=AssembleCoaddTask, 

1214 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1215 " naive/first-iteration model of the static sky.", 

1216 ) 

1217 detect = pexConfig.ConfigurableField( 

1218 target=SourceDetectionTask, 

1219 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1220 ) 

1221 detectTemplate = pexConfig.ConfigurableField( 

1222 target=SourceDetectionTask, 

1223 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1224 ) 

1225 maskStreaks = pexConfig.ConfigurableField( 

1226 target=MaskStreaksTask, 

1227 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1228 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1229 "streakMaskName" 

1230 ) 

1231 streakMaskName = pexConfig.Field( 

1232 dtype=str, 

1233 default="STREAK", 

1234 doc="Name of mask bit used for streaks" 

1235 ) 

1236 maxNumEpochs = pexConfig.Field( 

1237 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1238 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1239 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1240 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1241 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1242 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1243 "than transient and not masked.", 

1244 dtype=int, 

1245 default=2 

1246 ) 

1247 maxFractionEpochsLow = pexConfig.RangeField( 

1248 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1249 "Effective maxNumEpochs = " 

1250 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1251 dtype=float, 

1252 default=0.4, 

1253 min=0., max=1., 

1254 ) 

1255 maxFractionEpochsHigh = pexConfig.RangeField( 

1256 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1257 "Effective maxNumEpochs = " 

1258 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1259 dtype=float, 

1260 default=0.03, 

1261 min=0., max=1., 

1262 ) 

1263 spatialThreshold = pexConfig.RangeField( 

1264 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1265 "temporal criteria. If 0, clip all. If 1, clip none.", 

1266 dtype=float, 

1267 default=0.5, 

1268 min=0., max=1., 

1269 inclusiveMin=True, inclusiveMax=True 

1270 ) 

1271 doScaleWarpVariance = pexConfig.Field( 

1272 doc="Rescale Warp variance plane using empirical noise?", 

1273 dtype=bool, 

1274 default=True, 

1275 ) 

1276 scaleWarpVariance = pexConfig.ConfigurableField( 

1277 target=ScaleVarianceTask, 

1278 doc="Rescale variance on warps", 

1279 ) 

1280 doPreserveContainedBySource = pexConfig.Field( 

1281 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1282 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1283 dtype=bool, 

1284 default=True, 

1285 ) 

1286 doPrefilterArtifacts = pexConfig.Field( 

1287 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1288 "because they will be excluded anyway. This prevents them from contributing " 

1289 "to the outlier epoch count image and potentially being labeled as persistant." 

1290 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1291 dtype=bool, 

1292 default=True 

1293 ) 

1294 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1295 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1296 dtype=str, 

1297 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1298 ) 

1299 prefilterArtifactsRatio = pexConfig.Field( 

1300 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1301 dtype=float, 

1302 default=0.05 

1303 ) 

1304 doFilterMorphological = pexConfig.Field( 

1305 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1306 "be streaks.", 

1307 dtype=bool, 

1308 default=False 

1309 ) 

1310 growStreakFp = pexConfig.Field( 

1311 doc="Grow streak footprints by this number multiplied by the PSF width", 

1312 dtype=float, 

1313 default=5 

1314 ) 

1315 

1316 def setDefaults(self): 

1317 AssembleCoaddConfig.setDefaults(self) 

1318 self.statistic = 'MEAN' 

1319 self.doUsePsfMatchedPolygons = True 

1320 

1321 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1322 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1323 if "EDGE" in self.badMaskPlanes: 

1324 self.badMaskPlanes.remove('EDGE') 

1325 self.removeMaskPlanes.append('EDGE') 

1326 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1327 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1328 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1329 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1330 self.assembleStaticSkyModel.sigmaClip = 2.5 

1331 self.assembleStaticSkyModel.clipIter = 3 

1332 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1333 self.assembleStaticSkyModel.doWrite = False 

1334 self.detect.doTempLocalBackground = False 

1335 self.detect.reEstimateBackground = False 

1336 self.detect.returnOriginalFootprints = False 

1337 self.detect.thresholdPolarity = "both" 

1338 self.detect.thresholdValue = 5 

1339 self.detect.minPixels = 4 

1340 self.detect.isotropicGrow = True 

1341 self.detect.thresholdType = "pixel_stdev" 

1342 self.detect.nSigmaToGrow = 0.4 

1343 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1344 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1345 self.detectTemplate.nSigmaToGrow = 2.4 

1346 self.detectTemplate.doTempLocalBackground = False 

1347 self.detectTemplate.reEstimateBackground = False 

1348 self.detectTemplate.returnOriginalFootprints = False 

1349 

1350 def validate(self): 

1351 super().validate() 

1352 if self.assembleStaticSkyModel.doNImage: 

1353 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1354 "Please set assembleStaticSkyModel.doNImage=False") 

1355 

1356 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1357 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1358 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1359 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1360 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1361 

1362 

1363class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1364 """Assemble a compareWarp coadded image from a set of warps 

1365 by masking artifacts detected by comparing PSF-matched warps. 

1366 

1367 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1368 we clip outliers). The problem with doing this is that when computing the 

1369 coadd PSF at a given location, individual visit PSFs from visits with 

1370 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1371 In this task, we correct for this behavior by creating a new badMaskPlane 

1372 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1373 an artifact. We populate this plane on the input warps by comparing 

1374 PSF-matched warps with a PSF-matched median coadd which serves as a 

1375 model of the static sky. Any group of pixels that deviates from the 

1376 PSF-matched template coadd by more than config.detect.threshold sigma, 

1377 is an artifact candidate. The candidates are then filtered to remove 

1378 variable sources and sources that are difficult to subtract such as 

1379 bright stars. This filter is configured using the config parameters 

1380 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1381 the maximum fraction of epochs that the deviation can appear in and still 

1382 be considered an artifact. The spatialThreshold is the maximum fraction of 

1383 pixels in the footprint of the deviation that appear in other epochs 

1384 (where other epochs is defined by the temporalThreshold). If the deviant 

1385 region meets this criteria of having a significant percentage of pixels 

1386 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1387 set in the mask. These regions will not contribute to the final coadd. 

1388 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1389 of clipped regions. Note that the algorithm implemented by this task is 

1390 preliminary and works correctly for HSC data. Parameter modifications and 

1391 or considerable redesigning of the algorithm is likley required for other 

1392 surveys. 

1393 

1394 ``CompareWarpAssembleCoaddTask`` sub-classes 

1395 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1396 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1397 

1398 Notes 

1399 ----- 

1400 Debugging: 

1401 This task supports the following debug variables: 

1402 - ``saveCountIm`` 

1403 If True then save the Epoch Count Image as a fits file in the `figPath` 

1404 - ``figPath`` 

1405 Path to save the debug fits images and figures 

1406 """ 

1407 

1408 ConfigClass = CompareWarpAssembleCoaddConfig 

1409 _DefaultName = "compareWarpAssembleCoadd" 

1410 

1411 def __init__(self, *args, **kwargs): 

1412 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1413 self.makeSubtask("assembleStaticSkyModel") 

1414 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

1415 self.makeSubtask("detect", schema=detectionSchema) 

1416 if self.config.doPreserveContainedBySource: 

1417 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

1418 if self.config.doScaleWarpVariance: 

1419 self.makeSubtask("scaleWarpVariance") 

1420 if self.config.doFilterMorphological: 

1421 self.makeSubtask("maskStreaks") 

1422 

1423 @utils.inheritDoc(AssembleCoaddTask) 

1424 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs): 

1425 """Generate a templateCoadd to use as a naive model of static sky to 

1426 subtract from PSF-Matched warps. 

1427 

1428 Returns 

1429 ------- 

1430 result : `~lsst.pipe.base.Struct` 

1431 Results as a struct with attributes: 

1432 

1433 ``templateCoadd`` 

1434 Coadded exposure (`lsst.afw.image.Exposure`). 

1435 ``nImage`` 

1436 Keeps track of exposure count for each pixel (`lsst.afw.image.ImageU`). 

1437 

1438 Raises 

1439 ------ 

1440 RuntimeError 

1441 Raised if ``templateCoadd`` is `None`. 

1442 """ 

1443 # Ensure that psfMatchedWarps are used as input warps for template generation 

1444 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

1445 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

1446 

1447 # Because subtasks don't have connections we have to make one. 

1448 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

1449 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

1450 if self.config.assembleStaticSkyModel.doWrite: 

1451 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

1452 # Remove template coadd from both subtask's and main tasks outputs, 

1453 # because it is handled by the subtask as `coaddExposure` 

1454 del outputRefs.templateCoadd 

1455 del staticSkyModelOutputRefs.templateCoadd 

1456 

1457 # A PSF-Matched nImage does not exist as a dataset type 

1458 if 'nImage' in staticSkyModelOutputRefs.keys(): 

1459 del staticSkyModelOutputRefs.nImage 

1460 

1461 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

1462 staticSkyModelOutputRefs) 

1463 if templateCoadd is None: 

1464 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

1465 

1466 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

1467 nImage=templateCoadd.nImage, 

1468 warpRefList=templateCoadd.warpRefList, 

1469 imageScalerList=templateCoadd.imageScalerList, 

1470 weightList=templateCoadd.weightList) 

1471 

1472 def _noTemplateMessage(self, warpType): 

1473 warpName = (warpType[0].upper() + warpType[1:]) 

1474 message = """No %(warpName)s warps were found to build the template coadd which is 

1475 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

1476 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

1477 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

1478 

1479 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

1480 another algorithm like: 

1481 

1482 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

1483 config.assemble.retarget(SafeClipAssembleCoaddTask) 

1484 """ % {"warpName": warpName} 

1485 return message 

1486 

1487 @utils.inheritDoc(AssembleCoaddTask) 

1488 @timeMethod 

1489 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1490 supplementaryData): 

1491 """Assemble the coadd. 

1492 

1493 Find artifacts and apply them to the warps' masks creating a list of 

1494 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

1495 plane. Then pass these alternative masks to the base class's ``run`` 

1496 method. 

1497 """ 

1498 # Check and match the order of the supplementaryData 

1499 # (PSF-matched) inputs to the order of the direct inputs, 

1500 # so that the artifact mask is applied to the right warp 

1501 dataIds = [ref.dataId for ref in tempExpRefList] 

1502 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

1503 

1504 if dataIds != psfMatchedDataIds: 

1505 self.log.info("Reordering and or/padding PSF-matched visit input list") 

1506 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

1507 psfMatchedDataIds, dataIds) 

1508 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

1509 psfMatchedDataIds, dataIds) 

1510 

1511 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

1512 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

1513 supplementaryData.warpRefList, 

1514 supplementaryData.imageScalerList) 

1515 

1516 badMaskPlanes = self.config.badMaskPlanes[:] 

1517 badMaskPlanes.append("CLIPPED") 

1518 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1519 

1520 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1521 spanSetMaskList, mask=badPixelMask) 

1522 

1523 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

1524 # Psf-Matching moves the real edge inwards 

1525 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

1526 return result 

1527 

1528 def applyAltEdgeMask(self, mask, altMaskList): 

1529 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

1530 

1531 Parameters 

1532 ---------- 

1533 mask : `lsst.afw.image.Mask` 

1534 Original mask. 

1535 altMaskList : `list` of `dict` 

1536 List of Dicts containing ``spanSet`` lists. 

1537 Each element contains the new mask plane name (e.g. "CLIPPED 

1538 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

1539 the mask. 

1540 """ 

1541 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

1542 for visitMask in altMaskList: 

1543 if "EDGE" in visitMask: 

1544 for spanSet in visitMask['EDGE']: 

1545 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

1546 

1547 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

1548 """Find artifacts. 

1549 

1550 Loop through warps twice. The first loop builds a map with the count 

1551 of how many epochs each pixel deviates from the templateCoadd by more 

1552 than ``config.chiThreshold`` sigma. The second loop takes each 

1553 difference image and filters the artifacts detected in each using 

1554 count map to filter out variable sources and sources that are 

1555 difficult to subtract cleanly. 

1556 

1557 Parameters 

1558 ---------- 

1559 templateCoadd : `lsst.afw.image.Exposure` 

1560 Exposure to serve as model of static sky. 

1561 tempExpRefList : `list` 

1562 List of data references to warps. 

1563 imageScalerList : `list` 

1564 List of image scalers. 

1565 

1566 Returns 

1567 ------- 

1568 altMasks : `list` of `dict` 

1569 List of dicts containing information about CLIPPED 

1570 (i.e., artifacts), NO_DATA, and EDGE pixels. 

1571 """ 

1572 self.log.debug("Generating Count Image, and mask lists.") 

1573 coaddBBox = templateCoadd.getBBox() 

1574 slateIm = afwImage.ImageU(coaddBBox) 

1575 epochCountImage = afwImage.ImageU(coaddBBox) 

1576 nImage = afwImage.ImageU(coaddBBox) 

1577 spanSetArtifactList = [] 

1578 spanSetNoDataMaskList = [] 

1579 spanSetEdgeList = [] 

1580 spanSetBadMorphoList = [] 

1581 badPixelMask = self.getBadPixelMask() 

1582 

1583 # mask of the warp diffs should = that of only the warp 

1584 templateCoadd.mask.clearAllMaskPlanes() 

1585 

1586 if self.config.doPreserveContainedBySource: 

1587 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

1588 else: 

1589 templateFootprints = None 

1590 

1591 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

1592 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

1593 if warpDiffExp is not None: 

1594 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

1595 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

1596 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

1597 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

1598 fpSet.positive.merge(fpSet.negative) 

1599 footprints = fpSet.positive 

1600 slateIm.set(0) 

1601 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

1602 

1603 # Remove artifacts due to defects before they contribute to the epochCountImage 

1604 if self.config.doPrefilterArtifacts: 

1605 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

1606 

1607 # Clear mask before adding prefiltered spanSets 

1608 self.detect.clearMask(warpDiffExp.mask) 

1609 for spans in spanSetList: 

1610 spans.setImage(slateIm, 1, doClip=True) 

1611 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

1612 epochCountImage += slateIm 

1613 

1614 if self.config.doFilterMorphological: 

1615 maskName = self.config.streakMaskName 

1616 _ = self.maskStreaks.run(warpDiffExp) 

1617 streakMask = warpDiffExp.mask 

1618 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

1619 streakMask.getPlaneBitMask(maskName)).split() 

1620 # Pad the streaks to account for low-surface brightness wings 

1621 psf = warpDiffExp.getPsf() 

1622 for s, sset in enumerate(spanSetStreak): 

1623 psfShape = psf.computeShape(sset.computeCentroid()) 

1624 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius() 

1625 sset_dilated = sset.dilated(int(dilation)) 

1626 spanSetStreak[s] = sset_dilated 

1627 

1628 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

1629 # undergo a second convolution. Pixels with data in the direct warp 

1630 # but not in the PSF-matched warp will not have their artifacts detected. 

1631 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

1632 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

1633 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

1634 nansMask.setXY0(warpDiffExp.getXY0()) 

1635 edgeMask = warpDiffExp.mask 

1636 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

1637 edgeMask.getPlaneBitMask("EDGE")).split() 

1638 else: 

1639 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

1640 # In this case, mask the whole epoch 

1641 nansMask = afwImage.MaskX(coaddBBox, 1) 

1642 spanSetList = [] 

1643 spanSetEdgeMask = [] 

1644 spanSetStreak = [] 

1645 

1646 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

1647 

1648 spanSetNoDataMaskList.append(spanSetNoDataMask) 

1649 spanSetArtifactList.append(spanSetList) 

1650 spanSetEdgeList.append(spanSetEdgeMask) 

1651 if self.config.doFilterMorphological: 

1652 spanSetBadMorphoList.append(spanSetStreak) 

1653 

1654 if lsstDebug.Info(__name__).saveCountIm: 

1655 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

1656 epochCountImage.writeFits(path) 

1657 

1658 for i, spanSetList in enumerate(spanSetArtifactList): 

1659 if spanSetList: 

1660 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

1661 templateFootprints) 

1662 spanSetArtifactList[i] = filteredSpanSetList 

1663 if self.config.doFilterMorphological: 

1664 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

1665 

1666 altMasks = [] 

1667 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

1668 altMasks.append({'CLIPPED': artifacts, 

1669 'NO_DATA': noData, 

1670 'EDGE': edge}) 

1671 return altMasks 

1672 

1673 def prefilterArtifacts(self, spanSetList, exp): 

1674 """Remove artifact candidates covered by bad mask plane. 

1675 

1676 Any future editing of the candidate list that does not depend on 

1677 temporal information should go in this method. 

1678 

1679 Parameters 

1680 ---------- 

1681 spanSetList : `list` of `lsst.afw.geom.SpanSet` 

1682 List of SpanSets representing artifact candidates. 

1683 exp : `lsst.afw.image.Exposure` 

1684 Exposure containing mask planes used to prefilter. 

1685 

1686 Returns 

1687 ------- 

1688 returnSpanSetList : `list` of `lsst.afw.geom.SpanSet` 

1689 List of SpanSets with artifacts. 

1690 """ 

1691 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

1692 goodArr = (exp.mask.array & badPixelMask) == 0 

1693 returnSpanSetList = [] 

1694 bbox = exp.getBBox() 

1695 x0, y0 = exp.getXY0() 

1696 for i, span in enumerate(spanSetList): 

1697 y, x = span.clippedTo(bbox).indices() 

1698 yIndexLocal = numpy.array(y) - y0 

1699 xIndexLocal = numpy.array(x) - x0 

1700 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

1701 if goodRatio > self.config.prefilterArtifactsRatio: 

1702 returnSpanSetList.append(span) 

1703 return returnSpanSetList 

1704 

1705 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

1706 """Filter artifact candidates. 

1707 

1708 Parameters 

1709 ---------- 

1710 spanSetList : `list` of `lsst.afw.geom.SpanSet` 

1711 List of SpanSets representing artifact candidates. 

1712 epochCountImage : `lsst.afw.image.Image` 

1713 Image of accumulated number of warpDiff detections. 

1714 nImage : `lsst.afw.image.ImageU` 

1715 Image of the accumulated number of total epochs contributing. 

1716 

1717 Returns 

1718 ------- 

1719 maskSpanSetList : `list` 

1720 List of SpanSets with artifacts. 

1721 """ 

1722 maskSpanSetList = [] 

1723 x0, y0 = epochCountImage.getXY0() 

1724 for i, span in enumerate(spanSetList): 

1725 y, x = span.indices() 

1726 yIdxLocal = [y1 - y0 for y1 in y] 

1727 xIdxLocal = [x1 - x0 for x1 in x] 

1728 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

1729 totalN = nImage.array[yIdxLocal, xIdxLocal] 

1730 

1731 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

1732 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

1733 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

1734 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

1735 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

1736 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

1737 & (outlierN <= effectiveMaxNumEpochs)) 

1738 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

1739 if percentBelowThreshold > self.config.spatialThreshold: 

1740 maskSpanSetList.append(span) 

1741 

1742 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

1743 # If a candidate is contained by a footprint on the template coadd, do not clip 

1744 filteredMaskSpanSetList = [] 

1745 for span in maskSpanSetList: 

1746 doKeep = True 

1747 for footprint in footprintsToExclude.positive.getFootprints(): 

1748 if footprint.spans.contains(span): 

1749 doKeep = False 

1750 break 

1751 if doKeep: 

1752 filteredMaskSpanSetList.append(span) 

1753 maskSpanSetList = filteredMaskSpanSetList 

1754 

1755 return maskSpanSetList 

1756 

1757 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

1758 """Fetch a warp from the butler and return a warpDiff. 

1759 

1760 Parameters 

1761 ---------- 

1762 warpRef : `lsst.daf.butler.DeferredDatasetHandle` 

1763 Handle for the warp. 

1764 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

1765 An image scaler object. 

1766 templateCoadd : `lsst.afw.image.Exposure` 

1767 Exposure to be substracted from the scaled warp. 

1768 

1769 Returns 

1770 ------- 

1771 warp : `lsst.afw.image.Exposure` 

1772 Exposure of the image difference between the warp and template. 

1773 """ 

1774 # If the PSF-Matched warp did not exist for this direct warp 

1775 # None is holding its place to maintain order in Gen 3 

1776 if warpRef is None: 

1777 return None 

1778 

1779 warp = warpRef.get() 

1780 # direct image scaler OK for PSF-matched Warp 

1781 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

1782 mi = warp.getMaskedImage() 

1783 if self.config.doScaleWarpVariance: 

1784 try: 

1785 self.scaleWarpVariance.run(mi) 

1786 except Exception as exc: 

1787 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

1788 mi -= templateCoadd.getMaskedImage() 

1789 return warp