Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from lsst.meas.algorithms import SourceDetectionTask 

48from lsst.daf.butler import DeferredDatasetHandle 

49 

50__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

51 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

52 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

53 

54 

55class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

56 dimensions=("tract", "patch", "band", "skymap"), 

57 defaultTemplates={"inputCoaddName": "deep", 

58 "outputCoaddName": "deep", 

59 "warpType": "direct", 

60 "warpTypeSuffix": "", 

61 "fakesType": ""}): 

62 inputWarps = pipeBase.connectionTypes.Input( 

63 doc=("Input list of warps to be assemebled i.e. stacked." 

64 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

65 name="{inputCoaddName}Coadd_{warpType}Warp", 

66 storageClass="ExposureF", 

67 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

68 deferLoad=True, 

69 multiple=True 

70 ) 

71 skyMap = pipeBase.connectionTypes.Input( 

72 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

73 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

74 storageClass="SkyMap", 

75 dimensions=("skymap", ), 

76 ) 

77 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

78 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

79 " BRIGHT_OBJECT."), 

80 name="brightObjectMask", 

81 storageClass="ObjectMaskCatalog", 

82 dimensions=("tract", "patch", "skymap", "band"), 

83 ) 

84 coaddExposure = pipeBase.connectionTypes.Output( 

85 doc="Output coadded exposure, produced by stacking input warps", 

86 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

87 storageClass="ExposureF", 

88 dimensions=("tract", "patch", "skymap", "band"), 

89 ) 

90 nImage = pipeBase.connectionTypes.Output( 

91 doc="Output image of number of input images per pixel", 

92 name="{outputCoaddName}Coadd_nImage", 

93 storageClass="ImageU", 

94 dimensions=("tract", "patch", "skymap", "band"), 

95 ) 

96 

97 def __init__(self, *, config=None): 

98 super().__init__(config=config) 

99 

100 # Override the connection's name template with config to replicate Gen2 behavior 

101 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

102 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

103 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

104 templateValues['warpType'] = config.warpType 

105 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

106 if config.hasFakes: 

107 templateValues['fakesType'] = "_fakes" 

108 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

109 for name in self.allConnections} 

110 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

111 # End code to remove after deprecation 

112 

113 if not config.doMaskBrightObjects: 

114 self.prerequisiteInputs.remove("brightObjectMask") 

115 

116 if not config.doNImage: 

117 self.outputs.remove("nImage") 

118 

119 

120class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

121 pipelineConnections=AssembleCoaddConnections): 

122 """Configuration parameters for the `AssembleCoaddTask`. 

123 

124 Notes 

125 ----- 

126 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

127 only set the bitplane config.brightObjectMaskName. To make this useful you 

128 *must* also configure the flags.pixel algorithm, for example by adding 

129 

130 .. code-block:: none 

131 

132 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

133 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

134 

135 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

136 """ 

137 warpType = pexConfig.Field( 

138 doc="Warp name: one of 'direct' or 'psfMatched'", 

139 dtype=str, 

140 default="direct", 

141 ) 

142 subregionSize = pexConfig.ListField( 

143 dtype=int, 

144 doc="Width, height of stack subregion size; " 

145 "make small enough that a full stack of images will fit into memory at once.", 

146 length=2, 

147 default=(2000, 2000), 

148 ) 

149 statistic = pexConfig.Field( 

150 dtype=str, 

151 doc="Main stacking statistic for aggregating over the epochs.", 

152 default="MEANCLIP", 

153 ) 

154 doSigmaClip = pexConfig.Field( 

155 dtype=bool, 

156 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

157 default=False, 

158 ) 

159 sigmaClip = pexConfig.Field( 

160 dtype=float, 

161 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

162 default=3.0, 

163 ) 

164 clipIter = pexConfig.Field( 

165 dtype=int, 

166 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

167 default=2, 

168 ) 

169 calcErrorFromInputVariance = pexConfig.Field( 

170 dtype=bool, 

171 doc="Calculate coadd variance from input variance by stacking statistic." 

172 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

173 default=True, 

174 ) 

175 scaleZeroPoint = pexConfig.ConfigurableField( 

176 target=ScaleZeroPointTask, 

177 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

178 ) 

179 doInterp = pexConfig.Field( 

180 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

181 dtype=bool, 

182 default=True, 

183 ) 

184 interpImage = pexConfig.ConfigurableField( 

185 target=InterpImageTask, 

186 doc="Task to interpolate (and extrapolate) over NaN pixels", 

187 ) 

188 doWrite = pexConfig.Field( 

189 doc="Persist coadd?", 

190 dtype=bool, 

191 default=True, 

192 ) 

193 doNImage = pexConfig.Field( 

194 doc="Create image of number of contributing exposures for each pixel", 

195 dtype=bool, 

196 default=False, 

197 ) 

198 doUsePsfMatchedPolygons = pexConfig.Field( 

199 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

200 dtype=bool, 

201 default=False, 

202 ) 

203 maskPropagationThresholds = pexConfig.DictField( 

204 keytype=str, 

205 itemtype=float, 

206 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

207 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

208 "would have contributed exceeds this value."), 

209 default={"SAT": 0.1}, 

210 ) 

211 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

212 doc="Mask planes to remove before coadding") 

213 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

214 doc="Set mask and flag bits for bright objects?") 

215 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

216 doc="Name of mask bit used for bright objects") 

217 coaddPsf = pexConfig.ConfigField( 

218 doc="Configuration for CoaddPsf", 

219 dtype=measAlg.CoaddPsfConfig, 

220 ) 

221 doAttachTransmissionCurve = pexConfig.Field( 

222 dtype=bool, default=False, optional=False, 

223 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

224 "(requires all input Exposures to have TransmissionCurves).") 

225 ) 

226 hasFakes = pexConfig.Field( 

227 dtype=bool, 

228 default=False, 

229 doc="Should be set to True if fake sources have been inserted into the input data." 

230 ) 

231 

232 def setDefaults(self): 

233 super().setDefaults() 

234 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

235 

236 def validate(self): 

237 super().validate() 

238 if self.doPsfMatch: 

239 # Backwards compatibility. 

240 # Configs do not have loggers 

241 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

242 self.warpType = 'psfMatched' 

243 if self.doSigmaClip and self.statistic != "MEANCLIP": 

244 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

245 self.statistic = "MEANCLIP" 

246 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

247 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

248 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

249 

250 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

251 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

252 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

253 if str(k) not in unstackableStats] 

254 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

255 % (self.statistic, stackableStats)) 

256 

257 

258class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

259 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

260 

261 We want to assemble a coadded image from a set of Warps (also called 

262 coadded temporary exposures or ``coaddTempExps``). 

263 Each input Warp covers a patch on the sky and corresponds to a single 

264 run/visit/exposure of the covered patch. We provide the task with a list 

265 of Warps (``selectDataList``) from which it selects Warps that cover the 

266 specified patch (pointed at by ``dataRef``). 

267 Each Warp that goes into a coadd will typically have an independent 

268 photometric zero-point. Therefore, we must scale each Warp to set it to 

269 a common photometric zeropoint. WarpType may be one of 'direct' or 

270 'psfMatched', and the boolean configs `config.makeDirect` and 

271 `config.makePsfMatched` set which of the warp types will be coadded. 

272 The coadd is computed as a mean with optional outlier rejection. 

273 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

274 Finally, Warps can have bad 'NaN' pixels which received no input from the 

275 source calExps. We interpolate over these bad (NaN) pixels. 

276 

277 `AssembleCoaddTask` uses several sub-tasks. These are 

278 

279 - `ScaleZeroPointTask` 

280 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

281 - `InterpImageTask` 

282 - interpolate across bad pixels (NaN) in the final coadd 

283 

284 You can retarget these subtasks if you wish. 

285 

286 Notes 

287 ----- 

288 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

289 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

290 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

291 no debug variables of its own. Some of the subtasks may support debug 

292 variables. See the documentation for the subtasks for further information. 

293 

294 Examples 

295 -------- 

296 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

297 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

298 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

299 inputs: a data reference to the tract patch and filter to be coadded, and 

300 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

301 ``--selectId``, respectively: 

302 

303 .. code-block:: none 

304 

305 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

306 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

307 

308 Only the Warps that cover the specified tract and patch will be coadded. 

309 A list of the available optional arguments can be obtained by calling 

310 ``assembleCoadd.py`` with the ``--help`` command line argument: 

311 

312 .. code-block:: none 

313 

314 assembleCoadd.py --help 

315 

316 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

317 multi-band processing, we will generate the HSC-I & -R band coadds from 

318 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

319 assuming that the lsst stack has been already set up, we must set up the 

320 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

321 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

322 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

323 coadds, we must first 

324 

325 - processCcd 

326 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

327 - makeSkyMap 

328 - create a skymap that covers the area of the sky present in the raw exposures 

329 - makeCoaddTempExp 

330 - warp the individual calibrated exposures to the tangent plane of the coadd 

331 

332 We can perform all of these steps by running 

333 

334 .. code-block:: none 

335 

336 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

337 

338 This will produce warped exposures for each visit. To coadd the warped 

339 data, we call assembleCoadd.py as follows: 

340 

341 .. code-block:: none 

342 

343 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

344 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

345 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

346 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

347 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

348 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

349 --selectId visit=903988 ccd=24 

350 

351 that will process the HSC-I band data. The results are written in 

352 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

353 

354 You may also choose to run: 

355 

356 .. code-block:: none 

357 

358 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

359 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

360 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

361 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

362 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

363 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

364 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

365 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

366 

367 to generate the coadd for the HSC-R band if you are interested in 

368 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

369 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

370 rather than `AssembleCoaddTask` to make the coadd. 

371 """ 

372 ConfigClass = AssembleCoaddConfig 

373 _DefaultName = "assembleCoadd" 

374 

375 def __init__(self, *args, **kwargs): 

376 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

377 if args: 377 ↛ 378line 377 didn't jump to line 378, because the condition on line 377 was never true

378 argNames = ["config", "name", "parentTask", "log"] 

379 kwargs.update({k: v for k, v in zip(argNames, args)}) 

380 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

381 "PipelineTask will not take positional args" % argNames, FutureWarning) 

382 

383 super().__init__(**kwargs) 

384 self.makeSubtask("interpImage") 

385 self.makeSubtask("scaleZeroPoint") 

386 

387 if self.config.doMaskBrightObjects: 387 ↛ 388line 387 didn't jump to line 388, because the condition on line 387 was never true

388 mask = afwImage.Mask() 

389 try: 

390 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

391 except pexExceptions.LsstCppException: 

392 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

393 mask.getMaskPlaneDict().keys()) 

394 del mask 

395 

396 self.warpType = self.config.warpType 

397 

398 @utils.inheritDoc(pipeBase.PipelineTask) 

399 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

400 # Docstring to be formatted with info from PipelineTask.runQuantum 

401 """ 

402 Notes 

403 ----- 

404 Assemble a coadd from a set of Warps. 

405 

406 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

407 Analogous to `runDataRef`, it prepares all the data products to be 

408 passed to `run`, and processes the results before returning a struct 

409 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

410 Therefore, its inputs are accessed subregion by subregion 

411 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

412 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

413 correspond to an update in `runDataRef` while both entry points 

414 are used. 

415 """ 

416 inputData = butlerQC.get(inputRefs) 

417 

418 # Construct skyInfo expected by run 

419 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

420 skyMap = inputData["skyMap"] 

421 outputDataId = butlerQC.quantum.dataId 

422 

423 inputData['skyInfo'] = makeSkyInfo(skyMap, 

424 tractId=outputDataId['tract'], 

425 patchId=outputDataId['patch']) 

426 

427 # Construct list of input Deferred Datasets 

428 # These quack a bit like like Gen2 DataRefs 

429 warpRefList = inputData['inputWarps'] 

430 # Perform same middle steps as `runDataRef` does 

431 inputs = self.prepareInputs(warpRefList) 

432 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

433 self.getTempExpDatasetName(self.warpType)) 

434 if len(inputs.tempExpRefList) == 0: 

435 self.log.warn("No coadd temporary exposures found") 

436 return 

437 

438 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

439 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

440 inputs.weightList, supplementaryData=supplementaryData) 

441 

442 inputData.setdefault('brightObjectMask', None) 

443 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

444 

445 if self.config.doWrite: 

446 butlerQC.put(retStruct, outputRefs) 

447 return retStruct 

448 

449 @pipeBase.timeMethod 

450 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

451 """Assemble a coadd from a set of Warps. 

452 

453 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

454 Compute weights to be applied to each Warp and 

455 find scalings to match the photometric zeropoint to a reference Warp. 

456 Assemble the Warps using `run`. Interpolate over NaNs and 

457 optionally write the coadd to disk. Return the coadded exposure. 

458 

459 Parameters 

460 ---------- 

461 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

462 Data reference defining the patch for coaddition and the 

463 reference Warp (if ``config.autoReference=False``). 

464 Used to access the following data products: 

465 - ``self.config.coaddName + "Coadd_skyMap"`` 

466 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

467 - ``self.config.coaddName + "Coadd"`` 

468 selectDataList : `list` 

469 List of data references to Calexps. Data to be coadded will be 

470 selected from this list based on overlap with the patch defined 

471 by dataRef, grouped by visit, and converted to a list of data 

472 references to warps. 

473 warpRefList : `list` 

474 List of data references to Warps to be coadded. 

475 Note: `warpRefList` is just the new name for `tempExpRefList`. 

476 

477 Returns 

478 ------- 

479 retStruct : `lsst.pipe.base.Struct` 

480 Result struct with components: 

481 

482 - ``coaddExposure``: coadded exposure (``Exposure``). 

483 - ``nImage``: exposure count image (``Image``). 

484 """ 

485 if selectDataList and warpRefList: 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true

486 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

487 "and which to use is ambiguous. Please pass only one.") 

488 

489 skyInfo = self.getSkyInfo(dataRef) 

490 if warpRefList is None: 

491 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

492 if len(calExpRefList) == 0: 492 ↛ 493line 492 didn't jump to line 493, because the condition on line 492 was never true

493 self.log.warn("No exposures to coadd") 

494 return 

495 self.log.info("Coadding %d exposures", len(calExpRefList)) 

496 

497 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

498 

499 inputData = self.prepareInputs(warpRefList) 

500 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

501 self.getTempExpDatasetName(self.warpType)) 

502 if len(inputData.tempExpRefList) == 0: 502 ↛ 503line 502 didn't jump to line 503, because the condition on line 502 was never true

503 self.log.warn("No coadd temporary exposures found") 

504 return 

505 

506 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

507 

508 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

509 inputData.weightList, supplementaryData=supplementaryData) 

510 

511 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

512 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

513 

514 if self.config.doWrite: 

515 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 515 ↛ 516line 515 didn't jump to line 516, because the condition on line 515 was never true

516 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

517 else: 

518 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

519 self.log.info("Persisting %s" % coaddDatasetName) 

520 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

521 if self.config.doNImage and retStruct.nImage is not None: 521 ↛ 522line 521 didn't jump to line 522, because the condition on line 521 was never true

522 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

523 

524 return retStruct 

525 

526 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

527 """Interpolate over missing data and mask bright stars. 

528 

529 Parameters 

530 ---------- 

531 coaddExposure : `lsst.afw.image.Exposure` 

532 The coadded exposure to process. 

533 dataRef : `lsst.daf.persistence.ButlerDataRef` 

534 Butler data reference for supplementary data. 

535 """ 

536 if self.config.doInterp: 536 ↛ 543line 536 didn't jump to line 543, because the condition on line 536 was never false

537 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

538 # The variance must be positive; work around for DM-3201. 

539 varArray = coaddExposure.variance.array 

540 with numpy.errstate(invalid="ignore"): 

541 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

542 

543 if self.config.doMaskBrightObjects: 543 ↛ 544line 543 didn't jump to line 544, because the condition on line 543 was never true

544 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

545 

546 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

547 """Make additional inputs to run() specific to subclasses (Gen2) 

548 

549 Duplicates interface of `runDataRef` method 

550 Available to be implemented by subclasses only if they need the 

551 coadd dataRef for performing preliminary processing before 

552 assembling the coadd. 

553 

554 Parameters 

555 ---------- 

556 dataRef : `lsst.daf.persistence.ButlerDataRef` 

557 Butler data reference for supplementary data. 

558 selectDataList : `list` (optional) 

559 Optional List of data references to Calexps. 

560 warpRefList : `list` (optional) 

561 Optional List of data references to Warps. 

562 """ 

563 return pipeBase.Struct() 

564 

565 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

566 """Make additional inputs to run() specific to subclasses (Gen3) 

567 

568 Duplicates interface of `runQuantum` method. 

569 Available to be implemented by subclasses only if they need the 

570 coadd dataRef for performing preliminary processing before 

571 assembling the coadd. 

572 

573 Parameters 

574 ---------- 

575 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

576 Gen3 Butler object for fetching additional data products before 

577 running the Task specialized for quantum being processed 

578 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

579 Attributes are the names of the connections describing input dataset types. 

580 Values are DatasetRefs that task consumes for corresponding dataset type. 

581 DataIds are guaranteed to match data objects in ``inputData``. 

582 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

583 Attributes are the names of the connections describing output dataset types. 

584 Values are DatasetRefs that task is to produce 

585 for corresponding dataset type. 

586 """ 

587 return pipeBase.Struct() 

588 

589 def getTempExpRefList(self, patchRef, calExpRefList): 

590 """Generate list data references corresponding to warped exposures 

591 that lie within the patch to be coadded. 

592 

593 Parameters 

594 ---------- 

595 patchRef : `dataRef` 

596 Data reference for patch. 

597 calExpRefList : `list` 

598 List of data references for input calexps. 

599 

600 Returns 

601 ------- 

602 tempExpRefList : `list` 

603 List of Warp/CoaddTempExp data references. 

604 """ 

605 butler = patchRef.getButler() 

606 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

607 self.getTempExpDatasetName(self.warpType)) 

608 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

609 g, groupData.keys) for 

610 g in groupData.groups.keys()] 

611 return tempExpRefList 

612 

613 def prepareInputs(self, refList): 

614 """Prepare the input warps for coaddition by measuring the weight for 

615 each warp and the scaling for the photometric zero point. 

616 

617 Each Warp has its own photometric zeropoint and background variance. 

618 Before coadding these Warps together, compute a scale factor to 

619 normalize the photometric zeropoint and compute the weight for each Warp. 

620 

621 Parameters 

622 ---------- 

623 refList : `list` 

624 List of data references to tempExp 

625 

626 Returns 

627 ------- 

628 result : `lsst.pipe.base.Struct` 

629 Result struct with components: 

630 

631 - ``tempExprefList``: `list` of data references to tempExp. 

632 - ``weightList``: `list` of weightings. 

633 - ``imageScalerList``: `list` of image scalers. 

634 """ 

635 statsCtrl = afwMath.StatisticsControl() 

636 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

637 statsCtrl.setNumIter(self.config.clipIter) 

638 statsCtrl.setAndMask(self.getBadPixelMask()) 

639 statsCtrl.setNanSafe(True) 

640 # compute tempExpRefList: a list of tempExpRef that actually exist 

641 # and weightList: a list of the weight of the associated coadd tempExp 

642 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

643 tempExpRefList = [] 

644 weightList = [] 

645 imageScalerList = [] 

646 tempExpName = self.getTempExpDatasetName(self.warpType) 

647 for tempExpRef in refList: 

648 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

649 # therefore have no datasetExists() method 

650 if not isinstance(tempExpRef, DeferredDatasetHandle): 650 ↛ 655line 650 didn't jump to line 655, because the condition on line 650 was never false

651 if not tempExpRef.datasetExists(tempExpName): 

652 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

653 continue 

654 

655 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

656 # Ignore any input warp that is empty of data 

657 if numpy.isnan(tempExp.image.array).all(): 657 ↛ 658line 657 didn't jump to line 658, because the condition on line 657 was never true

658 continue 

659 maskedImage = tempExp.getMaskedImage() 

660 imageScaler = self.scaleZeroPoint.computeImageScaler( 

661 exposure=tempExp, 

662 dataRef=tempExpRef, 

663 ) 

664 try: 

665 imageScaler.scaleMaskedImage(maskedImage) 

666 except Exception as e: 

667 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

668 continue 

669 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

670 afwMath.MEANCLIP, statsCtrl) 

671 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

672 weight = 1.0 / float(meanVar) 

673 if not numpy.isfinite(weight): 673 ↛ 674line 673 didn't jump to line 674, because the condition on line 673 was never true

674 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

675 continue 

676 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

677 

678 del maskedImage 

679 del tempExp 

680 

681 tempExpRefList.append(tempExpRef) 

682 weightList.append(weight) 

683 imageScalerList.append(imageScaler) 

684 

685 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

686 imageScalerList=imageScalerList) 

687 

688 def prepareStats(self, mask=None): 

689 """Prepare the statistics for coadding images. 

690 

691 Parameters 

692 ---------- 

693 mask : `int`, optional 

694 Bit mask value to exclude from coaddition. 

695 

696 Returns 

697 ------- 

698 stats : `lsst.pipe.base.Struct` 

699 Statistics structure with the following fields: 

700 

701 - ``statsCtrl``: Statistics control object for coadd 

702 (`lsst.afw.math.StatisticsControl`) 

703 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

704 """ 

705 if mask is None: 

706 mask = self.getBadPixelMask() 

707 statsCtrl = afwMath.StatisticsControl() 

708 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

709 statsCtrl.setNumIter(self.config.clipIter) 

710 statsCtrl.setAndMask(mask) 

711 statsCtrl.setNanSafe(True) 

712 statsCtrl.setWeighted(True) 

713 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

714 for plane, threshold in self.config.maskPropagationThresholds.items(): 

715 bit = afwImage.Mask.getMaskPlane(plane) 

716 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

717 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

718 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

719 

720 @pipeBase.timeMethod 

721 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

722 altMaskList=None, mask=None, supplementaryData=None): 

723 """Assemble a coadd from input warps 

724 

725 Assemble the coadd using the provided list of coaddTempExps. Since 

726 the full coadd covers a patch (a large area), the assembly is 

727 performed over small areas on the image at a time in order to 

728 conserve memory usage. Iterate over subregions within the outer 

729 bbox of the patch using `assembleSubregion` to stack the corresponding 

730 subregions from the coaddTempExps with the statistic specified. 

731 Set the edge bits the coadd mask based on the weight map. 

732 

733 Parameters 

734 ---------- 

735 skyInfo : `lsst.pipe.base.Struct` 

736 Struct with geometric information about the patch. 

737 tempExpRefList : `list` 

738 List of data references to Warps (previously called CoaddTempExps). 

739 imageScalerList : `list` 

740 List of image scalers. 

741 weightList : `list` 

742 List of weights 

743 altMaskList : `list`, optional 

744 List of alternate masks to use rather than those stored with 

745 tempExp. 

746 mask : `int`, optional 

747 Bit mask value to exclude from coaddition. 

748 supplementaryData : lsst.pipe.base.Struct, optional 

749 Struct with additional data products needed to assemble coadd. 

750 Only used by subclasses that implement `makeSupplementaryData` 

751 and override `run`. 

752 

753 Returns 

754 ------- 

755 result : `lsst.pipe.base.Struct` 

756 Result struct with components: 

757 

758 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

759 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

760 - ``warpRefList``: input list of refs to the warps ( 

761 ``lsst.daf.butler.DeferredDatasetHandle`` or 

762 ``lsst.daf.persistence.ButlerDataRef``) 

763 (unmodified) 

764 - ``imageScalerList``: input list of image scalers (unmodified) 

765 - ``weightList``: input list of weights (unmodified) 

766 """ 

767 tempExpName = self.getTempExpDatasetName(self.warpType) 

768 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

769 stats = self.prepareStats(mask=mask) 

770 

771 if altMaskList is None: 

772 altMaskList = [None]*len(tempExpRefList) 

773 

774 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

775 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

776 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

777 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

778 coaddMaskedImage = coaddExposure.getMaskedImage() 

779 subregionSizeArr = self.config.subregionSize 

780 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

781 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

782 if self.config.doNImage: 782 ↛ 783line 782 didn't jump to line 783, because the condition on line 782 was never true

783 nImage = afwImage.ImageU(skyInfo.bbox) 

784 else: 

785 nImage = None 

786 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

787 try: 

788 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

789 weightList, altMaskList, stats.flags, stats.ctrl, 

790 nImage=nImage) 

791 except Exception as e: 

792 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

793 

794 self.setInexactPsf(coaddMaskedImage.getMask()) 

795 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

796 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

797 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

798 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

799 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

800 weightList=weightList) 

801 

802 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

803 """Set the metadata for the coadd. 

804 

805 This basic implementation sets the filter from the first input. 

806 

807 Parameters 

808 ---------- 

809 coaddExposure : `lsst.afw.image.Exposure` 

810 The target exposure for the coadd. 

811 tempExpRefList : `list` 

812 List of data references to tempExp. 

813 weightList : `list` 

814 List of weights. 

815 """ 

816 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

817 tempExpName = self.getTempExpDatasetName(self.warpType) 

818 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

819 # (and we need more than just the PropertySet that contains the header), which is not possible 

820 # with the current butler (see #2777). 

821 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

822 

823 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 823 ↛ 825line 823 didn't jump to line 825, because the condition on line 823 was never true

824 # Gen 3 API 

825 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

826 else: 

827 # Gen 2 API. Delete this when Gen 2 retired 

828 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

829 for tempExpRef in tempExpRefList] 

830 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

831 

832 # Set the coadd FilterLabel to the band of the first input exposure: 

833 # Coadds are calibrated, so the physical label is now meaningless. 

834 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

835 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

836 coaddInputs.ccds.reserve(numCcds) 

837 coaddInputs.visits.reserve(len(tempExpList)) 

838 

839 for tempExp, weight in zip(tempExpList, weightList): 

840 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

841 

842 if self.config.doUsePsfMatchedPolygons: 

843 self.shrinkValidPolygons(coaddInputs) 

844 

845 coaddInputs.visits.sort() 

846 if self.warpType == "psfMatched": 

847 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

848 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

849 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

850 # having the maximum width (sufficient because square) 

851 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

852 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

853 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

854 else: 

855 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

856 self.config.coaddPsf.makeControl()) 

857 coaddExposure.setPsf(psf) 

858 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

859 coaddExposure.getWcs()) 

860 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

861 if self.config.doAttachTransmissionCurve: 

862 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

863 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

864 

865 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

866 altMaskList, statsFlags, statsCtrl, nImage=None): 

867 """Assemble the coadd for a sub-region. 

868 

869 For each coaddTempExp, check for (and swap in) an alternative mask 

870 if one is passed. Remove mask planes listed in 

871 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

872 `lsst.afw.math.statisticsStack` with the statistic specified by 

873 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

874 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

875 an N-sigma clipped mean where N and iterations are specified by 

876 statsCtrl. Assign the stacked subregion back to the coadd. 

877 

878 Parameters 

879 ---------- 

880 coaddExposure : `lsst.afw.image.Exposure` 

881 The target exposure for the coadd. 

882 bbox : `lsst.geom.Box` 

883 Sub-region to coadd. 

884 tempExpRefList : `list` 

885 List of data reference to tempExp. 

886 imageScalerList : `list` 

887 List of image scalers. 

888 weightList : `list` 

889 List of weights. 

890 altMaskList : `list` 

891 List of alternate masks to use rather than those stored with 

892 tempExp, or None. Each element is dict with keys = mask plane 

893 name to which to add the spans. 

894 statsFlags : `lsst.afw.math.Property` 

895 Property object for statistic for coadd. 

896 statsCtrl : `lsst.afw.math.StatisticsControl` 

897 Statistics control object for coadd. 

898 nImage : `lsst.afw.image.ImageU`, optional 

899 Keeps track of exposure count for each pixel. 

900 """ 

901 self.log.debug("Computing coadd over %s", bbox) 

902 tempExpName = self.getTempExpDatasetName(self.warpType) 

903 coaddExposure.mask.addMaskPlane("REJECTED") 

904 coaddExposure.mask.addMaskPlane("CLIPPED") 

905 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

906 maskMap = self.setRejectedMaskMapping(statsCtrl) 

907 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

908 maskedImageList = [] 

909 if nImage is not None: 909 ↛ 910line 909 didn't jump to line 910, because the condition on line 909 was never true

910 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

911 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

912 

913 if isinstance(tempExpRef, DeferredDatasetHandle): 913 ↛ 915line 913 didn't jump to line 915, because the condition on line 913 was never true

914 # Gen 3 API 

915 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

916 else: 

917 # Gen 2 API. Delete this when Gen 2 retired 

918 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

919 

920 maskedImage = exposure.getMaskedImage() 

921 mask = maskedImage.getMask() 

922 if altMask is not None: 

923 self.applyAltMaskPlanes(mask, altMask) 

924 imageScaler.scaleMaskedImage(maskedImage) 

925 

926 # Add 1 for each pixel which is not excluded by the exclude mask. 

927 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

928 if nImage is not None: 928 ↛ 929line 928 didn't jump to line 929, because the condition on line 928 was never true

929 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

930 if self.config.removeMaskPlanes: 930 ↛ 932line 930 didn't jump to line 932, because the condition on line 930 was never false

931 self.removeMaskPlanes(maskedImage) 

932 maskedImageList.append(maskedImage) 

933 

934 with self.timer("stack"): 

935 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

936 clipped, # also set output to CLIPPED if sigma-clipped 

937 maskMap) 

938 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

939 if nImage is not None: 939 ↛ 940line 939 didn't jump to line 940, because the condition on line 939 was never true

940 nImage.assign(subNImage, bbox) 

941 

942 def removeMaskPlanes(self, maskedImage): 

943 """Unset the mask of an image for mask planes specified in the config. 

944 

945 Parameters 

946 ---------- 

947 maskedImage : `lsst.afw.image.MaskedImage` 

948 The masked image to be modified. 

949 """ 

950 mask = maskedImage.getMask() 

951 for maskPlane in self.config.removeMaskPlanes: 

952 try: 

953 mask &= ~mask.getPlaneBitMask(maskPlane) 

954 except pexExceptions.InvalidParameterError: 

955 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

956 maskPlane) 

957 

958 @staticmethod 

959 def setRejectedMaskMapping(statsCtrl): 

960 """Map certain mask planes of the warps to new planes for the coadd. 

961 

962 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

963 or CLIPPED, set it to REJECTED on the coadd. 

964 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

965 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

966 

967 Parameters 

968 ---------- 

969 statsCtrl : `lsst.afw.math.StatisticsControl` 

970 Statistics control object for coadd 

971 

972 Returns 

973 ------- 

974 maskMap : `list` of `tuple` of `int` 

975 A list of mappings of mask planes of the warped exposures to 

976 mask planes of the coadd. 

977 """ 

978 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

979 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

980 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

981 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

982 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

983 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

984 (clipped, clipped)] 

985 return maskMap 

986 

987 def applyAltMaskPlanes(self, mask, altMaskSpans): 

988 """Apply in place alt mask formatted as SpanSets to a mask. 

989 

990 Parameters 

991 ---------- 

992 mask : `lsst.afw.image.Mask` 

993 Original mask. 

994 altMaskSpans : `dict` 

995 SpanSet lists to apply. Each element contains the new mask 

996 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

997 and list of SpanSets to apply to the mask. 

998 

999 Returns 

1000 ------- 

1001 mask : `lsst.afw.image.Mask` 

1002 Updated mask. 

1003 """ 

1004 if self.config.doUsePsfMatchedPolygons: 

1005 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1005 ↛ 1013line 1005 didn't jump to line 1013, because the condition on line 1005 was never false

1006 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1007 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1008 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1009 # This mask-clearing step must only occur *before* applying the new masks below 

1010 for spanSet in altMaskSpans['NO_DATA']: 

1011 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1012 

1013 for plane, spanSetList in altMaskSpans.items(): 

1014 maskClipValue = mask.addMaskPlane(plane) 

1015 for spanSet in spanSetList: 

1016 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1017 return mask 

1018 

1019 def shrinkValidPolygons(self, coaddInputs): 

1020 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1021 

1022 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1023 does not have a validPolygon, create one from its bbox. 

1024 

1025 Parameters 

1026 ---------- 

1027 coaddInputs : `lsst.afw.image.coaddInputs` 

1028 Original mask. 

1029 

1030 """ 

1031 for ccd in coaddInputs.ccds: 

1032 polyOrig = ccd.getValidPolygon() 

1033 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1034 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1035 if polyOrig: 1035 ↛ 1036line 1035 didn't jump to line 1036, because the condition on line 1035 was never true

1036 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1037 else: 

1038 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1039 ccd.setValidPolygon(validPolygon) 

1040 

1041 def readBrightObjectMasks(self, dataRef): 

1042 """Retrieve the bright object masks. 

1043 

1044 Returns None on failure. 

1045 

1046 Parameters 

1047 ---------- 

1048 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1049 A Butler dataRef. 

1050 

1051 Returns 

1052 ------- 

1053 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1054 Bright object mask from the Butler object, or None if it cannot 

1055 be retrieved. 

1056 """ 

1057 try: 

1058 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1059 except Exception as e: 

1060 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1061 return None 

1062 

1063 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1064 """Set the bright object masks. 

1065 

1066 Parameters 

1067 ---------- 

1068 exposure : `lsst.afw.image.Exposure` 

1069 Exposure under consideration. 

1070 dataId : `lsst.daf.persistence.dataId` 

1071 Data identifier dict for patch. 

1072 brightObjectMasks : `lsst.afw.table` 

1073 Table of bright objects to mask. 

1074 """ 

1075 

1076 if brightObjectMasks is None: 

1077 self.log.warn("Unable to apply bright object mask: none supplied") 

1078 return 

1079 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1080 mask = exposure.getMaskedImage().getMask() 

1081 wcs = exposure.getWcs() 

1082 plateScale = wcs.getPixelScale().asArcseconds() 

1083 

1084 for rec in brightObjectMasks: 

1085 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1086 if rec["type"] == "box": 

1087 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1088 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1089 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1090 

1091 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1092 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1093 

1094 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1095 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1096 spans = afwGeom.SpanSet(bbox) 

1097 elif rec["type"] == "circle": 

1098 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1099 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1100 else: 

1101 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1102 continue 

1103 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1104 

1105 def setInexactPsf(self, mask): 

1106 """Set INEXACT_PSF mask plane. 

1107 

1108 If any of the input images isn't represented in the coadd (due to 

1109 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1110 these pixels. 

1111 

1112 Parameters 

1113 ---------- 

1114 mask : `lsst.afw.image.Mask` 

1115 Coadded exposure's mask, modified in-place. 

1116 """ 

1117 mask.addMaskPlane("INEXACT_PSF") 

1118 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1119 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1120 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1121 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1122 array = mask.getArray() 

1123 selected = array & (sensorEdge | clipped | rejected) > 0 

1124 array[selected] |= inexactPsf 

1125 

1126 @classmethod 

1127 def _makeArgumentParser(cls): 

1128 """Create an argument parser. 

1129 """ 

1130 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1131 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1132 + cls.ConfigClass().warpType + "Warp", 

1133 help="data ID, e.g. --id tract=12345 patch=1,2", 

1134 ContainerClass=AssembleCoaddDataIdContainer) 

1135 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1136 ContainerClass=SelectDataIdContainer) 

1137 return parser 

1138 

1139 @staticmethod 

1140 def _subBBoxIter(bbox, subregionSize): 

1141 """Iterate over subregions of a bbox. 

1142 

1143 Parameters 

1144 ---------- 

1145 bbox : `lsst.geom.Box2I` 

1146 Bounding box over which to iterate. 

1147 subregionSize: `lsst.geom.Extent2I` 

1148 Size of sub-bboxes. 

1149 

1150 Yields 

1151 ------ 

1152 subBBox : `lsst.geom.Box2I` 

1153 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1154 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1155 the edges of ``bbox``, but it will never be empty. 

1156 """ 

1157 if bbox.isEmpty(): 1157 ↛ 1158line 1157 didn't jump to line 1158, because the condition on line 1157 was never true

1158 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1159 if subregionSize[0] < 1 or subregionSize[1] < 1: 1159 ↛ 1160line 1159 didn't jump to line 1160, because the condition on line 1159 was never true

1160 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1161 

1162 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1163 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1164 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1165 subBBox.clip(bbox) 

1166 if subBBox.isEmpty(): 1166 ↛ 1167line 1166 didn't jump to line 1167, because the condition on line 1166 was never true

1167 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1168 "colShift=%s, rowShift=%s" % 

1169 (bbox, subregionSize, colShift, rowShift)) 

1170 yield subBBox 

1171 

1172 

1173class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1174 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1175 """ 

1176 

1177 def makeDataRefList(self, namespace): 

1178 """Make self.refList from self.idList. 

1179 

1180 Parameters 

1181 ---------- 

1182 namespace 

1183 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1184 """ 

1185 datasetType = namespace.config.coaddName + "Coadd" 

1186 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1187 

1188 for dataId in self.idList: 

1189 # tract and patch are required 

1190 for key in keysCoadd: 

1191 if key not in dataId: 

1192 raise RuntimeError("--id must include " + key) 

1193 

1194 dataRef = namespace.butler.dataRef( 

1195 datasetType=datasetType, 

1196 dataId=dataId, 

1197 ) 

1198 self.refList.append(dataRef) 

1199 

1200 

1201def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1202 """Function to count the number of pixels with a specific mask in a 

1203 footprint. 

1204 

1205 Find the intersection of mask & footprint. Count all pixels in the mask 

1206 that are in the intersection that have bitmask set but do not have 

1207 ignoreMask set. Return the count. 

1208 

1209 Parameters 

1210 ---------- 

1211 mask : `lsst.afw.image.Mask` 

1212 Mask to define intersection region by. 

1213 footprint : `lsst.afw.detection.Footprint` 

1214 Footprint to define the intersection region by. 

1215 bitmask 

1216 Specific mask that we wish to count the number of occurances of. 

1217 ignoreMask 

1218 Pixels to not consider. 

1219 

1220 Returns 

1221 ------- 

1222 result : `int` 

1223 Count of number of pixels in footprint with specified mask. 

1224 """ 

1225 bbox = footprint.getBBox() 

1226 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1227 fp = afwImage.Mask(bbox) 

1228 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1229 footprint.spans.setMask(fp, bitmask) 

1230 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1231 (subMask.getArray() & ignoreMask) == 0).sum() 

1232 

1233 

1234class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1235 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1236 """ 

1237 assembleMeanCoadd = pexConfig.ConfigurableField( 

1238 target=AssembleCoaddTask, 

1239 doc="Task to assemble an initial Coadd using the MEAN statistic.", 

1240 ) 

1241 assembleMeanClipCoadd = pexConfig.ConfigurableField( 

1242 target=AssembleCoaddTask, 

1243 doc="Task to assemble an initial Coadd using the MEANCLIP statistic.", 

1244 ) 

1245 clipDetection = pexConfig.ConfigurableField( 

1246 target=SourceDetectionTask, 

1247 doc="Detect sources on difference between unclipped and clipped coadd") 

1248 minClipFootOverlap = pexConfig.Field( 

1249 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1250 dtype=float, 

1251 default=0.6 

1252 ) 

1253 minClipFootOverlapSingle = pexConfig.Field( 

1254 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1255 "clipped when only one visit overlaps", 

1256 dtype=float, 

1257 default=0.5 

1258 ) 

1259 minClipFootOverlapDouble = pexConfig.Field( 

1260 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1261 "clipped when two visits overlap", 

1262 dtype=float, 

1263 default=0.45 

1264 ) 

1265 maxClipFootOverlapDouble = pexConfig.Field( 

1266 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1267 "considering two visits", 

1268 dtype=float, 

1269 default=0.15 

1270 ) 

1271 minBigOverlap = pexConfig.Field( 

1272 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1273 "when labeling clipped footprints", 

1274 dtype=int, 

1275 default=100 

1276 ) 

1277 

1278 def setDefaults(self): 

1279 """Set default values for clipDetection. 

1280 

1281 Notes 

1282 ----- 

1283 The numeric values for these configuration parameters were 

1284 empirically determined, future work may further refine them. 

1285 """ 

1286 AssembleCoaddConfig.setDefaults(self) 

1287 self.clipDetection.doTempLocalBackground = False 

1288 self.clipDetection.reEstimateBackground = False 

1289 self.clipDetection.returnOriginalFootprints = False 

1290 self.clipDetection.thresholdPolarity = "both" 

1291 self.clipDetection.thresholdValue = 2 

1292 self.clipDetection.nSigmaToGrow = 2 

1293 self.clipDetection.minPixels = 4 

1294 self.clipDetection.isotropicGrow = True 

1295 self.clipDetection.thresholdType = "pixel_stdev" 

1296 self.sigmaClip = 1.5 

1297 self.clipIter = 3 

1298 self.statistic = "MEAN" 

1299 self.assembleMeanCoadd.statistic = 'MEAN' 

1300 self.assembleMeanClipCoadd.statistic = 'MEANCLIP' 

1301 self.assembleMeanCoadd.doWrite = False 

1302 self.assembleMeanClipCoadd.doWrite = False 

1303 

1304 def validate(self): 

1305 if self.doSigmaClip: 

1306 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1307 "Ignoring doSigmaClip.") 

1308 self.doSigmaClip = False 

1309 if self.statistic != "MEAN": 

1310 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1311 "(%s chosen). Please set statistic to MEAN." 

1312 % (self.statistic)) 

1313 AssembleCoaddTask.ConfigClass.validate(self) 

1314 

1315 

1316class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1317 """Assemble a coadded image from a set of coadded temporary exposures, 

1318 being careful to clip & flag areas with potential artifacts. 

1319 

1320 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1321 we clip outliers). The problem with doing this is that when computing the 

1322 coadd PSF at a given location, individual visit PSFs from visits with 

1323 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1324 In this task, we correct for this behavior by creating a new 

1325 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1326 coaddTempExps and the final coadd where 

1327 

1328 i. difference imaging suggests that there is an outlier and 

1329 ii. this outlier appears on only one or two images. 

1330 

1331 Such regions will not contribute to the final coadd. Furthermore, any 

1332 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1333 Note that the algorithm implemented by this task is preliminary and works 

1334 correctly for HSC data. Parameter modifications and or considerable 

1335 redesigning of the algorithm is likley required for other surveys. 

1336 

1337 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1338 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1339 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1340 if you wish. 

1341 

1342 Notes 

1343 ----- 

1344 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1345 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1346 see `baseDebug` for more about ``debug.py`` files. 

1347 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1348 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1349 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1350 for further information. 

1351 

1352 Examples 

1353 -------- 

1354 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1355 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1356 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1357 

1358 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1359 and filter to be coadded (specified using 

1360 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1361 along with a list of coaddTempExps to attempt to coadd (specified using 

1362 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1363 Only the coaddTempExps that cover the specified tract and patch will be 

1364 coadded. A list of the available optional arguments can be obtained by 

1365 calling assembleCoadd.py with the --help command line argument: 

1366 

1367 .. code-block:: none 

1368 

1369 assembleCoadd.py --help 

1370 

1371 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1372 context of multi-band processing, we will generate the HSC-I & -R band 

1373 coadds from HSC engineering test data provided in the ci_hsc package. 

1374 To begin, assuming that the lsst stack has been already set up, we must 

1375 set up the obs_subaru and ci_hsc packages. This defines the environment 

1376 variable $CI_HSC_DIR and points at the location of the package. The raw 

1377 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1378 the coadds, we must first 

1379 

1380 - ``processCcd`` 

1381 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1382 - ``makeSkyMap`` 

1383 create a skymap that covers the area of the sky present in the raw exposures 

1384 - ``makeCoaddTempExp`` 

1385 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1386 

1387 We can perform all of these steps by running 

1388 

1389 .. code-block:: none 

1390 

1391 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1392 

1393 This will produce warped coaddTempExps for each visit. To coadd the 

1394 warped data, we call ``assembleCoadd.py`` as follows: 

1395 

1396 .. code-block:: none 

1397 

1398 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1399 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1400 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1401 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1402 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1403 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1404 --selectId visit=903988 ccd=24 

1405 

1406 This will process the HSC-I band data. The results are written in 

1407 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1408 

1409 You may also choose to run: 

1410 

1411 .. code-block:: none 

1412 

1413 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1414 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1415 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1416 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1417 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1418 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1419 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1420 --selectId visit=903346 ccd=12 

1421 

1422 to generate the coadd for the HSC-R band if you are interested in following 

1423 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1424 """ 

1425 ConfigClass = SafeClipAssembleCoaddConfig 

1426 _DefaultName = "safeClipAssembleCoadd" 

1427 

1428 def __init__(self, *args, **kwargs): 

1429 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1430 schema = afwTable.SourceTable.makeMinimalSchema() 

1431 self.makeSubtask("clipDetection", schema=schema) 

1432 self.makeSubtask("assembleMeanClipCoadd") 

1433 self.makeSubtask("assembleMeanCoadd") 

1434 

1435 @utils.inheritDoc(AssembleCoaddTask) 

1436 @pipeBase.timeMethod 

1437 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1438 """Assemble the coadd for a region. 

1439 

1440 Compute the difference of coadds created with and without outlier 

1441 rejection to identify coadd pixels that have outlier values in some 

1442 individual visits. 

1443 Detect clipped regions on the difference image and mark these regions 

1444 on the one or two individual coaddTempExps where they occur if there 

1445 is significant overlap between the clipped region and a source. This 

1446 leaves us with a set of footprints from the difference image that have 

1447 been identified as having occured on just one or two individual visits. 

1448 However, these footprints were generated from a difference image. It 

1449 is conceivable for a large diffuse source to have become broken up 

1450 into multiple footprints acrosss the coadd difference in this process. 

1451 Determine the clipped region from all overlapping footprints from the 

1452 detected sources in each visit - these are big footprints. 

1453 Combine the small and big clipped footprints and mark them on a new 

1454 bad mask plane. 

1455 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1456 removal. Clipped footprints will no longer make it into the coadd 

1457 because they are marked in the new bad mask plane. 

1458 

1459 Notes 

1460 ----- 

1461 args and kwargs are passed but ignored in order to match the call 

1462 signature expected by the parent task. 

1463 """ 

1464 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1465 mask = exp.getMaskedImage().getMask() 

1466 mask.addMaskPlane("CLIPPED") 

1467 

1468 result = self.detectClip(exp, tempExpRefList) 

1469 

1470 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1471 

1472 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1473 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1474 # Append big footprints from individual Warps to result.clipSpans 

1475 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1476 result.detectionFootprints, maskClipValue, maskDetValue, 

1477 exp.getBBox()) 

1478 # Create mask of the current clipped footprints 

1479 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1480 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1481 

1482 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1483 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1484 maskClip |= maskClipBig 

1485 

1486 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1487 badMaskPlanes = self.config.badMaskPlanes[:] 

1488 badMaskPlanes.append("CLIPPED") 

1489 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1490 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1491 result.clipSpans, mask=badPixelMask) 

1492 

1493 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1494 """Return an exposure that contains the difference between unclipped 

1495 and clipped coadds. 

1496 

1497 Generate a difference image between clipped and unclipped coadds. 

1498 Compute the difference image by subtracting an outlier-clipped coadd 

1499 from an outlier-unclipped coadd. Return the difference image. 

1500 

1501 Parameters 

1502 ---------- 

1503 skyInfo : `lsst.pipe.base.Struct` 

1504 Patch geometry information, from getSkyInfo 

1505 tempExpRefList : `list` 

1506 List of data reference to tempExp 

1507 imageScalerList : `list` 

1508 List of image scalers 

1509 weightList : `list` 

1510 List of weights 

1511 

1512 Returns 

1513 ------- 

1514 exp : `lsst.afw.image.Exposure` 

1515 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1516 """ 

1517 coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList, 

1518 imageScalerList, weightList).coaddExposure 

1519 

1520 coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList, 

1521 imageScalerList, weightList).coaddExposure 

1522 

1523 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1524 coaddDiff -= coaddClip.getMaskedImage() 

1525 exp = afwImage.ExposureF(coaddDiff) 

1526 exp.setPsf(coaddMean.getPsf()) 

1527 return exp 

1528 

1529 def detectClip(self, exp, tempExpRefList): 

1530 """Detect clipped regions on an exposure and set the mask on the 

1531 individual tempExp masks. 

1532 

1533 Detect footprints in the difference image after smoothing the 

1534 difference image with a Gaussian kernal. Identify footprints that 

1535 overlap with one or two input ``coaddTempExps`` by comparing the 

1536 computed overlap fraction to thresholds set in the config. A different 

1537 threshold is applied depending on the number of overlapping visits 

1538 (restricted to one or two). If the overlap exceeds the thresholds, 

1539 the footprint is considered "CLIPPED" and is marked as such on the 

1540 coaddTempExp. Return a struct with the clipped footprints, the indices 

1541 of the ``coaddTempExps`` that end up overlapping with the clipped 

1542 footprints, and a list of new masks for the ``coaddTempExps``. 

1543 

1544 Parameters 

1545 ---------- 

1546 exp : `lsst.afw.image.Exposure` 

1547 Exposure to run detection on. 

1548 tempExpRefList : `list` 

1549 List of data reference to tempExp. 

1550 

1551 Returns 

1552 ------- 

1553 result : `lsst.pipe.base.Struct` 

1554 Result struct with components: 

1555 

1556 - ``clipFootprints``: list of clipped footprints. 

1557 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1558 ``tempExpRefList``. 

1559 - ``clipSpans``: List of dictionaries containing spanSet lists 

1560 to clip. Each element contains the new maskplane name 

1561 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1562 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1563 compressed into footprints. 

1564 """ 

1565 mask = exp.getMaskedImage().getMask() 

1566 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1567 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1568 # Merge positive and negative together footprints together 

1569 fpSet.positive.merge(fpSet.negative) 

1570 footprints = fpSet.positive 

1571 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1572 ignoreMask = self.getBadPixelMask() 

1573 

1574 clipFootprints = [] 

1575 clipIndices = [] 

1576 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1577 

1578 # for use by detectClipBig 

1579 visitDetectionFootprints = [] 

1580 

1581 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1582 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1583 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1584 

1585 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1586 for i, warpRef in enumerate(tempExpRefList): 

1587 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1588 immediate=True).getMaskedImage().getMask() 

1589 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1590 afwImage.PARENT, True) 

1591 maskVisitDet &= maskDetValue 

1592 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1593 visitDetectionFootprints.append(visitFootprints) 

1594 

1595 for j, footprint in enumerate(footprints.getFootprints()): 

1596 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1597 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1598 

1599 # build a list of clipped spans for each visit 

1600 for j, footprint in enumerate(footprints.getFootprints()): 

1601 nPixel = footprint.getArea() 

1602 overlap = [] # hold the overlap with each visit 

1603 indexList = [] # index of visit in global list 

1604 for i in range(len(tempExpRefList)): 

1605 ignore = ignoreArr[i, j] 

1606 overlapDet = overlapDetArr[i, j] 

1607 totPixel = nPixel - ignore 

1608 

1609 # If we have more bad pixels than detection skip 

1610 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1610 ↛ 1612line 1610 didn't jump to line 1612, because the condition on line 1610 was never false

1611 continue 

1612 overlap.append(overlapDet/float(totPixel)) 

1613 indexList.append(i) 

1614 

1615 overlap = numpy.array(overlap) 

1616 if not len(overlap): 1616 ↛ 1619line 1616 didn't jump to line 1619, because the condition on line 1616 was never false

1617 continue 

1618 

1619 keep = False # Should this footprint be marked as clipped? 

1620 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1621 

1622 # If footprint only has one overlap use a lower threshold 

1623 if len(overlap) == 1: 

1624 if overlap[0] > self.config.minClipFootOverlapSingle: 

1625 keep = True 

1626 keepIndex = [0] 

1627 else: 

1628 # This is the general case where only visit should be clipped 

1629 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1630 if len(clipIndex) == 1: 

1631 keep = True 

1632 keepIndex = [clipIndex[0]] 

1633 

1634 # Test if there are clipped objects that overlap two different visits 

1635 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1636 if len(clipIndex) == 2 and len(overlap) > 3: 

1637 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1638 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1639 keep = True 

1640 keepIndex = clipIndex 

1641 

1642 if not keep: 

1643 continue 

1644 

1645 for index in keepIndex: 

1646 globalIndex = indexList[index] 

1647 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1648 

1649 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1650 clipFootprints.append(footprint) 

1651 

1652 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1653 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1654 

1655 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1656 maskClipValue, maskDetValue, coaddBBox): 

1657 """Return individual warp footprints for large artifacts and append 

1658 them to ``clipList`` in place. 

1659 

1660 Identify big footprints composed of many sources in the coadd 

1661 difference that may have originated in a large diffuse source in the 

1662 coadd. We do this by indentifying all clipped footprints that overlap 

1663 significantly with each source in all the coaddTempExps. 

1664 

1665 Parameters 

1666 ---------- 

1667 clipList : `list` 

1668 List of alt mask SpanSets with clipping information. Modified. 

1669 clipFootprints : `list` 

1670 List of clipped footprints. 

1671 clipIndices : `list` 

1672 List of which entries in tempExpClipList each footprint belongs to. 

1673 maskClipValue 

1674 Mask value of clipped pixels. 

1675 maskDetValue 

1676 Mask value of detected pixels. 

1677 coaddBBox : `lsst.geom.Box` 

1678 BBox of the coadd and warps. 

1679 

1680 Returns 

1681 ------- 

1682 bigFootprintsCoadd : `list` 

1683 List of big footprints 

1684 """ 

1685 bigFootprintsCoadd = [] 

1686 ignoreMask = self.getBadPixelMask() 

1687 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1688 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1689 for footprint in visitFootprints.getFootprints(): 1689 ↛ 1690line 1689 didn't jump to line 1690, because the loop on line 1689 never started

1690 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1691 

1692 # build a mask of clipped footprints that are in this visit 

1693 clippedFootprintsVisit = [] 

1694 for foot, clipIndex in zip(clipFootprints, clipIndices): 1694 ↛ 1695line 1694 didn't jump to line 1695, because the loop on line 1694 never started

1695 if index not in clipIndex: 

1696 continue 

1697 clippedFootprintsVisit.append(foot) 

1698 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1699 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1700 

1701 bigFootprintsVisit = [] 

1702 for foot in visitFootprints.getFootprints(): 1702 ↛ 1703line 1702 didn't jump to line 1703, because the loop on line 1702 never started

1703 if foot.getArea() < self.config.minBigOverlap: 

1704 continue 

1705 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1706 if nCount > self.config.minBigOverlap: 

1707 bigFootprintsVisit.append(foot) 

1708 bigFootprintsCoadd.append(foot) 

1709 

1710 for footprint in bigFootprintsVisit: 1710 ↛ 1711line 1710 didn't jump to line 1711, because the loop on line 1710 never started

1711 clippedSpans["CLIPPED"].append(footprint.spans) 

1712 

1713 return bigFootprintsCoadd 

1714 

1715 

1716class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1717 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1718 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1719 "Only PSF-Matched Warps make sense for image subtraction. " 

1720 "Therefore, they must be an additional declared input."), 

1721 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1722 storageClass="ExposureF", 

1723 dimensions=("tract", "patch", "skymap", "visit"), 

1724 deferLoad=True, 

1725 multiple=True 

1726 ) 

1727 templateCoadd = pipeBase.connectionTypes.Output( 

1728 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1729 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1730 name="{fakesType}{outputCoaddName}CoaddPsfMatched", 

1731 storageClass="ExposureF", 

1732 dimensions=("tract", "patch", "skymap", "band"), 

1733 ) 

1734 

1735 def __init__(self, *, config=None): 

1736 super().__init__(config=config) 

1737 if not config.assembleStaticSkyModel.doWrite: 

1738 self.outputs.remove("templateCoadd") 

1739 config.validate() 

1740 

1741 

1742class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1743 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1744 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1745 target=AssembleCoaddTask, 

1746 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1747 " naive/first-iteration model of the static sky.", 

1748 ) 

1749 detect = pexConfig.ConfigurableField( 

1750 target=SourceDetectionTask, 

1751 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1752 ) 

1753 detectTemplate = pexConfig.ConfigurableField( 

1754 target=SourceDetectionTask, 

1755 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1756 ) 

1757 maskStreaks = pexConfig.ConfigurableField( 

1758 target=MaskStreaksTask, 

1759 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1760 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1761 "streakMaskName" 

1762 ) 

1763 streakMaskName = pexConfig.Field( 

1764 dtype=str, 

1765 default="STREAK", 

1766 doc="Name of mask bit used for streaks" 

1767 ) 

1768 maxNumEpochs = pexConfig.Field( 

1769 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1770 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1771 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1772 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1773 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1774 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1775 "than transient and not masked.", 

1776 dtype=int, 

1777 default=2 

1778 ) 

1779 maxFractionEpochsLow = pexConfig.RangeField( 

1780 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1781 "Effective maxNumEpochs = " 

1782 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1783 dtype=float, 

1784 default=0.4, 

1785 min=0., max=1., 

1786 ) 

1787 maxFractionEpochsHigh = pexConfig.RangeField( 

1788 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1789 "Effective maxNumEpochs = " 

1790 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1791 dtype=float, 

1792 default=0.03, 

1793 min=0., max=1., 

1794 ) 

1795 spatialThreshold = pexConfig.RangeField( 

1796 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1797 "temporal criteria. If 0, clip all. If 1, clip none.", 

1798 dtype=float, 

1799 default=0.5, 

1800 min=0., max=1., 

1801 inclusiveMin=True, inclusiveMax=True 

1802 ) 

1803 doScaleWarpVariance = pexConfig.Field( 

1804 doc="Rescale Warp variance plane using empirical noise?", 

1805 dtype=bool, 

1806 default=True, 

1807 ) 

1808 scaleWarpVariance = pexConfig.ConfigurableField( 

1809 target=ScaleVarianceTask, 

1810 doc="Rescale variance on warps", 

1811 ) 

1812 doPreserveContainedBySource = pexConfig.Field( 

1813 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1814 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1815 dtype=bool, 

1816 default=True, 

1817 ) 

1818 doPrefilterArtifacts = pexConfig.Field( 

1819 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1820 "because they will be excluded anyway. This prevents them from contributing " 

1821 "to the outlier epoch count image and potentially being labeled as persistant." 

1822 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1823 dtype=bool, 

1824 default=True 

1825 ) 

1826 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1827 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1828 dtype=str, 

1829 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1830 ) 

1831 prefilterArtifactsRatio = pexConfig.Field( 

1832 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1833 dtype=float, 

1834 default=0.05 

1835 ) 

1836 doFilterMorphological = pexConfig.Field( 

1837 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1838 "be streaks.", 

1839 dtype=bool, 

1840 default=False 

1841 ) 

1842 

1843 def setDefaults(self): 

1844 AssembleCoaddConfig.setDefaults(self) 

1845 self.statistic = 'MEAN' 

1846 self.doUsePsfMatchedPolygons = True 

1847 

1848 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1849 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1850 if "EDGE" in self.badMaskPlanes: 1850 ↛ 1852line 1850 didn't jump to line 1852, because the condition on line 1850 was never false

1851 self.badMaskPlanes.remove('EDGE') 

1852 self.removeMaskPlanes.append('EDGE') 

1853 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1854 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1855 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1856 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1857 self.assembleStaticSkyModel.sigmaClip = 2.5 

1858 self.assembleStaticSkyModel.clipIter = 3 

1859 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1860 self.assembleStaticSkyModel.doWrite = False 

1861 self.detect.doTempLocalBackground = False 

1862 self.detect.reEstimateBackground = False 

1863 self.detect.returnOriginalFootprints = False 

1864 self.detect.thresholdPolarity = "both" 

1865 self.detect.thresholdValue = 5 

1866 self.detect.minPixels = 4 

1867 self.detect.isotropicGrow = True 

1868 self.detect.thresholdType = "pixel_stdev" 

1869 self.detect.nSigmaToGrow = 0.4 

1870 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1871 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1872 self.detectTemplate.nSigmaToGrow = 2.4 

1873 self.detectTemplate.doTempLocalBackground = False 

1874 self.detectTemplate.reEstimateBackground = False 

1875 self.detectTemplate.returnOriginalFootprints = False 

1876 

1877 def validate(self): 

1878 super().validate() 

1879 if self.assembleStaticSkyModel.doNImage: 

1880 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1881 "Please set assembleStaticSkyModel.doNImage=False") 

1882 

1883 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1884 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1885 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1886 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1887 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1888 

1889 

1890class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1891 """Assemble a compareWarp coadded image from a set of warps 

1892 by masking artifacts detected by comparing PSF-matched warps. 

1893 

1894 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1895 we clip outliers). The problem with doing this is that when computing the 

1896 coadd PSF at a given location, individual visit PSFs from visits with 

1897 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1898 In this task, we correct for this behavior by creating a new badMaskPlane 

1899 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1900 an artifact. We populate this plane on the input warps by comparing 

1901 PSF-matched warps with a PSF-matched median coadd which serves as a 

1902 model of the static sky. Any group of pixels that deviates from the 

1903 PSF-matched template coadd by more than config.detect.threshold sigma, 

1904 is an artifact candidate. The candidates are then filtered to remove 

1905 variable sources and sources that are difficult to subtract such as 

1906 bright stars. This filter is configured using the config parameters 

1907 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1908 the maximum fraction of epochs that the deviation can appear in and still 

1909 be considered an artifact. The spatialThreshold is the maximum fraction of 

1910 pixels in the footprint of the deviation that appear in other epochs 

1911 (where other epochs is defined by the temporalThreshold). If the deviant 

1912 region meets this criteria of having a significant percentage of pixels 

1913 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1914 set in the mask. These regions will not contribute to the final coadd. 

1915 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1916 of clipped regions. Note that the algorithm implemented by this task is 

1917 preliminary and works correctly for HSC data. Parameter modifications and 

1918 or considerable redesigning of the algorithm is likley required for other 

1919 surveys. 

1920 

1921 ``CompareWarpAssembleCoaddTask`` sub-classes 

1922 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1923 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1924 

1925 Notes 

1926 ----- 

1927 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1928 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

1929 ``baseDebug`` for more about ``debug.py`` files. 

1930 

1931 This task supports the following debug variables: 

1932 

1933 - ``saveCountIm`` 

1934 If True then save the Epoch Count Image as a fits file in the `figPath` 

1935 - ``figPath`` 

1936 Path to save the debug fits images and figures 

1937 

1938 For example, put something like: 

1939 

1940 .. code-block:: python 

1941 

1942 import lsstDebug 

1943 def DebugInfo(name): 

1944 di = lsstDebug.getInfo(name) 

1945 if name == "lsst.pipe.tasks.assembleCoadd": 

1946 di.saveCountIm = True 

1947 di.figPath = "/desired/path/to/debugging/output/images" 

1948 return di 

1949 lsstDebug.Info = DebugInfo 

1950 

1951 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

1952 ``--debug`` flag. Some subtasks may have their own debug variables; 

1953 see individual Task documentation. 

1954 

1955 Examples 

1956 -------- 

1957 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

1958 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

1959 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

1960 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1961 and filter to be coadded (specified using 

1962 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1963 along with a list of coaddTempExps to attempt to coadd (specified using 

1964 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1965 Only the warps that cover the specified tract and patch will be coadded. 

1966 A list of the available optional arguments can be obtained by calling 

1967 ``assembleCoadd.py`` with the ``--help`` command line argument: 

1968 

1969 .. code-block:: none 

1970 

1971 assembleCoadd.py --help 

1972 

1973 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

1974 context of multi-band processing, we will generate the HSC-I & -R band 

1975 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

1976 To begin, assuming that the lsst stack has been already set up, we must 

1977 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

1978 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

1979 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

1980 directory. To begin assembling the coadds, we must first 

1981 

1982 - processCcd 

1983 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1984 - makeSkyMap 

1985 create a skymap that covers the area of the sky present in the raw exposures 

1986 - makeCoaddTempExp 

1987 warp the individual calibrated exposures to the tangent plane of the coadd 

1988 

1989 We can perform all of these steps by running 

1990 

1991 .. code-block:: none 

1992 

1993 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1994 

1995 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

1996 warped data, we call ``assembleCoadd.py`` as follows: 

1997 

1998 .. code-block:: none 

1999 

2000 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2001 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2002 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2003 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2004 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2005 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2006 --selectId visit=903988 ccd=24 

2007 

2008 This will process the HSC-I band data. The results are written in 

2009 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2010 """ 

2011 ConfigClass = CompareWarpAssembleCoaddConfig 

2012 _DefaultName = "compareWarpAssembleCoadd" 

2013 

2014 def __init__(self, *args, **kwargs): 

2015 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2016 self.makeSubtask("assembleStaticSkyModel") 

2017 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2018 self.makeSubtask("detect", schema=detectionSchema) 

2019 if self.config.doPreserveContainedBySource: 2019 ↛ 2021line 2019 didn't jump to line 2021, because the condition on line 2019 was never false

2020 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2021 if self.config.doScaleWarpVariance: 2021 ↛ 2023line 2021 didn't jump to line 2023, because the condition on line 2021 was never false

2022 self.makeSubtask("scaleWarpVariance") 

2023 if self.config.doFilterMorphological: 2023 ↛ 2024line 2023 didn't jump to line 2024, because the condition on line 2023 was never true

2024 self.makeSubtask("maskStreaks") 

2025 

2026 @utils.inheritDoc(AssembleCoaddTask) 

2027 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2028 """ 

2029 Generate a templateCoadd to use as a naive model of static sky to 

2030 subtract from PSF-Matched warps. 

2031 

2032 Returns 

2033 ------- 

2034 result : `lsst.pipe.base.Struct` 

2035 Result struct with components: 

2036 

2037 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2038 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2039 """ 

2040 # Ensure that psfMatchedWarps are used as input warps for template generation 

2041 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2042 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2043 

2044 # Because subtasks don't have connections we have to make one. 

2045 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2046 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2047 if self.config.assembleStaticSkyModel.doWrite: 

2048 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2049 # Remove template coadd from both subtask's and main tasks outputs, 

2050 # because it is handled by the subtask as `coaddExposure` 

2051 del outputRefs.templateCoadd 

2052 del staticSkyModelOutputRefs.templateCoadd 

2053 

2054 # A PSF-Matched nImage does not exist as a dataset type 

2055 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2056 del staticSkyModelOutputRefs.nImage 

2057 

2058 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2059 staticSkyModelOutputRefs) 

2060 if templateCoadd is None: 

2061 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2062 

2063 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2064 nImage=templateCoadd.nImage, 

2065 warpRefList=templateCoadd.warpRefList, 

2066 imageScalerList=templateCoadd.imageScalerList, 

2067 weightList=templateCoadd.weightList) 

2068 

2069 @utils.inheritDoc(AssembleCoaddTask) 

2070 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2071 """ 

2072 Generate a templateCoadd to use as a naive model of static sky to 

2073 subtract from PSF-Matched warps. 

2074 

2075 Returns 

2076 ------- 

2077 result : `lsst.pipe.base.Struct` 

2078 Result struct with components: 

2079 

2080 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2081 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2082 """ 

2083 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2084 if templateCoadd is None: 2084 ↛ 2085line 2084 didn't jump to line 2085, because the condition on line 2084 was never true

2085 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2086 

2087 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2088 nImage=templateCoadd.nImage, 

2089 warpRefList=templateCoadd.warpRefList, 

2090 imageScalerList=templateCoadd.imageScalerList, 

2091 weightList=templateCoadd.weightList) 

2092 

2093 def _noTemplateMessage(self, warpType): 

2094 warpName = (warpType[0].upper() + warpType[1:]) 

2095 message = """No %(warpName)s warps were found to build the template coadd which is 

2096 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2097 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2098 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2099 

2100 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2101 another algorithm like: 

2102 

2103 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2104 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2105 """ % {"warpName": warpName} 

2106 return message 

2107 

2108 @utils.inheritDoc(AssembleCoaddTask) 

2109 @pipeBase.timeMethod 

2110 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2111 supplementaryData, *args, **kwargs): 

2112 """Assemble the coadd. 

2113 

2114 Find artifacts and apply them to the warps' masks creating a list of 

2115 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2116 plane. Then pass these alternative masks to the base class's `run` 

2117 method. 

2118 

2119 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2120 that must contain a ``templateCoadd`` that serves as the 

2121 model of the static sky. 

2122 """ 

2123 

2124 # Check and match the order of the supplementaryData 

2125 # (PSF-matched) inputs to the order of the direct inputs, 

2126 # so that the artifact mask is applied to the right warp 

2127 dataIds = [ref.dataId for ref in tempExpRefList] 

2128 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2129 

2130 if dataIds != psfMatchedDataIds: 

2131 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2132 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2133 psfMatchedDataIds, dataIds) 

2134 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2135 psfMatchedDataIds, dataIds) 

2136 

2137 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2138 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2139 supplementaryData.warpRefList, 

2140 supplementaryData.imageScalerList) 

2141 

2142 badMaskPlanes = self.config.badMaskPlanes[:] 

2143 badMaskPlanes.append("CLIPPED") 

2144 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2145 

2146 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2147 spanSetMaskList, mask=badPixelMask) 

2148 

2149 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2150 # Psf-Matching moves the real edge inwards 

2151 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2152 return result 

2153 

2154 def applyAltEdgeMask(self, mask, altMaskList): 

2155 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2156 

2157 Parameters 

2158 ---------- 

2159 mask : `lsst.afw.image.Mask` 

2160 Original mask. 

2161 altMaskList : `list` 

2162 List of Dicts containing ``spanSet`` lists. 

2163 Each element contains the new mask plane name (e.g. "CLIPPED 

2164 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2165 the mask. 

2166 """ 

2167 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2168 for visitMask in altMaskList: 

2169 if "EDGE" in visitMask: 2169 ↛ 2168line 2169 didn't jump to line 2168, because the condition on line 2169 was never false

2170 for spanSet in visitMask['EDGE']: 

2171 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2172 

2173 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2174 """Find artifacts. 

2175 

2176 Loop through warps twice. The first loop builds a map with the count 

2177 of how many epochs each pixel deviates from the templateCoadd by more 

2178 than ``config.chiThreshold`` sigma. The second loop takes each 

2179 difference image and filters the artifacts detected in each using 

2180 count map to filter out variable sources and sources that are 

2181 difficult to subtract cleanly. 

2182 

2183 Parameters 

2184 ---------- 

2185 templateCoadd : `lsst.afw.image.Exposure` 

2186 Exposure to serve as model of static sky. 

2187 tempExpRefList : `list` 

2188 List of data references to warps. 

2189 imageScalerList : `list` 

2190 List of image scalers. 

2191 

2192 Returns 

2193 ------- 

2194 altMasks : `list` 

2195 List of dicts containing information about CLIPPED 

2196 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2197 """ 

2198 

2199 self.log.debug("Generating Count Image, and mask lists.") 

2200 coaddBBox = templateCoadd.getBBox() 

2201 slateIm = afwImage.ImageU(coaddBBox) 

2202 epochCountImage = afwImage.ImageU(coaddBBox) 

2203 nImage = afwImage.ImageU(coaddBBox) 

2204 spanSetArtifactList = [] 

2205 spanSetNoDataMaskList = [] 

2206 spanSetEdgeList = [] 

2207 spanSetBadMorphoList = [] 

2208 badPixelMask = self.getBadPixelMask() 

2209 

2210 # mask of the warp diffs should = that of only the warp 

2211 templateCoadd.mask.clearAllMaskPlanes() 

2212 

2213 if self.config.doPreserveContainedBySource: 2213 ↛ 2216line 2213 didn't jump to line 2216, because the condition on line 2213 was never false

2214 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2215 else: 

2216 templateFootprints = None 

2217 

2218 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2219 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2220 if warpDiffExp is not None: 

2221 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2222 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2223 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2224 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2225 fpSet.positive.merge(fpSet.negative) 

2226 footprints = fpSet.positive 

2227 slateIm.set(0) 

2228 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2229 

2230 # Remove artifacts due to defects before they contribute to the epochCountImage 

2231 if self.config.doPrefilterArtifacts: 2231 ↛ 2235line 2231 didn't jump to line 2235, because the condition on line 2231 was never false

2232 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2233 

2234 # Clear mask before adding prefiltered spanSets 

2235 self.detect.clearMask(warpDiffExp.mask) 

2236 for spans in spanSetList: 

2237 spans.setImage(slateIm, 1, doClip=True) 

2238 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2239 epochCountImage += slateIm 

2240 

2241 if self.config.doFilterMorphological: 2241 ↛ 2242line 2241 didn't jump to line 2242, because the condition on line 2241 was never true

2242 maskName = self.config.streakMaskName 

2243 _ = self.maskStreaks.run(warpDiffExp) 

2244 streakMask = warpDiffExp.mask 

2245 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2246 streakMask.getPlaneBitMask(maskName)).split() 

2247 

2248 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2249 # undergo a second convolution. Pixels with data in the direct warp 

2250 # but not in the PSF-matched warp will not have their artifacts detected. 

2251 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2252 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2253 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2254 nansMask.setXY0(warpDiffExp.getXY0()) 

2255 edgeMask = warpDiffExp.mask 

2256 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2257 edgeMask.getPlaneBitMask("EDGE")).split() 

2258 else: 

2259 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2260 # In this case, mask the whole epoch 

2261 nansMask = afwImage.MaskX(coaddBBox, 1) 

2262 spanSetList = [] 

2263 spanSetEdgeMask = [] 

2264 spanSetStreak = [] 

2265 

2266 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2267 

2268 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2269 spanSetArtifactList.append(spanSetList) 

2270 spanSetEdgeList.append(spanSetEdgeMask) 

2271 if self.config.doFilterMorphological: 2271 ↛ 2272line 2271 didn't jump to line 2272, because the condition on line 2271 was never true

2272 spanSetBadMorphoList.append(spanSetStreak) 

2273 

2274 if lsstDebug.Info(__name__).saveCountIm: 2274 ↛ 2275line 2274 didn't jump to line 2275, because the condition on line 2274 was never true

2275 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2276 epochCountImage.writeFits(path) 

2277 

2278 for i, spanSetList in enumerate(spanSetArtifactList): 

2279 if spanSetList: 

2280 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2281 templateFootprints) 

2282 spanSetArtifactList[i] = filteredSpanSetList 

2283 if self.config.doFilterMorphological: 2283 ↛ 2284line 2283 didn't jump to line 2284, because the condition on line 2283 was never true

2284 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2285 

2286 altMasks = [] 

2287 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2288 altMasks.append({'CLIPPED': artifacts, 

2289 'NO_DATA': noData, 

2290 'EDGE': edge}) 

2291 return altMasks 

2292 

2293 def prefilterArtifacts(self, spanSetList, exp): 

2294 """Remove artifact candidates covered by bad mask plane. 

2295 

2296 Any future editing of the candidate list that does not depend on 

2297 temporal information should go in this method. 

2298 

2299 Parameters 

2300 ---------- 

2301 spanSetList : `list` 

2302 List of SpanSets representing artifact candidates. 

2303 exp : `lsst.afw.image.Exposure` 

2304 Exposure containing mask planes used to prefilter. 

2305 

2306 Returns 

2307 ------- 

2308 returnSpanSetList : `list` 

2309 List of SpanSets with artifacts. 

2310 """ 

2311 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2312 goodArr = (exp.mask.array & badPixelMask) == 0 

2313 returnSpanSetList = [] 

2314 bbox = exp.getBBox() 

2315 x0, y0 = exp.getXY0() 

2316 for i, span in enumerate(spanSetList): 

2317 y, x = span.clippedTo(bbox).indices() 

2318 yIndexLocal = numpy.array(y) - y0 

2319 xIndexLocal = numpy.array(x) - x0 

2320 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2321 if goodRatio > self.config.prefilterArtifactsRatio: 2321 ↛ 2316line 2321 didn't jump to line 2316, because the condition on line 2321 was never false

2322 returnSpanSetList.append(span) 

2323 return returnSpanSetList 

2324 

2325 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2326 """Filter artifact candidates. 

2327 

2328 Parameters 

2329 ---------- 

2330 spanSetList : `list` 

2331 List of SpanSets representing artifact candidates. 

2332 epochCountImage : `lsst.afw.image.Image` 

2333 Image of accumulated number of warpDiff detections. 

2334 nImage : `lsst.afw.image.Image` 

2335 Image of the accumulated number of total epochs contributing. 

2336 

2337 Returns 

2338 ------- 

2339 maskSpanSetList : `list` 

2340 List of SpanSets with artifacts. 

2341 """ 

2342 

2343 maskSpanSetList = [] 

2344 x0, y0 = epochCountImage.getXY0() 

2345 for i, span in enumerate(spanSetList): 

2346 y, x = span.indices() 

2347 yIdxLocal = [y1 - y0 for y1 in y] 

2348 xIdxLocal = [x1 - x0 for x1 in x] 

2349 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2350 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2351 

2352 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2353 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2354 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2355 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2356 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2357 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2358 & (outlierN <= effectiveMaxNumEpochs)) 

2359 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2360 if percentBelowThreshold > self.config.spatialThreshold: 

2361 maskSpanSetList.append(span) 

2362 

2363 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2363 ↛ 2376line 2363 didn't jump to line 2376, because the condition on line 2363 was never false

2364 # If a candidate is contained by a footprint on the template coadd, do not clip 

2365 filteredMaskSpanSetList = [] 

2366 for span in maskSpanSetList: 

2367 doKeep = True 

2368 for footprint in footprintsToExclude.positive.getFootprints(): 

2369 if footprint.spans.contains(span): 

2370 doKeep = False 

2371 break 

2372 if doKeep: 

2373 filteredMaskSpanSetList.append(span) 

2374 maskSpanSetList = filteredMaskSpanSetList 

2375 

2376 return maskSpanSetList 

2377 

2378 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2379 """Fetch a warp from the butler and return a warpDiff. 

2380 

2381 Parameters 

2382 ---------- 

2383 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2384 Butler dataRef for the warp. 

2385 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2386 An image scaler object. 

2387 templateCoadd : `lsst.afw.image.Exposure` 

2388 Exposure to be substracted from the scaled warp. 

2389 

2390 Returns 

2391 ------- 

2392 warp : `lsst.afw.image.Exposure` 

2393 Exposure of the image difference between the warp and template. 

2394 """ 

2395 

2396 # If the PSF-Matched warp did not exist for this direct warp 

2397 # None is holding its place to maintain order in Gen 3 

2398 if warpRef is None: 

2399 return None 

2400 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2401 warpName = self.getTempExpDatasetName('psfMatched') 

2402 if not isinstance(warpRef, DeferredDatasetHandle): 2402 ↛ 2406line 2402 didn't jump to line 2406, because the condition on line 2402 was never false

2403 if not warpRef.datasetExists(warpName): 2403 ↛ 2404line 2403 didn't jump to line 2404, because the condition on line 2403 was never true

2404 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2405 return None 

2406 warp = warpRef.get(datasetType=warpName, immediate=True) 

2407 # direct image scaler OK for PSF-matched Warp 

2408 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2409 mi = warp.getMaskedImage() 

2410 if self.config.doScaleWarpVariance: 2410 ↛ 2415line 2410 didn't jump to line 2415, because the condition on line 2410 was never false

2411 try: 

2412 self.scaleWarpVariance.run(mi) 

2413 except Exception as exc: 

2414 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2415 mi -= templateCoadd.getMaskedImage() 

2416 return warp 

2417 

2418 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2419 """Return a path to which to write debugging output. 

2420 

2421 Creates a hyphen-delimited string of dataId values for simple filenames. 

2422 

2423 Parameters 

2424 ---------- 

2425 prefix : `str` 

2426 Prefix for filename. 

2427 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2428 Butler dataRef to make the path from. 

2429 coaddLevel : `bool`, optional. 

2430 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2431 'filter', but no 'visit'). 

2432 

2433 Returns 

2434 ------- 

2435 result : `str` 

2436 Path for debugging output. 

2437 """ 

2438 if coaddLevel: 

2439 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2440 else: 

2441 keys = warpRef.dataId.keys() 

2442 keyList = sorted(keys, reverse=True) 

2443 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2444 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2445 return os.path.join(directory, filename) 

2446 

2447 

2448def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None): 

2449 """Match the order of one list to another, padding if necessary 

2450 

2451 Parameters 

2452 ---------- 

2453 inputList : list 

2454 List to be reordered and padded. Elements can be any type. 

2455 inputKeys : iterable 

2456 Iterable of values to be compared with outputKeys. 

2457 Length must match `inputList` 

2458 outputKeys : iterable 

2459 Iterable of values to be compared with inputKeys. 

2460 padWith : 

2461 Any value to be inserted where inputKey not in outputKeys 

2462 

2463 Returns 

2464 ------- 

2465 list 

2466 Copy of inputList reordered per outputKeys and padded with `padWith` 

2467 so that the length matches length of outputKeys. 

2468 """ 

2469 outputList = [] 

2470 for d in outputKeys: 

2471 if d in inputKeys: 

2472 outputList.append(inputList[inputKeys.index(d)]) 

2473 else: 

2474 outputList.append(padWith) 

2475 return outputList