Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from lsst.meas.algorithms import SourceDetectionTask 

48from lsst.daf.butler import DeferredDatasetHandle 

49 

50__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

51 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

52 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

53 

54 

55class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

56 dimensions=("tract", "patch", "band", "skymap"), 

57 defaultTemplates={"inputCoaddName": "deep", 

58 "outputCoaddName": "deep", 

59 "warpType": "direct", 

60 "warpTypeSuffix": "", 

61 "fakesType": ""}): 

62 inputWarps = pipeBase.connectionTypes.Input( 

63 doc=("Input list of warps to be assemebled i.e. stacked." 

64 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

65 name="{inputCoaddName}Coadd_{warpType}Warp", 

66 storageClass="ExposureF", 

67 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

68 deferLoad=True, 

69 multiple=True 

70 ) 

71 skyMap = pipeBase.connectionTypes.Input( 

72 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

73 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

74 storageClass="SkyMap", 

75 dimensions=("skymap", ), 

76 ) 

77 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

78 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

79 " BRIGHT_OBJECT."), 

80 name="brightObjectMask", 

81 storageClass="ObjectMaskCatalog", 

82 dimensions=("tract", "patch", "skymap", "band"), 

83 ) 

84 coaddExposure = pipeBase.connectionTypes.Output( 

85 doc="Output coadded exposure, produced by stacking input warps", 

86 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

87 storageClass="ExposureF", 

88 dimensions=("tract", "patch", "skymap", "band"), 

89 ) 

90 nImage = pipeBase.connectionTypes.Output( 

91 doc="Output image of number of input images per pixel", 

92 name="{outputCoaddName}Coadd_nImage", 

93 storageClass="ImageU", 

94 dimensions=("tract", "patch", "skymap", "band"), 

95 ) 

96 

97 def __init__(self, *, config=None): 

98 super().__init__(config=config) 

99 

100 # Override the connection's name template with config to replicate Gen2 behavior 

101 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

102 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

103 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

104 templateValues['warpType'] = config.warpType 

105 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

106 if config.hasFakes: 

107 templateValues['fakesType'] = "_fakes" 

108 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

109 for name in self.allConnections} 

110 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

111 # End code to remove after deprecation 

112 

113 if not config.doMaskBrightObjects: 

114 self.prerequisiteInputs.remove("brightObjectMask") 

115 

116 if not config.doNImage: 

117 self.outputs.remove("nImage") 

118 

119 

120class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

121 pipelineConnections=AssembleCoaddConnections): 

122 """Configuration parameters for the `AssembleCoaddTask`. 

123 

124 Notes 

125 ----- 

126 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

127 only set the bitplane config.brightObjectMaskName. To make this useful you 

128 *must* also configure the flags.pixel algorithm, for example by adding 

129 

130 .. code-block:: none 

131 

132 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

133 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

134 

135 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

136 """ 

137 warpType = pexConfig.Field( 

138 doc="Warp name: one of 'direct' or 'psfMatched'", 

139 dtype=str, 

140 default="direct", 

141 ) 

142 subregionSize = pexConfig.ListField( 

143 dtype=int, 

144 doc="Width, height of stack subregion size; " 

145 "make small enough that a full stack of images will fit into memory at once.", 

146 length=2, 

147 default=(2000, 2000), 

148 ) 

149 statistic = pexConfig.Field( 

150 dtype=str, 

151 doc="Main stacking statistic for aggregating over the epochs.", 

152 default="MEANCLIP", 

153 ) 

154 doSigmaClip = pexConfig.Field( 

155 dtype=bool, 

156 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

157 default=False, 

158 ) 

159 sigmaClip = pexConfig.Field( 

160 dtype=float, 

161 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

162 default=3.0, 

163 ) 

164 clipIter = pexConfig.Field( 

165 dtype=int, 

166 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

167 default=2, 

168 ) 

169 calcErrorFromInputVariance = pexConfig.Field( 

170 dtype=bool, 

171 doc="Calculate coadd variance from input variance by stacking statistic." 

172 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

173 default=True, 

174 ) 

175 scaleZeroPoint = pexConfig.ConfigurableField( 

176 target=ScaleZeroPointTask, 

177 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

178 ) 

179 doInterp = pexConfig.Field( 

180 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

181 dtype=bool, 

182 default=True, 

183 ) 

184 interpImage = pexConfig.ConfigurableField( 

185 target=InterpImageTask, 

186 doc="Task to interpolate (and extrapolate) over NaN pixels", 

187 ) 

188 doWrite = pexConfig.Field( 

189 doc="Persist coadd?", 

190 dtype=bool, 

191 default=True, 

192 ) 

193 doNImage = pexConfig.Field( 

194 doc="Create image of number of contributing exposures for each pixel", 

195 dtype=bool, 

196 default=False, 

197 ) 

198 doUsePsfMatchedPolygons = pexConfig.Field( 

199 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

200 dtype=bool, 

201 default=False, 

202 ) 

203 maskPropagationThresholds = pexConfig.DictField( 

204 keytype=str, 

205 itemtype=float, 

206 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

207 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

208 "would have contributed exceeds this value."), 

209 default={"SAT": 0.1}, 

210 ) 

211 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

212 doc="Mask planes to remove before coadding") 

213 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

214 doc="Set mask and flag bits for bright objects?") 

215 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

216 doc="Name of mask bit used for bright objects") 

217 coaddPsf = pexConfig.ConfigField( 

218 doc="Configuration for CoaddPsf", 

219 dtype=measAlg.CoaddPsfConfig, 

220 ) 

221 doAttachTransmissionCurve = pexConfig.Field( 

222 dtype=bool, default=False, optional=False, 

223 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

224 "(requires all input Exposures to have TransmissionCurves).") 

225 ) 

226 hasFakes = pexConfig.Field( 

227 dtype=bool, 

228 default=False, 

229 doc="Should be set to True if fake sources have been inserted into the input data." 

230 ) 

231 

232 def setDefaults(self): 

233 super().setDefaults() 

234 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

235 

236 def validate(self): 

237 super().validate() 

238 if self.doPsfMatch: 

239 # Backwards compatibility. 

240 # Configs do not have loggers 

241 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

242 self.warpType = 'psfMatched' 

243 if self.doSigmaClip and self.statistic != "MEANCLIP": 

244 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

245 self.statistic = "MEANCLIP" 

246 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

247 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

248 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

249 

250 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

251 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

252 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

253 if str(k) not in unstackableStats] 

254 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

255 % (self.statistic, stackableStats)) 

256 

257 

258class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

259 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

260 

261 We want to assemble a coadded image from a set of Warps (also called 

262 coadded temporary exposures or ``coaddTempExps``). 

263 Each input Warp covers a patch on the sky and corresponds to a single 

264 run/visit/exposure of the covered patch. We provide the task with a list 

265 of Warps (``selectDataList``) from which it selects Warps that cover the 

266 specified patch (pointed at by ``dataRef``). 

267 Each Warp that goes into a coadd will typically have an independent 

268 photometric zero-point. Therefore, we must scale each Warp to set it to 

269 a common photometric zeropoint. WarpType may be one of 'direct' or 

270 'psfMatched', and the boolean configs `config.makeDirect` and 

271 `config.makePsfMatched` set which of the warp types will be coadded. 

272 The coadd is computed as a mean with optional outlier rejection. 

273 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

274 Finally, Warps can have bad 'NaN' pixels which received no input from the 

275 source calExps. We interpolate over these bad (NaN) pixels. 

276 

277 `AssembleCoaddTask` uses several sub-tasks. These are 

278 

279 - `ScaleZeroPointTask` 

280 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

281 - `InterpImageTask` 

282 - interpolate across bad pixels (NaN) in the final coadd 

283 

284 You can retarget these subtasks if you wish. 

285 

286 Notes 

287 ----- 

288 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

289 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

290 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

291 no debug variables of its own. Some of the subtasks may support debug 

292 variables. See the documentation for the subtasks for further information. 

293 

294 Examples 

295 -------- 

296 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

297 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

298 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

299 inputs: a data reference to the tract patch and filter to be coadded, and 

300 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

301 ``--selectId``, respectively: 

302 

303 .. code-block:: none 

304 

305 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

306 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

307 

308 Only the Warps that cover the specified tract and patch will be coadded. 

309 A list of the available optional arguments can be obtained by calling 

310 ``assembleCoadd.py`` with the ``--help`` command line argument: 

311 

312 .. code-block:: none 

313 

314 assembleCoadd.py --help 

315 

316 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

317 multi-band processing, we will generate the HSC-I & -R band coadds from 

318 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

319 assuming that the lsst stack has been already set up, we must set up the 

320 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

321 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

322 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

323 coadds, we must first 

324 

325 - processCcd 

326 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

327 - makeSkyMap 

328 - create a skymap that covers the area of the sky present in the raw exposures 

329 - makeCoaddTempExp 

330 - warp the individual calibrated exposures to the tangent plane of the coadd 

331 

332 We can perform all of these steps by running 

333 

334 .. code-block:: none 

335 

336 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

337 

338 This will produce warped exposures for each visit. To coadd the warped 

339 data, we call assembleCoadd.py as follows: 

340 

341 .. code-block:: none 

342 

343 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

344 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

345 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

346 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

347 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

348 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

349 --selectId visit=903988 ccd=24 

350 

351 that will process the HSC-I band data. The results are written in 

352 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

353 

354 You may also choose to run: 

355 

356 .. code-block:: none 

357 

358 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

359 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

360 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

361 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

362 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

363 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

364 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

365 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

366 

367 to generate the coadd for the HSC-R band if you are interested in 

368 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

369 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

370 rather than `AssembleCoaddTask` to make the coadd. 

371 """ 

372 ConfigClass = AssembleCoaddConfig 

373 _DefaultName = "assembleCoadd" 

374 

375 def __init__(self, *args, **kwargs): 

376 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

377 if args: 377 ↛ 378line 377 didn't jump to line 378, because the condition on line 377 was never true

378 argNames = ["config", "name", "parentTask", "log"] 

379 kwargs.update({k: v for k, v in zip(argNames, args)}) 

380 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

381 "PipelineTask will not take positional args" % argNames, FutureWarning) 

382 

383 super().__init__(**kwargs) 

384 self.makeSubtask("interpImage") 

385 self.makeSubtask("scaleZeroPoint") 

386 

387 if self.config.doMaskBrightObjects: 387 ↛ 388line 387 didn't jump to line 388, because the condition on line 387 was never true

388 mask = afwImage.Mask() 

389 try: 

390 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

391 except pexExceptions.LsstCppException: 

392 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

393 mask.getMaskPlaneDict().keys()) 

394 del mask 

395 

396 self.warpType = self.config.warpType 

397 

398 @utils.inheritDoc(pipeBase.PipelineTask) 

399 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

400 # Docstring to be formatted with info from PipelineTask.runQuantum 

401 """ 

402 Notes 

403 ----- 

404 Assemble a coadd from a set of Warps. 

405 

406 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

407 Analogous to `runDataRef`, it prepares all the data products to be 

408 passed to `run`, and processes the results before returning a struct 

409 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

410 Therefore, its inputs are accessed subregion by subregion 

411 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

412 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

413 correspond to an update in `runDataRef` while both entry points 

414 are used. 

415 """ 

416 inputData = butlerQC.get(inputRefs) 

417 

418 # Construct skyInfo expected by run 

419 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

420 skyMap = inputData["skyMap"] 

421 outputDataId = butlerQC.quantum.dataId 

422 

423 inputData['skyInfo'] = makeSkyInfo(skyMap, 

424 tractId=outputDataId['tract'], 

425 patchId=outputDataId['patch']) 

426 

427 # Construct list of input Deferred Datasets 

428 # These quack a bit like like Gen2 DataRefs 

429 warpRefList = inputData['inputWarps'] 

430 # Perform same middle steps as `runDataRef` does 

431 inputs = self.prepareInputs(warpRefList) 

432 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

433 self.getTempExpDatasetName(self.warpType)) 

434 if len(inputs.tempExpRefList) == 0: 

435 self.log.warn("No coadd temporary exposures found") 

436 return 

437 

438 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

439 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

440 inputs.weightList, supplementaryData=supplementaryData) 

441 

442 inputData.setdefault('brightObjectMask', None) 

443 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

444 

445 if self.config.doWrite: 

446 butlerQC.put(retStruct, outputRefs) 

447 return retStruct 

448 

449 @pipeBase.timeMethod 

450 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

451 """Assemble a coadd from a set of Warps. 

452 

453 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

454 Compute weights to be applied to each Warp and 

455 find scalings to match the photometric zeropoint to a reference Warp. 

456 Assemble the Warps using `run`. Interpolate over NaNs and 

457 optionally write the coadd to disk. Return the coadded exposure. 

458 

459 Parameters 

460 ---------- 

461 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

462 Data reference defining the patch for coaddition and the 

463 reference Warp (if ``config.autoReference=False``). 

464 Used to access the following data products: 

465 - ``self.config.coaddName + "Coadd_skyMap"`` 

466 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

467 - ``self.config.coaddName + "Coadd"`` 

468 selectDataList : `list` 

469 List of data references to Calexps. Data to be coadded will be 

470 selected from this list based on overlap with the patch defined 

471 by dataRef, grouped by visit, and converted to a list of data 

472 references to warps. 

473 warpRefList : `list` 

474 List of data references to Warps to be coadded. 

475 Note: `warpRefList` is just the new name for `tempExpRefList`. 

476 

477 Returns 

478 ------- 

479 retStruct : `lsst.pipe.base.Struct` 

480 Result struct with components: 

481 

482 - ``coaddExposure``: coadded exposure (``Exposure``). 

483 - ``nImage``: exposure count image (``Image``). 

484 """ 

485 if selectDataList and warpRefList: 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true

486 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

487 "and which to use is ambiguous. Please pass only one.") 

488 

489 skyInfo = self.getSkyInfo(dataRef) 

490 if warpRefList is None: 

491 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

492 if len(calExpRefList) == 0: 492 ↛ 493line 492 didn't jump to line 493, because the condition on line 492 was never true

493 self.log.warn("No exposures to coadd") 

494 return 

495 self.log.info("Coadding %d exposures", len(calExpRefList)) 

496 

497 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

498 

499 inputData = self.prepareInputs(warpRefList) 

500 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

501 self.getTempExpDatasetName(self.warpType)) 

502 if len(inputData.tempExpRefList) == 0: 502 ↛ 503line 502 didn't jump to line 503, because the condition on line 502 was never true

503 self.log.warn("No coadd temporary exposures found") 

504 return 

505 

506 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

507 

508 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

509 inputData.weightList, supplementaryData=supplementaryData) 

510 

511 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

512 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

513 

514 if self.config.doWrite: 

515 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 515 ↛ 516line 515 didn't jump to line 516, because the condition on line 515 was never true

516 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

517 else: 

518 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

519 self.log.info("Persisting %s" % coaddDatasetName) 

520 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

521 if self.config.doNImage and retStruct.nImage is not None: 521 ↛ 522line 521 didn't jump to line 522, because the condition on line 521 was never true

522 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

523 

524 return retStruct 

525 

526 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

527 """Interpolate over missing data and mask bright stars. 

528 

529 Parameters 

530 ---------- 

531 coaddExposure : `lsst.afw.image.Exposure` 

532 The coadded exposure to process. 

533 dataRef : `lsst.daf.persistence.ButlerDataRef` 

534 Butler data reference for supplementary data. 

535 """ 

536 if self.config.doInterp: 536 ↛ 543line 536 didn't jump to line 543, because the condition on line 536 was never false

537 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

538 # The variance must be positive; work around for DM-3201. 

539 varArray = coaddExposure.variance.array 

540 with numpy.errstate(invalid="ignore"): 

541 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

542 

543 if self.config.doMaskBrightObjects: 543 ↛ 544line 543 didn't jump to line 544, because the condition on line 543 was never true

544 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

545 

546 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

547 """Make additional inputs to run() specific to subclasses (Gen2) 

548 

549 Duplicates interface of `runDataRef` method 

550 Available to be implemented by subclasses only if they need the 

551 coadd dataRef for performing preliminary processing before 

552 assembling the coadd. 

553 

554 Parameters 

555 ---------- 

556 dataRef : `lsst.daf.persistence.ButlerDataRef` 

557 Butler data reference for supplementary data. 

558 selectDataList : `list` (optional) 

559 Optional List of data references to Calexps. 

560 warpRefList : `list` (optional) 

561 Optional List of data references to Warps. 

562 """ 

563 return pipeBase.Struct() 

564 

565 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

566 """Make additional inputs to run() specific to subclasses (Gen3) 

567 

568 Duplicates interface of `runQuantum` method. 

569 Available to be implemented by subclasses only if they need the 

570 coadd dataRef for performing preliminary processing before 

571 assembling the coadd. 

572 

573 Parameters 

574 ---------- 

575 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

576 Gen3 Butler object for fetching additional data products before 

577 running the Task specialized for quantum being processed 

578 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

579 Attributes are the names of the connections describing input dataset types. 

580 Values are DatasetRefs that task consumes for corresponding dataset type. 

581 DataIds are guaranteed to match data objects in ``inputData``. 

582 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

583 Attributes are the names of the connections describing output dataset types. 

584 Values are DatasetRefs that task is to produce 

585 for corresponding dataset type. 

586 """ 

587 return pipeBase.Struct() 

588 

589 def getTempExpRefList(self, patchRef, calExpRefList): 

590 """Generate list data references corresponding to warped exposures 

591 that lie within the patch to be coadded. 

592 

593 Parameters 

594 ---------- 

595 patchRef : `dataRef` 

596 Data reference for patch. 

597 calExpRefList : `list` 

598 List of data references for input calexps. 

599 

600 Returns 

601 ------- 

602 tempExpRefList : `list` 

603 List of Warp/CoaddTempExp data references. 

604 """ 

605 butler = patchRef.getButler() 

606 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

607 self.getTempExpDatasetName(self.warpType)) 

608 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

609 g, groupData.keys) for 

610 g in groupData.groups.keys()] 

611 return tempExpRefList 

612 

613 def prepareInputs(self, refList): 

614 """Prepare the input warps for coaddition by measuring the weight for 

615 each warp and the scaling for the photometric zero point. 

616 

617 Each Warp has its own photometric zeropoint and background variance. 

618 Before coadding these Warps together, compute a scale factor to 

619 normalize the photometric zeropoint and compute the weight for each Warp. 

620 

621 Parameters 

622 ---------- 

623 refList : `list` 

624 List of data references to tempExp 

625 

626 Returns 

627 ------- 

628 result : `lsst.pipe.base.Struct` 

629 Result struct with components: 

630 

631 - ``tempExprefList``: `list` of data references to tempExp. 

632 - ``weightList``: `list` of weightings. 

633 - ``imageScalerList``: `list` of image scalers. 

634 """ 

635 statsCtrl = afwMath.StatisticsControl() 

636 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

637 statsCtrl.setNumIter(self.config.clipIter) 

638 statsCtrl.setAndMask(self.getBadPixelMask()) 

639 statsCtrl.setNanSafe(True) 

640 # compute tempExpRefList: a list of tempExpRef that actually exist 

641 # and weightList: a list of the weight of the associated coadd tempExp 

642 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

643 tempExpRefList = [] 

644 weightList = [] 

645 imageScalerList = [] 

646 tempExpName = self.getTempExpDatasetName(self.warpType) 

647 for tempExpRef in refList: 

648 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

649 # therefore have no datasetExists() method 

650 if not isinstance(tempExpRef, DeferredDatasetHandle): 650 ↛ 655line 650 didn't jump to line 655, because the condition on line 650 was never false

651 if not tempExpRef.datasetExists(tempExpName): 

652 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

653 continue 

654 

655 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

656 # Ignore any input warp that is empty of data 

657 if numpy.isnan(tempExp.image.array).all(): 657 ↛ 658line 657 didn't jump to line 658, because the condition on line 657 was never true

658 continue 

659 maskedImage = tempExp.getMaskedImage() 

660 imageScaler = self.scaleZeroPoint.computeImageScaler( 

661 exposure=tempExp, 

662 dataRef=tempExpRef, 

663 ) 

664 try: 

665 imageScaler.scaleMaskedImage(maskedImage) 

666 except Exception as e: 

667 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

668 continue 

669 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

670 afwMath.MEANCLIP, statsCtrl) 

671 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

672 weight = 1.0 / float(meanVar) 

673 if not numpy.isfinite(weight): 673 ↛ 674line 673 didn't jump to line 674, because the condition on line 673 was never true

674 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

675 continue 

676 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

677 

678 del maskedImage 

679 del tempExp 

680 

681 tempExpRefList.append(tempExpRef) 

682 weightList.append(weight) 

683 imageScalerList.append(imageScaler) 

684 

685 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

686 imageScalerList=imageScalerList) 

687 

688 def prepareStats(self, mask=None): 

689 """Prepare the statistics for coadding images. 

690 

691 Parameters 

692 ---------- 

693 mask : `int`, optional 

694 Bit mask value to exclude from coaddition. 

695 

696 Returns 

697 ------- 

698 stats : `lsst.pipe.base.Struct` 

699 Statistics structure with the following fields: 

700 

701 - ``statsCtrl``: Statistics control object for coadd 

702 (`lsst.afw.math.StatisticsControl`) 

703 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

704 """ 

705 if mask is None: 

706 mask = self.getBadPixelMask() 

707 statsCtrl = afwMath.StatisticsControl() 

708 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

709 statsCtrl.setNumIter(self.config.clipIter) 

710 statsCtrl.setAndMask(mask) 

711 statsCtrl.setNanSafe(True) 

712 statsCtrl.setWeighted(True) 

713 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

714 for plane, threshold in self.config.maskPropagationThresholds.items(): 

715 bit = afwImage.Mask.getMaskPlane(plane) 

716 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

717 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

718 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

719 

720 @pipeBase.timeMethod 

721 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

722 altMaskList=None, mask=None, supplementaryData=None): 

723 """Assemble a coadd from input warps 

724 

725 Assemble the coadd using the provided list of coaddTempExps. Since 

726 the full coadd covers a patch (a large area), the assembly is 

727 performed over small areas on the image at a time in order to 

728 conserve memory usage. Iterate over subregions within the outer 

729 bbox of the patch using `assembleSubregion` to stack the corresponding 

730 subregions from the coaddTempExps with the statistic specified. 

731 Set the edge bits the coadd mask based on the weight map. 

732 

733 Parameters 

734 ---------- 

735 skyInfo : `lsst.pipe.base.Struct` 

736 Struct with geometric information about the patch. 

737 tempExpRefList : `list` 

738 List of data references to Warps (previously called CoaddTempExps). 

739 imageScalerList : `list` 

740 List of image scalers. 

741 weightList : `list` 

742 List of weights 

743 altMaskList : `list`, optional 

744 List of alternate masks to use rather than those stored with 

745 tempExp. 

746 mask : `int`, optional 

747 Bit mask value to exclude from coaddition. 

748 supplementaryData : lsst.pipe.base.Struct, optional 

749 Struct with additional data products needed to assemble coadd. 

750 Only used by subclasses that implement `makeSupplementaryData` 

751 and override `run`. 

752 

753 Returns 

754 ------- 

755 result : `lsst.pipe.base.Struct` 

756 Result struct with components: 

757 

758 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

759 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

760 - ``warpRefList``: input list of refs to the warps ( 

761 ``lsst.daf.butler.DeferredDatasetHandle`` or 

762 ``lsst.daf.persistence.ButlerDataRef``) 

763 (unmodified) 

764 - ``imageScalerList``: input list of image scalers (unmodified) 

765 - ``weightList``: input list of weights (unmodified) 

766 """ 

767 tempExpName = self.getTempExpDatasetName(self.warpType) 

768 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

769 stats = self.prepareStats(mask=mask) 

770 

771 if altMaskList is None: 

772 altMaskList = [None]*len(tempExpRefList) 

773 

774 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

775 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

776 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

777 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

778 coaddMaskedImage = coaddExposure.getMaskedImage() 

779 subregionSizeArr = self.config.subregionSize 

780 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

781 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

782 if self.config.doNImage: 782 ↛ 783line 782 didn't jump to line 783, because the condition on line 782 was never true

783 nImage = afwImage.ImageU(skyInfo.bbox) 

784 else: 

785 nImage = None 

786 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

787 try: 

788 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

789 weightList, altMaskList, stats.flags, stats.ctrl, 

790 nImage=nImage) 

791 except Exception as e: 

792 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

793 

794 self.setInexactPsf(coaddMaskedImage.getMask()) 

795 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

796 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

797 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

798 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

799 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

800 weightList=weightList) 

801 

802 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

803 """Set the metadata for the coadd. 

804 

805 This basic implementation sets the filter from the first input. 

806 

807 Parameters 

808 ---------- 

809 coaddExposure : `lsst.afw.image.Exposure` 

810 The target exposure for the coadd. 

811 tempExpRefList : `list` 

812 List of data references to tempExp. 

813 weightList : `list` 

814 List of weights. 

815 """ 

816 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

817 tempExpName = self.getTempExpDatasetName(self.warpType) 

818 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

819 # (and we need more than just the PropertySet that contains the header), which is not possible 

820 # with the current butler (see #2777). 

821 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

822 

823 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 823 ↛ 825line 823 didn't jump to line 825, because the condition on line 823 was never true

824 # Gen 3 API 

825 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

826 else: 

827 # Gen 2 API. Delete this when Gen 2 retired 

828 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

829 for tempExpRef in tempExpRefList] 

830 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

831 

832 coaddExposure.setFilter(tempExpList[0].getFilter()) 

833 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

834 coaddInputs.ccds.reserve(numCcds) 

835 coaddInputs.visits.reserve(len(tempExpList)) 

836 

837 for tempExp, weight in zip(tempExpList, weightList): 

838 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

839 

840 if self.config.doUsePsfMatchedPolygons: 

841 self.shrinkValidPolygons(coaddInputs) 

842 

843 coaddInputs.visits.sort() 

844 if self.warpType == "psfMatched": 

845 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

846 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

847 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

848 # having the maximum width (sufficient because square) 

849 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

850 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

851 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

852 else: 

853 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

854 self.config.coaddPsf.makeControl()) 

855 coaddExposure.setPsf(psf) 

856 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

857 coaddExposure.getWcs()) 

858 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

859 if self.config.doAttachTransmissionCurve: 

860 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

861 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

862 

863 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

864 altMaskList, statsFlags, statsCtrl, nImage=None): 

865 """Assemble the coadd for a sub-region. 

866 

867 For each coaddTempExp, check for (and swap in) an alternative mask 

868 if one is passed. Remove mask planes listed in 

869 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

870 `lsst.afw.math.statisticsStack` with the statistic specified by 

871 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

872 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

873 an N-sigma clipped mean where N and iterations are specified by 

874 statsCtrl. Assign the stacked subregion back to the coadd. 

875 

876 Parameters 

877 ---------- 

878 coaddExposure : `lsst.afw.image.Exposure` 

879 The target exposure for the coadd. 

880 bbox : `lsst.geom.Box` 

881 Sub-region to coadd. 

882 tempExpRefList : `list` 

883 List of data reference to tempExp. 

884 imageScalerList : `list` 

885 List of image scalers. 

886 weightList : `list` 

887 List of weights. 

888 altMaskList : `list` 

889 List of alternate masks to use rather than those stored with 

890 tempExp, or None. Each element is dict with keys = mask plane 

891 name to which to add the spans. 

892 statsFlags : `lsst.afw.math.Property` 

893 Property object for statistic for coadd. 

894 statsCtrl : `lsst.afw.math.StatisticsControl` 

895 Statistics control object for coadd. 

896 nImage : `lsst.afw.image.ImageU`, optional 

897 Keeps track of exposure count for each pixel. 

898 """ 

899 self.log.debug("Computing coadd over %s", bbox) 

900 tempExpName = self.getTempExpDatasetName(self.warpType) 

901 coaddExposure.mask.addMaskPlane("REJECTED") 

902 coaddExposure.mask.addMaskPlane("CLIPPED") 

903 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

904 maskMap = self.setRejectedMaskMapping(statsCtrl) 

905 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

906 maskedImageList = [] 

907 if nImage is not None: 907 ↛ 908line 907 didn't jump to line 908, because the condition on line 907 was never true

908 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

909 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

910 

911 if isinstance(tempExpRef, DeferredDatasetHandle): 911 ↛ 913line 911 didn't jump to line 913, because the condition on line 911 was never true

912 # Gen 3 API 

913 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

914 else: 

915 # Gen 2 API. Delete this when Gen 2 retired 

916 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

917 

918 maskedImage = exposure.getMaskedImage() 

919 mask = maskedImage.getMask() 

920 if altMask is not None: 

921 self.applyAltMaskPlanes(mask, altMask) 

922 imageScaler.scaleMaskedImage(maskedImage) 

923 

924 # Add 1 for each pixel which is not excluded by the exclude mask. 

925 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

926 if nImage is not None: 926 ↛ 927line 926 didn't jump to line 927, because the condition on line 926 was never true

927 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

928 if self.config.removeMaskPlanes: 928 ↛ 930line 928 didn't jump to line 930, because the condition on line 928 was never false

929 self.removeMaskPlanes(maskedImage) 

930 maskedImageList.append(maskedImage) 

931 

932 with self.timer("stack"): 

933 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

934 clipped, # also set output to CLIPPED if sigma-clipped 

935 maskMap) 

936 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

937 if nImage is not None: 937 ↛ 938line 937 didn't jump to line 938, because the condition on line 937 was never true

938 nImage.assign(subNImage, bbox) 

939 

940 def removeMaskPlanes(self, maskedImage): 

941 """Unset the mask of an image for mask planes specified in the config. 

942 

943 Parameters 

944 ---------- 

945 maskedImage : `lsst.afw.image.MaskedImage` 

946 The masked image to be modified. 

947 """ 

948 mask = maskedImage.getMask() 

949 for maskPlane in self.config.removeMaskPlanes: 

950 try: 

951 mask &= ~mask.getPlaneBitMask(maskPlane) 

952 except pexExceptions.InvalidParameterError: 

953 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

954 maskPlane) 

955 

956 @staticmethod 

957 def setRejectedMaskMapping(statsCtrl): 

958 """Map certain mask planes of the warps to new planes for the coadd. 

959 

960 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

961 or CLIPPED, set it to REJECTED on the coadd. 

962 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

963 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

964 

965 Parameters 

966 ---------- 

967 statsCtrl : `lsst.afw.math.StatisticsControl` 

968 Statistics control object for coadd 

969 

970 Returns 

971 ------- 

972 maskMap : `list` of `tuple` of `int` 

973 A list of mappings of mask planes of the warped exposures to 

974 mask planes of the coadd. 

975 """ 

976 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

977 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

978 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

979 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

980 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

981 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

982 (clipped, clipped)] 

983 return maskMap 

984 

985 def applyAltMaskPlanes(self, mask, altMaskSpans): 

986 """Apply in place alt mask formatted as SpanSets to a mask. 

987 

988 Parameters 

989 ---------- 

990 mask : `lsst.afw.image.Mask` 

991 Original mask. 

992 altMaskSpans : `dict` 

993 SpanSet lists to apply. Each element contains the new mask 

994 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

995 and list of SpanSets to apply to the mask. 

996 

997 Returns 

998 ------- 

999 mask : `lsst.afw.image.Mask` 

1000 Updated mask. 

1001 """ 

1002 if self.config.doUsePsfMatchedPolygons: 

1003 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1003 ↛ 1011line 1003 didn't jump to line 1011, because the condition on line 1003 was never false

1004 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1005 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1006 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1007 # This mask-clearing step must only occur *before* applying the new masks below 

1008 for spanSet in altMaskSpans['NO_DATA']: 

1009 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1010 

1011 for plane, spanSetList in altMaskSpans.items(): 

1012 maskClipValue = mask.addMaskPlane(plane) 

1013 for spanSet in spanSetList: 

1014 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1015 return mask 

1016 

1017 def shrinkValidPolygons(self, coaddInputs): 

1018 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1019 

1020 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1021 does not have a validPolygon, create one from its bbox. 

1022 

1023 Parameters 

1024 ---------- 

1025 coaddInputs : `lsst.afw.image.coaddInputs` 

1026 Original mask. 

1027 

1028 """ 

1029 for ccd in coaddInputs.ccds: 

1030 polyOrig = ccd.getValidPolygon() 

1031 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1032 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1033 if polyOrig: 1033 ↛ 1034line 1033 didn't jump to line 1034, because the condition on line 1033 was never true

1034 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1035 else: 

1036 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1037 ccd.setValidPolygon(validPolygon) 

1038 

1039 def readBrightObjectMasks(self, dataRef): 

1040 """Retrieve the bright object masks. 

1041 

1042 Returns None on failure. 

1043 

1044 Parameters 

1045 ---------- 

1046 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1047 A Butler dataRef. 

1048 

1049 Returns 

1050 ------- 

1051 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1052 Bright object mask from the Butler object, or None if it cannot 

1053 be retrieved. 

1054 """ 

1055 try: 

1056 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1057 except Exception as e: 

1058 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1059 return None 

1060 

1061 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1062 """Set the bright object masks. 

1063 

1064 Parameters 

1065 ---------- 

1066 exposure : `lsst.afw.image.Exposure` 

1067 Exposure under consideration. 

1068 dataId : `lsst.daf.persistence.dataId` 

1069 Data identifier dict for patch. 

1070 brightObjectMasks : `lsst.afw.table` 

1071 Table of bright objects to mask. 

1072 """ 

1073 

1074 if brightObjectMasks is None: 

1075 self.log.warn("Unable to apply bright object mask: none supplied") 

1076 return 

1077 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1078 mask = exposure.getMaskedImage().getMask() 

1079 wcs = exposure.getWcs() 

1080 plateScale = wcs.getPixelScale().asArcseconds() 

1081 

1082 for rec in brightObjectMasks: 

1083 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1084 if rec["type"] == "box": 

1085 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1086 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1087 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1088 

1089 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1090 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1091 

1092 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1093 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1094 spans = afwGeom.SpanSet(bbox) 

1095 elif rec["type"] == "circle": 

1096 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1097 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1098 else: 

1099 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1100 continue 

1101 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1102 

1103 def setInexactPsf(self, mask): 

1104 """Set INEXACT_PSF mask plane. 

1105 

1106 If any of the input images isn't represented in the coadd (due to 

1107 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1108 these pixels. 

1109 

1110 Parameters 

1111 ---------- 

1112 mask : `lsst.afw.image.Mask` 

1113 Coadded exposure's mask, modified in-place. 

1114 """ 

1115 mask.addMaskPlane("INEXACT_PSF") 

1116 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1117 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1118 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1119 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1120 array = mask.getArray() 

1121 selected = array & (sensorEdge | clipped | rejected) > 0 

1122 array[selected] |= inexactPsf 

1123 

1124 @classmethod 

1125 def _makeArgumentParser(cls): 

1126 """Create an argument parser. 

1127 """ 

1128 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1129 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1130 + cls.ConfigClass().warpType + "Warp", 

1131 help="data ID, e.g. --id tract=12345 patch=1,2", 

1132 ContainerClass=AssembleCoaddDataIdContainer) 

1133 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1134 ContainerClass=SelectDataIdContainer) 

1135 return parser 

1136 

1137 @staticmethod 

1138 def _subBBoxIter(bbox, subregionSize): 

1139 """Iterate over subregions of a bbox. 

1140 

1141 Parameters 

1142 ---------- 

1143 bbox : `lsst.geom.Box2I` 

1144 Bounding box over which to iterate. 

1145 subregionSize: `lsst.geom.Extent2I` 

1146 Size of sub-bboxes. 

1147 

1148 Yields 

1149 ------ 

1150 subBBox : `lsst.geom.Box2I` 

1151 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1152 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1153 the edges of ``bbox``, but it will never be empty. 

1154 """ 

1155 if bbox.isEmpty(): 1155 ↛ 1156line 1155 didn't jump to line 1156, because the condition on line 1155 was never true

1156 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1157 if subregionSize[0] < 1 or subregionSize[1] < 1: 1157 ↛ 1158line 1157 didn't jump to line 1158, because the condition on line 1157 was never true

1158 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1159 

1160 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1161 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1162 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1163 subBBox.clip(bbox) 

1164 if subBBox.isEmpty(): 1164 ↛ 1165line 1164 didn't jump to line 1165, because the condition on line 1164 was never true

1165 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1166 "colShift=%s, rowShift=%s" % 

1167 (bbox, subregionSize, colShift, rowShift)) 

1168 yield subBBox 

1169 

1170 

1171class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1172 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1173 """ 

1174 

1175 def makeDataRefList(self, namespace): 

1176 """Make self.refList from self.idList. 

1177 

1178 Parameters 

1179 ---------- 

1180 namespace 

1181 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1182 """ 

1183 datasetType = namespace.config.coaddName + "Coadd" 

1184 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1185 

1186 for dataId in self.idList: 

1187 # tract and patch are required 

1188 for key in keysCoadd: 

1189 if key not in dataId: 

1190 raise RuntimeError("--id must include " + key) 

1191 

1192 dataRef = namespace.butler.dataRef( 

1193 datasetType=datasetType, 

1194 dataId=dataId, 

1195 ) 

1196 self.refList.append(dataRef) 

1197 

1198 

1199def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1200 """Function to count the number of pixels with a specific mask in a 

1201 footprint. 

1202 

1203 Find the intersection of mask & footprint. Count all pixels in the mask 

1204 that are in the intersection that have bitmask set but do not have 

1205 ignoreMask set. Return the count. 

1206 

1207 Parameters 

1208 ---------- 

1209 mask : `lsst.afw.image.Mask` 

1210 Mask to define intersection region by. 

1211 footprint : `lsst.afw.detection.Footprint` 

1212 Footprint to define the intersection region by. 

1213 bitmask 

1214 Specific mask that we wish to count the number of occurances of. 

1215 ignoreMask 

1216 Pixels to not consider. 

1217 

1218 Returns 

1219 ------- 

1220 result : `int` 

1221 Count of number of pixels in footprint with specified mask. 

1222 """ 

1223 bbox = footprint.getBBox() 

1224 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1225 fp = afwImage.Mask(bbox) 

1226 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1227 footprint.spans.setMask(fp, bitmask) 

1228 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1229 (subMask.getArray() & ignoreMask) == 0).sum() 

1230 

1231 

1232class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1233 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1234 """ 

1235 assembleMeanCoadd = pexConfig.ConfigurableField( 

1236 target=AssembleCoaddTask, 

1237 doc="Task to assemble an initial Coadd using the MEAN statistic.", 

1238 ) 

1239 assembleMeanClipCoadd = pexConfig.ConfigurableField( 

1240 target=AssembleCoaddTask, 

1241 doc="Task to assemble an initial Coadd using the MEANCLIP statistic.", 

1242 ) 

1243 clipDetection = pexConfig.ConfigurableField( 

1244 target=SourceDetectionTask, 

1245 doc="Detect sources on difference between unclipped and clipped coadd") 

1246 minClipFootOverlap = pexConfig.Field( 

1247 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1248 dtype=float, 

1249 default=0.6 

1250 ) 

1251 minClipFootOverlapSingle = pexConfig.Field( 

1252 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1253 "clipped when only one visit overlaps", 

1254 dtype=float, 

1255 default=0.5 

1256 ) 

1257 minClipFootOverlapDouble = pexConfig.Field( 

1258 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1259 "clipped when two visits overlap", 

1260 dtype=float, 

1261 default=0.45 

1262 ) 

1263 maxClipFootOverlapDouble = pexConfig.Field( 

1264 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1265 "considering two visits", 

1266 dtype=float, 

1267 default=0.15 

1268 ) 

1269 minBigOverlap = pexConfig.Field( 

1270 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1271 "when labeling clipped footprints", 

1272 dtype=int, 

1273 default=100 

1274 ) 

1275 

1276 def setDefaults(self): 

1277 """Set default values for clipDetection. 

1278 

1279 Notes 

1280 ----- 

1281 The numeric values for these configuration parameters were 

1282 empirically determined, future work may further refine them. 

1283 """ 

1284 AssembleCoaddConfig.setDefaults(self) 

1285 self.clipDetection.doTempLocalBackground = False 

1286 self.clipDetection.reEstimateBackground = False 

1287 self.clipDetection.returnOriginalFootprints = False 

1288 self.clipDetection.thresholdPolarity = "both" 

1289 self.clipDetection.thresholdValue = 2 

1290 self.clipDetection.nSigmaToGrow = 2 

1291 self.clipDetection.minPixels = 4 

1292 self.clipDetection.isotropicGrow = True 

1293 self.clipDetection.thresholdType = "pixel_stdev" 

1294 self.sigmaClip = 1.5 

1295 self.clipIter = 3 

1296 self.statistic = "MEAN" 

1297 self.assembleMeanCoadd.statistic = 'MEAN' 

1298 self.assembleMeanClipCoadd.statistic = 'MEANCLIP' 

1299 self.assembleMeanCoadd.doWrite = False 

1300 self.assembleMeanClipCoadd.doWrite = False 

1301 

1302 def validate(self): 

1303 if self.doSigmaClip: 

1304 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1305 "Ignoring doSigmaClip.") 

1306 self.doSigmaClip = False 

1307 if self.statistic != "MEAN": 

1308 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1309 "(%s chosen). Please set statistic to MEAN." 

1310 % (self.statistic)) 

1311 AssembleCoaddTask.ConfigClass.validate(self) 

1312 

1313 

1314class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1315 """Assemble a coadded image from a set of coadded temporary exposures, 

1316 being careful to clip & flag areas with potential artifacts. 

1317 

1318 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1319 we clip outliers). The problem with doing this is that when computing the 

1320 coadd PSF at a given location, individual visit PSFs from visits with 

1321 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1322 In this task, we correct for this behavior by creating a new 

1323 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1324 coaddTempExps and the final coadd where 

1325 

1326 i. difference imaging suggests that there is an outlier and 

1327 ii. this outlier appears on only one or two images. 

1328 

1329 Such regions will not contribute to the final coadd. Furthermore, any 

1330 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1331 Note that the algorithm implemented by this task is preliminary and works 

1332 correctly for HSC data. Parameter modifications and or considerable 

1333 redesigning of the algorithm is likley required for other surveys. 

1334 

1335 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1336 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1337 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1338 if you wish. 

1339 

1340 Notes 

1341 ----- 

1342 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1343 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1344 see `baseDebug` for more about ``debug.py`` files. 

1345 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1346 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1347 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1348 for further information. 

1349 

1350 Examples 

1351 -------- 

1352 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1353 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1354 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1355 

1356 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1357 and filter to be coadded (specified using 

1358 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1359 along with a list of coaddTempExps to attempt to coadd (specified using 

1360 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1361 Only the coaddTempExps that cover the specified tract and patch will be 

1362 coadded. A list of the available optional arguments can be obtained by 

1363 calling assembleCoadd.py with the --help command line argument: 

1364 

1365 .. code-block:: none 

1366 

1367 assembleCoadd.py --help 

1368 

1369 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1370 context of multi-band processing, we will generate the HSC-I & -R band 

1371 coadds from HSC engineering test data provided in the ci_hsc package. 

1372 To begin, assuming that the lsst stack has been already set up, we must 

1373 set up the obs_subaru and ci_hsc packages. This defines the environment 

1374 variable $CI_HSC_DIR and points at the location of the package. The raw 

1375 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1376 the coadds, we must first 

1377 

1378 - ``processCcd`` 

1379 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1380 - ``makeSkyMap`` 

1381 create a skymap that covers the area of the sky present in the raw exposures 

1382 - ``makeCoaddTempExp`` 

1383 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1384 

1385 We can perform all of these steps by running 

1386 

1387 .. code-block:: none 

1388 

1389 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1390 

1391 This will produce warped coaddTempExps for each visit. To coadd the 

1392 warped data, we call ``assembleCoadd.py`` as follows: 

1393 

1394 .. code-block:: none 

1395 

1396 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1397 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1398 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1399 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1400 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1401 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1402 --selectId visit=903988 ccd=24 

1403 

1404 This will process the HSC-I band data. The results are written in 

1405 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1406 

1407 You may also choose to run: 

1408 

1409 .. code-block:: none 

1410 

1411 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1412 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1413 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1414 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1415 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1416 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1417 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1418 --selectId visit=903346 ccd=12 

1419 

1420 to generate the coadd for the HSC-R band if you are interested in following 

1421 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1422 """ 

1423 ConfigClass = SafeClipAssembleCoaddConfig 

1424 _DefaultName = "safeClipAssembleCoadd" 

1425 

1426 def __init__(self, *args, **kwargs): 

1427 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1428 schema = afwTable.SourceTable.makeMinimalSchema() 

1429 self.makeSubtask("clipDetection", schema=schema) 

1430 self.makeSubtask("assembleMeanClipCoadd") 

1431 self.makeSubtask("assembleMeanCoadd") 

1432 

1433 @utils.inheritDoc(AssembleCoaddTask) 

1434 @pipeBase.timeMethod 

1435 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1436 """Assemble the coadd for a region. 

1437 

1438 Compute the difference of coadds created with and without outlier 

1439 rejection to identify coadd pixels that have outlier values in some 

1440 individual visits. 

1441 Detect clipped regions on the difference image and mark these regions 

1442 on the one or two individual coaddTempExps where they occur if there 

1443 is significant overlap between the clipped region and a source. This 

1444 leaves us with a set of footprints from the difference image that have 

1445 been identified as having occured on just one or two individual visits. 

1446 However, these footprints were generated from a difference image. It 

1447 is conceivable for a large diffuse source to have become broken up 

1448 into multiple footprints acrosss the coadd difference in this process. 

1449 Determine the clipped region from all overlapping footprints from the 

1450 detected sources in each visit - these are big footprints. 

1451 Combine the small and big clipped footprints and mark them on a new 

1452 bad mask plane. 

1453 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1454 removal. Clipped footprints will no longer make it into the coadd 

1455 because they are marked in the new bad mask plane. 

1456 

1457 Notes 

1458 ----- 

1459 args and kwargs are passed but ignored in order to match the call 

1460 signature expected by the parent task. 

1461 """ 

1462 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1463 mask = exp.getMaskedImage().getMask() 

1464 mask.addMaskPlane("CLIPPED") 

1465 

1466 result = self.detectClip(exp, tempExpRefList) 

1467 

1468 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1469 

1470 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1471 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1472 # Append big footprints from individual Warps to result.clipSpans 

1473 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1474 result.detectionFootprints, maskClipValue, maskDetValue, 

1475 exp.getBBox()) 

1476 # Create mask of the current clipped footprints 

1477 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1478 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1479 

1480 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1481 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1482 maskClip |= maskClipBig 

1483 

1484 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1485 badMaskPlanes = self.config.badMaskPlanes[:] 

1486 badMaskPlanes.append("CLIPPED") 

1487 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1488 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1489 result.clipSpans, mask=badPixelMask) 

1490 

1491 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1492 """Return an exposure that contains the difference between unclipped 

1493 and clipped coadds. 

1494 

1495 Generate a difference image between clipped and unclipped coadds. 

1496 Compute the difference image by subtracting an outlier-clipped coadd 

1497 from an outlier-unclipped coadd. Return the difference image. 

1498 

1499 Parameters 

1500 ---------- 

1501 skyInfo : `lsst.pipe.base.Struct` 

1502 Patch geometry information, from getSkyInfo 

1503 tempExpRefList : `list` 

1504 List of data reference to tempExp 

1505 imageScalerList : `list` 

1506 List of image scalers 

1507 weightList : `list` 

1508 List of weights 

1509 

1510 Returns 

1511 ------- 

1512 exp : `lsst.afw.image.Exposure` 

1513 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1514 """ 

1515 coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList, 

1516 imageScalerList, weightList).coaddExposure 

1517 

1518 coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList, 

1519 imageScalerList, weightList).coaddExposure 

1520 

1521 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1522 coaddDiff -= coaddClip.getMaskedImage() 

1523 exp = afwImage.ExposureF(coaddDiff) 

1524 exp.setPsf(coaddMean.getPsf()) 

1525 return exp 

1526 

1527 def detectClip(self, exp, tempExpRefList): 

1528 """Detect clipped regions on an exposure and set the mask on the 

1529 individual tempExp masks. 

1530 

1531 Detect footprints in the difference image after smoothing the 

1532 difference image with a Gaussian kernal. Identify footprints that 

1533 overlap with one or two input ``coaddTempExps`` by comparing the 

1534 computed overlap fraction to thresholds set in the config. A different 

1535 threshold is applied depending on the number of overlapping visits 

1536 (restricted to one or two). If the overlap exceeds the thresholds, 

1537 the footprint is considered "CLIPPED" and is marked as such on the 

1538 coaddTempExp. Return a struct with the clipped footprints, the indices 

1539 of the ``coaddTempExps`` that end up overlapping with the clipped 

1540 footprints, and a list of new masks for the ``coaddTempExps``. 

1541 

1542 Parameters 

1543 ---------- 

1544 exp : `lsst.afw.image.Exposure` 

1545 Exposure to run detection on. 

1546 tempExpRefList : `list` 

1547 List of data reference to tempExp. 

1548 

1549 Returns 

1550 ------- 

1551 result : `lsst.pipe.base.Struct` 

1552 Result struct with components: 

1553 

1554 - ``clipFootprints``: list of clipped footprints. 

1555 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1556 ``tempExpRefList``. 

1557 - ``clipSpans``: List of dictionaries containing spanSet lists 

1558 to clip. Each element contains the new maskplane name 

1559 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1560 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1561 compressed into footprints. 

1562 """ 

1563 mask = exp.getMaskedImage().getMask() 

1564 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1565 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1566 # Merge positive and negative together footprints together 

1567 fpSet.positive.merge(fpSet.negative) 

1568 footprints = fpSet.positive 

1569 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1570 ignoreMask = self.getBadPixelMask() 

1571 

1572 clipFootprints = [] 

1573 clipIndices = [] 

1574 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1575 

1576 # for use by detectClipBig 

1577 visitDetectionFootprints = [] 

1578 

1579 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1580 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1581 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1582 

1583 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1584 for i, warpRef in enumerate(tempExpRefList): 

1585 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1586 immediate=True).getMaskedImage().getMask() 

1587 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1588 afwImage.PARENT, True) 

1589 maskVisitDet &= maskDetValue 

1590 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1591 visitDetectionFootprints.append(visitFootprints) 

1592 

1593 for j, footprint in enumerate(footprints.getFootprints()): 

1594 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1595 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1596 

1597 # build a list of clipped spans for each visit 

1598 for j, footprint in enumerate(footprints.getFootprints()): 

1599 nPixel = footprint.getArea() 

1600 overlap = [] # hold the overlap with each visit 

1601 indexList = [] # index of visit in global list 

1602 for i in range(len(tempExpRefList)): 

1603 ignore = ignoreArr[i, j] 

1604 overlapDet = overlapDetArr[i, j] 

1605 totPixel = nPixel - ignore 

1606 

1607 # If we have more bad pixels than detection skip 

1608 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1608 ↛ 1610line 1608 didn't jump to line 1610, because the condition on line 1608 was never false

1609 continue 

1610 overlap.append(overlapDet/float(totPixel)) 

1611 indexList.append(i) 

1612 

1613 overlap = numpy.array(overlap) 

1614 if not len(overlap): 1614 ↛ 1617line 1614 didn't jump to line 1617, because the condition on line 1614 was never false

1615 continue 

1616 

1617 keep = False # Should this footprint be marked as clipped? 

1618 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1619 

1620 # If footprint only has one overlap use a lower threshold 

1621 if len(overlap) == 1: 

1622 if overlap[0] > self.config.minClipFootOverlapSingle: 

1623 keep = True 

1624 keepIndex = [0] 

1625 else: 

1626 # This is the general case where only visit should be clipped 

1627 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1628 if len(clipIndex) == 1: 

1629 keep = True 

1630 keepIndex = [clipIndex[0]] 

1631 

1632 # Test if there are clipped objects that overlap two different visits 

1633 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1634 if len(clipIndex) == 2 and len(overlap) > 3: 

1635 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1636 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1637 keep = True 

1638 keepIndex = clipIndex 

1639 

1640 if not keep: 

1641 continue 

1642 

1643 for index in keepIndex: 

1644 globalIndex = indexList[index] 

1645 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1646 

1647 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1648 clipFootprints.append(footprint) 

1649 

1650 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1651 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1652 

1653 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1654 maskClipValue, maskDetValue, coaddBBox): 

1655 """Return individual warp footprints for large artifacts and append 

1656 them to ``clipList`` in place. 

1657 

1658 Identify big footprints composed of many sources in the coadd 

1659 difference that may have originated in a large diffuse source in the 

1660 coadd. We do this by indentifying all clipped footprints that overlap 

1661 significantly with each source in all the coaddTempExps. 

1662 

1663 Parameters 

1664 ---------- 

1665 clipList : `list` 

1666 List of alt mask SpanSets with clipping information. Modified. 

1667 clipFootprints : `list` 

1668 List of clipped footprints. 

1669 clipIndices : `list` 

1670 List of which entries in tempExpClipList each footprint belongs to. 

1671 maskClipValue 

1672 Mask value of clipped pixels. 

1673 maskDetValue 

1674 Mask value of detected pixels. 

1675 coaddBBox : `lsst.geom.Box` 

1676 BBox of the coadd and warps. 

1677 

1678 Returns 

1679 ------- 

1680 bigFootprintsCoadd : `list` 

1681 List of big footprints 

1682 """ 

1683 bigFootprintsCoadd = [] 

1684 ignoreMask = self.getBadPixelMask() 

1685 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1686 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1687 for footprint in visitFootprints.getFootprints(): 1687 ↛ 1688line 1687 didn't jump to line 1688, because the loop on line 1687 never started

1688 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1689 

1690 # build a mask of clipped footprints that are in this visit 

1691 clippedFootprintsVisit = [] 

1692 for foot, clipIndex in zip(clipFootprints, clipIndices): 1692 ↛ 1693line 1692 didn't jump to line 1693, because the loop on line 1692 never started

1693 if index not in clipIndex: 

1694 continue 

1695 clippedFootprintsVisit.append(foot) 

1696 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1697 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1698 

1699 bigFootprintsVisit = [] 

1700 for foot in visitFootprints.getFootprints(): 1700 ↛ 1701line 1700 didn't jump to line 1701, because the loop on line 1700 never started

1701 if foot.getArea() < self.config.minBigOverlap: 

1702 continue 

1703 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1704 if nCount > self.config.minBigOverlap: 

1705 bigFootprintsVisit.append(foot) 

1706 bigFootprintsCoadd.append(foot) 

1707 

1708 for footprint in bigFootprintsVisit: 1708 ↛ 1709line 1708 didn't jump to line 1709, because the loop on line 1708 never started

1709 clippedSpans["CLIPPED"].append(footprint.spans) 

1710 

1711 return bigFootprintsCoadd 

1712 

1713 

1714class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1715 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1716 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1717 "Only PSF-Matched Warps make sense for image subtraction. " 

1718 "Therefore, they must be an additional declared input."), 

1719 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1720 storageClass="ExposureF", 

1721 dimensions=("tract", "patch", "skymap", "visit"), 

1722 deferLoad=True, 

1723 multiple=True 

1724 ) 

1725 templateCoadd = pipeBase.connectionTypes.Output( 

1726 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1727 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1728 name="{fakesType}{outputCoaddName}CoaddPsfMatched", 

1729 storageClass="ExposureF", 

1730 dimensions=("tract", "patch", "skymap", "band"), 

1731 ) 

1732 

1733 def __init__(self, *, config=None): 

1734 super().__init__(config=config) 

1735 if not config.assembleStaticSkyModel.doWrite: 

1736 self.outputs.remove("templateCoadd") 

1737 config.validate() 

1738 

1739 

1740class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1741 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1742 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1743 target=AssembleCoaddTask, 

1744 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1745 " naive/first-iteration model of the static sky.", 

1746 ) 

1747 detect = pexConfig.ConfigurableField( 

1748 target=SourceDetectionTask, 

1749 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1750 ) 

1751 detectTemplate = pexConfig.ConfigurableField( 

1752 target=SourceDetectionTask, 

1753 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1754 ) 

1755 maskStreaks = pexConfig.ConfigurableField( 

1756 target=MaskStreaksTask, 

1757 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1758 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1759 "streakMaskName" 

1760 ) 

1761 streakMaskName = pexConfig.Field( 

1762 dtype=str, 

1763 default="STREAK", 

1764 doc="Name of mask bit used for streaks" 

1765 ) 

1766 maxNumEpochs = pexConfig.Field( 

1767 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1768 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1769 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1770 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1771 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1772 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1773 "than transient and not masked.", 

1774 dtype=int, 

1775 default=2 

1776 ) 

1777 maxFractionEpochsLow = pexConfig.RangeField( 

1778 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1779 "Effective maxNumEpochs = " 

1780 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1781 dtype=float, 

1782 default=0.4, 

1783 min=0., max=1., 

1784 ) 

1785 maxFractionEpochsHigh = pexConfig.RangeField( 

1786 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1787 "Effective maxNumEpochs = " 

1788 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1789 dtype=float, 

1790 default=0.03, 

1791 min=0., max=1., 

1792 ) 

1793 spatialThreshold = pexConfig.RangeField( 

1794 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1795 "temporal criteria. If 0, clip all. If 1, clip none.", 

1796 dtype=float, 

1797 default=0.5, 

1798 min=0., max=1., 

1799 inclusiveMin=True, inclusiveMax=True 

1800 ) 

1801 doScaleWarpVariance = pexConfig.Field( 

1802 doc="Rescale Warp variance plane using empirical noise?", 

1803 dtype=bool, 

1804 default=True, 

1805 ) 

1806 scaleWarpVariance = pexConfig.ConfigurableField( 

1807 target=ScaleVarianceTask, 

1808 doc="Rescale variance on warps", 

1809 ) 

1810 doPreserveContainedBySource = pexConfig.Field( 

1811 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1812 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1813 dtype=bool, 

1814 default=True, 

1815 ) 

1816 doPrefilterArtifacts = pexConfig.Field( 

1817 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1818 "because they will be excluded anyway. This prevents them from contributing " 

1819 "to the outlier epoch count image and potentially being labeled as persistant." 

1820 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1821 dtype=bool, 

1822 default=True 

1823 ) 

1824 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1825 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1826 dtype=str, 

1827 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1828 ) 

1829 prefilterArtifactsRatio = pexConfig.Field( 

1830 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1831 dtype=float, 

1832 default=0.05 

1833 ) 

1834 doFilterMorphological = pexConfig.Field( 

1835 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1836 "be streaks.", 

1837 dtype=bool, 

1838 default=False 

1839 ) 

1840 

1841 def setDefaults(self): 

1842 AssembleCoaddConfig.setDefaults(self) 

1843 self.statistic = 'MEAN' 

1844 self.doUsePsfMatchedPolygons = True 

1845 

1846 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1847 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1848 if "EDGE" in self.badMaskPlanes: 1848 ↛ 1850line 1848 didn't jump to line 1850, because the condition on line 1848 was never false

1849 self.badMaskPlanes.remove('EDGE') 

1850 self.removeMaskPlanes.append('EDGE') 

1851 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1852 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1853 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1854 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1855 self.assembleStaticSkyModel.sigmaClip = 2.5 

1856 self.assembleStaticSkyModel.clipIter = 3 

1857 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1858 self.assembleStaticSkyModel.doWrite = False 

1859 self.detect.doTempLocalBackground = False 

1860 self.detect.reEstimateBackground = False 

1861 self.detect.returnOriginalFootprints = False 

1862 self.detect.thresholdPolarity = "both" 

1863 self.detect.thresholdValue = 5 

1864 self.detect.minPixels = 4 

1865 self.detect.isotropicGrow = True 

1866 self.detect.thresholdType = "pixel_stdev" 

1867 self.detect.nSigmaToGrow = 0.4 

1868 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1869 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1870 self.detectTemplate.nSigmaToGrow = 2.4 

1871 self.detectTemplate.doTempLocalBackground = False 

1872 self.detectTemplate.reEstimateBackground = False 

1873 self.detectTemplate.returnOriginalFootprints = False 

1874 

1875 def validate(self): 

1876 super().validate() 

1877 if self.assembleStaticSkyModel.doNImage: 

1878 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1879 "Please set assembleStaticSkyModel.doNImage=False") 

1880 

1881 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1882 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1883 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1884 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1885 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1886 

1887 

1888class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1889 """Assemble a compareWarp coadded image from a set of warps 

1890 by masking artifacts detected by comparing PSF-matched warps. 

1891 

1892 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1893 we clip outliers). The problem with doing this is that when computing the 

1894 coadd PSF at a given location, individual visit PSFs from visits with 

1895 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1896 In this task, we correct for this behavior by creating a new badMaskPlane 

1897 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1898 an artifact. We populate this plane on the input warps by comparing 

1899 PSF-matched warps with a PSF-matched median coadd which serves as a 

1900 model of the static sky. Any group of pixels that deviates from the 

1901 PSF-matched template coadd by more than config.detect.threshold sigma, 

1902 is an artifact candidate. The candidates are then filtered to remove 

1903 variable sources and sources that are difficult to subtract such as 

1904 bright stars. This filter is configured using the config parameters 

1905 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1906 the maximum fraction of epochs that the deviation can appear in and still 

1907 be considered an artifact. The spatialThreshold is the maximum fraction of 

1908 pixels in the footprint of the deviation that appear in other epochs 

1909 (where other epochs is defined by the temporalThreshold). If the deviant 

1910 region meets this criteria of having a significant percentage of pixels 

1911 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1912 set in the mask. These regions will not contribute to the final coadd. 

1913 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1914 of clipped regions. Note that the algorithm implemented by this task is 

1915 preliminary and works correctly for HSC data. Parameter modifications and 

1916 or considerable redesigning of the algorithm is likley required for other 

1917 surveys. 

1918 

1919 ``CompareWarpAssembleCoaddTask`` sub-classes 

1920 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1921 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1922 

1923 Notes 

1924 ----- 

1925 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1926 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

1927 ``baseDebug`` for more about ``debug.py`` files. 

1928 

1929 This task supports the following debug variables: 

1930 

1931 - ``saveCountIm`` 

1932 If True then save the Epoch Count Image as a fits file in the `figPath` 

1933 - ``figPath`` 

1934 Path to save the debug fits images and figures 

1935 

1936 For example, put something like: 

1937 

1938 .. code-block:: python 

1939 

1940 import lsstDebug 

1941 def DebugInfo(name): 

1942 di = lsstDebug.getInfo(name) 

1943 if name == "lsst.pipe.tasks.assembleCoadd": 

1944 di.saveCountIm = True 

1945 di.figPath = "/desired/path/to/debugging/output/images" 

1946 return di 

1947 lsstDebug.Info = DebugInfo 

1948 

1949 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

1950 ``--debug`` flag. Some subtasks may have their own debug variables; 

1951 see individual Task documentation. 

1952 

1953 Examples 

1954 -------- 

1955 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

1956 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

1957 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

1958 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1959 and filter to be coadded (specified using 

1960 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1961 along with a list of coaddTempExps to attempt to coadd (specified using 

1962 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1963 Only the warps that cover the specified tract and patch will be coadded. 

1964 A list of the available optional arguments can be obtained by calling 

1965 ``assembleCoadd.py`` with the ``--help`` command line argument: 

1966 

1967 .. code-block:: none 

1968 

1969 assembleCoadd.py --help 

1970 

1971 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

1972 context of multi-band processing, we will generate the HSC-I & -R band 

1973 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

1974 To begin, assuming that the lsst stack has been already set up, we must 

1975 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

1976 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

1977 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

1978 directory. To begin assembling the coadds, we must first 

1979 

1980 - processCcd 

1981 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1982 - makeSkyMap 

1983 create a skymap that covers the area of the sky present in the raw exposures 

1984 - makeCoaddTempExp 

1985 warp the individual calibrated exposures to the tangent plane of the coadd 

1986 

1987 We can perform all of these steps by running 

1988 

1989 .. code-block:: none 

1990 

1991 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1992 

1993 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

1994 warped data, we call ``assembleCoadd.py`` as follows: 

1995 

1996 .. code-block:: none 

1997 

1998 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1999 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2000 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2001 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2002 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2003 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2004 --selectId visit=903988 ccd=24 

2005 

2006 This will process the HSC-I band data. The results are written in 

2007 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2008 """ 

2009 ConfigClass = CompareWarpAssembleCoaddConfig 

2010 _DefaultName = "compareWarpAssembleCoadd" 

2011 

2012 def __init__(self, *args, **kwargs): 

2013 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2014 self.makeSubtask("assembleStaticSkyModel") 

2015 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2016 self.makeSubtask("detect", schema=detectionSchema) 

2017 if self.config.doPreserveContainedBySource: 2017 ↛ 2019line 2017 didn't jump to line 2019, because the condition on line 2017 was never false

2018 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2019 if self.config.doScaleWarpVariance: 2019 ↛ 2021line 2019 didn't jump to line 2021, because the condition on line 2019 was never false

2020 self.makeSubtask("scaleWarpVariance") 

2021 if self.config.doFilterMorphological: 2021 ↛ 2022line 2021 didn't jump to line 2022, because the condition on line 2021 was never true

2022 self.makeSubtask("maskStreaks") 

2023 

2024 @utils.inheritDoc(AssembleCoaddTask) 

2025 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2026 """ 

2027 Generate a templateCoadd to use as a naive model of static sky to 

2028 subtract from PSF-Matched warps. 

2029 

2030 Returns 

2031 ------- 

2032 result : `lsst.pipe.base.Struct` 

2033 Result struct with components: 

2034 

2035 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2036 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2037 """ 

2038 # Ensure that psfMatchedWarps are used as input warps for template generation 

2039 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2040 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2041 

2042 # Because subtasks don't have connections we have to make one. 

2043 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2044 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2045 if self.config.assembleStaticSkyModel.doWrite: 

2046 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2047 # Remove template coadd from both subtask's and main tasks outputs, 

2048 # because it is handled by the subtask as `coaddExposure` 

2049 del outputRefs.templateCoadd 

2050 del staticSkyModelOutputRefs.templateCoadd 

2051 

2052 # A PSF-Matched nImage does not exist as a dataset type 

2053 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2054 del staticSkyModelOutputRefs.nImage 

2055 

2056 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2057 staticSkyModelOutputRefs) 

2058 if templateCoadd is None: 

2059 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2060 

2061 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2062 nImage=templateCoadd.nImage, 

2063 warpRefList=templateCoadd.warpRefList, 

2064 imageScalerList=templateCoadd.imageScalerList, 

2065 weightList=templateCoadd.weightList) 

2066 

2067 @utils.inheritDoc(AssembleCoaddTask) 

2068 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2069 """ 

2070 Generate a templateCoadd to use as a naive model of static sky to 

2071 subtract from PSF-Matched warps. 

2072 

2073 Returns 

2074 ------- 

2075 result : `lsst.pipe.base.Struct` 

2076 Result struct with components: 

2077 

2078 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2079 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2080 """ 

2081 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2082 if templateCoadd is None: 2082 ↛ 2083line 2082 didn't jump to line 2083, because the condition on line 2082 was never true

2083 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2084 

2085 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2086 nImage=templateCoadd.nImage, 

2087 warpRefList=templateCoadd.warpRefList, 

2088 imageScalerList=templateCoadd.imageScalerList, 

2089 weightList=templateCoadd.weightList) 

2090 

2091 def _noTemplateMessage(self, warpType): 

2092 warpName = (warpType[0].upper() + warpType[1:]) 

2093 message = """No %(warpName)s warps were found to build the template coadd which is 

2094 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2095 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2096 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2097 

2098 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2099 another algorithm like: 

2100 

2101 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2102 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2103 """ % {"warpName": warpName} 

2104 return message 

2105 

2106 @utils.inheritDoc(AssembleCoaddTask) 

2107 @pipeBase.timeMethod 

2108 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2109 supplementaryData, *args, **kwargs): 

2110 """Assemble the coadd. 

2111 

2112 Find artifacts and apply them to the warps' masks creating a list of 

2113 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2114 plane. Then pass these alternative masks to the base class's `run` 

2115 method. 

2116 

2117 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2118 that must contain a ``templateCoadd`` that serves as the 

2119 model of the static sky. 

2120 """ 

2121 

2122 # Check and match the order of the supplementaryData 

2123 # (PSF-matched) inputs to the order of the direct inputs, 

2124 # so that the artifact mask is applied to the right warp 

2125 dataIds = [ref.dataId for ref in tempExpRefList] 

2126 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2127 

2128 if dataIds != psfMatchedDataIds: 

2129 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2130 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2131 psfMatchedDataIds, dataIds) 

2132 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2133 psfMatchedDataIds, dataIds) 

2134 

2135 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2136 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2137 supplementaryData.warpRefList, 

2138 supplementaryData.imageScalerList) 

2139 

2140 badMaskPlanes = self.config.badMaskPlanes[:] 

2141 badMaskPlanes.append("CLIPPED") 

2142 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2143 

2144 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2145 spanSetMaskList, mask=badPixelMask) 

2146 

2147 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2148 # Psf-Matching moves the real edge inwards 

2149 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2150 return result 

2151 

2152 def applyAltEdgeMask(self, mask, altMaskList): 

2153 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2154 

2155 Parameters 

2156 ---------- 

2157 mask : `lsst.afw.image.Mask` 

2158 Original mask. 

2159 altMaskList : `list` 

2160 List of Dicts containing ``spanSet`` lists. 

2161 Each element contains the new mask plane name (e.g. "CLIPPED 

2162 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2163 the mask. 

2164 """ 

2165 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2166 for visitMask in altMaskList: 

2167 if "EDGE" in visitMask: 2167 ↛ 2166line 2167 didn't jump to line 2166, because the condition on line 2167 was never false

2168 for spanSet in visitMask['EDGE']: 

2169 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2170 

2171 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2172 """Find artifacts. 

2173 

2174 Loop through warps twice. The first loop builds a map with the count 

2175 of how many epochs each pixel deviates from the templateCoadd by more 

2176 than ``config.chiThreshold`` sigma. The second loop takes each 

2177 difference image and filters the artifacts detected in each using 

2178 count map to filter out variable sources and sources that are 

2179 difficult to subtract cleanly. 

2180 

2181 Parameters 

2182 ---------- 

2183 templateCoadd : `lsst.afw.image.Exposure` 

2184 Exposure to serve as model of static sky. 

2185 tempExpRefList : `list` 

2186 List of data references to warps. 

2187 imageScalerList : `list` 

2188 List of image scalers. 

2189 

2190 Returns 

2191 ------- 

2192 altMasks : `list` 

2193 List of dicts containing information about CLIPPED 

2194 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2195 """ 

2196 

2197 self.log.debug("Generating Count Image, and mask lists.") 

2198 coaddBBox = templateCoadd.getBBox() 

2199 slateIm = afwImage.ImageU(coaddBBox) 

2200 epochCountImage = afwImage.ImageU(coaddBBox) 

2201 nImage = afwImage.ImageU(coaddBBox) 

2202 spanSetArtifactList = [] 

2203 spanSetNoDataMaskList = [] 

2204 spanSetEdgeList = [] 

2205 spanSetBadMorphoList = [] 

2206 badPixelMask = self.getBadPixelMask() 

2207 

2208 # mask of the warp diffs should = that of only the warp 

2209 templateCoadd.mask.clearAllMaskPlanes() 

2210 

2211 if self.config.doPreserveContainedBySource: 2211 ↛ 2214line 2211 didn't jump to line 2214, because the condition on line 2211 was never false

2212 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2213 else: 

2214 templateFootprints = None 

2215 

2216 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2217 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2218 if warpDiffExp is not None: 

2219 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2220 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2221 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2222 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2223 fpSet.positive.merge(fpSet.negative) 

2224 footprints = fpSet.positive 

2225 slateIm.set(0) 

2226 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2227 

2228 # Remove artifacts due to defects before they contribute to the epochCountImage 

2229 if self.config.doPrefilterArtifacts: 2229 ↛ 2233line 2229 didn't jump to line 2233, because the condition on line 2229 was never false

2230 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2231 

2232 # Clear mask before adding prefiltered spanSets 

2233 self.detect.clearMask(warpDiffExp.mask) 

2234 for spans in spanSetList: 

2235 spans.setImage(slateIm, 1, doClip=True) 

2236 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2237 epochCountImage += slateIm 

2238 

2239 if self.config.doFilterMorphological: 2239 ↛ 2240line 2239 didn't jump to line 2240, because the condition on line 2239 was never true

2240 maskName = self.config.streakMaskName 

2241 _ = self.maskStreaks.run(warpDiffExp) 

2242 streakMask = warpDiffExp.mask 

2243 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2244 streakMask.getPlaneBitMask(maskName)).split() 

2245 

2246 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2247 # undergo a second convolution. Pixels with data in the direct warp 

2248 # but not in the PSF-matched warp will not have their artifacts detected. 

2249 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2250 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2251 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2252 nansMask.setXY0(warpDiffExp.getXY0()) 

2253 edgeMask = warpDiffExp.mask 

2254 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2255 edgeMask.getPlaneBitMask("EDGE")).split() 

2256 else: 

2257 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2258 # In this case, mask the whole epoch 

2259 nansMask = afwImage.MaskX(coaddBBox, 1) 

2260 spanSetList = [] 

2261 spanSetEdgeMask = [] 

2262 spanSetStreak = [] 

2263 

2264 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2265 

2266 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2267 spanSetArtifactList.append(spanSetList) 

2268 spanSetEdgeList.append(spanSetEdgeMask) 

2269 if self.config.doFilterMorphological: 2269 ↛ 2270line 2269 didn't jump to line 2270, because the condition on line 2269 was never true

2270 spanSetBadMorphoList.append(spanSetStreak) 

2271 

2272 if lsstDebug.Info(__name__).saveCountIm: 2272 ↛ 2273line 2272 didn't jump to line 2273, because the condition on line 2272 was never true

2273 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2274 epochCountImage.writeFits(path) 

2275 

2276 for i, spanSetList in enumerate(spanSetArtifactList): 

2277 if spanSetList: 

2278 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2279 templateFootprints) 

2280 spanSetArtifactList[i] = filteredSpanSetList 

2281 if self.config.doFilterMorphological: 2281 ↛ 2282line 2281 didn't jump to line 2282, because the condition on line 2281 was never true

2282 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2283 

2284 altMasks = [] 

2285 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2286 altMasks.append({'CLIPPED': artifacts, 

2287 'NO_DATA': noData, 

2288 'EDGE': edge}) 

2289 return altMasks 

2290 

2291 def prefilterArtifacts(self, spanSetList, exp): 

2292 """Remove artifact candidates covered by bad mask plane. 

2293 

2294 Any future editing of the candidate list that does not depend on 

2295 temporal information should go in this method. 

2296 

2297 Parameters 

2298 ---------- 

2299 spanSetList : `list` 

2300 List of SpanSets representing artifact candidates. 

2301 exp : `lsst.afw.image.Exposure` 

2302 Exposure containing mask planes used to prefilter. 

2303 

2304 Returns 

2305 ------- 

2306 returnSpanSetList : `list` 

2307 List of SpanSets with artifacts. 

2308 """ 

2309 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2310 goodArr = (exp.mask.array & badPixelMask) == 0 

2311 returnSpanSetList = [] 

2312 bbox = exp.getBBox() 

2313 x0, y0 = exp.getXY0() 

2314 for i, span in enumerate(spanSetList): 

2315 y, x = span.clippedTo(bbox).indices() 

2316 yIndexLocal = numpy.array(y) - y0 

2317 xIndexLocal = numpy.array(x) - x0 

2318 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2319 if goodRatio > self.config.prefilterArtifactsRatio: 2319 ↛ 2314line 2319 didn't jump to line 2314, because the condition on line 2319 was never false

2320 returnSpanSetList.append(span) 

2321 return returnSpanSetList 

2322 

2323 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2324 """Filter artifact candidates. 

2325 

2326 Parameters 

2327 ---------- 

2328 spanSetList : `list` 

2329 List of SpanSets representing artifact candidates. 

2330 epochCountImage : `lsst.afw.image.Image` 

2331 Image of accumulated number of warpDiff detections. 

2332 nImage : `lsst.afw.image.Image` 

2333 Image of the accumulated number of total epochs contributing. 

2334 

2335 Returns 

2336 ------- 

2337 maskSpanSetList : `list` 

2338 List of SpanSets with artifacts. 

2339 """ 

2340 

2341 maskSpanSetList = [] 

2342 x0, y0 = epochCountImage.getXY0() 

2343 for i, span in enumerate(spanSetList): 

2344 y, x = span.indices() 

2345 yIdxLocal = [y1 - y0 for y1 in y] 

2346 xIdxLocal = [x1 - x0 for x1 in x] 

2347 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2348 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2349 

2350 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2351 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2352 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2353 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2354 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2355 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2356 & (outlierN <= effectiveMaxNumEpochs)) 

2357 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2358 if percentBelowThreshold > self.config.spatialThreshold: 

2359 maskSpanSetList.append(span) 

2360 

2361 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2361 ↛ 2374line 2361 didn't jump to line 2374, because the condition on line 2361 was never false

2362 # If a candidate is contained by a footprint on the template coadd, do not clip 

2363 filteredMaskSpanSetList = [] 

2364 for span in maskSpanSetList: 

2365 doKeep = True 

2366 for footprint in footprintsToExclude.positive.getFootprints(): 

2367 if footprint.spans.contains(span): 

2368 doKeep = False 

2369 break 

2370 if doKeep: 

2371 filteredMaskSpanSetList.append(span) 

2372 maskSpanSetList = filteredMaskSpanSetList 

2373 

2374 return maskSpanSetList 

2375 

2376 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2377 """Fetch a warp from the butler and return a warpDiff. 

2378 

2379 Parameters 

2380 ---------- 

2381 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2382 Butler dataRef for the warp. 

2383 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2384 An image scaler object. 

2385 templateCoadd : `lsst.afw.image.Exposure` 

2386 Exposure to be substracted from the scaled warp. 

2387 

2388 Returns 

2389 ------- 

2390 warp : `lsst.afw.image.Exposure` 

2391 Exposure of the image difference between the warp and template. 

2392 """ 

2393 

2394 # If the PSF-Matched warp did not exist for this direct warp 

2395 # None is holding its place to maintain order in Gen 3 

2396 if warpRef is None: 

2397 return None 

2398 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2399 warpName = self.getTempExpDatasetName('psfMatched') 

2400 if not isinstance(warpRef, DeferredDatasetHandle): 2400 ↛ 2404line 2400 didn't jump to line 2404, because the condition on line 2400 was never false

2401 if not warpRef.datasetExists(warpName): 2401 ↛ 2402line 2401 didn't jump to line 2402, because the condition on line 2401 was never true

2402 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2403 return None 

2404 warp = warpRef.get(datasetType=warpName, immediate=True) 

2405 # direct image scaler OK for PSF-matched Warp 

2406 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2407 mi = warp.getMaskedImage() 

2408 if self.config.doScaleWarpVariance: 2408 ↛ 2413line 2408 didn't jump to line 2413, because the condition on line 2408 was never false

2409 try: 

2410 self.scaleWarpVariance.run(mi) 

2411 except Exception as exc: 

2412 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2413 mi -= templateCoadd.getMaskedImage() 

2414 return warp 

2415 

2416 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2417 """Return a path to which to write debugging output. 

2418 

2419 Creates a hyphen-delimited string of dataId values for simple filenames. 

2420 

2421 Parameters 

2422 ---------- 

2423 prefix : `str` 

2424 Prefix for filename. 

2425 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2426 Butler dataRef to make the path from. 

2427 coaddLevel : `bool`, optional. 

2428 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2429 'filter', but no 'visit'). 

2430 

2431 Returns 

2432 ------- 

2433 result : `str` 

2434 Path for debugging output. 

2435 """ 

2436 if coaddLevel: 

2437 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2438 else: 

2439 keys = warpRef.dataId.keys() 

2440 keyList = sorted(keys, reverse=True) 

2441 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2442 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2443 return os.path.join(directory, filename) 

2444 

2445 

2446def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None): 

2447 """Match the order of one list to another, padding if necessary 

2448 

2449 Parameters 

2450 ---------- 

2451 inputList : list 

2452 List to be reordered and padded. Elements can be any type. 

2453 inputKeys : iterable 

2454 Iterable of values to be compared with outputKeys. 

2455 Length must match `inputList` 

2456 outputKeys : iterable 

2457 Iterable of values to be compared with inputKeys. 

2458 padWith : 

2459 Any value to be inserted where inputKey not in outputKeys 

2460 

2461 Returns 

2462 ------- 

2463 list 

2464 Copy of inputList reordered per outputKeys and padded with `padWith` 

2465 so that the length matches length of outputKeys. 

2466 """ 

2467 outputList = [] 

2468 for d in outputKeys: 

2469 if d in inputKeys: 

2470 outputList.append(inputList[inputKeys.index(d)]) 

2471 else: 

2472 outputList.append(padWith) 

2473 return outputList