Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from lsst.meas.algorithms import SourceDetectionTask 

48from lsst.daf.butler import DeferredDatasetHandle 

49 

50__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

51 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

52 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

53 

54 

55class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

56 dimensions=("tract", "patch", "band", "skymap"), 

57 defaultTemplates={"inputCoaddName": "deep", 

58 "outputCoaddName": "deep", 

59 "warpType": "direct", 

60 "warpTypeSuffix": "", 

61 "fakesType": ""}): 

62 inputWarps = pipeBase.connectionTypes.Input( 

63 doc=("Input list of warps to be assemebled i.e. stacked." 

64 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

65 name="{inputCoaddName}Coadd_{warpType}Warp", 

66 storageClass="ExposureF", 

67 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

68 deferLoad=True, 

69 multiple=True 

70 ) 

71 skyMap = pipeBase.connectionTypes.Input( 

72 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

73 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

74 storageClass="SkyMap", 

75 dimensions=("skymap", ), 

76 ) 

77 selectedVisits = pipeBase.connectionTypes.Input( 

78 doc="Selected visits to be coadded.", 

79 name="{outputCoaddName}VisitsDict", 

80 storageClass="StructuredDataDict", 

81 dimensions=("instrument", "tract", "patch", "skymap", "band") 

82 ) 

83 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

84 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

85 " BRIGHT_OBJECT."), 

86 name="brightObjectMask", 

87 storageClass="ObjectMaskCatalog", 

88 dimensions=("tract", "patch", "skymap", "band"), 

89 ) 

90 coaddExposure = pipeBase.connectionTypes.Output( 

91 doc="Output coadded exposure, produced by stacking input warps", 

92 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

93 storageClass="ExposureF", 

94 dimensions=("tract", "patch", "skymap", "band"), 

95 ) 

96 nImage = pipeBase.connectionTypes.Output( 

97 doc="Output image of number of input images per pixel", 

98 name="{outputCoaddName}Coadd_nImage", 

99 storageClass="ImageU", 

100 dimensions=("tract", "patch", "skymap", "band"), 

101 ) 

102 

103 def __init__(self, *, config=None): 

104 super().__init__(config=config) 

105 

106 # Override the connection's name template with config to replicate Gen2 behavior 

107 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

108 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

109 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

110 templateValues['warpType'] = config.warpType 

111 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

112 if config.hasFakes: 

113 templateValues['fakesType'] = "_fakes" 

114 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

115 for name in self.allConnections} 

116 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

117 # End code to remove after deprecation 

118 

119 if not config.doMaskBrightObjects: 

120 self.prerequisiteInputs.remove("brightObjectMask") 

121 

122 if not config.doSelectVisits: 

123 self.inputs.remove("selectedVisits") 

124 

125 if not config.doNImage: 

126 self.outputs.remove("nImage") 

127 

128 

129class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

130 pipelineConnections=AssembleCoaddConnections): 

131 """Configuration parameters for the `AssembleCoaddTask`. 

132 

133 Notes 

134 ----- 

135 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

136 only set the bitplane config.brightObjectMaskName. To make this useful you 

137 *must* also configure the flags.pixel algorithm, for example by adding 

138 

139 .. code-block:: none 

140 

141 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

142 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

143 

144 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

145 """ 

146 warpType = pexConfig.Field( 

147 doc="Warp name: one of 'direct' or 'psfMatched'", 

148 dtype=str, 

149 default="direct", 

150 ) 

151 subregionSize = pexConfig.ListField( 

152 dtype=int, 

153 doc="Width, height of stack subregion size; " 

154 "make small enough that a full stack of images will fit into memory at once.", 

155 length=2, 

156 default=(2000, 2000), 

157 ) 

158 statistic = pexConfig.Field( 

159 dtype=str, 

160 doc="Main stacking statistic for aggregating over the epochs.", 

161 default="MEANCLIP", 

162 ) 

163 doSigmaClip = pexConfig.Field( 

164 dtype=bool, 

165 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

166 default=False, 

167 ) 

168 sigmaClip = pexConfig.Field( 

169 dtype=float, 

170 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

171 default=3.0, 

172 ) 

173 clipIter = pexConfig.Field( 

174 dtype=int, 

175 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

176 default=2, 

177 ) 

178 calcErrorFromInputVariance = pexConfig.Field( 

179 dtype=bool, 

180 doc="Calculate coadd variance from input variance by stacking statistic." 

181 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

182 default=True, 

183 ) 

184 scaleZeroPoint = pexConfig.ConfigurableField( 

185 target=ScaleZeroPointTask, 

186 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

187 ) 

188 doInterp = pexConfig.Field( 

189 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

190 dtype=bool, 

191 default=True, 

192 ) 

193 interpImage = pexConfig.ConfigurableField( 

194 target=InterpImageTask, 

195 doc="Task to interpolate (and extrapolate) over NaN pixels", 

196 ) 

197 doWrite = pexConfig.Field( 

198 doc="Persist coadd?", 

199 dtype=bool, 

200 default=True, 

201 ) 

202 doNImage = pexConfig.Field( 

203 doc="Create image of number of contributing exposures for each pixel", 

204 dtype=bool, 

205 default=False, 

206 ) 

207 doUsePsfMatchedPolygons = pexConfig.Field( 

208 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

209 dtype=bool, 

210 default=False, 

211 ) 

212 maskPropagationThresholds = pexConfig.DictField( 

213 keytype=str, 

214 itemtype=float, 

215 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

216 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

217 "would have contributed exceeds this value."), 

218 default={"SAT": 0.1}, 

219 ) 

220 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

221 doc="Mask planes to remove before coadding") 

222 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

223 doc="Set mask and flag bits for bright objects?") 

224 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

225 doc="Name of mask bit used for bright objects") 

226 coaddPsf = pexConfig.ConfigField( 

227 doc="Configuration for CoaddPsf", 

228 dtype=measAlg.CoaddPsfConfig, 

229 ) 

230 doAttachTransmissionCurve = pexConfig.Field( 

231 dtype=bool, default=False, optional=False, 

232 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

233 "(requires all input Exposures to have TransmissionCurves).") 

234 ) 

235 hasFakes = pexConfig.Field( 

236 dtype=bool, 

237 default=False, 

238 doc="Should be set to True if fake sources have been inserted into the input data." 

239 ) 

240 doSelectVisits = pexConfig.Field( 

241 doc="Coadd only visits selected by a SelectVisitsTask", 

242 dtype=bool, 

243 default=False, 

244 ) 

245 

246 def setDefaults(self): 

247 super().setDefaults() 

248 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

249 

250 def validate(self): 

251 super().validate() 

252 if self.doPsfMatch: 

253 # Backwards compatibility. 

254 # Configs do not have loggers 

255 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

256 self.warpType = 'psfMatched' 

257 if self.doSigmaClip and self.statistic != "MEANCLIP": 

258 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

259 self.statistic = "MEANCLIP" 

260 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

261 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

262 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

263 

264 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

265 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

266 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

267 if str(k) not in unstackableStats] 

268 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

269 % (self.statistic, stackableStats)) 

270 

271 

272class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

273 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

274 

275 We want to assemble a coadded image from a set of Warps (also called 

276 coadded temporary exposures or ``coaddTempExps``). 

277 Each input Warp covers a patch on the sky and corresponds to a single 

278 run/visit/exposure of the covered patch. We provide the task with a list 

279 of Warps (``selectDataList``) from which it selects Warps that cover the 

280 specified patch (pointed at by ``dataRef``). 

281 Each Warp that goes into a coadd will typically have an independent 

282 photometric zero-point. Therefore, we must scale each Warp to set it to 

283 a common photometric zeropoint. WarpType may be one of 'direct' or 

284 'psfMatched', and the boolean configs `config.makeDirect` and 

285 `config.makePsfMatched` set which of the warp types will be coadded. 

286 The coadd is computed as a mean with optional outlier rejection. 

287 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

288 Finally, Warps can have bad 'NaN' pixels which received no input from the 

289 source calExps. We interpolate over these bad (NaN) pixels. 

290 

291 `AssembleCoaddTask` uses several sub-tasks. These are 

292 

293 - `ScaleZeroPointTask` 

294 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

295 - `InterpImageTask` 

296 - interpolate across bad pixels (NaN) in the final coadd 

297 

298 You can retarget these subtasks if you wish. 

299 

300 Notes 

301 ----- 

302 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

303 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

304 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

305 no debug variables of its own. Some of the subtasks may support debug 

306 variables. See the documentation for the subtasks for further information. 

307 

308 Examples 

309 -------- 

310 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

311 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

312 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

313 inputs: a data reference to the tract patch and filter to be coadded, and 

314 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

315 ``--selectId``, respectively: 

316 

317 .. code-block:: none 

318 

319 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

320 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

321 

322 Only the Warps that cover the specified tract and patch will be coadded. 

323 A list of the available optional arguments can be obtained by calling 

324 ``assembleCoadd.py`` with the ``--help`` command line argument: 

325 

326 .. code-block:: none 

327 

328 assembleCoadd.py --help 

329 

330 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

331 multi-band processing, we will generate the HSC-I & -R band coadds from 

332 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

333 assuming that the lsst stack has been already set up, we must set up the 

334 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

335 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

336 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

337 coadds, we must first 

338 

339 - processCcd 

340 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

341 - makeSkyMap 

342 - create a skymap that covers the area of the sky present in the raw exposures 

343 - makeCoaddTempExp 

344 - warp the individual calibrated exposures to the tangent plane of the coadd 

345 

346 We can perform all of these steps by running 

347 

348 .. code-block:: none 

349 

350 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

351 

352 This will produce warped exposures for each visit. To coadd the warped 

353 data, we call assembleCoadd.py as follows: 

354 

355 .. code-block:: none 

356 

357 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

358 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

359 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

360 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

361 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

362 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

363 --selectId visit=903988 ccd=24 

364 

365 that will process the HSC-I band data. The results are written in 

366 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

367 

368 You may also choose to run: 

369 

370 .. code-block:: none 

371 

372 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

373 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

374 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

375 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

376 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

377 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

378 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

379 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

380 

381 to generate the coadd for the HSC-R band if you are interested in 

382 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

383 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

384 rather than `AssembleCoaddTask` to make the coadd. 

385 """ 

386 ConfigClass = AssembleCoaddConfig 

387 _DefaultName = "assembleCoadd" 

388 

389 def __init__(self, *args, **kwargs): 

390 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

391 if args: 

392 argNames = ["config", "name", "parentTask", "log"] 

393 kwargs.update({k: v for k, v in zip(argNames, args)}) 

394 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

395 "PipelineTask will not take positional args" % argNames, FutureWarning) 

396 

397 super().__init__(**kwargs) 

398 self.makeSubtask("interpImage") 

399 self.makeSubtask("scaleZeroPoint") 

400 

401 if self.config.doMaskBrightObjects: 

402 mask = afwImage.Mask() 

403 try: 

404 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

405 except pexExceptions.LsstCppException: 

406 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

407 mask.getMaskPlaneDict().keys()) 

408 del mask 

409 

410 self.warpType = self.config.warpType 

411 

412 @utils.inheritDoc(pipeBase.PipelineTask) 

413 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

414 # Docstring to be formatted with info from PipelineTask.runQuantum 

415 """ 

416 Notes 

417 ----- 

418 Assemble a coadd from a set of Warps. 

419 

420 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

421 Analogous to `runDataRef`, it prepares all the data products to be 

422 passed to `run`, and processes the results before returning a struct 

423 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

424 Therefore, its inputs are accessed subregion by subregion 

425 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

426 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

427 correspond to an update in `runDataRef` while both entry points 

428 are used. 

429 """ 

430 inputData = butlerQC.get(inputRefs) 

431 

432 # Construct skyInfo expected by run 

433 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

434 skyMap = inputData["skyMap"] 

435 outputDataId = butlerQC.quantum.dataId 

436 

437 inputData['skyInfo'] = makeSkyInfo(skyMap, 

438 tractId=outputDataId['tract'], 

439 patchId=outputDataId['patch']) 

440 

441 if self.config.doSelectVisits: 

442 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

443 else: 

444 warpRefList = inputData['inputWarps'] 

445 

446 # Perform same middle steps as `runDataRef` does 

447 inputs = self.prepareInputs(warpRefList) 

448 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

449 self.getTempExpDatasetName(self.warpType)) 

450 if len(inputs.tempExpRefList) == 0: 

451 self.log.warn("No coadd temporary exposures found") 

452 return 

453 

454 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

455 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

456 inputs.weightList, supplementaryData=supplementaryData) 

457 

458 inputData.setdefault('brightObjectMask', None) 

459 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

460 

461 if self.config.doWrite: 

462 butlerQC.put(retStruct, outputRefs) 

463 return retStruct 

464 

465 @pipeBase.timeMethod 

466 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

467 """Assemble a coadd from a set of Warps. 

468 

469 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

470 Compute weights to be applied to each Warp and 

471 find scalings to match the photometric zeropoint to a reference Warp. 

472 Assemble the Warps using `run`. Interpolate over NaNs and 

473 optionally write the coadd to disk. Return the coadded exposure. 

474 

475 Parameters 

476 ---------- 

477 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

478 Data reference defining the patch for coaddition and the 

479 reference Warp (if ``config.autoReference=False``). 

480 Used to access the following data products: 

481 - ``self.config.coaddName + "Coadd_skyMap"`` 

482 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

483 - ``self.config.coaddName + "Coadd"`` 

484 selectDataList : `list` 

485 List of data references to Calexps. Data to be coadded will be 

486 selected from this list based on overlap with the patch defined 

487 by dataRef, grouped by visit, and converted to a list of data 

488 references to warps. 

489 warpRefList : `list` 

490 List of data references to Warps to be coadded. 

491 Note: `warpRefList` is just the new name for `tempExpRefList`. 

492 

493 Returns 

494 ------- 

495 retStruct : `lsst.pipe.base.Struct` 

496 Result struct with components: 

497 

498 - ``coaddExposure``: coadded exposure (``Exposure``). 

499 - ``nImage``: exposure count image (``Image``). 

500 """ 

501 if selectDataList and warpRefList: 

502 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

503 "and which to use is ambiguous. Please pass only one.") 

504 

505 skyInfo = self.getSkyInfo(dataRef) 

506 if warpRefList is None: 

507 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

508 if len(calExpRefList) == 0: 

509 self.log.warn("No exposures to coadd") 

510 return 

511 self.log.info("Coadding %d exposures", len(calExpRefList)) 

512 

513 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

514 

515 inputData = self.prepareInputs(warpRefList) 

516 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

517 self.getTempExpDatasetName(self.warpType)) 

518 if len(inputData.tempExpRefList) == 0: 

519 self.log.warn("No coadd temporary exposures found") 

520 return 

521 

522 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

523 

524 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

525 inputData.weightList, supplementaryData=supplementaryData) 

526 

527 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

528 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

529 

530 if self.config.doWrite: 

531 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 

532 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

533 else: 

534 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

535 self.log.info("Persisting %s" % coaddDatasetName) 

536 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

537 if self.config.doNImage and retStruct.nImage is not None: 

538 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

539 

540 return retStruct 

541 

542 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

543 """Interpolate over missing data and mask bright stars. 

544 

545 Parameters 

546 ---------- 

547 coaddExposure : `lsst.afw.image.Exposure` 

548 The coadded exposure to process. 

549 dataRef : `lsst.daf.persistence.ButlerDataRef` 

550 Butler data reference for supplementary data. 

551 """ 

552 if self.config.doInterp: 

553 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

554 # The variance must be positive; work around for DM-3201. 

555 varArray = coaddExposure.variance.array 

556 with numpy.errstate(invalid="ignore"): 

557 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

558 

559 if self.config.doMaskBrightObjects: 

560 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

561 

562 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

563 """Make additional inputs to run() specific to subclasses (Gen2) 

564 

565 Duplicates interface of `runDataRef` method 

566 Available to be implemented by subclasses only if they need the 

567 coadd dataRef for performing preliminary processing before 

568 assembling the coadd. 

569 

570 Parameters 

571 ---------- 

572 dataRef : `lsst.daf.persistence.ButlerDataRef` 

573 Butler data reference for supplementary data. 

574 selectDataList : `list` (optional) 

575 Optional List of data references to Calexps. 

576 warpRefList : `list` (optional) 

577 Optional List of data references to Warps. 

578 """ 

579 return pipeBase.Struct() 

580 

581 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

582 """Make additional inputs to run() specific to subclasses (Gen3) 

583 

584 Duplicates interface of `runQuantum` method. 

585 Available to be implemented by subclasses only if they need the 

586 coadd dataRef for performing preliminary processing before 

587 assembling the coadd. 

588 

589 Parameters 

590 ---------- 

591 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

592 Gen3 Butler object for fetching additional data products before 

593 running the Task specialized for quantum being processed 

594 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

595 Attributes are the names of the connections describing input dataset types. 

596 Values are DatasetRefs that task consumes for corresponding dataset type. 

597 DataIds are guaranteed to match data objects in ``inputData``. 

598 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

599 Attributes are the names of the connections describing output dataset types. 

600 Values are DatasetRefs that task is to produce 

601 for corresponding dataset type. 

602 """ 

603 return pipeBase.Struct() 

604 

605 def getTempExpRefList(self, patchRef, calExpRefList): 

606 """Generate list data references corresponding to warped exposures 

607 that lie within the patch to be coadded. 

608 

609 Parameters 

610 ---------- 

611 patchRef : `dataRef` 

612 Data reference for patch. 

613 calExpRefList : `list` 

614 List of data references for input calexps. 

615 

616 Returns 

617 ------- 

618 tempExpRefList : `list` 

619 List of Warp/CoaddTempExp data references. 

620 """ 

621 butler = patchRef.getButler() 

622 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

623 self.getTempExpDatasetName(self.warpType)) 

624 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

625 g, groupData.keys) for 

626 g in groupData.groups.keys()] 

627 return tempExpRefList 

628 

629 def prepareInputs(self, refList): 

630 """Prepare the input warps for coaddition by measuring the weight for 

631 each warp and the scaling for the photometric zero point. 

632 

633 Each Warp has its own photometric zeropoint and background variance. 

634 Before coadding these Warps together, compute a scale factor to 

635 normalize the photometric zeropoint and compute the weight for each Warp. 

636 

637 Parameters 

638 ---------- 

639 refList : `list` 

640 List of data references to tempExp 

641 

642 Returns 

643 ------- 

644 result : `lsst.pipe.base.Struct` 

645 Result struct with components: 

646 

647 - ``tempExprefList``: `list` of data references to tempExp. 

648 - ``weightList``: `list` of weightings. 

649 - ``imageScalerList``: `list` of image scalers. 

650 """ 

651 statsCtrl = afwMath.StatisticsControl() 

652 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

653 statsCtrl.setNumIter(self.config.clipIter) 

654 statsCtrl.setAndMask(self.getBadPixelMask()) 

655 statsCtrl.setNanSafe(True) 

656 # compute tempExpRefList: a list of tempExpRef that actually exist 

657 # and weightList: a list of the weight of the associated coadd tempExp 

658 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

659 tempExpRefList = [] 

660 weightList = [] 

661 imageScalerList = [] 

662 tempExpName = self.getTempExpDatasetName(self.warpType) 

663 for tempExpRef in refList: 

664 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

665 # therefore have no datasetExists() method 

666 if not isinstance(tempExpRef, DeferredDatasetHandle): 

667 if not tempExpRef.datasetExists(tempExpName): 

668 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

669 continue 

670 

671 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

672 # Ignore any input warp that is empty of data 

673 if numpy.isnan(tempExp.image.array).all(): 

674 continue 

675 maskedImage = tempExp.getMaskedImage() 

676 imageScaler = self.scaleZeroPoint.computeImageScaler( 

677 exposure=tempExp, 

678 dataRef=tempExpRef, 

679 ) 

680 try: 

681 imageScaler.scaleMaskedImage(maskedImage) 

682 except Exception as e: 

683 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

684 continue 

685 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

686 afwMath.MEANCLIP, statsCtrl) 

687 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

688 weight = 1.0 / float(meanVar) 

689 if not numpy.isfinite(weight): 

690 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

691 continue 

692 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

693 

694 del maskedImage 

695 del tempExp 

696 

697 tempExpRefList.append(tempExpRef) 

698 weightList.append(weight) 

699 imageScalerList.append(imageScaler) 

700 

701 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

702 imageScalerList=imageScalerList) 

703 

704 def prepareStats(self, mask=None): 

705 """Prepare the statistics for coadding images. 

706 

707 Parameters 

708 ---------- 

709 mask : `int`, optional 

710 Bit mask value to exclude from coaddition. 

711 

712 Returns 

713 ------- 

714 stats : `lsst.pipe.base.Struct` 

715 Statistics structure with the following fields: 

716 

717 - ``statsCtrl``: Statistics control object for coadd 

718 (`lsst.afw.math.StatisticsControl`) 

719 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

720 """ 

721 if mask is None: 

722 mask = self.getBadPixelMask() 

723 statsCtrl = afwMath.StatisticsControl() 

724 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

725 statsCtrl.setNumIter(self.config.clipIter) 

726 statsCtrl.setAndMask(mask) 

727 statsCtrl.setNanSafe(True) 

728 statsCtrl.setWeighted(True) 

729 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

730 for plane, threshold in self.config.maskPropagationThresholds.items(): 

731 bit = afwImage.Mask.getMaskPlane(plane) 

732 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

733 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

734 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

735 

736 @pipeBase.timeMethod 

737 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

738 altMaskList=None, mask=None, supplementaryData=None): 

739 """Assemble a coadd from input warps 

740 

741 Assemble the coadd using the provided list of coaddTempExps. Since 

742 the full coadd covers a patch (a large area), the assembly is 

743 performed over small areas on the image at a time in order to 

744 conserve memory usage. Iterate over subregions within the outer 

745 bbox of the patch using `assembleSubregion` to stack the corresponding 

746 subregions from the coaddTempExps with the statistic specified. 

747 Set the edge bits the coadd mask based on the weight map. 

748 

749 Parameters 

750 ---------- 

751 skyInfo : `lsst.pipe.base.Struct` 

752 Struct with geometric information about the patch. 

753 tempExpRefList : `list` 

754 List of data references to Warps (previously called CoaddTempExps). 

755 imageScalerList : `list` 

756 List of image scalers. 

757 weightList : `list` 

758 List of weights 

759 altMaskList : `list`, optional 

760 List of alternate masks to use rather than those stored with 

761 tempExp. 

762 mask : `int`, optional 

763 Bit mask value to exclude from coaddition. 

764 supplementaryData : lsst.pipe.base.Struct, optional 

765 Struct with additional data products needed to assemble coadd. 

766 Only used by subclasses that implement `makeSupplementaryData` 

767 and override `run`. 

768 

769 Returns 

770 ------- 

771 result : `lsst.pipe.base.Struct` 

772 Result struct with components: 

773 

774 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

775 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

776 - ``warpRefList``: input list of refs to the warps ( 

777 ``lsst.daf.butler.DeferredDatasetHandle`` or 

778 ``lsst.daf.persistence.ButlerDataRef``) 

779 (unmodified) 

780 - ``imageScalerList``: input list of image scalers (unmodified) 

781 - ``weightList``: input list of weights (unmodified) 

782 """ 

783 tempExpName = self.getTempExpDatasetName(self.warpType) 

784 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

785 stats = self.prepareStats(mask=mask) 

786 

787 if altMaskList is None: 

788 altMaskList = [None]*len(tempExpRefList) 

789 

790 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

791 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

792 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

793 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

794 coaddMaskedImage = coaddExposure.getMaskedImage() 

795 subregionSizeArr = self.config.subregionSize 

796 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

797 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

798 if self.config.doNImage: 

799 nImage = afwImage.ImageU(skyInfo.bbox) 

800 else: 

801 nImage = None 

802 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

803 try: 

804 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

805 weightList, altMaskList, stats.flags, stats.ctrl, 

806 nImage=nImage) 

807 except Exception as e: 

808 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

809 

810 self.setInexactPsf(coaddMaskedImage.getMask()) 

811 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

812 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

813 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

814 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

815 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

816 weightList=weightList) 

817 

818 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

819 """Set the metadata for the coadd. 

820 

821 This basic implementation sets the filter from the first input. 

822 

823 Parameters 

824 ---------- 

825 coaddExposure : `lsst.afw.image.Exposure` 

826 The target exposure for the coadd. 

827 tempExpRefList : `list` 

828 List of data references to tempExp. 

829 weightList : `list` 

830 List of weights. 

831 """ 

832 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

833 tempExpName = self.getTempExpDatasetName(self.warpType) 

834 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

835 # (and we need more than just the PropertySet that contains the header), which is not possible 

836 # with the current butler (see #2777). 

837 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

838 

839 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 

840 # Gen 3 API 

841 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

842 else: 

843 # Gen 2 API. Delete this when Gen 2 retired 

844 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

845 for tempExpRef in tempExpRefList] 

846 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

847 

848 # Set the coadd FilterLabel to the band of the first input exposure: 

849 # Coadds are calibrated, so the physical label is now meaningless. 

850 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

851 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

852 coaddInputs.ccds.reserve(numCcds) 

853 coaddInputs.visits.reserve(len(tempExpList)) 

854 

855 for tempExp, weight in zip(tempExpList, weightList): 

856 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

857 

858 if self.config.doUsePsfMatchedPolygons: 

859 self.shrinkValidPolygons(coaddInputs) 

860 

861 coaddInputs.visits.sort() 

862 if self.warpType == "psfMatched": 

863 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

864 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

865 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

866 # having the maximum width (sufficient because square) 

867 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

868 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

869 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

870 else: 

871 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

872 self.config.coaddPsf.makeControl()) 

873 coaddExposure.setPsf(psf) 

874 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

875 coaddExposure.getWcs()) 

876 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

877 if self.config.doAttachTransmissionCurve: 

878 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

879 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

880 

881 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

882 altMaskList, statsFlags, statsCtrl, nImage=None): 

883 """Assemble the coadd for a sub-region. 

884 

885 For each coaddTempExp, check for (and swap in) an alternative mask 

886 if one is passed. Remove mask planes listed in 

887 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

888 `lsst.afw.math.statisticsStack` with the statistic specified by 

889 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

890 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

891 an N-sigma clipped mean where N and iterations are specified by 

892 statsCtrl. Assign the stacked subregion back to the coadd. 

893 

894 Parameters 

895 ---------- 

896 coaddExposure : `lsst.afw.image.Exposure` 

897 The target exposure for the coadd. 

898 bbox : `lsst.geom.Box` 

899 Sub-region to coadd. 

900 tempExpRefList : `list` 

901 List of data reference to tempExp. 

902 imageScalerList : `list` 

903 List of image scalers. 

904 weightList : `list` 

905 List of weights. 

906 altMaskList : `list` 

907 List of alternate masks to use rather than those stored with 

908 tempExp, or None. Each element is dict with keys = mask plane 

909 name to which to add the spans. 

910 statsFlags : `lsst.afw.math.Property` 

911 Property object for statistic for coadd. 

912 statsCtrl : `lsst.afw.math.StatisticsControl` 

913 Statistics control object for coadd. 

914 nImage : `lsst.afw.image.ImageU`, optional 

915 Keeps track of exposure count for each pixel. 

916 """ 

917 self.log.debug("Computing coadd over %s", bbox) 

918 tempExpName = self.getTempExpDatasetName(self.warpType) 

919 coaddExposure.mask.addMaskPlane("REJECTED") 

920 coaddExposure.mask.addMaskPlane("CLIPPED") 

921 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

922 maskMap = self.setRejectedMaskMapping(statsCtrl) 

923 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

924 maskedImageList = [] 

925 if nImage is not None: 

926 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

927 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

928 

929 if isinstance(tempExpRef, DeferredDatasetHandle): 

930 # Gen 3 API 

931 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

932 else: 

933 # Gen 2 API. Delete this when Gen 2 retired 

934 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

935 

936 maskedImage = exposure.getMaskedImage() 

937 mask = maskedImage.getMask() 

938 if altMask is not None: 

939 self.applyAltMaskPlanes(mask, altMask) 

940 imageScaler.scaleMaskedImage(maskedImage) 

941 

942 # Add 1 for each pixel which is not excluded by the exclude mask. 

943 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

944 if nImage is not None: 

945 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

946 if self.config.removeMaskPlanes: 

947 self.removeMaskPlanes(maskedImage) 

948 maskedImageList.append(maskedImage) 

949 

950 with self.timer("stack"): 

951 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

952 clipped, # also set output to CLIPPED if sigma-clipped 

953 maskMap) 

954 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

955 if nImage is not None: 

956 nImage.assign(subNImage, bbox) 

957 

958 def removeMaskPlanes(self, maskedImage): 

959 """Unset the mask of an image for mask planes specified in the config. 

960 

961 Parameters 

962 ---------- 

963 maskedImage : `lsst.afw.image.MaskedImage` 

964 The masked image to be modified. 

965 """ 

966 mask = maskedImage.getMask() 

967 for maskPlane in self.config.removeMaskPlanes: 

968 try: 

969 mask &= ~mask.getPlaneBitMask(maskPlane) 

970 except pexExceptions.InvalidParameterError: 

971 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

972 maskPlane) 

973 

974 @staticmethod 

975 def setRejectedMaskMapping(statsCtrl): 

976 """Map certain mask planes of the warps to new planes for the coadd. 

977 

978 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

979 or CLIPPED, set it to REJECTED on the coadd. 

980 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

981 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

982 

983 Parameters 

984 ---------- 

985 statsCtrl : `lsst.afw.math.StatisticsControl` 

986 Statistics control object for coadd 

987 

988 Returns 

989 ------- 

990 maskMap : `list` of `tuple` of `int` 

991 A list of mappings of mask planes of the warped exposures to 

992 mask planes of the coadd. 

993 """ 

994 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

995 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

996 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

997 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

998 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

999 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1000 (clipped, clipped)] 

1001 return maskMap 

1002 

1003 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1004 """Apply in place alt mask formatted as SpanSets to a mask. 

1005 

1006 Parameters 

1007 ---------- 

1008 mask : `lsst.afw.image.Mask` 

1009 Original mask. 

1010 altMaskSpans : `dict` 

1011 SpanSet lists to apply. Each element contains the new mask 

1012 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1013 and list of SpanSets to apply to the mask. 

1014 

1015 Returns 

1016 ------- 

1017 mask : `lsst.afw.image.Mask` 

1018 Updated mask. 

1019 """ 

1020 if self.config.doUsePsfMatchedPolygons: 

1021 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

1022 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1023 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1024 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1025 # This mask-clearing step must only occur *before* applying the new masks below 

1026 for spanSet in altMaskSpans['NO_DATA']: 

1027 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1028 

1029 for plane, spanSetList in altMaskSpans.items(): 

1030 maskClipValue = mask.addMaskPlane(plane) 

1031 for spanSet in spanSetList: 

1032 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1033 return mask 

1034 

1035 def shrinkValidPolygons(self, coaddInputs): 

1036 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1037 

1038 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1039 does not have a validPolygon, create one from its bbox. 

1040 

1041 Parameters 

1042 ---------- 

1043 coaddInputs : `lsst.afw.image.coaddInputs` 

1044 Original mask. 

1045 

1046 """ 

1047 for ccd in coaddInputs.ccds: 

1048 polyOrig = ccd.getValidPolygon() 

1049 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1050 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1051 if polyOrig: 

1052 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1053 else: 

1054 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1055 ccd.setValidPolygon(validPolygon) 

1056 

1057 def readBrightObjectMasks(self, dataRef): 

1058 """Retrieve the bright object masks. 

1059 

1060 Returns None on failure. 

1061 

1062 Parameters 

1063 ---------- 

1064 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1065 A Butler dataRef. 

1066 

1067 Returns 

1068 ------- 

1069 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1070 Bright object mask from the Butler object, or None if it cannot 

1071 be retrieved. 

1072 """ 

1073 try: 

1074 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1075 except Exception as e: 

1076 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1077 return None 

1078 

1079 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1080 """Set the bright object masks. 

1081 

1082 Parameters 

1083 ---------- 

1084 exposure : `lsst.afw.image.Exposure` 

1085 Exposure under consideration. 

1086 dataId : `lsst.daf.persistence.dataId` 

1087 Data identifier dict for patch. 

1088 brightObjectMasks : `lsst.afw.table` 

1089 Table of bright objects to mask. 

1090 """ 

1091 

1092 if brightObjectMasks is None: 

1093 self.log.warn("Unable to apply bright object mask: none supplied") 

1094 return 

1095 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1096 mask = exposure.getMaskedImage().getMask() 

1097 wcs = exposure.getWcs() 

1098 plateScale = wcs.getPixelScale().asArcseconds() 

1099 

1100 for rec in brightObjectMasks: 

1101 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1102 if rec["type"] == "box": 

1103 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1104 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1105 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1106 

1107 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1108 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1109 

1110 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1111 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1112 spans = afwGeom.SpanSet(bbox) 

1113 elif rec["type"] == "circle": 

1114 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1115 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1116 else: 

1117 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1118 continue 

1119 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1120 

1121 def setInexactPsf(self, mask): 

1122 """Set INEXACT_PSF mask plane. 

1123 

1124 If any of the input images isn't represented in the coadd (due to 

1125 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1126 these pixels. 

1127 

1128 Parameters 

1129 ---------- 

1130 mask : `lsst.afw.image.Mask` 

1131 Coadded exposure's mask, modified in-place. 

1132 """ 

1133 mask.addMaskPlane("INEXACT_PSF") 

1134 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1135 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1136 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1137 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1138 array = mask.getArray() 

1139 selected = array & (sensorEdge | clipped | rejected) > 0 

1140 array[selected] |= inexactPsf 

1141 

1142 @classmethod 

1143 def _makeArgumentParser(cls): 

1144 """Create an argument parser. 

1145 """ 

1146 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1147 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1148 + cls.ConfigClass().warpType + "Warp", 

1149 help="data ID, e.g. --id tract=12345 patch=1,2", 

1150 ContainerClass=AssembleCoaddDataIdContainer) 

1151 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1152 ContainerClass=SelectDataIdContainer) 

1153 return parser 

1154 

1155 @staticmethod 

1156 def _subBBoxIter(bbox, subregionSize): 

1157 """Iterate over subregions of a bbox. 

1158 

1159 Parameters 

1160 ---------- 

1161 bbox : `lsst.geom.Box2I` 

1162 Bounding box over which to iterate. 

1163 subregionSize: `lsst.geom.Extent2I` 

1164 Size of sub-bboxes. 

1165 

1166 Yields 

1167 ------ 

1168 subBBox : `lsst.geom.Box2I` 

1169 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1170 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1171 the edges of ``bbox``, but it will never be empty. 

1172 """ 

1173 if bbox.isEmpty(): 

1174 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1175 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1176 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1177 

1178 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1179 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1180 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1181 subBBox.clip(bbox) 

1182 if subBBox.isEmpty(): 

1183 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1184 "colShift=%s, rowShift=%s" % 

1185 (bbox, subregionSize, colShift, rowShift)) 

1186 yield subBBox 

1187 

1188 def filterWarps(self, inputs, goodVisits): 

1189 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1190 

1191 Parameters 

1192 ---------- 

1193 inputs : list 

1194 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1195 goodVisit : `dict` 

1196 Dictionary with good visitIds as the keys. Value ignored. 

1197 

1198 Returns: 

1199 -------- 

1200 filteredInputs : `list` 

1201 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1202 """ 

1203 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1204 filteredInputs = [] 

1205 for visit in goodVisits.keys(): 

1206 filteredInputs.append(inputWarpDict[visit]) 

1207 return filteredInputs 

1208 

1209 

1210class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1211 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1212 """ 

1213 

1214 def makeDataRefList(self, namespace): 

1215 """Make self.refList from self.idList. 

1216 

1217 Parameters 

1218 ---------- 

1219 namespace 

1220 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1221 """ 

1222 datasetType = namespace.config.coaddName + "Coadd" 

1223 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1224 

1225 for dataId in self.idList: 

1226 # tract and patch are required 

1227 for key in keysCoadd: 

1228 if key not in dataId: 

1229 raise RuntimeError("--id must include " + key) 

1230 

1231 dataRef = namespace.butler.dataRef( 

1232 datasetType=datasetType, 

1233 dataId=dataId, 

1234 ) 

1235 self.refList.append(dataRef) 

1236 

1237 

1238def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1239 """Function to count the number of pixels with a specific mask in a 

1240 footprint. 

1241 

1242 Find the intersection of mask & footprint. Count all pixels in the mask 

1243 that are in the intersection that have bitmask set but do not have 

1244 ignoreMask set. Return the count. 

1245 

1246 Parameters 

1247 ---------- 

1248 mask : `lsst.afw.image.Mask` 

1249 Mask to define intersection region by. 

1250 footprint : `lsst.afw.detection.Footprint` 

1251 Footprint to define the intersection region by. 

1252 bitmask 

1253 Specific mask that we wish to count the number of occurances of. 

1254 ignoreMask 

1255 Pixels to not consider. 

1256 

1257 Returns 

1258 ------- 

1259 result : `int` 

1260 Count of number of pixels in footprint with specified mask. 

1261 """ 

1262 bbox = footprint.getBBox() 

1263 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1264 fp = afwImage.Mask(bbox) 

1265 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1266 footprint.spans.setMask(fp, bitmask) 

1267 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1268 (subMask.getArray() & ignoreMask) == 0).sum() 

1269 

1270 

1271class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1272 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1273 """ 

1274 clipDetection = pexConfig.ConfigurableField( 

1275 target=SourceDetectionTask, 

1276 doc="Detect sources on difference between unclipped and clipped coadd") 

1277 minClipFootOverlap = pexConfig.Field( 

1278 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1279 dtype=float, 

1280 default=0.6 

1281 ) 

1282 minClipFootOverlapSingle = pexConfig.Field( 

1283 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1284 "clipped when only one visit overlaps", 

1285 dtype=float, 

1286 default=0.5 

1287 ) 

1288 minClipFootOverlapDouble = pexConfig.Field( 

1289 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1290 "clipped when two visits overlap", 

1291 dtype=float, 

1292 default=0.45 

1293 ) 

1294 maxClipFootOverlapDouble = pexConfig.Field( 

1295 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1296 "considering two visits", 

1297 dtype=float, 

1298 default=0.15 

1299 ) 

1300 minBigOverlap = pexConfig.Field( 

1301 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1302 "when labeling clipped footprints", 

1303 dtype=int, 

1304 default=100 

1305 ) 

1306 

1307 def setDefaults(self): 

1308 """Set default values for clipDetection. 

1309 

1310 Notes 

1311 ----- 

1312 The numeric values for these configuration parameters were 

1313 empirically determined, future work may further refine them. 

1314 """ 

1315 AssembleCoaddConfig.setDefaults(self) 

1316 self.clipDetection.doTempLocalBackground = False 

1317 self.clipDetection.reEstimateBackground = False 

1318 self.clipDetection.returnOriginalFootprints = False 

1319 self.clipDetection.thresholdPolarity = "both" 

1320 self.clipDetection.thresholdValue = 2 

1321 self.clipDetection.nSigmaToGrow = 2 

1322 self.clipDetection.minPixels = 4 

1323 self.clipDetection.isotropicGrow = True 

1324 self.clipDetection.thresholdType = "pixel_stdev" 

1325 self.sigmaClip = 1.5 

1326 self.clipIter = 3 

1327 self.statistic = "MEAN" 

1328 

1329 def validate(self): 

1330 if self.doSigmaClip: 

1331 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1332 "Ignoring doSigmaClip.") 

1333 self.doSigmaClip = False 

1334 if self.statistic != "MEAN": 

1335 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1336 "(%s chosen). Please set statistic to MEAN." 

1337 % (self.statistic)) 

1338 AssembleCoaddTask.ConfigClass.validate(self) 

1339 

1340 

1341class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1342 """Assemble a coadded image from a set of coadded temporary exposures, 

1343 being careful to clip & flag areas with potential artifacts. 

1344 

1345 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1346 we clip outliers). The problem with doing this is that when computing the 

1347 coadd PSF at a given location, individual visit PSFs from visits with 

1348 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1349 In this task, we correct for this behavior by creating a new 

1350 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1351 coaddTempExps and the final coadd where 

1352 

1353 i. difference imaging suggests that there is an outlier and 

1354 ii. this outlier appears on only one or two images. 

1355 

1356 Such regions will not contribute to the final coadd. Furthermore, any 

1357 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1358 Note that the algorithm implemented by this task is preliminary and works 

1359 correctly for HSC data. Parameter modifications and or considerable 

1360 redesigning of the algorithm is likley required for other surveys. 

1361 

1362 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1363 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1364 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1365 if you wish. 

1366 

1367 Notes 

1368 ----- 

1369 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1370 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1371 see `baseDebug` for more about ``debug.py`` files. 

1372 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1373 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1374 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1375 for further information. 

1376 

1377 Examples 

1378 -------- 

1379 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1380 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1381 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1382 

1383 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1384 and filter to be coadded (specified using 

1385 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1386 along with a list of coaddTempExps to attempt to coadd (specified using 

1387 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1388 Only the coaddTempExps that cover the specified tract and patch will be 

1389 coadded. A list of the available optional arguments can be obtained by 

1390 calling assembleCoadd.py with the --help command line argument: 

1391 

1392 .. code-block:: none 

1393 

1394 assembleCoadd.py --help 

1395 

1396 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1397 context of multi-band processing, we will generate the HSC-I & -R band 

1398 coadds from HSC engineering test data provided in the ci_hsc package. 

1399 To begin, assuming that the lsst stack has been already set up, we must 

1400 set up the obs_subaru and ci_hsc packages. This defines the environment 

1401 variable $CI_HSC_DIR and points at the location of the package. The raw 

1402 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1403 the coadds, we must first 

1404 

1405 - ``processCcd`` 

1406 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1407 - ``makeSkyMap`` 

1408 create a skymap that covers the area of the sky present in the raw exposures 

1409 - ``makeCoaddTempExp`` 

1410 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1411 

1412 We can perform all of these steps by running 

1413 

1414 .. code-block:: none 

1415 

1416 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1417 

1418 This will produce warped coaddTempExps for each visit. To coadd the 

1419 warped data, we call ``assembleCoadd.py`` as follows: 

1420 

1421 .. code-block:: none 

1422 

1423 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1424 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1425 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1426 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1427 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1428 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1429 --selectId visit=903988 ccd=24 

1430 

1431 This will process the HSC-I band data. The results are written in 

1432 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1433 

1434 You may also choose to run: 

1435 

1436 .. code-block:: none 

1437 

1438 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1439 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1440 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1441 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1442 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1443 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1444 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1445 --selectId visit=903346 ccd=12 

1446 

1447 to generate the coadd for the HSC-R band if you are interested in following 

1448 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1449 """ 

1450 ConfigClass = SafeClipAssembleCoaddConfig 

1451 _DefaultName = "safeClipAssembleCoadd" 

1452 

1453 def __init__(self, *args, **kwargs): 

1454 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1455 schema = afwTable.SourceTable.makeMinimalSchema() 

1456 self.makeSubtask("clipDetection", schema=schema) 

1457 

1458 @utils.inheritDoc(AssembleCoaddTask) 

1459 @pipeBase.timeMethod 

1460 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1461 """Assemble the coadd for a region. 

1462 

1463 Compute the difference of coadds created with and without outlier 

1464 rejection to identify coadd pixels that have outlier values in some 

1465 individual visits. 

1466 Detect clipped regions on the difference image and mark these regions 

1467 on the one or two individual coaddTempExps where they occur if there 

1468 is significant overlap between the clipped region and a source. This 

1469 leaves us with a set of footprints from the difference image that have 

1470 been identified as having occured on just one or two individual visits. 

1471 However, these footprints were generated from a difference image. It 

1472 is conceivable for a large diffuse source to have become broken up 

1473 into multiple footprints acrosss the coadd difference in this process. 

1474 Determine the clipped region from all overlapping footprints from the 

1475 detected sources in each visit - these are big footprints. 

1476 Combine the small and big clipped footprints and mark them on a new 

1477 bad mask plane. 

1478 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1479 removal. Clipped footprints will no longer make it into the coadd 

1480 because they are marked in the new bad mask plane. 

1481 

1482 Notes 

1483 ----- 

1484 args and kwargs are passed but ignored in order to match the call 

1485 signature expected by the parent task. 

1486 """ 

1487 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1488 mask = exp.getMaskedImage().getMask() 

1489 mask.addMaskPlane("CLIPPED") 

1490 

1491 result = self.detectClip(exp, tempExpRefList) 

1492 

1493 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1494 

1495 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1496 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1497 # Append big footprints from individual Warps to result.clipSpans 

1498 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1499 result.detectionFootprints, maskClipValue, maskDetValue, 

1500 exp.getBBox()) 

1501 # Create mask of the current clipped footprints 

1502 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1503 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1504 

1505 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1506 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1507 maskClip |= maskClipBig 

1508 

1509 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1510 badMaskPlanes = self.config.badMaskPlanes[:] 

1511 badMaskPlanes.append("CLIPPED") 

1512 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1513 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1514 result.clipSpans, mask=badPixelMask) 

1515 

1516 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1517 """Return an exposure that contains the difference between unclipped 

1518 and clipped coadds. 

1519 

1520 Generate a difference image between clipped and unclipped coadds. 

1521 Compute the difference image by subtracting an outlier-clipped coadd 

1522 from an outlier-unclipped coadd. Return the difference image. 

1523 

1524 Parameters 

1525 ---------- 

1526 skyInfo : `lsst.pipe.base.Struct` 

1527 Patch geometry information, from getSkyInfo 

1528 tempExpRefList : `list` 

1529 List of data reference to tempExp 

1530 imageScalerList : `list` 

1531 List of image scalers 

1532 weightList : `list` 

1533 List of weights 

1534 

1535 Returns 

1536 ------- 

1537 exp : `lsst.afw.image.Exposure` 

1538 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1539 """ 

1540 config = AssembleCoaddConfig() 

1541 # getattr necessary because subtasks do not survive Config.toDict() 

1542 # exclude connections because the class of self.config.connections is not 

1543 # the same as AssembleCoaddConfig.connections, and the connections are not 

1544 # needed to run this task anyway. 

1545 configIntersection = {k: getattr(self.config, k) 

1546 for k, v in self.config.toDict().items() 

1547 if (k in config.keys() and k != "connections")} 

1548 config.update(**configIntersection) 

1549 

1550 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1551 config.statistic = 'MEAN' 

1552 task = AssembleCoaddTask(config=config) 

1553 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1554 

1555 config.statistic = 'MEANCLIP' 

1556 task = AssembleCoaddTask(config=config) 

1557 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1558 

1559 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1560 coaddDiff -= coaddClip.getMaskedImage() 

1561 exp = afwImage.ExposureF(coaddDiff) 

1562 exp.setPsf(coaddMean.getPsf()) 

1563 return exp 

1564 

1565 def detectClip(self, exp, tempExpRefList): 

1566 """Detect clipped regions on an exposure and set the mask on the 

1567 individual tempExp masks. 

1568 

1569 Detect footprints in the difference image after smoothing the 

1570 difference image with a Gaussian kernal. Identify footprints that 

1571 overlap with one or two input ``coaddTempExps`` by comparing the 

1572 computed overlap fraction to thresholds set in the config. A different 

1573 threshold is applied depending on the number of overlapping visits 

1574 (restricted to one or two). If the overlap exceeds the thresholds, 

1575 the footprint is considered "CLIPPED" and is marked as such on the 

1576 coaddTempExp. Return a struct with the clipped footprints, the indices 

1577 of the ``coaddTempExps`` that end up overlapping with the clipped 

1578 footprints, and a list of new masks for the ``coaddTempExps``. 

1579 

1580 Parameters 

1581 ---------- 

1582 exp : `lsst.afw.image.Exposure` 

1583 Exposure to run detection on. 

1584 tempExpRefList : `list` 

1585 List of data reference to tempExp. 

1586 

1587 Returns 

1588 ------- 

1589 result : `lsst.pipe.base.Struct` 

1590 Result struct with components: 

1591 

1592 - ``clipFootprints``: list of clipped footprints. 

1593 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1594 ``tempExpRefList``. 

1595 - ``clipSpans``: List of dictionaries containing spanSet lists 

1596 to clip. Each element contains the new maskplane name 

1597 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1598 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1599 compressed into footprints. 

1600 """ 

1601 mask = exp.getMaskedImage().getMask() 

1602 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1603 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1604 # Merge positive and negative together footprints together 

1605 fpSet.positive.merge(fpSet.negative) 

1606 footprints = fpSet.positive 

1607 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1608 ignoreMask = self.getBadPixelMask() 

1609 

1610 clipFootprints = [] 

1611 clipIndices = [] 

1612 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1613 

1614 # for use by detectClipBig 

1615 visitDetectionFootprints = [] 

1616 

1617 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1618 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1619 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1620 

1621 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1622 for i, warpRef in enumerate(tempExpRefList): 

1623 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1624 immediate=True).getMaskedImage().getMask() 

1625 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1626 afwImage.PARENT, True) 

1627 maskVisitDet &= maskDetValue 

1628 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1629 visitDetectionFootprints.append(visitFootprints) 

1630 

1631 for j, footprint in enumerate(footprints.getFootprints()): 

1632 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1633 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1634 

1635 # build a list of clipped spans for each visit 

1636 for j, footprint in enumerate(footprints.getFootprints()): 

1637 nPixel = footprint.getArea() 

1638 overlap = [] # hold the overlap with each visit 

1639 indexList = [] # index of visit in global list 

1640 for i in range(len(tempExpRefList)): 

1641 ignore = ignoreArr[i, j] 

1642 overlapDet = overlapDetArr[i, j] 

1643 totPixel = nPixel - ignore 

1644 

1645 # If we have more bad pixels than detection skip 

1646 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 

1647 continue 

1648 overlap.append(overlapDet/float(totPixel)) 

1649 indexList.append(i) 

1650 

1651 overlap = numpy.array(overlap) 

1652 if not len(overlap): 

1653 continue 

1654 

1655 keep = False # Should this footprint be marked as clipped? 

1656 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1657 

1658 # If footprint only has one overlap use a lower threshold 

1659 if len(overlap) == 1: 

1660 if overlap[0] > self.config.minClipFootOverlapSingle: 

1661 keep = True 

1662 keepIndex = [0] 

1663 else: 

1664 # This is the general case where only visit should be clipped 

1665 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1666 if len(clipIndex) == 1: 

1667 keep = True 

1668 keepIndex = [clipIndex[0]] 

1669 

1670 # Test if there are clipped objects that overlap two different visits 

1671 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1672 if len(clipIndex) == 2 and len(overlap) > 3: 

1673 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1674 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1675 keep = True 

1676 keepIndex = clipIndex 

1677 

1678 if not keep: 

1679 continue 

1680 

1681 for index in keepIndex: 

1682 globalIndex = indexList[index] 

1683 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1684 

1685 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1686 clipFootprints.append(footprint) 

1687 

1688 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1689 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1690 

1691 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1692 maskClipValue, maskDetValue, coaddBBox): 

1693 """Return individual warp footprints for large artifacts and append 

1694 them to ``clipList`` in place. 

1695 

1696 Identify big footprints composed of many sources in the coadd 

1697 difference that may have originated in a large diffuse source in the 

1698 coadd. We do this by indentifying all clipped footprints that overlap 

1699 significantly with each source in all the coaddTempExps. 

1700 

1701 Parameters 

1702 ---------- 

1703 clipList : `list` 

1704 List of alt mask SpanSets with clipping information. Modified. 

1705 clipFootprints : `list` 

1706 List of clipped footprints. 

1707 clipIndices : `list` 

1708 List of which entries in tempExpClipList each footprint belongs to. 

1709 maskClipValue 

1710 Mask value of clipped pixels. 

1711 maskDetValue 

1712 Mask value of detected pixels. 

1713 coaddBBox : `lsst.geom.Box` 

1714 BBox of the coadd and warps. 

1715 

1716 Returns 

1717 ------- 

1718 bigFootprintsCoadd : `list` 

1719 List of big footprints 

1720 """ 

1721 bigFootprintsCoadd = [] 

1722 ignoreMask = self.getBadPixelMask() 

1723 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1724 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1725 for footprint in visitFootprints.getFootprints(): 

1726 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1727 

1728 # build a mask of clipped footprints that are in this visit 

1729 clippedFootprintsVisit = [] 

1730 for foot, clipIndex in zip(clipFootprints, clipIndices): 

1731 if index not in clipIndex: 

1732 continue 

1733 clippedFootprintsVisit.append(foot) 

1734 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1735 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1736 

1737 bigFootprintsVisit = [] 

1738 for foot in visitFootprints.getFootprints(): 

1739 if foot.getArea() < self.config.minBigOverlap: 

1740 continue 

1741 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1742 if nCount > self.config.minBigOverlap: 

1743 bigFootprintsVisit.append(foot) 

1744 bigFootprintsCoadd.append(foot) 

1745 

1746 for footprint in bigFootprintsVisit: 

1747 clippedSpans["CLIPPED"].append(footprint.spans) 

1748 

1749 return bigFootprintsCoadd 

1750 

1751 

1752class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1753 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1754 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1755 "Only PSF-Matched Warps make sense for image subtraction. " 

1756 "Therefore, they must be an additional declared input."), 

1757 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1758 storageClass="ExposureF", 

1759 dimensions=("tract", "patch", "skymap", "visit"), 

1760 deferLoad=True, 

1761 multiple=True 

1762 ) 

1763 templateCoadd = pipeBase.connectionTypes.Output( 

1764 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1765 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1766 name="{fakesType}{outputCoaddName}CoaddPsfMatched", 

1767 storageClass="ExposureF", 

1768 dimensions=("tract", "patch", "skymap", "band"), 

1769 ) 

1770 

1771 def __init__(self, *, config=None): 

1772 super().__init__(config=config) 

1773 if not config.assembleStaticSkyModel.doWrite: 

1774 self.outputs.remove("templateCoadd") 

1775 config.validate() 

1776 

1777 

1778class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1779 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1780 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1781 target=AssembleCoaddTask, 

1782 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1783 " naive/first-iteration model of the static sky.", 

1784 ) 

1785 detect = pexConfig.ConfigurableField( 

1786 target=SourceDetectionTask, 

1787 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1788 ) 

1789 detectTemplate = pexConfig.ConfigurableField( 

1790 target=SourceDetectionTask, 

1791 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1792 ) 

1793 maskStreaks = pexConfig.ConfigurableField( 

1794 target=MaskStreaksTask, 

1795 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1796 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1797 "streakMaskName" 

1798 ) 

1799 streakMaskName = pexConfig.Field( 

1800 dtype=str, 

1801 default="STREAK", 

1802 doc="Name of mask bit used for streaks" 

1803 ) 

1804 maxNumEpochs = pexConfig.Field( 

1805 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1806 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1807 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1808 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1809 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1810 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1811 "than transient and not masked.", 

1812 dtype=int, 

1813 default=2 

1814 ) 

1815 maxFractionEpochsLow = pexConfig.RangeField( 

1816 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1817 "Effective maxNumEpochs = " 

1818 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1819 dtype=float, 

1820 default=0.4, 

1821 min=0., max=1., 

1822 ) 

1823 maxFractionEpochsHigh = pexConfig.RangeField( 

1824 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1825 "Effective maxNumEpochs = " 

1826 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1827 dtype=float, 

1828 default=0.03, 

1829 min=0., max=1., 

1830 ) 

1831 spatialThreshold = pexConfig.RangeField( 

1832 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1833 "temporal criteria. If 0, clip all. If 1, clip none.", 

1834 dtype=float, 

1835 default=0.5, 

1836 min=0., max=1., 

1837 inclusiveMin=True, inclusiveMax=True 

1838 ) 

1839 doScaleWarpVariance = pexConfig.Field( 

1840 doc="Rescale Warp variance plane using empirical noise?", 

1841 dtype=bool, 

1842 default=True, 

1843 ) 

1844 scaleWarpVariance = pexConfig.ConfigurableField( 

1845 target=ScaleVarianceTask, 

1846 doc="Rescale variance on warps", 

1847 ) 

1848 doPreserveContainedBySource = pexConfig.Field( 

1849 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1850 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1851 dtype=bool, 

1852 default=True, 

1853 ) 

1854 doPrefilterArtifacts = pexConfig.Field( 

1855 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1856 "because they will be excluded anyway. This prevents them from contributing " 

1857 "to the outlier epoch count image and potentially being labeled as persistant." 

1858 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1859 dtype=bool, 

1860 default=True 

1861 ) 

1862 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1863 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1864 dtype=str, 

1865 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1866 ) 

1867 prefilterArtifactsRatio = pexConfig.Field( 

1868 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1869 dtype=float, 

1870 default=0.05 

1871 ) 

1872 doFilterMorphological = pexConfig.Field( 

1873 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1874 "be streaks.", 

1875 dtype=bool, 

1876 default=False 

1877 ) 

1878 

1879 def setDefaults(self): 

1880 AssembleCoaddConfig.setDefaults(self) 

1881 self.statistic = 'MEAN' 

1882 self.doUsePsfMatchedPolygons = True 

1883 

1884 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1885 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1886 if "EDGE" in self.badMaskPlanes: 

1887 self.badMaskPlanes.remove('EDGE') 

1888 self.removeMaskPlanes.append('EDGE') 

1889 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1890 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1891 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1892 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1893 self.assembleStaticSkyModel.sigmaClip = 2.5 

1894 self.assembleStaticSkyModel.clipIter = 3 

1895 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1896 self.assembleStaticSkyModel.doWrite = False 

1897 self.detect.doTempLocalBackground = False 

1898 self.detect.reEstimateBackground = False 

1899 self.detect.returnOriginalFootprints = False 

1900 self.detect.thresholdPolarity = "both" 

1901 self.detect.thresholdValue = 5 

1902 self.detect.minPixels = 4 

1903 self.detect.isotropicGrow = True 

1904 self.detect.thresholdType = "pixel_stdev" 

1905 self.detect.nSigmaToGrow = 0.4 

1906 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1907 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1908 self.detectTemplate.nSigmaToGrow = 2.4 

1909 self.detectTemplate.doTempLocalBackground = False 

1910 self.detectTemplate.reEstimateBackground = False 

1911 self.detectTemplate.returnOriginalFootprints = False 

1912 

1913 def validate(self): 

1914 super().validate() 

1915 if self.assembleStaticSkyModel.doNImage: 

1916 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1917 "Please set assembleStaticSkyModel.doNImage=False") 

1918 

1919 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1920 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1921 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1922 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1923 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1924 

1925 

1926class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1927 """Assemble a compareWarp coadded image from a set of warps 

1928 by masking artifacts detected by comparing PSF-matched warps. 

1929 

1930 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1931 we clip outliers). The problem with doing this is that when computing the 

1932 coadd PSF at a given location, individual visit PSFs from visits with 

1933 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1934 In this task, we correct for this behavior by creating a new badMaskPlane 

1935 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1936 an artifact. We populate this plane on the input warps by comparing 

1937 PSF-matched warps with a PSF-matched median coadd which serves as a 

1938 model of the static sky. Any group of pixels that deviates from the 

1939 PSF-matched template coadd by more than config.detect.threshold sigma, 

1940 is an artifact candidate. The candidates are then filtered to remove 

1941 variable sources and sources that are difficult to subtract such as 

1942 bright stars. This filter is configured using the config parameters 

1943 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1944 the maximum fraction of epochs that the deviation can appear in and still 

1945 be considered an artifact. The spatialThreshold is the maximum fraction of 

1946 pixels in the footprint of the deviation that appear in other epochs 

1947 (where other epochs is defined by the temporalThreshold). If the deviant 

1948 region meets this criteria of having a significant percentage of pixels 

1949 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1950 set in the mask. These regions will not contribute to the final coadd. 

1951 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1952 of clipped regions. Note that the algorithm implemented by this task is 

1953 preliminary and works correctly for HSC data. Parameter modifications and 

1954 or considerable redesigning of the algorithm is likley required for other 

1955 surveys. 

1956 

1957 ``CompareWarpAssembleCoaddTask`` sub-classes 

1958 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1959 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1960 

1961 Notes 

1962 ----- 

1963 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1964 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

1965 ``baseDebug`` for more about ``debug.py`` files. 

1966 

1967 This task supports the following debug variables: 

1968 

1969 - ``saveCountIm`` 

1970 If True then save the Epoch Count Image as a fits file in the `figPath` 

1971 - ``figPath`` 

1972 Path to save the debug fits images and figures 

1973 

1974 For example, put something like: 

1975 

1976 .. code-block:: python 

1977 

1978 import lsstDebug 

1979 def DebugInfo(name): 

1980 di = lsstDebug.getInfo(name) 

1981 if name == "lsst.pipe.tasks.assembleCoadd": 

1982 di.saveCountIm = True 

1983 di.figPath = "/desired/path/to/debugging/output/images" 

1984 return di 

1985 lsstDebug.Info = DebugInfo 

1986 

1987 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

1988 ``--debug`` flag. Some subtasks may have their own debug variables; 

1989 see individual Task documentation. 

1990 

1991 Examples 

1992 -------- 

1993 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

1994 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

1995 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

1996 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1997 and filter to be coadded (specified using 

1998 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1999 along with a list of coaddTempExps to attempt to coadd (specified using 

2000 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2001 Only the warps that cover the specified tract and patch will be coadded. 

2002 A list of the available optional arguments can be obtained by calling 

2003 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2004 

2005 .. code-block:: none 

2006 

2007 assembleCoadd.py --help 

2008 

2009 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2010 context of multi-band processing, we will generate the HSC-I & -R band 

2011 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2012 To begin, assuming that the lsst stack has been already set up, we must 

2013 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2014 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2015 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2016 directory. To begin assembling the coadds, we must first 

2017 

2018 - processCcd 

2019 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2020 - makeSkyMap 

2021 create a skymap that covers the area of the sky present in the raw exposures 

2022 - makeCoaddTempExp 

2023 warp the individual calibrated exposures to the tangent plane of the coadd 

2024 

2025 We can perform all of these steps by running 

2026 

2027 .. code-block:: none 

2028 

2029 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2030 

2031 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2032 warped data, we call ``assembleCoadd.py`` as follows: 

2033 

2034 .. code-block:: none 

2035 

2036 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2037 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2038 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2039 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2040 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2041 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2042 --selectId visit=903988 ccd=24 

2043 

2044 This will process the HSC-I band data. The results are written in 

2045 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2046 """ 

2047 ConfigClass = CompareWarpAssembleCoaddConfig 

2048 _DefaultName = "compareWarpAssembleCoadd" 

2049 

2050 def __init__(self, *args, **kwargs): 

2051 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2052 self.makeSubtask("assembleStaticSkyModel") 

2053 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2054 self.makeSubtask("detect", schema=detectionSchema) 

2055 if self.config.doPreserveContainedBySource: 

2056 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2057 if self.config.doScaleWarpVariance: 

2058 self.makeSubtask("scaleWarpVariance") 

2059 if self.config.doFilterMorphological: 

2060 self.makeSubtask("maskStreaks") 

2061 

2062 @utils.inheritDoc(AssembleCoaddTask) 

2063 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2064 """ 

2065 Generate a templateCoadd to use as a naive model of static sky to 

2066 subtract from PSF-Matched warps. 

2067 

2068 Returns 

2069 ------- 

2070 result : `lsst.pipe.base.Struct` 

2071 Result struct with components: 

2072 

2073 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2074 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2075 """ 

2076 # Ensure that psfMatchedWarps are used as input warps for template generation 

2077 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2078 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2079 

2080 # Because subtasks don't have connections we have to make one. 

2081 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2082 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2083 if self.config.assembleStaticSkyModel.doWrite: 

2084 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2085 # Remove template coadd from both subtask's and main tasks outputs, 

2086 # because it is handled by the subtask as `coaddExposure` 

2087 del outputRefs.templateCoadd 

2088 del staticSkyModelOutputRefs.templateCoadd 

2089 

2090 # A PSF-Matched nImage does not exist as a dataset type 

2091 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2092 del staticSkyModelOutputRefs.nImage 

2093 

2094 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2095 staticSkyModelOutputRefs) 

2096 if templateCoadd is None: 

2097 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2098 

2099 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2100 nImage=templateCoadd.nImage, 

2101 warpRefList=templateCoadd.warpRefList, 

2102 imageScalerList=templateCoadd.imageScalerList, 

2103 weightList=templateCoadd.weightList) 

2104 

2105 @utils.inheritDoc(AssembleCoaddTask) 

2106 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2107 """ 

2108 Generate a templateCoadd to use as a naive model of static sky to 

2109 subtract from PSF-Matched warps. 

2110 

2111 Returns 

2112 ------- 

2113 result : `lsst.pipe.base.Struct` 

2114 Result struct with components: 

2115 

2116 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2117 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2118 """ 

2119 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2120 if templateCoadd is None: 

2121 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2122 

2123 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2124 nImage=templateCoadd.nImage, 

2125 warpRefList=templateCoadd.warpRefList, 

2126 imageScalerList=templateCoadd.imageScalerList, 

2127 weightList=templateCoadd.weightList) 

2128 

2129 def _noTemplateMessage(self, warpType): 

2130 warpName = (warpType[0].upper() + warpType[1:]) 

2131 message = """No %(warpName)s warps were found to build the template coadd which is 

2132 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2133 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2134 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2135 

2136 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2137 another algorithm like: 

2138 

2139 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2140 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2141 """ % {"warpName": warpName} 

2142 return message 

2143 

2144 @utils.inheritDoc(AssembleCoaddTask) 

2145 @pipeBase.timeMethod 

2146 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2147 supplementaryData, *args, **kwargs): 

2148 """Assemble the coadd. 

2149 

2150 Find artifacts and apply them to the warps' masks creating a list of 

2151 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2152 plane. Then pass these alternative masks to the base class's `run` 

2153 method. 

2154 

2155 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2156 that must contain a ``templateCoadd`` that serves as the 

2157 model of the static sky. 

2158 """ 

2159 

2160 # Check and match the order of the supplementaryData 

2161 # (PSF-matched) inputs to the order of the direct inputs, 

2162 # so that the artifact mask is applied to the right warp 

2163 dataIds = [ref.dataId for ref in tempExpRefList] 

2164 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2165 

2166 if dataIds != psfMatchedDataIds: 

2167 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2168 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2169 psfMatchedDataIds, dataIds) 

2170 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2171 psfMatchedDataIds, dataIds) 

2172 

2173 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2174 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2175 supplementaryData.warpRefList, 

2176 supplementaryData.imageScalerList) 

2177 

2178 badMaskPlanes = self.config.badMaskPlanes[:] 

2179 badMaskPlanes.append("CLIPPED") 

2180 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2181 

2182 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2183 spanSetMaskList, mask=badPixelMask) 

2184 

2185 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2186 # Psf-Matching moves the real edge inwards 

2187 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2188 return result 

2189 

2190 def applyAltEdgeMask(self, mask, altMaskList): 

2191 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2192 

2193 Parameters 

2194 ---------- 

2195 mask : `lsst.afw.image.Mask` 

2196 Original mask. 

2197 altMaskList : `list` 

2198 List of Dicts containing ``spanSet`` lists. 

2199 Each element contains the new mask plane name (e.g. "CLIPPED 

2200 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2201 the mask. 

2202 """ 

2203 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2204 for visitMask in altMaskList: 

2205 if "EDGE" in visitMask: 

2206 for spanSet in visitMask['EDGE']: 

2207 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2208 

2209 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2210 """Find artifacts. 

2211 

2212 Loop through warps twice. The first loop builds a map with the count 

2213 of how many epochs each pixel deviates from the templateCoadd by more 

2214 than ``config.chiThreshold`` sigma. The second loop takes each 

2215 difference image and filters the artifacts detected in each using 

2216 count map to filter out variable sources and sources that are 

2217 difficult to subtract cleanly. 

2218 

2219 Parameters 

2220 ---------- 

2221 templateCoadd : `lsst.afw.image.Exposure` 

2222 Exposure to serve as model of static sky. 

2223 tempExpRefList : `list` 

2224 List of data references to warps. 

2225 imageScalerList : `list` 

2226 List of image scalers. 

2227 

2228 Returns 

2229 ------- 

2230 altMasks : `list` 

2231 List of dicts containing information about CLIPPED 

2232 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2233 """ 

2234 

2235 self.log.debug("Generating Count Image, and mask lists.") 

2236 coaddBBox = templateCoadd.getBBox() 

2237 slateIm = afwImage.ImageU(coaddBBox) 

2238 epochCountImage = afwImage.ImageU(coaddBBox) 

2239 nImage = afwImage.ImageU(coaddBBox) 

2240 spanSetArtifactList = [] 

2241 spanSetNoDataMaskList = [] 

2242 spanSetEdgeList = [] 

2243 spanSetBadMorphoList = [] 

2244 badPixelMask = self.getBadPixelMask() 

2245 

2246 # mask of the warp diffs should = that of only the warp 

2247 templateCoadd.mask.clearAllMaskPlanes() 

2248 

2249 if self.config.doPreserveContainedBySource: 

2250 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2251 else: 

2252 templateFootprints = None 

2253 

2254 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2255 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2256 if warpDiffExp is not None: 

2257 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2258 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2259 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2260 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2261 fpSet.positive.merge(fpSet.negative) 

2262 footprints = fpSet.positive 

2263 slateIm.set(0) 

2264 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2265 

2266 # Remove artifacts due to defects before they contribute to the epochCountImage 

2267 if self.config.doPrefilterArtifacts: 

2268 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2269 

2270 # Clear mask before adding prefiltered spanSets 

2271 self.detect.clearMask(warpDiffExp.mask) 

2272 for spans in spanSetList: 

2273 spans.setImage(slateIm, 1, doClip=True) 

2274 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2275 epochCountImage += slateIm 

2276 

2277 if self.config.doFilterMorphological: 

2278 maskName = self.config.streakMaskName 

2279 _ = self.maskStreaks.run(warpDiffExp) 

2280 streakMask = warpDiffExp.mask 

2281 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2282 streakMask.getPlaneBitMask(maskName)).split() 

2283 

2284 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2285 # undergo a second convolution. Pixels with data in the direct warp 

2286 # but not in the PSF-matched warp will not have their artifacts detected. 

2287 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2288 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2289 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2290 nansMask.setXY0(warpDiffExp.getXY0()) 

2291 edgeMask = warpDiffExp.mask 

2292 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2293 edgeMask.getPlaneBitMask("EDGE")).split() 

2294 else: 

2295 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2296 # In this case, mask the whole epoch 

2297 nansMask = afwImage.MaskX(coaddBBox, 1) 

2298 spanSetList = [] 

2299 spanSetEdgeMask = [] 

2300 spanSetStreak = [] 

2301 

2302 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2303 

2304 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2305 spanSetArtifactList.append(spanSetList) 

2306 spanSetEdgeList.append(spanSetEdgeMask) 

2307 if self.config.doFilterMorphological: 

2308 spanSetBadMorphoList.append(spanSetStreak) 

2309 

2310 if lsstDebug.Info(__name__).saveCountIm: 

2311 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2312 epochCountImage.writeFits(path) 

2313 

2314 for i, spanSetList in enumerate(spanSetArtifactList): 

2315 if spanSetList: 

2316 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2317 templateFootprints) 

2318 spanSetArtifactList[i] = filteredSpanSetList 

2319 if self.config.doFilterMorphological: 

2320 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2321 

2322 altMasks = [] 

2323 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2324 altMasks.append({'CLIPPED': artifacts, 

2325 'NO_DATA': noData, 

2326 'EDGE': edge}) 

2327 return altMasks 

2328 

2329 def prefilterArtifacts(self, spanSetList, exp): 

2330 """Remove artifact candidates covered by bad mask plane. 

2331 

2332 Any future editing of the candidate list that does not depend on 

2333 temporal information should go in this method. 

2334 

2335 Parameters 

2336 ---------- 

2337 spanSetList : `list` 

2338 List of SpanSets representing artifact candidates. 

2339 exp : `lsst.afw.image.Exposure` 

2340 Exposure containing mask planes used to prefilter. 

2341 

2342 Returns 

2343 ------- 

2344 returnSpanSetList : `list` 

2345 List of SpanSets with artifacts. 

2346 """ 

2347 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2348 goodArr = (exp.mask.array & badPixelMask) == 0 

2349 returnSpanSetList = [] 

2350 bbox = exp.getBBox() 

2351 x0, y0 = exp.getXY0() 

2352 for i, span in enumerate(spanSetList): 

2353 y, x = span.clippedTo(bbox).indices() 

2354 yIndexLocal = numpy.array(y) - y0 

2355 xIndexLocal = numpy.array(x) - x0 

2356 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2357 if goodRatio > self.config.prefilterArtifactsRatio: 

2358 returnSpanSetList.append(span) 

2359 return returnSpanSetList 

2360 

2361 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2362 """Filter artifact candidates. 

2363 

2364 Parameters 

2365 ---------- 

2366 spanSetList : `list` 

2367 List of SpanSets representing artifact candidates. 

2368 epochCountImage : `lsst.afw.image.Image` 

2369 Image of accumulated number of warpDiff detections. 

2370 nImage : `lsst.afw.image.Image` 

2371 Image of the accumulated number of total epochs contributing. 

2372 

2373 Returns 

2374 ------- 

2375 maskSpanSetList : `list` 

2376 List of SpanSets with artifacts. 

2377 """ 

2378 

2379 maskSpanSetList = [] 

2380 x0, y0 = epochCountImage.getXY0() 

2381 for i, span in enumerate(spanSetList): 

2382 y, x = span.indices() 

2383 yIdxLocal = [y1 - y0 for y1 in y] 

2384 xIdxLocal = [x1 - x0 for x1 in x] 

2385 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2386 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2387 

2388 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2389 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2390 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2391 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2392 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2393 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2394 & (outlierN <= effectiveMaxNumEpochs)) 

2395 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2396 if percentBelowThreshold > self.config.spatialThreshold: 

2397 maskSpanSetList.append(span) 

2398 

2399 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

2400 # If a candidate is contained by a footprint on the template coadd, do not clip 

2401 filteredMaskSpanSetList = [] 

2402 for span in maskSpanSetList: 

2403 doKeep = True 

2404 for footprint in footprintsToExclude.positive.getFootprints(): 

2405 if footprint.spans.contains(span): 

2406 doKeep = False 

2407 break 

2408 if doKeep: 

2409 filteredMaskSpanSetList.append(span) 

2410 maskSpanSetList = filteredMaskSpanSetList 

2411 

2412 return maskSpanSetList 

2413 

2414 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2415 """Fetch a warp from the butler and return a warpDiff. 

2416 

2417 Parameters 

2418 ---------- 

2419 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2420 Butler dataRef for the warp. 

2421 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2422 An image scaler object. 

2423 templateCoadd : `lsst.afw.image.Exposure` 

2424 Exposure to be substracted from the scaled warp. 

2425 

2426 Returns 

2427 ------- 

2428 warp : `lsst.afw.image.Exposure` 

2429 Exposure of the image difference between the warp and template. 

2430 """ 

2431 

2432 # If the PSF-Matched warp did not exist for this direct warp 

2433 # None is holding its place to maintain order in Gen 3 

2434 if warpRef is None: 

2435 return None 

2436 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2437 warpName = self.getTempExpDatasetName('psfMatched') 

2438 if not isinstance(warpRef, DeferredDatasetHandle): 

2439 if not warpRef.datasetExists(warpName): 

2440 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2441 return None 

2442 warp = warpRef.get(datasetType=warpName, immediate=True) 

2443 # direct image scaler OK for PSF-matched Warp 

2444 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2445 mi = warp.getMaskedImage() 

2446 if self.config.doScaleWarpVariance: 

2447 try: 

2448 self.scaleWarpVariance.run(mi) 

2449 except Exception as exc: 

2450 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2451 mi -= templateCoadd.getMaskedImage() 

2452 return warp 

2453 

2454 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2455 """Return a path to which to write debugging output. 

2456 

2457 Creates a hyphen-delimited string of dataId values for simple filenames. 

2458 

2459 Parameters 

2460 ---------- 

2461 prefix : `str` 

2462 Prefix for filename. 

2463 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2464 Butler dataRef to make the path from. 

2465 coaddLevel : `bool`, optional. 

2466 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2467 'filter', but no 'visit'). 

2468 

2469 Returns 

2470 ------- 

2471 result : `str` 

2472 Path for debugging output. 

2473 """ 

2474 if coaddLevel: 

2475 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2476 else: 

2477 keys = warpRef.dataId.keys() 

2478 keyList = sorted(keys, reverse=True) 

2479 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2480 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2481 return os.path.join(directory, filename)