Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix 

41from .interpImage import InterpImageTask 

42from .scaleZeroPoint import ScaleZeroPointTask 

43from .coaddHelpers import groupPatchExposures, getGroupDataRef 

44from .scaleVariance import ScaleVarianceTask 

45from lsst.meas.algorithms import SourceDetectionTask 

46from lsst.daf.butler import DeferredDatasetHandle 

47 

48__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

49 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

50 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

51 

52 

53class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

54 dimensions=("tract", "patch", "abstract_filter", "skymap"), 

55 defaultTemplates={"inputCoaddName": "deep", 

56 "outputCoaddName": "deep", 

57 "warpType": "direct", 

58 "warpTypeSuffix": "", 

59 "fakesType": ""}): 

60 inputWarps = pipeBase.connectionTypes.Input( 

61 doc=("Input list of warps to be assemebled i.e. stacked." 

62 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

63 name="{inputCoaddName}Coadd_{warpType}Warp", 

64 storageClass="ExposureF", 

65 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

66 deferLoad=True, 

67 multiple=True 

68 ) 

69 skyMap = pipeBase.connectionTypes.Input( 

70 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

71 name="{inputCoaddName}Coadd_skyMap", 

72 storageClass="SkyMap", 

73 dimensions=("skymap", ), 

74 ) 

75 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

76 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

77 " BRIGHT_OBJECT."), 

78 name="brightObjectMask", 

79 storageClass="ObjectMaskCatalog", 

80 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

81 ) 

82 coaddExposure = pipeBase.connectionTypes.Output( 

83 doc="Output coadded exposure, produced by stacking input warps", 

84 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

85 storageClass="ExposureF", 

86 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

87 ) 

88 nImage = pipeBase.connectionTypes.Output( 

89 doc="Output image of number of input images per pixel", 

90 name="{outputCoaddName}Coadd_nImage", 

91 storageClass="ImageU", 

92 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

93 ) 

94 

95 def __init__(self, *, config=None): 

96 # Override the connection's name template with config to replicate Gen2 behavior 

97 config.connections.warpType = config.warpType 

98 config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType) 

99 

100 if config.hasFakes: 

101 config.connections.fakesType = "_fakes" 

102 

103 super().__init__(config=config) 

104 

105 if not config.doMaskBrightObjects: 

106 self.prerequisiteInputs.remove("brightObjectMask") 

107 

108 if not config.doNImage: 

109 self.outputs.remove("nImage") 

110 

111 

112class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

113 pipelineConnections=AssembleCoaddConnections): 

114 """Configuration parameters for the `AssembleCoaddTask`. 

115 

116 Notes 

117 ----- 

118 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

119 only set the bitplane config.brightObjectMaskName. To make this useful you 

120 *must* also configure the flags.pixel algorithm, for example by adding 

121 

122 .. code-block:: none 

123 

124 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

125 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

126 

127 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

128 """ 

129 warpType = pexConfig.Field( 

130 doc="Warp name: one of 'direct' or 'psfMatched'", 

131 dtype=str, 

132 default="direct", 

133 ) 

134 subregionSize = pexConfig.ListField( 

135 dtype=int, 

136 doc="Width, height of stack subregion size; " 

137 "make small enough that a full stack of images will fit into memory at once.", 

138 length=2, 

139 default=(2000, 2000), 

140 ) 

141 statistic = pexConfig.Field( 

142 dtype=str, 

143 doc="Main stacking statistic for aggregating over the epochs.", 

144 default="MEANCLIP", 

145 ) 

146 doSigmaClip = pexConfig.Field( 

147 dtype=bool, 

148 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

149 default=False, 

150 ) 

151 sigmaClip = pexConfig.Field( 

152 dtype=float, 

153 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

154 default=3.0, 

155 ) 

156 clipIter = pexConfig.Field( 

157 dtype=int, 

158 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

159 default=2, 

160 ) 

161 calcErrorFromInputVariance = pexConfig.Field( 

162 dtype=bool, 

163 doc="Calculate coadd variance from input variance by stacking statistic." 

164 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

165 default=True, 

166 ) 

167 scaleZeroPoint = pexConfig.ConfigurableField( 

168 target=ScaleZeroPointTask, 

169 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

170 ) 

171 doInterp = pexConfig.Field( 

172 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

173 dtype=bool, 

174 default=True, 

175 ) 

176 interpImage = pexConfig.ConfigurableField( 

177 target=InterpImageTask, 

178 doc="Task to interpolate (and extrapolate) over NaN pixels", 

179 ) 

180 doWrite = pexConfig.Field( 

181 doc="Persist coadd?", 

182 dtype=bool, 

183 default=True, 

184 ) 

185 doNImage = pexConfig.Field( 

186 doc="Create image of number of contributing exposures for each pixel", 

187 dtype=bool, 

188 default=False, 

189 ) 

190 doUsePsfMatchedPolygons = pexConfig.Field( 

191 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

192 dtype=bool, 

193 default=False, 

194 ) 

195 maskPropagationThresholds = pexConfig.DictField( 

196 keytype=str, 

197 itemtype=float, 

198 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

199 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

200 "would have contributed exceeds this value."), 

201 default={"SAT": 0.1}, 

202 ) 

203 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

204 doc="Mask planes to remove before coadding") 

205 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

206 doc="Set mask and flag bits for bright objects?") 

207 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

208 doc="Name of mask bit used for bright objects") 

209 coaddPsf = pexConfig.ConfigField( 

210 doc="Configuration for CoaddPsf", 

211 dtype=measAlg.CoaddPsfConfig, 

212 ) 

213 doAttachTransmissionCurve = pexConfig.Field( 

214 dtype=bool, default=False, optional=False, 

215 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

216 "(requires all input Exposures to have TransmissionCurves).") 

217 ) 

218 hasFakes = pexConfig.Field( 

219 dtype=bool, 

220 default=False, 

221 doc="Should be set to True if fake sources have been inserted into the input data." 

222 ) 

223 

224 def setDefaults(self): 

225 super().setDefaults() 

226 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

227 

228 def validate(self): 

229 super().validate() 

230 if self.doPsfMatch: 

231 # Backwards compatibility. 

232 # Configs do not have loggers 

233 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

234 self.warpType = 'psfMatched' 

235 if self.doSigmaClip and self.statistic != "MEANCLIP": 

236 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

237 self.statistic = "MEANCLIP" 

238 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

239 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

240 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

241 

242 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

243 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

244 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

245 if str(k) not in unstackableStats] 

246 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

247 % (self.statistic, stackableStats)) 

248 

249 

250class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

251 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

252 

253 We want to assemble a coadded image from a set of Warps (also called 

254 coadded temporary exposures or ``coaddTempExps``). 

255 Each input Warp covers a patch on the sky and corresponds to a single 

256 run/visit/exposure of the covered patch. We provide the task with a list 

257 of Warps (``selectDataList``) from which it selects Warps that cover the 

258 specified patch (pointed at by ``dataRef``). 

259 Each Warp that goes into a coadd will typically have an independent 

260 photometric zero-point. Therefore, we must scale each Warp to set it to 

261 a common photometric zeropoint. WarpType may be one of 'direct' or 

262 'psfMatched', and the boolean configs `config.makeDirect` and 

263 `config.makePsfMatched` set which of the warp types will be coadded. 

264 The coadd is computed as a mean with optional outlier rejection. 

265 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

266 Finally, Warps can have bad 'NaN' pixels which received no input from the 

267 source calExps. We interpolate over these bad (NaN) pixels. 

268 

269 `AssembleCoaddTask` uses several sub-tasks. These are 

270 

271 - `ScaleZeroPointTask` 

272 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

273 - `InterpImageTask` 

274 - interpolate across bad pixels (NaN) in the final coadd 

275 

276 You can retarget these subtasks if you wish. 

277 

278 Notes 

279 ----- 

280 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

281 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

282 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

283 no debug variables of its own. Some of the subtasks may support debug 

284 variables. See the documentation for the subtasks for further information. 

285 

286 Examples 

287 -------- 

288 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

289 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

290 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

291 inputs: a data reference to the tract patch and filter to be coadded, and 

292 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

293 ``--selectId``, respectively: 

294 

295 .. code-block:: none 

296 

297 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

298 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

299 

300 Only the Warps that cover the specified tract and patch will be coadded. 

301 A list of the available optional arguments can be obtained by calling 

302 ``assembleCoadd.py`` with the ``--help`` command line argument: 

303 

304 .. code-block:: none 

305 

306 assembleCoadd.py --help 

307 

308 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

309 multi-band processing, we will generate the HSC-I & -R band coadds from 

310 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

311 assuming that the lsst stack has been already set up, we must set up the 

312 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

313 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

314 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

315 coadds, we must first 

316 

317 - processCcd 

318 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

319 - makeSkyMap 

320 - create a skymap that covers the area of the sky present in the raw exposures 

321 - makeCoaddTempExp 

322 - warp the individual calibrated exposures to the tangent plane of the coadd 

323 

324 We can perform all of these steps by running 

325 

326 .. code-block:: none 

327 

328 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

329 

330 This will produce warped exposures for each visit. To coadd the warped 

331 data, we call assembleCoadd.py as follows: 

332 

333 .. code-block:: none 

334 

335 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

336 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

337 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

338 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

339 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

340 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

341 --selectId visit=903988 ccd=24 

342 

343 that will process the HSC-I band data. The results are written in 

344 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

345 

346 You may also choose to run: 

347 

348 .. code-block:: none 

349 

350 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

351 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

352 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

353 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

354 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

355 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

356 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

357 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

358 

359 to generate the coadd for the HSC-R band if you are interested in 

360 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

361 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

362 rather than `AssembleCoaddTask` to make the coadd. 

363 """ 

364 ConfigClass = AssembleCoaddConfig 

365 _DefaultName = "assembleCoadd" 

366 

367 def __init__(self, *args, **kwargs): 

368 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

369 if args: 369 ↛ 370line 369 didn't jump to line 370, because the condition on line 369 was never true

370 argNames = ["config", "name", "parentTask", "log"] 

371 kwargs.update({k: v for k, v in zip(argNames, args)}) 

372 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

373 "PipelineTask will not take positional args" % argNames, FutureWarning) 

374 

375 super().__init__(**kwargs) 

376 self.makeSubtask("interpImage") 

377 self.makeSubtask("scaleZeroPoint") 

378 

379 if self.config.doMaskBrightObjects: 379 ↛ 380line 379 didn't jump to line 380, because the condition on line 379 was never true

380 mask = afwImage.Mask() 

381 try: 

382 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

383 except pexExceptions.LsstCppException: 

384 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

385 mask.getMaskPlaneDict().keys()) 

386 del mask 

387 

388 self.warpType = self.config.warpType 

389 

390 @utils.inheritDoc(pipeBase.PipelineTask) 

391 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

392 # Docstring to be formatted with info from PipelineTask.runQuantum 

393 """ 

394 Notes 

395 ----- 

396 Assemble a coadd from a set of Warps. 

397 

398 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

399 Analogous to `runDataRef`, it prepares all the data products to be 

400 passed to `run`, and processes the results before returning a struct 

401 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

402 Therefore, its inputs are accessed subregion by subregion 

403 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

404 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

405 correspond to an update in `runDataRef` while both entry points 

406 are used. 

407 """ 

408 inputData = butlerQC.get(inputRefs) 

409 

410 # Construct skyInfo expected by run 

411 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

412 skyMap = inputData["skyMap"] 

413 outputDataId = butlerQC.quantum.dataId 

414 

415 inputData['skyInfo'] = makeSkyInfo(skyMap, 

416 tractId=outputDataId['tract'], 

417 patchId=outputDataId['patch']) 

418 

419 # Construct list of input Deferred Datasets 

420 # These quack a bit like like Gen2 DataRefs 

421 warpRefList = inputData['inputWarps'] 

422 # Perform same middle steps as `runDataRef` does 

423 inputs = self.prepareInputs(warpRefList) 

424 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

425 self.getTempExpDatasetName(self.warpType)) 

426 if len(inputs.tempExpRefList) == 0: 

427 self.log.warn("No coadd temporary exposures found") 

428 return 

429 

430 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

431 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

432 inputs.weightList, supplementaryData=supplementaryData) 

433 

434 inputData.setdefault('brightObjectMask', None) 

435 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

436 

437 if self.config.doWrite: 

438 butlerQC.put(retStruct, outputRefs) 

439 return retStruct 

440 

441 @pipeBase.timeMethod 

442 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

443 """Assemble a coadd from a set of Warps. 

444 

445 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

446 Compute weights to be applied to each Warp and 

447 find scalings to match the photometric zeropoint to a reference Warp. 

448 Assemble the Warps using `run`. Interpolate over NaNs and 

449 optionally write the coadd to disk. Return the coadded exposure. 

450 

451 Parameters 

452 ---------- 

453 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

454 Data reference defining the patch for coaddition and the 

455 reference Warp (if ``config.autoReference=False``). 

456 Used to access the following data products: 

457 - ``self.config.coaddName + "Coadd_skyMap"`` 

458 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

459 - ``self.config.coaddName + "Coadd"`` 

460 selectDataList : `list` 

461 List of data references to Calexps. Data to be coadded will be 

462 selected from this list based on overlap with the patch defined 

463 by dataRef, grouped by visit, and converted to a list of data 

464 references to warps. 

465 warpRefList : `list` 

466 List of data references to Warps to be coadded. 

467 Note: `warpRefList` is just the new name for `tempExpRefList`. 

468 

469 Returns 

470 ------- 

471 retStruct : `lsst.pipe.base.Struct` 

472 Result struct with components: 

473 

474 - ``coaddExposure``: coadded exposure (``Exposure``). 

475 - ``nImage``: exposure count image (``Image``). 

476 """ 

477 if selectDataList and warpRefList: 477 ↛ 478line 477 didn't jump to line 478, because the condition on line 477 was never true

478 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

479 "and which to use is ambiguous. Please pass only one.") 

480 

481 skyInfo = self.getSkyInfo(dataRef) 

482 if warpRefList is None: 

483 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

484 if len(calExpRefList) == 0: 484 ↛ 485line 484 didn't jump to line 485, because the condition on line 484 was never true

485 self.log.warn("No exposures to coadd") 

486 return 

487 self.log.info("Coadding %d exposures", len(calExpRefList)) 

488 

489 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

490 

491 inputData = self.prepareInputs(warpRefList) 

492 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

493 self.getTempExpDatasetName(self.warpType)) 

494 if len(inputData.tempExpRefList) == 0: 494 ↛ 495line 494 didn't jump to line 495, because the condition on line 494 was never true

495 self.log.warn("No coadd temporary exposures found") 

496 return 

497 

498 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

499 

500 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

501 inputData.weightList, supplementaryData=supplementaryData) 

502 

503 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

504 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

505 

506 if self.config.doWrite: 

507 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 507 ↛ 508line 507 didn't jump to line 508, because the condition on line 507 was never true

508 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

509 else: 

510 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

511 self.log.info("Persisting %s" % coaddDatasetName) 

512 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

513 if self.config.doNImage and retStruct.nImage is not None: 513 ↛ 514line 513 didn't jump to line 514, because the condition on line 513 was never true

514 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

515 

516 return retStruct 

517 

518 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

519 """Interpolate over missing data and mask bright stars. 

520 

521 Parameters 

522 ---------- 

523 coaddExposure : `lsst.afw.image.Exposure` 

524 The coadded exposure to process. 

525 dataRef : `lsst.daf.persistence.ButlerDataRef` 

526 Butler data reference for supplementary data. 

527 """ 

528 if self.config.doInterp: 528 ↛ 535line 528 didn't jump to line 535, because the condition on line 528 was never false

529 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

530 # The variance must be positive; work around for DM-3201. 

531 varArray = coaddExposure.variance.array 

532 with numpy.errstate(invalid="ignore"): 

533 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

534 

535 if self.config.doMaskBrightObjects: 535 ↛ 536line 535 didn't jump to line 536, because the condition on line 535 was never true

536 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

537 

538 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

539 """Make additional inputs to run() specific to subclasses (Gen2) 

540 

541 Duplicates interface of `runDataRef` method 

542 Available to be implemented by subclasses only if they need the 

543 coadd dataRef for performing preliminary processing before 

544 assembling the coadd. 

545 

546 Parameters 

547 ---------- 

548 dataRef : `lsst.daf.persistence.ButlerDataRef` 

549 Butler data reference for supplementary data. 

550 selectDataList : `list` (optional) 

551 Optional List of data references to Calexps. 

552 warpRefList : `list` (optional) 

553 Optional List of data references to Warps. 

554 """ 

555 return pipeBase.Struct() 

556 

557 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

558 """Make additional inputs to run() specific to subclasses (Gen3) 

559 

560 Duplicates interface of `runQuantum` method. 

561 Available to be implemented by subclasses only if they need the 

562 coadd dataRef for performing preliminary processing before 

563 assembling the coadd. 

564 

565 Parameters 

566 ---------- 

567 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

568 Gen3 Butler object for fetching additional data products before 

569 running the Task specialized for quantum being processed 

570 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

571 Attributes are the names of the connections describing input dataset types. 

572 Values are DatasetRefs that task consumes for corresponding dataset type. 

573 DataIds are guaranteed to match data objects in ``inputData``. 

574 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

575 Attributes are the names of the connections describing output dataset types. 

576 Values are DatasetRefs that task is to produce 

577 for corresponding dataset type. 

578 """ 

579 return pipeBase.Struct() 

580 

581 def getTempExpRefList(self, patchRef, calExpRefList): 

582 """Generate list data references corresponding to warped exposures 

583 that lie within the patch to be coadded. 

584 

585 Parameters 

586 ---------- 

587 patchRef : `dataRef` 

588 Data reference for patch. 

589 calExpRefList : `list` 

590 List of data references for input calexps. 

591 

592 Returns 

593 ------- 

594 tempExpRefList : `list` 

595 List of Warp/CoaddTempExp data references. 

596 """ 

597 butler = patchRef.getButler() 

598 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

599 self.getTempExpDatasetName(self.warpType)) 

600 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

601 g, groupData.keys) for 

602 g in groupData.groups.keys()] 

603 return tempExpRefList 

604 

605 def prepareInputs(self, refList): 

606 """Prepare the input warps for coaddition by measuring the weight for 

607 each warp and the scaling for the photometric zero point. 

608 

609 Each Warp has its own photometric zeropoint and background variance. 

610 Before coadding these Warps together, compute a scale factor to 

611 normalize the photometric zeropoint and compute the weight for each Warp. 

612 

613 Parameters 

614 ---------- 

615 refList : `list` 

616 List of data references to tempExp 

617 

618 Returns 

619 ------- 

620 result : `lsst.pipe.base.Struct` 

621 Result struct with components: 

622 

623 - ``tempExprefList``: `list` of data references to tempExp. 

624 - ``weightList``: `list` of weightings. 

625 - ``imageScalerList``: `list` of image scalers. 

626 """ 

627 statsCtrl = afwMath.StatisticsControl() 

628 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

629 statsCtrl.setNumIter(self.config.clipIter) 

630 statsCtrl.setAndMask(self.getBadPixelMask()) 

631 statsCtrl.setNanSafe(True) 

632 # compute tempExpRefList: a list of tempExpRef that actually exist 

633 # and weightList: a list of the weight of the associated coadd tempExp 

634 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

635 tempExpRefList = [] 

636 weightList = [] 

637 imageScalerList = [] 

638 tempExpName = self.getTempExpDatasetName(self.warpType) 

639 for tempExpRef in refList: 

640 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

641 # therefore have no datasetExists() method 

642 if not isinstance(tempExpRef, DeferredDatasetHandle): 

643 if not tempExpRef.datasetExists(tempExpName): 

644 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

645 continue 

646 

647 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

648 # Ignore any input warp that is empty of data 

649 if numpy.isnan(tempExp.image.array).all(): 649 ↛ 650line 649 didn't jump to line 650, because the condition on line 649 was never true

650 continue 

651 maskedImage = tempExp.getMaskedImage() 

652 imageScaler = self.scaleZeroPoint.computeImageScaler( 

653 exposure=tempExp, 

654 dataRef=tempExpRef, 

655 ) 

656 try: 

657 imageScaler.scaleMaskedImage(maskedImage) 

658 except Exception as e: 

659 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

660 continue 

661 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

662 afwMath.MEANCLIP, statsCtrl) 

663 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

664 weight = 1.0 / float(meanVar) 

665 if not numpy.isfinite(weight): 665 ↛ 666line 665 didn't jump to line 666, because the condition on line 665 was never true

666 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

667 continue 

668 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

669 

670 del maskedImage 

671 del tempExp 

672 

673 tempExpRefList.append(tempExpRef) 

674 weightList.append(weight) 

675 imageScalerList.append(imageScaler) 

676 

677 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

678 imageScalerList=imageScalerList) 

679 

680 def prepareStats(self, mask=None): 

681 """Prepare the statistics for coadding images. 

682 

683 Parameters 

684 ---------- 

685 mask : `int`, optional 

686 Bit mask value to exclude from coaddition. 

687 

688 Returns 

689 ------- 

690 stats : `lsst.pipe.base.Struct` 

691 Statistics structure with the following fields: 

692 

693 - ``statsCtrl``: Statistics control object for coadd 

694 (`lsst.afw.math.StatisticsControl`) 

695 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

696 """ 

697 if mask is None: 

698 mask = self.getBadPixelMask() 

699 statsCtrl = afwMath.StatisticsControl() 

700 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

701 statsCtrl.setNumIter(self.config.clipIter) 

702 statsCtrl.setAndMask(mask) 

703 statsCtrl.setNanSafe(True) 

704 statsCtrl.setWeighted(True) 

705 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

706 for plane, threshold in self.config.maskPropagationThresholds.items(): 

707 bit = afwImage.Mask.getMaskPlane(plane) 

708 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

709 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

710 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

711 

712 @pipeBase.timeMethod 

713 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

714 altMaskList=None, mask=None, supplementaryData=None): 

715 """Assemble a coadd from input warps 

716 

717 Assemble the coadd using the provided list of coaddTempExps. Since 

718 the full coadd covers a patch (a large area), the assembly is 

719 performed over small areas on the image at a time in order to 

720 conserve memory usage. Iterate over subregions within the outer 

721 bbox of the patch using `assembleSubregion` to stack the corresponding 

722 subregions from the coaddTempExps with the statistic specified. 

723 Set the edge bits the coadd mask based on the weight map. 

724 

725 Parameters 

726 ---------- 

727 skyInfo : `lsst.pipe.base.Struct` 

728 Struct with geometric information about the patch. 

729 tempExpRefList : `list` 

730 List of data references to Warps (previously called CoaddTempExps). 

731 imageScalerList : `list` 

732 List of image scalers. 

733 weightList : `list` 

734 List of weights 

735 altMaskList : `list`, optional 

736 List of alternate masks to use rather than those stored with 

737 tempExp. 

738 mask : `int`, optional 

739 Bit mask value to exclude from coaddition. 

740 supplementaryData : lsst.pipe.base.Struct, optional 

741 Struct with additional data products needed to assemble coadd. 

742 Only used by subclasses that implement `makeSupplementaryData` 

743 and override `run`. 

744 

745 Returns 

746 ------- 

747 result : `lsst.pipe.base.Struct` 

748 Result struct with components: 

749 

750 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

751 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

752 - ``warpRefList``: input list of refs to the warps ( 

753 ``lsst.daf.butler.DeferredDatasetHandle`` or 

754 ``lsst.daf.persistence.ButlerDataRef``) 

755 (unmodified) 

756 - ``imageScalerList``: input list of image scalers (unmodified) 

757 - ``weightList``: input list of weights (unmodified) 

758 """ 

759 tempExpName = self.getTempExpDatasetName(self.warpType) 

760 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

761 stats = self.prepareStats(mask=mask) 

762 

763 if altMaskList is None: 

764 altMaskList = [None]*len(tempExpRefList) 

765 

766 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

767 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

768 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

769 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

770 coaddMaskedImage = coaddExposure.getMaskedImage() 

771 subregionSizeArr = self.config.subregionSize 

772 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

773 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

774 if self.config.doNImage: 774 ↛ 775line 774 didn't jump to line 775, because the condition on line 774 was never true

775 nImage = afwImage.ImageU(skyInfo.bbox) 

776 else: 

777 nImage = None 

778 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

779 try: 

780 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

781 weightList, altMaskList, stats.flags, stats.ctrl, 

782 nImage=nImage) 

783 except Exception as e: 

784 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

785 

786 self.setInexactPsf(coaddMaskedImage.getMask()) 

787 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

788 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

789 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

790 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

791 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

792 weightList=weightList) 

793 

794 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

795 """Set the metadata for the coadd. 

796 

797 This basic implementation sets the filter from the first input. 

798 

799 Parameters 

800 ---------- 

801 coaddExposure : `lsst.afw.image.Exposure` 

802 The target exposure for the coadd. 

803 tempExpRefList : `list` 

804 List of data references to tempExp. 

805 weightList : `list` 

806 List of weights. 

807 """ 

808 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

809 tempExpName = self.getTempExpDatasetName(self.warpType) 

810 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

811 # (and we need more than just the PropertySet that contains the header), which is not possible 

812 # with the current butler (see #2777). 

813 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

814 

815 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 

816 # Gen 3 API 

817 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

818 else: 

819 # Gen 2 API. Delete this when Gen 2 retired 

820 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

821 for tempExpRef in tempExpRefList] 

822 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

823 

824 coaddExposure.setFilter(tempExpList[0].getFilter()) 

825 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

826 coaddInputs.ccds.reserve(numCcds) 

827 coaddInputs.visits.reserve(len(tempExpList)) 

828 

829 for tempExp, weight in zip(tempExpList, weightList): 

830 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

831 

832 if self.config.doUsePsfMatchedPolygons: 

833 self.shrinkValidPolygons(coaddInputs) 

834 

835 coaddInputs.visits.sort() 

836 if self.warpType == "psfMatched": 

837 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

838 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

839 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

840 # having the maximum width (sufficient because square) 

841 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

842 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

843 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

844 else: 

845 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

846 self.config.coaddPsf.makeControl()) 

847 coaddExposure.setPsf(psf) 

848 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

849 coaddExposure.getWcs()) 

850 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

851 if self.config.doAttachTransmissionCurve: 

852 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

853 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

854 

855 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

856 altMaskList, statsFlags, statsCtrl, nImage=None): 

857 """Assemble the coadd for a sub-region. 

858 

859 For each coaddTempExp, check for (and swap in) an alternative mask 

860 if one is passed. Remove mask planes listed in 

861 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

862 `lsst.afw.math.statisticsStack` with the statistic specified by 

863 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

864 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

865 an N-sigma clipped mean where N and iterations are specified by 

866 statsCtrl. Assign the stacked subregion back to the coadd. 

867 

868 Parameters 

869 ---------- 

870 coaddExposure : `lsst.afw.image.Exposure` 

871 The target exposure for the coadd. 

872 bbox : `lsst.geom.Box` 

873 Sub-region to coadd. 

874 tempExpRefList : `list` 

875 List of data reference to tempExp. 

876 imageScalerList : `list` 

877 List of image scalers. 

878 weightList : `list` 

879 List of weights. 

880 altMaskList : `list` 

881 List of alternate masks to use rather than those stored with 

882 tempExp, or None. Each element is dict with keys = mask plane 

883 name to which to add the spans. 

884 statsFlags : `lsst.afw.math.Property` 

885 Property object for statistic for coadd. 

886 statsCtrl : `lsst.afw.math.StatisticsControl` 

887 Statistics control object for coadd. 

888 nImage : `lsst.afw.image.ImageU`, optional 

889 Keeps track of exposure count for each pixel. 

890 """ 

891 self.log.debug("Computing coadd over %s", bbox) 

892 tempExpName = self.getTempExpDatasetName(self.warpType) 

893 coaddExposure.mask.addMaskPlane("REJECTED") 

894 coaddExposure.mask.addMaskPlane("CLIPPED") 

895 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

896 maskMap = self.setRejectedMaskMapping(statsCtrl) 

897 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

898 maskedImageList = [] 

899 if nImage is not None: 899 ↛ 900line 899 didn't jump to line 900, because the condition on line 899 was never true

900 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

901 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

902 

903 if isinstance(tempExpRef, DeferredDatasetHandle): 

904 # Gen 3 API 

905 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

906 else: 

907 # Gen 2 API. Delete this when Gen 2 retired 

908 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

909 

910 maskedImage = exposure.getMaskedImage() 

911 mask = maskedImage.getMask() 

912 if altMask is not None: 

913 self.applyAltMaskPlanes(mask, altMask) 

914 imageScaler.scaleMaskedImage(maskedImage) 

915 

916 # Add 1 for each pixel which is not excluded by the exclude mask. 

917 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

918 if nImage is not None: 918 ↛ 919line 918 didn't jump to line 919, because the condition on line 918 was never true

919 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

920 if self.config.removeMaskPlanes: 920 ↛ 922line 920 didn't jump to line 922, because the condition on line 920 was never false

921 self.removeMaskPlanes(maskedImage) 

922 maskedImageList.append(maskedImage) 

923 

924 with self.timer("stack"): 

925 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

926 clipped, # also set output to CLIPPED if sigma-clipped 

927 maskMap) 

928 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

929 if nImage is not None: 929 ↛ 930line 929 didn't jump to line 930, because the condition on line 929 was never true

930 nImage.assign(subNImage, bbox) 

931 

932 def removeMaskPlanes(self, maskedImage): 

933 """Unset the mask of an image for mask planes specified in the config. 

934 

935 Parameters 

936 ---------- 

937 maskedImage : `lsst.afw.image.MaskedImage` 

938 The masked image to be modified. 

939 """ 

940 mask = maskedImage.getMask() 

941 for maskPlane in self.config.removeMaskPlanes: 

942 try: 

943 mask &= ~mask.getPlaneBitMask(maskPlane) 

944 except pexExceptions.InvalidParameterError: 

945 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

946 maskPlane) 

947 

948 @staticmethod 

949 def setRejectedMaskMapping(statsCtrl): 

950 """Map certain mask planes of the warps to new planes for the coadd. 

951 

952 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

953 or CLIPPED, set it to REJECTED on the coadd. 

954 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

955 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

956 

957 Parameters 

958 ---------- 

959 statsCtrl : `lsst.afw.math.StatisticsControl` 

960 Statistics control object for coadd 

961 

962 Returns 

963 ------- 

964 maskMap : `list` of `tuple` of `int` 

965 A list of mappings of mask planes of the warped exposures to 

966 mask planes of the coadd. 

967 """ 

968 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

969 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

970 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

971 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

972 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

973 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

974 (clipped, clipped)] 

975 return maskMap 

976 

977 def applyAltMaskPlanes(self, mask, altMaskSpans): 

978 """Apply in place alt mask formatted as SpanSets to a mask. 

979 

980 Parameters 

981 ---------- 

982 mask : `lsst.afw.image.Mask` 

983 Original mask. 

984 altMaskSpans : `dict` 

985 SpanSet lists to apply. Each element contains the new mask 

986 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

987 and list of SpanSets to apply to the mask. 

988 

989 Returns 

990 ------- 

991 mask : `lsst.afw.image.Mask` 

992 Updated mask. 

993 """ 

994 if self.config.doUsePsfMatchedPolygons: 

995 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 995 ↛ 1003line 995 didn't jump to line 1003, because the condition on line 995 was never false

996 # Clear away any other masks outside the validPolygons. These pixels are no longer 

997 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

998 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

999 # This mask-clearing step must only occur *before* applying the new masks below 

1000 for spanSet in altMaskSpans['NO_DATA']: 

1001 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1002 

1003 for plane, spanSetList in altMaskSpans.items(): 

1004 maskClipValue = mask.addMaskPlane(plane) 

1005 for spanSet in spanSetList: 

1006 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1007 return mask 

1008 

1009 def shrinkValidPolygons(self, coaddInputs): 

1010 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1011 

1012 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1013 does not have a validPolygon, create one from its bbox. 

1014 

1015 Parameters 

1016 ---------- 

1017 coaddInputs : `lsst.afw.image.coaddInputs` 

1018 Original mask. 

1019 

1020 """ 

1021 for ccd in coaddInputs.ccds: 

1022 polyOrig = ccd.getValidPolygon() 

1023 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1024 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1025 if polyOrig: 

1026 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1027 else: 

1028 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1029 ccd.setValidPolygon(validPolygon) 

1030 

1031 def readBrightObjectMasks(self, dataRef): 

1032 """Retrieve the bright object masks. 

1033 

1034 Returns None on failure. 

1035 

1036 Parameters 

1037 ---------- 

1038 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1039 A Butler dataRef. 

1040 

1041 Returns 

1042 ------- 

1043 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1044 Bright object mask from the Butler object, or None if it cannot 

1045 be retrieved. 

1046 """ 

1047 try: 

1048 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1049 except Exception as e: 

1050 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1051 return None 

1052 

1053 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1054 """Set the bright object masks. 

1055 

1056 Parameters 

1057 ---------- 

1058 exposure : `lsst.afw.image.Exposure` 

1059 Exposure under consideration. 

1060 dataId : `lsst.daf.persistence.dataId` 

1061 Data identifier dict for patch. 

1062 brightObjectMasks : `lsst.afw.table` 

1063 Table of bright objects to mask. 

1064 """ 

1065 

1066 if brightObjectMasks is None: 

1067 self.log.warn("Unable to apply bright object mask: none supplied") 

1068 return 

1069 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1070 mask = exposure.getMaskedImage().getMask() 

1071 wcs = exposure.getWcs() 

1072 plateScale = wcs.getPixelScale().asArcseconds() 

1073 

1074 for rec in brightObjectMasks: 

1075 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1076 if rec["type"] == "box": 

1077 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1078 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1079 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1080 

1081 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1082 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1083 

1084 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1085 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1086 spans = afwGeom.SpanSet(bbox) 

1087 elif rec["type"] == "circle": 

1088 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1089 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1090 else: 

1091 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1092 continue 

1093 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1094 

1095 def setInexactPsf(self, mask): 

1096 """Set INEXACT_PSF mask plane. 

1097 

1098 If any of the input images isn't represented in the coadd (due to 

1099 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1100 these pixels. 

1101 

1102 Parameters 

1103 ---------- 

1104 mask : `lsst.afw.image.Mask` 

1105 Coadded exposure's mask, modified in-place. 

1106 """ 

1107 mask.addMaskPlane("INEXACT_PSF") 

1108 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1109 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1110 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1111 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1112 array = mask.getArray() 

1113 selected = array & (sensorEdge | clipped | rejected) > 0 

1114 array[selected] |= inexactPsf 

1115 

1116 @classmethod 

1117 def _makeArgumentParser(cls): 

1118 """Create an argument parser. 

1119 """ 

1120 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1121 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1122 + cls.ConfigClass().warpType + "Warp", 

1123 help="data ID, e.g. --id tract=12345 patch=1,2", 

1124 ContainerClass=AssembleCoaddDataIdContainer) 

1125 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1126 ContainerClass=SelectDataIdContainer) 

1127 return parser 

1128 

1129 @staticmethod 

1130 def _subBBoxIter(bbox, subregionSize): 

1131 """Iterate over subregions of a bbox. 

1132 

1133 Parameters 

1134 ---------- 

1135 bbox : `lsst.geom.Box2I` 

1136 Bounding box over which to iterate. 

1137 subregionSize: `lsst.geom.Extent2I` 

1138 Size of sub-bboxes. 

1139 

1140 Yields 

1141 ------ 

1142 subBBox : `lsst.geom.Box2I` 

1143 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1144 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1145 the edges of ``bbox``, but it will never be empty. 

1146 """ 

1147 if bbox.isEmpty(): 1147 ↛ 1148line 1147 didn't jump to line 1148, because the condition on line 1147 was never true

1148 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1149 if subregionSize[0] < 1 or subregionSize[1] < 1: 1149 ↛ 1150line 1149 didn't jump to line 1150, because the condition on line 1149 was never true

1150 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1151 

1152 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1153 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1154 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1155 subBBox.clip(bbox) 

1156 if subBBox.isEmpty(): 1156 ↛ 1157line 1156 didn't jump to line 1157, because the condition on line 1156 was never true

1157 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1158 "colShift=%s, rowShift=%s" % 

1159 (bbox, subregionSize, colShift, rowShift)) 

1160 yield subBBox 

1161 

1162 

1163class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1164 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1165 """ 

1166 

1167 def makeDataRefList(self, namespace): 

1168 """Make self.refList from self.idList. 

1169 

1170 Parameters 

1171 ---------- 

1172 namespace 

1173 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1174 """ 

1175 datasetType = namespace.config.coaddName + "Coadd" 

1176 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1177 

1178 for dataId in self.idList: 

1179 # tract and patch are required 

1180 for key in keysCoadd: 

1181 if key not in dataId: 

1182 raise RuntimeError("--id must include " + key) 

1183 

1184 dataRef = namespace.butler.dataRef( 

1185 datasetType=datasetType, 

1186 dataId=dataId, 

1187 ) 

1188 self.refList.append(dataRef) 

1189 

1190 

1191def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1192 """Function to count the number of pixels with a specific mask in a 

1193 footprint. 

1194 

1195 Find the intersection of mask & footprint. Count all pixels in the mask 

1196 that are in the intersection that have bitmask set but do not have 

1197 ignoreMask set. Return the count. 

1198 

1199 Parameters 

1200 ---------- 

1201 mask : `lsst.afw.image.Mask` 

1202 Mask to define intersection region by. 

1203 footprint : `lsst.afw.detection.Footprint` 

1204 Footprint to define the intersection region by. 

1205 bitmask 

1206 Specific mask that we wish to count the number of occurances of. 

1207 ignoreMask 

1208 Pixels to not consider. 

1209 

1210 Returns 

1211 ------- 

1212 result : `int` 

1213 Count of number of pixels in footprint with specified mask. 

1214 """ 

1215 bbox = footprint.getBBox() 

1216 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1217 fp = afwImage.Mask(bbox) 

1218 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1219 footprint.spans.setMask(fp, bitmask) 

1220 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1221 (subMask.getArray() & ignoreMask) == 0).sum() 

1222 

1223 

1224class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1225 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1226 """ 

1227 assembleMeanCoadd = pexConfig.ConfigurableField( 

1228 target=AssembleCoaddTask, 

1229 doc="Task to assemble an initial Coadd using the MEAN statistic.", 

1230 ) 

1231 assembleMeanClipCoadd = pexConfig.ConfigurableField( 

1232 target=AssembleCoaddTask, 

1233 doc="Task to assemble an initial Coadd using the MEANCLIP statistic.", 

1234 ) 

1235 clipDetection = pexConfig.ConfigurableField( 

1236 target=SourceDetectionTask, 

1237 doc="Detect sources on difference between unclipped and clipped coadd") 

1238 minClipFootOverlap = pexConfig.Field( 

1239 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1240 dtype=float, 

1241 default=0.6 

1242 ) 

1243 minClipFootOverlapSingle = pexConfig.Field( 

1244 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1245 "clipped when only one visit overlaps", 

1246 dtype=float, 

1247 default=0.5 

1248 ) 

1249 minClipFootOverlapDouble = pexConfig.Field( 

1250 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1251 "clipped when two visits overlap", 

1252 dtype=float, 

1253 default=0.45 

1254 ) 

1255 maxClipFootOverlapDouble = pexConfig.Field( 

1256 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1257 "considering two visits", 

1258 dtype=float, 

1259 default=0.15 

1260 ) 

1261 minBigOverlap = pexConfig.Field( 

1262 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1263 "when labeling clipped footprints", 

1264 dtype=int, 

1265 default=100 

1266 ) 

1267 

1268 def setDefaults(self): 

1269 """Set default values for clipDetection. 

1270 

1271 Notes 

1272 ----- 

1273 The numeric values for these configuration parameters were 

1274 empirically determined, future work may further refine them. 

1275 """ 

1276 AssembleCoaddConfig.setDefaults(self) 

1277 self.clipDetection.doTempLocalBackground = False 

1278 self.clipDetection.reEstimateBackground = False 

1279 self.clipDetection.returnOriginalFootprints = False 

1280 self.clipDetection.thresholdPolarity = "both" 

1281 self.clipDetection.thresholdValue = 2 

1282 self.clipDetection.nSigmaToGrow = 2 

1283 self.clipDetection.minPixels = 4 

1284 self.clipDetection.isotropicGrow = True 

1285 self.clipDetection.thresholdType = "pixel_stdev" 

1286 self.sigmaClip = 1.5 

1287 self.clipIter = 3 

1288 self.statistic = "MEAN" 

1289 self.assembleMeanCoadd.statistic = 'MEAN' 

1290 self.assembleMeanClipCoadd.statistic = 'MEANCLIP' 

1291 self.assembleMeanCoadd.doWrite = False 

1292 self.assembleMeanClipCoadd.doWrite = False 

1293 

1294 def validate(self): 

1295 if self.doSigmaClip: 

1296 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1297 "Ignoring doSigmaClip.") 

1298 self.doSigmaClip = False 

1299 if self.statistic != "MEAN": 

1300 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1301 "(%s chosen). Please set statistic to MEAN." 

1302 % (self.statistic)) 

1303 AssembleCoaddTask.ConfigClass.validate(self) 

1304 

1305 

1306class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1307 """Assemble a coadded image from a set of coadded temporary exposures, 

1308 being careful to clip & flag areas with potential artifacts. 

1309 

1310 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1311 we clip outliers). The problem with doing this is that when computing the 

1312 coadd PSF at a given location, individual visit PSFs from visits with 

1313 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1314 In this task, we correct for this behavior by creating a new 

1315 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1316 coaddTempExps and the final coadd where 

1317 

1318 i. difference imaging suggests that there is an outlier and 

1319 ii. this outlier appears on only one or two images. 

1320 

1321 Such regions will not contribute to the final coadd. Furthermore, any 

1322 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1323 Note that the algorithm implemented by this task is preliminary and works 

1324 correctly for HSC data. Parameter modifications and or considerable 

1325 redesigning of the algorithm is likley required for other surveys. 

1326 

1327 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1328 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1329 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1330 if you wish. 

1331 

1332 Notes 

1333 ----- 

1334 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1335 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1336 see `baseDebug` for more about ``debug.py`` files. 

1337 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1338 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1339 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1340 for further information. 

1341 

1342 Examples 

1343 -------- 

1344 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1345 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1346 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1347 

1348 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1349 and filter to be coadded (specified using 

1350 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1351 along with a list of coaddTempExps to attempt to coadd (specified using 

1352 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1353 Only the coaddTempExps that cover the specified tract and patch will be 

1354 coadded. A list of the available optional arguments can be obtained by 

1355 calling assembleCoadd.py with the --help command line argument: 

1356 

1357 .. code-block:: none 

1358 

1359 assembleCoadd.py --help 

1360 

1361 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1362 context of multi-band processing, we will generate the HSC-I & -R band 

1363 coadds from HSC engineering test data provided in the ci_hsc package. 

1364 To begin, assuming that the lsst stack has been already set up, we must 

1365 set up the obs_subaru and ci_hsc packages. This defines the environment 

1366 variable $CI_HSC_DIR and points at the location of the package. The raw 

1367 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1368 the coadds, we must first 

1369 

1370 - ``processCcd`` 

1371 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1372 - ``makeSkyMap`` 

1373 create a skymap that covers the area of the sky present in the raw exposures 

1374 - ``makeCoaddTempExp`` 

1375 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1376 

1377 We can perform all of these steps by running 

1378 

1379 .. code-block:: none 

1380 

1381 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1382 

1383 This will produce warped coaddTempExps for each visit. To coadd the 

1384 warped data, we call ``assembleCoadd.py`` as follows: 

1385 

1386 .. code-block:: none 

1387 

1388 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1389 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1390 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1391 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1392 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1393 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1394 --selectId visit=903988 ccd=24 

1395 

1396 This will process the HSC-I band data. The results are written in 

1397 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1398 

1399 You may also choose to run: 

1400 

1401 .. code-block:: none 

1402 

1403 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1404 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1405 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1406 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1407 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1408 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1409 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1410 --selectId visit=903346 ccd=12 

1411 

1412 to generate the coadd for the HSC-R band if you are interested in following 

1413 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1414 """ 

1415 ConfigClass = SafeClipAssembleCoaddConfig 

1416 _DefaultName = "safeClipAssembleCoadd" 

1417 

1418 def __init__(self, *args, **kwargs): 

1419 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1420 schema = afwTable.SourceTable.makeMinimalSchema() 

1421 self.makeSubtask("clipDetection", schema=schema) 

1422 self.makeSubtask("assembleMeanClipCoadd") 

1423 self.makeSubtask("assembleMeanCoadd") 

1424 

1425 @utils.inheritDoc(AssembleCoaddTask) 

1426 @pipeBase.timeMethod 

1427 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1428 """Assemble the coadd for a region. 

1429 

1430 Compute the difference of coadds created with and without outlier 

1431 rejection to identify coadd pixels that have outlier values in some 

1432 individual visits. 

1433 Detect clipped regions on the difference image and mark these regions 

1434 on the one or two individual coaddTempExps where they occur if there 

1435 is significant overlap between the clipped region and a source. This 

1436 leaves us with a set of footprints from the difference image that have 

1437 been identified as having occured on just one or two individual visits. 

1438 However, these footprints were generated from a difference image. It 

1439 is conceivable for a large diffuse source to have become broken up 

1440 into multiple footprints acrosss the coadd difference in this process. 

1441 Determine the clipped region from all overlapping footprints from the 

1442 detected sources in each visit - these are big footprints. 

1443 Combine the small and big clipped footprints and mark them on a new 

1444 bad mask plane. 

1445 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1446 removal. Clipped footprints will no longer make it into the coadd 

1447 because they are marked in the new bad mask plane. 

1448 

1449 Notes 

1450 ----- 

1451 args and kwargs are passed but ignored in order to match the call 

1452 signature expected by the parent task. 

1453 """ 

1454 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1455 mask = exp.getMaskedImage().getMask() 

1456 mask.addMaskPlane("CLIPPED") 

1457 

1458 result = self.detectClip(exp, tempExpRefList) 

1459 

1460 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1461 

1462 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1463 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1464 # Append big footprints from individual Warps to result.clipSpans 

1465 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1466 result.detectionFootprints, maskClipValue, maskDetValue, 

1467 exp.getBBox()) 

1468 # Create mask of the current clipped footprints 

1469 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1470 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1471 

1472 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1473 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1474 maskClip |= maskClipBig 

1475 

1476 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1477 badMaskPlanes = self.config.badMaskPlanes[:] 

1478 badMaskPlanes.append("CLIPPED") 

1479 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1480 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1481 result.clipSpans, mask=badPixelMask) 

1482 

1483 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1484 """Return an exposure that contains the difference between unclipped 

1485 and clipped coadds. 

1486 

1487 Generate a difference image between clipped and unclipped coadds. 

1488 Compute the difference image by subtracting an outlier-clipped coadd 

1489 from an outlier-unclipped coadd. Return the difference image. 

1490 

1491 Parameters 

1492 ---------- 

1493 skyInfo : `lsst.pipe.base.Struct` 

1494 Patch geometry information, from getSkyInfo 

1495 tempExpRefList : `list` 

1496 List of data reference to tempExp 

1497 imageScalerList : `list` 

1498 List of image scalers 

1499 weightList : `list` 

1500 List of weights 

1501 

1502 Returns 

1503 ------- 

1504 exp : `lsst.afw.image.Exposure` 

1505 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1506 """ 

1507 coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList, 

1508 imageScalerList, weightList).coaddExposure 

1509 

1510 coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList, 

1511 imageScalerList, weightList).coaddExposure 

1512 

1513 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1514 coaddDiff -= coaddClip.getMaskedImage() 

1515 exp = afwImage.ExposureF(coaddDiff) 

1516 exp.setPsf(coaddMean.getPsf()) 

1517 return exp 

1518 

1519 def detectClip(self, exp, tempExpRefList): 

1520 """Detect clipped regions on an exposure and set the mask on the 

1521 individual tempExp masks. 

1522 

1523 Detect footprints in the difference image after smoothing the 

1524 difference image with a Gaussian kernal. Identify footprints that 

1525 overlap with one or two input ``coaddTempExps`` by comparing the 

1526 computed overlap fraction to thresholds set in the config. A different 

1527 threshold is applied depending on the number of overlapping visits 

1528 (restricted to one or two). If the overlap exceeds the thresholds, 

1529 the footprint is considered "CLIPPED" and is marked as such on the 

1530 coaddTempExp. Return a struct with the clipped footprints, the indices 

1531 of the ``coaddTempExps`` that end up overlapping with the clipped 

1532 footprints, and a list of new masks for the ``coaddTempExps``. 

1533 

1534 Parameters 

1535 ---------- 

1536 exp : `lsst.afw.image.Exposure` 

1537 Exposure to run detection on. 

1538 tempExpRefList : `list` 

1539 List of data reference to tempExp. 

1540 

1541 Returns 

1542 ------- 

1543 result : `lsst.pipe.base.Struct` 

1544 Result struct with components: 

1545 

1546 - ``clipFootprints``: list of clipped footprints. 

1547 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1548 ``tempExpRefList``. 

1549 - ``clipSpans``: List of dictionaries containing spanSet lists 

1550 to clip. Each element contains the new maskplane name 

1551 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1552 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1553 compressed into footprints. 

1554 """ 

1555 mask = exp.getMaskedImage().getMask() 

1556 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1557 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1558 # Merge positive and negative together footprints together 

1559 fpSet.positive.merge(fpSet.negative) 

1560 footprints = fpSet.positive 

1561 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1562 ignoreMask = self.getBadPixelMask() 

1563 

1564 clipFootprints = [] 

1565 clipIndices = [] 

1566 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1567 

1568 # for use by detectClipBig 

1569 visitDetectionFootprints = [] 

1570 

1571 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1572 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1573 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1574 

1575 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1576 for i, warpRef in enumerate(tempExpRefList): 

1577 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1578 immediate=True).getMaskedImage().getMask() 

1579 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1580 afwImage.PARENT, True) 

1581 maskVisitDet &= maskDetValue 

1582 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1583 visitDetectionFootprints.append(visitFootprints) 

1584 

1585 for j, footprint in enumerate(footprints.getFootprints()): 

1586 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1587 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1588 

1589 # build a list of clipped spans for each visit 

1590 for j, footprint in enumerate(footprints.getFootprints()): 

1591 nPixel = footprint.getArea() 

1592 overlap = [] # hold the overlap with each visit 

1593 indexList = [] # index of visit in global list 

1594 for i in range(len(tempExpRefList)): 

1595 ignore = ignoreArr[i, j] 

1596 overlapDet = overlapDetArr[i, j] 

1597 totPixel = nPixel - ignore 

1598 

1599 # If we have more bad pixels than detection skip 

1600 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1600 ↛ 1602line 1600 didn't jump to line 1602, because the condition on line 1600 was never false

1601 continue 

1602 overlap.append(overlapDet/float(totPixel)) 

1603 indexList.append(i) 

1604 

1605 overlap = numpy.array(overlap) 

1606 if not len(overlap): 1606 ↛ 1609line 1606 didn't jump to line 1609, because the condition on line 1606 was never false

1607 continue 

1608 

1609 keep = False # Should this footprint be marked as clipped? 

1610 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1611 

1612 # If footprint only has one overlap use a lower threshold 

1613 if len(overlap) == 1: 

1614 if overlap[0] > self.config.minClipFootOverlapSingle: 

1615 keep = True 

1616 keepIndex = [0] 

1617 else: 

1618 # This is the general case where only visit should be clipped 

1619 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1620 if len(clipIndex) == 1: 

1621 keep = True 

1622 keepIndex = [clipIndex[0]] 

1623 

1624 # Test if there are clipped objects that overlap two different visits 

1625 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1626 if len(clipIndex) == 2 and len(overlap) > 3: 

1627 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1628 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1629 keep = True 

1630 keepIndex = clipIndex 

1631 

1632 if not keep: 

1633 continue 

1634 

1635 for index in keepIndex: 

1636 globalIndex = indexList[index] 

1637 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1638 

1639 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1640 clipFootprints.append(footprint) 

1641 

1642 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1643 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1644 

1645 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1646 maskClipValue, maskDetValue, coaddBBox): 

1647 """Return individual warp footprints for large artifacts and append 

1648 them to ``clipList`` in place. 

1649 

1650 Identify big footprints composed of many sources in the coadd 

1651 difference that may have originated in a large diffuse source in the 

1652 coadd. We do this by indentifying all clipped footprints that overlap 

1653 significantly with each source in all the coaddTempExps. 

1654 

1655 Parameters 

1656 ---------- 

1657 clipList : `list` 

1658 List of alt mask SpanSets with clipping information. Modified. 

1659 clipFootprints : `list` 

1660 List of clipped footprints. 

1661 clipIndices : `list` 

1662 List of which entries in tempExpClipList each footprint belongs to. 

1663 maskClipValue 

1664 Mask value of clipped pixels. 

1665 maskDetValue 

1666 Mask value of detected pixels. 

1667 coaddBBox : `lsst.geom.Box` 

1668 BBox of the coadd and warps. 

1669 

1670 Returns 

1671 ------- 

1672 bigFootprintsCoadd : `list` 

1673 List of big footprints 

1674 """ 

1675 bigFootprintsCoadd = [] 

1676 ignoreMask = self.getBadPixelMask() 

1677 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1678 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1679 for footprint in visitFootprints.getFootprints(): 

1680 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1681 

1682 # build a mask of clipped footprints that are in this visit 

1683 clippedFootprintsVisit = [] 

1684 for foot, clipIndex in zip(clipFootprints, clipIndices): 1684 ↛ 1685line 1684 didn't jump to line 1685, because the loop on line 1684 never started

1685 if index not in clipIndex: 

1686 continue 

1687 clippedFootprintsVisit.append(foot) 

1688 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1689 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1690 

1691 bigFootprintsVisit = [] 

1692 for foot in visitFootprints.getFootprints(): 

1693 if foot.getArea() < self.config.minBigOverlap: 

1694 continue 

1695 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1696 if nCount > self.config.minBigOverlap: 1696 ↛ 1697line 1696 didn't jump to line 1697, because the condition on line 1696 was never true

1697 bigFootprintsVisit.append(foot) 

1698 bigFootprintsCoadd.append(foot) 

1699 

1700 for footprint in bigFootprintsVisit: 1700 ↛ 1701line 1700 didn't jump to line 1701, because the loop on line 1700 never started

1701 clippedSpans["CLIPPED"].append(footprint.spans) 

1702 

1703 return bigFootprintsCoadd 

1704 

1705 

1706class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1707 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1708 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1709 "Only PSF-Matched Warps make sense for image subtraction. " 

1710 "Therefore, they must be an additional declared input."), 

1711 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1712 storageClass="ExposureF", 

1713 dimensions=("tract", "patch", "skymap", "visit"), 

1714 deferLoad=True, 

1715 multiple=True 

1716 ) 

1717 templateCoadd = pipeBase.connectionTypes.Output( 

1718 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1719 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1720 name="{fakesType}{outputCoaddName}CoaddPsfMatched", 

1721 storageClass="ExposureF", 

1722 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

1723 ) 

1724 

1725 def __init__(self, *, config=None): 

1726 super().__init__(config=config) 

1727 if not config.assembleStaticSkyModel.doWrite: 

1728 self.outputs.remove("templateCoadd") 

1729 config.validate() 

1730 

1731 

1732class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1733 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1734 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1735 target=AssembleCoaddTask, 

1736 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1737 " naive/first-iteration model of the static sky.", 

1738 ) 

1739 detect = pexConfig.ConfigurableField( 

1740 target=SourceDetectionTask, 

1741 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1742 ) 

1743 detectTemplate = pexConfig.ConfigurableField( 

1744 target=SourceDetectionTask, 

1745 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1746 ) 

1747 maxNumEpochs = pexConfig.Field( 

1748 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1749 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1750 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1751 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1752 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1753 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1754 "than transient and not masked.", 

1755 dtype=int, 

1756 default=2 

1757 ) 

1758 maxFractionEpochsLow = pexConfig.RangeField( 

1759 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1760 "Effective maxNumEpochs = " 

1761 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1762 dtype=float, 

1763 default=0.4, 

1764 min=0., max=1., 

1765 ) 

1766 maxFractionEpochsHigh = pexConfig.RangeField( 

1767 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1768 "Effective maxNumEpochs = " 

1769 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1770 dtype=float, 

1771 default=0.03, 

1772 min=0., max=1., 

1773 ) 

1774 spatialThreshold = pexConfig.RangeField( 

1775 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1776 "temporal criteria. If 0, clip all. If 1, clip none.", 

1777 dtype=float, 

1778 default=0.5, 

1779 min=0., max=1., 

1780 inclusiveMin=True, inclusiveMax=True 

1781 ) 

1782 doScaleWarpVariance = pexConfig.Field( 

1783 doc="Rescale Warp variance plane using empirical noise?", 

1784 dtype=bool, 

1785 default=True, 

1786 ) 

1787 scaleWarpVariance = pexConfig.ConfigurableField( 

1788 target=ScaleVarianceTask, 

1789 doc="Rescale variance on warps", 

1790 ) 

1791 doPreserveContainedBySource = pexConfig.Field( 

1792 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1793 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1794 dtype=bool, 

1795 default=True, 

1796 ) 

1797 doPrefilterArtifacts = pexConfig.Field( 

1798 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1799 "because they will be excluded anyway. This prevents them from contributing " 

1800 "to the outlier epoch count image and potentially being labeled as persistant." 

1801 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1802 dtype=bool, 

1803 default=True 

1804 ) 

1805 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1806 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1807 dtype=str, 

1808 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1809 ) 

1810 prefilterArtifactsRatio = pexConfig.Field( 

1811 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1812 dtype=float, 

1813 default=0.05 

1814 ) 

1815 

1816 def setDefaults(self): 

1817 AssembleCoaddConfig.setDefaults(self) 

1818 self.statistic = 'MEAN' 

1819 self.doUsePsfMatchedPolygons = True 

1820 

1821 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1822 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1823 if "EDGE" in self.badMaskPlanes: 1823 ↛ 1825line 1823 didn't jump to line 1825, because the condition on line 1823 was never false

1824 self.badMaskPlanes.remove('EDGE') 

1825 self.removeMaskPlanes.append('EDGE') 

1826 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1827 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1828 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1829 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1830 self.assembleStaticSkyModel.sigmaClip = 2.5 

1831 self.assembleStaticSkyModel.clipIter = 3 

1832 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1833 self.assembleStaticSkyModel.doWrite = False 

1834 self.detect.doTempLocalBackground = False 

1835 self.detect.reEstimateBackground = False 

1836 self.detect.returnOriginalFootprints = False 

1837 self.detect.thresholdPolarity = "both" 

1838 self.detect.thresholdValue = 5 

1839 self.detect.minPixels = 4 

1840 self.detect.isotropicGrow = True 

1841 self.detect.thresholdType = "pixel_stdev" 

1842 self.detect.nSigmaToGrow = 0.4 

1843 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1844 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1845 self.detectTemplate.nSigmaToGrow = 2.4 

1846 self.detectTemplate.doTempLocalBackground = False 

1847 self.detectTemplate.reEstimateBackground = False 

1848 self.detectTemplate.returnOriginalFootprints = False 

1849 

1850 def validate(self): 

1851 super().validate() 

1852 if self.assembleStaticSkyModel.doNImage: 

1853 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1854 "Please set assembleStaticSkyModel.doNImage=False") 

1855 

1856 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1857 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1858 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1859 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1860 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1861 

1862 

1863class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1864 """Assemble a compareWarp coadded image from a set of warps 

1865 by masking artifacts detected by comparing PSF-matched warps. 

1866 

1867 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1868 we clip outliers). The problem with doing this is that when computing the 

1869 coadd PSF at a given location, individual visit PSFs from visits with 

1870 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1871 In this task, we correct for this behavior by creating a new badMaskPlane 

1872 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1873 an artifact. We populate this plane on the input warps by comparing 

1874 PSF-matched warps with a PSF-matched median coadd which serves as a 

1875 model of the static sky. Any group of pixels that deviates from the 

1876 PSF-matched template coadd by more than config.detect.threshold sigma, 

1877 is an artifact candidate. The candidates are then filtered to remove 

1878 variable sources and sources that are difficult to subtract such as 

1879 bright stars. This filter is configured using the config parameters 

1880 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1881 the maximum fraction of epochs that the deviation can appear in and still 

1882 be considered an artifact. The spatialThreshold is the maximum fraction of 

1883 pixels in the footprint of the deviation that appear in other epochs 

1884 (where other epochs is defined by the temporalThreshold). If the deviant 

1885 region meets this criteria of having a significant percentage of pixels 

1886 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1887 set in the mask. These regions will not contribute to the final coadd. 

1888 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1889 of clipped regions. Note that the algorithm implemented by this task is 

1890 preliminary and works correctly for HSC data. Parameter modifications and 

1891 or considerable redesigning of the algorithm is likley required for other 

1892 surveys. 

1893 

1894 ``CompareWarpAssembleCoaddTask`` sub-classes 

1895 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1896 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1897 

1898 Notes 

1899 ----- 

1900 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1901 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

1902 ``baseDebug`` for more about ``debug.py`` files. 

1903 

1904 This task supports the following debug variables: 

1905 

1906 - ``saveCountIm`` 

1907 If True then save the Epoch Count Image as a fits file in the `figPath` 

1908 - ``figPath`` 

1909 Path to save the debug fits images and figures 

1910 

1911 For example, put something like: 

1912 

1913 .. code-block:: python 

1914 

1915 import lsstDebug 

1916 def DebugInfo(name): 

1917 di = lsstDebug.getInfo(name) 

1918 if name == "lsst.pipe.tasks.assembleCoadd": 

1919 di.saveCountIm = True 

1920 di.figPath = "/desired/path/to/debugging/output/images" 

1921 return di 

1922 lsstDebug.Info = DebugInfo 

1923 

1924 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

1925 ``--debug`` flag. Some subtasks may have their own debug variables; 

1926 see individual Task documentation. 

1927 

1928 Examples 

1929 -------- 

1930 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

1931 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

1932 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

1933 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1934 and filter to be coadded (specified using 

1935 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1936 along with a list of coaddTempExps to attempt to coadd (specified using 

1937 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1938 Only the warps that cover the specified tract and patch will be coadded. 

1939 A list of the available optional arguments can be obtained by calling 

1940 ``assembleCoadd.py`` with the ``--help`` command line argument: 

1941 

1942 .. code-block:: none 

1943 

1944 assembleCoadd.py --help 

1945 

1946 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

1947 context of multi-band processing, we will generate the HSC-I & -R band 

1948 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

1949 To begin, assuming that the lsst stack has been already set up, we must 

1950 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

1951 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

1952 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

1953 directory. To begin assembling the coadds, we must first 

1954 

1955 - processCcd 

1956 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1957 - makeSkyMap 

1958 create a skymap that covers the area of the sky present in the raw exposures 

1959 - makeCoaddTempExp 

1960 warp the individual calibrated exposures to the tangent plane of the coadd 

1961 

1962 We can perform all of these steps by running 

1963 

1964 .. code-block:: none 

1965 

1966 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1967 

1968 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

1969 warped data, we call ``assembleCoadd.py`` as follows: 

1970 

1971 .. code-block:: none 

1972 

1973 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1974 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1975 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1976 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1977 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1978 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1979 --selectId visit=903988 ccd=24 

1980 

1981 This will process the HSC-I band data. The results are written in 

1982 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1983 """ 

1984 ConfigClass = CompareWarpAssembleCoaddConfig 

1985 _DefaultName = "compareWarpAssembleCoadd" 

1986 

1987 def __init__(self, *args, **kwargs): 

1988 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1989 self.makeSubtask("assembleStaticSkyModel") 

1990 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

1991 self.makeSubtask("detect", schema=detectionSchema) 

1992 if self.config.doPreserveContainedBySource: 1992 ↛ 1994line 1992 didn't jump to line 1994, because the condition on line 1992 was never false

1993 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

1994 if self.config.doScaleWarpVariance: 1994 ↛ exitline 1994 didn't return from function '__init__', because the condition on line 1994 was never false

1995 self.makeSubtask("scaleWarpVariance") 

1996 

1997 @utils.inheritDoc(AssembleCoaddTask) 

1998 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

1999 """ 

2000 Generate a templateCoadd to use as a naive model of static sky to 

2001 subtract from PSF-Matched warps. 

2002 

2003 Returns 

2004 ------- 

2005 result : `lsst.pipe.base.Struct` 

2006 Result struct with components: 

2007 

2008 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2009 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2010 """ 

2011 # Ensure that psfMatchedWarps are used as input warps for template generation 

2012 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2013 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2014 

2015 # Because subtasks don't have connections we have to make one. 

2016 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2017 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2018 if self.config.assembleStaticSkyModel.doWrite: 

2019 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2020 # Remove template coadd from both subtask's and main tasks outputs, 

2021 # because it is handled by the subtask as `coaddExposure` 

2022 del outputRefs.templateCoadd 

2023 del staticSkyModelOutputRefs.templateCoadd 

2024 

2025 # A PSF-Matched nImage does not exist as a dataset type 

2026 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2027 del staticSkyModelOutputRefs.nImage 

2028 

2029 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2030 staticSkyModelOutputRefs) 

2031 if templateCoadd is None: 

2032 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2033 

2034 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2035 nImage=templateCoadd.nImage, 

2036 warpRefList=templateCoadd.warpRefList, 

2037 imageScalerList=templateCoadd.imageScalerList, 

2038 weightList=templateCoadd.weightList) 

2039 

2040 @utils.inheritDoc(AssembleCoaddTask) 

2041 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2042 """ 

2043 Generate a templateCoadd to use as a naive model of static sky to 

2044 subtract from PSF-Matched warps. 

2045 

2046 Returns 

2047 ------- 

2048 result : `lsst.pipe.base.Struct` 

2049 Result struct with components: 

2050 

2051 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2052 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2053 """ 

2054 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2055 if templateCoadd is None: 2055 ↛ 2056line 2055 didn't jump to line 2056, because the condition on line 2055 was never true

2056 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2057 

2058 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2059 nImage=templateCoadd.nImage, 

2060 warpRefList=templateCoadd.warpRefList, 

2061 imageScalerList=templateCoadd.imageScalerList, 

2062 weightList=templateCoadd.weightList) 

2063 

2064 def _noTemplateMessage(self, warpType): 

2065 warpName = (warpType[0].upper() + warpType[1:]) 

2066 message = """No %(warpName)s warps were found to build the template coadd which is 

2067 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2068 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2069 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2070 

2071 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2072 another algorithm like: 

2073 

2074 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2075 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2076 """ % {"warpName": warpName} 

2077 return message 

2078 

2079 @utils.inheritDoc(AssembleCoaddTask) 

2080 @pipeBase.timeMethod 

2081 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2082 supplementaryData, *args, **kwargs): 

2083 """Assemble the coadd. 

2084 

2085 Find artifacts and apply them to the warps' masks creating a list of 

2086 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2087 plane. Then pass these alternative masks to the base class's `run` 

2088 method. 

2089 

2090 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2091 that must contain a ``templateCoadd`` that serves as the 

2092 model of the static sky. 

2093 """ 

2094 

2095 # Check and match the order of the supplementaryData 

2096 # (PSF-matched) inputs to the order of the direct inputs, 

2097 # so that the artifact mask is applied to the right warp 

2098 dataIds = [ref.dataId for ref in tempExpRefList] 

2099 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2100 

2101 if dataIds != psfMatchedDataIds: 

2102 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2103 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2104 psfMatchedDataIds, dataIds) 

2105 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2106 psfMatchedDataIds, dataIds) 

2107 

2108 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2109 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2110 supplementaryData.warpRefList, 

2111 supplementaryData.imageScalerList) 

2112 

2113 badMaskPlanes = self.config.badMaskPlanes[:] 

2114 badMaskPlanes.append("CLIPPED") 

2115 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2116 

2117 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2118 spanSetMaskList, mask=badPixelMask) 

2119 

2120 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2121 # Psf-Matching moves the real edge inwards 

2122 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2123 return result 

2124 

2125 def applyAltEdgeMask(self, mask, altMaskList): 

2126 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2127 

2128 Parameters 

2129 ---------- 

2130 mask : `lsst.afw.image.Mask` 

2131 Original mask. 

2132 altMaskList : `list` 

2133 List of Dicts containing ``spanSet`` lists. 

2134 Each element contains the new mask plane name (e.g. "CLIPPED 

2135 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2136 the mask. 

2137 """ 

2138 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2139 for visitMask in altMaskList: 

2140 if "EDGE" in visitMask: 2140 ↛ 2139line 2140 didn't jump to line 2139, because the condition on line 2140 was never false

2141 for spanSet in visitMask['EDGE']: 

2142 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2143 

2144 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2145 """Find artifacts. 

2146 

2147 Loop through warps twice. The first loop builds a map with the count 

2148 of how many epochs each pixel deviates from the templateCoadd by more 

2149 than ``config.chiThreshold`` sigma. The second loop takes each 

2150 difference image and filters the artifacts detected in each using 

2151 count map to filter out variable sources and sources that are 

2152 difficult to subtract cleanly. 

2153 

2154 Parameters 

2155 ---------- 

2156 templateCoadd : `lsst.afw.image.Exposure` 

2157 Exposure to serve as model of static sky. 

2158 tempExpRefList : `list` 

2159 List of data references to warps. 

2160 imageScalerList : `list` 

2161 List of image scalers. 

2162 

2163 Returns 

2164 ------- 

2165 altMasks : `list` 

2166 List of dicts containing information about CLIPPED 

2167 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2168 """ 

2169 

2170 self.log.debug("Generating Count Image, and mask lists.") 

2171 coaddBBox = templateCoadd.getBBox() 

2172 slateIm = afwImage.ImageU(coaddBBox) 

2173 epochCountImage = afwImage.ImageU(coaddBBox) 

2174 nImage = afwImage.ImageU(coaddBBox) 

2175 spanSetArtifactList = [] 

2176 spanSetNoDataMaskList = [] 

2177 spanSetEdgeList = [] 

2178 badPixelMask = self.getBadPixelMask() 

2179 

2180 # mask of the warp diffs should = that of only the warp 

2181 templateCoadd.mask.clearAllMaskPlanes() 

2182 

2183 if self.config.doPreserveContainedBySource: 2183 ↛ 2186line 2183 didn't jump to line 2186, because the condition on line 2183 was never false

2184 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2185 else: 

2186 templateFootprints = None 

2187 

2188 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2189 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2190 if warpDiffExp is not None: 

2191 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2192 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2193 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2194 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2195 fpSet.positive.merge(fpSet.negative) 

2196 footprints = fpSet.positive 

2197 slateIm.set(0) 

2198 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2199 

2200 # Remove artifacts due to defects before they contribute to the epochCountImage 

2201 if self.config.doPrefilterArtifacts: 2201 ↛ 2203line 2201 didn't jump to line 2203, because the condition on line 2201 was never false

2202 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2203 for spans in spanSetList: 

2204 spans.setImage(slateIm, 1, doClip=True) 

2205 epochCountImage += slateIm 

2206 

2207 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2208 # undergo a second convolution. Pixels with data in the direct warp 

2209 # but not in the PSF-matched warp will not have their artifacts detected. 

2210 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2211 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2212 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2213 nansMask.setXY0(warpDiffExp.getXY0()) 

2214 edgeMask = warpDiffExp.mask 

2215 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2216 edgeMask.getPlaneBitMask("EDGE")).split() 

2217 else: 

2218 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2219 # In this case, mask the whole epoch 

2220 nansMask = afwImage.MaskX(coaddBBox, 1) 

2221 spanSetList = [] 

2222 spanSetEdgeMask = [] 

2223 

2224 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2225 

2226 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2227 spanSetArtifactList.append(spanSetList) 

2228 spanSetEdgeList.append(spanSetEdgeMask) 

2229 

2230 if lsstDebug.Info(__name__).saveCountIm: 2230 ↛ 2231line 2230 didn't jump to line 2231, because the condition on line 2230 was never true

2231 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2232 epochCountImage.writeFits(path) 

2233 

2234 for i, spanSetList in enumerate(spanSetArtifactList): 

2235 if spanSetList: 

2236 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2237 templateFootprints) 

2238 spanSetArtifactList[i] = filteredSpanSetList 

2239 

2240 altMasks = [] 

2241 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2242 altMasks.append({'CLIPPED': artifacts, 

2243 'NO_DATA': noData, 

2244 'EDGE': edge}) 

2245 return altMasks 

2246 

2247 def prefilterArtifacts(self, spanSetList, exp): 

2248 """Remove artifact candidates covered by bad mask plane. 

2249 

2250 Any future editing of the candidate list that does not depend on 

2251 temporal information should go in this method. 

2252 

2253 Parameters 

2254 ---------- 

2255 spanSetList : `list` 

2256 List of SpanSets representing artifact candidates. 

2257 exp : `lsst.afw.image.Exposure` 

2258 Exposure containing mask planes used to prefilter. 

2259 

2260 Returns 

2261 ------- 

2262 returnSpanSetList : `list` 

2263 List of SpanSets with artifacts. 

2264 """ 

2265 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2266 goodArr = (exp.mask.array & badPixelMask) == 0 

2267 returnSpanSetList = [] 

2268 bbox = exp.getBBox() 

2269 x0, y0 = exp.getXY0() 

2270 for i, span in enumerate(spanSetList): 

2271 y, x = span.clippedTo(bbox).indices() 

2272 yIndexLocal = numpy.array(y) - y0 

2273 xIndexLocal = numpy.array(x) - x0 

2274 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2275 if goodRatio > self.config.prefilterArtifactsRatio: 2275 ↛ 2270line 2275 didn't jump to line 2270, because the condition on line 2275 was never false

2276 returnSpanSetList.append(span) 

2277 return returnSpanSetList 

2278 

2279 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2280 """Filter artifact candidates. 

2281 

2282 Parameters 

2283 ---------- 

2284 spanSetList : `list` 

2285 List of SpanSets representing artifact candidates. 

2286 epochCountImage : `lsst.afw.image.Image` 

2287 Image of accumulated number of warpDiff detections. 

2288 nImage : `lsst.afw.image.Image` 

2289 Image of the accumulated number of total epochs contributing. 

2290 

2291 Returns 

2292 ------- 

2293 maskSpanSetList : `list` 

2294 List of SpanSets with artifacts. 

2295 """ 

2296 

2297 maskSpanSetList = [] 

2298 x0, y0 = epochCountImage.getXY0() 

2299 for i, span in enumerate(spanSetList): 

2300 y, x = span.indices() 

2301 yIdxLocal = [y1 - y0 for y1 in y] 

2302 xIdxLocal = [x1 - x0 for x1 in x] 

2303 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2304 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2305 

2306 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2307 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2308 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2309 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2310 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2311 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2312 & (outlierN <= effectiveMaxNumEpochs)) 

2313 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2314 if percentBelowThreshold > self.config.spatialThreshold: 

2315 maskSpanSetList.append(span) 

2316 

2317 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2317 ↛ 2330line 2317 didn't jump to line 2330, because the condition on line 2317 was never false

2318 # If a candidate is contained by a footprint on the template coadd, do not clip 

2319 filteredMaskSpanSetList = [] 

2320 for span in maskSpanSetList: 

2321 doKeep = True 

2322 for footprint in footprintsToExclude.positive.getFootprints(): 

2323 if footprint.spans.contains(span): 

2324 doKeep = False 

2325 break 

2326 if doKeep: 

2327 filteredMaskSpanSetList.append(span) 

2328 maskSpanSetList = filteredMaskSpanSetList 

2329 

2330 return maskSpanSetList 

2331 

2332 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2333 """Fetch a warp from the butler and return a warpDiff. 

2334 

2335 Parameters 

2336 ---------- 

2337 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2338 Butler dataRef for the warp. 

2339 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2340 An image scaler object. 

2341 templateCoadd : `lsst.afw.image.Exposure` 

2342 Exposure to be substracted from the scaled warp. 

2343 

2344 Returns 

2345 ------- 

2346 warp : `lsst.afw.image.Exposure` 

2347 Exposure of the image difference between the warp and template. 

2348 """ 

2349 

2350 # If the PSF-Matched warp did not exist for this direct warp 

2351 # None is holding its place to maintain order in Gen 3 

2352 if warpRef is None: 

2353 return None 

2354 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2355 warpName = self.getTempExpDatasetName('psfMatched') 

2356 if not isinstance(warpRef, DeferredDatasetHandle): 

2357 if not warpRef.datasetExists(warpName): 2357 ↛ 2358line 2357 didn't jump to line 2358, because the condition on line 2357 was never true

2358 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2359 return None 

2360 warp = warpRef.get(datasetType=warpName, immediate=True) 

2361 # direct image scaler OK for PSF-matched Warp 

2362 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2363 mi = warp.getMaskedImage() 

2364 if self.config.doScaleWarpVariance: 2364 ↛ 2369line 2364 didn't jump to line 2369, because the condition on line 2364 was never false

2365 try: 

2366 self.scaleWarpVariance.run(mi) 

2367 except Exception as exc: 

2368 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2369 mi -= templateCoadd.getMaskedImage() 

2370 return warp 

2371 

2372 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2373 """Return a path to which to write debugging output. 

2374 

2375 Creates a hyphen-delimited string of dataId values for simple filenames. 

2376 

2377 Parameters 

2378 ---------- 

2379 prefix : `str` 

2380 Prefix for filename. 

2381 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2382 Butler dataRef to make the path from. 

2383 coaddLevel : `bool`, optional. 

2384 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2385 'filter', but no 'visit'). 

2386 

2387 Returns 

2388 ------- 

2389 result : `str` 

2390 Path for debugging output. 

2391 """ 

2392 if coaddLevel: 

2393 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2394 else: 

2395 keys = warpRef.dataId.keys() 

2396 keyList = sorted(keys, reverse=True) 

2397 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2398 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2399 return os.path.join(directory, filename) 

2400 

2401 

2402def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None): 

2403 """Match the order of one list to another, padding if necessary 

2404 

2405 Parameters 

2406 ---------- 

2407 inputList : list 

2408 List to be reordered and padded. Elements can be any type. 

2409 inputKeys : iterable 

2410 Iterable of values to be compared with outputKeys. 

2411 Length must match `inputList` 

2412 outputKeys : iterable 

2413 Iterable of values to be compared with inputKeys. 

2414 padWith : 

2415 Any value to be inserted where inputKey not in outputKeys 

2416 

2417 Returns 

2418 ------- 

2419 list 

2420 Copy of inputList reordered per outputKeys and padded with `padWith` 

2421 so that the length matches length of outputKeys. 

2422 """ 

2423 outputList = [] 

2424 for d in outputKeys: 

2425 if d in inputKeys: 

2426 outputList.append(inputList[inputKeys.index(d)]) 

2427 else: 

2428 outputList.append(padWith) 

2429 return outputList