Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix 

41from .interpImage import InterpImageTask 

42from .scaleZeroPoint import ScaleZeroPointTask 

43from .coaddHelpers import groupPatchExposures, getGroupDataRef 

44from .scaleVariance import ScaleVarianceTask 

45from .maskStreaks import MaskStreaksTask 

46from lsst.meas.algorithms import SourceDetectionTask 

47from lsst.daf.butler import DeferredDatasetHandle 

48 

49__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

50 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

51 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

52 

53 

54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

55 dimensions=("tract", "patch", "abstract_filter", "skymap"), 

56 defaultTemplates={"inputCoaddName": "deep", 

57 "outputCoaddName": "deep", 

58 "warpType": "direct", 

59 "warpTypeSuffix": "", 

60 "fakesType": ""}): 

61 inputWarps = pipeBase.connectionTypes.Input( 

62 doc=("Input list of warps to be assemebled i.e. stacked." 

63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

64 name="{inputCoaddName}Coadd_{warpType}Warp", 

65 storageClass="ExposureF", 

66 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

67 deferLoad=True, 

68 multiple=True 

69 ) 

70 skyMap = pipeBase.connectionTypes.Input( 

71 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

72 name="{inputCoaddName}Coadd_skyMap", 

73 storageClass="SkyMap", 

74 dimensions=("skymap", ), 

75 ) 

76 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

77 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

78 " BRIGHT_OBJECT."), 

79 name="brightObjectMask", 

80 storageClass="ObjectMaskCatalog", 

81 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

82 ) 

83 coaddExposure = pipeBase.connectionTypes.Output( 

84 doc="Output coadded exposure, produced by stacking input warps", 

85 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}", 

86 storageClass="ExposureF", 

87 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

88 ) 

89 nImage = pipeBase.connectionTypes.Output( 

90 doc="Output image of number of input images per pixel", 

91 name="{outputCoaddName}Coadd_nImage", 

92 storageClass="ImageU", 

93 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

94 ) 

95 

96 def __init__(self, *, config=None): 

97 # Override the connection's name template with config to replicate Gen2 behavior 

98 config.connections.warpType = config.warpType 

99 config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType) 

100 

101 if config.hasFakes: 

102 config.connections.fakesType = "_fakes" 

103 

104 super().__init__(config=config) 

105 

106 if not config.doMaskBrightObjects: 

107 self.prerequisiteInputs.remove("brightObjectMask") 

108 

109 if not config.doNImage: 

110 self.outputs.remove("nImage") 

111 

112 

113class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

114 pipelineConnections=AssembleCoaddConnections): 

115 """Configuration parameters for the `AssembleCoaddTask`. 

116 

117 Notes 

118 ----- 

119 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

120 only set the bitplane config.brightObjectMaskName. To make this useful you 

121 *must* also configure the flags.pixel algorithm, for example by adding 

122 

123 .. code-block:: none 

124 

125 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

126 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

127 

128 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

129 """ 

130 warpType = pexConfig.Field( 

131 doc="Warp name: one of 'direct' or 'psfMatched'", 

132 dtype=str, 

133 default="direct", 

134 ) 

135 subregionSize = pexConfig.ListField( 

136 dtype=int, 

137 doc="Width, height of stack subregion size; " 

138 "make small enough that a full stack of images will fit into memory at once.", 

139 length=2, 

140 default=(2000, 2000), 

141 ) 

142 statistic = pexConfig.Field( 

143 dtype=str, 

144 doc="Main stacking statistic for aggregating over the epochs.", 

145 default="MEANCLIP", 

146 ) 

147 doSigmaClip = pexConfig.Field( 

148 dtype=bool, 

149 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

150 default=False, 

151 ) 

152 sigmaClip = pexConfig.Field( 

153 dtype=float, 

154 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

155 default=3.0, 

156 ) 

157 clipIter = pexConfig.Field( 

158 dtype=int, 

159 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

160 default=2, 

161 ) 

162 calcErrorFromInputVariance = pexConfig.Field( 

163 dtype=bool, 

164 doc="Calculate coadd variance from input variance by stacking statistic." 

165 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

166 default=True, 

167 ) 

168 scaleZeroPoint = pexConfig.ConfigurableField( 

169 target=ScaleZeroPointTask, 

170 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

171 ) 

172 doInterp = pexConfig.Field( 

173 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

174 dtype=bool, 

175 default=True, 

176 ) 

177 interpImage = pexConfig.ConfigurableField( 

178 target=InterpImageTask, 

179 doc="Task to interpolate (and extrapolate) over NaN pixels", 

180 ) 

181 doWrite = pexConfig.Field( 

182 doc="Persist coadd?", 

183 dtype=bool, 

184 default=True, 

185 ) 

186 doNImage = pexConfig.Field( 

187 doc="Create image of number of contributing exposures for each pixel", 

188 dtype=bool, 

189 default=False, 

190 ) 

191 doUsePsfMatchedPolygons = pexConfig.Field( 

192 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

193 dtype=bool, 

194 default=False, 

195 ) 

196 maskPropagationThresholds = pexConfig.DictField( 

197 keytype=str, 

198 itemtype=float, 

199 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

200 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

201 "would have contributed exceeds this value."), 

202 default={"SAT": 0.1}, 

203 ) 

204 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

205 doc="Mask planes to remove before coadding") 

206 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

207 doc="Set mask and flag bits for bright objects?") 

208 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

209 doc="Name of mask bit used for bright objects") 

210 coaddPsf = pexConfig.ConfigField( 

211 doc="Configuration for CoaddPsf", 

212 dtype=measAlg.CoaddPsfConfig, 

213 ) 

214 doAttachTransmissionCurve = pexConfig.Field( 

215 dtype=bool, default=False, optional=False, 

216 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

217 "(requires all input Exposures to have TransmissionCurves).") 

218 ) 

219 hasFakes = pexConfig.Field( 

220 dtype=bool, 

221 default=False, 

222 doc="Should be set to True if fake sources have been inserted into the input data." 

223 ) 

224 

225 def setDefaults(self): 

226 super().setDefaults() 

227 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

228 

229 def validate(self): 

230 super().validate() 

231 if self.doPsfMatch: 

232 # Backwards compatibility. 

233 # Configs do not have loggers 

234 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

235 self.warpType = 'psfMatched' 

236 if self.doSigmaClip and self.statistic != "MEANCLIP": 

237 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

238 self.statistic = "MEANCLIP" 

239 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

240 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

241 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

242 

243 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

244 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

245 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

246 if str(k) not in unstackableStats] 

247 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

248 % (self.statistic, stackableStats)) 

249 

250 

251class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

252 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

253 

254 We want to assemble a coadded image from a set of Warps (also called 

255 coadded temporary exposures or ``coaddTempExps``). 

256 Each input Warp covers a patch on the sky and corresponds to a single 

257 run/visit/exposure of the covered patch. We provide the task with a list 

258 of Warps (``selectDataList``) from which it selects Warps that cover the 

259 specified patch (pointed at by ``dataRef``). 

260 Each Warp that goes into a coadd will typically have an independent 

261 photometric zero-point. Therefore, we must scale each Warp to set it to 

262 a common photometric zeropoint. WarpType may be one of 'direct' or 

263 'psfMatched', and the boolean configs `config.makeDirect` and 

264 `config.makePsfMatched` set which of the warp types will be coadded. 

265 The coadd is computed as a mean with optional outlier rejection. 

266 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

267 Finally, Warps can have bad 'NaN' pixels which received no input from the 

268 source calExps. We interpolate over these bad (NaN) pixels. 

269 

270 `AssembleCoaddTask` uses several sub-tasks. These are 

271 

272 - `ScaleZeroPointTask` 

273 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

274 - `InterpImageTask` 

275 - interpolate across bad pixels (NaN) in the final coadd 

276 

277 You can retarget these subtasks if you wish. 

278 

279 Notes 

280 ----- 

281 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

282 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

283 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

284 no debug variables of its own. Some of the subtasks may support debug 

285 variables. See the documentation for the subtasks for further information. 

286 

287 Examples 

288 -------- 

289 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

290 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

291 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

292 inputs: a data reference to the tract patch and filter to be coadded, and 

293 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

294 ``--selectId``, respectively: 

295 

296 .. code-block:: none 

297 

298 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

299 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

300 

301 Only the Warps that cover the specified tract and patch will be coadded. 

302 A list of the available optional arguments can be obtained by calling 

303 ``assembleCoadd.py`` with the ``--help`` command line argument: 

304 

305 .. code-block:: none 

306 

307 assembleCoadd.py --help 

308 

309 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

310 multi-band processing, we will generate the HSC-I & -R band coadds from 

311 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

312 assuming that the lsst stack has been already set up, we must set up the 

313 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

314 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

315 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

316 coadds, we must first 

317 

318 - processCcd 

319 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

320 - makeSkyMap 

321 - create a skymap that covers the area of the sky present in the raw exposures 

322 - makeCoaddTempExp 

323 - warp the individual calibrated exposures to the tangent plane of the coadd 

324 

325 We can perform all of these steps by running 

326 

327 .. code-block:: none 

328 

329 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

330 

331 This will produce warped exposures for each visit. To coadd the warped 

332 data, we call assembleCoadd.py as follows: 

333 

334 .. code-block:: none 

335 

336 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

337 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

338 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

339 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

340 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

341 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

342 --selectId visit=903988 ccd=24 

343 

344 that will process the HSC-I band data. The results are written in 

345 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

346 

347 You may also choose to run: 

348 

349 .. code-block:: none 

350 

351 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

352 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

353 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

354 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

355 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

356 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

357 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

358 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

359 

360 to generate the coadd for the HSC-R band if you are interested in 

361 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

362 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

363 rather than `AssembleCoaddTask` to make the coadd. 

364 """ 

365 ConfigClass = AssembleCoaddConfig 

366 _DefaultName = "assembleCoadd" 

367 

368 def __init__(self, *args, **kwargs): 

369 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

370 if args: 

371 argNames = ["config", "name", "parentTask", "log"] 

372 kwargs.update({k: v for k, v in zip(argNames, args)}) 

373 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

374 "PipelineTask will not take positional args" % argNames, FutureWarning) 

375 

376 super().__init__(**kwargs) 

377 self.makeSubtask("interpImage") 

378 self.makeSubtask("scaleZeroPoint") 

379 

380 if self.config.doMaskBrightObjects: 

381 mask = afwImage.Mask() 

382 try: 

383 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

384 except pexExceptions.LsstCppException: 

385 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

386 mask.getMaskPlaneDict().keys()) 

387 del mask 

388 

389 self.warpType = self.config.warpType 

390 

391 @utils.inheritDoc(pipeBase.PipelineTask) 

392 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

393 # Docstring to be formatted with info from PipelineTask.runQuantum 

394 """ 

395 Notes 

396 ----- 

397 Assemble a coadd from a set of Warps. 

398 

399 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

400 Analogous to `runDataRef`, it prepares all the data products to be 

401 passed to `run`, and processes the results before returning a struct 

402 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

403 Therefore, its inputs are accessed subregion by subregion 

404 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

405 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

406 correspond to an update in `runDataRef` while both entry points 

407 are used. 

408 """ 

409 inputData = butlerQC.get(inputRefs) 

410 

411 # Construct skyInfo expected by run 

412 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

413 skyMap = inputData["skyMap"] 

414 outputDataId = butlerQC.quantum.dataId 

415 

416 inputData['skyInfo'] = makeSkyInfo(skyMap, 

417 tractId=outputDataId['tract'], 

418 patchId=outputDataId['patch']) 

419 

420 # Construct list of input Deferred Datasets 

421 # These quack a bit like like Gen2 DataRefs 

422 warpRefList = inputData['inputWarps'] 

423 # Perform same middle steps as `runDataRef` does 

424 inputs = self.prepareInputs(warpRefList) 

425 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

426 self.getTempExpDatasetName(self.warpType)) 

427 if len(inputs.tempExpRefList) == 0: 

428 self.log.warn("No coadd temporary exposures found") 

429 return 

430 

431 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

432 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

433 inputs.weightList, supplementaryData=supplementaryData) 

434 

435 inputData.setdefault('brightObjectMask', None) 

436 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

437 

438 if self.config.doWrite: 

439 butlerQC.put(retStruct, outputRefs) 

440 return retStruct 

441 

442 @pipeBase.timeMethod 

443 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

444 """Assemble a coadd from a set of Warps. 

445 

446 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

447 Compute weights to be applied to each Warp and 

448 find scalings to match the photometric zeropoint to a reference Warp. 

449 Assemble the Warps using `run`. Interpolate over NaNs and 

450 optionally write the coadd to disk. Return the coadded exposure. 

451 

452 Parameters 

453 ---------- 

454 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

455 Data reference defining the patch for coaddition and the 

456 reference Warp (if ``config.autoReference=False``). 

457 Used to access the following data products: 

458 - ``self.config.coaddName + "Coadd_skyMap"`` 

459 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

460 - ``self.config.coaddName + "Coadd"`` 

461 selectDataList : `list` 

462 List of data references to Calexps. Data to be coadded will be 

463 selected from this list based on overlap with the patch defined 

464 by dataRef, grouped by visit, and converted to a list of data 

465 references to warps. 

466 warpRefList : `list` 

467 List of data references to Warps to be coadded. 

468 Note: `warpRefList` is just the new name for `tempExpRefList`. 

469 

470 Returns 

471 ------- 

472 retStruct : `lsst.pipe.base.Struct` 

473 Result struct with components: 

474 

475 - ``coaddExposure``: coadded exposure (``Exposure``). 

476 - ``nImage``: exposure count image (``Image``). 

477 """ 

478 if selectDataList and warpRefList: 

479 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

480 "and which to use is ambiguous. Please pass only one.") 

481 

482 skyInfo = self.getSkyInfo(dataRef) 

483 if warpRefList is None: 

484 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

485 if len(calExpRefList) == 0: 

486 self.log.warn("No exposures to coadd") 

487 return 

488 self.log.info("Coadding %d exposures", len(calExpRefList)) 

489 

490 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

491 

492 inputData = self.prepareInputs(warpRefList) 

493 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

494 self.getTempExpDatasetName(self.warpType)) 

495 if len(inputData.tempExpRefList) == 0: 

496 self.log.warn("No coadd temporary exposures found") 

497 return 

498 

499 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

500 

501 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

502 inputData.weightList, supplementaryData=supplementaryData) 

503 

504 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

505 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

506 

507 if self.config.doWrite: 

508 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 

509 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

510 else: 

511 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

512 self.log.info("Persisting %s" % coaddDatasetName) 

513 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

514 if self.config.doNImage and retStruct.nImage is not None: 

515 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

516 

517 return retStruct 

518 

519 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

520 """Interpolate over missing data and mask bright stars. 

521 

522 Parameters 

523 ---------- 

524 coaddExposure : `lsst.afw.image.Exposure` 

525 The coadded exposure to process. 

526 dataRef : `lsst.daf.persistence.ButlerDataRef` 

527 Butler data reference for supplementary data. 

528 """ 

529 if self.config.doInterp: 

530 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

531 # The variance must be positive; work around for DM-3201. 

532 varArray = coaddExposure.variance.array 

533 with numpy.errstate(invalid="ignore"): 

534 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

535 

536 if self.config.doMaskBrightObjects: 

537 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

538 

539 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

540 """Make additional inputs to run() specific to subclasses (Gen2) 

541 

542 Duplicates interface of `runDataRef` method 

543 Available to be implemented by subclasses only if they need the 

544 coadd dataRef for performing preliminary processing before 

545 assembling the coadd. 

546 

547 Parameters 

548 ---------- 

549 dataRef : `lsst.daf.persistence.ButlerDataRef` 

550 Butler data reference for supplementary data. 

551 selectDataList : `list` (optional) 

552 Optional List of data references to Calexps. 

553 warpRefList : `list` (optional) 

554 Optional List of data references to Warps. 

555 """ 

556 return pipeBase.Struct() 

557 

558 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

559 """Make additional inputs to run() specific to subclasses (Gen3) 

560 

561 Duplicates interface of `runQuantum` method. 

562 Available to be implemented by subclasses only if they need the 

563 coadd dataRef for performing preliminary processing before 

564 assembling the coadd. 

565 

566 Parameters 

567 ---------- 

568 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

569 Gen3 Butler object for fetching additional data products before 

570 running the Task specialized for quantum being processed 

571 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

572 Attributes are the names of the connections describing input dataset types. 

573 Values are DatasetRefs that task consumes for corresponding dataset type. 

574 DataIds are guaranteed to match data objects in ``inputData``. 

575 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

576 Attributes are the names of the connections describing output dataset types. 

577 Values are DatasetRefs that task is to produce 

578 for corresponding dataset type. 

579 """ 

580 return pipeBase.Struct() 

581 

582 def getTempExpRefList(self, patchRef, calExpRefList): 

583 """Generate list data references corresponding to warped exposures 

584 that lie within the patch to be coadded. 

585 

586 Parameters 

587 ---------- 

588 patchRef : `dataRef` 

589 Data reference for patch. 

590 calExpRefList : `list` 

591 List of data references for input calexps. 

592 

593 Returns 

594 ------- 

595 tempExpRefList : `list` 

596 List of Warp/CoaddTempExp data references. 

597 """ 

598 butler = patchRef.getButler() 

599 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

600 self.getTempExpDatasetName(self.warpType)) 

601 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

602 g, groupData.keys) for 

603 g in groupData.groups.keys()] 

604 return tempExpRefList 

605 

606 def prepareInputs(self, refList): 

607 """Prepare the input warps for coaddition by measuring the weight for 

608 each warp and the scaling for the photometric zero point. 

609 

610 Each Warp has its own photometric zeropoint and background variance. 

611 Before coadding these Warps together, compute a scale factor to 

612 normalize the photometric zeropoint and compute the weight for each Warp. 

613 

614 Parameters 

615 ---------- 

616 refList : `list` 

617 List of data references to tempExp 

618 

619 Returns 

620 ------- 

621 result : `lsst.pipe.base.Struct` 

622 Result struct with components: 

623 

624 - ``tempExprefList``: `list` of data references to tempExp. 

625 - ``weightList``: `list` of weightings. 

626 - ``imageScalerList``: `list` of image scalers. 

627 """ 

628 statsCtrl = afwMath.StatisticsControl() 

629 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

630 statsCtrl.setNumIter(self.config.clipIter) 

631 statsCtrl.setAndMask(self.getBadPixelMask()) 

632 statsCtrl.setNanSafe(True) 

633 # compute tempExpRefList: a list of tempExpRef that actually exist 

634 # and weightList: a list of the weight of the associated coadd tempExp 

635 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

636 tempExpRefList = [] 

637 weightList = [] 

638 imageScalerList = [] 

639 tempExpName = self.getTempExpDatasetName(self.warpType) 

640 for tempExpRef in refList: 

641 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

642 # therefore have no datasetExists() method 

643 if not isinstance(tempExpRef, DeferredDatasetHandle): 

644 if not tempExpRef.datasetExists(tempExpName): 

645 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

646 continue 

647 

648 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

649 # Ignore any input warp that is empty of data 

650 if numpy.isnan(tempExp.image.array).all(): 

651 continue 

652 maskedImage = tempExp.getMaskedImage() 

653 imageScaler = self.scaleZeroPoint.computeImageScaler( 

654 exposure=tempExp, 

655 dataRef=tempExpRef, 

656 ) 

657 try: 

658 imageScaler.scaleMaskedImage(maskedImage) 

659 except Exception as e: 

660 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

661 continue 

662 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

663 afwMath.MEANCLIP, statsCtrl) 

664 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

665 weight = 1.0 / float(meanVar) 

666 if not numpy.isfinite(weight): 

667 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

668 continue 

669 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

670 

671 del maskedImage 

672 del tempExp 

673 

674 tempExpRefList.append(tempExpRef) 

675 weightList.append(weight) 

676 imageScalerList.append(imageScaler) 

677 

678 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

679 imageScalerList=imageScalerList) 

680 

681 def prepareStats(self, mask=None): 

682 """Prepare the statistics for coadding images. 

683 

684 Parameters 

685 ---------- 

686 mask : `int`, optional 

687 Bit mask value to exclude from coaddition. 

688 

689 Returns 

690 ------- 

691 stats : `lsst.pipe.base.Struct` 

692 Statistics structure with the following fields: 

693 

694 - ``statsCtrl``: Statistics control object for coadd 

695 (`lsst.afw.math.StatisticsControl`) 

696 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

697 """ 

698 if mask is None: 

699 mask = self.getBadPixelMask() 

700 statsCtrl = afwMath.StatisticsControl() 

701 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

702 statsCtrl.setNumIter(self.config.clipIter) 

703 statsCtrl.setAndMask(mask) 

704 statsCtrl.setNanSafe(True) 

705 statsCtrl.setWeighted(True) 

706 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

707 for plane, threshold in self.config.maskPropagationThresholds.items(): 

708 bit = afwImage.Mask.getMaskPlane(plane) 

709 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

710 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

711 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

712 

713 @pipeBase.timeMethod 

714 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

715 altMaskList=None, mask=None, supplementaryData=None): 

716 """Assemble a coadd from input warps 

717 

718 Assemble the coadd using the provided list of coaddTempExps. Since 

719 the full coadd covers a patch (a large area), the assembly is 

720 performed over small areas on the image at a time in order to 

721 conserve memory usage. Iterate over subregions within the outer 

722 bbox of the patch using `assembleSubregion` to stack the corresponding 

723 subregions from the coaddTempExps with the statistic specified. 

724 Set the edge bits the coadd mask based on the weight map. 

725 

726 Parameters 

727 ---------- 

728 skyInfo : `lsst.pipe.base.Struct` 

729 Struct with geometric information about the patch. 

730 tempExpRefList : `list` 

731 List of data references to Warps (previously called CoaddTempExps). 

732 imageScalerList : `list` 

733 List of image scalers. 

734 weightList : `list` 

735 List of weights 

736 altMaskList : `list`, optional 

737 List of alternate masks to use rather than those stored with 

738 tempExp. 

739 mask : `int`, optional 

740 Bit mask value to exclude from coaddition. 

741 supplementaryData : lsst.pipe.base.Struct, optional 

742 Struct with additional data products needed to assemble coadd. 

743 Only used by subclasses that implement `makeSupplementaryData` 

744 and override `run`. 

745 

746 Returns 

747 ------- 

748 result : `lsst.pipe.base.Struct` 

749 Result struct with components: 

750 

751 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

752 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

753 - ``warpRefList``: input list of refs to the warps ( 

754 ``lsst.daf.butler.DeferredDatasetHandle`` or 

755 ``lsst.daf.persistence.ButlerDataRef``) 

756 (unmodified) 

757 - ``imageScalerList``: input list of image scalers (unmodified) 

758 - ``weightList``: input list of weights (unmodified) 

759 """ 

760 tempExpName = self.getTempExpDatasetName(self.warpType) 

761 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

762 stats = self.prepareStats(mask=mask) 

763 

764 if altMaskList is None: 

765 altMaskList = [None]*len(tempExpRefList) 

766 

767 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

768 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

769 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

770 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

771 coaddMaskedImage = coaddExposure.getMaskedImage() 

772 subregionSizeArr = self.config.subregionSize 

773 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

774 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

775 if self.config.doNImage: 

776 nImage = afwImage.ImageU(skyInfo.bbox) 

777 else: 

778 nImage = None 

779 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

780 try: 

781 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

782 weightList, altMaskList, stats.flags, stats.ctrl, 

783 nImage=nImage) 

784 except Exception as e: 

785 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

786 

787 self.setInexactPsf(coaddMaskedImage.getMask()) 

788 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

789 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

790 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

791 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

792 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

793 weightList=weightList) 

794 

795 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

796 """Set the metadata for the coadd. 

797 

798 This basic implementation sets the filter from the first input. 

799 

800 Parameters 

801 ---------- 

802 coaddExposure : `lsst.afw.image.Exposure` 

803 The target exposure for the coadd. 

804 tempExpRefList : `list` 

805 List of data references to tempExp. 

806 weightList : `list` 

807 List of weights. 

808 """ 

809 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

810 tempExpName = self.getTempExpDatasetName(self.warpType) 

811 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

812 # (and we need more than just the PropertySet that contains the header), which is not possible 

813 # with the current butler (see #2777). 

814 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

815 

816 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 

817 # Gen 3 API 

818 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

819 else: 

820 # Gen 2 API. Delete this when Gen 2 retired 

821 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

822 for tempExpRef in tempExpRefList] 

823 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

824 

825 coaddExposure.setFilter(tempExpList[0].getFilter()) 

826 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

827 coaddInputs.ccds.reserve(numCcds) 

828 coaddInputs.visits.reserve(len(tempExpList)) 

829 

830 for tempExp, weight in zip(tempExpList, weightList): 

831 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

832 

833 if self.config.doUsePsfMatchedPolygons: 

834 self.shrinkValidPolygons(coaddInputs) 

835 

836 coaddInputs.visits.sort() 

837 if self.warpType == "psfMatched": 

838 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

839 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

840 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

841 # having the maximum width (sufficient because square) 

842 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

843 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

844 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

845 else: 

846 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

847 self.config.coaddPsf.makeControl()) 

848 coaddExposure.setPsf(psf) 

849 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

850 coaddExposure.getWcs()) 

851 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

852 if self.config.doAttachTransmissionCurve: 

853 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

854 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

855 

856 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

857 altMaskList, statsFlags, statsCtrl, nImage=None): 

858 """Assemble the coadd for a sub-region. 

859 

860 For each coaddTempExp, check for (and swap in) an alternative mask 

861 if one is passed. Remove mask planes listed in 

862 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

863 `lsst.afw.math.statisticsStack` with the statistic specified by 

864 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

865 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

866 an N-sigma clipped mean where N and iterations are specified by 

867 statsCtrl. Assign the stacked subregion back to the coadd. 

868 

869 Parameters 

870 ---------- 

871 coaddExposure : `lsst.afw.image.Exposure` 

872 The target exposure for the coadd. 

873 bbox : `lsst.geom.Box` 

874 Sub-region to coadd. 

875 tempExpRefList : `list` 

876 List of data reference to tempExp. 

877 imageScalerList : `list` 

878 List of image scalers. 

879 weightList : `list` 

880 List of weights. 

881 altMaskList : `list` 

882 List of alternate masks to use rather than those stored with 

883 tempExp, or None. Each element is dict with keys = mask plane 

884 name to which to add the spans. 

885 statsFlags : `lsst.afw.math.Property` 

886 Property object for statistic for coadd. 

887 statsCtrl : `lsst.afw.math.StatisticsControl` 

888 Statistics control object for coadd. 

889 nImage : `lsst.afw.image.ImageU`, optional 

890 Keeps track of exposure count for each pixel. 

891 """ 

892 self.log.debug("Computing coadd over %s", bbox) 

893 tempExpName = self.getTempExpDatasetName(self.warpType) 

894 coaddExposure.mask.addMaskPlane("REJECTED") 

895 coaddExposure.mask.addMaskPlane("CLIPPED") 

896 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

897 maskMap = self.setRejectedMaskMapping(statsCtrl) 

898 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

899 maskedImageList = [] 

900 if nImage is not None: 

901 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

902 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

903 

904 if isinstance(tempExpRef, DeferredDatasetHandle): 

905 # Gen 3 API 

906 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

907 else: 

908 # Gen 2 API. Delete this when Gen 2 retired 

909 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

910 

911 maskedImage = exposure.getMaskedImage() 

912 mask = maskedImage.getMask() 

913 if altMask is not None: 

914 self.applyAltMaskPlanes(mask, altMask) 

915 imageScaler.scaleMaskedImage(maskedImage) 

916 

917 # Add 1 for each pixel which is not excluded by the exclude mask. 

918 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

919 if nImage is not None: 

920 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

921 if self.config.removeMaskPlanes: 

922 self.removeMaskPlanes(maskedImage) 

923 maskedImageList.append(maskedImage) 

924 

925 with self.timer("stack"): 

926 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

927 clipped, # also set output to CLIPPED if sigma-clipped 

928 maskMap) 

929 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

930 if nImage is not None: 

931 nImage.assign(subNImage, bbox) 

932 

933 def removeMaskPlanes(self, maskedImage): 

934 """Unset the mask of an image for mask planes specified in the config. 

935 

936 Parameters 

937 ---------- 

938 maskedImage : `lsst.afw.image.MaskedImage` 

939 The masked image to be modified. 

940 """ 

941 mask = maskedImage.getMask() 

942 for maskPlane in self.config.removeMaskPlanes: 

943 try: 

944 mask &= ~mask.getPlaneBitMask(maskPlane) 

945 except pexExceptions.InvalidParameterError: 

946 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

947 maskPlane) 

948 

949 @staticmethod 

950 def setRejectedMaskMapping(statsCtrl): 

951 """Map certain mask planes of the warps to new planes for the coadd. 

952 

953 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

954 or CLIPPED, set it to REJECTED on the coadd. 

955 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

956 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

957 

958 Parameters 

959 ---------- 

960 statsCtrl : `lsst.afw.math.StatisticsControl` 

961 Statistics control object for coadd 

962 

963 Returns 

964 ------- 

965 maskMap : `list` of `tuple` of `int` 

966 A list of mappings of mask planes of the warped exposures to 

967 mask planes of the coadd. 

968 """ 

969 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

970 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

971 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

972 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

973 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

974 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

975 (clipped, clipped)] 

976 return maskMap 

977 

978 def applyAltMaskPlanes(self, mask, altMaskSpans): 

979 """Apply in place alt mask formatted as SpanSets to a mask. 

980 

981 Parameters 

982 ---------- 

983 mask : `lsst.afw.image.Mask` 

984 Original mask. 

985 altMaskSpans : `dict` 

986 SpanSet lists to apply. Each element contains the new mask 

987 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

988 and list of SpanSets to apply to the mask. 

989 

990 Returns 

991 ------- 

992 mask : `lsst.afw.image.Mask` 

993 Updated mask. 

994 """ 

995 if self.config.doUsePsfMatchedPolygons: 

996 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

997 # Clear away any other masks outside the validPolygons. These pixels are no longer 

998 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

999 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1000 # This mask-clearing step must only occur *before* applying the new masks below 

1001 for spanSet in altMaskSpans['NO_DATA']: 

1002 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1003 

1004 for plane, spanSetList in altMaskSpans.items(): 

1005 maskClipValue = mask.addMaskPlane(plane) 

1006 for spanSet in spanSetList: 

1007 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1008 return mask 

1009 

1010 def shrinkValidPolygons(self, coaddInputs): 

1011 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1012 

1013 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1014 does not have a validPolygon, create one from its bbox. 

1015 

1016 Parameters 

1017 ---------- 

1018 coaddInputs : `lsst.afw.image.coaddInputs` 

1019 Original mask. 

1020 

1021 """ 

1022 for ccd in coaddInputs.ccds: 

1023 polyOrig = ccd.getValidPolygon() 

1024 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1025 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1026 if polyOrig: 

1027 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1028 else: 

1029 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1030 ccd.setValidPolygon(validPolygon) 

1031 

1032 def readBrightObjectMasks(self, dataRef): 

1033 """Retrieve the bright object masks. 

1034 

1035 Returns None on failure. 

1036 

1037 Parameters 

1038 ---------- 

1039 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1040 A Butler dataRef. 

1041 

1042 Returns 

1043 ------- 

1044 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1045 Bright object mask from the Butler object, or None if it cannot 

1046 be retrieved. 

1047 """ 

1048 try: 

1049 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1050 except Exception as e: 

1051 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1052 return None 

1053 

1054 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1055 """Set the bright object masks. 

1056 

1057 Parameters 

1058 ---------- 

1059 exposure : `lsst.afw.image.Exposure` 

1060 Exposure under consideration. 

1061 dataId : `lsst.daf.persistence.dataId` 

1062 Data identifier dict for patch. 

1063 brightObjectMasks : `lsst.afw.table` 

1064 Table of bright objects to mask. 

1065 """ 

1066 

1067 if brightObjectMasks is None: 

1068 self.log.warn("Unable to apply bright object mask: none supplied") 

1069 return 

1070 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1071 mask = exposure.getMaskedImage().getMask() 

1072 wcs = exposure.getWcs() 

1073 plateScale = wcs.getPixelScale().asArcseconds() 

1074 

1075 for rec in brightObjectMasks: 

1076 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1077 if rec["type"] == "box": 

1078 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1079 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1080 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1081 

1082 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1083 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1084 

1085 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1086 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1087 spans = afwGeom.SpanSet(bbox) 

1088 elif rec["type"] == "circle": 

1089 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1090 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1091 else: 

1092 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1093 continue 

1094 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1095 

1096 def setInexactPsf(self, mask): 

1097 """Set INEXACT_PSF mask plane. 

1098 

1099 If any of the input images isn't represented in the coadd (due to 

1100 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1101 these pixels. 

1102 

1103 Parameters 

1104 ---------- 

1105 mask : `lsst.afw.image.Mask` 

1106 Coadded exposure's mask, modified in-place. 

1107 """ 

1108 mask.addMaskPlane("INEXACT_PSF") 

1109 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1110 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1111 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1112 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1113 array = mask.getArray() 

1114 selected = array & (sensorEdge | clipped | rejected) > 0 

1115 array[selected] |= inexactPsf 

1116 

1117 @classmethod 

1118 def _makeArgumentParser(cls): 

1119 """Create an argument parser. 

1120 """ 

1121 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1122 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1123 + cls.ConfigClass().warpType + "Warp", 

1124 help="data ID, e.g. --id tract=12345 patch=1,2", 

1125 ContainerClass=AssembleCoaddDataIdContainer) 

1126 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1127 ContainerClass=SelectDataIdContainer) 

1128 return parser 

1129 

1130 @staticmethod 

1131 def _subBBoxIter(bbox, subregionSize): 

1132 """Iterate over subregions of a bbox. 

1133 

1134 Parameters 

1135 ---------- 

1136 bbox : `lsst.geom.Box2I` 

1137 Bounding box over which to iterate. 

1138 subregionSize: `lsst.geom.Extent2I` 

1139 Size of sub-bboxes. 

1140 

1141 Yields 

1142 ------ 

1143 subBBox : `lsst.geom.Box2I` 

1144 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1145 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1146 the edges of ``bbox``, but it will never be empty. 

1147 """ 

1148 if bbox.isEmpty(): 

1149 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1150 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1151 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1152 

1153 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1154 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1155 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1156 subBBox.clip(bbox) 

1157 if subBBox.isEmpty(): 

1158 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1159 "colShift=%s, rowShift=%s" % 

1160 (bbox, subregionSize, colShift, rowShift)) 

1161 yield subBBox 

1162 

1163 

1164class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1165 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1166 """ 

1167 

1168 def makeDataRefList(self, namespace): 

1169 """Make self.refList from self.idList. 

1170 

1171 Parameters 

1172 ---------- 

1173 namespace 

1174 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1175 """ 

1176 datasetType = namespace.config.coaddName + "Coadd" 

1177 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1178 

1179 for dataId in self.idList: 

1180 # tract and patch are required 

1181 for key in keysCoadd: 

1182 if key not in dataId: 

1183 raise RuntimeError("--id must include " + key) 

1184 

1185 dataRef = namespace.butler.dataRef( 

1186 datasetType=datasetType, 

1187 dataId=dataId, 

1188 ) 

1189 self.refList.append(dataRef) 

1190 

1191 

1192def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1193 """Function to count the number of pixels with a specific mask in a 

1194 footprint. 

1195 

1196 Find the intersection of mask & footprint. Count all pixels in the mask 

1197 that are in the intersection that have bitmask set but do not have 

1198 ignoreMask set. Return the count. 

1199 

1200 Parameters 

1201 ---------- 

1202 mask : `lsst.afw.image.Mask` 

1203 Mask to define intersection region by. 

1204 footprint : `lsst.afw.detection.Footprint` 

1205 Footprint to define the intersection region by. 

1206 bitmask 

1207 Specific mask that we wish to count the number of occurances of. 

1208 ignoreMask 

1209 Pixels to not consider. 

1210 

1211 Returns 

1212 ------- 

1213 result : `int` 

1214 Count of number of pixels in footprint with specified mask. 

1215 """ 

1216 bbox = footprint.getBBox() 

1217 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1218 fp = afwImage.Mask(bbox) 

1219 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1220 footprint.spans.setMask(fp, bitmask) 

1221 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1222 (subMask.getArray() & ignoreMask) == 0).sum() 

1223 

1224 

1225class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1226 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1227 """ 

1228 assembleMeanCoadd = pexConfig.ConfigurableField( 

1229 target=AssembleCoaddTask, 

1230 doc="Task to assemble an initial Coadd using the MEAN statistic.", 

1231 ) 

1232 assembleMeanClipCoadd = pexConfig.ConfigurableField( 

1233 target=AssembleCoaddTask, 

1234 doc="Task to assemble an initial Coadd using the MEANCLIP statistic.", 

1235 ) 

1236 clipDetection = pexConfig.ConfigurableField( 

1237 target=SourceDetectionTask, 

1238 doc="Detect sources on difference between unclipped and clipped coadd") 

1239 minClipFootOverlap = pexConfig.Field( 

1240 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1241 dtype=float, 

1242 default=0.6 

1243 ) 

1244 minClipFootOverlapSingle = pexConfig.Field( 

1245 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1246 "clipped when only one visit overlaps", 

1247 dtype=float, 

1248 default=0.5 

1249 ) 

1250 minClipFootOverlapDouble = pexConfig.Field( 

1251 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1252 "clipped when two visits overlap", 

1253 dtype=float, 

1254 default=0.45 

1255 ) 

1256 maxClipFootOverlapDouble = pexConfig.Field( 

1257 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1258 "considering two visits", 

1259 dtype=float, 

1260 default=0.15 

1261 ) 

1262 minBigOverlap = pexConfig.Field( 

1263 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1264 "when labeling clipped footprints", 

1265 dtype=int, 

1266 default=100 

1267 ) 

1268 

1269 def setDefaults(self): 

1270 """Set default values for clipDetection. 

1271 

1272 Notes 

1273 ----- 

1274 The numeric values for these configuration parameters were 

1275 empirically determined, future work may further refine them. 

1276 """ 

1277 AssembleCoaddConfig.setDefaults(self) 

1278 self.clipDetection.doTempLocalBackground = False 

1279 self.clipDetection.reEstimateBackground = False 

1280 self.clipDetection.returnOriginalFootprints = False 

1281 self.clipDetection.thresholdPolarity = "both" 

1282 self.clipDetection.thresholdValue = 2 

1283 self.clipDetection.nSigmaToGrow = 2 

1284 self.clipDetection.minPixels = 4 

1285 self.clipDetection.isotropicGrow = True 

1286 self.clipDetection.thresholdType = "pixel_stdev" 

1287 self.sigmaClip = 1.5 

1288 self.clipIter = 3 

1289 self.statistic = "MEAN" 

1290 self.assembleMeanCoadd.statistic = 'MEAN' 

1291 self.assembleMeanClipCoadd.statistic = 'MEANCLIP' 

1292 self.assembleMeanCoadd.doWrite = False 

1293 self.assembleMeanClipCoadd.doWrite = False 

1294 

1295 def validate(self): 

1296 if self.doSigmaClip: 

1297 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1298 "Ignoring doSigmaClip.") 

1299 self.doSigmaClip = False 

1300 if self.statistic != "MEAN": 

1301 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1302 "(%s chosen). Please set statistic to MEAN." 

1303 % (self.statistic)) 

1304 AssembleCoaddTask.ConfigClass.validate(self) 

1305 

1306 

1307class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1308 """Assemble a coadded image from a set of coadded temporary exposures, 

1309 being careful to clip & flag areas with potential artifacts. 

1310 

1311 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1312 we clip outliers). The problem with doing this is that when computing the 

1313 coadd PSF at a given location, individual visit PSFs from visits with 

1314 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1315 In this task, we correct for this behavior by creating a new 

1316 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1317 coaddTempExps and the final coadd where 

1318 

1319 i. difference imaging suggests that there is an outlier and 

1320 ii. this outlier appears on only one or two images. 

1321 

1322 Such regions will not contribute to the final coadd. Furthermore, any 

1323 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1324 Note that the algorithm implemented by this task is preliminary and works 

1325 correctly for HSC data. Parameter modifications and or considerable 

1326 redesigning of the algorithm is likley required for other surveys. 

1327 

1328 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1329 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1330 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1331 if you wish. 

1332 

1333 Notes 

1334 ----- 

1335 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1336 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1337 see `baseDebug` for more about ``debug.py`` files. 

1338 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1339 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1340 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1341 for further information. 

1342 

1343 Examples 

1344 -------- 

1345 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1346 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1347 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1348 

1349 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1350 and filter to be coadded (specified using 

1351 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1352 along with a list of coaddTempExps to attempt to coadd (specified using 

1353 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1354 Only the coaddTempExps that cover the specified tract and patch will be 

1355 coadded. A list of the available optional arguments can be obtained by 

1356 calling assembleCoadd.py with the --help command line argument: 

1357 

1358 .. code-block:: none 

1359 

1360 assembleCoadd.py --help 

1361 

1362 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1363 context of multi-band processing, we will generate the HSC-I & -R band 

1364 coadds from HSC engineering test data provided in the ci_hsc package. 

1365 To begin, assuming that the lsst stack has been already set up, we must 

1366 set up the obs_subaru and ci_hsc packages. This defines the environment 

1367 variable $CI_HSC_DIR and points at the location of the package. The raw 

1368 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1369 the coadds, we must first 

1370 

1371 - ``processCcd`` 

1372 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1373 - ``makeSkyMap`` 

1374 create a skymap that covers the area of the sky present in the raw exposures 

1375 - ``makeCoaddTempExp`` 

1376 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1377 

1378 We can perform all of these steps by running 

1379 

1380 .. code-block:: none 

1381 

1382 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1383 

1384 This will produce warped coaddTempExps for each visit. To coadd the 

1385 warped data, we call ``assembleCoadd.py`` as follows: 

1386 

1387 .. code-block:: none 

1388 

1389 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1390 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1391 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1392 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1393 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1394 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1395 --selectId visit=903988 ccd=24 

1396 

1397 This will process the HSC-I band data. The results are written in 

1398 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1399 

1400 You may also choose to run: 

1401 

1402 .. code-block:: none 

1403 

1404 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1405 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1406 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1407 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1408 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1409 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1410 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1411 --selectId visit=903346 ccd=12 

1412 

1413 to generate the coadd for the HSC-R band if you are interested in following 

1414 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1415 """ 

1416 ConfigClass = SafeClipAssembleCoaddConfig 

1417 _DefaultName = "safeClipAssembleCoadd" 

1418 

1419 def __init__(self, *args, **kwargs): 

1420 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1421 schema = afwTable.SourceTable.makeMinimalSchema() 

1422 self.makeSubtask("clipDetection", schema=schema) 

1423 self.makeSubtask("assembleMeanClipCoadd") 

1424 self.makeSubtask("assembleMeanCoadd") 

1425 

1426 @utils.inheritDoc(AssembleCoaddTask) 

1427 @pipeBase.timeMethod 

1428 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1429 """Assemble the coadd for a region. 

1430 

1431 Compute the difference of coadds created with and without outlier 

1432 rejection to identify coadd pixels that have outlier values in some 

1433 individual visits. 

1434 Detect clipped regions on the difference image and mark these regions 

1435 on the one or two individual coaddTempExps where they occur if there 

1436 is significant overlap between the clipped region and a source. This 

1437 leaves us with a set of footprints from the difference image that have 

1438 been identified as having occured on just one or two individual visits. 

1439 However, these footprints were generated from a difference image. It 

1440 is conceivable for a large diffuse source to have become broken up 

1441 into multiple footprints acrosss the coadd difference in this process. 

1442 Determine the clipped region from all overlapping footprints from the 

1443 detected sources in each visit - these are big footprints. 

1444 Combine the small and big clipped footprints and mark them on a new 

1445 bad mask plane. 

1446 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1447 removal. Clipped footprints will no longer make it into the coadd 

1448 because they are marked in the new bad mask plane. 

1449 

1450 Notes 

1451 ----- 

1452 args and kwargs are passed but ignored in order to match the call 

1453 signature expected by the parent task. 

1454 """ 

1455 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1456 mask = exp.getMaskedImage().getMask() 

1457 mask.addMaskPlane("CLIPPED") 

1458 

1459 result = self.detectClip(exp, tempExpRefList) 

1460 

1461 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1462 

1463 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1464 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1465 # Append big footprints from individual Warps to result.clipSpans 

1466 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1467 result.detectionFootprints, maskClipValue, maskDetValue, 

1468 exp.getBBox()) 

1469 # Create mask of the current clipped footprints 

1470 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1471 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1472 

1473 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1474 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1475 maskClip |= maskClipBig 

1476 

1477 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1478 badMaskPlanes = self.config.badMaskPlanes[:] 

1479 badMaskPlanes.append("CLIPPED") 

1480 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1481 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1482 result.clipSpans, mask=badPixelMask) 

1483 

1484 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1485 """Return an exposure that contains the difference between unclipped 

1486 and clipped coadds. 

1487 

1488 Generate a difference image between clipped and unclipped coadds. 

1489 Compute the difference image by subtracting an outlier-clipped coadd 

1490 from an outlier-unclipped coadd. Return the difference image. 

1491 

1492 Parameters 

1493 ---------- 

1494 skyInfo : `lsst.pipe.base.Struct` 

1495 Patch geometry information, from getSkyInfo 

1496 tempExpRefList : `list` 

1497 List of data reference to tempExp 

1498 imageScalerList : `list` 

1499 List of image scalers 

1500 weightList : `list` 

1501 List of weights 

1502 

1503 Returns 

1504 ------- 

1505 exp : `lsst.afw.image.Exposure` 

1506 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1507 """ 

1508 coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList, 

1509 imageScalerList, weightList).coaddExposure 

1510 

1511 coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList, 

1512 imageScalerList, weightList).coaddExposure 

1513 

1514 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1515 coaddDiff -= coaddClip.getMaskedImage() 

1516 exp = afwImage.ExposureF(coaddDiff) 

1517 exp.setPsf(coaddMean.getPsf()) 

1518 return exp 

1519 

1520 def detectClip(self, exp, tempExpRefList): 

1521 """Detect clipped regions on an exposure and set the mask on the 

1522 individual tempExp masks. 

1523 

1524 Detect footprints in the difference image after smoothing the 

1525 difference image with a Gaussian kernal. Identify footprints that 

1526 overlap with one or two input ``coaddTempExps`` by comparing the 

1527 computed overlap fraction to thresholds set in the config. A different 

1528 threshold is applied depending on the number of overlapping visits 

1529 (restricted to one or two). If the overlap exceeds the thresholds, 

1530 the footprint is considered "CLIPPED" and is marked as such on the 

1531 coaddTempExp. Return a struct with the clipped footprints, the indices 

1532 of the ``coaddTempExps`` that end up overlapping with the clipped 

1533 footprints, and a list of new masks for the ``coaddTempExps``. 

1534 

1535 Parameters 

1536 ---------- 

1537 exp : `lsst.afw.image.Exposure` 

1538 Exposure to run detection on. 

1539 tempExpRefList : `list` 

1540 List of data reference to tempExp. 

1541 

1542 Returns 

1543 ------- 

1544 result : `lsst.pipe.base.Struct` 

1545 Result struct with components: 

1546 

1547 - ``clipFootprints``: list of clipped footprints. 

1548 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1549 ``tempExpRefList``. 

1550 - ``clipSpans``: List of dictionaries containing spanSet lists 

1551 to clip. Each element contains the new maskplane name 

1552 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1553 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1554 compressed into footprints. 

1555 """ 

1556 mask = exp.getMaskedImage().getMask() 

1557 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1558 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1559 # Merge positive and negative together footprints together 

1560 fpSet.positive.merge(fpSet.negative) 

1561 footprints = fpSet.positive 

1562 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1563 ignoreMask = self.getBadPixelMask() 

1564 

1565 clipFootprints = [] 

1566 clipIndices = [] 

1567 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1568 

1569 # for use by detectClipBig 

1570 visitDetectionFootprints = [] 

1571 

1572 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1573 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1574 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1575 

1576 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1577 for i, warpRef in enumerate(tempExpRefList): 

1578 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1579 immediate=True).getMaskedImage().getMask() 

1580 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1581 afwImage.PARENT, True) 

1582 maskVisitDet &= maskDetValue 

1583 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1584 visitDetectionFootprints.append(visitFootprints) 

1585 

1586 for j, footprint in enumerate(footprints.getFootprints()): 

1587 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1588 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1589 

1590 # build a list of clipped spans for each visit 

1591 for j, footprint in enumerate(footprints.getFootprints()): 

1592 nPixel = footprint.getArea() 

1593 overlap = [] # hold the overlap with each visit 

1594 indexList = [] # index of visit in global list 

1595 for i in range(len(tempExpRefList)): 

1596 ignore = ignoreArr[i, j] 

1597 overlapDet = overlapDetArr[i, j] 

1598 totPixel = nPixel - ignore 

1599 

1600 # If we have more bad pixels than detection skip 

1601 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 

1602 continue 

1603 overlap.append(overlapDet/float(totPixel)) 

1604 indexList.append(i) 

1605 

1606 overlap = numpy.array(overlap) 

1607 if not len(overlap): 

1608 continue 

1609 

1610 keep = False # Should this footprint be marked as clipped? 

1611 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1612 

1613 # If footprint only has one overlap use a lower threshold 

1614 if len(overlap) == 1: 

1615 if overlap[0] > self.config.minClipFootOverlapSingle: 

1616 keep = True 

1617 keepIndex = [0] 

1618 else: 

1619 # This is the general case where only visit should be clipped 

1620 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1621 if len(clipIndex) == 1: 

1622 keep = True 

1623 keepIndex = [clipIndex[0]] 

1624 

1625 # Test if there are clipped objects that overlap two different visits 

1626 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1627 if len(clipIndex) == 2 and len(overlap) > 3: 

1628 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1629 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1630 keep = True 

1631 keepIndex = clipIndex 

1632 

1633 if not keep: 

1634 continue 

1635 

1636 for index in keepIndex: 

1637 globalIndex = indexList[index] 

1638 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1639 

1640 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1641 clipFootprints.append(footprint) 

1642 

1643 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1644 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1645 

1646 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1647 maskClipValue, maskDetValue, coaddBBox): 

1648 """Return individual warp footprints for large artifacts and append 

1649 them to ``clipList`` in place. 

1650 

1651 Identify big footprints composed of many sources in the coadd 

1652 difference that may have originated in a large diffuse source in the 

1653 coadd. We do this by indentifying all clipped footprints that overlap 

1654 significantly with each source in all the coaddTempExps. 

1655 

1656 Parameters 

1657 ---------- 

1658 clipList : `list` 

1659 List of alt mask SpanSets with clipping information. Modified. 

1660 clipFootprints : `list` 

1661 List of clipped footprints. 

1662 clipIndices : `list` 

1663 List of which entries in tempExpClipList each footprint belongs to. 

1664 maskClipValue 

1665 Mask value of clipped pixels. 

1666 maskDetValue 

1667 Mask value of detected pixels. 

1668 coaddBBox : `lsst.geom.Box` 

1669 BBox of the coadd and warps. 

1670 

1671 Returns 

1672 ------- 

1673 bigFootprintsCoadd : `list` 

1674 List of big footprints 

1675 """ 

1676 bigFootprintsCoadd = [] 

1677 ignoreMask = self.getBadPixelMask() 

1678 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1679 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1680 for footprint in visitFootprints.getFootprints(): 

1681 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1682 

1683 # build a mask of clipped footprints that are in this visit 

1684 clippedFootprintsVisit = [] 

1685 for foot, clipIndex in zip(clipFootprints, clipIndices): 

1686 if index not in clipIndex: 

1687 continue 

1688 clippedFootprintsVisit.append(foot) 

1689 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1690 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1691 

1692 bigFootprintsVisit = [] 

1693 for foot in visitFootprints.getFootprints(): 

1694 if foot.getArea() < self.config.minBigOverlap: 

1695 continue 

1696 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1697 if nCount > self.config.minBigOverlap: 

1698 bigFootprintsVisit.append(foot) 

1699 bigFootprintsCoadd.append(foot) 

1700 

1701 for footprint in bigFootprintsVisit: 

1702 clippedSpans["CLIPPED"].append(footprint.spans) 

1703 

1704 return bigFootprintsCoadd 

1705 

1706 

1707class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1708 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1709 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1710 "Only PSF-Matched Warps make sense for image subtraction. " 

1711 "Therefore, they must be an additional declared input."), 

1712 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1713 storageClass="ExposureF", 

1714 dimensions=("tract", "patch", "skymap", "visit"), 

1715 deferLoad=True, 

1716 multiple=True 

1717 ) 

1718 templateCoadd = pipeBase.connectionTypes.Output( 

1719 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1720 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1721 name="{fakesType}{outputCoaddName}CoaddPsfMatched", 

1722 storageClass="ExposureF", 

1723 dimensions=("tract", "patch", "skymap", "abstract_filter"), 

1724 ) 

1725 

1726 def __init__(self, *, config=None): 

1727 super().__init__(config=config) 

1728 if not config.assembleStaticSkyModel.doWrite: 

1729 self.outputs.remove("templateCoadd") 

1730 config.validate() 

1731 

1732 

1733class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1734 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1735 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1736 target=AssembleCoaddTask, 

1737 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1738 " naive/first-iteration model of the static sky.", 

1739 ) 

1740 detect = pexConfig.ConfigurableField( 

1741 target=SourceDetectionTask, 

1742 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1743 ) 

1744 detectTemplate = pexConfig.ConfigurableField( 

1745 target=SourceDetectionTask, 

1746 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1747 ) 

1748 maskStreaks = pexConfig.ConfigurableField( 

1749 target=MaskStreaksTask, 

1750 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1751 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1752 "streakMaskName" 

1753 ) 

1754 streakMaskName = pexConfig.Field( 

1755 dtype=str, 

1756 default="STREAK", 

1757 doc="Name of mask bit used for streaks" 

1758 ) 

1759 maxNumEpochs = pexConfig.Field( 

1760 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1761 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1762 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1763 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1764 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1765 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1766 "than transient and not masked.", 

1767 dtype=int, 

1768 default=2 

1769 ) 

1770 maxFractionEpochsLow = pexConfig.RangeField( 

1771 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1772 "Effective maxNumEpochs = " 

1773 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1774 dtype=float, 

1775 default=0.4, 

1776 min=0., max=1., 

1777 ) 

1778 maxFractionEpochsHigh = pexConfig.RangeField( 

1779 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1780 "Effective maxNumEpochs = " 

1781 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1782 dtype=float, 

1783 default=0.03, 

1784 min=0., max=1., 

1785 ) 

1786 spatialThreshold = pexConfig.RangeField( 

1787 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1788 "temporal criteria. If 0, clip all. If 1, clip none.", 

1789 dtype=float, 

1790 default=0.5, 

1791 min=0., max=1., 

1792 inclusiveMin=True, inclusiveMax=True 

1793 ) 

1794 doScaleWarpVariance = pexConfig.Field( 

1795 doc="Rescale Warp variance plane using empirical noise?", 

1796 dtype=bool, 

1797 default=True, 

1798 ) 

1799 scaleWarpVariance = pexConfig.ConfigurableField( 

1800 target=ScaleVarianceTask, 

1801 doc="Rescale variance on warps", 

1802 ) 

1803 doPreserveContainedBySource = pexConfig.Field( 

1804 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1805 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1806 dtype=bool, 

1807 default=True, 

1808 ) 

1809 doPrefilterArtifacts = pexConfig.Field( 

1810 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1811 "because they will be excluded anyway. This prevents them from contributing " 

1812 "to the outlier epoch count image and potentially being labeled as persistant." 

1813 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1814 dtype=bool, 

1815 default=True 

1816 ) 

1817 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1818 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1819 dtype=str, 

1820 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1821 ) 

1822 prefilterArtifactsRatio = pexConfig.Field( 

1823 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1824 dtype=float, 

1825 default=0.05 

1826 ) 

1827 doFilterMorphological = pexConfig.Field( 

1828 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1829 "be streaks.", 

1830 dtype=bool, 

1831 default=False 

1832 ) 

1833 

1834 def setDefaults(self): 

1835 AssembleCoaddConfig.setDefaults(self) 

1836 self.statistic = 'MEAN' 

1837 self.doUsePsfMatchedPolygons = True 

1838 

1839 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1840 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1841 if "EDGE" in self.badMaskPlanes: 

1842 self.badMaskPlanes.remove('EDGE') 

1843 self.removeMaskPlanes.append('EDGE') 

1844 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1845 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1846 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1847 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1848 self.assembleStaticSkyModel.sigmaClip = 2.5 

1849 self.assembleStaticSkyModel.clipIter = 3 

1850 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1851 self.assembleStaticSkyModel.doWrite = False 

1852 self.detect.doTempLocalBackground = False 

1853 self.detect.reEstimateBackground = False 

1854 self.detect.returnOriginalFootprints = False 

1855 self.detect.thresholdPolarity = "both" 

1856 self.detect.thresholdValue = 5 

1857 self.detect.minPixels = 4 

1858 self.detect.isotropicGrow = True 

1859 self.detect.thresholdType = "pixel_stdev" 

1860 self.detect.nSigmaToGrow = 0.4 

1861 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1862 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1863 self.detectTemplate.nSigmaToGrow = 2.4 

1864 self.detectTemplate.doTempLocalBackground = False 

1865 self.detectTemplate.reEstimateBackground = False 

1866 self.detectTemplate.returnOriginalFootprints = False 

1867 

1868 def validate(self): 

1869 super().validate() 

1870 if self.assembleStaticSkyModel.doNImage: 

1871 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1872 "Please set assembleStaticSkyModel.doNImage=False") 

1873 

1874 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1875 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1876 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1877 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1878 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1879 

1880 

1881class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1882 """Assemble a compareWarp coadded image from a set of warps 

1883 by masking artifacts detected by comparing PSF-matched warps. 

1884 

1885 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1886 we clip outliers). The problem with doing this is that when computing the 

1887 coadd PSF at a given location, individual visit PSFs from visits with 

1888 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1889 In this task, we correct for this behavior by creating a new badMaskPlane 

1890 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1891 an artifact. We populate this plane on the input warps by comparing 

1892 PSF-matched warps with a PSF-matched median coadd which serves as a 

1893 model of the static sky. Any group of pixels that deviates from the 

1894 PSF-matched template coadd by more than config.detect.threshold sigma, 

1895 is an artifact candidate. The candidates are then filtered to remove 

1896 variable sources and sources that are difficult to subtract such as 

1897 bright stars. This filter is configured using the config parameters 

1898 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1899 the maximum fraction of epochs that the deviation can appear in and still 

1900 be considered an artifact. The spatialThreshold is the maximum fraction of 

1901 pixels in the footprint of the deviation that appear in other epochs 

1902 (where other epochs is defined by the temporalThreshold). If the deviant 

1903 region meets this criteria of having a significant percentage of pixels 

1904 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1905 set in the mask. These regions will not contribute to the final coadd. 

1906 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1907 of clipped regions. Note that the algorithm implemented by this task is 

1908 preliminary and works correctly for HSC data. Parameter modifications and 

1909 or considerable redesigning of the algorithm is likley required for other 

1910 surveys. 

1911 

1912 ``CompareWarpAssembleCoaddTask`` sub-classes 

1913 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1914 as a subtask to generate the TemplateCoadd (the model of the static sky). 

1915 

1916 Notes 

1917 ----- 

1918 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1919 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

1920 ``baseDebug`` for more about ``debug.py`` files. 

1921 

1922 This task supports the following debug variables: 

1923 

1924 - ``saveCountIm`` 

1925 If True then save the Epoch Count Image as a fits file in the `figPath` 

1926 - ``figPath`` 

1927 Path to save the debug fits images and figures 

1928 

1929 For example, put something like: 

1930 

1931 .. code-block:: python 

1932 

1933 import lsstDebug 

1934 def DebugInfo(name): 

1935 di = lsstDebug.getInfo(name) 

1936 if name == "lsst.pipe.tasks.assembleCoadd": 

1937 di.saveCountIm = True 

1938 di.figPath = "/desired/path/to/debugging/output/images" 

1939 return di 

1940 lsstDebug.Info = DebugInfo 

1941 

1942 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

1943 ``--debug`` flag. Some subtasks may have their own debug variables; 

1944 see individual Task documentation. 

1945 

1946 Examples 

1947 -------- 

1948 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

1949 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

1950 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

1951 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1952 and filter to be coadded (specified using 

1953 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1954 along with a list of coaddTempExps to attempt to coadd (specified using 

1955 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1956 Only the warps that cover the specified tract and patch will be coadded. 

1957 A list of the available optional arguments can be obtained by calling 

1958 ``assembleCoadd.py`` with the ``--help`` command line argument: 

1959 

1960 .. code-block:: none 

1961 

1962 assembleCoadd.py --help 

1963 

1964 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

1965 context of multi-band processing, we will generate the HSC-I & -R band 

1966 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

1967 To begin, assuming that the lsst stack has been already set up, we must 

1968 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

1969 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

1970 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

1971 directory. To begin assembling the coadds, we must first 

1972 

1973 - processCcd 

1974 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1975 - makeSkyMap 

1976 create a skymap that covers the area of the sky present in the raw exposures 

1977 - makeCoaddTempExp 

1978 warp the individual calibrated exposures to the tangent plane of the coadd 

1979 

1980 We can perform all of these steps by running 

1981 

1982 .. code-block:: none 

1983 

1984 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1985 

1986 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

1987 warped data, we call ``assembleCoadd.py`` as follows: 

1988 

1989 .. code-block:: none 

1990 

1991 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1992 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1993 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1994 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1995 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1996 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1997 --selectId visit=903988 ccd=24 

1998 

1999 This will process the HSC-I band data. The results are written in 

2000 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2001 """ 

2002 ConfigClass = CompareWarpAssembleCoaddConfig 

2003 _DefaultName = "compareWarpAssembleCoadd" 

2004 

2005 def __init__(self, *args, **kwargs): 

2006 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2007 self.makeSubtask("assembleStaticSkyModel") 

2008 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2009 self.makeSubtask("detect", schema=detectionSchema) 

2010 if self.config.doPreserveContainedBySource: 

2011 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2012 if self.config.doScaleWarpVariance: 

2013 self.makeSubtask("scaleWarpVariance") 

2014 if self.config.doFilterMorphological: 

2015 self.makeSubtask("maskStreaks") 

2016 

2017 @utils.inheritDoc(AssembleCoaddTask) 

2018 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2019 """ 

2020 Generate a templateCoadd to use as a naive model of static sky to 

2021 subtract from PSF-Matched warps. 

2022 

2023 Returns 

2024 ------- 

2025 result : `lsst.pipe.base.Struct` 

2026 Result struct with components: 

2027 

2028 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2029 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2030 """ 

2031 # Ensure that psfMatchedWarps are used as input warps for template generation 

2032 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2033 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2034 

2035 # Because subtasks don't have connections we have to make one. 

2036 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2037 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2038 if self.config.assembleStaticSkyModel.doWrite: 

2039 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2040 # Remove template coadd from both subtask's and main tasks outputs, 

2041 # because it is handled by the subtask as `coaddExposure` 

2042 del outputRefs.templateCoadd 

2043 del staticSkyModelOutputRefs.templateCoadd 

2044 

2045 # A PSF-Matched nImage does not exist as a dataset type 

2046 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2047 del staticSkyModelOutputRefs.nImage 

2048 

2049 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2050 staticSkyModelOutputRefs) 

2051 if templateCoadd is None: 

2052 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2053 

2054 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2055 nImage=templateCoadd.nImage, 

2056 warpRefList=templateCoadd.warpRefList, 

2057 imageScalerList=templateCoadd.imageScalerList, 

2058 weightList=templateCoadd.weightList) 

2059 

2060 @utils.inheritDoc(AssembleCoaddTask) 

2061 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2062 """ 

2063 Generate a templateCoadd to use as a naive model of static sky to 

2064 subtract from PSF-Matched warps. 

2065 

2066 Returns 

2067 ------- 

2068 result : `lsst.pipe.base.Struct` 

2069 Result struct with components: 

2070 

2071 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2072 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2073 """ 

2074 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2075 if templateCoadd is None: 

2076 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2077 

2078 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2079 nImage=templateCoadd.nImage, 

2080 warpRefList=templateCoadd.warpRefList, 

2081 imageScalerList=templateCoadd.imageScalerList, 

2082 weightList=templateCoadd.weightList) 

2083 

2084 def _noTemplateMessage(self, warpType): 

2085 warpName = (warpType[0].upper() + warpType[1:]) 

2086 message = """No %(warpName)s warps were found to build the template coadd which is 

2087 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2088 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2089 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2090 

2091 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2092 another algorithm like: 

2093 

2094 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2095 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2096 """ % {"warpName": warpName} 

2097 return message 

2098 

2099 @utils.inheritDoc(AssembleCoaddTask) 

2100 @pipeBase.timeMethod 

2101 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2102 supplementaryData, *args, **kwargs): 

2103 """Assemble the coadd. 

2104 

2105 Find artifacts and apply them to the warps' masks creating a list of 

2106 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2107 plane. Then pass these alternative masks to the base class's `run` 

2108 method. 

2109 

2110 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2111 that must contain a ``templateCoadd`` that serves as the 

2112 model of the static sky. 

2113 """ 

2114 

2115 # Check and match the order of the supplementaryData 

2116 # (PSF-matched) inputs to the order of the direct inputs, 

2117 # so that the artifact mask is applied to the right warp 

2118 dataIds = [ref.dataId for ref in tempExpRefList] 

2119 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2120 

2121 if dataIds != psfMatchedDataIds: 

2122 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2123 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2124 psfMatchedDataIds, dataIds) 

2125 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2126 psfMatchedDataIds, dataIds) 

2127 

2128 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2129 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2130 supplementaryData.warpRefList, 

2131 supplementaryData.imageScalerList) 

2132 

2133 badMaskPlanes = self.config.badMaskPlanes[:] 

2134 badMaskPlanes.append("CLIPPED") 

2135 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2136 

2137 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2138 spanSetMaskList, mask=badPixelMask) 

2139 

2140 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2141 # Psf-Matching moves the real edge inwards 

2142 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2143 return result 

2144 

2145 def applyAltEdgeMask(self, mask, altMaskList): 

2146 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2147 

2148 Parameters 

2149 ---------- 

2150 mask : `lsst.afw.image.Mask` 

2151 Original mask. 

2152 altMaskList : `list` 

2153 List of Dicts containing ``spanSet`` lists. 

2154 Each element contains the new mask plane name (e.g. "CLIPPED 

2155 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2156 the mask. 

2157 """ 

2158 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2159 for visitMask in altMaskList: 

2160 if "EDGE" in visitMask: 

2161 for spanSet in visitMask['EDGE']: 

2162 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2163 

2164 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2165 """Find artifacts. 

2166 

2167 Loop through warps twice. The first loop builds a map with the count 

2168 of how many epochs each pixel deviates from the templateCoadd by more 

2169 than ``config.chiThreshold`` sigma. The second loop takes each 

2170 difference image and filters the artifacts detected in each using 

2171 count map to filter out variable sources and sources that are 

2172 difficult to subtract cleanly. 

2173 

2174 Parameters 

2175 ---------- 

2176 templateCoadd : `lsst.afw.image.Exposure` 

2177 Exposure to serve as model of static sky. 

2178 tempExpRefList : `list` 

2179 List of data references to warps. 

2180 imageScalerList : `list` 

2181 List of image scalers. 

2182 

2183 Returns 

2184 ------- 

2185 altMasks : `list` 

2186 List of dicts containing information about CLIPPED 

2187 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2188 """ 

2189 

2190 self.log.debug("Generating Count Image, and mask lists.") 

2191 coaddBBox = templateCoadd.getBBox() 

2192 slateIm = afwImage.ImageU(coaddBBox) 

2193 epochCountImage = afwImage.ImageU(coaddBBox) 

2194 nImage = afwImage.ImageU(coaddBBox) 

2195 spanSetArtifactList = [] 

2196 spanSetNoDataMaskList = [] 

2197 spanSetEdgeList = [] 

2198 spanSetBadMorphoList = [] 

2199 badPixelMask = self.getBadPixelMask() 

2200 

2201 # mask of the warp diffs should = that of only the warp 

2202 templateCoadd.mask.clearAllMaskPlanes() 

2203 

2204 if self.config.doPreserveContainedBySource: 

2205 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2206 else: 

2207 templateFootprints = None 

2208 

2209 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2210 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2211 if warpDiffExp is not None: 

2212 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2213 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2214 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2215 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2216 fpSet.positive.merge(fpSet.negative) 

2217 footprints = fpSet.positive 

2218 slateIm.set(0) 

2219 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2220 

2221 # Remove artifacts due to defects before they contribute to the epochCountImage 

2222 if self.config.doPrefilterArtifacts: 

2223 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2224 

2225 # Clear mask before adding prefiltered spanSets 

2226 self.detect.clearMask(warpDiffExp.mask) 

2227 for spans in spanSetList: 

2228 spans.setImage(slateIm, 1, doClip=True) 

2229 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2230 epochCountImage += slateIm 

2231 

2232 if self.config.doFilterMorphological: 

2233 maskName = self.config.streakMaskName 

2234 _ = self.maskStreaks.run(warpDiffExp) 

2235 streakMask = warpDiffExp.mask 

2236 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2237 streakMask.getPlaneBitMask(maskName)).split() 

2238 

2239 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2240 # undergo a second convolution. Pixels with data in the direct warp 

2241 # but not in the PSF-matched warp will not have their artifacts detected. 

2242 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2243 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2244 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2245 nansMask.setXY0(warpDiffExp.getXY0()) 

2246 edgeMask = warpDiffExp.mask 

2247 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2248 edgeMask.getPlaneBitMask("EDGE")).split() 

2249 else: 

2250 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2251 # In this case, mask the whole epoch 

2252 nansMask = afwImage.MaskX(coaddBBox, 1) 

2253 spanSetList = [] 

2254 spanSetEdgeMask = [] 

2255 spanSetStreak = [] 

2256 

2257 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2258 

2259 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2260 spanSetArtifactList.append(spanSetList) 

2261 spanSetEdgeList.append(spanSetEdgeMask) 

2262 if self.config.doFilterMorphological: 

2263 spanSetBadMorphoList.append(spanSetStreak) 

2264 

2265 if lsstDebug.Info(__name__).saveCountIm: 

2266 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2267 epochCountImage.writeFits(path) 

2268 

2269 for i, spanSetList in enumerate(spanSetArtifactList): 

2270 if spanSetList: 

2271 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2272 templateFootprints) 

2273 spanSetArtifactList[i] = filteredSpanSetList 

2274 if self.config.doFilterMorphological: 

2275 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2276 

2277 altMasks = [] 

2278 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2279 altMasks.append({'CLIPPED': artifacts, 

2280 'NO_DATA': noData, 

2281 'EDGE': edge}) 

2282 return altMasks 

2283 

2284 def prefilterArtifacts(self, spanSetList, exp): 

2285 """Remove artifact candidates covered by bad mask plane. 

2286 

2287 Any future editing of the candidate list that does not depend on 

2288 temporal information should go in this method. 

2289 

2290 Parameters 

2291 ---------- 

2292 spanSetList : `list` 

2293 List of SpanSets representing artifact candidates. 

2294 exp : `lsst.afw.image.Exposure` 

2295 Exposure containing mask planes used to prefilter. 

2296 

2297 Returns 

2298 ------- 

2299 returnSpanSetList : `list` 

2300 List of SpanSets with artifacts. 

2301 """ 

2302 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2303 goodArr = (exp.mask.array & badPixelMask) == 0 

2304 returnSpanSetList = [] 

2305 bbox = exp.getBBox() 

2306 x0, y0 = exp.getXY0() 

2307 for i, span in enumerate(spanSetList): 

2308 y, x = span.clippedTo(bbox).indices() 

2309 yIndexLocal = numpy.array(y) - y0 

2310 xIndexLocal = numpy.array(x) - x0 

2311 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2312 if goodRatio > self.config.prefilterArtifactsRatio: 

2313 returnSpanSetList.append(span) 

2314 return returnSpanSetList 

2315 

2316 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2317 """Filter artifact candidates. 

2318 

2319 Parameters 

2320 ---------- 

2321 spanSetList : `list` 

2322 List of SpanSets representing artifact candidates. 

2323 epochCountImage : `lsst.afw.image.Image` 

2324 Image of accumulated number of warpDiff detections. 

2325 nImage : `lsst.afw.image.Image` 

2326 Image of the accumulated number of total epochs contributing. 

2327 

2328 Returns 

2329 ------- 

2330 maskSpanSetList : `list` 

2331 List of SpanSets with artifacts. 

2332 """ 

2333 

2334 maskSpanSetList = [] 

2335 x0, y0 = epochCountImage.getXY0() 

2336 for i, span in enumerate(spanSetList): 

2337 y, x = span.indices() 

2338 yIdxLocal = [y1 - y0 for y1 in y] 

2339 xIdxLocal = [x1 - x0 for x1 in x] 

2340 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2341 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2342 

2343 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2344 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2345 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2346 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2347 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2348 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2349 & (outlierN <= effectiveMaxNumEpochs)) 

2350 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2351 if percentBelowThreshold > self.config.spatialThreshold: 

2352 maskSpanSetList.append(span) 

2353 

2354 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

2355 # If a candidate is contained by a footprint on the template coadd, do not clip 

2356 filteredMaskSpanSetList = [] 

2357 for span in maskSpanSetList: 

2358 doKeep = True 

2359 for footprint in footprintsToExclude.positive.getFootprints(): 

2360 if footprint.spans.contains(span): 

2361 doKeep = False 

2362 break 

2363 if doKeep: 

2364 filteredMaskSpanSetList.append(span) 

2365 maskSpanSetList = filteredMaskSpanSetList 

2366 

2367 return maskSpanSetList 

2368 

2369 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2370 """Fetch a warp from the butler and return a warpDiff. 

2371 

2372 Parameters 

2373 ---------- 

2374 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2375 Butler dataRef for the warp. 

2376 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2377 An image scaler object. 

2378 templateCoadd : `lsst.afw.image.Exposure` 

2379 Exposure to be substracted from the scaled warp. 

2380 

2381 Returns 

2382 ------- 

2383 warp : `lsst.afw.image.Exposure` 

2384 Exposure of the image difference between the warp and template. 

2385 """ 

2386 

2387 # If the PSF-Matched warp did not exist for this direct warp 

2388 # None is holding its place to maintain order in Gen 3 

2389 if warpRef is None: 

2390 return None 

2391 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2392 warpName = self.getTempExpDatasetName('psfMatched') 

2393 if not isinstance(warpRef, DeferredDatasetHandle): 

2394 if not warpRef.datasetExists(warpName): 

2395 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2396 return None 

2397 warp = warpRef.get(datasetType=warpName, immediate=True) 

2398 # direct image scaler OK for PSF-matched Warp 

2399 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2400 mi = warp.getMaskedImage() 

2401 if self.config.doScaleWarpVariance: 

2402 try: 

2403 self.scaleWarpVariance.run(mi) 

2404 except Exception as exc: 

2405 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2406 mi -= templateCoadd.getMaskedImage() 

2407 return warp 

2408 

2409 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2410 """Return a path to which to write debugging output. 

2411 

2412 Creates a hyphen-delimited string of dataId values for simple filenames. 

2413 

2414 Parameters 

2415 ---------- 

2416 prefix : `str` 

2417 Prefix for filename. 

2418 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2419 Butler dataRef to make the path from. 

2420 coaddLevel : `bool`, optional. 

2421 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2422 'filter', but no 'visit'). 

2423 

2424 Returns 

2425 ------- 

2426 result : `str` 

2427 Path for debugging output. 

2428 """ 

2429 if coaddLevel: 

2430 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2431 else: 

2432 keys = warpRef.dataId.keys() 

2433 keyList = sorted(keys, reverse=True) 

2434 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2435 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2436 return os.path.join(directory, filename) 

2437 

2438 

2439def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None): 

2440 """Match the order of one list to another, padding if necessary 

2441 

2442 Parameters 

2443 ---------- 

2444 inputList : list 

2445 List to be reordered and padded. Elements can be any type. 

2446 inputKeys : iterable 

2447 Iterable of values to be compared with outputKeys. 

2448 Length must match `inputList` 

2449 outputKeys : iterable 

2450 Iterable of values to be compared with inputKeys. 

2451 padWith : 

2452 Any value to be inserted where inputKey not in outputKeys 

2453 

2454 Returns 

2455 ------- 

2456 list 

2457 Copy of inputList reordered per outputKeys and padded with `padWith` 

2458 so that the length matches length of outputKeys. 

2459 """ 

2460 outputList = [] 

2461 for d in outputKeys: 

2462 if d in inputKeys: 

2463 outputList.append(inputList[inputKeys.index(d)]) 

2464 else: 

2465 outputList.append(padWith) 

2466 return outputList