Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

893 statements  

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import logging 

27import lsst.pex.config as pexConfig 

28import lsst.pex.exceptions as pexExceptions 

29import lsst.geom as geom 

30import lsst.afw.geom as afwGeom 

31import lsst.afw.image as afwImage 

32import lsst.afw.math as afwMath 

33import lsst.afw.table as afwTable 

34import lsst.afw.detection as afwDet 

35import lsst.coadd.utils as coaddUtils 

36import lsst.pipe.base as pipeBase 

37import lsst.meas.algorithms as measAlg 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from .healSparseMapping import HealSparseInputMapTask 

48from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack 

49from lsst.daf.butler import DeferredDatasetHandle 

50from lsst.utils.timer import timeMethod 

51 

52__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

53 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

54 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

55 

56log = logging.getLogger(__name__) 

57 

58 

59class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("tract", "patch", "band", "skymap"), 

61 defaultTemplates={"inputCoaddName": "deep", 

62 "outputCoaddName": "deep", 

63 "warpType": "direct", 

64 "warpTypeSuffix": ""}): 

65 

66 inputWarps = pipeBase.connectionTypes.Input( 

67 doc=("Input list of warps to be assemebled i.e. stacked." 

68 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

69 name="{inputCoaddName}Coadd_{warpType}Warp", 

70 storageClass="ExposureF", 

71 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

72 deferLoad=True, 

73 multiple=True 

74 ) 

75 skyMap = pipeBase.connectionTypes.Input( 

76 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

78 storageClass="SkyMap", 

79 dimensions=("skymap", ), 

80 ) 

81 selectedVisits = pipeBase.connectionTypes.Input( 

82 doc="Selected visits to be coadded.", 

83 name="{outputCoaddName}Visits", 

84 storageClass="StructuredDataDict", 

85 dimensions=("instrument", "tract", "patch", "skymap", "band") 

86 ) 

87 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

88 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

89 " BRIGHT_OBJECT."), 

90 name="brightObjectMask", 

91 storageClass="ObjectMaskCatalog", 

92 dimensions=("tract", "patch", "skymap", "band"), 

93 ) 

94 coaddExposure = pipeBase.connectionTypes.Output( 

95 doc="Output coadded exposure, produced by stacking input warps", 

96 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

97 storageClass="ExposureF", 

98 dimensions=("tract", "patch", "skymap", "band"), 

99 ) 

100 nImage = pipeBase.connectionTypes.Output( 

101 doc="Output image of number of input images per pixel", 

102 name="{outputCoaddName}Coadd_nImage", 

103 storageClass="ImageU", 

104 dimensions=("tract", "patch", "skymap", "band"), 

105 ) 

106 inputMap = pipeBase.connectionTypes.Output( 

107 doc="Output healsparse map of input images", 

108 name="{outputCoaddName}Coadd_inputMap", 

109 storageClass="HealSparseMap", 

110 dimensions=("tract", "patch", "skymap", "band"), 

111 ) 

112 

113 def __init__(self, *, config=None): 

114 super().__init__(config=config) 

115 

116 # Override the connection's name template with config to replicate Gen2 behavior 

117 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

118 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

119 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

120 templateValues['warpType'] = config.warpType 

121 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

123 for name in self.allConnections} 

124 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

125 # End code to remove after deprecation 

126 

127 if not config.doMaskBrightObjects: 

128 self.prerequisiteInputs.remove("brightObjectMask") 

129 

130 if not config.doSelectVisits: 

131 self.inputs.remove("selectedVisits") 

132 

133 if not config.doNImage: 

134 self.outputs.remove("nImage") 

135 

136 if not self.config.doInputMap: 

137 self.outputs.remove("inputMap") 

138 

139 

140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

141 pipelineConnections=AssembleCoaddConnections): 

142 """Configuration parameters for the `AssembleCoaddTask`. 

143 

144 Notes 

145 ----- 

146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

147 only set the bitplane config.brightObjectMaskName. To make this useful you 

148 *must* also configure the flags.pixel algorithm, for example by adding 

149 

150 .. code-block:: none 

151 

152 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

153 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

154 

155 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

156 """ 

157 warpType = pexConfig.Field( 

158 doc="Warp name: one of 'direct' or 'psfMatched'", 

159 dtype=str, 

160 default="direct", 

161 ) 

162 subregionSize = pexConfig.ListField( 

163 dtype=int, 

164 doc="Width, height of stack subregion size; " 

165 "make small enough that a full stack of images will fit into memory at once.", 

166 length=2, 

167 default=(2000, 2000), 

168 ) 

169 statistic = pexConfig.Field( 

170 dtype=str, 

171 doc="Main stacking statistic for aggregating over the epochs.", 

172 default="MEANCLIP", 

173 ) 

174 doOnlineForMean = pexConfig.Field( 

175 dtype=bool, 

176 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

177 default=False, 

178 ) 

179 doSigmaClip = pexConfig.Field( 

180 dtype=bool, 

181 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

182 default=False, 

183 ) 

184 sigmaClip = pexConfig.Field( 

185 dtype=float, 

186 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

187 default=3.0, 

188 ) 

189 clipIter = pexConfig.Field( 

190 dtype=int, 

191 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

192 default=2, 

193 ) 

194 calcErrorFromInputVariance = pexConfig.Field( 

195 dtype=bool, 

196 doc="Calculate coadd variance from input variance by stacking statistic." 

197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

198 default=True, 

199 ) 

200 scaleZeroPoint = pexConfig.ConfigurableField( 

201 target=ScaleZeroPointTask, 

202 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

203 ) 

204 doInterp = pexConfig.Field( 

205 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

206 dtype=bool, 

207 default=True, 

208 ) 

209 interpImage = pexConfig.ConfigurableField( 

210 target=InterpImageTask, 

211 doc="Task to interpolate (and extrapolate) over NaN pixels", 

212 ) 

213 doWrite = pexConfig.Field( 

214 doc="Persist coadd?", 

215 dtype=bool, 

216 default=True, 

217 ) 

218 doNImage = pexConfig.Field( 

219 doc="Create image of number of contributing exposures for each pixel", 

220 dtype=bool, 

221 default=False, 

222 ) 

223 doUsePsfMatchedPolygons = pexConfig.Field( 

224 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

225 dtype=bool, 

226 default=False, 

227 ) 

228 maskPropagationThresholds = pexConfig.DictField( 

229 keytype=str, 

230 itemtype=float, 

231 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

233 "would have contributed exceeds this value."), 

234 default={"SAT": 0.1}, 

235 ) 

236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

237 doc="Mask planes to remove before coadding") 

238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

239 doc="Set mask and flag bits for bright objects?") 

240 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

241 doc="Name of mask bit used for bright objects") 

242 coaddPsf = pexConfig.ConfigField( 

243 doc="Configuration for CoaddPsf", 

244 dtype=measAlg.CoaddPsfConfig, 

245 ) 

246 doAttachTransmissionCurve = pexConfig.Field( 

247 dtype=bool, default=False, optional=False, 

248 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

249 "(requires all input Exposures to have TransmissionCurves).") 

250 ) 

251 hasFakes = pexConfig.Field( 

252 dtype=bool, 

253 default=False, 

254 doc="Should be set to True if fake sources have been inserted into the input data." 

255 ) 

256 doSelectVisits = pexConfig.Field( 

257 doc="Coadd only visits selected by a SelectVisitsTask", 

258 dtype=bool, 

259 default=False, 

260 ) 

261 doInputMap = pexConfig.Field( 

262 doc="Create a bitwise map of coadd inputs", 

263 dtype=bool, 

264 default=False, 

265 ) 

266 inputMapper = pexConfig.ConfigurableField( 

267 doc="Input map creation subtask.", 

268 target=HealSparseInputMapTask, 

269 ) 

270 

271 def setDefaults(self): 

272 super().setDefaults() 

273 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

274 

275 def validate(self): 

276 super().validate() 

277 if self.doPsfMatch: 

278 # Backwards compatibility. 

279 # Configs do not have loggers 

280 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

281 self.warpType = 'psfMatched' 

282 if self.doSigmaClip and self.statistic != "MEANCLIP": 

283 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

284 self.statistic = "MEANCLIP" 

285 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

286 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

287 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

288 

289 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

290 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

291 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

292 if str(k) not in unstackableStats] 

293 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

294 % (self.statistic, stackableStats)) 

295 

296 

297class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

298 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

299 

300 We want to assemble a coadded image from a set of Warps (also called 

301 coadded temporary exposures or ``coaddTempExps``). 

302 Each input Warp covers a patch on the sky and corresponds to a single 

303 run/visit/exposure of the covered patch. We provide the task with a list 

304 of Warps (``selectDataList``) from which it selects Warps that cover the 

305 specified patch (pointed at by ``dataRef``). 

306 Each Warp that goes into a coadd will typically have an independent 

307 photometric zero-point. Therefore, we must scale each Warp to set it to 

308 a common photometric zeropoint. WarpType may be one of 'direct' or 

309 'psfMatched', and the boolean configs `config.makeDirect` and 

310 `config.makePsfMatched` set which of the warp types will be coadded. 

311 The coadd is computed as a mean with optional outlier rejection. 

312 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

313 Finally, Warps can have bad 'NaN' pixels which received no input from the 

314 source calExps. We interpolate over these bad (NaN) pixels. 

315 

316 `AssembleCoaddTask` uses several sub-tasks. These are 

317 

318 - `ScaleZeroPointTask` 

319 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

320 - `InterpImageTask` 

321 - interpolate across bad pixels (NaN) in the final coadd 

322 

323 You can retarget these subtasks if you wish. 

324 

325 Notes 

326 ----- 

327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

328 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

329 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

330 no debug variables of its own. Some of the subtasks may support debug 

331 variables. See the documentation for the subtasks for further information. 

332 

333 Examples 

334 -------- 

335 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

337 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

338 inputs: a data reference to the tract patch and filter to be coadded, and 

339 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

340 ``--selectId``, respectively: 

341 

342 .. code-block:: none 

343 

344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

346 

347 Only the Warps that cover the specified tract and patch will be coadded. 

348 A list of the available optional arguments can be obtained by calling 

349 ``assembleCoadd.py`` with the ``--help`` command line argument: 

350 

351 .. code-block:: none 

352 

353 assembleCoadd.py --help 

354 

355 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

356 multi-band processing, we will generate the HSC-I & -R band coadds from 

357 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

358 assuming that the lsst stack has been already set up, we must set up the 

359 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

360 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

361 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

362 coadds, we must first 

363 

364 - processCcd 

365 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

366 - makeSkyMap 

367 - create a skymap that covers the area of the sky present in the raw exposures 

368 - makeCoaddTempExp 

369 - warp the individual calibrated exposures to the tangent plane of the coadd 

370 

371 We can perform all of these steps by running 

372 

373 .. code-block:: none 

374 

375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

376 

377 This will produce warped exposures for each visit. To coadd the warped 

378 data, we call assembleCoadd.py as follows: 

379 

380 .. code-block:: none 

381 

382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

388 --selectId visit=903988 ccd=24 

389 

390 that will process the HSC-I band data. The results are written in 

391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

392 

393 You may also choose to run: 

394 

395 .. code-block:: none 

396 

397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

405 

406 to generate the coadd for the HSC-R band if you are interested in 

407 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

408 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

409 rather than `AssembleCoaddTask` to make the coadd. 

410 """ 

411 ConfigClass = AssembleCoaddConfig 

412 _DefaultName = "assembleCoadd" 

413 

414 def __init__(self, *args, **kwargs): 

415 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

416 if args: 

417 argNames = ["config", "name", "parentTask", "log"] 

418 kwargs.update({k: v for k, v in zip(argNames, args)}) 

419 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

420 "PipelineTask will not take positional args" % argNames, FutureWarning) 

421 

422 super().__init__(**kwargs) 

423 self.makeSubtask("interpImage") 

424 self.makeSubtask("scaleZeroPoint") 

425 

426 if self.config.doMaskBrightObjects: 

427 mask = afwImage.Mask() 

428 try: 

429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

430 except pexExceptions.LsstCppException: 

431 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

432 mask.getMaskPlaneDict().keys()) 

433 del mask 

434 

435 if self.config.doInputMap: 

436 self.makeSubtask("inputMapper") 

437 

438 self.warpType = self.config.warpType 

439 

440 @utils.inheritDoc(pipeBase.PipelineTask) 

441 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

442 # Docstring to be formatted with info from PipelineTask.runQuantum 

443 """ 

444 Notes 

445 ----- 

446 Assemble a coadd from a set of Warps. 

447 

448 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

449 Analogous to `runDataRef`, it prepares all the data products to be 

450 passed to `run`, and processes the results before returning a struct 

451 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

452 Therefore, its inputs are accessed subregion by subregion 

453 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

455 correspond to an update in `runDataRef` while both entry points 

456 are used. 

457 """ 

458 inputData = butlerQC.get(inputRefs) 

459 

460 # Construct skyInfo expected by run 

461 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

462 skyMap = inputData["skyMap"] 

463 outputDataId = butlerQC.quantum.dataId 

464 

465 inputData['skyInfo'] = makeSkyInfo(skyMap, 

466 tractId=outputDataId['tract'], 

467 patchId=outputDataId['patch']) 

468 

469 if self.config.doSelectVisits: 

470 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

471 else: 

472 warpRefList = inputData['inputWarps'] 

473 

474 # Perform same middle steps as `runDataRef` does 

475 inputs = self.prepareInputs(warpRefList) 

476 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

477 self.getTempExpDatasetName(self.warpType)) 

478 if len(inputs.tempExpRefList) == 0: 

479 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

480 

481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

482 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

483 inputs.weightList, supplementaryData=supplementaryData) 

484 

485 inputData.setdefault('brightObjectMask', None) 

486 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

487 

488 if self.config.doWrite: 

489 butlerQC.put(retStruct, outputRefs) 

490 return retStruct 

491 

492 @timeMethod 

493 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

494 """Assemble a coadd from a set of Warps. 

495 

496 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

497 Compute weights to be applied to each Warp and 

498 find scalings to match the photometric zeropoint to a reference Warp. 

499 Assemble the Warps using `run`. Interpolate over NaNs and 

500 optionally write the coadd to disk. Return the coadded exposure. 

501 

502 Parameters 

503 ---------- 

504 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

505 Data reference defining the patch for coaddition and the 

506 reference Warp (if ``config.autoReference=False``). 

507 Used to access the following data products: 

508 - ``self.config.coaddName + "Coadd_skyMap"`` 

509 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

510 - ``self.config.coaddName + "Coadd"`` 

511 selectDataList : `list` 

512 List of data references to Calexps. Data to be coadded will be 

513 selected from this list based on overlap with the patch defined 

514 by dataRef, grouped by visit, and converted to a list of data 

515 references to warps. 

516 warpRefList : `list` 

517 List of data references to Warps to be coadded. 

518 Note: `warpRefList` is just the new name for `tempExpRefList`. 

519 

520 Returns 

521 ------- 

522 retStruct : `lsst.pipe.base.Struct` 

523 Result struct with components: 

524 

525 - ``coaddExposure``: coadded exposure (``Exposure``). 

526 - ``nImage``: exposure count image (``Image``). 

527 """ 

528 if selectDataList and warpRefList: 

529 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

530 "and which to use is ambiguous. Please pass only one.") 

531 

532 skyInfo = self.getSkyInfo(dataRef) 

533 if warpRefList is None: 

534 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

535 if len(calExpRefList) == 0: 

536 self.log.warning("No exposures to coadd") 

537 return 

538 self.log.info("Coadding %d exposures", len(calExpRefList)) 

539 

540 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

541 

542 inputData = self.prepareInputs(warpRefList) 

543 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

544 self.getTempExpDatasetName(self.warpType)) 

545 if len(inputData.tempExpRefList) == 0: 

546 self.log.warning("No coadd temporary exposures found") 

547 return 

548 

549 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

550 

551 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

552 inputData.weightList, supplementaryData=supplementaryData) 

553 

554 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

555 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

556 

557 if self.config.doWrite: 

558 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 

559 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

560 else: 

561 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

562 self.log.info("Persisting %s", coaddDatasetName) 

563 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

564 if self.config.doNImage and retStruct.nImage is not None: 

565 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

566 

567 return retStruct 

568 

569 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

570 """Interpolate over missing data and mask bright stars. 

571 

572 Parameters 

573 ---------- 

574 coaddExposure : `lsst.afw.image.Exposure` 

575 The coadded exposure to process. 

576 dataRef : `lsst.daf.persistence.ButlerDataRef` 

577 Butler data reference for supplementary data. 

578 """ 

579 if self.config.doInterp: 

580 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

581 # The variance must be positive; work around for DM-3201. 

582 varArray = coaddExposure.variance.array 

583 with numpy.errstate(invalid="ignore"): 

584 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

585 

586 if self.config.doMaskBrightObjects: 

587 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

588 

589 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

590 """Make additional inputs to run() specific to subclasses (Gen2) 

591 

592 Duplicates interface of `runDataRef` method 

593 Available to be implemented by subclasses only if they need the 

594 coadd dataRef for performing preliminary processing before 

595 assembling the coadd. 

596 

597 Parameters 

598 ---------- 

599 dataRef : `lsst.daf.persistence.ButlerDataRef` 

600 Butler data reference for supplementary data. 

601 selectDataList : `list` (optional) 

602 Optional List of data references to Calexps. 

603 warpRefList : `list` (optional) 

604 Optional List of data references to Warps. 

605 """ 

606 return pipeBase.Struct() 

607 

608 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

609 """Make additional inputs to run() specific to subclasses (Gen3) 

610 

611 Duplicates interface of `runQuantum` method. 

612 Available to be implemented by subclasses only if they need the 

613 coadd dataRef for performing preliminary processing before 

614 assembling the coadd. 

615 

616 Parameters 

617 ---------- 

618 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

619 Gen3 Butler object for fetching additional data products before 

620 running the Task specialized for quantum being processed 

621 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

622 Attributes are the names of the connections describing input dataset types. 

623 Values are DatasetRefs that task consumes for corresponding dataset type. 

624 DataIds are guaranteed to match data objects in ``inputData``. 

625 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

626 Attributes are the names of the connections describing output dataset types. 

627 Values are DatasetRefs that task is to produce 

628 for corresponding dataset type. 

629 """ 

630 return pipeBase.Struct() 

631 

632 def getTempExpRefList(self, patchRef, calExpRefList): 

633 """Generate list data references corresponding to warped exposures 

634 that lie within the patch to be coadded. 

635 

636 Parameters 

637 ---------- 

638 patchRef : `dataRef` 

639 Data reference for patch. 

640 calExpRefList : `list` 

641 List of data references for input calexps. 

642 

643 Returns 

644 ------- 

645 tempExpRefList : `list` 

646 List of Warp/CoaddTempExp data references. 

647 """ 

648 butler = patchRef.getButler() 

649 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

650 self.getTempExpDatasetName(self.warpType)) 

651 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

652 g, groupData.keys) for 

653 g in groupData.groups.keys()] 

654 return tempExpRefList 

655 

656 def prepareInputs(self, refList): 

657 """Prepare the input warps for coaddition by measuring the weight for 

658 each warp and the scaling for the photometric zero point. 

659 

660 Each Warp has its own photometric zeropoint and background variance. 

661 Before coadding these Warps together, compute a scale factor to 

662 normalize the photometric zeropoint and compute the weight for each Warp. 

663 

664 Parameters 

665 ---------- 

666 refList : `list` 

667 List of data references to tempExp 

668 

669 Returns 

670 ------- 

671 result : `lsst.pipe.base.Struct` 

672 Result struct with components: 

673 

674 - ``tempExprefList``: `list` of data references to tempExp. 

675 - ``weightList``: `list` of weightings. 

676 - ``imageScalerList``: `list` of image scalers. 

677 """ 

678 statsCtrl = afwMath.StatisticsControl() 

679 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

680 statsCtrl.setNumIter(self.config.clipIter) 

681 statsCtrl.setAndMask(self.getBadPixelMask()) 

682 statsCtrl.setNanSafe(True) 

683 # compute tempExpRefList: a list of tempExpRef that actually exist 

684 # and weightList: a list of the weight of the associated coadd tempExp 

685 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

686 tempExpRefList = [] 

687 weightList = [] 

688 imageScalerList = [] 

689 tempExpName = self.getTempExpDatasetName(self.warpType) 

690 for tempExpRef in refList: 

691 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

692 # therefore have no datasetExists() method 

693 if not isinstance(tempExpRef, DeferredDatasetHandle): 

694 if not tempExpRef.datasetExists(tempExpName): 

695 self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

696 continue 

697 

698 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

699 # Ignore any input warp that is empty of data 

700 if numpy.isnan(tempExp.image.array).all(): 

701 continue 

702 maskedImage = tempExp.getMaskedImage() 

703 imageScaler = self.scaleZeroPoint.computeImageScaler( 

704 exposure=tempExp, 

705 dataRef=tempExpRef, 

706 ) 

707 try: 

708 imageScaler.scaleMaskedImage(maskedImage) 

709 except Exception as e: 

710 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

711 continue 

712 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

713 afwMath.MEANCLIP, statsCtrl) 

714 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

715 weight = 1.0 / float(meanVar) 

716 if not numpy.isfinite(weight): 

717 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

718 continue 

719 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

720 

721 del maskedImage 

722 del tempExp 

723 

724 tempExpRefList.append(tempExpRef) 

725 weightList.append(weight) 

726 imageScalerList.append(imageScaler) 

727 

728 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

729 imageScalerList=imageScalerList) 

730 

731 def prepareStats(self, mask=None): 

732 """Prepare the statistics for coadding images. 

733 

734 Parameters 

735 ---------- 

736 mask : `int`, optional 

737 Bit mask value to exclude from coaddition. 

738 

739 Returns 

740 ------- 

741 stats : `lsst.pipe.base.Struct` 

742 Statistics structure with the following fields: 

743 

744 - ``statsCtrl``: Statistics control object for coadd 

745 (`lsst.afw.math.StatisticsControl`) 

746 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

747 """ 

748 if mask is None: 

749 mask = self.getBadPixelMask() 

750 statsCtrl = afwMath.StatisticsControl() 

751 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

752 statsCtrl.setNumIter(self.config.clipIter) 

753 statsCtrl.setAndMask(mask) 

754 statsCtrl.setNanSafe(True) 

755 statsCtrl.setWeighted(True) 

756 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

757 for plane, threshold in self.config.maskPropagationThresholds.items(): 

758 bit = afwImage.Mask.getMaskPlane(plane) 

759 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

760 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

761 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

762 

763 @timeMethod 

764 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

765 altMaskList=None, mask=None, supplementaryData=None): 

766 """Assemble a coadd from input warps 

767 

768 Assemble the coadd using the provided list of coaddTempExps. Since 

769 the full coadd covers a patch (a large area), the assembly is 

770 performed over small areas on the image at a time in order to 

771 conserve memory usage. Iterate over subregions within the outer 

772 bbox of the patch using `assembleSubregion` to stack the corresponding 

773 subregions from the coaddTempExps with the statistic specified. 

774 Set the edge bits the coadd mask based on the weight map. 

775 

776 Parameters 

777 ---------- 

778 skyInfo : `lsst.pipe.base.Struct` 

779 Struct with geometric information about the patch. 

780 tempExpRefList : `list` 

781 List of data references to Warps (previously called CoaddTempExps). 

782 imageScalerList : `list` 

783 List of image scalers. 

784 weightList : `list` 

785 List of weights 

786 altMaskList : `list`, optional 

787 List of alternate masks to use rather than those stored with 

788 tempExp. 

789 mask : `int`, optional 

790 Bit mask value to exclude from coaddition. 

791 supplementaryData : lsst.pipe.base.Struct, optional 

792 Struct with additional data products needed to assemble coadd. 

793 Only used by subclasses that implement `makeSupplementaryData` 

794 and override `run`. 

795 

796 Returns 

797 ------- 

798 result : `lsst.pipe.base.Struct` 

799 Result struct with components: 

800 

801 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

802 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

803 - ``inputMap``: bit-wise map of inputs, if requested. 

804 - ``warpRefList``: input list of refs to the warps ( 

805 ``lsst.daf.butler.DeferredDatasetHandle`` or 

806 ``lsst.daf.persistence.ButlerDataRef``) 

807 (unmodified) 

808 - ``imageScalerList``: input list of image scalers (unmodified) 

809 - ``weightList``: input list of weights (unmodified) 

810 """ 

811 tempExpName = self.getTempExpDatasetName(self.warpType) 

812 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

813 stats = self.prepareStats(mask=mask) 

814 

815 if altMaskList is None: 

816 altMaskList = [None]*len(tempExpRefList) 

817 

818 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

819 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

820 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

821 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

822 coaddMaskedImage = coaddExposure.getMaskedImage() 

823 subregionSizeArr = self.config.subregionSize 

824 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

825 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

826 if self.config.doNImage: 

827 nImage = afwImage.ImageU(skyInfo.bbox) 

828 else: 

829 nImage = None 

830 # If inputMap is requested, create the initial version that can be masked in 

831 # assembleSubregion. 

832 if self.config.doInputMap: 

833 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

834 skyInfo.wcs, 

835 coaddExposure.getInfo().getCoaddInputs().ccds) 

836 

837 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 

838 try: 

839 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

840 weightList, altMaskList, stats.ctrl, 

841 nImage=nImage) 

842 except Exception as e: 

843 self.log.exception("Cannot compute online coadd %s", e) 

844 raise 

845 else: 

846 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

847 try: 

848 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

849 weightList, altMaskList, stats.flags, stats.ctrl, 

850 nImage=nImage) 

851 except Exception as e: 

852 self.log.exception("Cannot compute coadd %s: %s", subBBox, e) 

853 raise 

854 

855 # If inputMap is requested, we must finalize the map after the accumulation. 

856 if self.config.doInputMap: 

857 self.inputMapper.finalize_ccd_input_map_mask() 

858 inputMap = self.inputMapper.ccd_input_map 

859 else: 

860 inputMap = None 

861 

862 self.setInexactPsf(coaddMaskedImage.getMask()) 

863 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

864 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

865 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

866 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

867 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

868 weightList=weightList, inputMap=inputMap) 

869 

870 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

871 """Set the metadata for the coadd. 

872 

873 This basic implementation sets the filter from the first input. 

874 

875 Parameters 

876 ---------- 

877 coaddExposure : `lsst.afw.image.Exposure` 

878 The target exposure for the coadd. 

879 tempExpRefList : `list` 

880 List of data references to tempExp. 

881 weightList : `list` 

882 List of weights. 

883 """ 

884 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

885 tempExpName = self.getTempExpDatasetName(self.warpType) 

886 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

887 # (and we need more than just the PropertySet that contains the header), which is not possible 

888 # with the current butler (see #2777). 

889 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

890 

891 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 

892 # Gen 3 API 

893 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

894 else: 

895 # Gen 2 API. Delete this when Gen 2 retired 

896 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

897 for tempExpRef in tempExpRefList] 

898 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

899 

900 # Set the coadd FilterLabel to the band of the first input exposure: 

901 # Coadds are calibrated, so the physical label is now meaningless. 

902 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

903 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

904 coaddInputs.ccds.reserve(numCcds) 

905 coaddInputs.visits.reserve(len(tempExpList)) 

906 

907 for tempExp, weight in zip(tempExpList, weightList): 

908 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

909 

910 if self.config.doUsePsfMatchedPolygons: 

911 self.shrinkValidPolygons(coaddInputs) 

912 

913 coaddInputs.visits.sort() 

914 coaddInputs.ccds.sort() 

915 if self.warpType == "psfMatched": 

916 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

917 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

918 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

919 # having the maximum width (sufficient because square) 

920 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

921 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

922 for modelPsf in modelPsfList] 

923 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

924 else: 

925 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

926 self.config.coaddPsf.makeControl()) 

927 coaddExposure.setPsf(psf) 

928 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

929 coaddExposure.getWcs()) 

930 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

931 if self.config.doAttachTransmissionCurve: 

932 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

933 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

934 

935 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

936 altMaskList, statsFlags, statsCtrl, nImage=None): 

937 """Assemble the coadd for a sub-region. 

938 

939 For each coaddTempExp, check for (and swap in) an alternative mask 

940 if one is passed. Remove mask planes listed in 

941 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

942 `lsst.afw.math.statisticsStack` with the statistic specified by 

943 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

944 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

945 an N-sigma clipped mean where N and iterations are specified by 

946 statsCtrl. Assign the stacked subregion back to the coadd. 

947 

948 Parameters 

949 ---------- 

950 coaddExposure : `lsst.afw.image.Exposure` 

951 The target exposure for the coadd. 

952 bbox : `lsst.geom.Box` 

953 Sub-region to coadd. 

954 tempExpRefList : `list` 

955 List of data reference to tempExp. 

956 imageScalerList : `list` 

957 List of image scalers. 

958 weightList : `list` 

959 List of weights. 

960 altMaskList : `list` 

961 List of alternate masks to use rather than those stored with 

962 tempExp, or None. Each element is dict with keys = mask plane 

963 name to which to add the spans. 

964 statsFlags : `lsst.afw.math.Property` 

965 Property object for statistic for coadd. 

966 statsCtrl : `lsst.afw.math.StatisticsControl` 

967 Statistics control object for coadd. 

968 nImage : `lsst.afw.image.ImageU`, optional 

969 Keeps track of exposure count for each pixel. 

970 """ 

971 self.log.debug("Computing coadd over %s", bbox) 

972 tempExpName = self.getTempExpDatasetName(self.warpType) 

973 coaddExposure.mask.addMaskPlane("REJECTED") 

974 coaddExposure.mask.addMaskPlane("CLIPPED") 

975 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

976 maskMap = self.setRejectedMaskMapping(statsCtrl) 

977 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

978 maskedImageList = [] 

979 if nImage is not None: 

980 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

981 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

982 

983 if isinstance(tempExpRef, DeferredDatasetHandle): 

984 # Gen 3 API 

985 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

986 else: 

987 # Gen 2 API. Delete this when Gen 2 retired 

988 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

989 

990 maskedImage = exposure.getMaskedImage() 

991 mask = maskedImage.getMask() 

992 if altMask is not None: 

993 self.applyAltMaskPlanes(mask, altMask) 

994 imageScaler.scaleMaskedImage(maskedImage) 

995 

996 # Add 1 for each pixel which is not excluded by the exclude mask. 

997 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

998 if nImage is not None: 

999 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

1000 if self.config.removeMaskPlanes: 

1001 self.removeMaskPlanes(maskedImage) 

1002 maskedImageList.append(maskedImage) 

1003 

1004 if self.config.doInputMap: 

1005 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

1006 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

1007 

1008 with self.timer("stack"): 

1009 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

1010 clipped, # also set output to CLIPPED if sigma-clipped 

1011 maskMap) 

1012 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

1013 if nImage is not None: 

1014 nImage.assign(subNImage, bbox) 

1015 

1016 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

1017 altMaskList, statsCtrl, nImage=None): 

1018 """Assemble the coadd using the "online" method. 

1019 

1020 This method takes a running sum of images and weights to save memory. 

1021 It only works for MEAN statistics. 

1022 

1023 Parameters 

1024 ---------- 

1025 coaddExposure : `lsst.afw.image.Exposure` 

1026 The target exposure for the coadd. 

1027 tempExpRefList : `list` 

1028 List of data reference to tempExp. 

1029 imageScalerList : `list` 

1030 List of image scalers. 

1031 weightList : `list` 

1032 List of weights. 

1033 altMaskList : `list` 

1034 List of alternate masks to use rather than those stored with 

1035 tempExp, or None. Each element is dict with keys = mask plane 

1036 name to which to add the spans. 

1037 statsCtrl : `lsst.afw.math.StatisticsControl` 

1038 Statistics control object for coadd 

1039 nImage : `lsst.afw.image.ImageU`, optional 

1040 Keeps track of exposure count for each pixel. 

1041 """ 

1042 self.log.debug("Computing online coadd.") 

1043 tempExpName = self.getTempExpDatasetName(self.warpType) 

1044 coaddExposure.mask.addMaskPlane("REJECTED") 

1045 coaddExposure.mask.addMaskPlane("CLIPPED") 

1046 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

1047 maskMap = self.setRejectedMaskMapping(statsCtrl) 

1048 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

1049 

1050 bbox = coaddExposure.maskedImage.getBBox() 

1051 

1052 stacker = AccumulatorMeanStack( 

1053 coaddExposure.image.array.shape, 

1054 statsCtrl.getAndMask(), 

1055 mask_threshold_dict=thresholdDict, 

1056 mask_map=maskMap, 

1057 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

1058 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

1059 compute_n_image=(nImage is not None) 

1060 ) 

1061 

1062 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

1063 imageScalerList, 

1064 altMaskList, 

1065 weightList): 

1066 if isinstance(tempExpRef, DeferredDatasetHandle): 

1067 # Gen 3 API 

1068 exposure = tempExpRef.get() 

1069 else: 

1070 # Gen 2 API. Delete this when Gen 2 retired 

1071 exposure = tempExpRef.get(tempExpName) 

1072 

1073 maskedImage = exposure.getMaskedImage() 

1074 mask = maskedImage.getMask() 

1075 if altMask is not None: 

1076 self.applyAltMaskPlanes(mask, altMask) 

1077 imageScaler.scaleMaskedImage(maskedImage) 

1078 if self.config.removeMaskPlanes: 

1079 self.removeMaskPlanes(maskedImage) 

1080 

1081 stacker.add_masked_image(maskedImage, weight=weight) 

1082 

1083 if self.config.doInputMap: 

1084 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

1085 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

1086 

1087 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

1088 

1089 if nImage is not None: 

1090 nImage.array[:, :] = stacker.n_image 

1091 

1092 def removeMaskPlanes(self, maskedImage): 

1093 """Unset the mask of an image for mask planes specified in the config. 

1094 

1095 Parameters 

1096 ---------- 

1097 maskedImage : `lsst.afw.image.MaskedImage` 

1098 The masked image to be modified. 

1099 """ 

1100 mask = maskedImage.getMask() 

1101 for maskPlane in self.config.removeMaskPlanes: 

1102 try: 

1103 mask &= ~mask.getPlaneBitMask(maskPlane) 

1104 except pexExceptions.InvalidParameterError: 

1105 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1106 maskPlane) 

1107 

1108 @staticmethod 

1109 def setRejectedMaskMapping(statsCtrl): 

1110 """Map certain mask planes of the warps to new planes for the coadd. 

1111 

1112 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1113 or CLIPPED, set it to REJECTED on the coadd. 

1114 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1115 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1116 

1117 Parameters 

1118 ---------- 

1119 statsCtrl : `lsst.afw.math.StatisticsControl` 

1120 Statistics control object for coadd 

1121 

1122 Returns 

1123 ------- 

1124 maskMap : `list` of `tuple` of `int` 

1125 A list of mappings of mask planes of the warped exposures to 

1126 mask planes of the coadd. 

1127 """ 

1128 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1129 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1130 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1131 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1132 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1133 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1134 (clipped, clipped)] 

1135 return maskMap 

1136 

1137 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1138 """Apply in place alt mask formatted as SpanSets to a mask. 

1139 

1140 Parameters 

1141 ---------- 

1142 mask : `lsst.afw.image.Mask` 

1143 Original mask. 

1144 altMaskSpans : `dict` 

1145 SpanSet lists to apply. Each element contains the new mask 

1146 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1147 and list of SpanSets to apply to the mask. 

1148 

1149 Returns 

1150 ------- 

1151 mask : `lsst.afw.image.Mask` 

1152 Updated mask. 

1153 """ 

1154 if self.config.doUsePsfMatchedPolygons: 

1155 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

1156 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1157 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1158 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1159 # This mask-clearing step must only occur *before* applying the new masks below 

1160 for spanSet in altMaskSpans['NO_DATA']: 

1161 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1162 

1163 for plane, spanSetList in altMaskSpans.items(): 

1164 maskClipValue = mask.addMaskPlane(plane) 

1165 for spanSet in spanSetList: 

1166 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1167 return mask 

1168 

1169 def shrinkValidPolygons(self, coaddInputs): 

1170 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1171 

1172 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1173 does not have a validPolygon, create one from its bbox. 

1174 

1175 Parameters 

1176 ---------- 

1177 coaddInputs : `lsst.afw.image.coaddInputs` 

1178 Original mask. 

1179 

1180 """ 

1181 for ccd in coaddInputs.ccds: 

1182 polyOrig = ccd.getValidPolygon() 

1183 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1184 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1185 if polyOrig: 

1186 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1187 else: 

1188 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1189 ccd.setValidPolygon(validPolygon) 

1190 

1191 def readBrightObjectMasks(self, dataRef): 

1192 """Retrieve the bright object masks. 

1193 

1194 Returns None on failure. 

1195 

1196 Parameters 

1197 ---------- 

1198 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1199 A Butler dataRef. 

1200 

1201 Returns 

1202 ------- 

1203 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1204 Bright object mask from the Butler object, or None if it cannot 

1205 be retrieved. 

1206 """ 

1207 try: 

1208 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1209 except Exception as e: 

1210 self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1211 return None 

1212 

1213 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1214 """Set the bright object masks. 

1215 

1216 Parameters 

1217 ---------- 

1218 exposure : `lsst.afw.image.Exposure` 

1219 Exposure under consideration. 

1220 dataId : `lsst.daf.persistence.dataId` 

1221 Data identifier dict for patch. 

1222 brightObjectMasks : `lsst.afw.table` 

1223 Table of bright objects to mask. 

1224 """ 

1225 

1226 if brightObjectMasks is None: 

1227 self.log.warning("Unable to apply bright object mask: none supplied") 

1228 return 

1229 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1230 mask = exposure.getMaskedImage().getMask() 

1231 wcs = exposure.getWcs() 

1232 plateScale = wcs.getPixelScale().asArcseconds() 

1233 

1234 for rec in brightObjectMasks: 

1235 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1236 if rec["type"] == "box": 

1237 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1238 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1239 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1240 

1241 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1242 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1243 

1244 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1245 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1246 spans = afwGeom.SpanSet(bbox) 

1247 elif rec["type"] == "circle": 

1248 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1249 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1250 else: 

1251 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1252 continue 

1253 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1254 

1255 def setInexactPsf(self, mask): 

1256 """Set INEXACT_PSF mask plane. 

1257 

1258 If any of the input images isn't represented in the coadd (due to 

1259 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1260 these pixels. 

1261 

1262 Parameters 

1263 ---------- 

1264 mask : `lsst.afw.image.Mask` 

1265 Coadded exposure's mask, modified in-place. 

1266 """ 

1267 mask.addMaskPlane("INEXACT_PSF") 

1268 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1269 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1270 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1271 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1272 array = mask.getArray() 

1273 selected = array & (sensorEdge | clipped | rejected) > 0 

1274 array[selected] |= inexactPsf 

1275 

1276 @classmethod 

1277 def _makeArgumentParser(cls): 

1278 """Create an argument parser. 

1279 """ 

1280 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1281 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1282 + cls.ConfigClass().warpType + "Warp", 

1283 help="data ID, e.g. --id tract=12345 patch=1,2", 

1284 ContainerClass=AssembleCoaddDataIdContainer) 

1285 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1286 ContainerClass=SelectDataIdContainer) 

1287 return parser 

1288 

1289 @staticmethod 

1290 def _subBBoxIter(bbox, subregionSize): 

1291 """Iterate over subregions of a bbox. 

1292 

1293 Parameters 

1294 ---------- 

1295 bbox : `lsst.geom.Box2I` 

1296 Bounding box over which to iterate. 

1297 subregionSize: `lsst.geom.Extent2I` 

1298 Size of sub-bboxes. 

1299 

1300 Yields 

1301 ------ 

1302 subBBox : `lsst.geom.Box2I` 

1303 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1304 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1305 the edges of ``bbox``, but it will never be empty. 

1306 """ 

1307 if bbox.isEmpty(): 

1308 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1309 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1310 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1311 

1312 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1313 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1314 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1315 subBBox.clip(bbox) 

1316 if subBBox.isEmpty(): 

1317 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1318 "colShift=%s, rowShift=%s" % 

1319 (bbox, subregionSize, colShift, rowShift)) 

1320 yield subBBox 

1321 

1322 def filterWarps(self, inputs, goodVisits): 

1323 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1324 

1325 Parameters 

1326 ---------- 

1327 inputs : list 

1328 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1329 goodVisit : `dict` 

1330 Dictionary with good visitIds as the keys. Value ignored. 

1331 

1332 Returns: 

1333 -------- 

1334 filteredInputs : `list` 

1335 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1336 """ 

1337 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1338 filteredInputs = [] 

1339 for visit in goodVisits.keys(): 

1340 if visit in inputWarpDict: 

1341 filteredInputs.append(inputWarpDict[visit]) 

1342 return filteredInputs 

1343 

1344 

1345class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1346 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1347 """ 

1348 

1349 def makeDataRefList(self, namespace): 

1350 """Make self.refList from self.idList. 

1351 

1352 Parameters 

1353 ---------- 

1354 namespace 

1355 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1356 """ 

1357 datasetType = namespace.config.coaddName + "Coadd" 

1358 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1359 

1360 for dataId in self.idList: 

1361 # tract and patch are required 

1362 for key in keysCoadd: 

1363 if key not in dataId: 

1364 raise RuntimeError("--id must include " + key) 

1365 

1366 dataRef = namespace.butler.dataRef( 

1367 datasetType=datasetType, 

1368 dataId=dataId, 

1369 ) 

1370 self.refList.append(dataRef) 

1371 

1372 

1373def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1374 """Function to count the number of pixels with a specific mask in a 

1375 footprint. 

1376 

1377 Find the intersection of mask & footprint. Count all pixels in the mask 

1378 that are in the intersection that have bitmask set but do not have 

1379 ignoreMask set. Return the count. 

1380 

1381 Parameters 

1382 ---------- 

1383 mask : `lsst.afw.image.Mask` 

1384 Mask to define intersection region by. 

1385 footprint : `lsst.afw.detection.Footprint` 

1386 Footprint to define the intersection region by. 

1387 bitmask 

1388 Specific mask that we wish to count the number of occurances of. 

1389 ignoreMask 

1390 Pixels to not consider. 

1391 

1392 Returns 

1393 ------- 

1394 result : `int` 

1395 Count of number of pixels in footprint with specified mask. 

1396 """ 

1397 bbox = footprint.getBBox() 

1398 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1399 fp = afwImage.Mask(bbox) 

1400 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1401 footprint.spans.setMask(fp, bitmask) 

1402 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1403 (subMask.getArray() & ignoreMask) == 0).sum() 

1404 

1405 

1406class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1407 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1408 """ 

1409 clipDetection = pexConfig.ConfigurableField( 

1410 target=SourceDetectionTask, 

1411 doc="Detect sources on difference between unclipped and clipped coadd") 

1412 minClipFootOverlap = pexConfig.Field( 

1413 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1414 dtype=float, 

1415 default=0.6 

1416 ) 

1417 minClipFootOverlapSingle = pexConfig.Field( 

1418 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1419 "clipped when only one visit overlaps", 

1420 dtype=float, 

1421 default=0.5 

1422 ) 

1423 minClipFootOverlapDouble = pexConfig.Field( 

1424 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1425 "clipped when two visits overlap", 

1426 dtype=float, 

1427 default=0.45 

1428 ) 

1429 maxClipFootOverlapDouble = pexConfig.Field( 

1430 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1431 "considering two visits", 

1432 dtype=float, 

1433 default=0.15 

1434 ) 

1435 minBigOverlap = pexConfig.Field( 

1436 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1437 "when labeling clipped footprints", 

1438 dtype=int, 

1439 default=100 

1440 ) 

1441 

1442 def setDefaults(self): 

1443 """Set default values for clipDetection. 

1444 

1445 Notes 

1446 ----- 

1447 The numeric values for these configuration parameters were 

1448 empirically determined, future work may further refine them. 

1449 """ 

1450 AssembleCoaddConfig.setDefaults(self) 

1451 self.clipDetection.doTempLocalBackground = False 

1452 self.clipDetection.reEstimateBackground = False 

1453 self.clipDetection.returnOriginalFootprints = False 

1454 self.clipDetection.thresholdPolarity = "both" 

1455 self.clipDetection.thresholdValue = 2 

1456 self.clipDetection.nSigmaToGrow = 2 

1457 self.clipDetection.minPixels = 4 

1458 self.clipDetection.isotropicGrow = True 

1459 self.clipDetection.thresholdType = "pixel_stdev" 

1460 self.sigmaClip = 1.5 

1461 self.clipIter = 3 

1462 self.statistic = "MEAN" 

1463 

1464 def validate(self): 

1465 if self.doSigmaClip: 

1466 log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1467 "Ignoring doSigmaClip.") 

1468 self.doSigmaClip = False 

1469 if self.statistic != "MEAN": 

1470 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1471 "(%s chosen). Please set statistic to MEAN." 

1472 % (self.statistic)) 

1473 AssembleCoaddTask.ConfigClass.validate(self) 

1474 

1475 

1476class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1477 """Assemble a coadded image from a set of coadded temporary exposures, 

1478 being careful to clip & flag areas with potential artifacts. 

1479 

1480 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1481 we clip outliers). The problem with doing this is that when computing the 

1482 coadd PSF at a given location, individual visit PSFs from visits with 

1483 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1484 In this task, we correct for this behavior by creating a new 

1485 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1486 coaddTempExps and the final coadd where 

1487 

1488 i. difference imaging suggests that there is an outlier and 

1489 ii. this outlier appears on only one or two images. 

1490 

1491 Such regions will not contribute to the final coadd. Furthermore, any 

1492 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1493 Note that the algorithm implemented by this task is preliminary and works 

1494 correctly for HSC data. Parameter modifications and or considerable 

1495 redesigning of the algorithm is likley required for other surveys. 

1496 

1497 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1498 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1499 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1500 if you wish. 

1501 

1502 Notes 

1503 ----- 

1504 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1505 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1506 see `baseDebug` for more about ``debug.py`` files. 

1507 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1508 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1509 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1510 for further information. 

1511 

1512 Examples 

1513 -------- 

1514 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1515 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1516 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1517 

1518 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1519 and filter to be coadded (specified using 

1520 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1521 along with a list of coaddTempExps to attempt to coadd (specified using 

1522 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1523 Only the coaddTempExps that cover the specified tract and patch will be 

1524 coadded. A list of the available optional arguments can be obtained by 

1525 calling assembleCoadd.py with the --help command line argument: 

1526 

1527 .. code-block:: none 

1528 

1529 assembleCoadd.py --help 

1530 

1531 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1532 context of multi-band processing, we will generate the HSC-I & -R band 

1533 coadds from HSC engineering test data provided in the ci_hsc package. 

1534 To begin, assuming that the lsst stack has been already set up, we must 

1535 set up the obs_subaru and ci_hsc packages. This defines the environment 

1536 variable $CI_HSC_DIR and points at the location of the package. The raw 

1537 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1538 the coadds, we must first 

1539 

1540 - ``processCcd`` 

1541 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1542 - ``makeSkyMap`` 

1543 create a skymap that covers the area of the sky present in the raw exposures 

1544 - ``makeCoaddTempExp`` 

1545 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1546 

1547 We can perform all of these steps by running 

1548 

1549 .. code-block:: none 

1550 

1551 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1552 

1553 This will produce warped coaddTempExps for each visit. To coadd the 

1554 warped data, we call ``assembleCoadd.py`` as follows: 

1555 

1556 .. code-block:: none 

1557 

1558 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1559 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1560 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1561 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1562 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1563 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1564 --selectId visit=903988 ccd=24 

1565 

1566 This will process the HSC-I band data. The results are written in 

1567 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1568 

1569 You may also choose to run: 

1570 

1571 .. code-block:: none 

1572 

1573 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1574 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1575 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1576 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1577 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1578 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1579 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1580 --selectId visit=903346 ccd=12 

1581 

1582 to generate the coadd for the HSC-R band if you are interested in following 

1583 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1584 """ 

1585 ConfigClass = SafeClipAssembleCoaddConfig 

1586 _DefaultName = "safeClipAssembleCoadd" 

1587 

1588 def __init__(self, *args, **kwargs): 

1589 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1590 schema = afwTable.SourceTable.makeMinimalSchema() 

1591 self.makeSubtask("clipDetection", schema=schema) 

1592 

1593 @utils.inheritDoc(AssembleCoaddTask) 

1594 @timeMethod 

1595 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1596 """Assemble the coadd for a region. 

1597 

1598 Compute the difference of coadds created with and without outlier 

1599 rejection to identify coadd pixels that have outlier values in some 

1600 individual visits. 

1601 Detect clipped regions on the difference image and mark these regions 

1602 on the one or two individual coaddTempExps where they occur if there 

1603 is significant overlap between the clipped region and a source. This 

1604 leaves us with a set of footprints from the difference image that have 

1605 been identified as having occured on just one or two individual visits. 

1606 However, these footprints were generated from a difference image. It 

1607 is conceivable for a large diffuse source to have become broken up 

1608 into multiple footprints acrosss the coadd difference in this process. 

1609 Determine the clipped region from all overlapping footprints from the 

1610 detected sources in each visit - these are big footprints. 

1611 Combine the small and big clipped footprints and mark them on a new 

1612 bad mask plane. 

1613 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1614 removal. Clipped footprints will no longer make it into the coadd 

1615 because they are marked in the new bad mask plane. 

1616 

1617 Notes 

1618 ----- 

1619 args and kwargs are passed but ignored in order to match the call 

1620 signature expected by the parent task. 

1621 """ 

1622 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1623 mask = exp.getMaskedImage().getMask() 

1624 mask.addMaskPlane("CLIPPED") 

1625 

1626 result = self.detectClip(exp, tempExpRefList) 

1627 

1628 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1629 

1630 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1631 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1632 # Append big footprints from individual Warps to result.clipSpans 

1633 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1634 result.detectionFootprints, maskClipValue, maskDetValue, 

1635 exp.getBBox()) 

1636 # Create mask of the current clipped footprints 

1637 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1638 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1639 

1640 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1641 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1642 maskClip |= maskClipBig 

1643 

1644 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1645 badMaskPlanes = self.config.badMaskPlanes[:] 

1646 badMaskPlanes.append("CLIPPED") 

1647 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1648 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1649 result.clipSpans, mask=badPixelMask) 

1650 

1651 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1652 """Return an exposure that contains the difference between unclipped 

1653 and clipped coadds. 

1654 

1655 Generate a difference image between clipped and unclipped coadds. 

1656 Compute the difference image by subtracting an outlier-clipped coadd 

1657 from an outlier-unclipped coadd. Return the difference image. 

1658 

1659 Parameters 

1660 ---------- 

1661 skyInfo : `lsst.pipe.base.Struct` 

1662 Patch geometry information, from getSkyInfo 

1663 tempExpRefList : `list` 

1664 List of data reference to tempExp 

1665 imageScalerList : `list` 

1666 List of image scalers 

1667 weightList : `list` 

1668 List of weights 

1669 

1670 Returns 

1671 ------- 

1672 exp : `lsst.afw.image.Exposure` 

1673 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1674 """ 

1675 config = AssembleCoaddConfig() 

1676 # getattr necessary because subtasks do not survive Config.toDict() 

1677 # exclude connections because the class of self.config.connections is not 

1678 # the same as AssembleCoaddConfig.connections, and the connections are not 

1679 # needed to run this task anyway. 

1680 configIntersection = {k: getattr(self.config, k) 

1681 for k, v in self.config.toDict().items() 

1682 if (k in config.keys() and k != "connections")} 

1683 configIntersection['doInputMap'] = False 

1684 configIntersection['doNImage'] = False 

1685 config.update(**configIntersection) 

1686 

1687 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1688 config.statistic = 'MEAN' 

1689 task = AssembleCoaddTask(config=config) 

1690 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1691 

1692 config.statistic = 'MEANCLIP' 

1693 task = AssembleCoaddTask(config=config) 

1694 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1695 

1696 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1697 coaddDiff -= coaddClip.getMaskedImage() 

1698 exp = afwImage.ExposureF(coaddDiff) 

1699 exp.setPsf(coaddMean.getPsf()) 

1700 return exp 

1701 

1702 def detectClip(self, exp, tempExpRefList): 

1703 """Detect clipped regions on an exposure and set the mask on the 

1704 individual tempExp masks. 

1705 

1706 Detect footprints in the difference image after smoothing the 

1707 difference image with a Gaussian kernal. Identify footprints that 

1708 overlap with one or two input ``coaddTempExps`` by comparing the 

1709 computed overlap fraction to thresholds set in the config. A different 

1710 threshold is applied depending on the number of overlapping visits 

1711 (restricted to one or two). If the overlap exceeds the thresholds, 

1712 the footprint is considered "CLIPPED" and is marked as such on the 

1713 coaddTempExp. Return a struct with the clipped footprints, the indices 

1714 of the ``coaddTempExps`` that end up overlapping with the clipped 

1715 footprints, and a list of new masks for the ``coaddTempExps``. 

1716 

1717 Parameters 

1718 ---------- 

1719 exp : `lsst.afw.image.Exposure` 

1720 Exposure to run detection on. 

1721 tempExpRefList : `list` 

1722 List of data reference to tempExp. 

1723 

1724 Returns 

1725 ------- 

1726 result : `lsst.pipe.base.Struct` 

1727 Result struct with components: 

1728 

1729 - ``clipFootprints``: list of clipped footprints. 

1730 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1731 ``tempExpRefList``. 

1732 - ``clipSpans``: List of dictionaries containing spanSet lists 

1733 to clip. Each element contains the new maskplane name 

1734 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1735 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1736 compressed into footprints. 

1737 """ 

1738 mask = exp.getMaskedImage().getMask() 

1739 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1740 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1741 # Merge positive and negative together footprints together 

1742 fpSet.positive.merge(fpSet.negative) 

1743 footprints = fpSet.positive 

1744 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1745 ignoreMask = self.getBadPixelMask() 

1746 

1747 clipFootprints = [] 

1748 clipIndices = [] 

1749 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1750 

1751 # for use by detectClipBig 

1752 visitDetectionFootprints = [] 

1753 

1754 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1755 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1756 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1757 

1758 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1759 for i, warpRef in enumerate(tempExpRefList): 

1760 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1761 immediate=True).getMaskedImage().getMask() 

1762 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1763 afwImage.PARENT, True) 

1764 maskVisitDet &= maskDetValue 

1765 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1766 visitDetectionFootprints.append(visitFootprints) 

1767 

1768 for j, footprint in enumerate(footprints.getFootprints()): 

1769 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1770 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1771 

1772 # build a list of clipped spans for each visit 

1773 for j, footprint in enumerate(footprints.getFootprints()): 

1774 nPixel = footprint.getArea() 

1775 overlap = [] # hold the overlap with each visit 

1776 indexList = [] # index of visit in global list 

1777 for i in range(len(tempExpRefList)): 

1778 ignore = ignoreArr[i, j] 

1779 overlapDet = overlapDetArr[i, j] 

1780 totPixel = nPixel - ignore 

1781 

1782 # If we have more bad pixels than detection skip 

1783 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 

1784 continue 

1785 overlap.append(overlapDet/float(totPixel)) 

1786 indexList.append(i) 

1787 

1788 overlap = numpy.array(overlap) 

1789 if not len(overlap): 

1790 continue 

1791 

1792 keep = False # Should this footprint be marked as clipped? 

1793 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1794 

1795 # If footprint only has one overlap use a lower threshold 

1796 if len(overlap) == 1: 

1797 if overlap[0] > self.config.minClipFootOverlapSingle: 

1798 keep = True 

1799 keepIndex = [0] 

1800 else: 

1801 # This is the general case where only visit should be clipped 

1802 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1803 if len(clipIndex) == 1: 

1804 keep = True 

1805 keepIndex = [clipIndex[0]] 

1806 

1807 # Test if there are clipped objects that overlap two different visits 

1808 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1809 if len(clipIndex) == 2 and len(overlap) > 3: 

1810 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1811 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1812 keep = True 

1813 keepIndex = clipIndex 

1814 

1815 if not keep: 

1816 continue 

1817 

1818 for index in keepIndex: 

1819 globalIndex = indexList[index] 

1820 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1821 

1822 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1823 clipFootprints.append(footprint) 

1824 

1825 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1826 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1827 

1828 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1829 maskClipValue, maskDetValue, coaddBBox): 

1830 """Return individual warp footprints for large artifacts and append 

1831 them to ``clipList`` in place. 

1832 

1833 Identify big footprints composed of many sources in the coadd 

1834 difference that may have originated in a large diffuse source in the 

1835 coadd. We do this by indentifying all clipped footprints that overlap 

1836 significantly with each source in all the coaddTempExps. 

1837 

1838 Parameters 

1839 ---------- 

1840 clipList : `list` 

1841 List of alt mask SpanSets with clipping information. Modified. 

1842 clipFootprints : `list` 

1843 List of clipped footprints. 

1844 clipIndices : `list` 

1845 List of which entries in tempExpClipList each footprint belongs to. 

1846 maskClipValue 

1847 Mask value of clipped pixels. 

1848 maskDetValue 

1849 Mask value of detected pixels. 

1850 coaddBBox : `lsst.geom.Box` 

1851 BBox of the coadd and warps. 

1852 

1853 Returns 

1854 ------- 

1855 bigFootprintsCoadd : `list` 

1856 List of big footprints 

1857 """ 

1858 bigFootprintsCoadd = [] 

1859 ignoreMask = self.getBadPixelMask() 

1860 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1861 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1862 for footprint in visitFootprints.getFootprints(): 

1863 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1864 

1865 # build a mask of clipped footprints that are in this visit 

1866 clippedFootprintsVisit = [] 

1867 for foot, clipIndex in zip(clipFootprints, clipIndices): 

1868 if index not in clipIndex: 

1869 continue 

1870 clippedFootprintsVisit.append(foot) 

1871 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1872 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1873 

1874 bigFootprintsVisit = [] 

1875 for foot in visitFootprints.getFootprints(): 

1876 if foot.getArea() < self.config.minBigOverlap: 

1877 continue 

1878 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1879 if nCount > self.config.minBigOverlap: 

1880 bigFootprintsVisit.append(foot) 

1881 bigFootprintsCoadd.append(foot) 

1882 

1883 for footprint in bigFootprintsVisit: 

1884 clippedSpans["CLIPPED"].append(footprint.spans) 

1885 

1886 return bigFootprintsCoadd 

1887 

1888 

1889class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1890 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1891 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1892 "Only PSF-Matched Warps make sense for image subtraction. " 

1893 "Therefore, they must be an additional declared input."), 

1894 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1895 storageClass="ExposureF", 

1896 dimensions=("tract", "patch", "skymap", "visit"), 

1897 deferLoad=True, 

1898 multiple=True 

1899 ) 

1900 templateCoadd = pipeBase.connectionTypes.Output( 

1901 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1902 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1903 name="{outputCoaddName}CoaddPsfMatched", 

1904 storageClass="ExposureF", 

1905 dimensions=("tract", "patch", "skymap", "band"), 

1906 ) 

1907 

1908 def __init__(self, *, config=None): 

1909 super().__init__(config=config) 

1910 if not config.assembleStaticSkyModel.doWrite: 

1911 self.outputs.remove("templateCoadd") 

1912 config.validate() 

1913 

1914 

1915class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1916 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1917 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1918 target=AssembleCoaddTask, 

1919 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1920 " naive/first-iteration model of the static sky.", 

1921 ) 

1922 detect = pexConfig.ConfigurableField( 

1923 target=SourceDetectionTask, 

1924 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1925 ) 

1926 detectTemplate = pexConfig.ConfigurableField( 

1927 target=SourceDetectionTask, 

1928 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1929 ) 

1930 maskStreaks = pexConfig.ConfigurableField( 

1931 target=MaskStreaksTask, 

1932 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1933 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1934 "streakMaskName" 

1935 ) 

1936 streakMaskName = pexConfig.Field( 

1937 dtype=str, 

1938 default="STREAK", 

1939 doc="Name of mask bit used for streaks" 

1940 ) 

1941 maxNumEpochs = pexConfig.Field( 

1942 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1943 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1944 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1945 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1946 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1947 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1948 "than transient and not masked.", 

1949 dtype=int, 

1950 default=2 

1951 ) 

1952 maxFractionEpochsLow = pexConfig.RangeField( 

1953 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1954 "Effective maxNumEpochs = " 

1955 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1956 dtype=float, 

1957 default=0.4, 

1958 min=0., max=1., 

1959 ) 

1960 maxFractionEpochsHigh = pexConfig.RangeField( 

1961 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1962 "Effective maxNumEpochs = " 

1963 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1964 dtype=float, 

1965 default=0.03, 

1966 min=0., max=1., 

1967 ) 

1968 spatialThreshold = pexConfig.RangeField( 

1969 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1970 "temporal criteria. If 0, clip all. If 1, clip none.", 

1971 dtype=float, 

1972 default=0.5, 

1973 min=0., max=1., 

1974 inclusiveMin=True, inclusiveMax=True 

1975 ) 

1976 doScaleWarpVariance = pexConfig.Field( 

1977 doc="Rescale Warp variance plane using empirical noise?", 

1978 dtype=bool, 

1979 default=True, 

1980 ) 

1981 scaleWarpVariance = pexConfig.ConfigurableField( 

1982 target=ScaleVarianceTask, 

1983 doc="Rescale variance on warps", 

1984 ) 

1985 doPreserveContainedBySource = pexConfig.Field( 

1986 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1987 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1988 dtype=bool, 

1989 default=True, 

1990 ) 

1991 doPrefilterArtifacts = pexConfig.Field( 

1992 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1993 "because they will be excluded anyway. This prevents them from contributing " 

1994 "to the outlier epoch count image and potentially being labeled as persistant." 

1995 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1996 dtype=bool, 

1997 default=True 

1998 ) 

1999 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

2000 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

2001 dtype=str, 

2002 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

2003 ) 

2004 prefilterArtifactsRatio = pexConfig.Field( 

2005 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

2006 dtype=float, 

2007 default=0.05 

2008 ) 

2009 doFilterMorphological = pexConfig.Field( 

2010 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

2011 "be streaks.", 

2012 dtype=bool, 

2013 default=False 

2014 ) 

2015 

2016 def setDefaults(self): 

2017 AssembleCoaddConfig.setDefaults(self) 

2018 self.statistic = 'MEAN' 

2019 self.doUsePsfMatchedPolygons = True 

2020 

2021 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

2022 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

2023 if "EDGE" in self.badMaskPlanes: 

2024 self.badMaskPlanes.remove('EDGE') 

2025 self.removeMaskPlanes.append('EDGE') 

2026 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

2027 self.assembleStaticSkyModel.warpType = 'psfMatched' 

2028 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

2029 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

2030 self.assembleStaticSkyModel.sigmaClip = 2.5 

2031 self.assembleStaticSkyModel.clipIter = 3 

2032 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

2033 self.assembleStaticSkyModel.doWrite = False 

2034 self.detect.doTempLocalBackground = False 

2035 self.detect.reEstimateBackground = False 

2036 self.detect.returnOriginalFootprints = False 

2037 self.detect.thresholdPolarity = "both" 

2038 self.detect.thresholdValue = 5 

2039 self.detect.minPixels = 4 

2040 self.detect.isotropicGrow = True 

2041 self.detect.thresholdType = "pixel_stdev" 

2042 self.detect.nSigmaToGrow = 0.4 

2043 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

2044 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

2045 self.detectTemplate.nSigmaToGrow = 2.4 

2046 self.detectTemplate.doTempLocalBackground = False 

2047 self.detectTemplate.reEstimateBackground = False 

2048 self.detectTemplate.returnOriginalFootprints = False 

2049 

2050 def validate(self): 

2051 super().validate() 

2052 if self.assembleStaticSkyModel.doNImage: 

2053 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

2054 "Please set assembleStaticSkyModel.doNImage=False") 

2055 

2056 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

2057 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

2058 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

2059 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

2060 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

2061 

2062 

2063class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

2064 """Assemble a compareWarp coadded image from a set of warps 

2065 by masking artifacts detected by comparing PSF-matched warps. 

2066 

2067 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

2068 we clip outliers). The problem with doing this is that when computing the 

2069 coadd PSF at a given location, individual visit PSFs from visits with 

2070 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

2071 In this task, we correct for this behavior by creating a new badMaskPlane 

2072 'CLIPPED' which marks pixels in the individual warps suspected to contain 

2073 an artifact. We populate this plane on the input warps by comparing 

2074 PSF-matched warps with a PSF-matched median coadd which serves as a 

2075 model of the static sky. Any group of pixels that deviates from the 

2076 PSF-matched template coadd by more than config.detect.threshold sigma, 

2077 is an artifact candidate. The candidates are then filtered to remove 

2078 variable sources and sources that are difficult to subtract such as 

2079 bright stars. This filter is configured using the config parameters 

2080 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

2081 the maximum fraction of epochs that the deviation can appear in and still 

2082 be considered an artifact. The spatialThreshold is the maximum fraction of 

2083 pixels in the footprint of the deviation that appear in other epochs 

2084 (where other epochs is defined by the temporalThreshold). If the deviant 

2085 region meets this criteria of having a significant percentage of pixels 

2086 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

2087 set in the mask. These regions will not contribute to the final coadd. 

2088 Furthermore, any routine to determine the coadd PSF can now be cognizant 

2089 of clipped regions. Note that the algorithm implemented by this task is 

2090 preliminary and works correctly for HSC data. Parameter modifications and 

2091 or considerable redesigning of the algorithm is likley required for other 

2092 surveys. 

2093 

2094 ``CompareWarpAssembleCoaddTask`` sub-classes 

2095 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

2096 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2097 

2098 Notes 

2099 ----- 

2100 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2101 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2102 ``baseDebug`` for more about ``debug.py`` files. 

2103 

2104 This task supports the following debug variables: 

2105 

2106 - ``saveCountIm`` 

2107 If True then save the Epoch Count Image as a fits file in the `figPath` 

2108 - ``figPath`` 

2109 Path to save the debug fits images and figures 

2110 

2111 For example, put something like: 

2112 

2113 .. code-block:: python 

2114 

2115 import lsstDebug 

2116 def DebugInfo(name): 

2117 di = lsstDebug.getInfo(name) 

2118 if name == "lsst.pipe.tasks.assembleCoadd": 

2119 di.saveCountIm = True 

2120 di.figPath = "/desired/path/to/debugging/output/images" 

2121 return di 

2122 lsstDebug.Info = DebugInfo 

2123 

2124 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2125 ``--debug`` flag. Some subtasks may have their own debug variables; 

2126 see individual Task documentation. 

2127 

2128 Examples 

2129 -------- 

2130 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2131 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2132 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2133 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2134 and filter to be coadded (specified using 

2135 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2136 along with a list of coaddTempExps to attempt to coadd (specified using 

2137 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2138 Only the warps that cover the specified tract and patch will be coadded. 

2139 A list of the available optional arguments can be obtained by calling 

2140 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2141 

2142 .. code-block:: none 

2143 

2144 assembleCoadd.py --help 

2145 

2146 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2147 context of multi-band processing, we will generate the HSC-I & -R band 

2148 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2149 To begin, assuming that the lsst stack has been already set up, we must 

2150 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2151 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2152 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2153 directory. To begin assembling the coadds, we must first 

2154 

2155 - processCcd 

2156 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2157 - makeSkyMap 

2158 create a skymap that covers the area of the sky present in the raw exposures 

2159 - makeCoaddTempExp 

2160 warp the individual calibrated exposures to the tangent plane of the coadd 

2161 

2162 We can perform all of these steps by running 

2163 

2164 .. code-block:: none 

2165 

2166 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2167 

2168 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2169 warped data, we call ``assembleCoadd.py`` as follows: 

2170 

2171 .. code-block:: none 

2172 

2173 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2174 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2175 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2176 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2177 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2178 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2179 --selectId visit=903988 ccd=24 

2180 

2181 This will process the HSC-I band data. The results are written in 

2182 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2183 """ 

2184 ConfigClass = CompareWarpAssembleCoaddConfig 

2185 _DefaultName = "compareWarpAssembleCoadd" 

2186 

2187 def __init__(self, *args, **kwargs): 

2188 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2189 self.makeSubtask("assembleStaticSkyModel") 

2190 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2191 self.makeSubtask("detect", schema=detectionSchema) 

2192 if self.config.doPreserveContainedBySource: 

2193 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2194 if self.config.doScaleWarpVariance: 

2195 self.makeSubtask("scaleWarpVariance") 

2196 if self.config.doFilterMorphological: 

2197 self.makeSubtask("maskStreaks") 

2198 

2199 @utils.inheritDoc(AssembleCoaddTask) 

2200 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2201 """ 

2202 Generate a templateCoadd to use as a naive model of static sky to 

2203 subtract from PSF-Matched warps. 

2204 

2205 Returns 

2206 ------- 

2207 result : `lsst.pipe.base.Struct` 

2208 Result struct with components: 

2209 

2210 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2211 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2212 """ 

2213 # Ensure that psfMatchedWarps are used as input warps for template generation 

2214 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2215 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2216 

2217 # Because subtasks don't have connections we have to make one. 

2218 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2219 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2220 if self.config.assembleStaticSkyModel.doWrite: 

2221 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2222 # Remove template coadd from both subtask's and main tasks outputs, 

2223 # because it is handled by the subtask as `coaddExposure` 

2224 del outputRefs.templateCoadd 

2225 del staticSkyModelOutputRefs.templateCoadd 

2226 

2227 # A PSF-Matched nImage does not exist as a dataset type 

2228 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2229 del staticSkyModelOutputRefs.nImage 

2230 

2231 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2232 staticSkyModelOutputRefs) 

2233 if templateCoadd is None: 

2234 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2235 

2236 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2237 nImage=templateCoadd.nImage, 

2238 warpRefList=templateCoadd.warpRefList, 

2239 imageScalerList=templateCoadd.imageScalerList, 

2240 weightList=templateCoadd.weightList) 

2241 

2242 @utils.inheritDoc(AssembleCoaddTask) 

2243 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2244 """ 

2245 Generate a templateCoadd to use as a naive model of static sky to 

2246 subtract from PSF-Matched warps. 

2247 

2248 Returns 

2249 ------- 

2250 result : `lsst.pipe.base.Struct` 

2251 Result struct with components: 

2252 

2253 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2254 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2255 """ 

2256 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2257 if templateCoadd is None: 

2258 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2259 

2260 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2261 nImage=templateCoadd.nImage, 

2262 warpRefList=templateCoadd.warpRefList, 

2263 imageScalerList=templateCoadd.imageScalerList, 

2264 weightList=templateCoadd.weightList) 

2265 

2266 def _noTemplateMessage(self, warpType): 

2267 warpName = (warpType[0].upper() + warpType[1:]) 

2268 message = """No %(warpName)s warps were found to build the template coadd which is 

2269 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2270 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2271 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2272 

2273 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2274 another algorithm like: 

2275 

2276 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2277 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2278 """ % {"warpName": warpName} 

2279 return message 

2280 

2281 @utils.inheritDoc(AssembleCoaddTask) 

2282 @timeMethod 

2283 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2284 supplementaryData, *args, **kwargs): 

2285 """Assemble the coadd. 

2286 

2287 Find artifacts and apply them to the warps' masks creating a list of 

2288 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2289 plane. Then pass these alternative masks to the base class's `run` 

2290 method. 

2291 

2292 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2293 that must contain a ``templateCoadd`` that serves as the 

2294 model of the static sky. 

2295 """ 

2296 

2297 # Check and match the order of the supplementaryData 

2298 # (PSF-matched) inputs to the order of the direct inputs, 

2299 # so that the artifact mask is applied to the right warp 

2300 dataIds = [ref.dataId for ref in tempExpRefList] 

2301 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2302 

2303 if dataIds != psfMatchedDataIds: 

2304 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2305 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2306 psfMatchedDataIds, dataIds) 

2307 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2308 psfMatchedDataIds, dataIds) 

2309 

2310 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2311 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2312 supplementaryData.warpRefList, 

2313 supplementaryData.imageScalerList) 

2314 

2315 badMaskPlanes = self.config.badMaskPlanes[:] 

2316 badMaskPlanes.append("CLIPPED") 

2317 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2318 

2319 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2320 spanSetMaskList, mask=badPixelMask) 

2321 

2322 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2323 # Psf-Matching moves the real edge inwards 

2324 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2325 return result 

2326 

2327 def applyAltEdgeMask(self, mask, altMaskList): 

2328 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2329 

2330 Parameters 

2331 ---------- 

2332 mask : `lsst.afw.image.Mask` 

2333 Original mask. 

2334 altMaskList : `list` 

2335 List of Dicts containing ``spanSet`` lists. 

2336 Each element contains the new mask plane name (e.g. "CLIPPED 

2337 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2338 the mask. 

2339 """ 

2340 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2341 for visitMask in altMaskList: 

2342 if "EDGE" in visitMask: 

2343 for spanSet in visitMask['EDGE']: 

2344 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2345 

2346 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2347 """Find artifacts. 

2348 

2349 Loop through warps twice. The first loop builds a map with the count 

2350 of how many epochs each pixel deviates from the templateCoadd by more 

2351 than ``config.chiThreshold`` sigma. The second loop takes each 

2352 difference image and filters the artifacts detected in each using 

2353 count map to filter out variable sources and sources that are 

2354 difficult to subtract cleanly. 

2355 

2356 Parameters 

2357 ---------- 

2358 templateCoadd : `lsst.afw.image.Exposure` 

2359 Exposure to serve as model of static sky. 

2360 tempExpRefList : `list` 

2361 List of data references to warps. 

2362 imageScalerList : `list` 

2363 List of image scalers. 

2364 

2365 Returns 

2366 ------- 

2367 altMasks : `list` 

2368 List of dicts containing information about CLIPPED 

2369 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2370 """ 

2371 

2372 self.log.debug("Generating Count Image, and mask lists.") 

2373 coaddBBox = templateCoadd.getBBox() 

2374 slateIm = afwImage.ImageU(coaddBBox) 

2375 epochCountImage = afwImage.ImageU(coaddBBox) 

2376 nImage = afwImage.ImageU(coaddBBox) 

2377 spanSetArtifactList = [] 

2378 spanSetNoDataMaskList = [] 

2379 spanSetEdgeList = [] 

2380 spanSetBadMorphoList = [] 

2381 badPixelMask = self.getBadPixelMask() 

2382 

2383 # mask of the warp diffs should = that of only the warp 

2384 templateCoadd.mask.clearAllMaskPlanes() 

2385 

2386 if self.config.doPreserveContainedBySource: 

2387 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2388 else: 

2389 templateFootprints = None 

2390 

2391 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2392 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2393 if warpDiffExp is not None: 

2394 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2395 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2396 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2397 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2398 fpSet.positive.merge(fpSet.negative) 

2399 footprints = fpSet.positive 

2400 slateIm.set(0) 

2401 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2402 

2403 # Remove artifacts due to defects before they contribute to the epochCountImage 

2404 if self.config.doPrefilterArtifacts: 

2405 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2406 

2407 # Clear mask before adding prefiltered spanSets 

2408 self.detect.clearMask(warpDiffExp.mask) 

2409 for spans in spanSetList: 

2410 spans.setImage(slateIm, 1, doClip=True) 

2411 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2412 epochCountImage += slateIm 

2413 

2414 if self.config.doFilterMorphological: 

2415 maskName = self.config.streakMaskName 

2416 _ = self.maskStreaks.run(warpDiffExp) 

2417 streakMask = warpDiffExp.mask 

2418 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2419 streakMask.getPlaneBitMask(maskName)).split() 

2420 

2421 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2422 # undergo a second convolution. Pixels with data in the direct warp 

2423 # but not in the PSF-matched warp will not have their artifacts detected. 

2424 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2425 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2426 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2427 nansMask.setXY0(warpDiffExp.getXY0()) 

2428 edgeMask = warpDiffExp.mask 

2429 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2430 edgeMask.getPlaneBitMask("EDGE")).split() 

2431 else: 

2432 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2433 # In this case, mask the whole epoch 

2434 nansMask = afwImage.MaskX(coaddBBox, 1) 

2435 spanSetList = [] 

2436 spanSetEdgeMask = [] 

2437 spanSetStreak = [] 

2438 

2439 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2440 

2441 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2442 spanSetArtifactList.append(spanSetList) 

2443 spanSetEdgeList.append(spanSetEdgeMask) 

2444 if self.config.doFilterMorphological: 

2445 spanSetBadMorphoList.append(spanSetStreak) 

2446 

2447 if lsstDebug.Info(__name__).saveCountIm: 

2448 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2449 epochCountImage.writeFits(path) 

2450 

2451 for i, spanSetList in enumerate(spanSetArtifactList): 

2452 if spanSetList: 

2453 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2454 templateFootprints) 

2455 spanSetArtifactList[i] = filteredSpanSetList 

2456 if self.config.doFilterMorphological: 

2457 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2458 

2459 altMasks = [] 

2460 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2461 altMasks.append({'CLIPPED': artifacts, 

2462 'NO_DATA': noData, 

2463 'EDGE': edge}) 

2464 return altMasks 

2465 

2466 def prefilterArtifacts(self, spanSetList, exp): 

2467 """Remove artifact candidates covered by bad mask plane. 

2468 

2469 Any future editing of the candidate list that does not depend on 

2470 temporal information should go in this method. 

2471 

2472 Parameters 

2473 ---------- 

2474 spanSetList : `list` 

2475 List of SpanSets representing artifact candidates. 

2476 exp : `lsst.afw.image.Exposure` 

2477 Exposure containing mask planes used to prefilter. 

2478 

2479 Returns 

2480 ------- 

2481 returnSpanSetList : `list` 

2482 List of SpanSets with artifacts. 

2483 """ 

2484 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2485 goodArr = (exp.mask.array & badPixelMask) == 0 

2486 returnSpanSetList = [] 

2487 bbox = exp.getBBox() 

2488 x0, y0 = exp.getXY0() 

2489 for i, span in enumerate(spanSetList): 

2490 y, x = span.clippedTo(bbox).indices() 

2491 yIndexLocal = numpy.array(y) - y0 

2492 xIndexLocal = numpy.array(x) - x0 

2493 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2494 if goodRatio > self.config.prefilterArtifactsRatio: 

2495 returnSpanSetList.append(span) 

2496 return returnSpanSetList 

2497 

2498 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2499 """Filter artifact candidates. 

2500 

2501 Parameters 

2502 ---------- 

2503 spanSetList : `list` 

2504 List of SpanSets representing artifact candidates. 

2505 epochCountImage : `lsst.afw.image.Image` 

2506 Image of accumulated number of warpDiff detections. 

2507 nImage : `lsst.afw.image.Image` 

2508 Image of the accumulated number of total epochs contributing. 

2509 

2510 Returns 

2511 ------- 

2512 maskSpanSetList : `list` 

2513 List of SpanSets with artifacts. 

2514 """ 

2515 

2516 maskSpanSetList = [] 

2517 x0, y0 = epochCountImage.getXY0() 

2518 for i, span in enumerate(spanSetList): 

2519 y, x = span.indices() 

2520 yIdxLocal = [y1 - y0 for y1 in y] 

2521 xIdxLocal = [x1 - x0 for x1 in x] 

2522 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2523 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2524 

2525 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2526 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2527 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2528 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2529 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2530 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2531 & (outlierN <= effectiveMaxNumEpochs)) 

2532 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2533 if percentBelowThreshold > self.config.spatialThreshold: 

2534 maskSpanSetList.append(span) 

2535 

2536 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

2537 # If a candidate is contained by a footprint on the template coadd, do not clip 

2538 filteredMaskSpanSetList = [] 

2539 for span in maskSpanSetList: 

2540 doKeep = True 

2541 for footprint in footprintsToExclude.positive.getFootprints(): 

2542 if footprint.spans.contains(span): 

2543 doKeep = False 

2544 break 

2545 if doKeep: 

2546 filteredMaskSpanSetList.append(span) 

2547 maskSpanSetList = filteredMaskSpanSetList 

2548 

2549 return maskSpanSetList 

2550 

2551 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2552 """Fetch a warp from the butler and return a warpDiff. 

2553 

2554 Parameters 

2555 ---------- 

2556 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2557 Butler dataRef for the warp. 

2558 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2559 An image scaler object. 

2560 templateCoadd : `lsst.afw.image.Exposure` 

2561 Exposure to be substracted from the scaled warp. 

2562 

2563 Returns 

2564 ------- 

2565 warp : `lsst.afw.image.Exposure` 

2566 Exposure of the image difference between the warp and template. 

2567 """ 

2568 

2569 # If the PSF-Matched warp did not exist for this direct warp 

2570 # None is holding its place to maintain order in Gen 3 

2571 if warpRef is None: 

2572 return None 

2573 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2574 warpName = self.getTempExpDatasetName('psfMatched') 

2575 if not isinstance(warpRef, DeferredDatasetHandle): 

2576 if not warpRef.datasetExists(warpName): 

2577 self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2578 return None 

2579 warp = warpRef.get(datasetType=warpName, immediate=True) 

2580 # direct image scaler OK for PSF-matched Warp 

2581 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2582 mi = warp.getMaskedImage() 

2583 if self.config.doScaleWarpVariance: 

2584 try: 

2585 self.scaleWarpVariance.run(mi) 

2586 except Exception as exc: 

2587 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

2588 mi -= templateCoadd.getMaskedImage() 

2589 return warp 

2590 

2591 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2592 """Return a path to which to write debugging output. 

2593 

2594 Creates a hyphen-delimited string of dataId values for simple filenames. 

2595 

2596 Parameters 

2597 ---------- 

2598 prefix : `str` 

2599 Prefix for filename. 

2600 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2601 Butler dataRef to make the path from. 

2602 coaddLevel : `bool`, optional. 

2603 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2604 'filter', but no 'visit'). 

2605 

2606 Returns 

2607 ------- 

2608 result : `str` 

2609 Path for debugging output. 

2610 """ 

2611 if coaddLevel: 

2612 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2613 else: 

2614 keys = warpRef.dataId.keys() 

2615 keyList = sorted(keys, reverse=True) 

2616 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2617 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2618 return os.path.join(directory, filename)