Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 66%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

890 statements  

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import logging 

27import lsst.pex.config as pexConfig 

28import lsst.pex.exceptions as pexExceptions 

29import lsst.geom as geom 

30import lsst.afw.geom as afwGeom 

31import lsst.afw.image as afwImage 

32import lsst.afw.math as afwMath 

33import lsst.afw.table as afwTable 

34import lsst.afw.detection as afwDet 

35import lsst.coadd.utils as coaddUtils 

36import lsst.pipe.base as pipeBase 

37import lsst.meas.algorithms as measAlg 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from .healSparseMapping import HealSparseInputMapTask 

48from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack 

49from lsst.daf.butler import DeferredDatasetHandle 

50from lsst.utils.timer import timeMethod 

51 

52__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

53 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

54 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

55 

56log = logging.getLogger(__name__) 

57 

58 

59class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("tract", "patch", "band", "skymap"), 

61 defaultTemplates={"inputCoaddName": "deep", 

62 "outputCoaddName": "deep", 

63 "warpType": "direct", 

64 "warpTypeSuffix": ""}): 

65 

66 inputWarps = pipeBase.connectionTypes.Input( 

67 doc=("Input list of warps to be assemebled i.e. stacked." 

68 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

69 name="{inputCoaddName}Coadd_{warpType}Warp", 

70 storageClass="ExposureF", 

71 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

72 deferLoad=True, 

73 multiple=True 

74 ) 

75 skyMap = pipeBase.connectionTypes.Input( 

76 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

78 storageClass="SkyMap", 

79 dimensions=("skymap", ), 

80 ) 

81 selectedVisits = pipeBase.connectionTypes.Input( 

82 doc="Selected visits to be coadded.", 

83 name="{outputCoaddName}Visits", 

84 storageClass="StructuredDataDict", 

85 dimensions=("instrument", "tract", "patch", "skymap", "band") 

86 ) 

87 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

88 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

89 " BRIGHT_OBJECT."), 

90 name="brightObjectMask", 

91 storageClass="ObjectMaskCatalog", 

92 dimensions=("tract", "patch", "skymap", "band"), 

93 ) 

94 coaddExposure = pipeBase.connectionTypes.Output( 

95 doc="Output coadded exposure, produced by stacking input warps", 

96 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

97 storageClass="ExposureF", 

98 dimensions=("tract", "patch", "skymap", "band"), 

99 ) 

100 nImage = pipeBase.connectionTypes.Output( 

101 doc="Output image of number of input images per pixel", 

102 name="{outputCoaddName}Coadd_nImage", 

103 storageClass="ImageU", 

104 dimensions=("tract", "patch", "skymap", "band"), 

105 ) 

106 inputMap = pipeBase.connectionTypes.Output( 

107 doc="Output healsparse map of input images", 

108 name="{outputCoaddName}Coadd_inputMap", 

109 storageClass="HealSparseMap", 

110 dimensions=("tract", "patch", "skymap", "band"), 

111 ) 

112 

113 def __init__(self, *, config=None): 

114 super().__init__(config=config) 

115 

116 # Override the connection's name template with config to replicate Gen2 behavior 

117 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

118 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

119 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

120 templateValues['warpType'] = config.warpType 

121 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

123 for name in self.allConnections} 

124 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

125 # End code to remove after deprecation 

126 

127 if not config.doMaskBrightObjects: 

128 self.prerequisiteInputs.remove("brightObjectMask") 

129 

130 if not config.doSelectVisits: 

131 self.inputs.remove("selectedVisits") 

132 

133 if not config.doNImage: 

134 self.outputs.remove("nImage") 

135 

136 if not self.config.doInputMap: 

137 self.outputs.remove("inputMap") 

138 

139 

140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

141 pipelineConnections=AssembleCoaddConnections): 

142 """Configuration parameters for the `AssembleCoaddTask`. 

143 

144 Notes 

145 ----- 

146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

147 only set the bitplane config.brightObjectMaskName. To make this useful you 

148 *must* also configure the flags.pixel algorithm, for example by adding 

149 

150 .. code-block:: none 

151 

152 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

153 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

154 

155 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

156 """ 

157 warpType = pexConfig.Field( 

158 doc="Warp name: one of 'direct' or 'psfMatched'", 

159 dtype=str, 

160 default="direct", 

161 ) 

162 subregionSize = pexConfig.ListField( 

163 dtype=int, 

164 doc="Width, height of stack subregion size; " 

165 "make small enough that a full stack of images will fit into memory at once.", 

166 length=2, 

167 default=(2000, 2000), 

168 ) 

169 statistic = pexConfig.Field( 

170 dtype=str, 

171 doc="Main stacking statistic for aggregating over the epochs.", 

172 default="MEANCLIP", 

173 ) 

174 doOnlineForMean = pexConfig.Field( 

175 dtype=bool, 

176 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

177 default=False, 

178 ) 

179 doSigmaClip = pexConfig.Field( 

180 dtype=bool, 

181 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

182 default=False, 

183 ) 

184 sigmaClip = pexConfig.Field( 

185 dtype=float, 

186 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

187 default=3.0, 

188 ) 

189 clipIter = pexConfig.Field( 

190 dtype=int, 

191 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

192 default=2, 

193 ) 

194 calcErrorFromInputVariance = pexConfig.Field( 

195 dtype=bool, 

196 doc="Calculate coadd variance from input variance by stacking statistic." 

197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

198 default=True, 

199 ) 

200 scaleZeroPoint = pexConfig.ConfigurableField( 

201 target=ScaleZeroPointTask, 

202 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

203 ) 

204 doInterp = pexConfig.Field( 

205 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

206 dtype=bool, 

207 default=True, 

208 ) 

209 interpImage = pexConfig.ConfigurableField( 

210 target=InterpImageTask, 

211 doc="Task to interpolate (and extrapolate) over NaN pixels", 

212 ) 

213 doWrite = pexConfig.Field( 

214 doc="Persist coadd?", 

215 dtype=bool, 

216 default=True, 

217 ) 

218 doNImage = pexConfig.Field( 

219 doc="Create image of number of contributing exposures for each pixel", 

220 dtype=bool, 

221 default=False, 

222 ) 

223 doUsePsfMatchedPolygons = pexConfig.Field( 

224 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

225 dtype=bool, 

226 default=False, 

227 ) 

228 maskPropagationThresholds = pexConfig.DictField( 

229 keytype=str, 

230 itemtype=float, 

231 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

233 "would have contributed exceeds this value."), 

234 default={"SAT": 0.1}, 

235 ) 

236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

237 doc="Mask planes to remove before coadding") 

238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

239 doc="Set mask and flag bits for bright objects?") 

240 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

241 doc="Name of mask bit used for bright objects") 

242 coaddPsf = pexConfig.ConfigField( 

243 doc="Configuration for CoaddPsf", 

244 dtype=measAlg.CoaddPsfConfig, 

245 ) 

246 doAttachTransmissionCurve = pexConfig.Field( 

247 dtype=bool, default=False, optional=False, 

248 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

249 "(requires all input Exposures to have TransmissionCurves).") 

250 ) 

251 hasFakes = pexConfig.Field( 

252 dtype=bool, 

253 default=False, 

254 doc="Should be set to True if fake sources have been inserted into the input data." 

255 ) 

256 doSelectVisits = pexConfig.Field( 

257 doc="Coadd only visits selected by a SelectVisitsTask", 

258 dtype=bool, 

259 default=False, 

260 ) 

261 doInputMap = pexConfig.Field( 

262 doc="Create a bitwise map of coadd inputs", 

263 dtype=bool, 

264 default=False, 

265 ) 

266 inputMapper = pexConfig.ConfigurableField( 

267 doc="Input map creation subtask.", 

268 target=HealSparseInputMapTask, 

269 ) 

270 

271 def setDefaults(self): 

272 super().setDefaults() 

273 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

274 

275 def validate(self): 

276 super().validate() 

277 if self.doPsfMatch: 

278 # Backwards compatibility. 

279 # Configs do not have loggers 

280 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

281 self.warpType = 'psfMatched' 

282 if self.doSigmaClip and self.statistic != "MEANCLIP": 

283 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

284 self.statistic = "MEANCLIP" 

285 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

286 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

287 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

288 

289 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

290 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

291 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

292 if str(k) not in unstackableStats] 

293 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

294 % (self.statistic, stackableStats)) 

295 

296 

297class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

298 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

299 

300 We want to assemble a coadded image from a set of Warps (also called 

301 coadded temporary exposures or ``coaddTempExps``). 

302 Each input Warp covers a patch on the sky and corresponds to a single 

303 run/visit/exposure of the covered patch. We provide the task with a list 

304 of Warps (``selectDataList``) from which it selects Warps that cover the 

305 specified patch (pointed at by ``dataRef``). 

306 Each Warp that goes into a coadd will typically have an independent 

307 photometric zero-point. Therefore, we must scale each Warp to set it to 

308 a common photometric zeropoint. WarpType may be one of 'direct' or 

309 'psfMatched', and the boolean configs `config.makeDirect` and 

310 `config.makePsfMatched` set which of the warp types will be coadded. 

311 The coadd is computed as a mean with optional outlier rejection. 

312 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

313 Finally, Warps can have bad 'NaN' pixels which received no input from the 

314 source calExps. We interpolate over these bad (NaN) pixels. 

315 

316 `AssembleCoaddTask` uses several sub-tasks. These are 

317 

318 - `ScaleZeroPointTask` 

319 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

320 - `InterpImageTask` 

321 - interpolate across bad pixels (NaN) in the final coadd 

322 

323 You can retarget these subtasks if you wish. 

324 

325 Notes 

326 ----- 

327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

328 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

329 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

330 no debug variables of its own. Some of the subtasks may support debug 

331 variables. See the documentation for the subtasks for further information. 

332 

333 Examples 

334 -------- 

335 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

337 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

338 inputs: a data reference to the tract patch and filter to be coadded, and 

339 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

340 ``--selectId``, respectively: 

341 

342 .. code-block:: none 

343 

344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

346 

347 Only the Warps that cover the specified tract and patch will be coadded. 

348 A list of the available optional arguments can be obtained by calling 

349 ``assembleCoadd.py`` with the ``--help`` command line argument: 

350 

351 .. code-block:: none 

352 

353 assembleCoadd.py --help 

354 

355 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

356 multi-band processing, we will generate the HSC-I & -R band coadds from 

357 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

358 assuming that the lsst stack has been already set up, we must set up the 

359 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

360 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

361 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

362 coadds, we must first 

363 

364 - processCcd 

365 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

366 - makeSkyMap 

367 - create a skymap that covers the area of the sky present in the raw exposures 

368 - makeCoaddTempExp 

369 - warp the individual calibrated exposures to the tangent plane of the coadd 

370 

371 We can perform all of these steps by running 

372 

373 .. code-block:: none 

374 

375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

376 

377 This will produce warped exposures for each visit. To coadd the warped 

378 data, we call assembleCoadd.py as follows: 

379 

380 .. code-block:: none 

381 

382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

388 --selectId visit=903988 ccd=24 

389 

390 that will process the HSC-I band data. The results are written in 

391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

392 

393 You may also choose to run: 

394 

395 .. code-block:: none 

396 

397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

405 

406 to generate the coadd for the HSC-R band if you are interested in 

407 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

408 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

409 rather than `AssembleCoaddTask` to make the coadd. 

410 """ 

411 ConfigClass = AssembleCoaddConfig 

412 _DefaultName = "assembleCoadd" 

413 

414 def __init__(self, *args, **kwargs): 

415 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

416 if args: 416 ↛ 417line 416 didn't jump to line 417, because the condition on line 416 was never true

417 argNames = ["config", "name", "parentTask", "log"] 

418 kwargs.update({k: v for k, v in zip(argNames, args)}) 

419 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

420 "PipelineTask will not take positional args" % argNames, FutureWarning) 

421 

422 super().__init__(**kwargs) 

423 self.makeSubtask("interpImage") 

424 self.makeSubtask("scaleZeroPoint") 

425 

426 if self.config.doMaskBrightObjects: 426 ↛ 427line 426 didn't jump to line 427, because the condition on line 426 was never true

427 mask = afwImage.Mask() 

428 try: 

429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

430 except pexExceptions.LsstCppException: 

431 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

432 mask.getMaskPlaneDict().keys()) 

433 del mask 

434 

435 if self.config.doInputMap: 435 ↛ 436line 435 didn't jump to line 436, because the condition on line 435 was never true

436 self.makeSubtask("inputMapper") 

437 

438 self.warpType = self.config.warpType 

439 

440 @utils.inheritDoc(pipeBase.PipelineTask) 

441 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

442 # Docstring to be formatted with info from PipelineTask.runQuantum 

443 """ 

444 Notes 

445 ----- 

446 Assemble a coadd from a set of Warps. 

447 

448 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

449 Analogous to `runDataRef`, it prepares all the data products to be 

450 passed to `run`, and processes the results before returning a struct 

451 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

452 Therefore, its inputs are accessed subregion by subregion 

453 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

455 correspond to an update in `runDataRef` while both entry points 

456 are used. 

457 """ 

458 inputData = butlerQC.get(inputRefs) 

459 

460 # Construct skyInfo expected by run 

461 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

462 skyMap = inputData["skyMap"] 

463 outputDataId = butlerQC.quantum.dataId 

464 

465 inputData['skyInfo'] = makeSkyInfo(skyMap, 

466 tractId=outputDataId['tract'], 

467 patchId=outputDataId['patch']) 

468 

469 if self.config.doSelectVisits: 

470 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

471 else: 

472 warpRefList = inputData['inputWarps'] 

473 

474 # Perform same middle steps as `runDataRef` does 

475 inputs = self.prepareInputs(warpRefList) 

476 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

477 self.getTempExpDatasetName(self.warpType)) 

478 if len(inputs.tempExpRefList) == 0: 

479 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

480 

481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

482 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

483 inputs.weightList, supplementaryData=supplementaryData) 

484 

485 inputData.setdefault('brightObjectMask', None) 

486 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

487 

488 if self.config.doWrite: 

489 butlerQC.put(retStruct, outputRefs) 

490 return retStruct 

491 

492 @timeMethod 

493 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

494 """Assemble a coadd from a set of Warps. 

495 

496 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

497 Compute weights to be applied to each Warp and 

498 find scalings to match the photometric zeropoint to a reference Warp. 

499 Assemble the Warps using `run`. Interpolate over NaNs and 

500 optionally write the coadd to disk. Return the coadded exposure. 

501 

502 Parameters 

503 ---------- 

504 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

505 Data reference defining the patch for coaddition and the 

506 reference Warp (if ``config.autoReference=False``). 

507 Used to access the following data products: 

508 - ``self.config.coaddName + "Coadd_skyMap"`` 

509 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

510 - ``self.config.coaddName + "Coadd"`` 

511 selectDataList : `list` 

512 List of data references to Calexps. Data to be coadded will be 

513 selected from this list based on overlap with the patch defined 

514 by dataRef, grouped by visit, and converted to a list of data 

515 references to warps. 

516 warpRefList : `list` 

517 List of data references to Warps to be coadded. 

518 Note: `warpRefList` is just the new name for `tempExpRefList`. 

519 

520 Returns 

521 ------- 

522 retStruct : `lsst.pipe.base.Struct` 

523 Result struct with components: 

524 

525 - ``coaddExposure``: coadded exposure (``Exposure``). 

526 - ``nImage``: exposure count image (``Image``). 

527 """ 

528 if selectDataList and warpRefList: 528 ↛ 529line 528 didn't jump to line 529, because the condition on line 528 was never true

529 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

530 "and which to use is ambiguous. Please pass only one.") 

531 

532 skyInfo = self.getSkyInfo(dataRef) 

533 if warpRefList is None: 

534 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

535 if len(calExpRefList) == 0: 535 ↛ 536line 535 didn't jump to line 536, because the condition on line 535 was never true

536 self.log.warning("No exposures to coadd") 

537 return 

538 self.log.info("Coadding %d exposures", len(calExpRefList)) 

539 

540 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

541 

542 inputData = self.prepareInputs(warpRefList) 

543 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

544 self.getTempExpDatasetName(self.warpType)) 

545 if len(inputData.tempExpRefList) == 0: 545 ↛ 546line 545 didn't jump to line 546, because the condition on line 545 was never true

546 self.log.warning("No coadd temporary exposures found") 

547 return 

548 

549 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

550 

551 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

552 inputData.weightList, supplementaryData=supplementaryData) 

553 

554 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

555 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

556 

557 if self.config.doWrite: 

558 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 558 ↛ 559line 558 didn't jump to line 559, because the condition on line 558 was never true

559 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

560 else: 

561 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

562 self.log.info("Persisting %s", coaddDatasetName) 

563 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

564 if self.config.doNImage and retStruct.nImage is not None: 564 ↛ 565line 564 didn't jump to line 565, because the condition on line 564 was never true

565 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

566 

567 return retStruct 

568 

569 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

570 """Interpolate over missing data and mask bright stars. 

571 

572 Parameters 

573 ---------- 

574 coaddExposure : `lsst.afw.image.Exposure` 

575 The coadded exposure to process. 

576 dataRef : `lsst.daf.persistence.ButlerDataRef` 

577 Butler data reference for supplementary data. 

578 """ 

579 if self.config.doInterp: 579 ↛ 586line 579 didn't jump to line 586, because the condition on line 579 was never false

580 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

581 # The variance must be positive; work around for DM-3201. 

582 varArray = coaddExposure.variance.array 

583 with numpy.errstate(invalid="ignore"): 

584 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

585 

586 if self.config.doMaskBrightObjects: 586 ↛ 587line 586 didn't jump to line 587, because the condition on line 586 was never true

587 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

588 

589 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

590 """Make additional inputs to run() specific to subclasses (Gen2) 

591 

592 Duplicates interface of `runDataRef` method 

593 Available to be implemented by subclasses only if they need the 

594 coadd dataRef for performing preliminary processing before 

595 assembling the coadd. 

596 

597 Parameters 

598 ---------- 

599 dataRef : `lsst.daf.persistence.ButlerDataRef` 

600 Butler data reference for supplementary data. 

601 selectDataList : `list` (optional) 

602 Optional List of data references to Calexps. 

603 warpRefList : `list` (optional) 

604 Optional List of data references to Warps. 

605 """ 

606 return pipeBase.Struct() 

607 

608 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

609 """Make additional inputs to run() specific to subclasses (Gen3) 

610 

611 Duplicates interface of `runQuantum` method. 

612 Available to be implemented by subclasses only if they need the 

613 coadd dataRef for performing preliminary processing before 

614 assembling the coadd. 

615 

616 Parameters 

617 ---------- 

618 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

619 Gen3 Butler object for fetching additional data products before 

620 running the Task specialized for quantum being processed 

621 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

622 Attributes are the names of the connections describing input dataset types. 

623 Values are DatasetRefs that task consumes for corresponding dataset type. 

624 DataIds are guaranteed to match data objects in ``inputData``. 

625 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

626 Attributes are the names of the connections describing output dataset types. 

627 Values are DatasetRefs that task is to produce 

628 for corresponding dataset type. 

629 """ 

630 return pipeBase.Struct() 

631 

632 def getTempExpRefList(self, patchRef, calExpRefList): 

633 """Generate list data references corresponding to warped exposures 

634 that lie within the patch to be coadded. 

635 

636 Parameters 

637 ---------- 

638 patchRef : `dataRef` 

639 Data reference for patch. 

640 calExpRefList : `list` 

641 List of data references for input calexps. 

642 

643 Returns 

644 ------- 

645 tempExpRefList : `list` 

646 List of Warp/CoaddTempExp data references. 

647 """ 

648 butler = patchRef.getButler() 

649 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

650 self.getTempExpDatasetName(self.warpType)) 

651 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

652 g, groupData.keys) for 

653 g in groupData.groups.keys()] 

654 return tempExpRefList 

655 

656 def prepareInputs(self, refList): 

657 """Prepare the input warps for coaddition by measuring the weight for 

658 each warp and the scaling for the photometric zero point. 

659 

660 Each Warp has its own photometric zeropoint and background variance. 

661 Before coadding these Warps together, compute a scale factor to 

662 normalize the photometric zeropoint and compute the weight for each Warp. 

663 

664 Parameters 

665 ---------- 

666 refList : `list` 

667 List of data references to tempExp 

668 

669 Returns 

670 ------- 

671 result : `lsst.pipe.base.Struct` 

672 Result struct with components: 

673 

674 - ``tempExprefList``: `list` of data references to tempExp. 

675 - ``weightList``: `list` of weightings. 

676 - ``imageScalerList``: `list` of image scalers. 

677 """ 

678 statsCtrl = afwMath.StatisticsControl() 

679 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

680 statsCtrl.setNumIter(self.config.clipIter) 

681 statsCtrl.setAndMask(self.getBadPixelMask()) 

682 statsCtrl.setNanSafe(True) 

683 # compute tempExpRefList: a list of tempExpRef that actually exist 

684 # and weightList: a list of the weight of the associated coadd tempExp 

685 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

686 tempExpRefList = [] 

687 weightList = [] 

688 imageScalerList = [] 

689 tempExpName = self.getTempExpDatasetName(self.warpType) 

690 for tempExpRef in refList: 

691 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

692 # therefore have no datasetExists() method 

693 if not isinstance(tempExpRef, DeferredDatasetHandle): 693 ↛ 698line 693 didn't jump to line 698, because the condition on line 693 was never false

694 if not tempExpRef.datasetExists(tempExpName): 

695 self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

696 continue 

697 

698 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

699 # Ignore any input warp that is empty of data 

700 if numpy.isnan(tempExp.image.array).all(): 700 ↛ 701line 700 didn't jump to line 701, because the condition on line 700 was never true

701 continue 

702 maskedImage = tempExp.getMaskedImage() 

703 imageScaler = self.scaleZeroPoint.computeImageScaler( 

704 exposure=tempExp, 

705 dataRef=tempExpRef, 

706 ) 

707 try: 

708 imageScaler.scaleMaskedImage(maskedImage) 

709 except Exception as e: 

710 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

711 continue 

712 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

713 afwMath.MEANCLIP, statsCtrl) 

714 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

715 weight = 1.0 / float(meanVar) 

716 if not numpy.isfinite(weight): 716 ↛ 717line 716 didn't jump to line 717, because the condition on line 716 was never true

717 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

718 continue 

719 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

720 

721 del maskedImage 

722 del tempExp 

723 

724 tempExpRefList.append(tempExpRef) 

725 weightList.append(weight) 

726 imageScalerList.append(imageScaler) 

727 

728 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

729 imageScalerList=imageScalerList) 

730 

731 def prepareStats(self, mask=None): 

732 """Prepare the statistics for coadding images. 

733 

734 Parameters 

735 ---------- 

736 mask : `int`, optional 

737 Bit mask value to exclude from coaddition. 

738 

739 Returns 

740 ------- 

741 stats : `lsst.pipe.base.Struct` 

742 Statistics structure with the following fields: 

743 

744 - ``statsCtrl``: Statistics control object for coadd 

745 (`lsst.afw.math.StatisticsControl`) 

746 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

747 """ 

748 if mask is None: 

749 mask = self.getBadPixelMask() 

750 statsCtrl = afwMath.StatisticsControl() 

751 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

752 statsCtrl.setNumIter(self.config.clipIter) 

753 statsCtrl.setAndMask(mask) 

754 statsCtrl.setNanSafe(True) 

755 statsCtrl.setWeighted(True) 

756 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

757 for plane, threshold in self.config.maskPropagationThresholds.items(): 

758 bit = afwImage.Mask.getMaskPlane(plane) 

759 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

760 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

761 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

762 

763 @timeMethod 

764 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

765 altMaskList=None, mask=None, supplementaryData=None): 

766 """Assemble a coadd from input warps 

767 

768 Assemble the coadd using the provided list of coaddTempExps. Since 

769 the full coadd covers a patch (a large area), the assembly is 

770 performed over small areas on the image at a time in order to 

771 conserve memory usage. Iterate over subregions within the outer 

772 bbox of the patch using `assembleSubregion` to stack the corresponding 

773 subregions from the coaddTempExps with the statistic specified. 

774 Set the edge bits the coadd mask based on the weight map. 

775 

776 Parameters 

777 ---------- 

778 skyInfo : `lsst.pipe.base.Struct` 

779 Struct with geometric information about the patch. 

780 tempExpRefList : `list` 

781 List of data references to Warps (previously called CoaddTempExps). 

782 imageScalerList : `list` 

783 List of image scalers. 

784 weightList : `list` 

785 List of weights 

786 altMaskList : `list`, optional 

787 List of alternate masks to use rather than those stored with 

788 tempExp. 

789 mask : `int`, optional 

790 Bit mask value to exclude from coaddition. 

791 supplementaryData : lsst.pipe.base.Struct, optional 

792 Struct with additional data products needed to assemble coadd. 

793 Only used by subclasses that implement `makeSupplementaryData` 

794 and override `run`. 

795 

796 Returns 

797 ------- 

798 result : `lsst.pipe.base.Struct` 

799 Result struct with components: 

800 

801 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

802 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

803 - ``inputMap``: bit-wise map of inputs, if requested. 

804 - ``warpRefList``: input list of refs to the warps ( 

805 ``lsst.daf.butler.DeferredDatasetHandle`` or 

806 ``lsst.daf.persistence.ButlerDataRef``) 

807 (unmodified) 

808 - ``imageScalerList``: input list of image scalers (unmodified) 

809 - ``weightList``: input list of weights (unmodified) 

810 """ 

811 tempExpName = self.getTempExpDatasetName(self.warpType) 

812 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

813 stats = self.prepareStats(mask=mask) 

814 

815 if altMaskList is None: 

816 altMaskList = [None]*len(tempExpRefList) 

817 

818 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

819 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

820 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

821 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

822 coaddMaskedImage = coaddExposure.getMaskedImage() 

823 subregionSizeArr = self.config.subregionSize 

824 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

825 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

826 if self.config.doNImage: 826 ↛ 827line 826 didn't jump to line 827, because the condition on line 826 was never true

827 nImage = afwImage.ImageU(skyInfo.bbox) 

828 else: 

829 nImage = None 

830 # If inputMap is requested, create the initial version that can be masked in 

831 # assembleSubregion. 

832 if self.config.doInputMap: 832 ↛ 833line 832 didn't jump to line 833, because the condition on line 832 was never true

833 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

834 skyInfo.wcs, 

835 coaddExposure.getInfo().getCoaddInputs().ccds) 

836 

837 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 837 ↛ 838line 837 didn't jump to line 838, because the condition on line 837 was never true

838 try: 

839 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

840 weightList, altMaskList, stats.ctrl, 

841 nImage=nImage) 

842 except Exception as e: 

843 self.log.fatal("Cannot compute online coadd %s", e) 

844 else: 

845 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

846 try: 

847 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

848 weightList, altMaskList, stats.flags, stats.ctrl, 

849 nImage=nImage) 

850 except Exception as e: 

851 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

852 

853 # If inputMap is requested, we must finalize the map after the accumulation. 

854 if self.config.doInputMap: 854 ↛ 855line 854 didn't jump to line 855, because the condition on line 854 was never true

855 self.inputMapper.finalize_ccd_input_map_mask() 

856 inputMap = self.inputMapper.ccd_input_map 

857 else: 

858 inputMap = None 

859 

860 self.setInexactPsf(coaddMaskedImage.getMask()) 

861 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

862 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

863 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

864 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

865 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

866 weightList=weightList, inputMap=inputMap) 

867 

868 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

869 """Set the metadata for the coadd. 

870 

871 This basic implementation sets the filter from the first input. 

872 

873 Parameters 

874 ---------- 

875 coaddExposure : `lsst.afw.image.Exposure` 

876 The target exposure for the coadd. 

877 tempExpRefList : `list` 

878 List of data references to tempExp. 

879 weightList : `list` 

880 List of weights. 

881 """ 

882 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

883 tempExpName = self.getTempExpDatasetName(self.warpType) 

884 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

885 # (and we need more than just the PropertySet that contains the header), which is not possible 

886 # with the current butler (see #2777). 

887 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

888 

889 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 889 ↛ 891line 889 didn't jump to line 891, because the condition on line 889 was never true

890 # Gen 3 API 

891 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

892 else: 

893 # Gen 2 API. Delete this when Gen 2 retired 

894 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

895 for tempExpRef in tempExpRefList] 

896 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

897 

898 # Set the coadd FilterLabel to the band of the first input exposure: 

899 # Coadds are calibrated, so the physical label is now meaningless. 

900 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

901 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

902 coaddInputs.ccds.reserve(numCcds) 

903 coaddInputs.visits.reserve(len(tempExpList)) 

904 

905 for tempExp, weight in zip(tempExpList, weightList): 

906 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

907 

908 if self.config.doUsePsfMatchedPolygons: 

909 self.shrinkValidPolygons(coaddInputs) 

910 

911 coaddInputs.visits.sort() 

912 if self.warpType == "psfMatched": 

913 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

914 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

915 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

916 # having the maximum width (sufficient because square) 

917 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

918 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

919 for modelPsf in modelPsfList] 

920 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

921 else: 

922 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

923 self.config.coaddPsf.makeControl()) 

924 coaddExposure.setPsf(psf) 

925 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

926 coaddExposure.getWcs()) 

927 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

928 if self.config.doAttachTransmissionCurve: 

929 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

930 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

931 

932 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

933 altMaskList, statsFlags, statsCtrl, nImage=None): 

934 """Assemble the coadd for a sub-region. 

935 

936 For each coaddTempExp, check for (and swap in) an alternative mask 

937 if one is passed. Remove mask planes listed in 

938 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

939 `lsst.afw.math.statisticsStack` with the statistic specified by 

940 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

941 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

942 an N-sigma clipped mean where N and iterations are specified by 

943 statsCtrl. Assign the stacked subregion back to the coadd. 

944 

945 Parameters 

946 ---------- 

947 coaddExposure : `lsst.afw.image.Exposure` 

948 The target exposure for the coadd. 

949 bbox : `lsst.geom.Box` 

950 Sub-region to coadd. 

951 tempExpRefList : `list` 

952 List of data reference to tempExp. 

953 imageScalerList : `list` 

954 List of image scalers. 

955 weightList : `list` 

956 List of weights. 

957 altMaskList : `list` 

958 List of alternate masks to use rather than those stored with 

959 tempExp, or None. Each element is dict with keys = mask plane 

960 name to which to add the spans. 

961 statsFlags : `lsst.afw.math.Property` 

962 Property object for statistic for coadd. 

963 statsCtrl : `lsst.afw.math.StatisticsControl` 

964 Statistics control object for coadd. 

965 nImage : `lsst.afw.image.ImageU`, optional 

966 Keeps track of exposure count for each pixel. 

967 """ 

968 self.log.debug("Computing coadd over %s", bbox) 

969 tempExpName = self.getTempExpDatasetName(self.warpType) 

970 coaddExposure.mask.addMaskPlane("REJECTED") 

971 coaddExposure.mask.addMaskPlane("CLIPPED") 

972 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

973 maskMap = self.setRejectedMaskMapping(statsCtrl) 

974 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

975 maskedImageList = [] 

976 if nImage is not None: 976 ↛ 977line 976 didn't jump to line 977, because the condition on line 976 was never true

977 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

978 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

979 

980 if isinstance(tempExpRef, DeferredDatasetHandle): 980 ↛ 982line 980 didn't jump to line 982, because the condition on line 980 was never true

981 # Gen 3 API 

982 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

983 else: 

984 # Gen 2 API. Delete this when Gen 2 retired 

985 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

986 

987 maskedImage = exposure.getMaskedImage() 

988 mask = maskedImage.getMask() 

989 if altMask is not None: 

990 self.applyAltMaskPlanes(mask, altMask) 

991 imageScaler.scaleMaskedImage(maskedImage) 

992 

993 # Add 1 for each pixel which is not excluded by the exclude mask. 

994 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

995 if nImage is not None: 995 ↛ 996line 995 didn't jump to line 996, because the condition on line 995 was never true

996 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

997 if self.config.removeMaskPlanes: 997 ↛ 999line 997 didn't jump to line 999, because the condition on line 997 was never false

998 self.removeMaskPlanes(maskedImage) 

999 maskedImageList.append(maskedImage) 

1000 

1001 if self.config.doInputMap: 1001 ↛ 1002line 1001 didn't jump to line 1002, because the condition on line 1001 was never true

1002 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

1003 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

1004 

1005 with self.timer("stack"): 

1006 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

1007 clipped, # also set output to CLIPPED if sigma-clipped 

1008 maskMap) 

1009 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

1010 if nImage is not None: 1010 ↛ 1011line 1010 didn't jump to line 1011, because the condition on line 1010 was never true

1011 nImage.assign(subNImage, bbox) 

1012 

1013 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

1014 altMaskList, statsCtrl, nImage=None): 

1015 """Assemble the coadd using the "online" method. 

1016 

1017 This method takes a running sum of images and weights to save memory. 

1018 It only works for MEAN statistics. 

1019 

1020 Parameters 

1021 ---------- 

1022 coaddExposure : `lsst.afw.image.Exposure` 

1023 The target exposure for the coadd. 

1024 tempExpRefList : `list` 

1025 List of data reference to tempExp. 

1026 imageScalerList : `list` 

1027 List of image scalers. 

1028 weightList : `list` 

1029 List of weights. 

1030 altMaskList : `list` 

1031 List of alternate masks to use rather than those stored with 

1032 tempExp, or None. Each element is dict with keys = mask plane 

1033 name to which to add the spans. 

1034 statsCtrl : `lsst.afw.math.StatisticsControl` 

1035 Statistics control object for coadd 

1036 nImage : `lsst.afw.image.ImageU`, optional 

1037 Keeps track of exposure count for each pixel. 

1038 """ 

1039 self.log.debug("Computing online coadd.") 

1040 tempExpName = self.getTempExpDatasetName(self.warpType) 

1041 coaddExposure.mask.addMaskPlane("REJECTED") 

1042 coaddExposure.mask.addMaskPlane("CLIPPED") 

1043 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

1044 maskMap = self.setRejectedMaskMapping(statsCtrl) 

1045 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

1046 

1047 bbox = coaddExposure.maskedImage.getBBox() 

1048 

1049 stacker = AccumulatorMeanStack( 

1050 coaddExposure.image.array.shape, 

1051 statsCtrl.getAndMask(), 

1052 mask_threshold_dict=thresholdDict, 

1053 mask_map=maskMap, 

1054 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

1055 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

1056 compute_n_image=(nImage is not None) 

1057 ) 

1058 

1059 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

1060 imageScalerList, 

1061 altMaskList, 

1062 weightList): 

1063 if isinstance(tempExpRef, DeferredDatasetHandle): 

1064 # Gen 3 API 

1065 exposure = tempExpRef.get() 

1066 else: 

1067 # Gen 2 API. Delete this when Gen 2 retired 

1068 exposure = tempExpRef.get(tempExpName) 

1069 

1070 maskedImage = exposure.getMaskedImage() 

1071 mask = maskedImage.getMask() 

1072 if altMask is not None: 

1073 self.applyAltMaskPlanes(mask, altMask) 

1074 imageScaler.scaleMaskedImage(maskedImage) 

1075 if self.config.removeMaskPlanes: 

1076 self.removeMaskPlanes(maskedImage) 

1077 

1078 stacker.add_masked_image(maskedImage, weight=weight) 

1079 

1080 if self.config.doInputMap: 

1081 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

1082 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

1083 

1084 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

1085 

1086 if nImage is not None: 

1087 nImage.array[:, :] = stacker.n_image 

1088 

1089 def removeMaskPlanes(self, maskedImage): 

1090 """Unset the mask of an image for mask planes specified in the config. 

1091 

1092 Parameters 

1093 ---------- 

1094 maskedImage : `lsst.afw.image.MaskedImage` 

1095 The masked image to be modified. 

1096 """ 

1097 mask = maskedImage.getMask() 

1098 for maskPlane in self.config.removeMaskPlanes: 

1099 try: 

1100 mask &= ~mask.getPlaneBitMask(maskPlane) 

1101 except pexExceptions.InvalidParameterError: 

1102 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1103 maskPlane) 

1104 

1105 @staticmethod 

1106 def setRejectedMaskMapping(statsCtrl): 

1107 """Map certain mask planes of the warps to new planes for the coadd. 

1108 

1109 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1110 or CLIPPED, set it to REJECTED on the coadd. 

1111 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1112 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1113 

1114 Parameters 

1115 ---------- 

1116 statsCtrl : `lsst.afw.math.StatisticsControl` 

1117 Statistics control object for coadd 

1118 

1119 Returns 

1120 ------- 

1121 maskMap : `list` of `tuple` of `int` 

1122 A list of mappings of mask planes of the warped exposures to 

1123 mask planes of the coadd. 

1124 """ 

1125 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1126 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1127 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1128 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1129 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1130 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1131 (clipped, clipped)] 

1132 return maskMap 

1133 

1134 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1135 """Apply in place alt mask formatted as SpanSets to a mask. 

1136 

1137 Parameters 

1138 ---------- 

1139 mask : `lsst.afw.image.Mask` 

1140 Original mask. 

1141 altMaskSpans : `dict` 

1142 SpanSet lists to apply. Each element contains the new mask 

1143 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1144 and list of SpanSets to apply to the mask. 

1145 

1146 Returns 

1147 ------- 

1148 mask : `lsst.afw.image.Mask` 

1149 Updated mask. 

1150 """ 

1151 if self.config.doUsePsfMatchedPolygons: 

1152 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1152 ↛ 1160line 1152 didn't jump to line 1160, because the condition on line 1152 was never false

1153 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1154 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1155 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1156 # This mask-clearing step must only occur *before* applying the new masks below 

1157 for spanSet in altMaskSpans['NO_DATA']: 

1158 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1159 

1160 for plane, spanSetList in altMaskSpans.items(): 

1161 maskClipValue = mask.addMaskPlane(plane) 

1162 for spanSet in spanSetList: 

1163 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1164 return mask 

1165 

1166 def shrinkValidPolygons(self, coaddInputs): 

1167 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1168 

1169 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1170 does not have a validPolygon, create one from its bbox. 

1171 

1172 Parameters 

1173 ---------- 

1174 coaddInputs : `lsst.afw.image.coaddInputs` 

1175 Original mask. 

1176 

1177 """ 

1178 for ccd in coaddInputs.ccds: 

1179 polyOrig = ccd.getValidPolygon() 

1180 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1181 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1182 if polyOrig: 1182 ↛ 1183line 1182 didn't jump to line 1183, because the condition on line 1182 was never true

1183 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1184 else: 

1185 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1186 ccd.setValidPolygon(validPolygon) 

1187 

1188 def readBrightObjectMasks(self, dataRef): 

1189 """Retrieve the bright object masks. 

1190 

1191 Returns None on failure. 

1192 

1193 Parameters 

1194 ---------- 

1195 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1196 A Butler dataRef. 

1197 

1198 Returns 

1199 ------- 

1200 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1201 Bright object mask from the Butler object, or None if it cannot 

1202 be retrieved. 

1203 """ 

1204 try: 

1205 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1206 except Exception as e: 

1207 self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1208 return None 

1209 

1210 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1211 """Set the bright object masks. 

1212 

1213 Parameters 

1214 ---------- 

1215 exposure : `lsst.afw.image.Exposure` 

1216 Exposure under consideration. 

1217 dataId : `lsst.daf.persistence.dataId` 

1218 Data identifier dict for patch. 

1219 brightObjectMasks : `lsst.afw.table` 

1220 Table of bright objects to mask. 

1221 """ 

1222 

1223 if brightObjectMasks is None: 

1224 self.log.warning("Unable to apply bright object mask: none supplied") 

1225 return 

1226 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1227 mask = exposure.getMaskedImage().getMask() 

1228 wcs = exposure.getWcs() 

1229 plateScale = wcs.getPixelScale().asArcseconds() 

1230 

1231 for rec in brightObjectMasks: 

1232 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1233 if rec["type"] == "box": 

1234 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1235 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1236 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1237 

1238 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1239 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1240 

1241 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1242 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1243 spans = afwGeom.SpanSet(bbox) 

1244 elif rec["type"] == "circle": 

1245 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1246 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1247 else: 

1248 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1249 continue 

1250 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1251 

1252 def setInexactPsf(self, mask): 

1253 """Set INEXACT_PSF mask plane. 

1254 

1255 If any of the input images isn't represented in the coadd (due to 

1256 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1257 these pixels. 

1258 

1259 Parameters 

1260 ---------- 

1261 mask : `lsst.afw.image.Mask` 

1262 Coadded exposure's mask, modified in-place. 

1263 """ 

1264 mask.addMaskPlane("INEXACT_PSF") 

1265 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1266 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1267 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1268 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1269 array = mask.getArray() 

1270 selected = array & (sensorEdge | clipped | rejected) > 0 

1271 array[selected] |= inexactPsf 

1272 

1273 @classmethod 

1274 def _makeArgumentParser(cls): 

1275 """Create an argument parser. 

1276 """ 

1277 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1278 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1279 + cls.ConfigClass().warpType + "Warp", 

1280 help="data ID, e.g. --id tract=12345 patch=1,2", 

1281 ContainerClass=AssembleCoaddDataIdContainer) 

1282 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1283 ContainerClass=SelectDataIdContainer) 

1284 return parser 

1285 

1286 @staticmethod 

1287 def _subBBoxIter(bbox, subregionSize): 

1288 """Iterate over subregions of a bbox. 

1289 

1290 Parameters 

1291 ---------- 

1292 bbox : `lsst.geom.Box2I` 

1293 Bounding box over which to iterate. 

1294 subregionSize: `lsst.geom.Extent2I` 

1295 Size of sub-bboxes. 

1296 

1297 Yields 

1298 ------ 

1299 subBBox : `lsst.geom.Box2I` 

1300 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1301 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1302 the edges of ``bbox``, but it will never be empty. 

1303 """ 

1304 if bbox.isEmpty(): 1304 ↛ 1305line 1304 didn't jump to line 1305, because the condition on line 1304 was never true

1305 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1306 if subregionSize[0] < 1 or subregionSize[1] < 1: 1306 ↛ 1307line 1306 didn't jump to line 1307, because the condition on line 1306 was never true

1307 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1308 

1309 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1310 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1311 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1312 subBBox.clip(bbox) 

1313 if subBBox.isEmpty(): 1313 ↛ 1314line 1313 didn't jump to line 1314, because the condition on line 1313 was never true

1314 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1315 "colShift=%s, rowShift=%s" % 

1316 (bbox, subregionSize, colShift, rowShift)) 

1317 yield subBBox 

1318 

1319 def filterWarps(self, inputs, goodVisits): 

1320 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1321 

1322 Parameters 

1323 ---------- 

1324 inputs : list 

1325 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1326 goodVisit : `dict` 

1327 Dictionary with good visitIds as the keys. Value ignored. 

1328 

1329 Returns: 

1330 -------- 

1331 filteredInputs : `list` 

1332 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1333 """ 

1334 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1335 filteredInputs = [] 

1336 for visit in goodVisits.keys(): 

1337 if visit in inputWarpDict: 

1338 filteredInputs.append(inputWarpDict[visit]) 

1339 return filteredInputs 

1340 

1341 

1342class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1343 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1344 """ 

1345 

1346 def makeDataRefList(self, namespace): 

1347 """Make self.refList from self.idList. 

1348 

1349 Parameters 

1350 ---------- 

1351 namespace 

1352 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1353 """ 

1354 datasetType = namespace.config.coaddName + "Coadd" 

1355 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1356 

1357 for dataId in self.idList: 

1358 # tract and patch are required 

1359 for key in keysCoadd: 

1360 if key not in dataId: 

1361 raise RuntimeError("--id must include " + key) 

1362 

1363 dataRef = namespace.butler.dataRef( 

1364 datasetType=datasetType, 

1365 dataId=dataId, 

1366 ) 

1367 self.refList.append(dataRef) 

1368 

1369 

1370def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1371 """Function to count the number of pixels with a specific mask in a 

1372 footprint. 

1373 

1374 Find the intersection of mask & footprint. Count all pixels in the mask 

1375 that are in the intersection that have bitmask set but do not have 

1376 ignoreMask set. Return the count. 

1377 

1378 Parameters 

1379 ---------- 

1380 mask : `lsst.afw.image.Mask` 

1381 Mask to define intersection region by. 

1382 footprint : `lsst.afw.detection.Footprint` 

1383 Footprint to define the intersection region by. 

1384 bitmask 

1385 Specific mask that we wish to count the number of occurances of. 

1386 ignoreMask 

1387 Pixels to not consider. 

1388 

1389 Returns 

1390 ------- 

1391 result : `int` 

1392 Count of number of pixels in footprint with specified mask. 

1393 """ 

1394 bbox = footprint.getBBox() 

1395 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1396 fp = afwImage.Mask(bbox) 

1397 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1398 footprint.spans.setMask(fp, bitmask) 

1399 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1400 (subMask.getArray() & ignoreMask) == 0).sum() 

1401 

1402 

1403class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1404 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1405 """ 

1406 clipDetection = pexConfig.ConfigurableField( 

1407 target=SourceDetectionTask, 

1408 doc="Detect sources on difference between unclipped and clipped coadd") 

1409 minClipFootOverlap = pexConfig.Field( 

1410 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1411 dtype=float, 

1412 default=0.6 

1413 ) 

1414 minClipFootOverlapSingle = pexConfig.Field( 

1415 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1416 "clipped when only one visit overlaps", 

1417 dtype=float, 

1418 default=0.5 

1419 ) 

1420 minClipFootOverlapDouble = pexConfig.Field( 

1421 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1422 "clipped when two visits overlap", 

1423 dtype=float, 

1424 default=0.45 

1425 ) 

1426 maxClipFootOverlapDouble = pexConfig.Field( 

1427 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1428 "considering two visits", 

1429 dtype=float, 

1430 default=0.15 

1431 ) 

1432 minBigOverlap = pexConfig.Field( 

1433 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1434 "when labeling clipped footprints", 

1435 dtype=int, 

1436 default=100 

1437 ) 

1438 

1439 def setDefaults(self): 

1440 """Set default values for clipDetection. 

1441 

1442 Notes 

1443 ----- 

1444 The numeric values for these configuration parameters were 

1445 empirically determined, future work may further refine them. 

1446 """ 

1447 AssembleCoaddConfig.setDefaults(self) 

1448 self.clipDetection.doTempLocalBackground = False 

1449 self.clipDetection.reEstimateBackground = False 

1450 self.clipDetection.returnOriginalFootprints = False 

1451 self.clipDetection.thresholdPolarity = "both" 

1452 self.clipDetection.thresholdValue = 2 

1453 self.clipDetection.nSigmaToGrow = 2 

1454 self.clipDetection.minPixels = 4 

1455 self.clipDetection.isotropicGrow = True 

1456 self.clipDetection.thresholdType = "pixel_stdev" 

1457 self.sigmaClip = 1.5 

1458 self.clipIter = 3 

1459 self.statistic = "MEAN" 

1460 

1461 def validate(self): 

1462 if self.doSigmaClip: 

1463 log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1464 "Ignoring doSigmaClip.") 

1465 self.doSigmaClip = False 

1466 if self.statistic != "MEAN": 

1467 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1468 "(%s chosen). Please set statistic to MEAN." 

1469 % (self.statistic)) 

1470 AssembleCoaddTask.ConfigClass.validate(self) 

1471 

1472 

1473class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1474 """Assemble a coadded image from a set of coadded temporary exposures, 

1475 being careful to clip & flag areas with potential artifacts. 

1476 

1477 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1478 we clip outliers). The problem with doing this is that when computing the 

1479 coadd PSF at a given location, individual visit PSFs from visits with 

1480 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1481 In this task, we correct for this behavior by creating a new 

1482 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1483 coaddTempExps and the final coadd where 

1484 

1485 i. difference imaging suggests that there is an outlier and 

1486 ii. this outlier appears on only one or two images. 

1487 

1488 Such regions will not contribute to the final coadd. Furthermore, any 

1489 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1490 Note that the algorithm implemented by this task is preliminary and works 

1491 correctly for HSC data. Parameter modifications and or considerable 

1492 redesigning of the algorithm is likley required for other surveys. 

1493 

1494 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1495 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1496 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1497 if you wish. 

1498 

1499 Notes 

1500 ----- 

1501 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1502 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1503 see `baseDebug` for more about ``debug.py`` files. 

1504 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1505 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1506 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1507 for further information. 

1508 

1509 Examples 

1510 -------- 

1511 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1512 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1513 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1514 

1515 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1516 and filter to be coadded (specified using 

1517 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1518 along with a list of coaddTempExps to attempt to coadd (specified using 

1519 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1520 Only the coaddTempExps that cover the specified tract and patch will be 

1521 coadded. A list of the available optional arguments can be obtained by 

1522 calling assembleCoadd.py with the --help command line argument: 

1523 

1524 .. code-block:: none 

1525 

1526 assembleCoadd.py --help 

1527 

1528 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1529 context of multi-band processing, we will generate the HSC-I & -R band 

1530 coadds from HSC engineering test data provided in the ci_hsc package. 

1531 To begin, assuming that the lsst stack has been already set up, we must 

1532 set up the obs_subaru and ci_hsc packages. This defines the environment 

1533 variable $CI_HSC_DIR and points at the location of the package. The raw 

1534 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1535 the coadds, we must first 

1536 

1537 - ``processCcd`` 

1538 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1539 - ``makeSkyMap`` 

1540 create a skymap that covers the area of the sky present in the raw exposures 

1541 - ``makeCoaddTempExp`` 

1542 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1543 

1544 We can perform all of these steps by running 

1545 

1546 .. code-block:: none 

1547 

1548 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1549 

1550 This will produce warped coaddTempExps for each visit. To coadd the 

1551 warped data, we call ``assembleCoadd.py`` as follows: 

1552 

1553 .. code-block:: none 

1554 

1555 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1556 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1557 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1558 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1559 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1560 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1561 --selectId visit=903988 ccd=24 

1562 

1563 This will process the HSC-I band data. The results are written in 

1564 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1565 

1566 You may also choose to run: 

1567 

1568 .. code-block:: none 

1569 

1570 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1571 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1572 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1573 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1574 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1575 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1576 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1577 --selectId visit=903346 ccd=12 

1578 

1579 to generate the coadd for the HSC-R band if you are interested in following 

1580 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1581 """ 

1582 ConfigClass = SafeClipAssembleCoaddConfig 

1583 _DefaultName = "safeClipAssembleCoadd" 

1584 

1585 def __init__(self, *args, **kwargs): 

1586 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1587 schema = afwTable.SourceTable.makeMinimalSchema() 

1588 self.makeSubtask("clipDetection", schema=schema) 

1589 

1590 @utils.inheritDoc(AssembleCoaddTask) 

1591 @timeMethod 

1592 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1593 """Assemble the coadd for a region. 

1594 

1595 Compute the difference of coadds created with and without outlier 

1596 rejection to identify coadd pixels that have outlier values in some 

1597 individual visits. 

1598 Detect clipped regions on the difference image and mark these regions 

1599 on the one or two individual coaddTempExps where they occur if there 

1600 is significant overlap between the clipped region and a source. This 

1601 leaves us with a set of footprints from the difference image that have 

1602 been identified as having occured on just one or two individual visits. 

1603 However, these footprints were generated from a difference image. It 

1604 is conceivable for a large diffuse source to have become broken up 

1605 into multiple footprints acrosss the coadd difference in this process. 

1606 Determine the clipped region from all overlapping footprints from the 

1607 detected sources in each visit - these are big footprints. 

1608 Combine the small and big clipped footprints and mark them on a new 

1609 bad mask plane. 

1610 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1611 removal. Clipped footprints will no longer make it into the coadd 

1612 because they are marked in the new bad mask plane. 

1613 

1614 Notes 

1615 ----- 

1616 args and kwargs are passed but ignored in order to match the call 

1617 signature expected by the parent task. 

1618 """ 

1619 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1620 mask = exp.getMaskedImage().getMask() 

1621 mask.addMaskPlane("CLIPPED") 

1622 

1623 result = self.detectClip(exp, tempExpRefList) 

1624 

1625 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1626 

1627 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1628 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1629 # Append big footprints from individual Warps to result.clipSpans 

1630 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1631 result.detectionFootprints, maskClipValue, maskDetValue, 

1632 exp.getBBox()) 

1633 # Create mask of the current clipped footprints 

1634 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1635 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1636 

1637 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1638 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1639 maskClip |= maskClipBig 

1640 

1641 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1642 badMaskPlanes = self.config.badMaskPlanes[:] 

1643 badMaskPlanes.append("CLIPPED") 

1644 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1645 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1646 result.clipSpans, mask=badPixelMask) 

1647 

1648 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1649 """Return an exposure that contains the difference between unclipped 

1650 and clipped coadds. 

1651 

1652 Generate a difference image between clipped and unclipped coadds. 

1653 Compute the difference image by subtracting an outlier-clipped coadd 

1654 from an outlier-unclipped coadd. Return the difference image. 

1655 

1656 Parameters 

1657 ---------- 

1658 skyInfo : `lsst.pipe.base.Struct` 

1659 Patch geometry information, from getSkyInfo 

1660 tempExpRefList : `list` 

1661 List of data reference to tempExp 

1662 imageScalerList : `list` 

1663 List of image scalers 

1664 weightList : `list` 

1665 List of weights 

1666 

1667 Returns 

1668 ------- 

1669 exp : `lsst.afw.image.Exposure` 

1670 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1671 """ 

1672 config = AssembleCoaddConfig() 

1673 # getattr necessary because subtasks do not survive Config.toDict() 

1674 # exclude connections because the class of self.config.connections is not 

1675 # the same as AssembleCoaddConfig.connections, and the connections are not 

1676 # needed to run this task anyway. 

1677 configIntersection = {k: getattr(self.config, k) 

1678 for k, v in self.config.toDict().items() 

1679 if (k in config.keys() and k != "connections")} 

1680 configIntersection['doInputMap'] = False 

1681 configIntersection['doNImage'] = False 

1682 config.update(**configIntersection) 

1683 

1684 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1685 config.statistic = 'MEAN' 

1686 task = AssembleCoaddTask(config=config) 

1687 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1688 

1689 config.statistic = 'MEANCLIP' 

1690 task = AssembleCoaddTask(config=config) 

1691 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1692 

1693 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1694 coaddDiff -= coaddClip.getMaskedImage() 

1695 exp = afwImage.ExposureF(coaddDiff) 

1696 exp.setPsf(coaddMean.getPsf()) 

1697 return exp 

1698 

1699 def detectClip(self, exp, tempExpRefList): 

1700 """Detect clipped regions on an exposure and set the mask on the 

1701 individual tempExp masks. 

1702 

1703 Detect footprints in the difference image after smoothing the 

1704 difference image with a Gaussian kernal. Identify footprints that 

1705 overlap with one or two input ``coaddTempExps`` by comparing the 

1706 computed overlap fraction to thresholds set in the config. A different 

1707 threshold is applied depending on the number of overlapping visits 

1708 (restricted to one or two). If the overlap exceeds the thresholds, 

1709 the footprint is considered "CLIPPED" and is marked as such on the 

1710 coaddTempExp. Return a struct with the clipped footprints, the indices 

1711 of the ``coaddTempExps`` that end up overlapping with the clipped 

1712 footprints, and a list of new masks for the ``coaddTempExps``. 

1713 

1714 Parameters 

1715 ---------- 

1716 exp : `lsst.afw.image.Exposure` 

1717 Exposure to run detection on. 

1718 tempExpRefList : `list` 

1719 List of data reference to tempExp. 

1720 

1721 Returns 

1722 ------- 

1723 result : `lsst.pipe.base.Struct` 

1724 Result struct with components: 

1725 

1726 - ``clipFootprints``: list of clipped footprints. 

1727 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1728 ``tempExpRefList``. 

1729 - ``clipSpans``: List of dictionaries containing spanSet lists 

1730 to clip. Each element contains the new maskplane name 

1731 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1732 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1733 compressed into footprints. 

1734 """ 

1735 mask = exp.getMaskedImage().getMask() 

1736 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1737 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1738 # Merge positive and negative together footprints together 

1739 fpSet.positive.merge(fpSet.negative) 

1740 footprints = fpSet.positive 

1741 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1742 ignoreMask = self.getBadPixelMask() 

1743 

1744 clipFootprints = [] 

1745 clipIndices = [] 

1746 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1747 

1748 # for use by detectClipBig 

1749 visitDetectionFootprints = [] 

1750 

1751 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1752 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1753 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1754 

1755 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1756 for i, warpRef in enumerate(tempExpRefList): 

1757 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1758 immediate=True).getMaskedImage().getMask() 

1759 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1760 afwImage.PARENT, True) 

1761 maskVisitDet &= maskDetValue 

1762 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1763 visitDetectionFootprints.append(visitFootprints) 

1764 

1765 for j, footprint in enumerate(footprints.getFootprints()): 

1766 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1767 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1768 

1769 # build a list of clipped spans for each visit 

1770 for j, footprint in enumerate(footprints.getFootprints()): 

1771 nPixel = footprint.getArea() 

1772 overlap = [] # hold the overlap with each visit 

1773 indexList = [] # index of visit in global list 

1774 for i in range(len(tempExpRefList)): 

1775 ignore = ignoreArr[i, j] 

1776 overlapDet = overlapDetArr[i, j] 

1777 totPixel = nPixel - ignore 

1778 

1779 # If we have more bad pixels than detection skip 

1780 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1780 ↛ 1782line 1780 didn't jump to line 1782, because the condition on line 1780 was never false

1781 continue 

1782 overlap.append(overlapDet/float(totPixel)) 

1783 indexList.append(i) 

1784 

1785 overlap = numpy.array(overlap) 

1786 if not len(overlap): 1786 ↛ 1789line 1786 didn't jump to line 1789, because the condition on line 1786 was never false

1787 continue 

1788 

1789 keep = False # Should this footprint be marked as clipped? 

1790 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1791 

1792 # If footprint only has one overlap use a lower threshold 

1793 if len(overlap) == 1: 

1794 if overlap[0] > self.config.minClipFootOverlapSingle: 

1795 keep = True 

1796 keepIndex = [0] 

1797 else: 

1798 # This is the general case where only visit should be clipped 

1799 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1800 if len(clipIndex) == 1: 

1801 keep = True 

1802 keepIndex = [clipIndex[0]] 

1803 

1804 # Test if there are clipped objects that overlap two different visits 

1805 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1806 if len(clipIndex) == 2 and len(overlap) > 3: 

1807 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1808 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1809 keep = True 

1810 keepIndex = clipIndex 

1811 

1812 if not keep: 

1813 continue 

1814 

1815 for index in keepIndex: 

1816 globalIndex = indexList[index] 

1817 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1818 

1819 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1820 clipFootprints.append(footprint) 

1821 

1822 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1823 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1824 

1825 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1826 maskClipValue, maskDetValue, coaddBBox): 

1827 """Return individual warp footprints for large artifacts and append 

1828 them to ``clipList`` in place. 

1829 

1830 Identify big footprints composed of many sources in the coadd 

1831 difference that may have originated in a large diffuse source in the 

1832 coadd. We do this by indentifying all clipped footprints that overlap 

1833 significantly with each source in all the coaddTempExps. 

1834 

1835 Parameters 

1836 ---------- 

1837 clipList : `list` 

1838 List of alt mask SpanSets with clipping information. Modified. 

1839 clipFootprints : `list` 

1840 List of clipped footprints. 

1841 clipIndices : `list` 

1842 List of which entries in tempExpClipList each footprint belongs to. 

1843 maskClipValue 

1844 Mask value of clipped pixels. 

1845 maskDetValue 

1846 Mask value of detected pixels. 

1847 coaddBBox : `lsst.geom.Box` 

1848 BBox of the coadd and warps. 

1849 

1850 Returns 

1851 ------- 

1852 bigFootprintsCoadd : `list` 

1853 List of big footprints 

1854 """ 

1855 bigFootprintsCoadd = [] 

1856 ignoreMask = self.getBadPixelMask() 

1857 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1858 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1859 for footprint in visitFootprints.getFootprints(): 1859 ↛ 1860line 1859 didn't jump to line 1860, because the loop on line 1859 never started

1860 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1861 

1862 # build a mask of clipped footprints that are in this visit 

1863 clippedFootprintsVisit = [] 

1864 for foot, clipIndex in zip(clipFootprints, clipIndices): 1864 ↛ 1865line 1864 didn't jump to line 1865, because the loop on line 1864 never started

1865 if index not in clipIndex: 

1866 continue 

1867 clippedFootprintsVisit.append(foot) 

1868 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1869 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1870 

1871 bigFootprintsVisit = [] 

1872 for foot in visitFootprints.getFootprints(): 1872 ↛ 1873line 1872 didn't jump to line 1873, because the loop on line 1872 never started

1873 if foot.getArea() < self.config.minBigOverlap: 

1874 continue 

1875 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1876 if nCount > self.config.minBigOverlap: 

1877 bigFootprintsVisit.append(foot) 

1878 bigFootprintsCoadd.append(foot) 

1879 

1880 for footprint in bigFootprintsVisit: 1880 ↛ 1881line 1880 didn't jump to line 1881, because the loop on line 1880 never started

1881 clippedSpans["CLIPPED"].append(footprint.spans) 

1882 

1883 return bigFootprintsCoadd 

1884 

1885 

1886class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1887 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1888 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1889 "Only PSF-Matched Warps make sense for image subtraction. " 

1890 "Therefore, they must be an additional declared input."), 

1891 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1892 storageClass="ExposureF", 

1893 dimensions=("tract", "patch", "skymap", "visit"), 

1894 deferLoad=True, 

1895 multiple=True 

1896 ) 

1897 templateCoadd = pipeBase.connectionTypes.Output( 

1898 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1899 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1900 name="{outputCoaddName}CoaddPsfMatched", 

1901 storageClass="ExposureF", 

1902 dimensions=("tract", "patch", "skymap", "band"), 

1903 ) 

1904 

1905 def __init__(self, *, config=None): 

1906 super().__init__(config=config) 

1907 if not config.assembleStaticSkyModel.doWrite: 

1908 self.outputs.remove("templateCoadd") 

1909 config.validate() 

1910 

1911 

1912class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1913 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1914 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1915 target=AssembleCoaddTask, 

1916 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1917 " naive/first-iteration model of the static sky.", 

1918 ) 

1919 detect = pexConfig.ConfigurableField( 

1920 target=SourceDetectionTask, 

1921 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1922 ) 

1923 detectTemplate = pexConfig.ConfigurableField( 

1924 target=SourceDetectionTask, 

1925 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1926 ) 

1927 maskStreaks = pexConfig.ConfigurableField( 

1928 target=MaskStreaksTask, 

1929 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1930 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1931 "streakMaskName" 

1932 ) 

1933 streakMaskName = pexConfig.Field( 

1934 dtype=str, 

1935 default="STREAK", 

1936 doc="Name of mask bit used for streaks" 

1937 ) 

1938 maxNumEpochs = pexConfig.Field( 

1939 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1940 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1941 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1942 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1943 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1944 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1945 "than transient and not masked.", 

1946 dtype=int, 

1947 default=2 

1948 ) 

1949 maxFractionEpochsLow = pexConfig.RangeField( 

1950 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1951 "Effective maxNumEpochs = " 

1952 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1953 dtype=float, 

1954 default=0.4, 

1955 min=0., max=1., 

1956 ) 

1957 maxFractionEpochsHigh = pexConfig.RangeField( 

1958 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1959 "Effective maxNumEpochs = " 

1960 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1961 dtype=float, 

1962 default=0.03, 

1963 min=0., max=1., 

1964 ) 

1965 spatialThreshold = pexConfig.RangeField( 

1966 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1967 "temporal criteria. If 0, clip all. If 1, clip none.", 

1968 dtype=float, 

1969 default=0.5, 

1970 min=0., max=1., 

1971 inclusiveMin=True, inclusiveMax=True 

1972 ) 

1973 doScaleWarpVariance = pexConfig.Field( 

1974 doc="Rescale Warp variance plane using empirical noise?", 

1975 dtype=bool, 

1976 default=True, 

1977 ) 

1978 scaleWarpVariance = pexConfig.ConfigurableField( 

1979 target=ScaleVarianceTask, 

1980 doc="Rescale variance on warps", 

1981 ) 

1982 doPreserveContainedBySource = pexConfig.Field( 

1983 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1984 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1985 dtype=bool, 

1986 default=True, 

1987 ) 

1988 doPrefilterArtifacts = pexConfig.Field( 

1989 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1990 "because they will be excluded anyway. This prevents them from contributing " 

1991 "to the outlier epoch count image and potentially being labeled as persistant." 

1992 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1993 dtype=bool, 

1994 default=True 

1995 ) 

1996 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1997 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1998 dtype=str, 

1999 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

2000 ) 

2001 prefilterArtifactsRatio = pexConfig.Field( 

2002 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

2003 dtype=float, 

2004 default=0.05 

2005 ) 

2006 doFilterMorphological = pexConfig.Field( 

2007 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

2008 "be streaks.", 

2009 dtype=bool, 

2010 default=False 

2011 ) 

2012 

2013 def setDefaults(self): 

2014 AssembleCoaddConfig.setDefaults(self) 

2015 self.statistic = 'MEAN' 

2016 self.doUsePsfMatchedPolygons = True 

2017 

2018 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

2019 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

2020 if "EDGE" in self.badMaskPlanes: 2020 ↛ 2022line 2020 didn't jump to line 2022, because the condition on line 2020 was never false

2021 self.badMaskPlanes.remove('EDGE') 

2022 self.removeMaskPlanes.append('EDGE') 

2023 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

2024 self.assembleStaticSkyModel.warpType = 'psfMatched' 

2025 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

2026 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

2027 self.assembleStaticSkyModel.sigmaClip = 2.5 

2028 self.assembleStaticSkyModel.clipIter = 3 

2029 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

2030 self.assembleStaticSkyModel.doWrite = False 

2031 self.detect.doTempLocalBackground = False 

2032 self.detect.reEstimateBackground = False 

2033 self.detect.returnOriginalFootprints = False 

2034 self.detect.thresholdPolarity = "both" 

2035 self.detect.thresholdValue = 5 

2036 self.detect.minPixels = 4 

2037 self.detect.isotropicGrow = True 

2038 self.detect.thresholdType = "pixel_stdev" 

2039 self.detect.nSigmaToGrow = 0.4 

2040 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

2041 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

2042 self.detectTemplate.nSigmaToGrow = 2.4 

2043 self.detectTemplate.doTempLocalBackground = False 

2044 self.detectTemplate.reEstimateBackground = False 

2045 self.detectTemplate.returnOriginalFootprints = False 

2046 

2047 def validate(self): 

2048 super().validate() 

2049 if self.assembleStaticSkyModel.doNImage: 

2050 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

2051 "Please set assembleStaticSkyModel.doNImage=False") 

2052 

2053 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

2054 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

2055 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

2056 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

2057 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

2058 

2059 

2060class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

2061 """Assemble a compareWarp coadded image from a set of warps 

2062 by masking artifacts detected by comparing PSF-matched warps. 

2063 

2064 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

2065 we clip outliers). The problem with doing this is that when computing the 

2066 coadd PSF at a given location, individual visit PSFs from visits with 

2067 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

2068 In this task, we correct for this behavior by creating a new badMaskPlane 

2069 'CLIPPED' which marks pixels in the individual warps suspected to contain 

2070 an artifact. We populate this plane on the input warps by comparing 

2071 PSF-matched warps with a PSF-matched median coadd which serves as a 

2072 model of the static sky. Any group of pixels that deviates from the 

2073 PSF-matched template coadd by more than config.detect.threshold sigma, 

2074 is an artifact candidate. The candidates are then filtered to remove 

2075 variable sources and sources that are difficult to subtract such as 

2076 bright stars. This filter is configured using the config parameters 

2077 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

2078 the maximum fraction of epochs that the deviation can appear in and still 

2079 be considered an artifact. The spatialThreshold is the maximum fraction of 

2080 pixels in the footprint of the deviation that appear in other epochs 

2081 (where other epochs is defined by the temporalThreshold). If the deviant 

2082 region meets this criteria of having a significant percentage of pixels 

2083 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

2084 set in the mask. These regions will not contribute to the final coadd. 

2085 Furthermore, any routine to determine the coadd PSF can now be cognizant 

2086 of clipped regions. Note that the algorithm implemented by this task is 

2087 preliminary and works correctly for HSC data. Parameter modifications and 

2088 or considerable redesigning of the algorithm is likley required for other 

2089 surveys. 

2090 

2091 ``CompareWarpAssembleCoaddTask`` sub-classes 

2092 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

2093 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2094 

2095 Notes 

2096 ----- 

2097 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2098 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2099 ``baseDebug`` for more about ``debug.py`` files. 

2100 

2101 This task supports the following debug variables: 

2102 

2103 - ``saveCountIm`` 

2104 If True then save the Epoch Count Image as a fits file in the `figPath` 

2105 - ``figPath`` 

2106 Path to save the debug fits images and figures 

2107 

2108 For example, put something like: 

2109 

2110 .. code-block:: python 

2111 

2112 import lsstDebug 

2113 def DebugInfo(name): 

2114 di = lsstDebug.getInfo(name) 

2115 if name == "lsst.pipe.tasks.assembleCoadd": 

2116 di.saveCountIm = True 

2117 di.figPath = "/desired/path/to/debugging/output/images" 

2118 return di 

2119 lsstDebug.Info = DebugInfo 

2120 

2121 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2122 ``--debug`` flag. Some subtasks may have their own debug variables; 

2123 see individual Task documentation. 

2124 

2125 Examples 

2126 -------- 

2127 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2128 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2129 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2130 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2131 and filter to be coadded (specified using 

2132 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2133 along with a list of coaddTempExps to attempt to coadd (specified using 

2134 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2135 Only the warps that cover the specified tract and patch will be coadded. 

2136 A list of the available optional arguments can be obtained by calling 

2137 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2138 

2139 .. code-block:: none 

2140 

2141 assembleCoadd.py --help 

2142 

2143 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2144 context of multi-band processing, we will generate the HSC-I & -R band 

2145 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2146 To begin, assuming that the lsst stack has been already set up, we must 

2147 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2148 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2149 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2150 directory. To begin assembling the coadds, we must first 

2151 

2152 - processCcd 

2153 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2154 - makeSkyMap 

2155 create a skymap that covers the area of the sky present in the raw exposures 

2156 - makeCoaddTempExp 

2157 warp the individual calibrated exposures to the tangent plane of the coadd 

2158 

2159 We can perform all of these steps by running 

2160 

2161 .. code-block:: none 

2162 

2163 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2164 

2165 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2166 warped data, we call ``assembleCoadd.py`` as follows: 

2167 

2168 .. code-block:: none 

2169 

2170 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2171 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2172 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2173 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2174 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2175 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2176 --selectId visit=903988 ccd=24 

2177 

2178 This will process the HSC-I band data. The results are written in 

2179 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2180 """ 

2181 ConfigClass = CompareWarpAssembleCoaddConfig 

2182 _DefaultName = "compareWarpAssembleCoadd" 

2183 

2184 def __init__(self, *args, **kwargs): 

2185 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2186 self.makeSubtask("assembleStaticSkyModel") 

2187 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2188 self.makeSubtask("detect", schema=detectionSchema) 

2189 if self.config.doPreserveContainedBySource: 2189 ↛ 2191line 2189 didn't jump to line 2191, because the condition on line 2189 was never false

2190 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2191 if self.config.doScaleWarpVariance: 2191 ↛ 2193line 2191 didn't jump to line 2193, because the condition on line 2191 was never false

2192 self.makeSubtask("scaleWarpVariance") 

2193 if self.config.doFilterMorphological: 2193 ↛ 2194line 2193 didn't jump to line 2194, because the condition on line 2193 was never true

2194 self.makeSubtask("maskStreaks") 

2195 

2196 @utils.inheritDoc(AssembleCoaddTask) 

2197 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2198 """ 

2199 Generate a templateCoadd to use as a naive model of static sky to 

2200 subtract from PSF-Matched warps. 

2201 

2202 Returns 

2203 ------- 

2204 result : `lsst.pipe.base.Struct` 

2205 Result struct with components: 

2206 

2207 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2208 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2209 """ 

2210 # Ensure that psfMatchedWarps are used as input warps for template generation 

2211 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2212 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2213 

2214 # Because subtasks don't have connections we have to make one. 

2215 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2216 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2217 if self.config.assembleStaticSkyModel.doWrite: 

2218 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2219 # Remove template coadd from both subtask's and main tasks outputs, 

2220 # because it is handled by the subtask as `coaddExposure` 

2221 del outputRefs.templateCoadd 

2222 del staticSkyModelOutputRefs.templateCoadd 

2223 

2224 # A PSF-Matched nImage does not exist as a dataset type 

2225 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2226 del staticSkyModelOutputRefs.nImage 

2227 

2228 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2229 staticSkyModelOutputRefs) 

2230 if templateCoadd is None: 

2231 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2232 

2233 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2234 nImage=templateCoadd.nImage, 

2235 warpRefList=templateCoadd.warpRefList, 

2236 imageScalerList=templateCoadd.imageScalerList, 

2237 weightList=templateCoadd.weightList) 

2238 

2239 @utils.inheritDoc(AssembleCoaddTask) 

2240 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2241 """ 

2242 Generate a templateCoadd to use as a naive model of static sky to 

2243 subtract from PSF-Matched warps. 

2244 

2245 Returns 

2246 ------- 

2247 result : `lsst.pipe.base.Struct` 

2248 Result struct with components: 

2249 

2250 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2251 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2252 """ 

2253 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2254 if templateCoadd is None: 2254 ↛ 2255line 2254 didn't jump to line 2255, because the condition on line 2254 was never true

2255 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2256 

2257 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2258 nImage=templateCoadd.nImage, 

2259 warpRefList=templateCoadd.warpRefList, 

2260 imageScalerList=templateCoadd.imageScalerList, 

2261 weightList=templateCoadd.weightList) 

2262 

2263 def _noTemplateMessage(self, warpType): 

2264 warpName = (warpType[0].upper() + warpType[1:]) 

2265 message = """No %(warpName)s warps were found to build the template coadd which is 

2266 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2267 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2268 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2269 

2270 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2271 another algorithm like: 

2272 

2273 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2274 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2275 """ % {"warpName": warpName} 

2276 return message 

2277 

2278 @utils.inheritDoc(AssembleCoaddTask) 

2279 @timeMethod 

2280 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2281 supplementaryData, *args, **kwargs): 

2282 """Assemble the coadd. 

2283 

2284 Find artifacts and apply them to the warps' masks creating a list of 

2285 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2286 plane. Then pass these alternative masks to the base class's `run` 

2287 method. 

2288 

2289 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2290 that must contain a ``templateCoadd`` that serves as the 

2291 model of the static sky. 

2292 """ 

2293 

2294 # Check and match the order of the supplementaryData 

2295 # (PSF-matched) inputs to the order of the direct inputs, 

2296 # so that the artifact mask is applied to the right warp 

2297 dataIds = [ref.dataId for ref in tempExpRefList] 

2298 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2299 

2300 if dataIds != psfMatchedDataIds: 

2301 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2302 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2303 psfMatchedDataIds, dataIds) 

2304 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2305 psfMatchedDataIds, dataIds) 

2306 

2307 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2308 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2309 supplementaryData.warpRefList, 

2310 supplementaryData.imageScalerList) 

2311 

2312 badMaskPlanes = self.config.badMaskPlanes[:] 

2313 badMaskPlanes.append("CLIPPED") 

2314 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2315 

2316 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2317 spanSetMaskList, mask=badPixelMask) 

2318 

2319 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2320 # Psf-Matching moves the real edge inwards 

2321 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2322 return result 

2323 

2324 def applyAltEdgeMask(self, mask, altMaskList): 

2325 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2326 

2327 Parameters 

2328 ---------- 

2329 mask : `lsst.afw.image.Mask` 

2330 Original mask. 

2331 altMaskList : `list` 

2332 List of Dicts containing ``spanSet`` lists. 

2333 Each element contains the new mask plane name (e.g. "CLIPPED 

2334 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2335 the mask. 

2336 """ 

2337 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2338 for visitMask in altMaskList: 

2339 if "EDGE" in visitMask: 2339 ↛ 2338line 2339 didn't jump to line 2338, because the condition on line 2339 was never false

2340 for spanSet in visitMask['EDGE']: 

2341 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2342 

2343 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2344 """Find artifacts. 

2345 

2346 Loop through warps twice. The first loop builds a map with the count 

2347 of how many epochs each pixel deviates from the templateCoadd by more 

2348 than ``config.chiThreshold`` sigma. The second loop takes each 

2349 difference image and filters the artifacts detected in each using 

2350 count map to filter out variable sources and sources that are 

2351 difficult to subtract cleanly. 

2352 

2353 Parameters 

2354 ---------- 

2355 templateCoadd : `lsst.afw.image.Exposure` 

2356 Exposure to serve as model of static sky. 

2357 tempExpRefList : `list` 

2358 List of data references to warps. 

2359 imageScalerList : `list` 

2360 List of image scalers. 

2361 

2362 Returns 

2363 ------- 

2364 altMasks : `list` 

2365 List of dicts containing information about CLIPPED 

2366 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2367 """ 

2368 

2369 self.log.debug("Generating Count Image, and mask lists.") 

2370 coaddBBox = templateCoadd.getBBox() 

2371 slateIm = afwImage.ImageU(coaddBBox) 

2372 epochCountImage = afwImage.ImageU(coaddBBox) 

2373 nImage = afwImage.ImageU(coaddBBox) 

2374 spanSetArtifactList = [] 

2375 spanSetNoDataMaskList = [] 

2376 spanSetEdgeList = [] 

2377 spanSetBadMorphoList = [] 

2378 badPixelMask = self.getBadPixelMask() 

2379 

2380 # mask of the warp diffs should = that of only the warp 

2381 templateCoadd.mask.clearAllMaskPlanes() 

2382 

2383 if self.config.doPreserveContainedBySource: 2383 ↛ 2386line 2383 didn't jump to line 2386, because the condition on line 2383 was never false

2384 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2385 else: 

2386 templateFootprints = None 

2387 

2388 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2389 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2390 if warpDiffExp is not None: 

2391 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2392 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2393 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2394 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2395 fpSet.positive.merge(fpSet.negative) 

2396 footprints = fpSet.positive 

2397 slateIm.set(0) 

2398 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2399 

2400 # Remove artifacts due to defects before they contribute to the epochCountImage 

2401 if self.config.doPrefilterArtifacts: 2401 ↛ 2405line 2401 didn't jump to line 2405, because the condition on line 2401 was never false

2402 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2403 

2404 # Clear mask before adding prefiltered spanSets 

2405 self.detect.clearMask(warpDiffExp.mask) 

2406 for spans in spanSetList: 

2407 spans.setImage(slateIm, 1, doClip=True) 

2408 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2409 epochCountImage += slateIm 

2410 

2411 if self.config.doFilterMorphological: 2411 ↛ 2412line 2411 didn't jump to line 2412, because the condition on line 2411 was never true

2412 maskName = self.config.streakMaskName 

2413 _ = self.maskStreaks.run(warpDiffExp) 

2414 streakMask = warpDiffExp.mask 

2415 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2416 streakMask.getPlaneBitMask(maskName)).split() 

2417 

2418 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2419 # undergo a second convolution. Pixels with data in the direct warp 

2420 # but not in the PSF-matched warp will not have their artifacts detected. 

2421 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2422 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2423 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2424 nansMask.setXY0(warpDiffExp.getXY0()) 

2425 edgeMask = warpDiffExp.mask 

2426 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2427 edgeMask.getPlaneBitMask("EDGE")).split() 

2428 else: 

2429 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2430 # In this case, mask the whole epoch 

2431 nansMask = afwImage.MaskX(coaddBBox, 1) 

2432 spanSetList = [] 

2433 spanSetEdgeMask = [] 

2434 spanSetStreak = [] 

2435 

2436 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2437 

2438 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2439 spanSetArtifactList.append(spanSetList) 

2440 spanSetEdgeList.append(spanSetEdgeMask) 

2441 if self.config.doFilterMorphological: 2441 ↛ 2442line 2441 didn't jump to line 2442, because the condition on line 2441 was never true

2442 spanSetBadMorphoList.append(spanSetStreak) 

2443 

2444 if lsstDebug.Info(__name__).saveCountIm: 2444 ↛ 2445line 2444 didn't jump to line 2445, because the condition on line 2444 was never true

2445 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2446 epochCountImage.writeFits(path) 

2447 

2448 for i, spanSetList in enumerate(spanSetArtifactList): 

2449 if spanSetList: 

2450 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2451 templateFootprints) 

2452 spanSetArtifactList[i] = filteredSpanSetList 

2453 if self.config.doFilterMorphological: 2453 ↛ 2454line 2453 didn't jump to line 2454, because the condition on line 2453 was never true

2454 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2455 

2456 altMasks = [] 

2457 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2458 altMasks.append({'CLIPPED': artifacts, 

2459 'NO_DATA': noData, 

2460 'EDGE': edge}) 

2461 return altMasks 

2462 

2463 def prefilterArtifacts(self, spanSetList, exp): 

2464 """Remove artifact candidates covered by bad mask plane. 

2465 

2466 Any future editing of the candidate list that does not depend on 

2467 temporal information should go in this method. 

2468 

2469 Parameters 

2470 ---------- 

2471 spanSetList : `list` 

2472 List of SpanSets representing artifact candidates. 

2473 exp : `lsst.afw.image.Exposure` 

2474 Exposure containing mask planes used to prefilter. 

2475 

2476 Returns 

2477 ------- 

2478 returnSpanSetList : `list` 

2479 List of SpanSets with artifacts. 

2480 """ 

2481 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2482 goodArr = (exp.mask.array & badPixelMask) == 0 

2483 returnSpanSetList = [] 

2484 bbox = exp.getBBox() 

2485 x0, y0 = exp.getXY0() 

2486 for i, span in enumerate(spanSetList): 

2487 y, x = span.clippedTo(bbox).indices() 

2488 yIndexLocal = numpy.array(y) - y0 

2489 xIndexLocal = numpy.array(x) - x0 

2490 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2491 if goodRatio > self.config.prefilterArtifactsRatio: 2491 ↛ 2486line 2491 didn't jump to line 2486, because the condition on line 2491 was never false

2492 returnSpanSetList.append(span) 

2493 return returnSpanSetList 

2494 

2495 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2496 """Filter artifact candidates. 

2497 

2498 Parameters 

2499 ---------- 

2500 spanSetList : `list` 

2501 List of SpanSets representing artifact candidates. 

2502 epochCountImage : `lsst.afw.image.Image` 

2503 Image of accumulated number of warpDiff detections. 

2504 nImage : `lsst.afw.image.Image` 

2505 Image of the accumulated number of total epochs contributing. 

2506 

2507 Returns 

2508 ------- 

2509 maskSpanSetList : `list` 

2510 List of SpanSets with artifacts. 

2511 """ 

2512 

2513 maskSpanSetList = [] 

2514 x0, y0 = epochCountImage.getXY0() 

2515 for i, span in enumerate(spanSetList): 

2516 y, x = span.indices() 

2517 yIdxLocal = [y1 - y0 for y1 in y] 

2518 xIdxLocal = [x1 - x0 for x1 in x] 

2519 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2520 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2521 

2522 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2523 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2524 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2525 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2526 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2527 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2528 & (outlierN <= effectiveMaxNumEpochs)) 

2529 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2530 if percentBelowThreshold > self.config.spatialThreshold: 

2531 maskSpanSetList.append(span) 

2532 

2533 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2533 ↛ 2546line 2533 didn't jump to line 2546, because the condition on line 2533 was never false

2534 # If a candidate is contained by a footprint on the template coadd, do not clip 

2535 filteredMaskSpanSetList = [] 

2536 for span in maskSpanSetList: 

2537 doKeep = True 

2538 for footprint in footprintsToExclude.positive.getFootprints(): 

2539 if footprint.spans.contains(span): 

2540 doKeep = False 

2541 break 

2542 if doKeep: 

2543 filteredMaskSpanSetList.append(span) 

2544 maskSpanSetList = filteredMaskSpanSetList 

2545 

2546 return maskSpanSetList 

2547 

2548 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2549 """Fetch a warp from the butler and return a warpDiff. 

2550 

2551 Parameters 

2552 ---------- 

2553 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2554 Butler dataRef for the warp. 

2555 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2556 An image scaler object. 

2557 templateCoadd : `lsst.afw.image.Exposure` 

2558 Exposure to be substracted from the scaled warp. 

2559 

2560 Returns 

2561 ------- 

2562 warp : `lsst.afw.image.Exposure` 

2563 Exposure of the image difference between the warp and template. 

2564 """ 

2565 

2566 # If the PSF-Matched warp did not exist for this direct warp 

2567 # None is holding its place to maintain order in Gen 3 

2568 if warpRef is None: 

2569 return None 

2570 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2571 warpName = self.getTempExpDatasetName('psfMatched') 

2572 if not isinstance(warpRef, DeferredDatasetHandle): 2572 ↛ 2576line 2572 didn't jump to line 2576, because the condition on line 2572 was never false

2573 if not warpRef.datasetExists(warpName): 2573 ↛ 2574line 2573 didn't jump to line 2574, because the condition on line 2573 was never true

2574 self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2575 return None 

2576 warp = warpRef.get(datasetType=warpName, immediate=True) 

2577 # direct image scaler OK for PSF-matched Warp 

2578 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2579 mi = warp.getMaskedImage() 

2580 if self.config.doScaleWarpVariance: 2580 ↛ 2585line 2580 didn't jump to line 2585, because the condition on line 2580 was never false

2581 try: 

2582 self.scaleWarpVariance.run(mi) 

2583 except Exception as exc: 

2584 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

2585 mi -= templateCoadd.getMaskedImage() 

2586 return warp 

2587 

2588 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2589 """Return a path to which to write debugging output. 

2590 

2591 Creates a hyphen-delimited string of dataId values for simple filenames. 

2592 

2593 Parameters 

2594 ---------- 

2595 prefix : `str` 

2596 Prefix for filename. 

2597 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2598 Butler dataRef to make the path from. 

2599 coaddLevel : `bool`, optional. 

2600 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2601 'filter', but no 'visit'). 

2602 

2603 Returns 

2604 ------- 

2605 result : `str` 

2606 Path for debugging output. 

2607 """ 

2608 if coaddLevel: 

2609 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2610 else: 

2611 keys = warpRef.dataId.keys() 

2612 keyList = sorted(keys, reverse=True) 

2613 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2614 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2615 return os.path.join(directory, filename)