Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 13%

901 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2024-03-14 10:10 +0000

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import logging 

27import lsst.pex.config as pexConfig 

28import lsst.pex.exceptions as pexExceptions 

29import lsst.geom as geom 

30import lsst.afw.geom as afwGeom 

31import lsst.afw.image as afwImage 

32import lsst.afw.math as afwMath 

33import lsst.afw.table as afwTable 

34import lsst.afw.detection as afwDet 

35import lsst.coadd.utils as coaddUtils 

36import lsst.pipe.base as pipeBase 

37import lsst.meas.algorithms as measAlg 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .maskStreaks import MaskStreaksTask 

46from .healSparseMapping import HealSparseInputMapTask 

47from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask 

48from lsst.daf.butler import DeferredDatasetHandle 

49from lsst.utils.timer import timeMethod 

50 

51__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

52 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

53 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

54 

55log = logging.getLogger(__name__) 

56 

57 

58class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

59 dimensions=("tract", "patch", "band", "skymap"), 

60 defaultTemplates={"inputCoaddName": "deep", 

61 "outputCoaddName": "deep", 

62 "warpType": "direct", 

63 "warpTypeSuffix": ""}): 

64 

65 inputWarps = pipeBase.connectionTypes.Input( 

66 doc=("Input list of warps to be assemebled i.e. stacked." 

67 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

68 name="{inputCoaddName}Coadd_{warpType}Warp", 

69 storageClass="ExposureF", 

70 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

71 deferLoad=True, 

72 multiple=True 

73 ) 

74 skyMap = pipeBase.connectionTypes.Input( 

75 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

77 storageClass="SkyMap", 

78 dimensions=("skymap", ), 

79 ) 

80 selectedVisits = pipeBase.connectionTypes.Input( 

81 doc="Selected visits to be coadded.", 

82 name="{outputCoaddName}Visits", 

83 storageClass="StructuredDataDict", 

84 dimensions=("instrument", "tract", "patch", "skymap", "band") 

85 ) 

86 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

87 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

88 " BRIGHT_OBJECT."), 

89 name="brightObjectMask", 

90 storageClass="ObjectMaskCatalog", 

91 dimensions=("tract", "patch", "skymap", "band"), 

92 minimum=0, 

93 ) 

94 coaddExposure = pipeBase.connectionTypes.Output( 

95 doc="Output coadded exposure, produced by stacking input warps", 

96 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

97 storageClass="ExposureF", 

98 dimensions=("tract", "patch", "skymap", "band"), 

99 ) 

100 nImage = pipeBase.connectionTypes.Output( 

101 doc="Output image of number of input images per pixel", 

102 name="{outputCoaddName}Coadd_nImage", 

103 storageClass="ImageU", 

104 dimensions=("tract", "patch", "skymap", "band"), 

105 ) 

106 inputMap = pipeBase.connectionTypes.Output( 

107 doc="Output healsparse map of input images", 

108 name="{outputCoaddName}Coadd_inputMap", 

109 storageClass="HealSparseMap", 

110 dimensions=("tract", "patch", "skymap", "band"), 

111 ) 

112 

113 def __init__(self, *, config=None): 

114 super().__init__(config=config) 

115 

116 # Override the connection's name template with config to replicate Gen2 behavior 

117 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

118 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

119 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

120 templateValues['warpType'] = config.warpType 

121 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

123 for name in self.allConnections} 

124 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

125 # End code to remove after deprecation 

126 

127 if not config.doMaskBrightObjects: 

128 self.prerequisiteInputs.remove("brightObjectMask") 

129 

130 if not config.doSelectVisits: 

131 self.inputs.remove("selectedVisits") 

132 

133 if not config.doNImage: 

134 self.outputs.remove("nImage") 

135 

136 if not self.config.doInputMap: 

137 self.outputs.remove("inputMap") 

138 

139 

140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

141 pipelineConnections=AssembleCoaddConnections): 

142 """Configuration parameters for the `AssembleCoaddTask`. 

143 

144 Notes 

145 ----- 

146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

147 only set the bitplane config.brightObjectMaskName. To make this useful you 

148 *must* also configure the flags.pixel algorithm, for example by adding 

149 

150 .. code-block:: none 

151 

152 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

153 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

154 

155 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

156 """ 

157 warpType = pexConfig.Field( 

158 doc="Warp name: one of 'direct' or 'psfMatched'", 

159 dtype=str, 

160 default="direct", 

161 ) 

162 subregionSize = pexConfig.ListField( 

163 dtype=int, 

164 doc="Width, height of stack subregion size; " 

165 "make small enough that a full stack of images will fit into memory at once.", 

166 length=2, 

167 default=(2000, 2000), 

168 ) 

169 statistic = pexConfig.Field( 

170 dtype=str, 

171 doc="Main stacking statistic for aggregating over the epochs.", 

172 default="MEANCLIP", 

173 ) 

174 doOnlineForMean = pexConfig.Field( 

175 dtype=bool, 

176 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?", 

177 default=False, 

178 ) 

179 doSigmaClip = pexConfig.Field( 

180 dtype=bool, 

181 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

182 default=False, 

183 ) 

184 sigmaClip = pexConfig.Field( 

185 dtype=float, 

186 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

187 default=3.0, 

188 ) 

189 clipIter = pexConfig.Field( 

190 dtype=int, 

191 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

192 default=2, 

193 ) 

194 calcErrorFromInputVariance = pexConfig.Field( 

195 dtype=bool, 

196 doc="Calculate coadd variance from input variance by stacking statistic." 

197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

198 default=True, 

199 ) 

200 scaleZeroPoint = pexConfig.ConfigurableField( 

201 target=ScaleZeroPointTask, 

202 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

203 ) 

204 doInterp = pexConfig.Field( 

205 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

206 dtype=bool, 

207 default=True, 

208 ) 

209 interpImage = pexConfig.ConfigurableField( 

210 target=InterpImageTask, 

211 doc="Task to interpolate (and extrapolate) over NaN pixels", 

212 ) 

213 doWrite = pexConfig.Field( 

214 doc="Persist coadd?", 

215 dtype=bool, 

216 default=True, 

217 ) 

218 doNImage = pexConfig.Field( 

219 doc="Create image of number of contributing exposures for each pixel", 

220 dtype=bool, 

221 default=False, 

222 ) 

223 doUsePsfMatchedPolygons = pexConfig.Field( 

224 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

225 dtype=bool, 

226 default=False, 

227 ) 

228 maskPropagationThresholds = pexConfig.DictField( 

229 keytype=str, 

230 itemtype=float, 

231 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

233 "would have contributed exceeds this value."), 

234 default={"SAT": 0.1}, 

235 ) 

236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

237 doc="Mask planes to remove before coadding") 

238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

239 doc="Set mask and flag bits for bright objects?") 

240 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

241 doc="Name of mask bit used for bright objects") 

242 coaddPsf = pexConfig.ConfigField( 

243 doc="Configuration for CoaddPsf", 

244 dtype=measAlg.CoaddPsfConfig, 

245 ) 

246 doAttachTransmissionCurve = pexConfig.Field( 

247 dtype=bool, default=False, optional=False, 

248 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

249 "(requires all input Exposures to have TransmissionCurves).") 

250 ) 

251 hasFakes = pexConfig.Field( 

252 dtype=bool, 

253 default=False, 

254 doc="Should be set to True if fake sources have been inserted into the input data." 

255 ) 

256 doSelectVisits = pexConfig.Field( 

257 doc="Coadd only visits selected by a SelectVisitsTask", 

258 dtype=bool, 

259 default=False, 

260 ) 

261 doInputMap = pexConfig.Field( 

262 doc="Create a bitwise map of coadd inputs", 

263 dtype=bool, 

264 default=False, 

265 ) 

266 inputMapper = pexConfig.ConfigurableField( 

267 doc="Input map creation subtask.", 

268 target=HealSparseInputMapTask, 

269 ) 

270 

271 def setDefaults(self): 

272 super().setDefaults() 

273 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

274 

275 def validate(self): 

276 super().validate() 

277 if self.doPsfMatch: 

278 # Backwards compatibility. 

279 # Configs do not have loggers 

280 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

281 self.warpType = 'psfMatched' 

282 if self.doSigmaClip and self.statistic != "MEANCLIP": 

283 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

284 self.statistic = "MEANCLIP" 

285 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

286 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

287 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

288 

289 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

290 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

291 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

292 if str(k) not in unstackableStats] 

293 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

294 % (self.statistic, stackableStats)) 

295 

296 

297class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

298 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

299 

300 We want to assemble a coadded image from a set of Warps (also called 

301 coadded temporary exposures or ``coaddTempExps``). 

302 Each input Warp covers a patch on the sky and corresponds to a single 

303 run/visit/exposure of the covered patch. We provide the task with a list 

304 of Warps (``selectDataList``) from which it selects Warps that cover the 

305 specified patch (pointed at by ``dataRef``). 

306 Each Warp that goes into a coadd will typically have an independent 

307 photometric zero-point. Therefore, we must scale each Warp to set it to 

308 a common photometric zeropoint. WarpType may be one of 'direct' or 

309 'psfMatched', and the boolean configs `config.makeDirect` and 

310 `config.makePsfMatched` set which of the warp types will be coadded. 

311 The coadd is computed as a mean with optional outlier rejection. 

312 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

313 Finally, Warps can have bad 'NaN' pixels which received no input from the 

314 source calExps. We interpolate over these bad (NaN) pixels. 

315 

316 `AssembleCoaddTask` uses several sub-tasks. These are 

317 

318 - `ScaleZeroPointTask` 

319 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

320 - `InterpImageTask` 

321 - interpolate across bad pixels (NaN) in the final coadd 

322 

323 You can retarget these subtasks if you wish. 

324 

325 Notes 

326 ----- 

327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

328 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

329 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

330 no debug variables of its own. Some of the subtasks may support debug 

331 variables. See the documentation for the subtasks for further information. 

332 

333 Examples 

334 -------- 

335 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

337 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

338 inputs: a data reference to the tract patch and filter to be coadded, and 

339 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

340 ``--selectId``, respectively: 

341 

342 .. code-block:: none 

343 

344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

346 

347 Only the Warps that cover the specified tract and patch will be coadded. 

348 A list of the available optional arguments can be obtained by calling 

349 ``assembleCoadd.py`` with the ``--help`` command line argument: 

350 

351 .. code-block:: none 

352 

353 assembleCoadd.py --help 

354 

355 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

356 multi-band processing, we will generate the HSC-I & -R band coadds from 

357 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

358 assuming that the lsst stack has been already set up, we must set up the 

359 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

360 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

361 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

362 coadds, we must first 

363 

364 - processCcd 

365 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

366 - makeSkyMap 

367 - create a skymap that covers the area of the sky present in the raw exposures 

368 - makeCoaddTempExp 

369 - warp the individual calibrated exposures to the tangent plane of the coadd 

370 

371 We can perform all of these steps by running 

372 

373 .. code-block:: none 

374 

375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

376 

377 This will produce warped exposures for each visit. To coadd the warped 

378 data, we call assembleCoadd.py as follows: 

379 

380 .. code-block:: none 

381 

382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

388 --selectId visit=903988 ccd=24 

389 

390 that will process the HSC-I band data. The results are written in 

391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

392 

393 You may also choose to run: 

394 

395 .. code-block:: none 

396 

397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

405 

406 to generate the coadd for the HSC-R band if you are interested in 

407 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

408 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

409 rather than `AssembleCoaddTask` to make the coadd. 

410 """ 

411 ConfigClass = AssembleCoaddConfig 

412 _DefaultName = "assembleCoadd" 

413 

414 def __init__(self, *args, **kwargs): 

415 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

416 if args: 

417 argNames = ["config", "name", "parentTask", "log"] 

418 kwargs.update({k: v for k, v in zip(argNames, args)}) 

419 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

420 "PipelineTask will not take positional args" % argNames, FutureWarning) 

421 

422 super().__init__(**kwargs) 

423 self.makeSubtask("interpImage") 

424 self.makeSubtask("scaleZeroPoint") 

425 

426 if self.config.doMaskBrightObjects: 

427 mask = afwImage.Mask() 

428 try: 

429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

430 except pexExceptions.LsstCppException: 

431 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

432 mask.getMaskPlaneDict().keys()) 

433 del mask 

434 

435 if self.config.doInputMap: 

436 self.makeSubtask("inputMapper") 

437 

438 self.warpType = self.config.warpType 

439 

440 @utils.inheritDoc(pipeBase.PipelineTask) 

441 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

442 # Docstring to be formatted with info from PipelineTask.runQuantum 

443 """ 

444 Notes 

445 ----- 

446 Assemble a coadd from a set of Warps. 

447 

448 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

449 Analogous to `runDataRef`, it prepares all the data products to be 

450 passed to `run`, and processes the results before returning a struct 

451 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

452 Therefore, its inputs are accessed subregion by subregion 

453 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

455 correspond to an update in `runDataRef` while both entry points 

456 are used. 

457 """ 

458 inputData = butlerQC.get(inputRefs) 

459 

460 # Construct skyInfo expected by run 

461 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

462 skyMap = inputData["skyMap"] 

463 outputDataId = butlerQC.quantum.dataId 

464 

465 inputData['skyInfo'] = makeSkyInfo(skyMap, 

466 tractId=outputDataId['tract'], 

467 patchId=outputDataId['patch']) 

468 

469 if self.config.doSelectVisits: 

470 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

471 else: 

472 warpRefList = inputData['inputWarps'] 

473 

474 # Perform same middle steps as `runDataRef` does 

475 inputs = self.prepareInputs(warpRefList) 

476 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

477 self.getTempExpDatasetName(self.warpType)) 

478 if len(inputs.tempExpRefList) == 0: 

479 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

480 

481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

482 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

483 inputs.weightList, supplementaryData=supplementaryData) 

484 

485 inputData.setdefault('brightObjectMask', None) 

486 if self.config.doMaskBrightObjects and inputData["brightObjectMask"] is None: 

487 log.warning("doMaskBrightObjects is set to True, but brightObjectMask not loaded") 

488 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

489 

490 if self.config.doWrite: 

491 butlerQC.put(retStruct, outputRefs) 

492 return retStruct 

493 

494 @timeMethod 

495 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

496 """Assemble a coadd from a set of Warps. 

497 

498 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

499 Compute weights to be applied to each Warp and 

500 find scalings to match the photometric zeropoint to a reference Warp. 

501 Assemble the Warps using `run`. Interpolate over NaNs and 

502 optionally write the coadd to disk. Return the coadded exposure. 

503 

504 Parameters 

505 ---------- 

506 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

507 Data reference defining the patch for coaddition and the 

508 reference Warp (if ``config.autoReference=False``). 

509 Used to access the following data products: 

510 - ``self.config.coaddName + "Coadd_skyMap"`` 

511 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

512 - ``self.config.coaddName + "Coadd"`` 

513 selectDataList : `list` 

514 List of data references to Calexps. Data to be coadded will be 

515 selected from this list based on overlap with the patch defined 

516 by dataRef, grouped by visit, and converted to a list of data 

517 references to warps. 

518 warpRefList : `list` 

519 List of data references to Warps to be coadded. 

520 Note: `warpRefList` is just the new name for `tempExpRefList`. 

521 

522 Returns 

523 ------- 

524 retStruct : `lsst.pipe.base.Struct` 

525 Result struct with components: 

526 

527 - ``coaddExposure``: coadded exposure (``Exposure``). 

528 - ``nImage``: exposure count image (``Image``). 

529 """ 

530 if selectDataList and warpRefList: 

531 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

532 "and which to use is ambiguous. Please pass only one.") 

533 

534 skyInfo = self.getSkyInfo(dataRef) 

535 if warpRefList is None: 

536 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

537 if len(calExpRefList) == 0: 

538 self.log.warning("No exposures to coadd") 

539 return 

540 self.log.info("Coadding %d exposures", len(calExpRefList)) 

541 

542 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

543 

544 inputData = self.prepareInputs(warpRefList) 

545 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

546 self.getTempExpDatasetName(self.warpType)) 

547 if len(inputData.tempExpRefList) == 0: 

548 self.log.warning("No coadd temporary exposures found") 

549 return 

550 

551 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

552 

553 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

554 inputData.weightList, supplementaryData=supplementaryData) 

555 

556 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

557 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

558 

559 if self.config.doWrite: 

560 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 

561 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

562 else: 

563 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

564 self.log.info("Persisting %s", coaddDatasetName) 

565 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

566 if self.config.doNImage and retStruct.nImage is not None: 

567 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

568 

569 return retStruct 

570 

571 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

572 """Interpolate over missing data and mask bright stars. 

573 

574 Parameters 

575 ---------- 

576 coaddExposure : `lsst.afw.image.Exposure` 

577 The coadded exposure to process. 

578 dataRef : `lsst.daf.persistence.ButlerDataRef` 

579 Butler data reference for supplementary data. 

580 """ 

581 if self.config.doInterp: 

582 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

583 # The variance must be positive; work around for DM-3201. 

584 varArray = coaddExposure.variance.array 

585 with numpy.errstate(invalid="ignore"): 

586 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

587 

588 if self.config.doMaskBrightObjects: 

589 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

590 

591 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

592 """Make additional inputs to run() specific to subclasses (Gen2) 

593 

594 Duplicates interface of `runDataRef` method 

595 Available to be implemented by subclasses only if they need the 

596 coadd dataRef for performing preliminary processing before 

597 assembling the coadd. 

598 

599 Parameters 

600 ---------- 

601 dataRef : `lsst.daf.persistence.ButlerDataRef` 

602 Butler data reference for supplementary data. 

603 selectDataList : `list` (optional) 

604 Optional List of data references to Calexps. 

605 warpRefList : `list` (optional) 

606 Optional List of data references to Warps. 

607 """ 

608 return pipeBase.Struct() 

609 

610 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

611 """Make additional inputs to run() specific to subclasses (Gen3) 

612 

613 Duplicates interface of `runQuantum` method. 

614 Available to be implemented by subclasses only if they need the 

615 coadd dataRef for performing preliminary processing before 

616 assembling the coadd. 

617 

618 Parameters 

619 ---------- 

620 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

621 Gen3 Butler object for fetching additional data products before 

622 running the Task specialized for quantum being processed 

623 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

624 Attributes are the names of the connections describing input dataset types. 

625 Values are DatasetRefs that task consumes for corresponding dataset type. 

626 DataIds are guaranteed to match data objects in ``inputData``. 

627 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

628 Attributes are the names of the connections describing output dataset types. 

629 Values are DatasetRefs that task is to produce 

630 for corresponding dataset type. 

631 """ 

632 return pipeBase.Struct() 

633 

634 def getTempExpRefList(self, patchRef, calExpRefList): 

635 """Generate list data references corresponding to warped exposures 

636 that lie within the patch to be coadded. 

637 

638 Parameters 

639 ---------- 

640 patchRef : `dataRef` 

641 Data reference for patch. 

642 calExpRefList : `list` 

643 List of data references for input calexps. 

644 

645 Returns 

646 ------- 

647 tempExpRefList : `list` 

648 List of Warp/CoaddTempExp data references. 

649 """ 

650 butler = patchRef.getButler() 

651 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

652 self.getTempExpDatasetName(self.warpType)) 

653 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

654 g, groupData.keys) for 

655 g in groupData.groups.keys()] 

656 return tempExpRefList 

657 

658 def prepareInputs(self, refList): 

659 """Prepare the input warps for coaddition by measuring the weight for 

660 each warp and the scaling for the photometric zero point. 

661 

662 Each Warp has its own photometric zeropoint and background variance. 

663 Before coadding these Warps together, compute a scale factor to 

664 normalize the photometric zeropoint and compute the weight for each Warp. 

665 

666 Parameters 

667 ---------- 

668 refList : `list` 

669 List of data references to tempExp 

670 

671 Returns 

672 ------- 

673 result : `lsst.pipe.base.Struct` 

674 Result struct with components: 

675 

676 - ``tempExprefList``: `list` of data references to tempExp. 

677 - ``weightList``: `list` of weightings. 

678 - ``imageScalerList``: `list` of image scalers. 

679 """ 

680 statsCtrl = afwMath.StatisticsControl() 

681 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

682 statsCtrl.setNumIter(self.config.clipIter) 

683 statsCtrl.setAndMask(self.getBadPixelMask()) 

684 statsCtrl.setNanSafe(True) 

685 # compute tempExpRefList: a list of tempExpRef that actually exist 

686 # and weightList: a list of the weight of the associated coadd tempExp 

687 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

688 tempExpRefList = [] 

689 weightList = [] 

690 imageScalerList = [] 

691 tempExpName = self.getTempExpDatasetName(self.warpType) 

692 for tempExpRef in refList: 

693 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

694 # therefore have no datasetExists() method 

695 if not isinstance(tempExpRef, DeferredDatasetHandle): 

696 if not tempExpRef.datasetExists(tempExpName): 

697 self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

698 continue 

699 

700 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

701 # Ignore any input warp that is empty of data 

702 if numpy.isnan(tempExp.image.array).all(): 

703 continue 

704 maskedImage = tempExp.getMaskedImage() 

705 imageScaler = self.scaleZeroPoint.computeImageScaler( 

706 exposure=tempExp, 

707 dataRef=tempExpRef, 

708 ) 

709 try: 

710 imageScaler.scaleMaskedImage(maskedImage) 

711 except Exception as e: 

712 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

713 continue 

714 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

715 afwMath.MEANCLIP, statsCtrl) 

716 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

717 weight = 1.0 / float(meanVar) 

718 if not numpy.isfinite(weight): 

719 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

720 continue 

721 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

722 

723 del maskedImage 

724 del tempExp 

725 

726 tempExpRefList.append(tempExpRef) 

727 weightList.append(weight) 

728 imageScalerList.append(imageScaler) 

729 

730 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

731 imageScalerList=imageScalerList) 

732 

733 def prepareStats(self, mask=None): 

734 """Prepare the statistics for coadding images. 

735 

736 Parameters 

737 ---------- 

738 mask : `int`, optional 

739 Bit mask value to exclude from coaddition. 

740 

741 Returns 

742 ------- 

743 stats : `lsst.pipe.base.Struct` 

744 Statistics structure with the following fields: 

745 

746 - ``statsCtrl``: Statistics control object for coadd 

747 (`lsst.afw.math.StatisticsControl`) 

748 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

749 """ 

750 if mask is None: 

751 mask = self.getBadPixelMask() 

752 statsCtrl = afwMath.StatisticsControl() 

753 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

754 statsCtrl.setNumIter(self.config.clipIter) 

755 statsCtrl.setAndMask(mask) 

756 statsCtrl.setNanSafe(True) 

757 statsCtrl.setWeighted(True) 

758 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

759 for plane, threshold in self.config.maskPropagationThresholds.items(): 

760 bit = afwImage.Mask.getMaskPlane(plane) 

761 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

762 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

763 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

764 

765 @timeMethod 

766 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

767 altMaskList=None, mask=None, supplementaryData=None): 

768 """Assemble a coadd from input warps 

769 

770 Assemble the coadd using the provided list of coaddTempExps. Since 

771 the full coadd covers a patch (a large area), the assembly is 

772 performed over small areas on the image at a time in order to 

773 conserve memory usage. Iterate over subregions within the outer 

774 bbox of the patch using `assembleSubregion` to stack the corresponding 

775 subregions from the coaddTempExps with the statistic specified. 

776 Set the edge bits the coadd mask based on the weight map. 

777 

778 Parameters 

779 ---------- 

780 skyInfo : `lsst.pipe.base.Struct` 

781 Struct with geometric information about the patch. 

782 tempExpRefList : `list` 

783 List of data references to Warps (previously called CoaddTempExps). 

784 imageScalerList : `list` 

785 List of image scalers. 

786 weightList : `list` 

787 List of weights 

788 altMaskList : `list`, optional 

789 List of alternate masks to use rather than those stored with 

790 tempExp. 

791 mask : `int`, optional 

792 Bit mask value to exclude from coaddition. 

793 supplementaryData : lsst.pipe.base.Struct, optional 

794 Struct with additional data products needed to assemble coadd. 

795 Only used by subclasses that implement `makeSupplementaryData` 

796 and override `run`. 

797 

798 Returns 

799 ------- 

800 result : `lsst.pipe.base.Struct` 

801 Result struct with components: 

802 

803 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

804 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

805 - ``inputMap``: bit-wise map of inputs, if requested. 

806 - ``warpRefList``: input list of refs to the warps ( 

807 ``lsst.daf.butler.DeferredDatasetHandle`` or 

808 ``lsst.daf.persistence.ButlerDataRef``) 

809 (unmodified) 

810 - ``imageScalerList``: input list of image scalers (unmodified) 

811 - ``weightList``: input list of weights (unmodified) 

812 """ 

813 tempExpName = self.getTempExpDatasetName(self.warpType) 

814 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

815 stats = self.prepareStats(mask=mask) 

816 

817 if altMaskList is None: 

818 altMaskList = [None]*len(tempExpRefList) 

819 

820 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

821 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

822 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

823 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

824 coaddMaskedImage = coaddExposure.getMaskedImage() 

825 subregionSizeArr = self.config.subregionSize 

826 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

827 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

828 if self.config.doNImage: 

829 nImage = afwImage.ImageU(skyInfo.bbox) 

830 else: 

831 nImage = None 

832 # If inputMap is requested, create the initial version that can be masked in 

833 # assembleSubregion. 

834 if self.config.doInputMap: 

835 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

836 skyInfo.wcs, 

837 coaddExposure.getInfo().getCoaddInputs().ccds) 

838 

839 if self.config.doOnlineForMean and self.config.statistic == "MEAN": 

840 try: 

841 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList, 

842 weightList, altMaskList, stats.ctrl, 

843 nImage=nImage) 

844 except Exception as e: 

845 self.log.exception("Cannot compute online coadd %s", e) 

846 raise 

847 else: 

848 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

849 try: 

850 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

851 weightList, altMaskList, stats.flags, stats.ctrl, 

852 nImage=nImage) 

853 except Exception as e: 

854 self.log.exception("Cannot compute coadd %s: %s", subBBox, e) 

855 raise 

856 

857 # If inputMap is requested, we must finalize the map after the accumulation. 

858 if self.config.doInputMap: 

859 self.inputMapper.finalize_ccd_input_map_mask() 

860 inputMap = self.inputMapper.ccd_input_map 

861 else: 

862 inputMap = None 

863 

864 self.setInexactPsf(coaddMaskedImage.getMask()) 

865 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

866 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

867 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

868 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

869 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

870 weightList=weightList, inputMap=inputMap) 

871 

872 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

873 """Set the metadata for the coadd. 

874 

875 This basic implementation sets the filter from the first input. 

876 

877 Parameters 

878 ---------- 

879 coaddExposure : `lsst.afw.image.Exposure` 

880 The target exposure for the coadd. 

881 tempExpRefList : `list` 

882 List of data references to tempExp. 

883 weightList : `list` 

884 List of weights. 

885 """ 

886 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

887 tempExpName = self.getTempExpDatasetName(self.warpType) 

888 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

889 # (and we need more than just the PropertySet that contains the header), which is not possible 

890 # with the current butler (see #2777). 

891 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

892 

893 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 

894 # Gen 3 API 

895 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

896 else: 

897 # Gen 2 API. Delete this when Gen 2 retired 

898 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

899 for tempExpRef in tempExpRefList] 

900 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

901 

902 # Set the coadd FilterLabel to the band of the first input exposure: 

903 # Coadds are calibrated, so the physical label is now meaningless. 

904 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel)) 

905 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

906 coaddInputs.ccds.reserve(numCcds) 

907 coaddInputs.visits.reserve(len(tempExpList)) 

908 

909 for tempExp, weight in zip(tempExpList, weightList): 

910 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

911 

912 if self.config.doUsePsfMatchedPolygons: 

913 self.shrinkValidPolygons(coaddInputs) 

914 

915 coaddInputs.visits.sort() 

916 coaddInputs.ccds.sort() 

917 if self.warpType == "psfMatched": 

918 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

919 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

920 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

921 # having the maximum width (sufficient because square) 

922 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

923 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() 

924 for modelPsf in modelPsfList] 

925 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

926 else: 

927 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

928 self.config.coaddPsf.makeControl()) 

929 coaddExposure.setPsf(psf) 

930 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

931 coaddExposure.getWcs()) 

932 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

933 if self.config.doAttachTransmissionCurve: 

934 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

935 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

936 

937 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

938 altMaskList, statsFlags, statsCtrl, nImage=None): 

939 """Assemble the coadd for a sub-region. 

940 

941 For each coaddTempExp, check for (and swap in) an alternative mask 

942 if one is passed. Remove mask planes listed in 

943 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

944 `lsst.afw.math.statisticsStack` with the statistic specified by 

945 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

946 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

947 an N-sigma clipped mean where N and iterations are specified by 

948 statsCtrl. Assign the stacked subregion back to the coadd. 

949 

950 Parameters 

951 ---------- 

952 coaddExposure : `lsst.afw.image.Exposure` 

953 The target exposure for the coadd. 

954 bbox : `lsst.geom.Box` 

955 Sub-region to coadd. 

956 tempExpRefList : `list` 

957 List of data reference to tempExp. 

958 imageScalerList : `list` 

959 List of image scalers. 

960 weightList : `list` 

961 List of weights. 

962 altMaskList : `list` 

963 List of alternate masks to use rather than those stored with 

964 tempExp, or None. Each element is dict with keys = mask plane 

965 name to which to add the spans. 

966 statsFlags : `lsst.afw.math.Property` 

967 Property object for statistic for coadd. 

968 statsCtrl : `lsst.afw.math.StatisticsControl` 

969 Statistics control object for coadd. 

970 nImage : `lsst.afw.image.ImageU`, optional 

971 Keeps track of exposure count for each pixel. 

972 """ 

973 self.log.debug("Computing coadd over %s", bbox) 

974 tempExpName = self.getTempExpDatasetName(self.warpType) 

975 coaddExposure.mask.addMaskPlane("REJECTED") 

976 coaddExposure.mask.addMaskPlane("CLIPPED") 

977 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

978 maskMap = self.setRejectedMaskMapping(statsCtrl) 

979 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

980 maskedImageList = [] 

981 if nImage is not None: 

982 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

983 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

984 

985 if isinstance(tempExpRef, DeferredDatasetHandle): 

986 # Gen 3 API 

987 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

988 else: 

989 # Gen 2 API. Delete this when Gen 2 retired 

990 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

991 

992 maskedImage = exposure.getMaskedImage() 

993 mask = maskedImage.getMask() 

994 if altMask is not None: 

995 self.applyAltMaskPlanes(mask, altMask) 

996 imageScaler.scaleMaskedImage(maskedImage) 

997 

998 # Add 1 for each pixel which is not excluded by the exclude mask. 

999 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

1000 if nImage is not None: 

1001 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

1002 if self.config.removeMaskPlanes: 

1003 self.removeMaskPlanes(maskedImage) 

1004 maskedImageList.append(maskedImage) 

1005 

1006 if self.config.doInputMap: 

1007 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

1008 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

1009 

1010 with self.timer("stack"): 

1011 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

1012 clipped, # also set output to CLIPPED if sigma-clipped 

1013 maskMap) 

1014 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

1015 if nImage is not None: 

1016 nImage.assign(subNImage, bbox) 

1017 

1018 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, 

1019 altMaskList, statsCtrl, nImage=None): 

1020 """Assemble the coadd using the "online" method. 

1021 

1022 This method takes a running sum of images and weights to save memory. 

1023 It only works for MEAN statistics. 

1024 

1025 Parameters 

1026 ---------- 

1027 coaddExposure : `lsst.afw.image.Exposure` 

1028 The target exposure for the coadd. 

1029 tempExpRefList : `list` 

1030 List of data reference to tempExp. 

1031 imageScalerList : `list` 

1032 List of image scalers. 

1033 weightList : `list` 

1034 List of weights. 

1035 altMaskList : `list` 

1036 List of alternate masks to use rather than those stored with 

1037 tempExp, or None. Each element is dict with keys = mask plane 

1038 name to which to add the spans. 

1039 statsCtrl : `lsst.afw.math.StatisticsControl` 

1040 Statistics control object for coadd 

1041 nImage : `lsst.afw.image.ImageU`, optional 

1042 Keeps track of exposure count for each pixel. 

1043 """ 

1044 self.log.debug("Computing online coadd.") 

1045 tempExpName = self.getTempExpDatasetName(self.warpType) 

1046 coaddExposure.mask.addMaskPlane("REJECTED") 

1047 coaddExposure.mask.addMaskPlane("CLIPPED") 

1048 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

1049 maskMap = self.setRejectedMaskMapping(statsCtrl) 

1050 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl) 

1051 

1052 bbox = coaddExposure.maskedImage.getBBox() 

1053 

1054 stacker = AccumulatorMeanStack( 

1055 coaddExposure.image.array.shape, 

1056 statsCtrl.getAndMask(), 

1057 mask_threshold_dict=thresholdDict, 

1058 mask_map=maskMap, 

1059 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(), 

1060 calc_error_from_input_variance=self.config.calcErrorFromInputVariance, 

1061 compute_n_image=(nImage is not None) 

1062 ) 

1063 

1064 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList, 

1065 imageScalerList, 

1066 altMaskList, 

1067 weightList): 

1068 if isinstance(tempExpRef, DeferredDatasetHandle): 

1069 # Gen 3 API 

1070 exposure = tempExpRef.get() 

1071 else: 

1072 # Gen 2 API. Delete this when Gen 2 retired 

1073 exposure = tempExpRef.get(tempExpName) 

1074 

1075 maskedImage = exposure.getMaskedImage() 

1076 mask = maskedImage.getMask() 

1077 if altMask is not None: 

1078 self.applyAltMaskPlanes(mask, altMask) 

1079 imageScaler.scaleMaskedImage(maskedImage) 

1080 if self.config.removeMaskPlanes: 

1081 self.removeMaskPlanes(maskedImage) 

1082 

1083 stacker.add_masked_image(maskedImage, weight=weight) 

1084 

1085 if self.config.doInputMap: 

1086 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

1087 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

1088 

1089 stacker.fill_stacked_masked_image(coaddExposure.maskedImage) 

1090 

1091 if nImage is not None: 

1092 nImage.array[:, :] = stacker.n_image 

1093 

1094 def removeMaskPlanes(self, maskedImage): 

1095 """Unset the mask of an image for mask planes specified in the config. 

1096 

1097 Parameters 

1098 ---------- 

1099 maskedImage : `lsst.afw.image.MaskedImage` 

1100 The masked image to be modified. 

1101 """ 

1102 mask = maskedImage.getMask() 

1103 for maskPlane in self.config.removeMaskPlanes: 

1104 try: 

1105 mask &= ~mask.getPlaneBitMask(maskPlane) 

1106 except pexExceptions.InvalidParameterError: 

1107 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1108 maskPlane) 

1109 

1110 @staticmethod 

1111 def setRejectedMaskMapping(statsCtrl): 

1112 """Map certain mask planes of the warps to new planes for the coadd. 

1113 

1114 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1115 or CLIPPED, set it to REJECTED on the coadd. 

1116 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1117 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1118 

1119 Parameters 

1120 ---------- 

1121 statsCtrl : `lsst.afw.math.StatisticsControl` 

1122 Statistics control object for coadd 

1123 

1124 Returns 

1125 ------- 

1126 maskMap : `list` of `tuple` of `int` 

1127 A list of mappings of mask planes of the warped exposures to 

1128 mask planes of the coadd. 

1129 """ 

1130 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1131 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1132 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1133 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1134 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1135 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1136 (clipped, clipped)] 

1137 return maskMap 

1138 

1139 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1140 """Apply in place alt mask formatted as SpanSets to a mask. 

1141 

1142 Parameters 

1143 ---------- 

1144 mask : `lsst.afw.image.Mask` 

1145 Original mask. 

1146 altMaskSpans : `dict` 

1147 SpanSet lists to apply. Each element contains the new mask 

1148 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1149 and list of SpanSets to apply to the mask. 

1150 

1151 Returns 

1152 ------- 

1153 mask : `lsst.afw.image.Mask` 

1154 Updated mask. 

1155 """ 

1156 if self.config.doUsePsfMatchedPolygons: 

1157 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 

1158 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1159 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1160 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1161 # This mask-clearing step must only occur *before* applying the new masks below 

1162 for spanSet in altMaskSpans['NO_DATA']: 

1163 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1164 

1165 for plane, spanSetList in altMaskSpans.items(): 

1166 maskClipValue = mask.addMaskPlane(plane) 

1167 for spanSet in spanSetList: 

1168 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1169 return mask 

1170 

1171 def shrinkValidPolygons(self, coaddInputs): 

1172 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1173 

1174 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1175 does not have a validPolygon, create one from its bbox. 

1176 

1177 Parameters 

1178 ---------- 

1179 coaddInputs : `lsst.afw.image.coaddInputs` 

1180 Original mask. 

1181 

1182 """ 

1183 for ccd in coaddInputs.ccds: 

1184 polyOrig = ccd.getValidPolygon() 

1185 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1186 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1187 if polyOrig: 

1188 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1189 else: 

1190 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1191 ccd.setValidPolygon(validPolygon) 

1192 

1193 def readBrightObjectMasks(self, dataRef): 

1194 """Retrieve the bright object masks. 

1195 

1196 Returns None on failure. 

1197 

1198 Parameters 

1199 ---------- 

1200 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1201 A Butler dataRef. 

1202 

1203 Returns 

1204 ------- 

1205 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1206 Bright object mask from the Butler object, or None if it cannot 

1207 be retrieved. 

1208 """ 

1209 try: 

1210 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1211 except Exception as e: 

1212 self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1213 return None 

1214 

1215 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1216 """Set the bright object masks. 

1217 

1218 Parameters 

1219 ---------- 

1220 exposure : `lsst.afw.image.Exposure` 

1221 Exposure under consideration. 

1222 dataId : `lsst.daf.persistence.dataId` 

1223 Data identifier dict for patch. 

1224 brightObjectMasks : `lsst.afw.table` 

1225 Table of bright objects to mask. 

1226 """ 

1227 

1228 if brightObjectMasks is None: 

1229 self.log.warning("Unable to apply bright object mask: none supplied") 

1230 return 

1231 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1232 mask = exposure.getMaskedImage().getMask() 

1233 wcs = exposure.getWcs() 

1234 plateScale = wcs.getPixelScale().asArcseconds() 

1235 

1236 for rec in brightObjectMasks: 

1237 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1238 if rec["type"] == "box": 

1239 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1240 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1241 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1242 

1243 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1244 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1245 

1246 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1247 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1248 spans = afwGeom.SpanSet(bbox) 

1249 elif rec["type"] == "circle": 

1250 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1251 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1252 else: 

1253 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1254 continue 

1255 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1256 

1257 def setInexactPsf(self, mask): 

1258 """Set INEXACT_PSF mask plane. 

1259 

1260 If any of the input images isn't represented in the coadd (due to 

1261 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1262 these pixels. 

1263 

1264 Parameters 

1265 ---------- 

1266 mask : `lsst.afw.image.Mask` 

1267 Coadded exposure's mask, modified in-place. 

1268 """ 

1269 mask.addMaskPlane("INEXACT_PSF") 

1270 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1271 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1272 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1273 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1274 array = mask.getArray() 

1275 selected = array & (sensorEdge | clipped | rejected) > 0 

1276 array[selected] |= inexactPsf 

1277 

1278 @classmethod 

1279 def _makeArgumentParser(cls): 

1280 """Create an argument parser. 

1281 """ 

1282 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1283 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1284 + cls.ConfigClass().warpType + "Warp", 

1285 help="data ID, e.g. --id tract=12345 patch=1,2", 

1286 ContainerClass=AssembleCoaddDataIdContainer) 

1287 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1288 ContainerClass=SelectDataIdContainer) 

1289 return parser 

1290 

1291 @staticmethod 

1292 def _subBBoxIter(bbox, subregionSize): 

1293 """Iterate over subregions of a bbox. 

1294 

1295 Parameters 

1296 ---------- 

1297 bbox : `lsst.geom.Box2I` 

1298 Bounding box over which to iterate. 

1299 subregionSize: `lsst.geom.Extent2I` 

1300 Size of sub-bboxes. 

1301 

1302 Yields 

1303 ------ 

1304 subBBox : `lsst.geom.Box2I` 

1305 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1306 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1307 the edges of ``bbox``, but it will never be empty. 

1308 """ 

1309 if bbox.isEmpty(): 

1310 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1311 if subregionSize[0] < 1 or subregionSize[1] < 1: 

1312 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1313 

1314 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1315 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1316 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1317 subBBox.clip(bbox) 

1318 if subBBox.isEmpty(): 

1319 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1320 "colShift=%s, rowShift=%s" % 

1321 (bbox, subregionSize, colShift, rowShift)) 

1322 yield subBBox 

1323 

1324 def filterWarps(self, inputs, goodVisits): 

1325 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1326 

1327 Parameters 

1328 ---------- 

1329 inputs : list 

1330 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1331 goodVisit : `dict` 

1332 Dictionary with good visitIds as the keys. Value ignored. 

1333 

1334 Returns: 

1335 -------- 

1336 filteredInputs : `list` 

1337 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1338 """ 

1339 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1340 filteredInputs = [] 

1341 for visit in goodVisits.keys(): 

1342 if visit in inputWarpDict: 

1343 filteredInputs.append(inputWarpDict[visit]) 

1344 return filteredInputs 

1345 

1346 

1347class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1348 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1349 """ 

1350 

1351 def makeDataRefList(self, namespace): 

1352 """Make self.refList from self.idList. 

1353 

1354 Parameters 

1355 ---------- 

1356 namespace 

1357 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1358 """ 

1359 datasetType = namespace.config.coaddName + "Coadd" 

1360 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1361 

1362 for dataId in self.idList: 

1363 # tract and patch are required 

1364 for key in keysCoadd: 

1365 if key not in dataId: 

1366 raise RuntimeError("--id must include " + key) 

1367 

1368 dataRef = namespace.butler.dataRef( 

1369 datasetType=datasetType, 

1370 dataId=dataId, 

1371 ) 

1372 self.refList.append(dataRef) 

1373 

1374 

1375def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1376 """Function to count the number of pixels with a specific mask in a 

1377 footprint. 

1378 

1379 Find the intersection of mask & footprint. Count all pixels in the mask 

1380 that are in the intersection that have bitmask set but do not have 

1381 ignoreMask set. Return the count. 

1382 

1383 Parameters 

1384 ---------- 

1385 mask : `lsst.afw.image.Mask` 

1386 Mask to define intersection region by. 

1387 footprint : `lsst.afw.detection.Footprint` 

1388 Footprint to define the intersection region by. 

1389 bitmask 

1390 Specific mask that we wish to count the number of occurances of. 

1391 ignoreMask 

1392 Pixels to not consider. 

1393 

1394 Returns 

1395 ------- 

1396 result : `int` 

1397 Count of number of pixels in footprint with specified mask. 

1398 """ 

1399 bbox = footprint.getBBox() 

1400 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1401 fp = afwImage.Mask(bbox) 

1402 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1403 footprint.spans.setMask(fp, bitmask) 

1404 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1405 (subMask.getArray() & ignoreMask) == 0).sum() 

1406 

1407 

1408class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1409 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1410 """ 

1411 clipDetection = pexConfig.ConfigurableField( 

1412 target=SourceDetectionTask, 

1413 doc="Detect sources on difference between unclipped and clipped coadd") 

1414 minClipFootOverlap = pexConfig.Field( 

1415 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1416 dtype=float, 

1417 default=0.6 

1418 ) 

1419 minClipFootOverlapSingle = pexConfig.Field( 

1420 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1421 "clipped when only one visit overlaps", 

1422 dtype=float, 

1423 default=0.5 

1424 ) 

1425 minClipFootOverlapDouble = pexConfig.Field( 

1426 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1427 "clipped when two visits overlap", 

1428 dtype=float, 

1429 default=0.45 

1430 ) 

1431 maxClipFootOverlapDouble = pexConfig.Field( 

1432 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1433 "considering two visits", 

1434 dtype=float, 

1435 default=0.15 

1436 ) 

1437 minBigOverlap = pexConfig.Field( 

1438 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1439 "when labeling clipped footprints", 

1440 dtype=int, 

1441 default=100 

1442 ) 

1443 

1444 def setDefaults(self): 

1445 """Set default values for clipDetection. 

1446 

1447 Notes 

1448 ----- 

1449 The numeric values for these configuration parameters were 

1450 empirically determined, future work may further refine them. 

1451 """ 

1452 AssembleCoaddConfig.setDefaults(self) 

1453 self.clipDetection.doTempLocalBackground = False 

1454 self.clipDetection.reEstimateBackground = False 

1455 self.clipDetection.returnOriginalFootprints = False 

1456 self.clipDetection.thresholdPolarity = "both" 

1457 self.clipDetection.thresholdValue = 2 

1458 self.clipDetection.nSigmaToGrow = 2 

1459 self.clipDetection.minPixels = 4 

1460 self.clipDetection.isotropicGrow = True 

1461 self.clipDetection.thresholdType = "pixel_stdev" 

1462 self.sigmaClip = 1.5 

1463 self.clipIter = 3 

1464 self.statistic = "MEAN" 

1465 

1466 def validate(self): 

1467 if self.doSigmaClip: 

1468 log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1469 "Ignoring doSigmaClip.") 

1470 self.doSigmaClip = False 

1471 if self.statistic != "MEAN": 

1472 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1473 "(%s chosen). Please set statistic to MEAN." 

1474 % (self.statistic)) 

1475 AssembleCoaddTask.ConfigClass.validate(self) 

1476 

1477 

1478class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1479 """Assemble a coadded image from a set of coadded temporary exposures, 

1480 being careful to clip & flag areas with potential artifacts. 

1481 

1482 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1483 we clip outliers). The problem with doing this is that when computing the 

1484 coadd PSF at a given location, individual visit PSFs from visits with 

1485 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1486 In this task, we correct for this behavior by creating a new 

1487 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1488 coaddTempExps and the final coadd where 

1489 

1490 i. difference imaging suggests that there is an outlier and 

1491 ii. this outlier appears on only one or two images. 

1492 

1493 Such regions will not contribute to the final coadd. Furthermore, any 

1494 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1495 Note that the algorithm implemented by this task is preliminary and works 

1496 correctly for HSC data. Parameter modifications and or considerable 

1497 redesigning of the algorithm is likley required for other surveys. 

1498 

1499 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1500 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1501 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1502 if you wish. 

1503 

1504 Notes 

1505 ----- 

1506 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1507 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1508 see `baseDebug` for more about ``debug.py`` files. 

1509 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1510 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1511 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1512 for further information. 

1513 

1514 Examples 

1515 -------- 

1516 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1517 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1518 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1519 

1520 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1521 and filter to be coadded (specified using 

1522 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1523 along with a list of coaddTempExps to attempt to coadd (specified using 

1524 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1525 Only the coaddTempExps that cover the specified tract and patch will be 

1526 coadded. A list of the available optional arguments can be obtained by 

1527 calling assembleCoadd.py with the --help command line argument: 

1528 

1529 .. code-block:: none 

1530 

1531 assembleCoadd.py --help 

1532 

1533 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1534 context of multi-band processing, we will generate the HSC-I & -R band 

1535 coadds from HSC engineering test data provided in the ci_hsc package. 

1536 To begin, assuming that the lsst stack has been already set up, we must 

1537 set up the obs_subaru and ci_hsc packages. This defines the environment 

1538 variable $CI_HSC_DIR and points at the location of the package. The raw 

1539 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1540 the coadds, we must first 

1541 

1542 - ``processCcd`` 

1543 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1544 - ``makeSkyMap`` 

1545 create a skymap that covers the area of the sky present in the raw exposures 

1546 - ``makeCoaddTempExp`` 

1547 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1548 

1549 We can perform all of these steps by running 

1550 

1551 .. code-block:: none 

1552 

1553 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1554 

1555 This will produce warped coaddTempExps for each visit. To coadd the 

1556 warped data, we call ``assembleCoadd.py`` as follows: 

1557 

1558 .. code-block:: none 

1559 

1560 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1561 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1562 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1563 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1564 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1565 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1566 --selectId visit=903988 ccd=24 

1567 

1568 This will process the HSC-I band data. The results are written in 

1569 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1570 

1571 You may also choose to run: 

1572 

1573 .. code-block:: none 

1574 

1575 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1576 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1577 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1578 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1579 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1580 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1581 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1582 --selectId visit=903346 ccd=12 

1583 

1584 to generate the coadd for the HSC-R band if you are interested in following 

1585 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1586 """ 

1587 ConfigClass = SafeClipAssembleCoaddConfig 

1588 _DefaultName = "safeClipAssembleCoadd" 

1589 

1590 def __init__(self, *args, **kwargs): 

1591 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1592 schema = afwTable.SourceTable.makeMinimalSchema() 

1593 self.makeSubtask("clipDetection", schema=schema) 

1594 

1595 @utils.inheritDoc(AssembleCoaddTask) 

1596 @timeMethod 

1597 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1598 """Assemble the coadd for a region. 

1599 

1600 Compute the difference of coadds created with and without outlier 

1601 rejection to identify coadd pixels that have outlier values in some 

1602 individual visits. 

1603 Detect clipped regions on the difference image and mark these regions 

1604 on the one or two individual coaddTempExps where they occur if there 

1605 is significant overlap between the clipped region and a source. This 

1606 leaves us with a set of footprints from the difference image that have 

1607 been identified as having occured on just one or two individual visits. 

1608 However, these footprints were generated from a difference image. It 

1609 is conceivable for a large diffuse source to have become broken up 

1610 into multiple footprints acrosss the coadd difference in this process. 

1611 Determine the clipped region from all overlapping footprints from the 

1612 detected sources in each visit - these are big footprints. 

1613 Combine the small and big clipped footprints and mark them on a new 

1614 bad mask plane. 

1615 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1616 removal. Clipped footprints will no longer make it into the coadd 

1617 because they are marked in the new bad mask plane. 

1618 

1619 Notes 

1620 ----- 

1621 args and kwargs are passed but ignored in order to match the call 

1622 signature expected by the parent task. 

1623 """ 

1624 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1625 mask = exp.getMaskedImage().getMask() 

1626 mask.addMaskPlane("CLIPPED") 

1627 

1628 result = self.detectClip(exp, tempExpRefList) 

1629 

1630 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1631 

1632 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1633 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1634 # Append big footprints from individual Warps to result.clipSpans 

1635 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1636 result.detectionFootprints, maskClipValue, maskDetValue, 

1637 exp.getBBox()) 

1638 # Create mask of the current clipped footprints 

1639 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1640 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1641 

1642 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1643 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1644 maskClip |= maskClipBig 

1645 

1646 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1647 badMaskPlanes = self.config.badMaskPlanes[:] 

1648 badMaskPlanes.append("CLIPPED") 

1649 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1650 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1651 result.clipSpans, mask=badPixelMask) 

1652 

1653 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1654 """Return an exposure that contains the difference between unclipped 

1655 and clipped coadds. 

1656 

1657 Generate a difference image between clipped and unclipped coadds. 

1658 Compute the difference image by subtracting an outlier-clipped coadd 

1659 from an outlier-unclipped coadd. Return the difference image. 

1660 

1661 Parameters 

1662 ---------- 

1663 skyInfo : `lsst.pipe.base.Struct` 

1664 Patch geometry information, from getSkyInfo 

1665 tempExpRefList : `list` 

1666 List of data reference to tempExp 

1667 imageScalerList : `list` 

1668 List of image scalers 

1669 weightList : `list` 

1670 List of weights 

1671 

1672 Returns 

1673 ------- 

1674 exp : `lsst.afw.image.Exposure` 

1675 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1676 """ 

1677 config = AssembleCoaddConfig() 

1678 # getattr necessary because subtasks do not survive Config.toDict() 

1679 # exclude connections because the class of self.config.connections is not 

1680 # the same as AssembleCoaddConfig.connections, and the connections are not 

1681 # needed to run this task anyway. 

1682 configIntersection = {k: getattr(self.config, k) 

1683 for k, v in self.config.toDict().items() 

1684 if (k in config.keys() and k != "connections")} 

1685 configIntersection['doInputMap'] = False 

1686 configIntersection['doNImage'] = False 

1687 config.update(**configIntersection) 

1688 

1689 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1690 config.statistic = 'MEAN' 

1691 task = AssembleCoaddTask(config=config) 

1692 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1693 

1694 config.statistic = 'MEANCLIP' 

1695 task = AssembleCoaddTask(config=config) 

1696 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1697 

1698 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1699 coaddDiff -= coaddClip.getMaskedImage() 

1700 exp = afwImage.ExposureF(coaddDiff) 

1701 exp.setPsf(coaddMean.getPsf()) 

1702 return exp 

1703 

1704 def detectClip(self, exp, tempExpRefList): 

1705 """Detect clipped regions on an exposure and set the mask on the 

1706 individual tempExp masks. 

1707 

1708 Detect footprints in the difference image after smoothing the 

1709 difference image with a Gaussian kernal. Identify footprints that 

1710 overlap with one or two input ``coaddTempExps`` by comparing the 

1711 computed overlap fraction to thresholds set in the config. A different 

1712 threshold is applied depending on the number of overlapping visits 

1713 (restricted to one or two). If the overlap exceeds the thresholds, 

1714 the footprint is considered "CLIPPED" and is marked as such on the 

1715 coaddTempExp. Return a struct with the clipped footprints, the indices 

1716 of the ``coaddTempExps`` that end up overlapping with the clipped 

1717 footprints, and a list of new masks for the ``coaddTempExps``. 

1718 

1719 Parameters 

1720 ---------- 

1721 exp : `lsst.afw.image.Exposure` 

1722 Exposure to run detection on. 

1723 tempExpRefList : `list` 

1724 List of data reference to tempExp. 

1725 

1726 Returns 

1727 ------- 

1728 result : `lsst.pipe.base.Struct` 

1729 Result struct with components: 

1730 

1731 - ``clipFootprints``: list of clipped footprints. 

1732 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1733 ``tempExpRefList``. 

1734 - ``clipSpans``: List of dictionaries containing spanSet lists 

1735 to clip. Each element contains the new maskplane name 

1736 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1737 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1738 compressed into footprints. 

1739 """ 

1740 mask = exp.getMaskedImage().getMask() 

1741 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1742 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1743 # Merge positive and negative together footprints together 

1744 fpSet.positive.merge(fpSet.negative) 

1745 footprints = fpSet.positive 

1746 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1747 ignoreMask = self.getBadPixelMask() 

1748 

1749 clipFootprints = [] 

1750 clipIndices = [] 

1751 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1752 

1753 # for use by detectClipBig 

1754 visitDetectionFootprints = [] 

1755 

1756 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1757 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1758 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1759 

1760 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1761 for i, warpRef in enumerate(tempExpRefList): 

1762 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1763 immediate=True).getMaskedImage().getMask() 

1764 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1765 afwImage.PARENT, True) 

1766 maskVisitDet &= maskDetValue 

1767 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1768 visitDetectionFootprints.append(visitFootprints) 

1769 

1770 for j, footprint in enumerate(footprints.getFootprints()): 

1771 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1772 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1773 

1774 # build a list of clipped spans for each visit 

1775 for j, footprint in enumerate(footprints.getFootprints()): 

1776 nPixel = footprint.getArea() 

1777 overlap = [] # hold the overlap with each visit 

1778 indexList = [] # index of visit in global list 

1779 for i in range(len(tempExpRefList)): 

1780 ignore = ignoreArr[i, j] 

1781 overlapDet = overlapDetArr[i, j] 

1782 totPixel = nPixel - ignore 

1783 

1784 # If we have more bad pixels than detection skip 

1785 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 

1786 continue 

1787 overlap.append(overlapDet/float(totPixel)) 

1788 indexList.append(i) 

1789 

1790 overlap = numpy.array(overlap) 

1791 if not len(overlap): 

1792 continue 

1793 

1794 keep = False # Should this footprint be marked as clipped? 

1795 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1796 

1797 # If footprint only has one overlap use a lower threshold 

1798 if len(overlap) == 1: 

1799 if overlap[0] > self.config.minClipFootOverlapSingle: 

1800 keep = True 

1801 keepIndex = [0] 

1802 else: 

1803 # This is the general case where only visit should be clipped 

1804 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1805 if len(clipIndex) == 1: 

1806 keep = True 

1807 keepIndex = [clipIndex[0]] 

1808 

1809 # Test if there are clipped objects that overlap two different visits 

1810 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1811 if len(clipIndex) == 2 and len(overlap) > 3: 

1812 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1813 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1814 keep = True 

1815 keepIndex = clipIndex 

1816 

1817 if not keep: 

1818 continue 

1819 

1820 for index in keepIndex: 

1821 globalIndex = indexList[index] 

1822 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1823 

1824 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1825 clipFootprints.append(footprint) 

1826 

1827 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1828 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1829 

1830 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1831 maskClipValue, maskDetValue, coaddBBox): 

1832 """Return individual warp footprints for large artifacts and append 

1833 them to ``clipList`` in place. 

1834 

1835 Identify big footprints composed of many sources in the coadd 

1836 difference that may have originated in a large diffuse source in the 

1837 coadd. We do this by indentifying all clipped footprints that overlap 

1838 significantly with each source in all the coaddTempExps. 

1839 

1840 Parameters 

1841 ---------- 

1842 clipList : `list` 

1843 List of alt mask SpanSets with clipping information. Modified. 

1844 clipFootprints : `list` 

1845 List of clipped footprints. 

1846 clipIndices : `list` 

1847 List of which entries in tempExpClipList each footprint belongs to. 

1848 maskClipValue 

1849 Mask value of clipped pixels. 

1850 maskDetValue 

1851 Mask value of detected pixels. 

1852 coaddBBox : `lsst.geom.Box` 

1853 BBox of the coadd and warps. 

1854 

1855 Returns 

1856 ------- 

1857 bigFootprintsCoadd : `list` 

1858 List of big footprints 

1859 """ 

1860 bigFootprintsCoadd = [] 

1861 ignoreMask = self.getBadPixelMask() 

1862 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1863 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1864 for footprint in visitFootprints.getFootprints(): 

1865 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1866 

1867 # build a mask of clipped footprints that are in this visit 

1868 clippedFootprintsVisit = [] 

1869 for foot, clipIndex in zip(clipFootprints, clipIndices): 

1870 if index not in clipIndex: 

1871 continue 

1872 clippedFootprintsVisit.append(foot) 

1873 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1874 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1875 

1876 bigFootprintsVisit = [] 

1877 for foot in visitFootprints.getFootprints(): 

1878 if foot.getArea() < self.config.minBigOverlap: 

1879 continue 

1880 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1881 if nCount > self.config.minBigOverlap: 

1882 bigFootprintsVisit.append(foot) 

1883 bigFootprintsCoadd.append(foot) 

1884 

1885 for footprint in bigFootprintsVisit: 

1886 clippedSpans["CLIPPED"].append(footprint.spans) 

1887 

1888 return bigFootprintsCoadd 

1889 

1890 

1891class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1892 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1893 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1894 "Only PSF-Matched Warps make sense for image subtraction. " 

1895 "Therefore, they must be an additional declared input."), 

1896 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1897 storageClass="ExposureF", 

1898 dimensions=("tract", "patch", "skymap", "visit"), 

1899 deferLoad=True, 

1900 multiple=True 

1901 ) 

1902 templateCoadd = pipeBase.connectionTypes.Output( 

1903 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1904 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1905 name="{outputCoaddName}CoaddPsfMatched", 

1906 storageClass="ExposureF", 

1907 dimensions=("tract", "patch", "skymap", "band"), 

1908 ) 

1909 

1910 def __init__(self, *, config=None): 

1911 super().__init__(config=config) 

1912 if not config.assembleStaticSkyModel.doWrite: 

1913 self.outputs.remove("templateCoadd") 

1914 config.validate() 

1915 

1916 

1917class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1918 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1919 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1920 target=AssembleCoaddTask, 

1921 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1922 " naive/first-iteration model of the static sky.", 

1923 ) 

1924 detect = pexConfig.ConfigurableField( 

1925 target=SourceDetectionTask, 

1926 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1927 ) 

1928 detectTemplate = pexConfig.ConfigurableField( 

1929 target=SourceDetectionTask, 

1930 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1931 ) 

1932 maskStreaks = pexConfig.ConfigurableField( 

1933 target=MaskStreaksTask, 

1934 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1935 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1936 "streakMaskName" 

1937 ) 

1938 streakMaskName = pexConfig.Field( 

1939 dtype=str, 

1940 default="STREAK", 

1941 doc="Name of mask bit used for streaks" 

1942 ) 

1943 maxNumEpochs = pexConfig.Field( 

1944 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1945 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1946 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1947 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1948 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1949 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1950 "than transient and not masked.", 

1951 dtype=int, 

1952 default=2 

1953 ) 

1954 maxFractionEpochsLow = pexConfig.RangeField( 

1955 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1956 "Effective maxNumEpochs = " 

1957 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1958 dtype=float, 

1959 default=0.4, 

1960 min=0., max=1., 

1961 ) 

1962 maxFractionEpochsHigh = pexConfig.RangeField( 

1963 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1964 "Effective maxNumEpochs = " 

1965 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1966 dtype=float, 

1967 default=0.03, 

1968 min=0., max=1., 

1969 ) 

1970 spatialThreshold = pexConfig.RangeField( 

1971 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1972 "temporal criteria. If 0, clip all. If 1, clip none.", 

1973 dtype=float, 

1974 default=0.5, 

1975 min=0., max=1., 

1976 inclusiveMin=True, inclusiveMax=True 

1977 ) 

1978 doScaleWarpVariance = pexConfig.Field( 

1979 doc="Rescale Warp variance plane using empirical noise?", 

1980 dtype=bool, 

1981 default=True, 

1982 ) 

1983 scaleWarpVariance = pexConfig.ConfigurableField( 

1984 target=ScaleVarianceTask, 

1985 doc="Rescale variance on warps", 

1986 ) 

1987 doPreserveContainedBySource = pexConfig.Field( 

1988 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1989 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1990 dtype=bool, 

1991 default=True, 

1992 ) 

1993 doPrefilterArtifacts = pexConfig.Field( 

1994 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1995 "because they will be excluded anyway. This prevents them from contributing " 

1996 "to the outlier epoch count image and potentially being labeled as persistant." 

1997 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1998 dtype=bool, 

1999 default=True 

2000 ) 

2001 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

2002 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

2003 dtype=str, 

2004 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

2005 ) 

2006 prefilterArtifactsRatio = pexConfig.Field( 

2007 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

2008 dtype=float, 

2009 default=0.05 

2010 ) 

2011 doFilterMorphological = pexConfig.Field( 

2012 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

2013 "be streaks.", 

2014 dtype=bool, 

2015 default=False 

2016 ) 

2017 growStreakFp = pexConfig.Field( 

2018 doc="Grow streak footprints by this number multiplied by the PSF width", 

2019 dtype=float, 

2020 default=5 

2021 ) 

2022 

2023 def setDefaults(self): 

2024 AssembleCoaddConfig.setDefaults(self) 

2025 self.statistic = 'MEAN' 

2026 self.doUsePsfMatchedPolygons = True 

2027 

2028 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

2029 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

2030 if "EDGE" in self.badMaskPlanes: 

2031 self.badMaskPlanes.remove('EDGE') 

2032 self.removeMaskPlanes.append('EDGE') 

2033 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

2034 self.assembleStaticSkyModel.warpType = 'psfMatched' 

2035 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

2036 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

2037 self.assembleStaticSkyModel.sigmaClip = 2.5 

2038 self.assembleStaticSkyModel.clipIter = 3 

2039 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

2040 self.assembleStaticSkyModel.doWrite = False 

2041 self.detect.doTempLocalBackground = False 

2042 self.detect.reEstimateBackground = False 

2043 self.detect.returnOriginalFootprints = False 

2044 self.detect.thresholdPolarity = "both" 

2045 self.detect.thresholdValue = 5 

2046 self.detect.minPixels = 4 

2047 self.detect.isotropicGrow = True 

2048 self.detect.thresholdType = "pixel_stdev" 

2049 self.detect.nSigmaToGrow = 0.4 

2050 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

2051 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

2052 self.detectTemplate.nSigmaToGrow = 2.4 

2053 self.detectTemplate.doTempLocalBackground = False 

2054 self.detectTemplate.reEstimateBackground = False 

2055 self.detectTemplate.returnOriginalFootprints = False 

2056 

2057 def validate(self): 

2058 super().validate() 

2059 if self.assembleStaticSkyModel.doNImage: 

2060 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

2061 "Please set assembleStaticSkyModel.doNImage=False") 

2062 

2063 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

2064 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

2065 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

2066 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

2067 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

2068 

2069 

2070class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

2071 """Assemble a compareWarp coadded image from a set of warps 

2072 by masking artifacts detected by comparing PSF-matched warps. 

2073 

2074 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

2075 we clip outliers). The problem with doing this is that when computing the 

2076 coadd PSF at a given location, individual visit PSFs from visits with 

2077 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

2078 In this task, we correct for this behavior by creating a new badMaskPlane 

2079 'CLIPPED' which marks pixels in the individual warps suspected to contain 

2080 an artifact. We populate this plane on the input warps by comparing 

2081 PSF-matched warps with a PSF-matched median coadd which serves as a 

2082 model of the static sky. Any group of pixels that deviates from the 

2083 PSF-matched template coadd by more than config.detect.threshold sigma, 

2084 is an artifact candidate. The candidates are then filtered to remove 

2085 variable sources and sources that are difficult to subtract such as 

2086 bright stars. This filter is configured using the config parameters 

2087 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

2088 the maximum fraction of epochs that the deviation can appear in and still 

2089 be considered an artifact. The spatialThreshold is the maximum fraction of 

2090 pixels in the footprint of the deviation that appear in other epochs 

2091 (where other epochs is defined by the temporalThreshold). If the deviant 

2092 region meets this criteria of having a significant percentage of pixels 

2093 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

2094 set in the mask. These regions will not contribute to the final coadd. 

2095 Furthermore, any routine to determine the coadd PSF can now be cognizant 

2096 of clipped regions. Note that the algorithm implemented by this task is 

2097 preliminary and works correctly for HSC data. Parameter modifications and 

2098 or considerable redesigning of the algorithm is likley required for other 

2099 surveys. 

2100 

2101 ``CompareWarpAssembleCoaddTask`` sub-classes 

2102 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

2103 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2104 

2105 Notes 

2106 ----- 

2107 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2108 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2109 ``baseDebug`` for more about ``debug.py`` files. 

2110 

2111 This task supports the following debug variables: 

2112 

2113 - ``saveCountIm`` 

2114 If True then save the Epoch Count Image as a fits file in the `figPath` 

2115 - ``figPath`` 

2116 Path to save the debug fits images and figures 

2117 

2118 For example, put something like: 

2119 

2120 .. code-block:: python 

2121 

2122 import lsstDebug 

2123 def DebugInfo(name): 

2124 di = lsstDebug.getInfo(name) 

2125 if name == "lsst.pipe.tasks.assembleCoadd": 

2126 di.saveCountIm = True 

2127 di.figPath = "/desired/path/to/debugging/output/images" 

2128 return di 

2129 lsstDebug.Info = DebugInfo 

2130 

2131 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2132 ``--debug`` flag. Some subtasks may have their own debug variables; 

2133 see individual Task documentation. 

2134 

2135 Examples 

2136 -------- 

2137 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2138 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2139 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2140 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2141 and filter to be coadded (specified using 

2142 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2143 along with a list of coaddTempExps to attempt to coadd (specified using 

2144 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2145 Only the warps that cover the specified tract and patch will be coadded. 

2146 A list of the available optional arguments can be obtained by calling 

2147 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2148 

2149 .. code-block:: none 

2150 

2151 assembleCoadd.py --help 

2152 

2153 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2154 context of multi-band processing, we will generate the HSC-I & -R band 

2155 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2156 To begin, assuming that the lsst stack has been already set up, we must 

2157 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2158 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2159 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2160 directory. To begin assembling the coadds, we must first 

2161 

2162 - processCcd 

2163 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2164 - makeSkyMap 

2165 create a skymap that covers the area of the sky present in the raw exposures 

2166 - makeCoaddTempExp 

2167 warp the individual calibrated exposures to the tangent plane of the coadd 

2168 

2169 We can perform all of these steps by running 

2170 

2171 .. code-block:: none 

2172 

2173 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2174 

2175 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2176 warped data, we call ``assembleCoadd.py`` as follows: 

2177 

2178 .. code-block:: none 

2179 

2180 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2181 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2182 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2183 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2184 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2185 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2186 --selectId visit=903988 ccd=24 

2187 

2188 This will process the HSC-I band data. The results are written in 

2189 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2190 """ 

2191 ConfigClass = CompareWarpAssembleCoaddConfig 

2192 _DefaultName = "compareWarpAssembleCoadd" 

2193 

2194 def __init__(self, *args, **kwargs): 

2195 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2196 self.makeSubtask("assembleStaticSkyModel") 

2197 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2198 self.makeSubtask("detect", schema=detectionSchema) 

2199 if self.config.doPreserveContainedBySource: 

2200 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2201 if self.config.doScaleWarpVariance: 

2202 self.makeSubtask("scaleWarpVariance") 

2203 if self.config.doFilterMorphological: 

2204 self.makeSubtask("maskStreaks") 

2205 

2206 @utils.inheritDoc(AssembleCoaddTask) 

2207 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2208 """ 

2209 Generate a templateCoadd to use as a naive model of static sky to 

2210 subtract from PSF-Matched warps. 

2211 

2212 Returns 

2213 ------- 

2214 result : `lsst.pipe.base.Struct` 

2215 Result struct with components: 

2216 

2217 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2218 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2219 """ 

2220 # Ensure that psfMatchedWarps are used as input warps for template generation 

2221 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2222 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2223 

2224 # Because subtasks don't have connections we have to make one. 

2225 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2226 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2227 if self.config.assembleStaticSkyModel.doWrite: 

2228 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2229 # Remove template coadd from both subtask's and main tasks outputs, 

2230 # because it is handled by the subtask as `coaddExposure` 

2231 del outputRefs.templateCoadd 

2232 del staticSkyModelOutputRefs.templateCoadd 

2233 

2234 # A PSF-Matched nImage does not exist as a dataset type 

2235 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2236 del staticSkyModelOutputRefs.nImage 

2237 

2238 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2239 staticSkyModelOutputRefs) 

2240 if templateCoadd is None: 

2241 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2242 

2243 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2244 nImage=templateCoadd.nImage, 

2245 warpRefList=templateCoadd.warpRefList, 

2246 imageScalerList=templateCoadd.imageScalerList, 

2247 weightList=templateCoadd.weightList) 

2248 

2249 @utils.inheritDoc(AssembleCoaddTask) 

2250 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2251 """ 

2252 Generate a templateCoadd to use as a naive model of static sky to 

2253 subtract from PSF-Matched warps. 

2254 

2255 Returns 

2256 ------- 

2257 result : `lsst.pipe.base.Struct` 

2258 Result struct with components: 

2259 

2260 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2261 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2262 """ 

2263 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2264 if templateCoadd is None: 

2265 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2266 

2267 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2268 nImage=templateCoadd.nImage, 

2269 warpRefList=templateCoadd.warpRefList, 

2270 imageScalerList=templateCoadd.imageScalerList, 

2271 weightList=templateCoadd.weightList) 

2272 

2273 def _noTemplateMessage(self, warpType): 

2274 warpName = (warpType[0].upper() + warpType[1:]) 

2275 message = """No %(warpName)s warps were found to build the template coadd which is 

2276 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2277 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2278 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2279 

2280 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2281 another algorithm like: 

2282 

2283 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2284 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2285 """ % {"warpName": warpName} 

2286 return message 

2287 

2288 @utils.inheritDoc(AssembleCoaddTask) 

2289 @timeMethod 

2290 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2291 supplementaryData, *args, **kwargs): 

2292 """Assemble the coadd. 

2293 

2294 Find artifacts and apply them to the warps' masks creating a list of 

2295 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2296 plane. Then pass these alternative masks to the base class's `run` 

2297 method. 

2298 

2299 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2300 that must contain a ``templateCoadd`` that serves as the 

2301 model of the static sky. 

2302 """ 

2303 

2304 # Check and match the order of the supplementaryData 

2305 # (PSF-matched) inputs to the order of the direct inputs, 

2306 # so that the artifact mask is applied to the right warp 

2307 dataIds = [ref.dataId for ref in tempExpRefList] 

2308 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2309 

2310 if dataIds != psfMatchedDataIds: 

2311 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2312 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2313 psfMatchedDataIds, dataIds) 

2314 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2315 psfMatchedDataIds, dataIds) 

2316 

2317 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2318 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2319 supplementaryData.warpRefList, 

2320 supplementaryData.imageScalerList) 

2321 

2322 badMaskPlanes = self.config.badMaskPlanes[:] 

2323 badMaskPlanes.append("CLIPPED") 

2324 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2325 

2326 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2327 spanSetMaskList, mask=badPixelMask) 

2328 

2329 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2330 # Psf-Matching moves the real edge inwards 

2331 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2332 return result 

2333 

2334 def applyAltEdgeMask(self, mask, altMaskList): 

2335 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2336 

2337 Parameters 

2338 ---------- 

2339 mask : `lsst.afw.image.Mask` 

2340 Original mask. 

2341 altMaskList : `list` 

2342 List of Dicts containing ``spanSet`` lists. 

2343 Each element contains the new mask plane name (e.g. "CLIPPED 

2344 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2345 the mask. 

2346 """ 

2347 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2348 for visitMask in altMaskList: 

2349 if "EDGE" in visitMask: 

2350 for spanSet in visitMask['EDGE']: 

2351 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2352 

2353 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2354 """Find artifacts. 

2355 

2356 Loop through warps twice. The first loop builds a map with the count 

2357 of how many epochs each pixel deviates from the templateCoadd by more 

2358 than ``config.chiThreshold`` sigma. The second loop takes each 

2359 difference image and filters the artifacts detected in each using 

2360 count map to filter out variable sources and sources that are 

2361 difficult to subtract cleanly. 

2362 

2363 Parameters 

2364 ---------- 

2365 templateCoadd : `lsst.afw.image.Exposure` 

2366 Exposure to serve as model of static sky. 

2367 tempExpRefList : `list` 

2368 List of data references to warps. 

2369 imageScalerList : `list` 

2370 List of image scalers. 

2371 

2372 Returns 

2373 ------- 

2374 altMasks : `list` 

2375 List of dicts containing information about CLIPPED 

2376 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2377 """ 

2378 

2379 self.log.debug("Generating Count Image, and mask lists.") 

2380 coaddBBox = templateCoadd.getBBox() 

2381 slateIm = afwImage.ImageU(coaddBBox) 

2382 epochCountImage = afwImage.ImageU(coaddBBox) 

2383 nImage = afwImage.ImageU(coaddBBox) 

2384 spanSetArtifactList = [] 

2385 spanSetNoDataMaskList = [] 

2386 spanSetEdgeList = [] 

2387 spanSetBadMorphoList = [] 

2388 badPixelMask = self.getBadPixelMask() 

2389 

2390 # mask of the warp diffs should = that of only the warp 

2391 templateCoadd.mask.clearAllMaskPlanes() 

2392 

2393 if self.config.doPreserveContainedBySource: 

2394 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2395 else: 

2396 templateFootprints = None 

2397 

2398 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2399 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2400 if warpDiffExp is not None: 

2401 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2402 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2403 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2404 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2405 fpSet.positive.merge(fpSet.negative) 

2406 footprints = fpSet.positive 

2407 slateIm.set(0) 

2408 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2409 

2410 # Remove artifacts due to defects before they contribute to the epochCountImage 

2411 if self.config.doPrefilterArtifacts: 

2412 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2413 

2414 # Clear mask before adding prefiltered spanSets 

2415 self.detect.clearMask(warpDiffExp.mask) 

2416 for spans in spanSetList: 

2417 spans.setImage(slateIm, 1, doClip=True) 

2418 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2419 epochCountImage += slateIm 

2420 

2421 if self.config.doFilterMorphological: 

2422 maskName = self.config.streakMaskName 

2423 _ = self.maskStreaks.run(warpDiffExp) 

2424 streakMask = warpDiffExp.mask 

2425 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2426 streakMask.getPlaneBitMask(maskName)).split() 

2427 # Pad the streaks to account for low-surface brightness wings 

2428 psf = warpDiffExp.getPsf() 

2429 for s, sset in enumerate(spanSetStreak): 

2430 psfShape = psf.computeShape(sset.computeCentroid()) 

2431 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius() 

2432 sset_dilated = sset.dilated(int(dilation)) 

2433 spanSetStreak[s] = sset_dilated 

2434 

2435 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2436 # undergo a second convolution. Pixels with data in the direct warp 

2437 # but not in the PSF-matched warp will not have their artifacts detected. 

2438 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2439 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2440 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2441 nansMask.setXY0(warpDiffExp.getXY0()) 

2442 edgeMask = warpDiffExp.mask 

2443 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2444 edgeMask.getPlaneBitMask("EDGE")).split() 

2445 else: 

2446 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2447 # In this case, mask the whole epoch 

2448 nansMask = afwImage.MaskX(coaddBBox, 1) 

2449 spanSetList = [] 

2450 spanSetEdgeMask = [] 

2451 spanSetStreak = [] 

2452 

2453 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2454 

2455 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2456 spanSetArtifactList.append(spanSetList) 

2457 spanSetEdgeList.append(spanSetEdgeMask) 

2458 if self.config.doFilterMorphological: 

2459 spanSetBadMorphoList.append(spanSetStreak) 

2460 

2461 if lsstDebug.Info(__name__).saveCountIm: 

2462 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2463 epochCountImage.writeFits(path) 

2464 

2465 for i, spanSetList in enumerate(spanSetArtifactList): 

2466 if spanSetList: 

2467 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2468 templateFootprints) 

2469 spanSetArtifactList[i] = filteredSpanSetList 

2470 if self.config.doFilterMorphological: 

2471 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2472 

2473 altMasks = [] 

2474 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2475 altMasks.append({'CLIPPED': artifacts, 

2476 'NO_DATA': noData, 

2477 'EDGE': edge}) 

2478 return altMasks 

2479 

2480 def prefilterArtifacts(self, spanSetList, exp): 

2481 """Remove artifact candidates covered by bad mask plane. 

2482 

2483 Any future editing of the candidate list that does not depend on 

2484 temporal information should go in this method. 

2485 

2486 Parameters 

2487 ---------- 

2488 spanSetList : `list` 

2489 List of SpanSets representing artifact candidates. 

2490 exp : `lsst.afw.image.Exposure` 

2491 Exposure containing mask planes used to prefilter. 

2492 

2493 Returns 

2494 ------- 

2495 returnSpanSetList : `list` 

2496 List of SpanSets with artifacts. 

2497 """ 

2498 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2499 goodArr = (exp.mask.array & badPixelMask) == 0 

2500 returnSpanSetList = [] 

2501 bbox = exp.getBBox() 

2502 x0, y0 = exp.getXY0() 

2503 for i, span in enumerate(spanSetList): 

2504 y, x = span.clippedTo(bbox).indices() 

2505 yIndexLocal = numpy.array(y) - y0 

2506 xIndexLocal = numpy.array(x) - x0 

2507 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2508 if goodRatio > self.config.prefilterArtifactsRatio: 

2509 returnSpanSetList.append(span) 

2510 return returnSpanSetList 

2511 

2512 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2513 """Filter artifact candidates. 

2514 

2515 Parameters 

2516 ---------- 

2517 spanSetList : `list` 

2518 List of SpanSets representing artifact candidates. 

2519 epochCountImage : `lsst.afw.image.Image` 

2520 Image of accumulated number of warpDiff detections. 

2521 nImage : `lsst.afw.image.Image` 

2522 Image of the accumulated number of total epochs contributing. 

2523 

2524 Returns 

2525 ------- 

2526 maskSpanSetList : `list` 

2527 List of SpanSets with artifacts. 

2528 """ 

2529 

2530 maskSpanSetList = [] 

2531 x0, y0 = epochCountImage.getXY0() 

2532 for i, span in enumerate(spanSetList): 

2533 y, x = span.indices() 

2534 yIdxLocal = [y1 - y0 for y1 in y] 

2535 xIdxLocal = [x1 - x0 for x1 in x] 

2536 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2537 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2538 

2539 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2540 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2541 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2542 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2543 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2544 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2545 & (outlierN <= effectiveMaxNumEpochs)) 

2546 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2547 if percentBelowThreshold > self.config.spatialThreshold: 

2548 maskSpanSetList.append(span) 

2549 

2550 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 

2551 # If a candidate is contained by a footprint on the template coadd, do not clip 

2552 filteredMaskSpanSetList = [] 

2553 for span in maskSpanSetList: 

2554 doKeep = True 

2555 for footprint in footprintsToExclude.positive.getFootprints(): 

2556 if footprint.spans.contains(span): 

2557 doKeep = False 

2558 break 

2559 if doKeep: 

2560 filteredMaskSpanSetList.append(span) 

2561 maskSpanSetList = filteredMaskSpanSetList 

2562 

2563 return maskSpanSetList 

2564 

2565 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2566 """Fetch a warp from the butler and return a warpDiff. 

2567 

2568 Parameters 

2569 ---------- 

2570 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2571 Butler dataRef for the warp. 

2572 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2573 An image scaler object. 

2574 templateCoadd : `lsst.afw.image.Exposure` 

2575 Exposure to be substracted from the scaled warp. 

2576 

2577 Returns 

2578 ------- 

2579 warp : `lsst.afw.image.Exposure` 

2580 Exposure of the image difference between the warp and template. 

2581 """ 

2582 

2583 # If the PSF-Matched warp did not exist for this direct warp 

2584 # None is holding its place to maintain order in Gen 3 

2585 if warpRef is None: 

2586 return None 

2587 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2588 warpName = self.getTempExpDatasetName('psfMatched') 

2589 if not isinstance(warpRef, DeferredDatasetHandle): 

2590 if not warpRef.datasetExists(warpName): 

2591 self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2592 return None 

2593 warp = warpRef.get(datasetType=warpName, immediate=True) 

2594 # direct image scaler OK for PSF-matched Warp 

2595 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2596 mi = warp.getMaskedImage() 

2597 if self.config.doScaleWarpVariance: 

2598 try: 

2599 self.scaleWarpVariance.run(mi) 

2600 except Exception as exc: 

2601 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

2602 mi -= templateCoadd.getMaskedImage() 

2603 return warp 

2604 

2605 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2606 """Return a path to which to write debugging output. 

2607 

2608 Creates a hyphen-delimited string of dataId values for simple filenames. 

2609 

2610 Parameters 

2611 ---------- 

2612 prefix : `str` 

2613 Prefix for filename. 

2614 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2615 Butler dataRef to make the path from. 

2616 coaddLevel : `bool`, optional. 

2617 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2618 'filter', but no 'visit'). 

2619 

2620 Returns 

2621 ------- 

2622 result : `str` 

2623 Path for debugging output. 

2624 """ 

2625 if coaddLevel: 

2626 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2627 else: 

2628 keys = warpRef.dataId.keys() 

2629 keyList = sorted(keys, reverse=True) 

2630 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2631 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2632 return os.path.join(directory, filename)