Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from .healSparseMapping import HealSparseInputMapTask 

48from lsst.meas.algorithms import SourceDetectionTask 

49from lsst.daf.butler import DeferredDatasetHandle 

50 

51__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

52 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

53 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

54 

55 

56class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

57 dimensions=("tract", "patch", "band", "skymap"), 

58 defaultTemplates={"inputCoaddName": "deep", 

59 "outputCoaddName": "deep", 

60 "warpType": "direct", 

61 "warpTypeSuffix": ""}): 

62 

63 inputWarps = pipeBase.connectionTypes.Input( 

64 doc=("Input list of warps to be assemebled i.e. stacked." 

65 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

66 name="{inputCoaddName}Coadd_{warpType}Warp", 

67 storageClass="ExposureF", 

68 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

69 deferLoad=True, 

70 multiple=True 

71 ) 

72 skyMap = pipeBase.connectionTypes.Input( 

73 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

75 storageClass="SkyMap", 

76 dimensions=("skymap", ), 

77 ) 

78 selectedVisits = pipeBase.connectionTypes.Input( 

79 doc="Selected visits to be coadded.", 

80 name="{outputCoaddName}Visits", 

81 storageClass="StructuredDataDict", 

82 dimensions=("instrument", "tract", "patch", "skymap", "band") 

83 ) 

84 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

85 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

86 " BRIGHT_OBJECT."), 

87 name="brightObjectMask", 

88 storageClass="ObjectMaskCatalog", 

89 dimensions=("tract", "patch", "skymap", "band"), 

90 ) 

91 coaddExposure = pipeBase.connectionTypes.Output( 

92 doc="Output coadded exposure, produced by stacking input warps", 

93 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

94 storageClass="ExposureF", 

95 dimensions=("tract", "patch", "skymap", "band"), 

96 ) 

97 nImage = pipeBase.connectionTypes.Output( 

98 doc="Output image of number of input images per pixel", 

99 name="{outputCoaddName}Coadd_nImage", 

100 storageClass="ImageU", 

101 dimensions=("tract", "patch", "skymap", "band"), 

102 ) 

103 inputMap = pipeBase.connectionTypes.Output( 

104 doc="Output healsparse map of input images", 

105 name="{outputCoaddName}Coadd_inputMap", 

106 storageClass="HealSparseMap", 

107 dimensions=("tract", "patch", "skymap", "band"), 

108 ) 

109 

110 def __init__(self, *, config=None): 

111 super().__init__(config=config) 

112 

113 # Override the connection's name template with config to replicate Gen2 behavior 

114 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

115 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

116 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

117 templateValues['warpType'] = config.warpType 

118 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

119 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

120 for name in self.allConnections} 

121 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

122 # End code to remove after deprecation 

123 

124 if not config.doMaskBrightObjects: 

125 self.prerequisiteInputs.remove("brightObjectMask") 

126 

127 if not config.doSelectVisits: 

128 self.inputs.remove("selectedVisits") 

129 

130 if not config.doNImage: 

131 self.outputs.remove("nImage") 

132 

133 if not self.config.doInputMap: 

134 self.outputs.remove("inputMap") 

135 

136 

137class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

138 pipelineConnections=AssembleCoaddConnections): 

139 """Configuration parameters for the `AssembleCoaddTask`. 

140 

141 Notes 

142 ----- 

143 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

144 only set the bitplane config.brightObjectMaskName. To make this useful you 

145 *must* also configure the flags.pixel algorithm, for example by adding 

146 

147 .. code-block:: none 

148 

149 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

150 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

151 

152 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

153 """ 

154 warpType = pexConfig.Field( 

155 doc="Warp name: one of 'direct' or 'psfMatched'", 

156 dtype=str, 

157 default="direct", 

158 ) 

159 subregionSize = pexConfig.ListField( 

160 dtype=int, 

161 doc="Width, height of stack subregion size; " 

162 "make small enough that a full stack of images will fit into memory at once.", 

163 length=2, 

164 default=(2000, 2000), 

165 ) 

166 statistic = pexConfig.Field( 

167 dtype=str, 

168 doc="Main stacking statistic for aggregating over the epochs.", 

169 default="MEANCLIP", 

170 ) 

171 doSigmaClip = pexConfig.Field( 

172 dtype=bool, 

173 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

174 default=False, 

175 ) 

176 sigmaClip = pexConfig.Field( 

177 dtype=float, 

178 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

179 default=3.0, 

180 ) 

181 clipIter = pexConfig.Field( 

182 dtype=int, 

183 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

184 default=2, 

185 ) 

186 calcErrorFromInputVariance = pexConfig.Field( 

187 dtype=bool, 

188 doc="Calculate coadd variance from input variance by stacking statistic." 

189 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

190 default=True, 

191 ) 

192 scaleZeroPoint = pexConfig.ConfigurableField( 

193 target=ScaleZeroPointTask, 

194 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

195 ) 

196 doInterp = pexConfig.Field( 

197 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

198 dtype=bool, 

199 default=True, 

200 ) 

201 interpImage = pexConfig.ConfigurableField( 

202 target=InterpImageTask, 

203 doc="Task to interpolate (and extrapolate) over NaN pixels", 

204 ) 

205 doWrite = pexConfig.Field( 

206 doc="Persist coadd?", 

207 dtype=bool, 

208 default=True, 

209 ) 

210 doNImage = pexConfig.Field( 

211 doc="Create image of number of contributing exposures for each pixel", 

212 dtype=bool, 

213 default=False, 

214 ) 

215 doUsePsfMatchedPolygons = pexConfig.Field( 

216 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

217 dtype=bool, 

218 default=False, 

219 ) 

220 maskPropagationThresholds = pexConfig.DictField( 

221 keytype=str, 

222 itemtype=float, 

223 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

224 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

225 "would have contributed exceeds this value."), 

226 default={"SAT": 0.1}, 

227 ) 

228 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

229 doc="Mask planes to remove before coadding") 

230 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

231 doc="Set mask and flag bits for bright objects?") 

232 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

233 doc="Name of mask bit used for bright objects") 

234 coaddPsf = pexConfig.ConfigField( 

235 doc="Configuration for CoaddPsf", 

236 dtype=measAlg.CoaddPsfConfig, 

237 ) 

238 doAttachTransmissionCurve = pexConfig.Field( 

239 dtype=bool, default=False, optional=False, 

240 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

241 "(requires all input Exposures to have TransmissionCurves).") 

242 ) 

243 hasFakes = pexConfig.Field( 

244 dtype=bool, 

245 default=False, 

246 doc="Should be set to True if fake sources have been inserted into the input data." 

247 ) 

248 doSelectVisits = pexConfig.Field( 

249 doc="Coadd only visits selected by a SelectVisitsTask", 

250 dtype=bool, 

251 default=False, 

252 ) 

253 doInputMap = pexConfig.Field( 

254 doc="Create a bitwise map of coadd inputs", 

255 dtype=bool, 

256 default=False, 

257 ) 

258 inputMapper = pexConfig.ConfigurableField( 

259 doc="Input map creation subtask.", 

260 target=HealSparseInputMapTask, 

261 ) 

262 

263 def setDefaults(self): 

264 super().setDefaults() 

265 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

266 

267 def validate(self): 

268 super().validate() 

269 if self.doPsfMatch: 

270 # Backwards compatibility. 

271 # Configs do not have loggers 

272 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

273 self.warpType = 'psfMatched' 

274 if self.doSigmaClip and self.statistic != "MEANCLIP": 

275 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

276 self.statistic = "MEANCLIP" 

277 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

278 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

279 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

280 

281 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

282 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

283 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

284 if str(k) not in unstackableStats] 

285 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

286 % (self.statistic, stackableStats)) 

287 

288 

289class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

290 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

291 

292 We want to assemble a coadded image from a set of Warps (also called 

293 coadded temporary exposures or ``coaddTempExps``). 

294 Each input Warp covers a patch on the sky and corresponds to a single 

295 run/visit/exposure of the covered patch. We provide the task with a list 

296 of Warps (``selectDataList``) from which it selects Warps that cover the 

297 specified patch (pointed at by ``dataRef``). 

298 Each Warp that goes into a coadd will typically have an independent 

299 photometric zero-point. Therefore, we must scale each Warp to set it to 

300 a common photometric zeropoint. WarpType may be one of 'direct' or 

301 'psfMatched', and the boolean configs `config.makeDirect` and 

302 `config.makePsfMatched` set which of the warp types will be coadded. 

303 The coadd is computed as a mean with optional outlier rejection. 

304 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

305 Finally, Warps can have bad 'NaN' pixels which received no input from the 

306 source calExps. We interpolate over these bad (NaN) pixels. 

307 

308 `AssembleCoaddTask` uses several sub-tasks. These are 

309 

310 - `ScaleZeroPointTask` 

311 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

312 - `InterpImageTask` 

313 - interpolate across bad pixels (NaN) in the final coadd 

314 

315 You can retarget these subtasks if you wish. 

316 

317 Notes 

318 ----- 

319 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

320 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

321 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

322 no debug variables of its own. Some of the subtasks may support debug 

323 variables. See the documentation for the subtasks for further information. 

324 

325 Examples 

326 -------- 

327 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

328 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

329 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

330 inputs: a data reference to the tract patch and filter to be coadded, and 

331 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

332 ``--selectId``, respectively: 

333 

334 .. code-block:: none 

335 

336 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

337 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

338 

339 Only the Warps that cover the specified tract and patch will be coadded. 

340 A list of the available optional arguments can be obtained by calling 

341 ``assembleCoadd.py`` with the ``--help`` command line argument: 

342 

343 .. code-block:: none 

344 

345 assembleCoadd.py --help 

346 

347 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

348 multi-band processing, we will generate the HSC-I & -R band coadds from 

349 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

350 assuming that the lsst stack has been already set up, we must set up the 

351 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

352 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

353 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

354 coadds, we must first 

355 

356 - processCcd 

357 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

358 - makeSkyMap 

359 - create a skymap that covers the area of the sky present in the raw exposures 

360 - makeCoaddTempExp 

361 - warp the individual calibrated exposures to the tangent plane of the coadd 

362 

363 We can perform all of these steps by running 

364 

365 .. code-block:: none 

366 

367 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

368 

369 This will produce warped exposures for each visit. To coadd the warped 

370 data, we call assembleCoadd.py as follows: 

371 

372 .. code-block:: none 

373 

374 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

375 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

376 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

377 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

378 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

379 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

380 --selectId visit=903988 ccd=24 

381 

382 that will process the HSC-I band data. The results are written in 

383 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

384 

385 You may also choose to run: 

386 

387 .. code-block:: none 

388 

389 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

390 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

391 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

392 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

393 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

394 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

395 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

396 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

397 

398 to generate the coadd for the HSC-R band if you are interested in 

399 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

400 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

401 rather than `AssembleCoaddTask` to make the coadd. 

402 """ 

403 ConfigClass = AssembleCoaddConfig 

404 _DefaultName = "assembleCoadd" 

405 

406 def __init__(self, *args, **kwargs): 

407 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

408 if args: 408 ↛ 409line 408 didn't jump to line 409, because the condition on line 408 was never true

409 argNames = ["config", "name", "parentTask", "log"] 

410 kwargs.update({k: v for k, v in zip(argNames, args)}) 

411 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

412 "PipelineTask will not take positional args" % argNames, FutureWarning) 

413 

414 super().__init__(**kwargs) 

415 self.makeSubtask("interpImage") 

416 self.makeSubtask("scaleZeroPoint") 

417 

418 if self.config.doMaskBrightObjects: 418 ↛ 419line 418 didn't jump to line 419, because the condition on line 418 was never true

419 mask = afwImage.Mask() 

420 try: 

421 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

422 except pexExceptions.LsstCppException: 

423 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

424 mask.getMaskPlaneDict().keys()) 

425 del mask 

426 

427 if self.config.doInputMap: 427 ↛ 428line 427 didn't jump to line 428, because the condition on line 427 was never true

428 self.makeSubtask("inputMapper") 

429 

430 self.warpType = self.config.warpType 

431 

432 @utils.inheritDoc(pipeBase.PipelineTask) 

433 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

434 # Docstring to be formatted with info from PipelineTask.runQuantum 

435 """ 

436 Notes 

437 ----- 

438 Assemble a coadd from a set of Warps. 

439 

440 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

441 Analogous to `runDataRef`, it prepares all the data products to be 

442 passed to `run`, and processes the results before returning a struct 

443 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

444 Therefore, its inputs are accessed subregion by subregion 

445 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

446 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

447 correspond to an update in `runDataRef` while both entry points 

448 are used. 

449 """ 

450 inputData = butlerQC.get(inputRefs) 

451 

452 # Construct skyInfo expected by run 

453 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

454 skyMap = inputData["skyMap"] 

455 outputDataId = butlerQC.quantum.dataId 

456 

457 inputData['skyInfo'] = makeSkyInfo(skyMap, 

458 tractId=outputDataId['tract'], 

459 patchId=outputDataId['patch']) 

460 

461 if self.config.doSelectVisits: 

462 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

463 else: 

464 warpRefList = inputData['inputWarps'] 

465 

466 # Perform same middle steps as `runDataRef` does 

467 inputs = self.prepareInputs(warpRefList) 

468 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

469 self.getTempExpDatasetName(self.warpType)) 

470 if len(inputs.tempExpRefList) == 0: 

471 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

472 

473 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

474 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

475 inputs.weightList, supplementaryData=supplementaryData) 

476 

477 inputData.setdefault('brightObjectMask', None) 

478 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

479 

480 if self.config.doWrite: 

481 butlerQC.put(retStruct, outputRefs) 

482 return retStruct 

483 

484 @pipeBase.timeMethod 

485 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

486 """Assemble a coadd from a set of Warps. 

487 

488 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

489 Compute weights to be applied to each Warp and 

490 find scalings to match the photometric zeropoint to a reference Warp. 

491 Assemble the Warps using `run`. Interpolate over NaNs and 

492 optionally write the coadd to disk. Return the coadded exposure. 

493 

494 Parameters 

495 ---------- 

496 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

497 Data reference defining the patch for coaddition and the 

498 reference Warp (if ``config.autoReference=False``). 

499 Used to access the following data products: 

500 - ``self.config.coaddName + "Coadd_skyMap"`` 

501 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

502 - ``self.config.coaddName + "Coadd"`` 

503 selectDataList : `list` 

504 List of data references to Calexps. Data to be coadded will be 

505 selected from this list based on overlap with the patch defined 

506 by dataRef, grouped by visit, and converted to a list of data 

507 references to warps. 

508 warpRefList : `list` 

509 List of data references to Warps to be coadded. 

510 Note: `warpRefList` is just the new name for `tempExpRefList`. 

511 

512 Returns 

513 ------- 

514 retStruct : `lsst.pipe.base.Struct` 

515 Result struct with components: 

516 

517 - ``coaddExposure``: coadded exposure (``Exposure``). 

518 - ``nImage``: exposure count image (``Image``). 

519 """ 

520 if selectDataList and warpRefList: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true

521 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

522 "and which to use is ambiguous. Please pass only one.") 

523 

524 skyInfo = self.getSkyInfo(dataRef) 

525 if warpRefList is None: 

526 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

527 if len(calExpRefList) == 0: 527 ↛ 528line 527 didn't jump to line 528, because the condition on line 527 was never true

528 self.log.warning("No exposures to coadd") 

529 return 

530 self.log.info("Coadding %d exposures", len(calExpRefList)) 

531 

532 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

533 

534 inputData = self.prepareInputs(warpRefList) 

535 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

536 self.getTempExpDatasetName(self.warpType)) 

537 if len(inputData.tempExpRefList) == 0: 537 ↛ 538line 537 didn't jump to line 538, because the condition on line 537 was never true

538 self.log.warning("No coadd temporary exposures found") 

539 return 

540 

541 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

542 

543 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

544 inputData.weightList, supplementaryData=supplementaryData) 

545 

546 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

547 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

548 

549 if self.config.doWrite: 

550 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 550 ↛ 551line 550 didn't jump to line 551, because the condition on line 550 was never true

551 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

552 else: 

553 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

554 self.log.info("Persisting %s", coaddDatasetName) 

555 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

556 if self.config.doNImage and retStruct.nImage is not None: 556 ↛ 557line 556 didn't jump to line 557, because the condition on line 556 was never true

557 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

558 

559 return retStruct 

560 

561 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

562 """Interpolate over missing data and mask bright stars. 

563 

564 Parameters 

565 ---------- 

566 coaddExposure : `lsst.afw.image.Exposure` 

567 The coadded exposure to process. 

568 dataRef : `lsst.daf.persistence.ButlerDataRef` 

569 Butler data reference for supplementary data. 

570 """ 

571 if self.config.doInterp: 571 ↛ 578line 571 didn't jump to line 578, because the condition on line 571 was never false

572 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

573 # The variance must be positive; work around for DM-3201. 

574 varArray = coaddExposure.variance.array 

575 with numpy.errstate(invalid="ignore"): 

576 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

577 

578 if self.config.doMaskBrightObjects: 578 ↛ 579line 578 didn't jump to line 579, because the condition on line 578 was never true

579 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

580 

581 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

582 """Make additional inputs to run() specific to subclasses (Gen2) 

583 

584 Duplicates interface of `runDataRef` method 

585 Available to be implemented by subclasses only if they need the 

586 coadd dataRef for performing preliminary processing before 

587 assembling the coadd. 

588 

589 Parameters 

590 ---------- 

591 dataRef : `lsst.daf.persistence.ButlerDataRef` 

592 Butler data reference for supplementary data. 

593 selectDataList : `list` (optional) 

594 Optional List of data references to Calexps. 

595 warpRefList : `list` (optional) 

596 Optional List of data references to Warps. 

597 """ 

598 return pipeBase.Struct() 

599 

600 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

601 """Make additional inputs to run() specific to subclasses (Gen3) 

602 

603 Duplicates interface of `runQuantum` method. 

604 Available to be implemented by subclasses only if they need the 

605 coadd dataRef for performing preliminary processing before 

606 assembling the coadd. 

607 

608 Parameters 

609 ---------- 

610 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

611 Gen3 Butler object for fetching additional data products before 

612 running the Task specialized for quantum being processed 

613 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

614 Attributes are the names of the connections describing input dataset types. 

615 Values are DatasetRefs that task consumes for corresponding dataset type. 

616 DataIds are guaranteed to match data objects in ``inputData``. 

617 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

618 Attributes are the names of the connections describing output dataset types. 

619 Values are DatasetRefs that task is to produce 

620 for corresponding dataset type. 

621 """ 

622 return pipeBase.Struct() 

623 

624 def getTempExpRefList(self, patchRef, calExpRefList): 

625 """Generate list data references corresponding to warped exposures 

626 that lie within the patch to be coadded. 

627 

628 Parameters 

629 ---------- 

630 patchRef : `dataRef` 

631 Data reference for patch. 

632 calExpRefList : `list` 

633 List of data references for input calexps. 

634 

635 Returns 

636 ------- 

637 tempExpRefList : `list` 

638 List of Warp/CoaddTempExp data references. 

639 """ 

640 butler = patchRef.getButler() 

641 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

642 self.getTempExpDatasetName(self.warpType)) 

643 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

644 g, groupData.keys) for 

645 g in groupData.groups.keys()] 

646 return tempExpRefList 

647 

648 def prepareInputs(self, refList): 

649 """Prepare the input warps for coaddition by measuring the weight for 

650 each warp and the scaling for the photometric zero point. 

651 

652 Each Warp has its own photometric zeropoint and background variance. 

653 Before coadding these Warps together, compute a scale factor to 

654 normalize the photometric zeropoint and compute the weight for each Warp. 

655 

656 Parameters 

657 ---------- 

658 refList : `list` 

659 List of data references to tempExp 

660 

661 Returns 

662 ------- 

663 result : `lsst.pipe.base.Struct` 

664 Result struct with components: 

665 

666 - ``tempExprefList``: `list` of data references to tempExp. 

667 - ``weightList``: `list` of weightings. 

668 - ``imageScalerList``: `list` of image scalers. 

669 """ 

670 statsCtrl = afwMath.StatisticsControl() 

671 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

672 statsCtrl.setNumIter(self.config.clipIter) 

673 statsCtrl.setAndMask(self.getBadPixelMask()) 

674 statsCtrl.setNanSafe(True) 

675 # compute tempExpRefList: a list of tempExpRef that actually exist 

676 # and weightList: a list of the weight of the associated coadd tempExp 

677 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

678 tempExpRefList = [] 

679 weightList = [] 

680 imageScalerList = [] 

681 tempExpName = self.getTempExpDatasetName(self.warpType) 

682 for tempExpRef in refList: 

683 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

684 # therefore have no datasetExists() method 

685 if not isinstance(tempExpRef, DeferredDatasetHandle): 685 ↛ 690line 685 didn't jump to line 690, because the condition on line 685 was never false

686 if not tempExpRef.datasetExists(tempExpName): 

687 self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

688 continue 

689 

690 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

691 # Ignore any input warp that is empty of data 

692 if numpy.isnan(tempExp.image.array).all(): 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true

693 continue 

694 maskedImage = tempExp.getMaskedImage() 

695 imageScaler = self.scaleZeroPoint.computeImageScaler( 

696 exposure=tempExp, 

697 dataRef=tempExpRef, 

698 ) 

699 try: 

700 imageScaler.scaleMaskedImage(maskedImage) 

701 except Exception as e: 

702 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

703 continue 

704 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

705 afwMath.MEANCLIP, statsCtrl) 

706 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

707 weight = 1.0 / float(meanVar) 

708 if not numpy.isfinite(weight): 708 ↛ 709line 708 didn't jump to line 709, because the condition on line 708 was never true

709 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

710 continue 

711 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

712 

713 del maskedImage 

714 del tempExp 

715 

716 tempExpRefList.append(tempExpRef) 

717 weightList.append(weight) 

718 imageScalerList.append(imageScaler) 

719 

720 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

721 imageScalerList=imageScalerList) 

722 

723 def prepareStats(self, mask=None): 

724 """Prepare the statistics for coadding images. 

725 

726 Parameters 

727 ---------- 

728 mask : `int`, optional 

729 Bit mask value to exclude from coaddition. 

730 

731 Returns 

732 ------- 

733 stats : `lsst.pipe.base.Struct` 

734 Statistics structure with the following fields: 

735 

736 - ``statsCtrl``: Statistics control object for coadd 

737 (`lsst.afw.math.StatisticsControl`) 

738 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

739 """ 

740 if mask is None: 

741 mask = self.getBadPixelMask() 

742 statsCtrl = afwMath.StatisticsControl() 

743 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

744 statsCtrl.setNumIter(self.config.clipIter) 

745 statsCtrl.setAndMask(mask) 

746 statsCtrl.setNanSafe(True) 

747 statsCtrl.setWeighted(True) 

748 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

749 for plane, threshold in self.config.maskPropagationThresholds.items(): 

750 bit = afwImage.Mask.getMaskPlane(plane) 

751 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

752 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

753 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

754 

755 @pipeBase.timeMethod 

756 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

757 altMaskList=None, mask=None, supplementaryData=None): 

758 """Assemble a coadd from input warps 

759 

760 Assemble the coadd using the provided list of coaddTempExps. Since 

761 the full coadd covers a patch (a large area), the assembly is 

762 performed over small areas on the image at a time in order to 

763 conserve memory usage. Iterate over subregions within the outer 

764 bbox of the patch using `assembleSubregion` to stack the corresponding 

765 subregions from the coaddTempExps with the statistic specified. 

766 Set the edge bits the coadd mask based on the weight map. 

767 

768 Parameters 

769 ---------- 

770 skyInfo : `lsst.pipe.base.Struct` 

771 Struct with geometric information about the patch. 

772 tempExpRefList : `list` 

773 List of data references to Warps (previously called CoaddTempExps). 

774 imageScalerList : `list` 

775 List of image scalers. 

776 weightList : `list` 

777 List of weights 

778 altMaskList : `list`, optional 

779 List of alternate masks to use rather than those stored with 

780 tempExp. 

781 mask : `int`, optional 

782 Bit mask value to exclude from coaddition. 

783 supplementaryData : lsst.pipe.base.Struct, optional 

784 Struct with additional data products needed to assemble coadd. 

785 Only used by subclasses that implement `makeSupplementaryData` 

786 and override `run`. 

787 

788 Returns 

789 ------- 

790 result : `lsst.pipe.base.Struct` 

791 Result struct with components: 

792 

793 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

794 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

795 - ``inputMap``: bit-wise map of inputs, if requested. 

796 - ``warpRefList``: input list of refs to the warps ( 

797 ``lsst.daf.butler.DeferredDatasetHandle`` or 

798 ``lsst.daf.persistence.ButlerDataRef``) 

799 (unmodified) 

800 - ``imageScalerList``: input list of image scalers (unmodified) 

801 - ``weightList``: input list of weights (unmodified) 

802 """ 

803 tempExpName = self.getTempExpDatasetName(self.warpType) 

804 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

805 stats = self.prepareStats(mask=mask) 

806 

807 if altMaskList is None: 

808 altMaskList = [None]*len(tempExpRefList) 

809 

810 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

811 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

812 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

813 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

814 coaddMaskedImage = coaddExposure.getMaskedImage() 

815 subregionSizeArr = self.config.subregionSize 

816 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

817 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

818 if self.config.doNImage: 818 ↛ 819line 818 didn't jump to line 819, because the condition on line 818 was never true

819 nImage = afwImage.ImageU(skyInfo.bbox) 

820 else: 

821 nImage = None 

822 # If inputMap is requested, create the initial version that can be masked in 

823 # assembleSubregion. 

824 if self.config.doInputMap: 824 ↛ 825line 824 didn't jump to line 825, because the condition on line 824 was never true

825 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

826 skyInfo.wcs, 

827 coaddExposure.getInfo().getCoaddInputs().ccds) 

828 

829 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

830 try: 

831 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

832 weightList, altMaskList, stats.flags, stats.ctrl, 

833 nImage=nImage) 

834 except Exception as e: 

835 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

836 

837 # If inputMap is requested, we must finalize the map after the accumulation. 

838 if self.config.doInputMap: 838 ↛ 839line 838 didn't jump to line 839, because the condition on line 838 was never true

839 self.inputMapper.finalize_ccd_input_map_mask() 

840 inputMap = self.inputMapper.ccd_input_map 

841 else: 

842 inputMap = None 

843 

844 self.setInexactPsf(coaddMaskedImage.getMask()) 

845 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

846 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

847 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

848 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

849 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

850 weightList=weightList, inputMap=inputMap) 

851 

852 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

853 """Set the metadata for the coadd. 

854 

855 This basic implementation sets the filter from the first input. 

856 

857 Parameters 

858 ---------- 

859 coaddExposure : `lsst.afw.image.Exposure` 

860 The target exposure for the coadd. 

861 tempExpRefList : `list` 

862 List of data references to tempExp. 

863 weightList : `list` 

864 List of weights. 

865 """ 

866 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

867 tempExpName = self.getTempExpDatasetName(self.warpType) 

868 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

869 # (and we need more than just the PropertySet that contains the header), which is not possible 

870 # with the current butler (see #2777). 

871 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

872 

873 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 873 ↛ 875line 873 didn't jump to line 875, because the condition on line 873 was never true

874 # Gen 3 API 

875 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

876 else: 

877 # Gen 2 API. Delete this when Gen 2 retired 

878 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

879 for tempExpRef in tempExpRefList] 

880 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

881 

882 # Set the coadd FilterLabel to the band of the first input exposure: 

883 # Coadds are calibrated, so the physical label is now meaningless. 

884 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

885 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

886 coaddInputs.ccds.reserve(numCcds) 

887 coaddInputs.visits.reserve(len(tempExpList)) 

888 

889 for tempExp, weight in zip(tempExpList, weightList): 

890 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

891 

892 if self.config.doUsePsfMatchedPolygons: 

893 self.shrinkValidPolygons(coaddInputs) 

894 

895 coaddInputs.visits.sort() 

896 if self.warpType == "psfMatched": 

897 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

898 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

899 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

900 # having the maximum width (sufficient because square) 

901 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

902 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

903 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

904 else: 

905 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

906 self.config.coaddPsf.makeControl()) 

907 coaddExposure.setPsf(psf) 

908 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

909 coaddExposure.getWcs()) 

910 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

911 if self.config.doAttachTransmissionCurve: 

912 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

913 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

914 

915 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

916 altMaskList, statsFlags, statsCtrl, nImage=None): 

917 """Assemble the coadd for a sub-region. 

918 

919 For each coaddTempExp, check for (and swap in) an alternative mask 

920 if one is passed. Remove mask planes listed in 

921 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

922 `lsst.afw.math.statisticsStack` with the statistic specified by 

923 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

924 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

925 an N-sigma clipped mean where N and iterations are specified by 

926 statsCtrl. Assign the stacked subregion back to the coadd. 

927 

928 Parameters 

929 ---------- 

930 coaddExposure : `lsst.afw.image.Exposure` 

931 The target exposure for the coadd. 

932 bbox : `lsst.geom.Box` 

933 Sub-region to coadd. 

934 tempExpRefList : `list` 

935 List of data reference to tempExp. 

936 imageScalerList : `list` 

937 List of image scalers. 

938 weightList : `list` 

939 List of weights. 

940 altMaskList : `list` 

941 List of alternate masks to use rather than those stored with 

942 tempExp, or None. Each element is dict with keys = mask plane 

943 name to which to add the spans. 

944 statsFlags : `lsst.afw.math.Property` 

945 Property object for statistic for coadd. 

946 statsCtrl : `lsst.afw.math.StatisticsControl` 

947 Statistics control object for coadd. 

948 nImage : `lsst.afw.image.ImageU`, optional 

949 Keeps track of exposure count for each pixel. 

950 """ 

951 self.log.debug("Computing coadd over %s", bbox) 

952 tempExpName = self.getTempExpDatasetName(self.warpType) 

953 coaddExposure.mask.addMaskPlane("REJECTED") 

954 coaddExposure.mask.addMaskPlane("CLIPPED") 

955 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

956 maskMap = self.setRejectedMaskMapping(statsCtrl) 

957 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

958 maskedImageList = [] 

959 if nImage is not None: 959 ↛ 960line 959 didn't jump to line 960, because the condition on line 959 was never true

960 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

961 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

962 

963 if isinstance(tempExpRef, DeferredDatasetHandle): 963 ↛ 965line 963 didn't jump to line 965, because the condition on line 963 was never true

964 # Gen 3 API 

965 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

966 else: 

967 # Gen 2 API. Delete this when Gen 2 retired 

968 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

969 

970 maskedImage = exposure.getMaskedImage() 

971 mask = maskedImage.getMask() 

972 if altMask is not None: 

973 self.applyAltMaskPlanes(mask, altMask) 

974 imageScaler.scaleMaskedImage(maskedImage) 

975 

976 # Add 1 for each pixel which is not excluded by the exclude mask. 

977 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

978 if nImage is not None: 978 ↛ 979line 978 didn't jump to line 979, because the condition on line 978 was never true

979 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

980 if self.config.removeMaskPlanes: 980 ↛ 982line 980 didn't jump to line 982, because the condition on line 980 was never false

981 self.removeMaskPlanes(maskedImage) 

982 maskedImageList.append(maskedImage) 

983 

984 if self.config.doInputMap: 984 ↛ 985line 984 didn't jump to line 985, because the condition on line 984 was never true

985 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

986 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

987 

988 with self.timer("stack"): 

989 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

990 clipped, # also set output to CLIPPED if sigma-clipped 

991 maskMap) 

992 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

993 if nImage is not None: 993 ↛ 994line 993 didn't jump to line 994, because the condition on line 993 was never true

994 nImage.assign(subNImage, bbox) 

995 

996 def removeMaskPlanes(self, maskedImage): 

997 """Unset the mask of an image for mask planes specified in the config. 

998 

999 Parameters 

1000 ---------- 

1001 maskedImage : `lsst.afw.image.MaskedImage` 

1002 The masked image to be modified. 

1003 """ 

1004 mask = maskedImage.getMask() 

1005 for maskPlane in self.config.removeMaskPlanes: 

1006 try: 

1007 mask &= ~mask.getPlaneBitMask(maskPlane) 

1008 except pexExceptions.InvalidParameterError: 

1009 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1010 maskPlane) 

1011 

1012 @staticmethod 

1013 def setRejectedMaskMapping(statsCtrl): 

1014 """Map certain mask planes of the warps to new planes for the coadd. 

1015 

1016 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1017 or CLIPPED, set it to REJECTED on the coadd. 

1018 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1019 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1020 

1021 Parameters 

1022 ---------- 

1023 statsCtrl : `lsst.afw.math.StatisticsControl` 

1024 Statistics control object for coadd 

1025 

1026 Returns 

1027 ------- 

1028 maskMap : `list` of `tuple` of `int` 

1029 A list of mappings of mask planes of the warped exposures to 

1030 mask planes of the coadd. 

1031 """ 

1032 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1033 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1034 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1035 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1036 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1037 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1038 (clipped, clipped)] 

1039 return maskMap 

1040 

1041 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1042 """Apply in place alt mask formatted as SpanSets to a mask. 

1043 

1044 Parameters 

1045 ---------- 

1046 mask : `lsst.afw.image.Mask` 

1047 Original mask. 

1048 altMaskSpans : `dict` 

1049 SpanSet lists to apply. Each element contains the new mask 

1050 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1051 and list of SpanSets to apply to the mask. 

1052 

1053 Returns 

1054 ------- 

1055 mask : `lsst.afw.image.Mask` 

1056 Updated mask. 

1057 """ 

1058 if self.config.doUsePsfMatchedPolygons: 

1059 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1059 ↛ 1067line 1059 didn't jump to line 1067, because the condition on line 1059 was never false

1060 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1061 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1062 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1063 # This mask-clearing step must only occur *before* applying the new masks below 

1064 for spanSet in altMaskSpans['NO_DATA']: 

1065 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1066 

1067 for plane, spanSetList in altMaskSpans.items(): 

1068 maskClipValue = mask.addMaskPlane(plane) 

1069 for spanSet in spanSetList: 

1070 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1071 return mask 

1072 

1073 def shrinkValidPolygons(self, coaddInputs): 

1074 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1075 

1076 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1077 does not have a validPolygon, create one from its bbox. 

1078 

1079 Parameters 

1080 ---------- 

1081 coaddInputs : `lsst.afw.image.coaddInputs` 

1082 Original mask. 

1083 

1084 """ 

1085 for ccd in coaddInputs.ccds: 

1086 polyOrig = ccd.getValidPolygon() 

1087 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1088 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1089 if polyOrig: 1089 ↛ 1090line 1089 didn't jump to line 1090, because the condition on line 1089 was never true

1090 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1091 else: 

1092 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1093 ccd.setValidPolygon(validPolygon) 

1094 

1095 def readBrightObjectMasks(self, dataRef): 

1096 """Retrieve the bright object masks. 

1097 

1098 Returns None on failure. 

1099 

1100 Parameters 

1101 ---------- 

1102 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1103 A Butler dataRef. 

1104 

1105 Returns 

1106 ------- 

1107 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1108 Bright object mask from the Butler object, or None if it cannot 

1109 be retrieved. 

1110 """ 

1111 try: 

1112 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1113 except Exception as e: 

1114 self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1115 return None 

1116 

1117 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1118 """Set the bright object masks. 

1119 

1120 Parameters 

1121 ---------- 

1122 exposure : `lsst.afw.image.Exposure` 

1123 Exposure under consideration. 

1124 dataId : `lsst.daf.persistence.dataId` 

1125 Data identifier dict for patch. 

1126 brightObjectMasks : `lsst.afw.table` 

1127 Table of bright objects to mask. 

1128 """ 

1129 

1130 if brightObjectMasks is None: 

1131 self.log.warning("Unable to apply bright object mask: none supplied") 

1132 return 

1133 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1134 mask = exposure.getMaskedImage().getMask() 

1135 wcs = exposure.getWcs() 

1136 plateScale = wcs.getPixelScale().asArcseconds() 

1137 

1138 for rec in brightObjectMasks: 

1139 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1140 if rec["type"] == "box": 

1141 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1142 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1143 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1144 

1145 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1146 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1147 

1148 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1149 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1150 spans = afwGeom.SpanSet(bbox) 

1151 elif rec["type"] == "circle": 

1152 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1153 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1154 else: 

1155 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1156 continue 

1157 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1158 

1159 def setInexactPsf(self, mask): 

1160 """Set INEXACT_PSF mask plane. 

1161 

1162 If any of the input images isn't represented in the coadd (due to 

1163 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1164 these pixels. 

1165 

1166 Parameters 

1167 ---------- 

1168 mask : `lsst.afw.image.Mask` 

1169 Coadded exposure's mask, modified in-place. 

1170 """ 

1171 mask.addMaskPlane("INEXACT_PSF") 

1172 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1173 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1174 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1175 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1176 array = mask.getArray() 

1177 selected = array & (sensorEdge | clipped | rejected) > 0 

1178 array[selected] |= inexactPsf 

1179 

1180 @classmethod 

1181 def _makeArgumentParser(cls): 

1182 """Create an argument parser. 

1183 """ 

1184 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1185 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1186 + cls.ConfigClass().warpType + "Warp", 

1187 help="data ID, e.g. --id tract=12345 patch=1,2", 

1188 ContainerClass=AssembleCoaddDataIdContainer) 

1189 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1190 ContainerClass=SelectDataIdContainer) 

1191 return parser 

1192 

1193 @staticmethod 

1194 def _subBBoxIter(bbox, subregionSize): 

1195 """Iterate over subregions of a bbox. 

1196 

1197 Parameters 

1198 ---------- 

1199 bbox : `lsst.geom.Box2I` 

1200 Bounding box over which to iterate. 

1201 subregionSize: `lsst.geom.Extent2I` 

1202 Size of sub-bboxes. 

1203 

1204 Yields 

1205 ------ 

1206 subBBox : `lsst.geom.Box2I` 

1207 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1208 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1209 the edges of ``bbox``, but it will never be empty. 

1210 """ 

1211 if bbox.isEmpty(): 1211 ↛ 1212line 1211 didn't jump to line 1212, because the condition on line 1211 was never true

1212 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1213 if subregionSize[0] < 1 or subregionSize[1] < 1: 1213 ↛ 1214line 1213 didn't jump to line 1214, because the condition on line 1213 was never true

1214 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1215 

1216 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1217 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1218 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1219 subBBox.clip(bbox) 

1220 if subBBox.isEmpty(): 1220 ↛ 1221line 1220 didn't jump to line 1221, because the condition on line 1220 was never true

1221 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1222 "colShift=%s, rowShift=%s" % 

1223 (bbox, subregionSize, colShift, rowShift)) 

1224 yield subBBox 

1225 

1226 def filterWarps(self, inputs, goodVisits): 

1227 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1228 

1229 Parameters 

1230 ---------- 

1231 inputs : list 

1232 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1233 goodVisit : `dict` 

1234 Dictionary with good visitIds as the keys. Value ignored. 

1235 

1236 Returns: 

1237 -------- 

1238 filteredInputs : `list` 

1239 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1240 """ 

1241 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1242 filteredInputs = [] 

1243 for visit in goodVisits.keys(): 

1244 filteredInputs.append(inputWarpDict[visit]) 

1245 return filteredInputs 

1246 

1247 

1248class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1249 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1250 """ 

1251 

1252 def makeDataRefList(self, namespace): 

1253 """Make self.refList from self.idList. 

1254 

1255 Parameters 

1256 ---------- 

1257 namespace 

1258 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1259 """ 

1260 datasetType = namespace.config.coaddName + "Coadd" 

1261 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1262 

1263 for dataId in self.idList: 

1264 # tract and patch are required 

1265 for key in keysCoadd: 

1266 if key not in dataId: 

1267 raise RuntimeError("--id must include " + key) 

1268 

1269 dataRef = namespace.butler.dataRef( 

1270 datasetType=datasetType, 

1271 dataId=dataId, 

1272 ) 

1273 self.refList.append(dataRef) 

1274 

1275 

1276def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1277 """Function to count the number of pixels with a specific mask in a 

1278 footprint. 

1279 

1280 Find the intersection of mask & footprint. Count all pixels in the mask 

1281 that are in the intersection that have bitmask set but do not have 

1282 ignoreMask set. Return the count. 

1283 

1284 Parameters 

1285 ---------- 

1286 mask : `lsst.afw.image.Mask` 

1287 Mask to define intersection region by. 

1288 footprint : `lsst.afw.detection.Footprint` 

1289 Footprint to define the intersection region by. 

1290 bitmask 

1291 Specific mask that we wish to count the number of occurances of. 

1292 ignoreMask 

1293 Pixels to not consider. 

1294 

1295 Returns 

1296 ------- 

1297 result : `int` 

1298 Count of number of pixels in footprint with specified mask. 

1299 """ 

1300 bbox = footprint.getBBox() 

1301 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1302 fp = afwImage.Mask(bbox) 

1303 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1304 footprint.spans.setMask(fp, bitmask) 

1305 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1306 (subMask.getArray() & ignoreMask) == 0).sum() 

1307 

1308 

1309class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1310 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1311 """ 

1312 clipDetection = pexConfig.ConfigurableField( 

1313 target=SourceDetectionTask, 

1314 doc="Detect sources on difference between unclipped and clipped coadd") 

1315 minClipFootOverlap = pexConfig.Field( 

1316 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1317 dtype=float, 

1318 default=0.6 

1319 ) 

1320 minClipFootOverlapSingle = pexConfig.Field( 

1321 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1322 "clipped when only one visit overlaps", 

1323 dtype=float, 

1324 default=0.5 

1325 ) 

1326 minClipFootOverlapDouble = pexConfig.Field( 

1327 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1328 "clipped when two visits overlap", 

1329 dtype=float, 

1330 default=0.45 

1331 ) 

1332 maxClipFootOverlapDouble = pexConfig.Field( 

1333 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1334 "considering two visits", 

1335 dtype=float, 

1336 default=0.15 

1337 ) 

1338 minBigOverlap = pexConfig.Field( 

1339 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1340 "when labeling clipped footprints", 

1341 dtype=int, 

1342 default=100 

1343 ) 

1344 

1345 def setDefaults(self): 

1346 """Set default values for clipDetection. 

1347 

1348 Notes 

1349 ----- 

1350 The numeric values for these configuration parameters were 

1351 empirically determined, future work may further refine them. 

1352 """ 

1353 AssembleCoaddConfig.setDefaults(self) 

1354 self.clipDetection.doTempLocalBackground = False 

1355 self.clipDetection.reEstimateBackground = False 

1356 self.clipDetection.returnOriginalFootprints = False 

1357 self.clipDetection.thresholdPolarity = "both" 

1358 self.clipDetection.thresholdValue = 2 

1359 self.clipDetection.nSigmaToGrow = 2 

1360 self.clipDetection.minPixels = 4 

1361 self.clipDetection.isotropicGrow = True 

1362 self.clipDetection.thresholdType = "pixel_stdev" 

1363 self.sigmaClip = 1.5 

1364 self.clipIter = 3 

1365 self.statistic = "MEAN" 

1366 

1367 def validate(self): 

1368 if self.doSigmaClip: 

1369 log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1370 "Ignoring doSigmaClip.") 

1371 self.doSigmaClip = False 

1372 if self.statistic != "MEAN": 

1373 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1374 "(%s chosen). Please set statistic to MEAN." 

1375 % (self.statistic)) 

1376 AssembleCoaddTask.ConfigClass.validate(self) 

1377 

1378 

1379class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1380 """Assemble a coadded image from a set of coadded temporary exposures, 

1381 being careful to clip & flag areas with potential artifacts. 

1382 

1383 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1384 we clip outliers). The problem with doing this is that when computing the 

1385 coadd PSF at a given location, individual visit PSFs from visits with 

1386 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1387 In this task, we correct for this behavior by creating a new 

1388 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1389 coaddTempExps and the final coadd where 

1390 

1391 i. difference imaging suggests that there is an outlier and 

1392 ii. this outlier appears on only one or two images. 

1393 

1394 Such regions will not contribute to the final coadd. Furthermore, any 

1395 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1396 Note that the algorithm implemented by this task is preliminary and works 

1397 correctly for HSC data. Parameter modifications and or considerable 

1398 redesigning of the algorithm is likley required for other surveys. 

1399 

1400 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1401 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1402 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1403 if you wish. 

1404 

1405 Notes 

1406 ----- 

1407 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1408 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1409 see `baseDebug` for more about ``debug.py`` files. 

1410 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1411 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1412 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1413 for further information. 

1414 

1415 Examples 

1416 -------- 

1417 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1418 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1419 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1420 

1421 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1422 and filter to be coadded (specified using 

1423 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1424 along with a list of coaddTempExps to attempt to coadd (specified using 

1425 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1426 Only the coaddTempExps that cover the specified tract and patch will be 

1427 coadded. A list of the available optional arguments can be obtained by 

1428 calling assembleCoadd.py with the --help command line argument: 

1429 

1430 .. code-block:: none 

1431 

1432 assembleCoadd.py --help 

1433 

1434 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1435 context of multi-band processing, we will generate the HSC-I & -R band 

1436 coadds from HSC engineering test data provided in the ci_hsc package. 

1437 To begin, assuming that the lsst stack has been already set up, we must 

1438 set up the obs_subaru and ci_hsc packages. This defines the environment 

1439 variable $CI_HSC_DIR and points at the location of the package. The raw 

1440 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1441 the coadds, we must first 

1442 

1443 - ``processCcd`` 

1444 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1445 - ``makeSkyMap`` 

1446 create a skymap that covers the area of the sky present in the raw exposures 

1447 - ``makeCoaddTempExp`` 

1448 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1449 

1450 We can perform all of these steps by running 

1451 

1452 .. code-block:: none 

1453 

1454 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1455 

1456 This will produce warped coaddTempExps for each visit. To coadd the 

1457 warped data, we call ``assembleCoadd.py`` as follows: 

1458 

1459 .. code-block:: none 

1460 

1461 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1462 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1463 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1464 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1465 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1466 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1467 --selectId visit=903988 ccd=24 

1468 

1469 This will process the HSC-I band data. The results are written in 

1470 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1471 

1472 You may also choose to run: 

1473 

1474 .. code-block:: none 

1475 

1476 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1477 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1478 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1479 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1480 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1481 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1482 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1483 --selectId visit=903346 ccd=12 

1484 

1485 to generate the coadd for the HSC-R band if you are interested in following 

1486 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1487 """ 

1488 ConfigClass = SafeClipAssembleCoaddConfig 

1489 _DefaultName = "safeClipAssembleCoadd" 

1490 

1491 def __init__(self, *args, **kwargs): 

1492 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1493 schema = afwTable.SourceTable.makeMinimalSchema() 

1494 self.makeSubtask("clipDetection", schema=schema) 

1495 

1496 @utils.inheritDoc(AssembleCoaddTask) 

1497 @pipeBase.timeMethod 

1498 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1499 """Assemble the coadd for a region. 

1500 

1501 Compute the difference of coadds created with and without outlier 

1502 rejection to identify coadd pixels that have outlier values in some 

1503 individual visits. 

1504 Detect clipped regions on the difference image and mark these regions 

1505 on the one or two individual coaddTempExps where they occur if there 

1506 is significant overlap between the clipped region and a source. This 

1507 leaves us with a set of footprints from the difference image that have 

1508 been identified as having occured on just one or two individual visits. 

1509 However, these footprints were generated from a difference image. It 

1510 is conceivable for a large diffuse source to have become broken up 

1511 into multiple footprints acrosss the coadd difference in this process. 

1512 Determine the clipped region from all overlapping footprints from the 

1513 detected sources in each visit - these are big footprints. 

1514 Combine the small and big clipped footprints and mark them on a new 

1515 bad mask plane. 

1516 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1517 removal. Clipped footprints will no longer make it into the coadd 

1518 because they are marked in the new bad mask plane. 

1519 

1520 Notes 

1521 ----- 

1522 args and kwargs are passed but ignored in order to match the call 

1523 signature expected by the parent task. 

1524 """ 

1525 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1526 mask = exp.getMaskedImage().getMask() 

1527 mask.addMaskPlane("CLIPPED") 

1528 

1529 result = self.detectClip(exp, tempExpRefList) 

1530 

1531 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1532 

1533 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1534 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1535 # Append big footprints from individual Warps to result.clipSpans 

1536 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1537 result.detectionFootprints, maskClipValue, maskDetValue, 

1538 exp.getBBox()) 

1539 # Create mask of the current clipped footprints 

1540 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1541 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1542 

1543 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1544 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1545 maskClip |= maskClipBig 

1546 

1547 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1548 badMaskPlanes = self.config.badMaskPlanes[:] 

1549 badMaskPlanes.append("CLIPPED") 

1550 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1551 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1552 result.clipSpans, mask=badPixelMask) 

1553 

1554 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1555 """Return an exposure that contains the difference between unclipped 

1556 and clipped coadds. 

1557 

1558 Generate a difference image between clipped and unclipped coadds. 

1559 Compute the difference image by subtracting an outlier-clipped coadd 

1560 from an outlier-unclipped coadd. Return the difference image. 

1561 

1562 Parameters 

1563 ---------- 

1564 skyInfo : `lsst.pipe.base.Struct` 

1565 Patch geometry information, from getSkyInfo 

1566 tempExpRefList : `list` 

1567 List of data reference to tempExp 

1568 imageScalerList : `list` 

1569 List of image scalers 

1570 weightList : `list` 

1571 List of weights 

1572 

1573 Returns 

1574 ------- 

1575 exp : `lsst.afw.image.Exposure` 

1576 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1577 """ 

1578 config = AssembleCoaddConfig() 

1579 # getattr necessary because subtasks do not survive Config.toDict() 

1580 # exclude connections because the class of self.config.connections is not 

1581 # the same as AssembleCoaddConfig.connections, and the connections are not 

1582 # needed to run this task anyway. 

1583 configIntersection = {k: getattr(self.config, k) 

1584 for k, v in self.config.toDict().items() 

1585 if (k in config.keys() and k != "connections")} 

1586 configIntersection['doInputMap'] = False 

1587 configIntersection['doNImage'] = False 

1588 config.update(**configIntersection) 

1589 

1590 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1591 config.statistic = 'MEAN' 

1592 task = AssembleCoaddTask(config=config) 

1593 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1594 

1595 config.statistic = 'MEANCLIP' 

1596 task = AssembleCoaddTask(config=config) 

1597 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1598 

1599 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1600 coaddDiff -= coaddClip.getMaskedImage() 

1601 exp = afwImage.ExposureF(coaddDiff) 

1602 exp.setPsf(coaddMean.getPsf()) 

1603 return exp 

1604 

1605 def detectClip(self, exp, tempExpRefList): 

1606 """Detect clipped regions on an exposure and set the mask on the 

1607 individual tempExp masks. 

1608 

1609 Detect footprints in the difference image after smoothing the 

1610 difference image with a Gaussian kernal. Identify footprints that 

1611 overlap with one or two input ``coaddTempExps`` by comparing the 

1612 computed overlap fraction to thresholds set in the config. A different 

1613 threshold is applied depending on the number of overlapping visits 

1614 (restricted to one or two). If the overlap exceeds the thresholds, 

1615 the footprint is considered "CLIPPED" and is marked as such on the 

1616 coaddTempExp. Return a struct with the clipped footprints, the indices 

1617 of the ``coaddTempExps`` that end up overlapping with the clipped 

1618 footprints, and a list of new masks for the ``coaddTempExps``. 

1619 

1620 Parameters 

1621 ---------- 

1622 exp : `lsst.afw.image.Exposure` 

1623 Exposure to run detection on. 

1624 tempExpRefList : `list` 

1625 List of data reference to tempExp. 

1626 

1627 Returns 

1628 ------- 

1629 result : `lsst.pipe.base.Struct` 

1630 Result struct with components: 

1631 

1632 - ``clipFootprints``: list of clipped footprints. 

1633 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1634 ``tempExpRefList``. 

1635 - ``clipSpans``: List of dictionaries containing spanSet lists 

1636 to clip. Each element contains the new maskplane name 

1637 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1638 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1639 compressed into footprints. 

1640 """ 

1641 mask = exp.getMaskedImage().getMask() 

1642 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1643 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1644 # Merge positive and negative together footprints together 

1645 fpSet.positive.merge(fpSet.negative) 

1646 footprints = fpSet.positive 

1647 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1648 ignoreMask = self.getBadPixelMask() 

1649 

1650 clipFootprints = [] 

1651 clipIndices = [] 

1652 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1653 

1654 # for use by detectClipBig 

1655 visitDetectionFootprints = [] 

1656 

1657 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1658 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1659 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1660 

1661 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1662 for i, warpRef in enumerate(tempExpRefList): 

1663 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1664 immediate=True).getMaskedImage().getMask() 

1665 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1666 afwImage.PARENT, True) 

1667 maskVisitDet &= maskDetValue 

1668 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1669 visitDetectionFootprints.append(visitFootprints) 

1670 

1671 for j, footprint in enumerate(footprints.getFootprints()): 

1672 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1673 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1674 

1675 # build a list of clipped spans for each visit 

1676 for j, footprint in enumerate(footprints.getFootprints()): 

1677 nPixel = footprint.getArea() 

1678 overlap = [] # hold the overlap with each visit 

1679 indexList = [] # index of visit in global list 

1680 for i in range(len(tempExpRefList)): 

1681 ignore = ignoreArr[i, j] 

1682 overlapDet = overlapDetArr[i, j] 

1683 totPixel = nPixel - ignore 

1684 

1685 # If we have more bad pixels than detection skip 

1686 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1686 ↛ 1688line 1686 didn't jump to line 1688, because the condition on line 1686 was never false

1687 continue 

1688 overlap.append(overlapDet/float(totPixel)) 

1689 indexList.append(i) 

1690 

1691 overlap = numpy.array(overlap) 

1692 if not len(overlap): 1692 ↛ 1695line 1692 didn't jump to line 1695, because the condition on line 1692 was never false

1693 continue 

1694 

1695 keep = False # Should this footprint be marked as clipped? 

1696 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1697 

1698 # If footprint only has one overlap use a lower threshold 

1699 if len(overlap) == 1: 

1700 if overlap[0] > self.config.minClipFootOverlapSingle: 

1701 keep = True 

1702 keepIndex = [0] 

1703 else: 

1704 # This is the general case where only visit should be clipped 

1705 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1706 if len(clipIndex) == 1: 

1707 keep = True 

1708 keepIndex = [clipIndex[0]] 

1709 

1710 # Test if there are clipped objects that overlap two different visits 

1711 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1712 if len(clipIndex) == 2 and len(overlap) > 3: 

1713 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1714 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1715 keep = True 

1716 keepIndex = clipIndex 

1717 

1718 if not keep: 

1719 continue 

1720 

1721 for index in keepIndex: 

1722 globalIndex = indexList[index] 

1723 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1724 

1725 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1726 clipFootprints.append(footprint) 

1727 

1728 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1729 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1730 

1731 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1732 maskClipValue, maskDetValue, coaddBBox): 

1733 """Return individual warp footprints for large artifacts and append 

1734 them to ``clipList`` in place. 

1735 

1736 Identify big footprints composed of many sources in the coadd 

1737 difference that may have originated in a large diffuse source in the 

1738 coadd. We do this by indentifying all clipped footprints that overlap 

1739 significantly with each source in all the coaddTempExps. 

1740 

1741 Parameters 

1742 ---------- 

1743 clipList : `list` 

1744 List of alt mask SpanSets with clipping information. Modified. 

1745 clipFootprints : `list` 

1746 List of clipped footprints. 

1747 clipIndices : `list` 

1748 List of which entries in tempExpClipList each footprint belongs to. 

1749 maskClipValue 

1750 Mask value of clipped pixels. 

1751 maskDetValue 

1752 Mask value of detected pixels. 

1753 coaddBBox : `lsst.geom.Box` 

1754 BBox of the coadd and warps. 

1755 

1756 Returns 

1757 ------- 

1758 bigFootprintsCoadd : `list` 

1759 List of big footprints 

1760 """ 

1761 bigFootprintsCoadd = [] 

1762 ignoreMask = self.getBadPixelMask() 

1763 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1764 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1765 for footprint in visitFootprints.getFootprints(): 1765 ↛ 1766line 1765 didn't jump to line 1766, because the loop on line 1765 never started

1766 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1767 

1768 # build a mask of clipped footprints that are in this visit 

1769 clippedFootprintsVisit = [] 

1770 for foot, clipIndex in zip(clipFootprints, clipIndices): 1770 ↛ 1771line 1770 didn't jump to line 1771, because the loop on line 1770 never started

1771 if index not in clipIndex: 

1772 continue 

1773 clippedFootprintsVisit.append(foot) 

1774 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1775 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1776 

1777 bigFootprintsVisit = [] 

1778 for foot in visitFootprints.getFootprints(): 1778 ↛ 1779line 1778 didn't jump to line 1779, because the loop on line 1778 never started

1779 if foot.getArea() < self.config.minBigOverlap: 

1780 continue 

1781 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1782 if nCount > self.config.minBigOverlap: 

1783 bigFootprintsVisit.append(foot) 

1784 bigFootprintsCoadd.append(foot) 

1785 

1786 for footprint in bigFootprintsVisit: 1786 ↛ 1787line 1786 didn't jump to line 1787, because the loop on line 1786 never started

1787 clippedSpans["CLIPPED"].append(footprint.spans) 

1788 

1789 return bigFootprintsCoadd 

1790 

1791 

1792class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1793 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1794 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1795 "Only PSF-Matched Warps make sense for image subtraction. " 

1796 "Therefore, they must be an additional declared input."), 

1797 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1798 storageClass="ExposureF", 

1799 dimensions=("tract", "patch", "skymap", "visit"), 

1800 deferLoad=True, 

1801 multiple=True 

1802 ) 

1803 templateCoadd = pipeBase.connectionTypes.Output( 

1804 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1805 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1806 name="{outputCoaddName}CoaddPsfMatched", 

1807 storageClass="ExposureF", 

1808 dimensions=("tract", "patch", "skymap", "band"), 

1809 ) 

1810 

1811 def __init__(self, *, config=None): 

1812 super().__init__(config=config) 

1813 if not config.assembleStaticSkyModel.doWrite: 

1814 self.outputs.remove("templateCoadd") 

1815 config.validate() 

1816 

1817 

1818class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1819 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1820 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1821 target=AssembleCoaddTask, 

1822 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1823 " naive/first-iteration model of the static sky.", 

1824 ) 

1825 detect = pexConfig.ConfigurableField( 

1826 target=SourceDetectionTask, 

1827 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1828 ) 

1829 detectTemplate = pexConfig.ConfigurableField( 

1830 target=SourceDetectionTask, 

1831 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1832 ) 

1833 maskStreaks = pexConfig.ConfigurableField( 

1834 target=MaskStreaksTask, 

1835 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1836 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1837 "streakMaskName" 

1838 ) 

1839 streakMaskName = pexConfig.Field( 

1840 dtype=str, 

1841 default="STREAK", 

1842 doc="Name of mask bit used for streaks" 

1843 ) 

1844 maxNumEpochs = pexConfig.Field( 

1845 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1846 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1847 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1848 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1849 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1850 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1851 "than transient and not masked.", 

1852 dtype=int, 

1853 default=2 

1854 ) 

1855 maxFractionEpochsLow = pexConfig.RangeField( 

1856 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1857 "Effective maxNumEpochs = " 

1858 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1859 dtype=float, 

1860 default=0.4, 

1861 min=0., max=1., 

1862 ) 

1863 maxFractionEpochsHigh = pexConfig.RangeField( 

1864 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1865 "Effective maxNumEpochs = " 

1866 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1867 dtype=float, 

1868 default=0.03, 

1869 min=0., max=1., 

1870 ) 

1871 spatialThreshold = pexConfig.RangeField( 

1872 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1873 "temporal criteria. If 0, clip all. If 1, clip none.", 

1874 dtype=float, 

1875 default=0.5, 

1876 min=0., max=1., 

1877 inclusiveMin=True, inclusiveMax=True 

1878 ) 

1879 doScaleWarpVariance = pexConfig.Field( 

1880 doc="Rescale Warp variance plane using empirical noise?", 

1881 dtype=bool, 

1882 default=True, 

1883 ) 

1884 scaleWarpVariance = pexConfig.ConfigurableField( 

1885 target=ScaleVarianceTask, 

1886 doc="Rescale variance on warps", 

1887 ) 

1888 doPreserveContainedBySource = pexConfig.Field( 

1889 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1890 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1891 dtype=bool, 

1892 default=True, 

1893 ) 

1894 doPrefilterArtifacts = pexConfig.Field( 

1895 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1896 "because they will be excluded anyway. This prevents them from contributing " 

1897 "to the outlier epoch count image and potentially being labeled as persistant." 

1898 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1899 dtype=bool, 

1900 default=True 

1901 ) 

1902 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1903 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1904 dtype=str, 

1905 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1906 ) 

1907 prefilterArtifactsRatio = pexConfig.Field( 

1908 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1909 dtype=float, 

1910 default=0.05 

1911 ) 

1912 doFilterMorphological = pexConfig.Field( 

1913 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1914 "be streaks.", 

1915 dtype=bool, 

1916 default=False 

1917 ) 

1918 

1919 def setDefaults(self): 

1920 AssembleCoaddConfig.setDefaults(self) 

1921 self.statistic = 'MEAN' 

1922 self.doUsePsfMatchedPolygons = True 

1923 

1924 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1925 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1926 if "EDGE" in self.badMaskPlanes: 1926 ↛ 1928line 1926 didn't jump to line 1928, because the condition on line 1926 was never false

1927 self.badMaskPlanes.remove('EDGE') 

1928 self.removeMaskPlanes.append('EDGE') 

1929 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1930 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1931 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1932 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1933 self.assembleStaticSkyModel.sigmaClip = 2.5 

1934 self.assembleStaticSkyModel.clipIter = 3 

1935 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1936 self.assembleStaticSkyModel.doWrite = False 

1937 self.detect.doTempLocalBackground = False 

1938 self.detect.reEstimateBackground = False 

1939 self.detect.returnOriginalFootprints = False 

1940 self.detect.thresholdPolarity = "both" 

1941 self.detect.thresholdValue = 5 

1942 self.detect.minPixels = 4 

1943 self.detect.isotropicGrow = True 

1944 self.detect.thresholdType = "pixel_stdev" 

1945 self.detect.nSigmaToGrow = 0.4 

1946 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1947 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1948 self.detectTemplate.nSigmaToGrow = 2.4 

1949 self.detectTemplate.doTempLocalBackground = False 

1950 self.detectTemplate.reEstimateBackground = False 

1951 self.detectTemplate.returnOriginalFootprints = False 

1952 

1953 def validate(self): 

1954 super().validate() 

1955 if self.assembleStaticSkyModel.doNImage: 

1956 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1957 "Please set assembleStaticSkyModel.doNImage=False") 

1958 

1959 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1960 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1961 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1962 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1963 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1964 

1965 

1966class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1967 """Assemble a compareWarp coadded image from a set of warps 

1968 by masking artifacts detected by comparing PSF-matched warps. 

1969 

1970 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1971 we clip outliers). The problem with doing this is that when computing the 

1972 coadd PSF at a given location, individual visit PSFs from visits with 

1973 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1974 In this task, we correct for this behavior by creating a new badMaskPlane 

1975 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1976 an artifact. We populate this plane on the input warps by comparing 

1977 PSF-matched warps with a PSF-matched median coadd which serves as a 

1978 model of the static sky. Any group of pixels that deviates from the 

1979 PSF-matched template coadd by more than config.detect.threshold sigma, 

1980 is an artifact candidate. The candidates are then filtered to remove 

1981 variable sources and sources that are difficult to subtract such as 

1982 bright stars. This filter is configured using the config parameters 

1983 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1984 the maximum fraction of epochs that the deviation can appear in and still 

1985 be considered an artifact. The spatialThreshold is the maximum fraction of 

1986 pixels in the footprint of the deviation that appear in other epochs 

1987 (where other epochs is defined by the temporalThreshold). If the deviant 

1988 region meets this criteria of having a significant percentage of pixels 

1989 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1990 set in the mask. These regions will not contribute to the final coadd. 

1991 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1992 of clipped regions. Note that the algorithm implemented by this task is 

1993 preliminary and works correctly for HSC data. Parameter modifications and 

1994 or considerable redesigning of the algorithm is likley required for other 

1995 surveys. 

1996 

1997 ``CompareWarpAssembleCoaddTask`` sub-classes 

1998 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

1999 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2000 

2001 Notes 

2002 ----- 

2003 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2004 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2005 ``baseDebug`` for more about ``debug.py`` files. 

2006 

2007 This task supports the following debug variables: 

2008 

2009 - ``saveCountIm`` 

2010 If True then save the Epoch Count Image as a fits file in the `figPath` 

2011 - ``figPath`` 

2012 Path to save the debug fits images and figures 

2013 

2014 For example, put something like: 

2015 

2016 .. code-block:: python 

2017 

2018 import lsstDebug 

2019 def DebugInfo(name): 

2020 di = lsstDebug.getInfo(name) 

2021 if name == "lsst.pipe.tasks.assembleCoadd": 

2022 di.saveCountIm = True 

2023 di.figPath = "/desired/path/to/debugging/output/images" 

2024 return di 

2025 lsstDebug.Info = DebugInfo 

2026 

2027 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2028 ``--debug`` flag. Some subtasks may have their own debug variables; 

2029 see individual Task documentation. 

2030 

2031 Examples 

2032 -------- 

2033 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2034 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2035 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2036 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2037 and filter to be coadded (specified using 

2038 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2039 along with a list of coaddTempExps to attempt to coadd (specified using 

2040 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2041 Only the warps that cover the specified tract and patch will be coadded. 

2042 A list of the available optional arguments can be obtained by calling 

2043 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2044 

2045 .. code-block:: none 

2046 

2047 assembleCoadd.py --help 

2048 

2049 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2050 context of multi-band processing, we will generate the HSC-I & -R band 

2051 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2052 To begin, assuming that the lsst stack has been already set up, we must 

2053 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2054 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2055 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2056 directory. To begin assembling the coadds, we must first 

2057 

2058 - processCcd 

2059 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2060 - makeSkyMap 

2061 create a skymap that covers the area of the sky present in the raw exposures 

2062 - makeCoaddTempExp 

2063 warp the individual calibrated exposures to the tangent plane of the coadd 

2064 

2065 We can perform all of these steps by running 

2066 

2067 .. code-block:: none 

2068 

2069 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2070 

2071 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2072 warped data, we call ``assembleCoadd.py`` as follows: 

2073 

2074 .. code-block:: none 

2075 

2076 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2077 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2078 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2079 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2080 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2081 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2082 --selectId visit=903988 ccd=24 

2083 

2084 This will process the HSC-I band data. The results are written in 

2085 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2086 """ 

2087 ConfigClass = CompareWarpAssembleCoaddConfig 

2088 _DefaultName = "compareWarpAssembleCoadd" 

2089 

2090 def __init__(self, *args, **kwargs): 

2091 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2092 self.makeSubtask("assembleStaticSkyModel") 

2093 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2094 self.makeSubtask("detect", schema=detectionSchema) 

2095 if self.config.doPreserveContainedBySource: 2095 ↛ 2097line 2095 didn't jump to line 2097, because the condition on line 2095 was never false

2096 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2097 if self.config.doScaleWarpVariance: 2097 ↛ 2099line 2097 didn't jump to line 2099, because the condition on line 2097 was never false

2098 self.makeSubtask("scaleWarpVariance") 

2099 if self.config.doFilterMorphological: 2099 ↛ 2100line 2099 didn't jump to line 2100, because the condition on line 2099 was never true

2100 self.makeSubtask("maskStreaks") 

2101 

2102 @utils.inheritDoc(AssembleCoaddTask) 

2103 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2104 """ 

2105 Generate a templateCoadd to use as a naive model of static sky to 

2106 subtract from PSF-Matched warps. 

2107 

2108 Returns 

2109 ------- 

2110 result : `lsst.pipe.base.Struct` 

2111 Result struct with components: 

2112 

2113 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2114 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2115 """ 

2116 # Ensure that psfMatchedWarps are used as input warps for template generation 

2117 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2118 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2119 

2120 # Because subtasks don't have connections we have to make one. 

2121 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2122 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2123 if self.config.assembleStaticSkyModel.doWrite: 

2124 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2125 # Remove template coadd from both subtask's and main tasks outputs, 

2126 # because it is handled by the subtask as `coaddExposure` 

2127 del outputRefs.templateCoadd 

2128 del staticSkyModelOutputRefs.templateCoadd 

2129 

2130 # A PSF-Matched nImage does not exist as a dataset type 

2131 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2132 del staticSkyModelOutputRefs.nImage 

2133 

2134 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2135 staticSkyModelOutputRefs) 

2136 if templateCoadd is None: 

2137 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2138 

2139 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2140 nImage=templateCoadd.nImage, 

2141 warpRefList=templateCoadd.warpRefList, 

2142 imageScalerList=templateCoadd.imageScalerList, 

2143 weightList=templateCoadd.weightList) 

2144 

2145 @utils.inheritDoc(AssembleCoaddTask) 

2146 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2147 """ 

2148 Generate a templateCoadd to use as a naive model of static sky to 

2149 subtract from PSF-Matched warps. 

2150 

2151 Returns 

2152 ------- 

2153 result : `lsst.pipe.base.Struct` 

2154 Result struct with components: 

2155 

2156 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2157 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2158 """ 

2159 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2160 if templateCoadd is None: 2160 ↛ 2161line 2160 didn't jump to line 2161, because the condition on line 2160 was never true

2161 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2162 

2163 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2164 nImage=templateCoadd.nImage, 

2165 warpRefList=templateCoadd.warpRefList, 

2166 imageScalerList=templateCoadd.imageScalerList, 

2167 weightList=templateCoadd.weightList) 

2168 

2169 def _noTemplateMessage(self, warpType): 

2170 warpName = (warpType[0].upper() + warpType[1:]) 

2171 message = """No %(warpName)s warps were found to build the template coadd which is 

2172 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2173 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2174 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2175 

2176 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2177 another algorithm like: 

2178 

2179 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2180 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2181 """ % {"warpName": warpName} 

2182 return message 

2183 

2184 @utils.inheritDoc(AssembleCoaddTask) 

2185 @pipeBase.timeMethod 

2186 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2187 supplementaryData, *args, **kwargs): 

2188 """Assemble the coadd. 

2189 

2190 Find artifacts and apply them to the warps' masks creating a list of 

2191 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2192 plane. Then pass these alternative masks to the base class's `run` 

2193 method. 

2194 

2195 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2196 that must contain a ``templateCoadd`` that serves as the 

2197 model of the static sky. 

2198 """ 

2199 

2200 # Check and match the order of the supplementaryData 

2201 # (PSF-matched) inputs to the order of the direct inputs, 

2202 # so that the artifact mask is applied to the right warp 

2203 dataIds = [ref.dataId for ref in tempExpRefList] 

2204 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2205 

2206 if dataIds != psfMatchedDataIds: 

2207 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2208 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2209 psfMatchedDataIds, dataIds) 

2210 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2211 psfMatchedDataIds, dataIds) 

2212 

2213 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2214 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2215 supplementaryData.warpRefList, 

2216 supplementaryData.imageScalerList) 

2217 

2218 badMaskPlanes = self.config.badMaskPlanes[:] 

2219 badMaskPlanes.append("CLIPPED") 

2220 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2221 

2222 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2223 spanSetMaskList, mask=badPixelMask) 

2224 

2225 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2226 # Psf-Matching moves the real edge inwards 

2227 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2228 return result 

2229 

2230 def applyAltEdgeMask(self, mask, altMaskList): 

2231 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2232 

2233 Parameters 

2234 ---------- 

2235 mask : `lsst.afw.image.Mask` 

2236 Original mask. 

2237 altMaskList : `list` 

2238 List of Dicts containing ``spanSet`` lists. 

2239 Each element contains the new mask plane name (e.g. "CLIPPED 

2240 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2241 the mask. 

2242 """ 

2243 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2244 for visitMask in altMaskList: 

2245 if "EDGE" in visitMask: 2245 ↛ 2244line 2245 didn't jump to line 2244, because the condition on line 2245 was never false

2246 for spanSet in visitMask['EDGE']: 

2247 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2248 

2249 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2250 """Find artifacts. 

2251 

2252 Loop through warps twice. The first loop builds a map with the count 

2253 of how many epochs each pixel deviates from the templateCoadd by more 

2254 than ``config.chiThreshold`` sigma. The second loop takes each 

2255 difference image and filters the artifacts detected in each using 

2256 count map to filter out variable sources and sources that are 

2257 difficult to subtract cleanly. 

2258 

2259 Parameters 

2260 ---------- 

2261 templateCoadd : `lsst.afw.image.Exposure` 

2262 Exposure to serve as model of static sky. 

2263 tempExpRefList : `list` 

2264 List of data references to warps. 

2265 imageScalerList : `list` 

2266 List of image scalers. 

2267 

2268 Returns 

2269 ------- 

2270 altMasks : `list` 

2271 List of dicts containing information about CLIPPED 

2272 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2273 """ 

2274 

2275 self.log.debug("Generating Count Image, and mask lists.") 

2276 coaddBBox = templateCoadd.getBBox() 

2277 slateIm = afwImage.ImageU(coaddBBox) 

2278 epochCountImage = afwImage.ImageU(coaddBBox) 

2279 nImage = afwImage.ImageU(coaddBBox) 

2280 spanSetArtifactList = [] 

2281 spanSetNoDataMaskList = [] 

2282 spanSetEdgeList = [] 

2283 spanSetBadMorphoList = [] 

2284 badPixelMask = self.getBadPixelMask() 

2285 

2286 # mask of the warp diffs should = that of only the warp 

2287 templateCoadd.mask.clearAllMaskPlanes() 

2288 

2289 if self.config.doPreserveContainedBySource: 2289 ↛ 2292line 2289 didn't jump to line 2292, because the condition on line 2289 was never false

2290 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2291 else: 

2292 templateFootprints = None 

2293 

2294 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2295 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2296 if warpDiffExp is not None: 

2297 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2298 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2299 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2300 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2301 fpSet.positive.merge(fpSet.negative) 

2302 footprints = fpSet.positive 

2303 slateIm.set(0) 

2304 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2305 

2306 # Remove artifacts due to defects before they contribute to the epochCountImage 

2307 if self.config.doPrefilterArtifacts: 2307 ↛ 2311line 2307 didn't jump to line 2311, because the condition on line 2307 was never false

2308 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2309 

2310 # Clear mask before adding prefiltered spanSets 

2311 self.detect.clearMask(warpDiffExp.mask) 

2312 for spans in spanSetList: 

2313 spans.setImage(slateIm, 1, doClip=True) 

2314 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2315 epochCountImage += slateIm 

2316 

2317 if self.config.doFilterMorphological: 2317 ↛ 2318line 2317 didn't jump to line 2318, because the condition on line 2317 was never true

2318 maskName = self.config.streakMaskName 

2319 _ = self.maskStreaks.run(warpDiffExp) 

2320 streakMask = warpDiffExp.mask 

2321 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2322 streakMask.getPlaneBitMask(maskName)).split() 

2323 

2324 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2325 # undergo a second convolution. Pixels with data in the direct warp 

2326 # but not in the PSF-matched warp will not have their artifacts detected. 

2327 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2328 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2329 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2330 nansMask.setXY0(warpDiffExp.getXY0()) 

2331 edgeMask = warpDiffExp.mask 

2332 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2333 edgeMask.getPlaneBitMask("EDGE")).split() 

2334 else: 

2335 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2336 # In this case, mask the whole epoch 

2337 nansMask = afwImage.MaskX(coaddBBox, 1) 

2338 spanSetList = [] 

2339 spanSetEdgeMask = [] 

2340 spanSetStreak = [] 

2341 

2342 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2343 

2344 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2345 spanSetArtifactList.append(spanSetList) 

2346 spanSetEdgeList.append(spanSetEdgeMask) 

2347 if self.config.doFilterMorphological: 2347 ↛ 2348line 2347 didn't jump to line 2348, because the condition on line 2347 was never true

2348 spanSetBadMorphoList.append(spanSetStreak) 

2349 

2350 if lsstDebug.Info(__name__).saveCountIm: 2350 ↛ 2351line 2350 didn't jump to line 2351, because the condition on line 2350 was never true

2351 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2352 epochCountImage.writeFits(path) 

2353 

2354 for i, spanSetList in enumerate(spanSetArtifactList): 

2355 if spanSetList: 

2356 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2357 templateFootprints) 

2358 spanSetArtifactList[i] = filteredSpanSetList 

2359 if self.config.doFilterMorphological: 2359 ↛ 2360line 2359 didn't jump to line 2360, because the condition on line 2359 was never true

2360 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2361 

2362 altMasks = [] 

2363 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2364 altMasks.append({'CLIPPED': artifacts, 

2365 'NO_DATA': noData, 

2366 'EDGE': edge}) 

2367 return altMasks 

2368 

2369 def prefilterArtifacts(self, spanSetList, exp): 

2370 """Remove artifact candidates covered by bad mask plane. 

2371 

2372 Any future editing of the candidate list that does not depend on 

2373 temporal information should go in this method. 

2374 

2375 Parameters 

2376 ---------- 

2377 spanSetList : `list` 

2378 List of SpanSets representing artifact candidates. 

2379 exp : `lsst.afw.image.Exposure` 

2380 Exposure containing mask planes used to prefilter. 

2381 

2382 Returns 

2383 ------- 

2384 returnSpanSetList : `list` 

2385 List of SpanSets with artifacts. 

2386 """ 

2387 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2388 goodArr = (exp.mask.array & badPixelMask) == 0 

2389 returnSpanSetList = [] 

2390 bbox = exp.getBBox() 

2391 x0, y0 = exp.getXY0() 

2392 for i, span in enumerate(spanSetList): 

2393 y, x = span.clippedTo(bbox).indices() 

2394 yIndexLocal = numpy.array(y) - y0 

2395 xIndexLocal = numpy.array(x) - x0 

2396 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2397 if goodRatio > self.config.prefilterArtifactsRatio: 2397 ↛ 2392line 2397 didn't jump to line 2392, because the condition on line 2397 was never false

2398 returnSpanSetList.append(span) 

2399 return returnSpanSetList 

2400 

2401 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2402 """Filter artifact candidates. 

2403 

2404 Parameters 

2405 ---------- 

2406 spanSetList : `list` 

2407 List of SpanSets representing artifact candidates. 

2408 epochCountImage : `lsst.afw.image.Image` 

2409 Image of accumulated number of warpDiff detections. 

2410 nImage : `lsst.afw.image.Image` 

2411 Image of the accumulated number of total epochs contributing. 

2412 

2413 Returns 

2414 ------- 

2415 maskSpanSetList : `list` 

2416 List of SpanSets with artifacts. 

2417 """ 

2418 

2419 maskSpanSetList = [] 

2420 x0, y0 = epochCountImage.getXY0() 

2421 for i, span in enumerate(spanSetList): 

2422 y, x = span.indices() 

2423 yIdxLocal = [y1 - y0 for y1 in y] 

2424 xIdxLocal = [x1 - x0 for x1 in x] 

2425 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2426 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2427 

2428 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2429 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2430 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2431 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2432 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2433 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2434 & (outlierN <= effectiveMaxNumEpochs)) 

2435 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2436 if percentBelowThreshold > self.config.spatialThreshold: 

2437 maskSpanSetList.append(span) 

2438 

2439 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2439 ↛ 2452line 2439 didn't jump to line 2452, because the condition on line 2439 was never false

2440 # If a candidate is contained by a footprint on the template coadd, do not clip 

2441 filteredMaskSpanSetList = [] 

2442 for span in maskSpanSetList: 

2443 doKeep = True 

2444 for footprint in footprintsToExclude.positive.getFootprints(): 

2445 if footprint.spans.contains(span): 

2446 doKeep = False 

2447 break 

2448 if doKeep: 

2449 filteredMaskSpanSetList.append(span) 

2450 maskSpanSetList = filteredMaskSpanSetList 

2451 

2452 return maskSpanSetList 

2453 

2454 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2455 """Fetch a warp from the butler and return a warpDiff. 

2456 

2457 Parameters 

2458 ---------- 

2459 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2460 Butler dataRef for the warp. 

2461 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2462 An image scaler object. 

2463 templateCoadd : `lsst.afw.image.Exposure` 

2464 Exposure to be substracted from the scaled warp. 

2465 

2466 Returns 

2467 ------- 

2468 warp : `lsst.afw.image.Exposure` 

2469 Exposure of the image difference between the warp and template. 

2470 """ 

2471 

2472 # If the PSF-Matched warp did not exist for this direct warp 

2473 # None is holding its place to maintain order in Gen 3 

2474 if warpRef is None: 

2475 return None 

2476 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2477 warpName = self.getTempExpDatasetName('psfMatched') 

2478 if not isinstance(warpRef, DeferredDatasetHandle): 2478 ↛ 2482line 2478 didn't jump to line 2482, because the condition on line 2478 was never false

2479 if not warpRef.datasetExists(warpName): 2479 ↛ 2480line 2479 didn't jump to line 2480, because the condition on line 2479 was never true

2480 self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2481 return None 

2482 warp = warpRef.get(datasetType=warpName, immediate=True) 

2483 # direct image scaler OK for PSF-matched Warp 

2484 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2485 mi = warp.getMaskedImage() 

2486 if self.config.doScaleWarpVariance: 2486 ↛ 2491line 2486 didn't jump to line 2491, because the condition on line 2486 was never false

2487 try: 

2488 self.scaleWarpVariance.run(mi) 

2489 except Exception as exc: 

2490 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

2491 mi -= templateCoadd.getMaskedImage() 

2492 return warp 

2493 

2494 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2495 """Return a path to which to write debugging output. 

2496 

2497 Creates a hyphen-delimited string of dataId values for simple filenames. 

2498 

2499 Parameters 

2500 ---------- 

2501 prefix : `str` 

2502 Prefix for filename. 

2503 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2504 Butler dataRef to make the path from. 

2505 coaddLevel : `bool`, optional. 

2506 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2507 'filter', but no 'visit'). 

2508 

2509 Returns 

2510 ------- 

2511 result : `str` 

2512 Path for debugging output. 

2513 """ 

2514 if coaddLevel: 

2515 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2516 else: 

2517 keys = warpRef.dataId.keys() 

2518 keyList = sorted(keys, reverse=True) 

2519 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2520 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2521 return os.path.join(directory, filename)