Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExceptions 

28import lsst.geom as geom 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.afw.detection as afwDet 

34import lsst.coadd.utils as coaddUtils 

35import lsst.pipe.base as pipeBase 

36import lsst.meas.algorithms as measAlg 

37import lsst.log as log 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from .healSparseMapping import HealSparseInputMapTask 

48from lsst.meas.algorithms import SourceDetectionTask 

49from lsst.daf.butler import DeferredDatasetHandle 

50 

51__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

52 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

53 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

54 

55 

56class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

57 dimensions=("tract", "patch", "band", "skymap"), 

58 defaultTemplates={"inputCoaddName": "deep", 

59 "outputCoaddName": "deep", 

60 "warpType": "direct", 

61 "warpTypeSuffix": ""}): 

62 

63 inputWarps = pipeBase.connectionTypes.Input( 

64 doc=("Input list of warps to be assemebled i.e. stacked." 

65 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

66 name="{inputCoaddName}Coadd_{warpType}Warp", 

67 storageClass="ExposureF", 

68 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

69 deferLoad=True, 

70 multiple=True 

71 ) 

72 skyMap = pipeBase.connectionTypes.Input( 

73 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

75 storageClass="SkyMap", 

76 dimensions=("skymap", ), 

77 ) 

78 selectedVisits = pipeBase.connectionTypes.Input( 

79 doc="Selected visits to be coadded.", 

80 name="{outputCoaddName}Visits", 

81 storageClass="StructuredDataDict", 

82 dimensions=("instrument", "tract", "patch", "skymap", "band") 

83 ) 

84 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

85 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

86 " BRIGHT_OBJECT."), 

87 name="brightObjectMask", 

88 storageClass="ObjectMaskCatalog", 

89 dimensions=("tract", "patch", "skymap", "band"), 

90 ) 

91 coaddExposure = pipeBase.connectionTypes.Output( 

92 doc="Output coadded exposure, produced by stacking input warps", 

93 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

94 storageClass="ExposureF", 

95 dimensions=("tract", "patch", "skymap", "band"), 

96 ) 

97 nImage = pipeBase.connectionTypes.Output( 

98 doc="Output image of number of input images per pixel", 

99 name="{outputCoaddName}Coadd_nImage", 

100 storageClass="ImageU", 

101 dimensions=("tract", "patch", "skymap", "band"), 

102 ) 

103 inputMap = pipeBase.connectionTypes.Output( 

104 doc="Output healsparse map of input images", 

105 name="{outputCoaddName}Coadd_inputMap", 

106 storageClass="HealSparseMap", 

107 dimensions=("tract", "patch", "skymap", "band"), 

108 ) 

109 

110 def __init__(self, *, config=None): 

111 super().__init__(config=config) 

112 

113 # Override the connection's name template with config to replicate Gen2 behavior 

114 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

115 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

116 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

117 templateValues['warpType'] = config.warpType 

118 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

119 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

120 for name in self.allConnections} 

121 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

122 # End code to remove after deprecation 

123 

124 if not config.doMaskBrightObjects: 

125 self.prerequisiteInputs.remove("brightObjectMask") 

126 

127 if not config.doSelectVisits: 

128 self.inputs.remove("selectedVisits") 

129 

130 if not config.doNImage: 

131 self.outputs.remove("nImage") 

132 

133 if not self.config.doInputMap: 

134 self.outputs.remove("inputMap") 

135 

136 

137class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

138 pipelineConnections=AssembleCoaddConnections): 

139 """Configuration parameters for the `AssembleCoaddTask`. 

140 

141 Notes 

142 ----- 

143 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

144 only set the bitplane config.brightObjectMaskName. To make this useful you 

145 *must* also configure the flags.pixel algorithm, for example by adding 

146 

147 .. code-block:: none 

148 

149 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

150 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

151 

152 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

153 """ 

154 warpType = pexConfig.Field( 

155 doc="Warp name: one of 'direct' or 'psfMatched'", 

156 dtype=str, 

157 default="direct", 

158 ) 

159 subregionSize = pexConfig.ListField( 

160 dtype=int, 

161 doc="Width, height of stack subregion size; " 

162 "make small enough that a full stack of images will fit into memory at once.", 

163 length=2, 

164 default=(2000, 2000), 

165 ) 

166 statistic = pexConfig.Field( 

167 dtype=str, 

168 doc="Main stacking statistic for aggregating over the epochs.", 

169 default="MEANCLIP", 

170 ) 

171 doSigmaClip = pexConfig.Field( 

172 dtype=bool, 

173 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

174 default=False, 

175 ) 

176 sigmaClip = pexConfig.Field( 

177 dtype=float, 

178 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

179 default=3.0, 

180 ) 

181 clipIter = pexConfig.Field( 

182 dtype=int, 

183 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

184 default=2, 

185 ) 

186 calcErrorFromInputVariance = pexConfig.Field( 

187 dtype=bool, 

188 doc="Calculate coadd variance from input variance by stacking statistic." 

189 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

190 default=True, 

191 ) 

192 scaleZeroPoint = pexConfig.ConfigurableField( 

193 target=ScaleZeroPointTask, 

194 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

195 ) 

196 doInterp = pexConfig.Field( 

197 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

198 dtype=bool, 

199 default=True, 

200 ) 

201 interpImage = pexConfig.ConfigurableField( 

202 target=InterpImageTask, 

203 doc="Task to interpolate (and extrapolate) over NaN pixels", 

204 ) 

205 doWrite = pexConfig.Field( 

206 doc="Persist coadd?", 

207 dtype=bool, 

208 default=True, 

209 ) 

210 doNImage = pexConfig.Field( 

211 doc="Create image of number of contributing exposures for each pixel", 

212 dtype=bool, 

213 default=False, 

214 ) 

215 doUsePsfMatchedPolygons = pexConfig.Field( 

216 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

217 dtype=bool, 

218 default=False, 

219 ) 

220 maskPropagationThresholds = pexConfig.DictField( 

221 keytype=str, 

222 itemtype=float, 

223 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

224 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

225 "would have contributed exceeds this value."), 

226 default={"SAT": 0.1}, 

227 ) 

228 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

229 doc="Mask planes to remove before coadding") 

230 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

231 doc="Set mask and flag bits for bright objects?") 

232 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

233 doc="Name of mask bit used for bright objects") 

234 coaddPsf = pexConfig.ConfigField( 

235 doc="Configuration for CoaddPsf", 

236 dtype=measAlg.CoaddPsfConfig, 

237 ) 

238 doAttachTransmissionCurve = pexConfig.Field( 

239 dtype=bool, default=False, optional=False, 

240 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

241 "(requires all input Exposures to have TransmissionCurves).") 

242 ) 

243 hasFakes = pexConfig.Field( 

244 dtype=bool, 

245 default=False, 

246 doc="Should be set to True if fake sources have been inserted into the input data." 

247 ) 

248 doSelectVisits = pexConfig.Field( 

249 doc="Coadd only visits selected by a SelectVisitsTask", 

250 dtype=bool, 

251 default=False, 

252 ) 

253 doInputMap = pexConfig.Field( 

254 doc="Create a bitwise map of coadd inputs", 

255 dtype=bool, 

256 default=False, 

257 ) 

258 inputMapper = pexConfig.ConfigurableField( 

259 doc="Input map creation subtask.", 

260 target=HealSparseInputMapTask, 

261 ) 

262 

263 def setDefaults(self): 

264 super().setDefaults() 

265 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

266 

267 def validate(self): 

268 super().validate() 

269 if self.doPsfMatch: 

270 # Backwards compatibility. 

271 # Configs do not have loggers 

272 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

273 self.warpType = 'psfMatched' 

274 if self.doSigmaClip and self.statistic != "MEANCLIP": 

275 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

276 self.statistic = "MEANCLIP" 

277 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

278 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

279 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

280 

281 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

282 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

283 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

284 if str(k) not in unstackableStats] 

285 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

286 % (self.statistic, stackableStats)) 

287 

288 

289class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

290 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

291 

292 We want to assemble a coadded image from a set of Warps (also called 

293 coadded temporary exposures or ``coaddTempExps``). 

294 Each input Warp covers a patch on the sky and corresponds to a single 

295 run/visit/exposure of the covered patch. We provide the task with a list 

296 of Warps (``selectDataList``) from which it selects Warps that cover the 

297 specified patch (pointed at by ``dataRef``). 

298 Each Warp that goes into a coadd will typically have an independent 

299 photometric zero-point. Therefore, we must scale each Warp to set it to 

300 a common photometric zeropoint. WarpType may be one of 'direct' or 

301 'psfMatched', and the boolean configs `config.makeDirect` and 

302 `config.makePsfMatched` set which of the warp types will be coadded. 

303 The coadd is computed as a mean with optional outlier rejection. 

304 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

305 Finally, Warps can have bad 'NaN' pixels which received no input from the 

306 source calExps. We interpolate over these bad (NaN) pixels. 

307 

308 `AssembleCoaddTask` uses several sub-tasks. These are 

309 

310 - `ScaleZeroPointTask` 

311 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

312 - `InterpImageTask` 

313 - interpolate across bad pixels (NaN) in the final coadd 

314 

315 You can retarget these subtasks if you wish. 

316 

317 Notes 

318 ----- 

319 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

320 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

321 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

322 no debug variables of its own. Some of the subtasks may support debug 

323 variables. See the documentation for the subtasks for further information. 

324 

325 Examples 

326 -------- 

327 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

328 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

329 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

330 inputs: a data reference to the tract patch and filter to be coadded, and 

331 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

332 ``--selectId``, respectively: 

333 

334 .. code-block:: none 

335 

336 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

337 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

338 

339 Only the Warps that cover the specified tract and patch will be coadded. 

340 A list of the available optional arguments can be obtained by calling 

341 ``assembleCoadd.py`` with the ``--help`` command line argument: 

342 

343 .. code-block:: none 

344 

345 assembleCoadd.py --help 

346 

347 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

348 multi-band processing, we will generate the HSC-I & -R band coadds from 

349 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

350 assuming that the lsst stack has been already set up, we must set up the 

351 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

352 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

353 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

354 coadds, we must first 

355 

356 - processCcd 

357 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

358 - makeSkyMap 

359 - create a skymap that covers the area of the sky present in the raw exposures 

360 - makeCoaddTempExp 

361 - warp the individual calibrated exposures to the tangent plane of the coadd 

362 

363 We can perform all of these steps by running 

364 

365 .. code-block:: none 

366 

367 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

368 

369 This will produce warped exposures for each visit. To coadd the warped 

370 data, we call assembleCoadd.py as follows: 

371 

372 .. code-block:: none 

373 

374 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

375 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

376 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

377 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

378 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

379 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

380 --selectId visit=903988 ccd=24 

381 

382 that will process the HSC-I band data. The results are written in 

383 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

384 

385 You may also choose to run: 

386 

387 .. code-block:: none 

388 

389 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

390 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

391 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

392 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

393 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

394 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

395 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

396 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

397 

398 to generate the coadd for the HSC-R band if you are interested in 

399 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

400 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

401 rather than `AssembleCoaddTask` to make the coadd. 

402 """ 

403 ConfigClass = AssembleCoaddConfig 

404 _DefaultName = "assembleCoadd" 

405 

406 def __init__(self, *args, **kwargs): 

407 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

408 if args: 408 ↛ 409line 408 didn't jump to line 409, because the condition on line 408 was never true

409 argNames = ["config", "name", "parentTask", "log"] 

410 kwargs.update({k: v for k, v in zip(argNames, args)}) 

411 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

412 "PipelineTask will not take positional args" % argNames, FutureWarning) 

413 

414 super().__init__(**kwargs) 

415 self.makeSubtask("interpImage") 

416 self.makeSubtask("scaleZeroPoint") 

417 

418 if self.config.doMaskBrightObjects: 418 ↛ 419line 418 didn't jump to line 419, because the condition on line 418 was never true

419 mask = afwImage.Mask() 

420 try: 

421 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

422 except pexExceptions.LsstCppException: 

423 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

424 mask.getMaskPlaneDict().keys()) 

425 del mask 

426 

427 if self.config.doInputMap: 427 ↛ 428line 427 didn't jump to line 428, because the condition on line 427 was never true

428 self.makeSubtask("inputMapper") 

429 

430 self.warpType = self.config.warpType 

431 

432 @utils.inheritDoc(pipeBase.PipelineTask) 

433 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

434 # Docstring to be formatted with info from PipelineTask.runQuantum 

435 """ 

436 Notes 

437 ----- 

438 Assemble a coadd from a set of Warps. 

439 

440 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

441 Analogous to `runDataRef`, it prepares all the data products to be 

442 passed to `run`, and processes the results before returning a struct 

443 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

444 Therefore, its inputs are accessed subregion by subregion 

445 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

446 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

447 correspond to an update in `runDataRef` while both entry points 

448 are used. 

449 """ 

450 inputData = butlerQC.get(inputRefs) 

451 

452 # Construct skyInfo expected by run 

453 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

454 skyMap = inputData["skyMap"] 

455 outputDataId = butlerQC.quantum.dataId 

456 

457 inputData['skyInfo'] = makeSkyInfo(skyMap, 

458 tractId=outputDataId['tract'], 

459 patchId=outputDataId['patch']) 

460 

461 if self.config.doSelectVisits: 

462 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

463 else: 

464 warpRefList = inputData['inputWarps'] 

465 

466 # Perform same middle steps as `runDataRef` does 

467 inputs = self.prepareInputs(warpRefList) 

468 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

469 self.getTempExpDatasetName(self.warpType)) 

470 if len(inputs.tempExpRefList) == 0: 

471 self.log.warn("No coadd temporary exposures found") 

472 return 

473 

474 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

475 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

476 inputs.weightList, supplementaryData=supplementaryData) 

477 

478 inputData.setdefault('brightObjectMask', None) 

479 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

480 

481 if self.config.doWrite: 

482 butlerQC.put(retStruct, outputRefs) 

483 return retStruct 

484 

485 @pipeBase.timeMethod 

486 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

487 """Assemble a coadd from a set of Warps. 

488 

489 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

490 Compute weights to be applied to each Warp and 

491 find scalings to match the photometric zeropoint to a reference Warp. 

492 Assemble the Warps using `run`. Interpolate over NaNs and 

493 optionally write the coadd to disk. Return the coadded exposure. 

494 

495 Parameters 

496 ---------- 

497 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

498 Data reference defining the patch for coaddition and the 

499 reference Warp (if ``config.autoReference=False``). 

500 Used to access the following data products: 

501 - ``self.config.coaddName + "Coadd_skyMap"`` 

502 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

503 - ``self.config.coaddName + "Coadd"`` 

504 selectDataList : `list` 

505 List of data references to Calexps. Data to be coadded will be 

506 selected from this list based on overlap with the patch defined 

507 by dataRef, grouped by visit, and converted to a list of data 

508 references to warps. 

509 warpRefList : `list` 

510 List of data references to Warps to be coadded. 

511 Note: `warpRefList` is just the new name for `tempExpRefList`. 

512 

513 Returns 

514 ------- 

515 retStruct : `lsst.pipe.base.Struct` 

516 Result struct with components: 

517 

518 - ``coaddExposure``: coadded exposure (``Exposure``). 

519 - ``nImage``: exposure count image (``Image``). 

520 """ 

521 if selectDataList and warpRefList: 521 ↛ 522line 521 didn't jump to line 522, because the condition on line 521 was never true

522 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

523 "and which to use is ambiguous. Please pass only one.") 

524 

525 skyInfo = self.getSkyInfo(dataRef) 

526 if warpRefList is None: 

527 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

528 if len(calExpRefList) == 0: 528 ↛ 529line 528 didn't jump to line 529, because the condition on line 528 was never true

529 self.log.warn("No exposures to coadd") 

530 return 

531 self.log.info("Coadding %d exposures", len(calExpRefList)) 

532 

533 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

534 

535 inputData = self.prepareInputs(warpRefList) 

536 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

537 self.getTempExpDatasetName(self.warpType)) 

538 if len(inputData.tempExpRefList) == 0: 538 ↛ 539line 538 didn't jump to line 539, because the condition on line 538 was never true

539 self.log.warn("No coadd temporary exposures found") 

540 return 

541 

542 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

543 

544 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

545 inputData.weightList, supplementaryData=supplementaryData) 

546 

547 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

548 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

549 

550 if self.config.doWrite: 

551 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 551 ↛ 552line 551 didn't jump to line 552, because the condition on line 551 was never true

552 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

553 else: 

554 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

555 self.log.info("Persisting %s" % coaddDatasetName) 

556 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

557 if self.config.doNImage and retStruct.nImage is not None: 557 ↛ 558line 557 didn't jump to line 558, because the condition on line 557 was never true

558 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

559 

560 return retStruct 

561 

562 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

563 """Interpolate over missing data and mask bright stars. 

564 

565 Parameters 

566 ---------- 

567 coaddExposure : `lsst.afw.image.Exposure` 

568 The coadded exposure to process. 

569 dataRef : `lsst.daf.persistence.ButlerDataRef` 

570 Butler data reference for supplementary data. 

571 """ 

572 if self.config.doInterp: 572 ↛ 579line 572 didn't jump to line 579, because the condition on line 572 was never false

573 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

574 # The variance must be positive; work around for DM-3201. 

575 varArray = coaddExposure.variance.array 

576 with numpy.errstate(invalid="ignore"): 

577 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

578 

579 if self.config.doMaskBrightObjects: 579 ↛ 580line 579 didn't jump to line 580, because the condition on line 579 was never true

580 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

581 

582 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

583 """Make additional inputs to run() specific to subclasses (Gen2) 

584 

585 Duplicates interface of `runDataRef` method 

586 Available to be implemented by subclasses only if they need the 

587 coadd dataRef for performing preliminary processing before 

588 assembling the coadd. 

589 

590 Parameters 

591 ---------- 

592 dataRef : `lsst.daf.persistence.ButlerDataRef` 

593 Butler data reference for supplementary data. 

594 selectDataList : `list` (optional) 

595 Optional List of data references to Calexps. 

596 warpRefList : `list` (optional) 

597 Optional List of data references to Warps. 

598 """ 

599 return pipeBase.Struct() 

600 

601 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

602 """Make additional inputs to run() specific to subclasses (Gen3) 

603 

604 Duplicates interface of `runQuantum` method. 

605 Available to be implemented by subclasses only if they need the 

606 coadd dataRef for performing preliminary processing before 

607 assembling the coadd. 

608 

609 Parameters 

610 ---------- 

611 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

612 Gen3 Butler object for fetching additional data products before 

613 running the Task specialized for quantum being processed 

614 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

615 Attributes are the names of the connections describing input dataset types. 

616 Values are DatasetRefs that task consumes for corresponding dataset type. 

617 DataIds are guaranteed to match data objects in ``inputData``. 

618 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

619 Attributes are the names of the connections describing output dataset types. 

620 Values are DatasetRefs that task is to produce 

621 for corresponding dataset type. 

622 """ 

623 return pipeBase.Struct() 

624 

625 def getTempExpRefList(self, patchRef, calExpRefList): 

626 """Generate list data references corresponding to warped exposures 

627 that lie within the patch to be coadded. 

628 

629 Parameters 

630 ---------- 

631 patchRef : `dataRef` 

632 Data reference for patch. 

633 calExpRefList : `list` 

634 List of data references for input calexps. 

635 

636 Returns 

637 ------- 

638 tempExpRefList : `list` 

639 List of Warp/CoaddTempExp data references. 

640 """ 

641 butler = patchRef.getButler() 

642 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

643 self.getTempExpDatasetName(self.warpType)) 

644 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

645 g, groupData.keys) for 

646 g in groupData.groups.keys()] 

647 return tempExpRefList 

648 

649 def prepareInputs(self, refList): 

650 """Prepare the input warps for coaddition by measuring the weight for 

651 each warp and the scaling for the photometric zero point. 

652 

653 Each Warp has its own photometric zeropoint and background variance. 

654 Before coadding these Warps together, compute a scale factor to 

655 normalize the photometric zeropoint and compute the weight for each Warp. 

656 

657 Parameters 

658 ---------- 

659 refList : `list` 

660 List of data references to tempExp 

661 

662 Returns 

663 ------- 

664 result : `lsst.pipe.base.Struct` 

665 Result struct with components: 

666 

667 - ``tempExprefList``: `list` of data references to tempExp. 

668 - ``weightList``: `list` of weightings. 

669 - ``imageScalerList``: `list` of image scalers. 

670 """ 

671 statsCtrl = afwMath.StatisticsControl() 

672 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

673 statsCtrl.setNumIter(self.config.clipIter) 

674 statsCtrl.setAndMask(self.getBadPixelMask()) 

675 statsCtrl.setNanSafe(True) 

676 # compute tempExpRefList: a list of tempExpRef that actually exist 

677 # and weightList: a list of the weight of the associated coadd tempExp 

678 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

679 tempExpRefList = [] 

680 weightList = [] 

681 imageScalerList = [] 

682 tempExpName = self.getTempExpDatasetName(self.warpType) 

683 for tempExpRef in refList: 

684 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

685 # therefore have no datasetExists() method 

686 if not isinstance(tempExpRef, DeferredDatasetHandle): 686 ↛ 691line 686 didn't jump to line 691, because the condition on line 686 was never false

687 if not tempExpRef.datasetExists(tempExpName): 

688 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

689 continue 

690 

691 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

692 # Ignore any input warp that is empty of data 

693 if numpy.isnan(tempExp.image.array).all(): 693 ↛ 694line 693 didn't jump to line 694, because the condition on line 693 was never true

694 continue 

695 maskedImage = tempExp.getMaskedImage() 

696 imageScaler = self.scaleZeroPoint.computeImageScaler( 

697 exposure=tempExp, 

698 dataRef=tempExpRef, 

699 ) 

700 try: 

701 imageScaler.scaleMaskedImage(maskedImage) 

702 except Exception as e: 

703 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

704 continue 

705 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

706 afwMath.MEANCLIP, statsCtrl) 

707 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

708 weight = 1.0 / float(meanVar) 

709 if not numpy.isfinite(weight): 709 ↛ 710line 709 didn't jump to line 710, because the condition on line 709 was never true

710 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId) 

711 continue 

712 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

713 

714 del maskedImage 

715 del tempExp 

716 

717 tempExpRefList.append(tempExpRef) 

718 weightList.append(weight) 

719 imageScalerList.append(imageScaler) 

720 

721 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

722 imageScalerList=imageScalerList) 

723 

724 def prepareStats(self, mask=None): 

725 """Prepare the statistics for coadding images. 

726 

727 Parameters 

728 ---------- 

729 mask : `int`, optional 

730 Bit mask value to exclude from coaddition. 

731 

732 Returns 

733 ------- 

734 stats : `lsst.pipe.base.Struct` 

735 Statistics structure with the following fields: 

736 

737 - ``statsCtrl``: Statistics control object for coadd 

738 (`lsst.afw.math.StatisticsControl`) 

739 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

740 """ 

741 if mask is None: 

742 mask = self.getBadPixelMask() 

743 statsCtrl = afwMath.StatisticsControl() 

744 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

745 statsCtrl.setNumIter(self.config.clipIter) 

746 statsCtrl.setAndMask(mask) 

747 statsCtrl.setNanSafe(True) 

748 statsCtrl.setWeighted(True) 

749 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

750 for plane, threshold in self.config.maskPropagationThresholds.items(): 

751 bit = afwImage.Mask.getMaskPlane(plane) 

752 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

753 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

754 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

755 

756 @pipeBase.timeMethod 

757 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

758 altMaskList=None, mask=None, supplementaryData=None): 

759 """Assemble a coadd from input warps 

760 

761 Assemble the coadd using the provided list of coaddTempExps. Since 

762 the full coadd covers a patch (a large area), the assembly is 

763 performed over small areas on the image at a time in order to 

764 conserve memory usage. Iterate over subregions within the outer 

765 bbox of the patch using `assembleSubregion` to stack the corresponding 

766 subregions from the coaddTempExps with the statistic specified. 

767 Set the edge bits the coadd mask based on the weight map. 

768 

769 Parameters 

770 ---------- 

771 skyInfo : `lsst.pipe.base.Struct` 

772 Struct with geometric information about the patch. 

773 tempExpRefList : `list` 

774 List of data references to Warps (previously called CoaddTempExps). 

775 imageScalerList : `list` 

776 List of image scalers. 

777 weightList : `list` 

778 List of weights 

779 altMaskList : `list`, optional 

780 List of alternate masks to use rather than those stored with 

781 tempExp. 

782 mask : `int`, optional 

783 Bit mask value to exclude from coaddition. 

784 supplementaryData : lsst.pipe.base.Struct, optional 

785 Struct with additional data products needed to assemble coadd. 

786 Only used by subclasses that implement `makeSupplementaryData` 

787 and override `run`. 

788 

789 Returns 

790 ------- 

791 result : `lsst.pipe.base.Struct` 

792 Result struct with components: 

793 

794 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

795 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

796 - ``inputMap``: bit-wise map of inputs, if requested. 

797 - ``warpRefList``: input list of refs to the warps ( 

798 ``lsst.daf.butler.DeferredDatasetHandle`` or 

799 ``lsst.daf.persistence.ButlerDataRef``) 

800 (unmodified) 

801 - ``imageScalerList``: input list of image scalers (unmodified) 

802 - ``weightList``: input list of weights (unmodified) 

803 """ 

804 tempExpName = self.getTempExpDatasetName(self.warpType) 

805 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

806 stats = self.prepareStats(mask=mask) 

807 

808 if altMaskList is None: 

809 altMaskList = [None]*len(tempExpRefList) 

810 

811 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

812 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

813 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

814 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

815 coaddMaskedImage = coaddExposure.getMaskedImage() 

816 subregionSizeArr = self.config.subregionSize 

817 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

818 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

819 if self.config.doNImage: 819 ↛ 820line 819 didn't jump to line 820, because the condition on line 819 was never true

820 nImage = afwImage.ImageU(skyInfo.bbox) 

821 else: 

822 nImage = None 

823 # If inputMap is requested, create the initial version that can be masked in 

824 # assembleSubregion. 

825 if self.config.doInputMap: 825 ↛ 826line 825 didn't jump to line 826, because the condition on line 825 was never true

826 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

827 skyInfo.wcs, 

828 coaddExposure.getInfo().getCoaddInputs().ccds) 

829 

830 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

831 try: 

832 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

833 weightList, altMaskList, stats.flags, stats.ctrl, 

834 nImage=nImage) 

835 except Exception as e: 

836 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

837 

838 # If inputMap is requested, we must finalize the map after the accumulation. 

839 if self.config.doInputMap: 839 ↛ 840line 839 didn't jump to line 840, because the condition on line 839 was never true

840 self.inputMapper.finalize_ccd_input_map_mask() 

841 inputMap = self.inputMapper.ccd_input_map 

842 else: 

843 inputMap = None 

844 

845 self.setInexactPsf(coaddMaskedImage.getMask()) 

846 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

847 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

848 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

849 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

850 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

851 weightList=weightList, inputMap=inputMap) 

852 

853 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

854 """Set the metadata for the coadd. 

855 

856 This basic implementation sets the filter from the first input. 

857 

858 Parameters 

859 ---------- 

860 coaddExposure : `lsst.afw.image.Exposure` 

861 The target exposure for the coadd. 

862 tempExpRefList : `list` 

863 List of data references to tempExp. 

864 weightList : `list` 

865 List of weights. 

866 """ 

867 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

868 tempExpName = self.getTempExpDatasetName(self.warpType) 

869 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

870 # (and we need more than just the PropertySet that contains the header), which is not possible 

871 # with the current butler (see #2777). 

872 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

873 

874 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 874 ↛ 876line 874 didn't jump to line 876, because the condition on line 874 was never true

875 # Gen 3 API 

876 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

877 else: 

878 # Gen 2 API. Delete this when Gen 2 retired 

879 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

880 for tempExpRef in tempExpRefList] 

881 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

882 

883 # Set the coadd FilterLabel to the band of the first input exposure: 

884 # Coadds are calibrated, so the physical label is now meaningless. 

885 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

886 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

887 coaddInputs.ccds.reserve(numCcds) 

888 coaddInputs.visits.reserve(len(tempExpList)) 

889 

890 for tempExp, weight in zip(tempExpList, weightList): 

891 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

892 

893 if self.config.doUsePsfMatchedPolygons: 

894 self.shrinkValidPolygons(coaddInputs) 

895 

896 coaddInputs.visits.sort() 

897 if self.warpType == "psfMatched": 

898 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

899 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

900 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

901 # having the maximum width (sufficient because square) 

902 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

903 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

904 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

905 else: 

906 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

907 self.config.coaddPsf.makeControl()) 

908 coaddExposure.setPsf(psf) 

909 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

910 coaddExposure.getWcs()) 

911 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

912 if self.config.doAttachTransmissionCurve: 

913 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

914 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

915 

916 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

917 altMaskList, statsFlags, statsCtrl, nImage=None): 

918 """Assemble the coadd for a sub-region. 

919 

920 For each coaddTempExp, check for (and swap in) an alternative mask 

921 if one is passed. Remove mask planes listed in 

922 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

923 `lsst.afw.math.statisticsStack` with the statistic specified by 

924 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

925 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

926 an N-sigma clipped mean where N and iterations are specified by 

927 statsCtrl. Assign the stacked subregion back to the coadd. 

928 

929 Parameters 

930 ---------- 

931 coaddExposure : `lsst.afw.image.Exposure` 

932 The target exposure for the coadd. 

933 bbox : `lsst.geom.Box` 

934 Sub-region to coadd. 

935 tempExpRefList : `list` 

936 List of data reference to tempExp. 

937 imageScalerList : `list` 

938 List of image scalers. 

939 weightList : `list` 

940 List of weights. 

941 altMaskList : `list` 

942 List of alternate masks to use rather than those stored with 

943 tempExp, or None. Each element is dict with keys = mask plane 

944 name to which to add the spans. 

945 statsFlags : `lsst.afw.math.Property` 

946 Property object for statistic for coadd. 

947 statsCtrl : `lsst.afw.math.StatisticsControl` 

948 Statistics control object for coadd. 

949 nImage : `lsst.afw.image.ImageU`, optional 

950 Keeps track of exposure count for each pixel. 

951 """ 

952 self.log.debug("Computing coadd over %s", bbox) 

953 tempExpName = self.getTempExpDatasetName(self.warpType) 

954 coaddExposure.mask.addMaskPlane("REJECTED") 

955 coaddExposure.mask.addMaskPlane("CLIPPED") 

956 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

957 maskMap = self.setRejectedMaskMapping(statsCtrl) 

958 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

959 maskedImageList = [] 

960 if nImage is not None: 960 ↛ 961line 960 didn't jump to line 961, because the condition on line 960 was never true

961 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

962 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

963 

964 if isinstance(tempExpRef, DeferredDatasetHandle): 964 ↛ 966line 964 didn't jump to line 966, because the condition on line 964 was never true

965 # Gen 3 API 

966 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

967 else: 

968 # Gen 2 API. Delete this when Gen 2 retired 

969 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

970 

971 maskedImage = exposure.getMaskedImage() 

972 mask = maskedImage.getMask() 

973 if altMask is not None: 

974 self.applyAltMaskPlanes(mask, altMask) 

975 imageScaler.scaleMaskedImage(maskedImage) 

976 

977 # Add 1 for each pixel which is not excluded by the exclude mask. 

978 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

979 if nImage is not None: 979 ↛ 980line 979 didn't jump to line 980, because the condition on line 979 was never true

980 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

981 if self.config.removeMaskPlanes: 981 ↛ 983line 981 didn't jump to line 983, because the condition on line 981 was never false

982 self.removeMaskPlanes(maskedImage) 

983 maskedImageList.append(maskedImage) 

984 

985 if self.config.doInputMap: 985 ↛ 986line 985 didn't jump to line 986, because the condition on line 985 was never true

986 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

987 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

988 

989 with self.timer("stack"): 

990 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

991 clipped, # also set output to CLIPPED if sigma-clipped 

992 maskMap) 

993 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

994 if nImage is not None: 994 ↛ 995line 994 didn't jump to line 995, because the condition on line 994 was never true

995 nImage.assign(subNImage, bbox) 

996 

997 def removeMaskPlanes(self, maskedImage): 

998 """Unset the mask of an image for mask planes specified in the config. 

999 

1000 Parameters 

1001 ---------- 

1002 maskedImage : `lsst.afw.image.MaskedImage` 

1003 The masked image to be modified. 

1004 """ 

1005 mask = maskedImage.getMask() 

1006 for maskPlane in self.config.removeMaskPlanes: 

1007 try: 

1008 mask &= ~mask.getPlaneBitMask(maskPlane) 

1009 except pexExceptions.InvalidParameterError: 

1010 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1011 maskPlane) 

1012 

1013 @staticmethod 

1014 def setRejectedMaskMapping(statsCtrl): 

1015 """Map certain mask planes of the warps to new planes for the coadd. 

1016 

1017 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1018 or CLIPPED, set it to REJECTED on the coadd. 

1019 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1020 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1021 

1022 Parameters 

1023 ---------- 

1024 statsCtrl : `lsst.afw.math.StatisticsControl` 

1025 Statistics control object for coadd 

1026 

1027 Returns 

1028 ------- 

1029 maskMap : `list` of `tuple` of `int` 

1030 A list of mappings of mask planes of the warped exposures to 

1031 mask planes of the coadd. 

1032 """ 

1033 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1034 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1035 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1036 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1037 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1038 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1039 (clipped, clipped)] 

1040 return maskMap 

1041 

1042 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1043 """Apply in place alt mask formatted as SpanSets to a mask. 

1044 

1045 Parameters 

1046 ---------- 

1047 mask : `lsst.afw.image.Mask` 

1048 Original mask. 

1049 altMaskSpans : `dict` 

1050 SpanSet lists to apply. Each element contains the new mask 

1051 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1052 and list of SpanSets to apply to the mask. 

1053 

1054 Returns 

1055 ------- 

1056 mask : `lsst.afw.image.Mask` 

1057 Updated mask. 

1058 """ 

1059 if self.config.doUsePsfMatchedPolygons: 

1060 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1060 ↛ 1068line 1060 didn't jump to line 1068, because the condition on line 1060 was never false

1061 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1062 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1063 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1064 # This mask-clearing step must only occur *before* applying the new masks below 

1065 for spanSet in altMaskSpans['NO_DATA']: 

1066 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1067 

1068 for plane, spanSetList in altMaskSpans.items(): 

1069 maskClipValue = mask.addMaskPlane(plane) 

1070 for spanSet in spanSetList: 

1071 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1072 return mask 

1073 

1074 def shrinkValidPolygons(self, coaddInputs): 

1075 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1076 

1077 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1078 does not have a validPolygon, create one from its bbox. 

1079 

1080 Parameters 

1081 ---------- 

1082 coaddInputs : `lsst.afw.image.coaddInputs` 

1083 Original mask. 

1084 

1085 """ 

1086 for ccd in coaddInputs.ccds: 

1087 polyOrig = ccd.getValidPolygon() 

1088 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1089 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1090 if polyOrig: 1090 ↛ 1091line 1090 didn't jump to line 1091, because the condition on line 1090 was never true

1091 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1092 else: 

1093 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1094 ccd.setValidPolygon(validPolygon) 

1095 

1096 def readBrightObjectMasks(self, dataRef): 

1097 """Retrieve the bright object masks. 

1098 

1099 Returns None on failure. 

1100 

1101 Parameters 

1102 ---------- 

1103 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1104 A Butler dataRef. 

1105 

1106 Returns 

1107 ------- 

1108 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1109 Bright object mask from the Butler object, or None if it cannot 

1110 be retrieved. 

1111 """ 

1112 try: 

1113 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1114 except Exception as e: 

1115 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1116 return None 

1117 

1118 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1119 """Set the bright object masks. 

1120 

1121 Parameters 

1122 ---------- 

1123 exposure : `lsst.afw.image.Exposure` 

1124 Exposure under consideration. 

1125 dataId : `lsst.daf.persistence.dataId` 

1126 Data identifier dict for patch. 

1127 brightObjectMasks : `lsst.afw.table` 

1128 Table of bright objects to mask. 

1129 """ 

1130 

1131 if brightObjectMasks is None: 

1132 self.log.warn("Unable to apply bright object mask: none supplied") 

1133 return 

1134 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1135 mask = exposure.getMaskedImage().getMask() 

1136 wcs = exposure.getWcs() 

1137 plateScale = wcs.getPixelScale().asArcseconds() 

1138 

1139 for rec in brightObjectMasks: 

1140 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1141 if rec["type"] == "box": 

1142 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1143 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1144 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1145 

1146 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1147 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1148 

1149 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1150 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1151 spans = afwGeom.SpanSet(bbox) 

1152 elif rec["type"] == "circle": 

1153 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1154 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1155 else: 

1156 self.log.warn("Unexpected region type %s at %s" % rec["type"], center) 

1157 continue 

1158 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1159 

1160 def setInexactPsf(self, mask): 

1161 """Set INEXACT_PSF mask plane. 

1162 

1163 If any of the input images isn't represented in the coadd (due to 

1164 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1165 these pixels. 

1166 

1167 Parameters 

1168 ---------- 

1169 mask : `lsst.afw.image.Mask` 

1170 Coadded exposure's mask, modified in-place. 

1171 """ 

1172 mask.addMaskPlane("INEXACT_PSF") 

1173 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1174 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1175 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1176 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1177 array = mask.getArray() 

1178 selected = array & (sensorEdge | clipped | rejected) > 0 

1179 array[selected] |= inexactPsf 

1180 

1181 @classmethod 

1182 def _makeArgumentParser(cls): 

1183 """Create an argument parser. 

1184 """ 

1185 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1186 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1187 + cls.ConfigClass().warpType + "Warp", 

1188 help="data ID, e.g. --id tract=12345 patch=1,2", 

1189 ContainerClass=AssembleCoaddDataIdContainer) 

1190 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1191 ContainerClass=SelectDataIdContainer) 

1192 return parser 

1193 

1194 @staticmethod 

1195 def _subBBoxIter(bbox, subregionSize): 

1196 """Iterate over subregions of a bbox. 

1197 

1198 Parameters 

1199 ---------- 

1200 bbox : `lsst.geom.Box2I` 

1201 Bounding box over which to iterate. 

1202 subregionSize: `lsst.geom.Extent2I` 

1203 Size of sub-bboxes. 

1204 

1205 Yields 

1206 ------ 

1207 subBBox : `lsst.geom.Box2I` 

1208 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1209 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1210 the edges of ``bbox``, but it will never be empty. 

1211 """ 

1212 if bbox.isEmpty(): 1212 ↛ 1213line 1212 didn't jump to line 1213, because the condition on line 1212 was never true

1213 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1214 if subregionSize[0] < 1 or subregionSize[1] < 1: 1214 ↛ 1215line 1214 didn't jump to line 1215, because the condition on line 1214 was never true

1215 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1216 

1217 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1218 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1219 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1220 subBBox.clip(bbox) 

1221 if subBBox.isEmpty(): 1221 ↛ 1222line 1221 didn't jump to line 1222, because the condition on line 1221 was never true

1222 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1223 "colShift=%s, rowShift=%s" % 

1224 (bbox, subregionSize, colShift, rowShift)) 

1225 yield subBBox 

1226 

1227 def filterWarps(self, inputs, goodVisits): 

1228 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1229 

1230 Parameters 

1231 ---------- 

1232 inputs : list 

1233 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1234 goodVisit : `dict` 

1235 Dictionary with good visitIds as the keys. Value ignored. 

1236 

1237 Returns: 

1238 -------- 

1239 filteredInputs : `list` 

1240 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1241 """ 

1242 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1243 filteredInputs = [] 

1244 for visit in goodVisits.keys(): 

1245 filteredInputs.append(inputWarpDict[visit]) 

1246 return filteredInputs 

1247 

1248 

1249class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1250 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1251 """ 

1252 

1253 def makeDataRefList(self, namespace): 

1254 """Make self.refList from self.idList. 

1255 

1256 Parameters 

1257 ---------- 

1258 namespace 

1259 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1260 """ 

1261 datasetType = namespace.config.coaddName + "Coadd" 

1262 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1263 

1264 for dataId in self.idList: 

1265 # tract and patch are required 

1266 for key in keysCoadd: 

1267 if key not in dataId: 

1268 raise RuntimeError("--id must include " + key) 

1269 

1270 dataRef = namespace.butler.dataRef( 

1271 datasetType=datasetType, 

1272 dataId=dataId, 

1273 ) 

1274 self.refList.append(dataRef) 

1275 

1276 

1277def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1278 """Function to count the number of pixels with a specific mask in a 

1279 footprint. 

1280 

1281 Find the intersection of mask & footprint. Count all pixels in the mask 

1282 that are in the intersection that have bitmask set but do not have 

1283 ignoreMask set. Return the count. 

1284 

1285 Parameters 

1286 ---------- 

1287 mask : `lsst.afw.image.Mask` 

1288 Mask to define intersection region by. 

1289 footprint : `lsst.afw.detection.Footprint` 

1290 Footprint to define the intersection region by. 

1291 bitmask 

1292 Specific mask that we wish to count the number of occurances of. 

1293 ignoreMask 

1294 Pixels to not consider. 

1295 

1296 Returns 

1297 ------- 

1298 result : `int` 

1299 Count of number of pixels in footprint with specified mask. 

1300 """ 

1301 bbox = footprint.getBBox() 

1302 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1303 fp = afwImage.Mask(bbox) 

1304 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1305 footprint.spans.setMask(fp, bitmask) 

1306 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1307 (subMask.getArray() & ignoreMask) == 0).sum() 

1308 

1309 

1310class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1311 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1312 """ 

1313 clipDetection = pexConfig.ConfigurableField( 

1314 target=SourceDetectionTask, 

1315 doc="Detect sources on difference between unclipped and clipped coadd") 

1316 minClipFootOverlap = pexConfig.Field( 

1317 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1318 dtype=float, 

1319 default=0.6 

1320 ) 

1321 minClipFootOverlapSingle = pexConfig.Field( 

1322 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1323 "clipped when only one visit overlaps", 

1324 dtype=float, 

1325 default=0.5 

1326 ) 

1327 minClipFootOverlapDouble = pexConfig.Field( 

1328 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1329 "clipped when two visits overlap", 

1330 dtype=float, 

1331 default=0.45 

1332 ) 

1333 maxClipFootOverlapDouble = pexConfig.Field( 

1334 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1335 "considering two visits", 

1336 dtype=float, 

1337 default=0.15 

1338 ) 

1339 minBigOverlap = pexConfig.Field( 

1340 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1341 "when labeling clipped footprints", 

1342 dtype=int, 

1343 default=100 

1344 ) 

1345 

1346 def setDefaults(self): 

1347 """Set default values for clipDetection. 

1348 

1349 Notes 

1350 ----- 

1351 The numeric values for these configuration parameters were 

1352 empirically determined, future work may further refine them. 

1353 """ 

1354 AssembleCoaddConfig.setDefaults(self) 

1355 self.clipDetection.doTempLocalBackground = False 

1356 self.clipDetection.reEstimateBackground = False 

1357 self.clipDetection.returnOriginalFootprints = False 

1358 self.clipDetection.thresholdPolarity = "both" 

1359 self.clipDetection.thresholdValue = 2 

1360 self.clipDetection.nSigmaToGrow = 2 

1361 self.clipDetection.minPixels = 4 

1362 self.clipDetection.isotropicGrow = True 

1363 self.clipDetection.thresholdType = "pixel_stdev" 

1364 self.sigmaClip = 1.5 

1365 self.clipIter = 3 

1366 self.statistic = "MEAN" 

1367 

1368 def validate(self): 

1369 if self.doSigmaClip: 

1370 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1371 "Ignoring doSigmaClip.") 

1372 self.doSigmaClip = False 

1373 if self.statistic != "MEAN": 

1374 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1375 "(%s chosen). Please set statistic to MEAN." 

1376 % (self.statistic)) 

1377 AssembleCoaddTask.ConfigClass.validate(self) 

1378 

1379 

1380class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1381 """Assemble a coadded image from a set of coadded temporary exposures, 

1382 being careful to clip & flag areas with potential artifacts. 

1383 

1384 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1385 we clip outliers). The problem with doing this is that when computing the 

1386 coadd PSF at a given location, individual visit PSFs from visits with 

1387 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1388 In this task, we correct for this behavior by creating a new 

1389 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1390 coaddTempExps and the final coadd where 

1391 

1392 i. difference imaging suggests that there is an outlier and 

1393 ii. this outlier appears on only one or two images. 

1394 

1395 Such regions will not contribute to the final coadd. Furthermore, any 

1396 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1397 Note that the algorithm implemented by this task is preliminary and works 

1398 correctly for HSC data. Parameter modifications and or considerable 

1399 redesigning of the algorithm is likley required for other surveys. 

1400 

1401 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1402 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1403 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1404 if you wish. 

1405 

1406 Notes 

1407 ----- 

1408 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1409 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1410 see `baseDebug` for more about ``debug.py`` files. 

1411 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1412 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1413 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1414 for further information. 

1415 

1416 Examples 

1417 -------- 

1418 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1419 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1420 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1421 

1422 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1423 and filter to be coadded (specified using 

1424 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1425 along with a list of coaddTempExps to attempt to coadd (specified using 

1426 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1427 Only the coaddTempExps that cover the specified tract and patch will be 

1428 coadded. A list of the available optional arguments can be obtained by 

1429 calling assembleCoadd.py with the --help command line argument: 

1430 

1431 .. code-block:: none 

1432 

1433 assembleCoadd.py --help 

1434 

1435 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1436 context of multi-band processing, we will generate the HSC-I & -R band 

1437 coadds from HSC engineering test data provided in the ci_hsc package. 

1438 To begin, assuming that the lsst stack has been already set up, we must 

1439 set up the obs_subaru and ci_hsc packages. This defines the environment 

1440 variable $CI_HSC_DIR and points at the location of the package. The raw 

1441 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1442 the coadds, we must first 

1443 

1444 - ``processCcd`` 

1445 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1446 - ``makeSkyMap`` 

1447 create a skymap that covers the area of the sky present in the raw exposures 

1448 - ``makeCoaddTempExp`` 

1449 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1450 

1451 We can perform all of these steps by running 

1452 

1453 .. code-block:: none 

1454 

1455 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1456 

1457 This will produce warped coaddTempExps for each visit. To coadd the 

1458 warped data, we call ``assembleCoadd.py`` as follows: 

1459 

1460 .. code-block:: none 

1461 

1462 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1463 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1464 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1465 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1466 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1467 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1468 --selectId visit=903988 ccd=24 

1469 

1470 This will process the HSC-I band data. The results are written in 

1471 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1472 

1473 You may also choose to run: 

1474 

1475 .. code-block:: none 

1476 

1477 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1478 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1479 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1480 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1481 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1482 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1483 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1484 --selectId visit=903346 ccd=12 

1485 

1486 to generate the coadd for the HSC-R band if you are interested in following 

1487 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1488 """ 

1489 ConfigClass = SafeClipAssembleCoaddConfig 

1490 _DefaultName = "safeClipAssembleCoadd" 

1491 

1492 def __init__(self, *args, **kwargs): 

1493 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1494 schema = afwTable.SourceTable.makeMinimalSchema() 

1495 self.makeSubtask("clipDetection", schema=schema) 

1496 

1497 @utils.inheritDoc(AssembleCoaddTask) 

1498 @pipeBase.timeMethod 

1499 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1500 """Assemble the coadd for a region. 

1501 

1502 Compute the difference of coadds created with and without outlier 

1503 rejection to identify coadd pixels that have outlier values in some 

1504 individual visits. 

1505 Detect clipped regions on the difference image and mark these regions 

1506 on the one or two individual coaddTempExps where they occur if there 

1507 is significant overlap between the clipped region and a source. This 

1508 leaves us with a set of footprints from the difference image that have 

1509 been identified as having occured on just one or two individual visits. 

1510 However, these footprints were generated from a difference image. It 

1511 is conceivable for a large diffuse source to have become broken up 

1512 into multiple footprints acrosss the coadd difference in this process. 

1513 Determine the clipped region from all overlapping footprints from the 

1514 detected sources in each visit - these are big footprints. 

1515 Combine the small and big clipped footprints and mark them on a new 

1516 bad mask plane. 

1517 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1518 removal. Clipped footprints will no longer make it into the coadd 

1519 because they are marked in the new bad mask plane. 

1520 

1521 Notes 

1522 ----- 

1523 args and kwargs are passed but ignored in order to match the call 

1524 signature expected by the parent task. 

1525 """ 

1526 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1527 mask = exp.getMaskedImage().getMask() 

1528 mask.addMaskPlane("CLIPPED") 

1529 

1530 result = self.detectClip(exp, tempExpRefList) 

1531 

1532 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1533 

1534 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1535 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1536 # Append big footprints from individual Warps to result.clipSpans 

1537 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1538 result.detectionFootprints, maskClipValue, maskDetValue, 

1539 exp.getBBox()) 

1540 # Create mask of the current clipped footprints 

1541 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1542 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1543 

1544 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1545 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1546 maskClip |= maskClipBig 

1547 

1548 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1549 badMaskPlanes = self.config.badMaskPlanes[:] 

1550 badMaskPlanes.append("CLIPPED") 

1551 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1552 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1553 result.clipSpans, mask=badPixelMask) 

1554 

1555 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1556 """Return an exposure that contains the difference between unclipped 

1557 and clipped coadds. 

1558 

1559 Generate a difference image between clipped and unclipped coadds. 

1560 Compute the difference image by subtracting an outlier-clipped coadd 

1561 from an outlier-unclipped coadd. Return the difference image. 

1562 

1563 Parameters 

1564 ---------- 

1565 skyInfo : `lsst.pipe.base.Struct` 

1566 Patch geometry information, from getSkyInfo 

1567 tempExpRefList : `list` 

1568 List of data reference to tempExp 

1569 imageScalerList : `list` 

1570 List of image scalers 

1571 weightList : `list` 

1572 List of weights 

1573 

1574 Returns 

1575 ------- 

1576 exp : `lsst.afw.image.Exposure` 

1577 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1578 """ 

1579 config = AssembleCoaddConfig() 

1580 # getattr necessary because subtasks do not survive Config.toDict() 

1581 # exclude connections because the class of self.config.connections is not 

1582 # the same as AssembleCoaddConfig.connections, and the connections are not 

1583 # needed to run this task anyway. 

1584 configIntersection = {k: getattr(self.config, k) 

1585 for k, v in self.config.toDict().items() 

1586 if (k in config.keys() and k != "connections")} 

1587 configIntersection['doInputMap'] = False 

1588 configIntersection['doNImage'] = False 

1589 config.update(**configIntersection) 

1590 

1591 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1592 config.statistic = 'MEAN' 

1593 task = AssembleCoaddTask(config=config) 

1594 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1595 

1596 config.statistic = 'MEANCLIP' 

1597 task = AssembleCoaddTask(config=config) 

1598 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1599 

1600 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1601 coaddDiff -= coaddClip.getMaskedImage() 

1602 exp = afwImage.ExposureF(coaddDiff) 

1603 exp.setPsf(coaddMean.getPsf()) 

1604 return exp 

1605 

1606 def detectClip(self, exp, tempExpRefList): 

1607 """Detect clipped regions on an exposure and set the mask on the 

1608 individual tempExp masks. 

1609 

1610 Detect footprints in the difference image after smoothing the 

1611 difference image with a Gaussian kernal. Identify footprints that 

1612 overlap with one or two input ``coaddTempExps`` by comparing the 

1613 computed overlap fraction to thresholds set in the config. A different 

1614 threshold is applied depending on the number of overlapping visits 

1615 (restricted to one or two). If the overlap exceeds the thresholds, 

1616 the footprint is considered "CLIPPED" and is marked as such on the 

1617 coaddTempExp. Return a struct with the clipped footprints, the indices 

1618 of the ``coaddTempExps`` that end up overlapping with the clipped 

1619 footprints, and a list of new masks for the ``coaddTempExps``. 

1620 

1621 Parameters 

1622 ---------- 

1623 exp : `lsst.afw.image.Exposure` 

1624 Exposure to run detection on. 

1625 tempExpRefList : `list` 

1626 List of data reference to tempExp. 

1627 

1628 Returns 

1629 ------- 

1630 result : `lsst.pipe.base.Struct` 

1631 Result struct with components: 

1632 

1633 - ``clipFootprints``: list of clipped footprints. 

1634 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1635 ``tempExpRefList``. 

1636 - ``clipSpans``: List of dictionaries containing spanSet lists 

1637 to clip. Each element contains the new maskplane name 

1638 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1639 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1640 compressed into footprints. 

1641 """ 

1642 mask = exp.getMaskedImage().getMask() 

1643 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1644 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1645 # Merge positive and negative together footprints together 

1646 fpSet.positive.merge(fpSet.negative) 

1647 footprints = fpSet.positive 

1648 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1649 ignoreMask = self.getBadPixelMask() 

1650 

1651 clipFootprints = [] 

1652 clipIndices = [] 

1653 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1654 

1655 # for use by detectClipBig 

1656 visitDetectionFootprints = [] 

1657 

1658 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1659 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1660 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1661 

1662 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1663 for i, warpRef in enumerate(tempExpRefList): 

1664 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1665 immediate=True).getMaskedImage().getMask() 

1666 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1667 afwImage.PARENT, True) 

1668 maskVisitDet &= maskDetValue 

1669 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1670 visitDetectionFootprints.append(visitFootprints) 

1671 

1672 for j, footprint in enumerate(footprints.getFootprints()): 

1673 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1674 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1675 

1676 # build a list of clipped spans for each visit 

1677 for j, footprint in enumerate(footprints.getFootprints()): 

1678 nPixel = footprint.getArea() 

1679 overlap = [] # hold the overlap with each visit 

1680 indexList = [] # index of visit in global list 

1681 for i in range(len(tempExpRefList)): 

1682 ignore = ignoreArr[i, j] 

1683 overlapDet = overlapDetArr[i, j] 

1684 totPixel = nPixel - ignore 

1685 

1686 # If we have more bad pixels than detection skip 

1687 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1687 ↛ 1689line 1687 didn't jump to line 1689, because the condition on line 1687 was never false

1688 continue 

1689 overlap.append(overlapDet/float(totPixel)) 

1690 indexList.append(i) 

1691 

1692 overlap = numpy.array(overlap) 

1693 if not len(overlap): 1693 ↛ 1696line 1693 didn't jump to line 1696, because the condition on line 1693 was never false

1694 continue 

1695 

1696 keep = False # Should this footprint be marked as clipped? 

1697 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1698 

1699 # If footprint only has one overlap use a lower threshold 

1700 if len(overlap) == 1: 

1701 if overlap[0] > self.config.minClipFootOverlapSingle: 

1702 keep = True 

1703 keepIndex = [0] 

1704 else: 

1705 # This is the general case where only visit should be clipped 

1706 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1707 if len(clipIndex) == 1: 

1708 keep = True 

1709 keepIndex = [clipIndex[0]] 

1710 

1711 # Test if there are clipped objects that overlap two different visits 

1712 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1713 if len(clipIndex) == 2 and len(overlap) > 3: 

1714 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1715 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1716 keep = True 

1717 keepIndex = clipIndex 

1718 

1719 if not keep: 

1720 continue 

1721 

1722 for index in keepIndex: 

1723 globalIndex = indexList[index] 

1724 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1725 

1726 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1727 clipFootprints.append(footprint) 

1728 

1729 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1730 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1731 

1732 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1733 maskClipValue, maskDetValue, coaddBBox): 

1734 """Return individual warp footprints for large artifacts and append 

1735 them to ``clipList`` in place. 

1736 

1737 Identify big footprints composed of many sources in the coadd 

1738 difference that may have originated in a large diffuse source in the 

1739 coadd. We do this by indentifying all clipped footprints that overlap 

1740 significantly with each source in all the coaddTempExps. 

1741 

1742 Parameters 

1743 ---------- 

1744 clipList : `list` 

1745 List of alt mask SpanSets with clipping information. Modified. 

1746 clipFootprints : `list` 

1747 List of clipped footprints. 

1748 clipIndices : `list` 

1749 List of which entries in tempExpClipList each footprint belongs to. 

1750 maskClipValue 

1751 Mask value of clipped pixels. 

1752 maskDetValue 

1753 Mask value of detected pixels. 

1754 coaddBBox : `lsst.geom.Box` 

1755 BBox of the coadd and warps. 

1756 

1757 Returns 

1758 ------- 

1759 bigFootprintsCoadd : `list` 

1760 List of big footprints 

1761 """ 

1762 bigFootprintsCoadd = [] 

1763 ignoreMask = self.getBadPixelMask() 

1764 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1765 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1766 for footprint in visitFootprints.getFootprints(): 1766 ↛ 1767line 1766 didn't jump to line 1767, because the loop on line 1766 never started

1767 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1768 

1769 # build a mask of clipped footprints that are in this visit 

1770 clippedFootprintsVisit = [] 

1771 for foot, clipIndex in zip(clipFootprints, clipIndices): 1771 ↛ 1772line 1771 didn't jump to line 1772, because the loop on line 1771 never started

1772 if index not in clipIndex: 

1773 continue 

1774 clippedFootprintsVisit.append(foot) 

1775 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1776 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1777 

1778 bigFootprintsVisit = [] 

1779 for foot in visitFootprints.getFootprints(): 1779 ↛ 1780line 1779 didn't jump to line 1780, because the loop on line 1779 never started

1780 if foot.getArea() < self.config.minBigOverlap: 

1781 continue 

1782 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1783 if nCount > self.config.minBigOverlap: 

1784 bigFootprintsVisit.append(foot) 

1785 bigFootprintsCoadd.append(foot) 

1786 

1787 for footprint in bigFootprintsVisit: 1787 ↛ 1788line 1787 didn't jump to line 1788, because the loop on line 1787 never started

1788 clippedSpans["CLIPPED"].append(footprint.spans) 

1789 

1790 return bigFootprintsCoadd 

1791 

1792 

1793class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1794 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1795 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1796 "Only PSF-Matched Warps make sense for image subtraction. " 

1797 "Therefore, they must be an additional declared input."), 

1798 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1799 storageClass="ExposureF", 

1800 dimensions=("tract", "patch", "skymap", "visit"), 

1801 deferLoad=True, 

1802 multiple=True 

1803 ) 

1804 templateCoadd = pipeBase.connectionTypes.Output( 

1805 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1806 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1807 name="{outputCoaddName}CoaddPsfMatched", 

1808 storageClass="ExposureF", 

1809 dimensions=("tract", "patch", "skymap", "band"), 

1810 ) 

1811 

1812 def __init__(self, *, config=None): 

1813 super().__init__(config=config) 

1814 if not config.assembleStaticSkyModel.doWrite: 

1815 self.outputs.remove("templateCoadd") 

1816 config.validate() 

1817 

1818 

1819class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1820 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1821 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1822 target=AssembleCoaddTask, 

1823 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1824 " naive/first-iteration model of the static sky.", 

1825 ) 

1826 detect = pexConfig.ConfigurableField( 

1827 target=SourceDetectionTask, 

1828 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1829 ) 

1830 detectTemplate = pexConfig.ConfigurableField( 

1831 target=SourceDetectionTask, 

1832 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1833 ) 

1834 maskStreaks = pexConfig.ConfigurableField( 

1835 target=MaskStreaksTask, 

1836 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1837 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1838 "streakMaskName" 

1839 ) 

1840 streakMaskName = pexConfig.Field( 

1841 dtype=str, 

1842 default="STREAK", 

1843 doc="Name of mask bit used for streaks" 

1844 ) 

1845 maxNumEpochs = pexConfig.Field( 

1846 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1847 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1848 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1849 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1850 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1851 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1852 "than transient and not masked.", 

1853 dtype=int, 

1854 default=2 

1855 ) 

1856 maxFractionEpochsLow = pexConfig.RangeField( 

1857 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1858 "Effective maxNumEpochs = " 

1859 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1860 dtype=float, 

1861 default=0.4, 

1862 min=0., max=1., 

1863 ) 

1864 maxFractionEpochsHigh = pexConfig.RangeField( 

1865 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1866 "Effective maxNumEpochs = " 

1867 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1868 dtype=float, 

1869 default=0.03, 

1870 min=0., max=1., 

1871 ) 

1872 spatialThreshold = pexConfig.RangeField( 

1873 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1874 "temporal criteria. If 0, clip all. If 1, clip none.", 

1875 dtype=float, 

1876 default=0.5, 

1877 min=0., max=1., 

1878 inclusiveMin=True, inclusiveMax=True 

1879 ) 

1880 doScaleWarpVariance = pexConfig.Field( 

1881 doc="Rescale Warp variance plane using empirical noise?", 

1882 dtype=bool, 

1883 default=True, 

1884 ) 

1885 scaleWarpVariance = pexConfig.ConfigurableField( 

1886 target=ScaleVarianceTask, 

1887 doc="Rescale variance on warps", 

1888 ) 

1889 doPreserveContainedBySource = pexConfig.Field( 

1890 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1891 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1892 dtype=bool, 

1893 default=True, 

1894 ) 

1895 doPrefilterArtifacts = pexConfig.Field( 

1896 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1897 "because they will be excluded anyway. This prevents them from contributing " 

1898 "to the outlier epoch count image and potentially being labeled as persistant." 

1899 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1900 dtype=bool, 

1901 default=True 

1902 ) 

1903 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1904 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1905 dtype=str, 

1906 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1907 ) 

1908 prefilterArtifactsRatio = pexConfig.Field( 

1909 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1910 dtype=float, 

1911 default=0.05 

1912 ) 

1913 doFilterMorphological = pexConfig.Field( 

1914 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1915 "be streaks.", 

1916 dtype=bool, 

1917 default=False 

1918 ) 

1919 

1920 def setDefaults(self): 

1921 AssembleCoaddConfig.setDefaults(self) 

1922 self.statistic = 'MEAN' 

1923 self.doUsePsfMatchedPolygons = True 

1924 

1925 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1926 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1927 if "EDGE" in self.badMaskPlanes: 1927 ↛ 1929line 1927 didn't jump to line 1929, because the condition on line 1927 was never false

1928 self.badMaskPlanes.remove('EDGE') 

1929 self.removeMaskPlanes.append('EDGE') 

1930 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1931 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1932 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1933 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1934 self.assembleStaticSkyModel.sigmaClip = 2.5 

1935 self.assembleStaticSkyModel.clipIter = 3 

1936 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1937 self.assembleStaticSkyModel.doWrite = False 

1938 self.detect.doTempLocalBackground = False 

1939 self.detect.reEstimateBackground = False 

1940 self.detect.returnOriginalFootprints = False 

1941 self.detect.thresholdPolarity = "both" 

1942 self.detect.thresholdValue = 5 

1943 self.detect.minPixels = 4 

1944 self.detect.isotropicGrow = True 

1945 self.detect.thresholdType = "pixel_stdev" 

1946 self.detect.nSigmaToGrow = 0.4 

1947 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1948 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1949 self.detectTemplate.nSigmaToGrow = 2.4 

1950 self.detectTemplate.doTempLocalBackground = False 

1951 self.detectTemplate.reEstimateBackground = False 

1952 self.detectTemplate.returnOriginalFootprints = False 

1953 

1954 def validate(self): 

1955 super().validate() 

1956 if self.assembleStaticSkyModel.doNImage: 

1957 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1958 "Please set assembleStaticSkyModel.doNImage=False") 

1959 

1960 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1961 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1962 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1963 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1964 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1965 

1966 

1967class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1968 """Assemble a compareWarp coadded image from a set of warps 

1969 by masking artifacts detected by comparing PSF-matched warps. 

1970 

1971 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1972 we clip outliers). The problem with doing this is that when computing the 

1973 coadd PSF at a given location, individual visit PSFs from visits with 

1974 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1975 In this task, we correct for this behavior by creating a new badMaskPlane 

1976 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1977 an artifact. We populate this plane on the input warps by comparing 

1978 PSF-matched warps with a PSF-matched median coadd which serves as a 

1979 model of the static sky. Any group of pixels that deviates from the 

1980 PSF-matched template coadd by more than config.detect.threshold sigma, 

1981 is an artifact candidate. The candidates are then filtered to remove 

1982 variable sources and sources that are difficult to subtract such as 

1983 bright stars. This filter is configured using the config parameters 

1984 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1985 the maximum fraction of epochs that the deviation can appear in and still 

1986 be considered an artifact. The spatialThreshold is the maximum fraction of 

1987 pixels in the footprint of the deviation that appear in other epochs 

1988 (where other epochs is defined by the temporalThreshold). If the deviant 

1989 region meets this criteria of having a significant percentage of pixels 

1990 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1991 set in the mask. These regions will not contribute to the final coadd. 

1992 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1993 of clipped regions. Note that the algorithm implemented by this task is 

1994 preliminary and works correctly for HSC data. Parameter modifications and 

1995 or considerable redesigning of the algorithm is likley required for other 

1996 surveys. 

1997 

1998 ``CompareWarpAssembleCoaddTask`` sub-classes 

1999 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

2000 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2001 

2002 Notes 

2003 ----- 

2004 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2005 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2006 ``baseDebug`` for more about ``debug.py`` files. 

2007 

2008 This task supports the following debug variables: 

2009 

2010 - ``saveCountIm`` 

2011 If True then save the Epoch Count Image as a fits file in the `figPath` 

2012 - ``figPath`` 

2013 Path to save the debug fits images and figures 

2014 

2015 For example, put something like: 

2016 

2017 .. code-block:: python 

2018 

2019 import lsstDebug 

2020 def DebugInfo(name): 

2021 di = lsstDebug.getInfo(name) 

2022 if name == "lsst.pipe.tasks.assembleCoadd": 

2023 di.saveCountIm = True 

2024 di.figPath = "/desired/path/to/debugging/output/images" 

2025 return di 

2026 lsstDebug.Info = DebugInfo 

2027 

2028 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2029 ``--debug`` flag. Some subtasks may have their own debug variables; 

2030 see individual Task documentation. 

2031 

2032 Examples 

2033 -------- 

2034 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2035 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2036 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2037 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2038 and filter to be coadded (specified using 

2039 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2040 along with a list of coaddTempExps to attempt to coadd (specified using 

2041 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2042 Only the warps that cover the specified tract and patch will be coadded. 

2043 A list of the available optional arguments can be obtained by calling 

2044 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2045 

2046 .. code-block:: none 

2047 

2048 assembleCoadd.py --help 

2049 

2050 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2051 context of multi-band processing, we will generate the HSC-I & -R band 

2052 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2053 To begin, assuming that the lsst stack has been already set up, we must 

2054 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2055 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2056 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2057 directory. To begin assembling the coadds, we must first 

2058 

2059 - processCcd 

2060 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2061 - makeSkyMap 

2062 create a skymap that covers the area of the sky present in the raw exposures 

2063 - makeCoaddTempExp 

2064 warp the individual calibrated exposures to the tangent plane of the coadd 

2065 

2066 We can perform all of these steps by running 

2067 

2068 .. code-block:: none 

2069 

2070 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2071 

2072 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2073 warped data, we call ``assembleCoadd.py`` as follows: 

2074 

2075 .. code-block:: none 

2076 

2077 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2078 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2079 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2080 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2081 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2082 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2083 --selectId visit=903988 ccd=24 

2084 

2085 This will process the HSC-I band data. The results are written in 

2086 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2087 """ 

2088 ConfigClass = CompareWarpAssembleCoaddConfig 

2089 _DefaultName = "compareWarpAssembleCoadd" 

2090 

2091 def __init__(self, *args, **kwargs): 

2092 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2093 self.makeSubtask("assembleStaticSkyModel") 

2094 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2095 self.makeSubtask("detect", schema=detectionSchema) 

2096 if self.config.doPreserveContainedBySource: 2096 ↛ 2098line 2096 didn't jump to line 2098, because the condition on line 2096 was never false

2097 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2098 if self.config.doScaleWarpVariance: 2098 ↛ 2100line 2098 didn't jump to line 2100, because the condition on line 2098 was never false

2099 self.makeSubtask("scaleWarpVariance") 

2100 if self.config.doFilterMorphological: 2100 ↛ 2101line 2100 didn't jump to line 2101, because the condition on line 2100 was never true

2101 self.makeSubtask("maskStreaks") 

2102 

2103 @utils.inheritDoc(AssembleCoaddTask) 

2104 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2105 """ 

2106 Generate a templateCoadd to use as a naive model of static sky to 

2107 subtract from PSF-Matched warps. 

2108 

2109 Returns 

2110 ------- 

2111 result : `lsst.pipe.base.Struct` 

2112 Result struct with components: 

2113 

2114 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2115 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2116 """ 

2117 # Ensure that psfMatchedWarps are used as input warps for template generation 

2118 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2119 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2120 

2121 # Because subtasks don't have connections we have to make one. 

2122 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2123 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2124 if self.config.assembleStaticSkyModel.doWrite: 

2125 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2126 # Remove template coadd from both subtask's and main tasks outputs, 

2127 # because it is handled by the subtask as `coaddExposure` 

2128 del outputRefs.templateCoadd 

2129 del staticSkyModelOutputRefs.templateCoadd 

2130 

2131 # A PSF-Matched nImage does not exist as a dataset type 

2132 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2133 del staticSkyModelOutputRefs.nImage 

2134 

2135 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2136 staticSkyModelOutputRefs) 

2137 if templateCoadd is None: 

2138 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2139 

2140 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2141 nImage=templateCoadd.nImage, 

2142 warpRefList=templateCoadd.warpRefList, 

2143 imageScalerList=templateCoadd.imageScalerList, 

2144 weightList=templateCoadd.weightList) 

2145 

2146 @utils.inheritDoc(AssembleCoaddTask) 

2147 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2148 """ 

2149 Generate a templateCoadd to use as a naive model of static sky to 

2150 subtract from PSF-Matched warps. 

2151 

2152 Returns 

2153 ------- 

2154 result : `lsst.pipe.base.Struct` 

2155 Result struct with components: 

2156 

2157 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2158 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2159 """ 

2160 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2161 if templateCoadd is None: 2161 ↛ 2162line 2161 didn't jump to line 2162, because the condition on line 2161 was never true

2162 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2163 

2164 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2165 nImage=templateCoadd.nImage, 

2166 warpRefList=templateCoadd.warpRefList, 

2167 imageScalerList=templateCoadd.imageScalerList, 

2168 weightList=templateCoadd.weightList) 

2169 

2170 def _noTemplateMessage(self, warpType): 

2171 warpName = (warpType[0].upper() + warpType[1:]) 

2172 message = """No %(warpName)s warps were found to build the template coadd which is 

2173 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2174 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2175 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2176 

2177 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2178 another algorithm like: 

2179 

2180 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2181 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2182 """ % {"warpName": warpName} 

2183 return message 

2184 

2185 @utils.inheritDoc(AssembleCoaddTask) 

2186 @pipeBase.timeMethod 

2187 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2188 supplementaryData, *args, **kwargs): 

2189 """Assemble the coadd. 

2190 

2191 Find artifacts and apply them to the warps' masks creating a list of 

2192 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2193 plane. Then pass these alternative masks to the base class's `run` 

2194 method. 

2195 

2196 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2197 that must contain a ``templateCoadd`` that serves as the 

2198 model of the static sky. 

2199 """ 

2200 

2201 # Check and match the order of the supplementaryData 

2202 # (PSF-matched) inputs to the order of the direct inputs, 

2203 # so that the artifact mask is applied to the right warp 

2204 dataIds = [ref.dataId for ref in tempExpRefList] 

2205 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2206 

2207 if dataIds != psfMatchedDataIds: 

2208 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2209 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2210 psfMatchedDataIds, dataIds) 

2211 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2212 psfMatchedDataIds, dataIds) 

2213 

2214 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2215 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2216 supplementaryData.warpRefList, 

2217 supplementaryData.imageScalerList) 

2218 

2219 badMaskPlanes = self.config.badMaskPlanes[:] 

2220 badMaskPlanes.append("CLIPPED") 

2221 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2222 

2223 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2224 spanSetMaskList, mask=badPixelMask) 

2225 

2226 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2227 # Psf-Matching moves the real edge inwards 

2228 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2229 return result 

2230 

2231 def applyAltEdgeMask(self, mask, altMaskList): 

2232 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2233 

2234 Parameters 

2235 ---------- 

2236 mask : `lsst.afw.image.Mask` 

2237 Original mask. 

2238 altMaskList : `list` 

2239 List of Dicts containing ``spanSet`` lists. 

2240 Each element contains the new mask plane name (e.g. "CLIPPED 

2241 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2242 the mask. 

2243 """ 

2244 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2245 for visitMask in altMaskList: 

2246 if "EDGE" in visitMask: 2246 ↛ 2245line 2246 didn't jump to line 2245, because the condition on line 2246 was never false

2247 for spanSet in visitMask['EDGE']: 

2248 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2249 

2250 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2251 """Find artifacts. 

2252 

2253 Loop through warps twice. The first loop builds a map with the count 

2254 of how many epochs each pixel deviates from the templateCoadd by more 

2255 than ``config.chiThreshold`` sigma. The second loop takes each 

2256 difference image and filters the artifacts detected in each using 

2257 count map to filter out variable sources and sources that are 

2258 difficult to subtract cleanly. 

2259 

2260 Parameters 

2261 ---------- 

2262 templateCoadd : `lsst.afw.image.Exposure` 

2263 Exposure to serve as model of static sky. 

2264 tempExpRefList : `list` 

2265 List of data references to warps. 

2266 imageScalerList : `list` 

2267 List of image scalers. 

2268 

2269 Returns 

2270 ------- 

2271 altMasks : `list` 

2272 List of dicts containing information about CLIPPED 

2273 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2274 """ 

2275 

2276 self.log.debug("Generating Count Image, and mask lists.") 

2277 coaddBBox = templateCoadd.getBBox() 

2278 slateIm = afwImage.ImageU(coaddBBox) 

2279 epochCountImage = afwImage.ImageU(coaddBBox) 

2280 nImage = afwImage.ImageU(coaddBBox) 

2281 spanSetArtifactList = [] 

2282 spanSetNoDataMaskList = [] 

2283 spanSetEdgeList = [] 

2284 spanSetBadMorphoList = [] 

2285 badPixelMask = self.getBadPixelMask() 

2286 

2287 # mask of the warp diffs should = that of only the warp 

2288 templateCoadd.mask.clearAllMaskPlanes() 

2289 

2290 if self.config.doPreserveContainedBySource: 2290 ↛ 2293line 2290 didn't jump to line 2293, because the condition on line 2290 was never false

2291 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2292 else: 

2293 templateFootprints = None 

2294 

2295 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2296 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2297 if warpDiffExp is not None: 

2298 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2299 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2300 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2301 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2302 fpSet.positive.merge(fpSet.negative) 

2303 footprints = fpSet.positive 

2304 slateIm.set(0) 

2305 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2306 

2307 # Remove artifacts due to defects before they contribute to the epochCountImage 

2308 if self.config.doPrefilterArtifacts: 2308 ↛ 2312line 2308 didn't jump to line 2312, because the condition on line 2308 was never false

2309 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2310 

2311 # Clear mask before adding prefiltered spanSets 

2312 self.detect.clearMask(warpDiffExp.mask) 

2313 for spans in spanSetList: 

2314 spans.setImage(slateIm, 1, doClip=True) 

2315 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2316 epochCountImage += slateIm 

2317 

2318 if self.config.doFilterMorphological: 2318 ↛ 2319line 2318 didn't jump to line 2319, because the condition on line 2318 was never true

2319 maskName = self.config.streakMaskName 

2320 _ = self.maskStreaks.run(warpDiffExp) 

2321 streakMask = warpDiffExp.mask 

2322 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2323 streakMask.getPlaneBitMask(maskName)).split() 

2324 

2325 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2326 # undergo a second convolution. Pixels with data in the direct warp 

2327 # but not in the PSF-matched warp will not have their artifacts detected. 

2328 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2329 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2330 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2331 nansMask.setXY0(warpDiffExp.getXY0()) 

2332 edgeMask = warpDiffExp.mask 

2333 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2334 edgeMask.getPlaneBitMask("EDGE")).split() 

2335 else: 

2336 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2337 # In this case, mask the whole epoch 

2338 nansMask = afwImage.MaskX(coaddBBox, 1) 

2339 spanSetList = [] 

2340 spanSetEdgeMask = [] 

2341 spanSetStreak = [] 

2342 

2343 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2344 

2345 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2346 spanSetArtifactList.append(spanSetList) 

2347 spanSetEdgeList.append(spanSetEdgeMask) 

2348 if self.config.doFilterMorphological: 2348 ↛ 2349line 2348 didn't jump to line 2349, because the condition on line 2348 was never true

2349 spanSetBadMorphoList.append(spanSetStreak) 

2350 

2351 if lsstDebug.Info(__name__).saveCountIm: 2351 ↛ 2352line 2351 didn't jump to line 2352, because the condition on line 2351 was never true

2352 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2353 epochCountImage.writeFits(path) 

2354 

2355 for i, spanSetList in enumerate(spanSetArtifactList): 

2356 if spanSetList: 

2357 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2358 templateFootprints) 

2359 spanSetArtifactList[i] = filteredSpanSetList 

2360 if self.config.doFilterMorphological: 2360 ↛ 2361line 2360 didn't jump to line 2361, because the condition on line 2360 was never true

2361 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2362 

2363 altMasks = [] 

2364 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2365 altMasks.append({'CLIPPED': artifacts, 

2366 'NO_DATA': noData, 

2367 'EDGE': edge}) 

2368 return altMasks 

2369 

2370 def prefilterArtifacts(self, spanSetList, exp): 

2371 """Remove artifact candidates covered by bad mask plane. 

2372 

2373 Any future editing of the candidate list that does not depend on 

2374 temporal information should go in this method. 

2375 

2376 Parameters 

2377 ---------- 

2378 spanSetList : `list` 

2379 List of SpanSets representing artifact candidates. 

2380 exp : `lsst.afw.image.Exposure` 

2381 Exposure containing mask planes used to prefilter. 

2382 

2383 Returns 

2384 ------- 

2385 returnSpanSetList : `list` 

2386 List of SpanSets with artifacts. 

2387 """ 

2388 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2389 goodArr = (exp.mask.array & badPixelMask) == 0 

2390 returnSpanSetList = [] 

2391 bbox = exp.getBBox() 

2392 x0, y0 = exp.getXY0() 

2393 for i, span in enumerate(spanSetList): 

2394 y, x = span.clippedTo(bbox).indices() 

2395 yIndexLocal = numpy.array(y) - y0 

2396 xIndexLocal = numpy.array(x) - x0 

2397 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2398 if goodRatio > self.config.prefilterArtifactsRatio: 2398 ↛ 2393line 2398 didn't jump to line 2393, because the condition on line 2398 was never false

2399 returnSpanSetList.append(span) 

2400 return returnSpanSetList 

2401 

2402 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2403 """Filter artifact candidates. 

2404 

2405 Parameters 

2406 ---------- 

2407 spanSetList : `list` 

2408 List of SpanSets representing artifact candidates. 

2409 epochCountImage : `lsst.afw.image.Image` 

2410 Image of accumulated number of warpDiff detections. 

2411 nImage : `lsst.afw.image.Image` 

2412 Image of the accumulated number of total epochs contributing. 

2413 

2414 Returns 

2415 ------- 

2416 maskSpanSetList : `list` 

2417 List of SpanSets with artifacts. 

2418 """ 

2419 

2420 maskSpanSetList = [] 

2421 x0, y0 = epochCountImage.getXY0() 

2422 for i, span in enumerate(spanSetList): 

2423 y, x = span.indices() 

2424 yIdxLocal = [y1 - y0 for y1 in y] 

2425 xIdxLocal = [x1 - x0 for x1 in x] 

2426 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2427 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2428 

2429 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2430 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2431 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2432 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2433 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2434 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2435 & (outlierN <= effectiveMaxNumEpochs)) 

2436 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2437 if percentBelowThreshold > self.config.spatialThreshold: 

2438 maskSpanSetList.append(span) 

2439 

2440 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2440 ↛ 2453line 2440 didn't jump to line 2453, because the condition on line 2440 was never false

2441 # If a candidate is contained by a footprint on the template coadd, do not clip 

2442 filteredMaskSpanSetList = [] 

2443 for span in maskSpanSetList: 

2444 doKeep = True 

2445 for footprint in footprintsToExclude.positive.getFootprints(): 

2446 if footprint.spans.contains(span): 

2447 doKeep = False 

2448 break 

2449 if doKeep: 

2450 filteredMaskSpanSetList.append(span) 

2451 maskSpanSetList = filteredMaskSpanSetList 

2452 

2453 return maskSpanSetList 

2454 

2455 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2456 """Fetch a warp from the butler and return a warpDiff. 

2457 

2458 Parameters 

2459 ---------- 

2460 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2461 Butler dataRef for the warp. 

2462 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2463 An image scaler object. 

2464 templateCoadd : `lsst.afw.image.Exposure` 

2465 Exposure to be substracted from the scaled warp. 

2466 

2467 Returns 

2468 ------- 

2469 warp : `lsst.afw.image.Exposure` 

2470 Exposure of the image difference between the warp and template. 

2471 """ 

2472 

2473 # If the PSF-Matched warp did not exist for this direct warp 

2474 # None is holding its place to maintain order in Gen 3 

2475 if warpRef is None: 

2476 return None 

2477 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2478 warpName = self.getTempExpDatasetName('psfMatched') 

2479 if not isinstance(warpRef, DeferredDatasetHandle): 2479 ↛ 2483line 2479 didn't jump to line 2483, because the condition on line 2479 was never false

2480 if not warpRef.datasetExists(warpName): 2480 ↛ 2481line 2480 didn't jump to line 2481, because the condition on line 2480 was never true

2481 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2482 return None 

2483 warp = warpRef.get(datasetType=warpName, immediate=True) 

2484 # direct image scaler OK for PSF-matched Warp 

2485 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2486 mi = warp.getMaskedImage() 

2487 if self.config.doScaleWarpVariance: 2487 ↛ 2492line 2487 didn't jump to line 2492, because the condition on line 2487 was never false

2488 try: 

2489 self.scaleWarpVariance.run(mi) 

2490 except Exception as exc: 

2491 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,)) 

2492 mi -= templateCoadd.getMaskedImage() 

2493 return warp 

2494 

2495 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2496 """Return a path to which to write debugging output. 

2497 

2498 Creates a hyphen-delimited string of dataId values for simple filenames. 

2499 

2500 Parameters 

2501 ---------- 

2502 prefix : `str` 

2503 Prefix for filename. 

2504 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2505 Butler dataRef to make the path from. 

2506 coaddLevel : `bool`, optional. 

2507 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2508 'filter', but no 'visit'). 

2509 

2510 Returns 

2511 ------- 

2512 result : `str` 

2513 Path for debugging output. 

2514 """ 

2515 if coaddLevel: 

2516 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2517 else: 

2518 keys = warpRef.dataId.keys() 

2519 keyList = sorted(keys, reverse=True) 

2520 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2521 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2522 return os.path.join(directory, filename)