Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import copy 

24import numpy 

25import warnings 

26import logging 

27import lsst.pex.config as pexConfig 

28import lsst.pex.exceptions as pexExceptions 

29import lsst.geom as geom 

30import lsst.afw.geom as afwGeom 

31import lsst.afw.image as afwImage 

32import lsst.afw.math as afwMath 

33import lsst.afw.table as afwTable 

34import lsst.afw.detection as afwDet 

35import lsst.coadd.utils as coaddUtils 

36import lsst.pipe.base as pipeBase 

37import lsst.meas.algorithms as measAlg 

38import lsstDebug 

39import lsst.utils as utils 

40from lsst.skymap import BaseSkyMap 

41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList 

42from .interpImage import InterpImageTask 

43from .scaleZeroPoint import ScaleZeroPointTask 

44from .coaddHelpers import groupPatchExposures, getGroupDataRef 

45from .scaleVariance import ScaleVarianceTask 

46from .maskStreaks import MaskStreaksTask 

47from .healSparseMapping import HealSparseInputMapTask 

48from lsst.meas.algorithms import SourceDetectionTask 

49from lsst.daf.butler import DeferredDatasetHandle 

50 

51__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig", 

52 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig", 

53 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"] 

54 

55log = logging.getLogger(__name__.partition(".")[2]) 

56 

57 

58class AssembleCoaddConnections(pipeBase.PipelineTaskConnections, 

59 dimensions=("tract", "patch", "band", "skymap"), 

60 defaultTemplates={"inputCoaddName": "deep", 

61 "outputCoaddName": "deep", 

62 "warpType": "direct", 

63 "warpTypeSuffix": ""}): 

64 

65 inputWarps = pipeBase.connectionTypes.Input( 

66 doc=("Input list of warps to be assemebled i.e. stacked." 

67 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"), 

68 name="{inputCoaddName}Coadd_{warpType}Warp", 

69 storageClass="ExposureF", 

70 dimensions=("tract", "patch", "skymap", "visit", "instrument"), 

71 deferLoad=True, 

72 multiple=True 

73 ) 

74 skyMap = pipeBase.connectionTypes.Input( 

75 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

77 storageClass="SkyMap", 

78 dimensions=("skymap", ), 

79 ) 

80 selectedVisits = pipeBase.connectionTypes.Input( 

81 doc="Selected visits to be coadded.", 

82 name="{outputCoaddName}Visits", 

83 storageClass="StructuredDataDict", 

84 dimensions=("instrument", "tract", "patch", "skymap", "band") 

85 ) 

86 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput( 

87 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 

88 " BRIGHT_OBJECT."), 

89 name="brightObjectMask", 

90 storageClass="ObjectMaskCatalog", 

91 dimensions=("tract", "patch", "skymap", "band"), 

92 ) 

93 coaddExposure = pipeBase.connectionTypes.Output( 

94 doc="Output coadded exposure, produced by stacking input warps", 

95 name="{outputCoaddName}Coadd{warpTypeSuffix}", 

96 storageClass="ExposureF", 

97 dimensions=("tract", "patch", "skymap", "band"), 

98 ) 

99 nImage = pipeBase.connectionTypes.Output( 

100 doc="Output image of number of input images per pixel", 

101 name="{outputCoaddName}Coadd_nImage", 

102 storageClass="ImageU", 

103 dimensions=("tract", "patch", "skymap", "band"), 

104 ) 

105 inputMap = pipeBase.connectionTypes.Output( 

106 doc="Output healsparse map of input images", 

107 name="{outputCoaddName}Coadd_inputMap", 

108 storageClass="HealSparseMap", 

109 dimensions=("tract", "patch", "skymap", "band"), 

110 ) 

111 

112 def __init__(self, *, config=None): 

113 super().__init__(config=config) 

114 

115 # Override the connection's name template with config to replicate Gen2 behavior 

116 # This duplicates some of the logic in the base class, due to wanting Gen2 and 

117 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated 

118 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates} 

119 templateValues['warpType'] = config.warpType 

120 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType) 

121 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues) 

122 for name in self.allConnections} 

123 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

124 # End code to remove after deprecation 

125 

126 if not config.doMaskBrightObjects: 

127 self.prerequisiteInputs.remove("brightObjectMask") 

128 

129 if not config.doSelectVisits: 

130 self.inputs.remove("selectedVisits") 

131 

132 if not config.doNImage: 

133 self.outputs.remove("nImage") 

134 

135 if not self.config.doInputMap: 

136 self.outputs.remove("inputMap") 

137 

138 

139class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, 

140 pipelineConnections=AssembleCoaddConnections): 

141 """Configuration parameters for the `AssembleCoaddTask`. 

142 

143 Notes 

144 ----- 

145 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 

146 only set the bitplane config.brightObjectMaskName. To make this useful you 

147 *must* also configure the flags.pixel algorithm, for example by adding 

148 

149 .. code-block:: none 

150 

151 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 

152 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 

153 

154 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 

155 """ 

156 warpType = pexConfig.Field( 

157 doc="Warp name: one of 'direct' or 'psfMatched'", 

158 dtype=str, 

159 default="direct", 

160 ) 

161 subregionSize = pexConfig.ListField( 

162 dtype=int, 

163 doc="Width, height of stack subregion size; " 

164 "make small enough that a full stack of images will fit into memory at once.", 

165 length=2, 

166 default=(2000, 2000), 

167 ) 

168 statistic = pexConfig.Field( 

169 dtype=str, 

170 doc="Main stacking statistic for aggregating over the epochs.", 

171 default="MEANCLIP", 

172 ) 

173 doSigmaClip = pexConfig.Field( 

174 dtype=bool, 

175 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)", 

176 default=False, 

177 ) 

178 sigmaClip = pexConfig.Field( 

179 dtype=float, 

180 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.", 

181 default=3.0, 

182 ) 

183 clipIter = pexConfig.Field( 

184 dtype=int, 

185 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.", 

186 default=2, 

187 ) 

188 calcErrorFromInputVariance = pexConfig.Field( 

189 dtype=bool, 

190 doc="Calculate coadd variance from input variance by stacking statistic." 

191 "Passed to StatisticsControl.setCalcErrorFromInputVariance()", 

192 default=True, 

193 ) 

194 scaleZeroPoint = pexConfig.ConfigurableField( 

195 target=ScaleZeroPointTask, 

196 doc="Task to adjust the photometric zero point of the coadd temp exposures", 

197 ) 

198 doInterp = pexConfig.Field( 

199 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.", 

200 dtype=bool, 

201 default=True, 

202 ) 

203 interpImage = pexConfig.ConfigurableField( 

204 target=InterpImageTask, 

205 doc="Task to interpolate (and extrapolate) over NaN pixels", 

206 ) 

207 doWrite = pexConfig.Field( 

208 doc="Persist coadd?", 

209 dtype=bool, 

210 default=True, 

211 ) 

212 doNImage = pexConfig.Field( 

213 doc="Create image of number of contributing exposures for each pixel", 

214 dtype=bool, 

215 default=False, 

216 ) 

217 doUsePsfMatchedPolygons = pexConfig.Field( 

218 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.", 

219 dtype=bool, 

220 default=False, 

221 ) 

222 maskPropagationThresholds = pexConfig.DictField( 

223 keytype=str, 

224 itemtype=float, 

225 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 

226 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 

227 "would have contributed exceeds this value."), 

228 default={"SAT": 0.1}, 

229 ) 

230 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"], 

231 doc="Mask planes to remove before coadding") 

232 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False, 

233 doc="Set mask and flag bits for bright objects?") 

234 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT", 

235 doc="Name of mask bit used for bright objects") 

236 coaddPsf = pexConfig.ConfigField( 

237 doc="Configuration for CoaddPsf", 

238 dtype=measAlg.CoaddPsfConfig, 

239 ) 

240 doAttachTransmissionCurve = pexConfig.Field( 

241 dtype=bool, default=False, optional=False, 

242 doc=("Attach a piecewise TransmissionCurve for the coadd? " 

243 "(requires all input Exposures to have TransmissionCurves).") 

244 ) 

245 hasFakes = pexConfig.Field( 

246 dtype=bool, 

247 default=False, 

248 doc="Should be set to True if fake sources have been inserted into the input data." 

249 ) 

250 doSelectVisits = pexConfig.Field( 

251 doc="Coadd only visits selected by a SelectVisitsTask", 

252 dtype=bool, 

253 default=False, 

254 ) 

255 doInputMap = pexConfig.Field( 

256 doc="Create a bitwise map of coadd inputs", 

257 dtype=bool, 

258 default=False, 

259 ) 

260 inputMapper = pexConfig.ConfigurableField( 

261 doc="Input map creation subtask.", 

262 target=HealSparseInputMapTask, 

263 ) 

264 

265 def setDefaults(self): 

266 super().setDefaults() 

267 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"] 

268 

269 def validate(self): 

270 super().validate() 

271 if self.doPsfMatch: 

272 # Backwards compatibility. 

273 # Configs do not have loggers 

274 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'") 

275 self.warpType = 'psfMatched' 

276 if self.doSigmaClip and self.statistic != "MEANCLIP": 

277 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"') 

278 self.statistic = "MEANCLIP" 

279 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']: 

280 raise ValueError("Must set doInterp=False for statistic=%s, which does not " 

281 "compute and set a non-zero coadd variance estimate." % (self.statistic)) 

282 

283 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK'] 

284 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats: 

285 stackableStats = [str(k) for k in afwMath.Property.__members__.keys() 

286 if str(k) not in unstackableStats] 

287 raise ValueError("statistic %s is not allowed. Please choose one of %s." 

288 % (self.statistic, stackableStats)) 

289 

290 

291class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask): 

292 """Assemble a coadded image from a set of warps (coadded temporary exposures). 

293 

294 We want to assemble a coadded image from a set of Warps (also called 

295 coadded temporary exposures or ``coaddTempExps``). 

296 Each input Warp covers a patch on the sky and corresponds to a single 

297 run/visit/exposure of the covered patch. We provide the task with a list 

298 of Warps (``selectDataList``) from which it selects Warps that cover the 

299 specified patch (pointed at by ``dataRef``). 

300 Each Warp that goes into a coadd will typically have an independent 

301 photometric zero-point. Therefore, we must scale each Warp to set it to 

302 a common photometric zeropoint. WarpType may be one of 'direct' or 

303 'psfMatched', and the boolean configs `config.makeDirect` and 

304 `config.makePsfMatched` set which of the warp types will be coadded. 

305 The coadd is computed as a mean with optional outlier rejection. 

306 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 

307 Finally, Warps can have bad 'NaN' pixels which received no input from the 

308 source calExps. We interpolate over these bad (NaN) pixels. 

309 

310 `AssembleCoaddTask` uses several sub-tasks. These are 

311 

312 - `ScaleZeroPointTask` 

313 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 

314 - `InterpImageTask` 

315 - interpolate across bad pixels (NaN) in the final coadd 

316 

317 You can retarget these subtasks if you wish. 

318 

319 Notes 

320 ----- 

321 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

322 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

323 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 

324 no debug variables of its own. Some of the subtasks may support debug 

325 variables. See the documentation for the subtasks for further information. 

326 

327 Examples 

328 -------- 

329 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 

330 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 

331 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 

332 inputs: a data reference to the tract patch and filter to be coadded, and 

333 a list of Warps to attempt to coadd. These are specified using ``--id`` and 

334 ``--selectId``, respectively: 

335 

336 .. code-block:: none 

337 

338 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

339 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 

340 

341 Only the Warps that cover the specified tract and patch will be coadded. 

342 A list of the available optional arguments can be obtained by calling 

343 ``assembleCoadd.py`` with the ``--help`` command line argument: 

344 

345 .. code-block:: none 

346 

347 assembleCoadd.py --help 

348 

349 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 

350 multi-band processing, we will generate the HSC-I & -R band coadds from 

351 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 

352 assuming that the lsst stack has been already set up, we must set up the 

353 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 

354 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 

355 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 

356 coadds, we must first 

357 

358 - processCcd 

359 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

360 - makeSkyMap 

361 - create a skymap that covers the area of the sky present in the raw exposures 

362 - makeCoaddTempExp 

363 - warp the individual calibrated exposures to the tangent plane of the coadd 

364 

365 We can perform all of these steps by running 

366 

367 .. code-block:: none 

368 

369 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

370 

371 This will produce warped exposures for each visit. To coadd the warped 

372 data, we call assembleCoadd.py as follows: 

373 

374 .. code-block:: none 

375 

376 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

377 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

378 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

379 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

380 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

381 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

382 --selectId visit=903988 ccd=24 

383 

384 that will process the HSC-I band data. The results are written in 

385 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

386 

387 You may also choose to run: 

388 

389 .. code-block:: none 

390 

391 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 

392 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 

393 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 

394 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 

395 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 

396 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 

397 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 

398 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 

399 

400 to generate the coadd for the HSC-R band if you are interested in 

401 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 

402 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 

403 rather than `AssembleCoaddTask` to make the coadd. 

404 """ 

405 ConfigClass = AssembleCoaddConfig 

406 _DefaultName = "assembleCoadd" 

407 

408 def __init__(self, *args, **kwargs): 

409 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)` 

410 if args: 410 ↛ 411line 410 didn't jump to line 411, because the condition on line 410 was never true

411 argNames = ["config", "name", "parentTask", "log"] 

412 kwargs.update({k: v for k, v in zip(argNames, args)}) 

413 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. " 

414 "PipelineTask will not take positional args" % argNames, FutureWarning) 

415 

416 super().__init__(**kwargs) 

417 self.makeSubtask("interpImage") 

418 self.makeSubtask("scaleZeroPoint") 

419 

420 if self.config.doMaskBrightObjects: 420 ↛ 421line 420 didn't jump to line 421, because the condition on line 420 was never true

421 mask = afwImage.Mask() 

422 try: 

423 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName) 

424 except pexExceptions.LsstCppException: 

425 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" % 

426 mask.getMaskPlaneDict().keys()) 

427 del mask 

428 

429 if self.config.doInputMap: 429 ↛ 430line 429 didn't jump to line 430, because the condition on line 429 was never true

430 self.makeSubtask("inputMapper") 

431 

432 self.warpType = self.config.warpType 

433 

434 @utils.inheritDoc(pipeBase.PipelineTask) 

435 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

436 # Docstring to be formatted with info from PipelineTask.runQuantum 

437 """ 

438 Notes 

439 ----- 

440 Assemble a coadd from a set of Warps. 

441 

442 PipelineTask (Gen3) entry point to Coadd a set of Warps. 

443 Analogous to `runDataRef`, it prepares all the data products to be 

444 passed to `run`, and processes the results before returning a struct 

445 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 

446 Therefore, its inputs are accessed subregion by subregion 

447 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 

448 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 

449 correspond to an update in `runDataRef` while both entry points 

450 are used. 

451 """ 

452 inputData = butlerQC.get(inputRefs) 

453 

454 # Construct skyInfo expected by run 

455 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it 

456 skyMap = inputData["skyMap"] 

457 outputDataId = butlerQC.quantum.dataId 

458 

459 inputData['skyInfo'] = makeSkyInfo(skyMap, 

460 tractId=outputDataId['tract'], 

461 patchId=outputDataId['patch']) 

462 

463 if self.config.doSelectVisits: 

464 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits']) 

465 else: 

466 warpRefList = inputData['inputWarps'] 

467 

468 # Perform same middle steps as `runDataRef` does 

469 inputs = self.prepareInputs(warpRefList) 

470 self.log.info("Found %d %s", len(inputs.tempExpRefList), 

471 self.getTempExpDatasetName(self.warpType)) 

472 if len(inputs.tempExpRefList) == 0: 

473 raise pipeBase.NoWorkFound("No coadd temporary exposures found") 

474 

475 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs) 

476 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList, 

477 inputs.weightList, supplementaryData=supplementaryData) 

478 

479 inputData.setdefault('brightObjectMask', None) 

480 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId) 

481 

482 if self.config.doWrite: 

483 butlerQC.put(retStruct, outputRefs) 

484 return retStruct 

485 

486 @pipeBase.timeMethod 

487 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None): 

488 """Assemble a coadd from a set of Warps. 

489 

490 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 

491 Compute weights to be applied to each Warp and 

492 find scalings to match the photometric zeropoint to a reference Warp. 

493 Assemble the Warps using `run`. Interpolate over NaNs and 

494 optionally write the coadd to disk. Return the coadded exposure. 

495 

496 Parameters 

497 ---------- 

498 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

499 Data reference defining the patch for coaddition and the 

500 reference Warp (if ``config.autoReference=False``). 

501 Used to access the following data products: 

502 - ``self.config.coaddName + "Coadd_skyMap"`` 

503 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 

504 - ``self.config.coaddName + "Coadd"`` 

505 selectDataList : `list` 

506 List of data references to Calexps. Data to be coadded will be 

507 selected from this list based on overlap with the patch defined 

508 by dataRef, grouped by visit, and converted to a list of data 

509 references to warps. 

510 warpRefList : `list` 

511 List of data references to Warps to be coadded. 

512 Note: `warpRefList` is just the new name for `tempExpRefList`. 

513 

514 Returns 

515 ------- 

516 retStruct : `lsst.pipe.base.Struct` 

517 Result struct with components: 

518 

519 - ``coaddExposure``: coadded exposure (``Exposure``). 

520 - ``nImage``: exposure count image (``Image``). 

521 """ 

522 if selectDataList and warpRefList: 522 ↛ 523line 522 didn't jump to line 523, because the condition on line 522 was never true

523 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, " 

524 "and which to use is ambiguous. Please pass only one.") 

525 

526 skyInfo = self.getSkyInfo(dataRef) 

527 if warpRefList is None: 

528 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList) 

529 if len(calExpRefList) == 0: 529 ↛ 530line 529 didn't jump to line 530, because the condition on line 529 was never true

530 self.log.warning("No exposures to coadd") 

531 return 

532 self.log.info("Coadding %d exposures", len(calExpRefList)) 

533 

534 warpRefList = self.getTempExpRefList(dataRef, calExpRefList) 

535 

536 inputData = self.prepareInputs(warpRefList) 

537 self.log.info("Found %d %s", len(inputData.tempExpRefList), 

538 self.getTempExpDatasetName(self.warpType)) 

539 if len(inputData.tempExpRefList) == 0: 539 ↛ 540line 539 didn't jump to line 540, because the condition on line 539 was never true

540 self.log.warning("No coadd temporary exposures found") 

541 return 

542 

543 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList) 

544 

545 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList, 

546 inputData.weightList, supplementaryData=supplementaryData) 

547 

548 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None 

549 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId) 

550 

551 if self.config.doWrite: 

552 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 552 ↛ 553line 552 didn't jump to line 553, because the condition on line 552 was never true

553 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType) 

554 else: 

555 coaddDatasetName = self.getCoaddDatasetName(self.warpType) 

556 self.log.info("Persisting %s", coaddDatasetName) 

557 dataRef.put(retStruct.coaddExposure, coaddDatasetName) 

558 if self.config.doNImage and retStruct.nImage is not None: 558 ↛ 559line 558 didn't jump to line 559, because the condition on line 558 was never true

559 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage') 

560 

561 return retStruct 

562 

563 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None): 

564 """Interpolate over missing data and mask bright stars. 

565 

566 Parameters 

567 ---------- 

568 coaddExposure : `lsst.afw.image.Exposure` 

569 The coadded exposure to process. 

570 dataRef : `lsst.daf.persistence.ButlerDataRef` 

571 Butler data reference for supplementary data. 

572 """ 

573 if self.config.doInterp: 573 ↛ 580line 573 didn't jump to line 580, because the condition on line 573 was never false

574 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA") 

575 # The variance must be positive; work around for DM-3201. 

576 varArray = coaddExposure.variance.array 

577 with numpy.errstate(invalid="ignore"): 

578 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf) 

579 

580 if self.config.doMaskBrightObjects: 580 ↛ 581line 580 didn't jump to line 581, because the condition on line 580 was never true

581 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId) 

582 

583 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

584 """Make additional inputs to run() specific to subclasses (Gen2) 

585 

586 Duplicates interface of `runDataRef` method 

587 Available to be implemented by subclasses only if they need the 

588 coadd dataRef for performing preliminary processing before 

589 assembling the coadd. 

590 

591 Parameters 

592 ---------- 

593 dataRef : `lsst.daf.persistence.ButlerDataRef` 

594 Butler data reference for supplementary data. 

595 selectDataList : `list` (optional) 

596 Optional List of data references to Calexps. 

597 warpRefList : `list` (optional) 

598 Optional List of data references to Warps. 

599 """ 

600 return pipeBase.Struct() 

601 

602 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

603 """Make additional inputs to run() specific to subclasses (Gen3) 

604 

605 Duplicates interface of `runQuantum` method. 

606 Available to be implemented by subclasses only if they need the 

607 coadd dataRef for performing preliminary processing before 

608 assembling the coadd. 

609 

610 Parameters 

611 ---------- 

612 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

613 Gen3 Butler object for fetching additional data products before 

614 running the Task specialized for quantum being processed 

615 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 

616 Attributes are the names of the connections describing input dataset types. 

617 Values are DatasetRefs that task consumes for corresponding dataset type. 

618 DataIds are guaranteed to match data objects in ``inputData``. 

619 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 

620 Attributes are the names of the connections describing output dataset types. 

621 Values are DatasetRefs that task is to produce 

622 for corresponding dataset type. 

623 """ 

624 return pipeBase.Struct() 

625 

626 def getTempExpRefList(self, patchRef, calExpRefList): 

627 """Generate list data references corresponding to warped exposures 

628 that lie within the patch to be coadded. 

629 

630 Parameters 

631 ---------- 

632 patchRef : `dataRef` 

633 Data reference for patch. 

634 calExpRefList : `list` 

635 List of data references for input calexps. 

636 

637 Returns 

638 ------- 

639 tempExpRefList : `list` 

640 List of Warp/CoaddTempExp data references. 

641 """ 

642 butler = patchRef.getButler() 

643 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType), 

644 self.getTempExpDatasetName(self.warpType)) 

645 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType), 

646 g, groupData.keys) for 

647 g in groupData.groups.keys()] 

648 return tempExpRefList 

649 

650 def prepareInputs(self, refList): 

651 """Prepare the input warps for coaddition by measuring the weight for 

652 each warp and the scaling for the photometric zero point. 

653 

654 Each Warp has its own photometric zeropoint and background variance. 

655 Before coadding these Warps together, compute a scale factor to 

656 normalize the photometric zeropoint and compute the weight for each Warp. 

657 

658 Parameters 

659 ---------- 

660 refList : `list` 

661 List of data references to tempExp 

662 

663 Returns 

664 ------- 

665 result : `lsst.pipe.base.Struct` 

666 Result struct with components: 

667 

668 - ``tempExprefList``: `list` of data references to tempExp. 

669 - ``weightList``: `list` of weightings. 

670 - ``imageScalerList``: `list` of image scalers. 

671 """ 

672 statsCtrl = afwMath.StatisticsControl() 

673 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

674 statsCtrl.setNumIter(self.config.clipIter) 

675 statsCtrl.setAndMask(self.getBadPixelMask()) 

676 statsCtrl.setNanSafe(True) 

677 # compute tempExpRefList: a list of tempExpRef that actually exist 

678 # and weightList: a list of the weight of the associated coadd tempExp 

679 # and imageScalerList: a list of scale factors for the associated coadd tempExp 

680 tempExpRefList = [] 

681 weightList = [] 

682 imageScalerList = [] 

683 tempExpName = self.getTempExpDatasetName(self.warpType) 

684 for tempExpRef in refList: 

685 # Gen3's DeferredDatasetHandles are guaranteed to exist and 

686 # therefore have no datasetExists() method 

687 if not isinstance(tempExpRef, DeferredDatasetHandle): 687 ↛ 692line 687 didn't jump to line 692, because the condition on line 687 was never false

688 if not tempExpRef.datasetExists(tempExpName): 

689 self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId) 

690 continue 

691 

692 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True) 

693 # Ignore any input warp that is empty of data 

694 if numpy.isnan(tempExp.image.array).all(): 694 ↛ 695line 694 didn't jump to line 695, because the condition on line 694 was never true

695 continue 

696 maskedImage = tempExp.getMaskedImage() 

697 imageScaler = self.scaleZeroPoint.computeImageScaler( 

698 exposure=tempExp, 

699 dataRef=tempExpRef, 

700 ) 

701 try: 

702 imageScaler.scaleMaskedImage(maskedImage) 

703 except Exception as e: 

704 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e) 

705 continue 

706 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(), 

707 afwMath.MEANCLIP, statsCtrl) 

708 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP) 

709 weight = 1.0 / float(meanVar) 

710 if not numpy.isfinite(weight): 710 ↛ 711line 710 didn't jump to line 711, because the condition on line 710 was never true

711 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId) 

712 continue 

713 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight) 

714 

715 del maskedImage 

716 del tempExp 

717 

718 tempExpRefList.append(tempExpRef) 

719 weightList.append(weight) 

720 imageScalerList.append(imageScaler) 

721 

722 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList, 

723 imageScalerList=imageScalerList) 

724 

725 def prepareStats(self, mask=None): 

726 """Prepare the statistics for coadding images. 

727 

728 Parameters 

729 ---------- 

730 mask : `int`, optional 

731 Bit mask value to exclude from coaddition. 

732 

733 Returns 

734 ------- 

735 stats : `lsst.pipe.base.Struct` 

736 Statistics structure with the following fields: 

737 

738 - ``statsCtrl``: Statistics control object for coadd 

739 (`lsst.afw.math.StatisticsControl`) 

740 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 

741 """ 

742 if mask is None: 

743 mask = self.getBadPixelMask() 

744 statsCtrl = afwMath.StatisticsControl() 

745 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

746 statsCtrl.setNumIter(self.config.clipIter) 

747 statsCtrl.setAndMask(mask) 

748 statsCtrl.setNanSafe(True) 

749 statsCtrl.setWeighted(True) 

750 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance) 

751 for plane, threshold in self.config.maskPropagationThresholds.items(): 

752 bit = afwImage.Mask.getMaskPlane(plane) 

753 statsCtrl.setMaskPropagationThreshold(bit, threshold) 

754 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) 

755 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags) 

756 

757 @pipeBase.timeMethod 

758 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

759 altMaskList=None, mask=None, supplementaryData=None): 

760 """Assemble a coadd from input warps 

761 

762 Assemble the coadd using the provided list of coaddTempExps. Since 

763 the full coadd covers a patch (a large area), the assembly is 

764 performed over small areas on the image at a time in order to 

765 conserve memory usage. Iterate over subregions within the outer 

766 bbox of the patch using `assembleSubregion` to stack the corresponding 

767 subregions from the coaddTempExps with the statistic specified. 

768 Set the edge bits the coadd mask based on the weight map. 

769 

770 Parameters 

771 ---------- 

772 skyInfo : `lsst.pipe.base.Struct` 

773 Struct with geometric information about the patch. 

774 tempExpRefList : `list` 

775 List of data references to Warps (previously called CoaddTempExps). 

776 imageScalerList : `list` 

777 List of image scalers. 

778 weightList : `list` 

779 List of weights 

780 altMaskList : `list`, optional 

781 List of alternate masks to use rather than those stored with 

782 tempExp. 

783 mask : `int`, optional 

784 Bit mask value to exclude from coaddition. 

785 supplementaryData : lsst.pipe.base.Struct, optional 

786 Struct with additional data products needed to assemble coadd. 

787 Only used by subclasses that implement `makeSupplementaryData` 

788 and override `run`. 

789 

790 Returns 

791 ------- 

792 result : `lsst.pipe.base.Struct` 

793 Result struct with components: 

794 

795 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 

796 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 

797 - ``inputMap``: bit-wise map of inputs, if requested. 

798 - ``warpRefList``: input list of refs to the warps ( 

799 ``lsst.daf.butler.DeferredDatasetHandle`` or 

800 ``lsst.daf.persistence.ButlerDataRef``) 

801 (unmodified) 

802 - ``imageScalerList``: input list of image scalers (unmodified) 

803 - ``weightList``: input list of weights (unmodified) 

804 """ 

805 tempExpName = self.getTempExpDatasetName(self.warpType) 

806 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName) 

807 stats = self.prepareStats(mask=mask) 

808 

809 if altMaskList is None: 

810 altMaskList = [None]*len(tempExpRefList) 

811 

812 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) 

813 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib()) 

814 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs()) 

815 self.assembleMetadata(coaddExposure, tempExpRefList, weightList) 

816 coaddMaskedImage = coaddExposure.getMaskedImage() 

817 subregionSizeArr = self.config.subregionSize 

818 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

819 # if nImage is requested, create a zero one which can be passed to assembleSubregion 

820 if self.config.doNImage: 820 ↛ 821line 820 didn't jump to line 821, because the condition on line 820 was never true

821 nImage = afwImage.ImageU(skyInfo.bbox) 

822 else: 

823 nImage = None 

824 # If inputMap is requested, create the initial version that can be masked in 

825 # assembleSubregion. 

826 if self.config.doInputMap: 826 ↛ 827line 826 didn't jump to line 827, because the condition on line 826 was never true

827 self.inputMapper.build_ccd_input_map(skyInfo.bbox, 

828 skyInfo.wcs, 

829 coaddExposure.getInfo().getCoaddInputs().ccds) 

830 

831 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize): 

832 try: 

833 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList, 

834 weightList, altMaskList, stats.flags, stats.ctrl, 

835 nImage=nImage) 

836 except Exception as e: 

837 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e) 

838 

839 # If inputMap is requested, we must finalize the map after the accumulation. 

840 if self.config.doInputMap: 840 ↛ 841line 840 didn't jump to line 841, because the condition on line 840 was never true

841 self.inputMapper.finalize_ccd_input_map_mask() 

842 inputMap = self.inputMapper.ccd_input_map 

843 else: 

844 inputMap = None 

845 

846 self.setInexactPsf(coaddMaskedImage.getMask()) 

847 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies 

848 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field). 

849 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) 

850 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage, 

851 warpRefList=tempExpRefList, imageScalerList=imageScalerList, 

852 weightList=weightList, inputMap=inputMap) 

853 

854 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList): 

855 """Set the metadata for the coadd. 

856 

857 This basic implementation sets the filter from the first input. 

858 

859 Parameters 

860 ---------- 

861 coaddExposure : `lsst.afw.image.Exposure` 

862 The target exposure for the coadd. 

863 tempExpRefList : `list` 

864 List of data references to tempExp. 

865 weightList : `list` 

866 List of weights. 

867 """ 

868 assert len(tempExpRefList) == len(weightList), "Length mismatch" 

869 tempExpName = self.getTempExpDatasetName(self.warpType) 

870 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata 

871 # (and we need more than just the PropertySet that contains the header), which is not possible 

872 # with the current butler (see #2777). 

873 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1)) 

874 

875 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 875 ↛ 877line 875 didn't jump to line 877, because the condition on line 875 was never true

876 # Gen 3 API 

877 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList] 

878 else: 

879 # Gen 2 API. Delete this when Gen 2 retired 

880 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True) 

881 for tempExpRef in tempExpRefList] 

882 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList) 

883 

884 # Set the coadd FilterLabel to the band of the first input exposure: 

885 # Coadds are calibrated, so the physical label is now meaningless. 

886 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel)) 

887 coaddInputs = coaddExposure.getInfo().getCoaddInputs() 

888 coaddInputs.ccds.reserve(numCcds) 

889 coaddInputs.visits.reserve(len(tempExpList)) 

890 

891 for tempExp, weight in zip(tempExpList, weightList): 

892 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) 

893 

894 if self.config.doUsePsfMatchedPolygons: 

895 self.shrinkValidPolygons(coaddInputs) 

896 

897 coaddInputs.visits.sort() 

898 if self.warpType == "psfMatched": 

899 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by 

900 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf. 

901 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf 

902 # having the maximum width (sufficient because square) 

903 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList] 

904 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList] 

905 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))] 

906 else: 

907 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(), 

908 self.config.coaddPsf.makeControl()) 

909 coaddExposure.setPsf(psf) 

910 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), 

911 coaddExposure.getWcs()) 

912 coaddExposure.getInfo().setApCorrMap(apCorrMap) 

913 if self.config.doAttachTransmissionCurve: 

914 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds) 

915 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve) 

916 

917 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, 

918 altMaskList, statsFlags, statsCtrl, nImage=None): 

919 """Assemble the coadd for a sub-region. 

920 

921 For each coaddTempExp, check for (and swap in) an alternative mask 

922 if one is passed. Remove mask planes listed in 

923 `config.removeMaskPlanes`. Finally, stack the actual exposures using 

924 `lsst.afw.math.statisticsStack` with the statistic specified by 

925 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 

926 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 

927 an N-sigma clipped mean where N and iterations are specified by 

928 statsCtrl. Assign the stacked subregion back to the coadd. 

929 

930 Parameters 

931 ---------- 

932 coaddExposure : `lsst.afw.image.Exposure` 

933 The target exposure for the coadd. 

934 bbox : `lsst.geom.Box` 

935 Sub-region to coadd. 

936 tempExpRefList : `list` 

937 List of data reference to tempExp. 

938 imageScalerList : `list` 

939 List of image scalers. 

940 weightList : `list` 

941 List of weights. 

942 altMaskList : `list` 

943 List of alternate masks to use rather than those stored with 

944 tempExp, or None. Each element is dict with keys = mask plane 

945 name to which to add the spans. 

946 statsFlags : `lsst.afw.math.Property` 

947 Property object for statistic for coadd. 

948 statsCtrl : `lsst.afw.math.StatisticsControl` 

949 Statistics control object for coadd. 

950 nImage : `lsst.afw.image.ImageU`, optional 

951 Keeps track of exposure count for each pixel. 

952 """ 

953 self.log.debug("Computing coadd over %s", bbox) 

954 tempExpName = self.getTempExpDatasetName(self.warpType) 

955 coaddExposure.mask.addMaskPlane("REJECTED") 

956 coaddExposure.mask.addMaskPlane("CLIPPED") 

957 coaddExposure.mask.addMaskPlane("SENSOR_EDGE") 

958 maskMap = self.setRejectedMaskMapping(statsCtrl) 

959 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

960 maskedImageList = [] 

961 if nImage is not None: 961 ↛ 962line 961 didn't jump to line 962, because the condition on line 961 was never true

962 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight()) 

963 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList): 

964 

965 if isinstance(tempExpRef, DeferredDatasetHandle): 965 ↛ 967line 965 didn't jump to line 967, because the condition on line 965 was never true

966 # Gen 3 API 

967 exposure = tempExpRef.get(parameters={'bbox': bbox}) 

968 else: 

969 # Gen 2 API. Delete this when Gen 2 retired 

970 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox) 

971 

972 maskedImage = exposure.getMaskedImage() 

973 mask = maskedImage.getMask() 

974 if altMask is not None: 

975 self.applyAltMaskPlanes(mask, altMask) 

976 imageScaler.scaleMaskedImage(maskedImage) 

977 

978 # Add 1 for each pixel which is not excluded by the exclude mask. 

979 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack. 

980 if nImage is not None: 980 ↛ 981line 980 didn't jump to line 981, because the condition on line 980 was never true

981 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1 

982 if self.config.removeMaskPlanes: 982 ↛ 984line 982 didn't jump to line 984, because the condition on line 982 was never false

983 self.removeMaskPlanes(maskedImage) 

984 maskedImageList.append(maskedImage) 

985 

986 if self.config.doInputMap: 986 ↛ 987line 986 didn't jump to line 987, because the condition on line 986 was never true

987 visit = exposure.getInfo().getCoaddInputs().visits[0].getId() 

988 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask()) 

989 

990 with self.timer("stack"): 

991 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, 

992 clipped, # also set output to CLIPPED if sigma-clipped 

993 maskMap) 

994 coaddExposure.maskedImage.assign(coaddSubregion, bbox) 

995 if nImage is not None: 995 ↛ 996line 995 didn't jump to line 996, because the condition on line 995 was never true

996 nImage.assign(subNImage, bbox) 

997 

998 def removeMaskPlanes(self, maskedImage): 

999 """Unset the mask of an image for mask planes specified in the config. 

1000 

1001 Parameters 

1002 ---------- 

1003 maskedImage : `lsst.afw.image.MaskedImage` 

1004 The masked image to be modified. 

1005 """ 

1006 mask = maskedImage.getMask() 

1007 for maskPlane in self.config.removeMaskPlanes: 

1008 try: 

1009 mask &= ~mask.getPlaneBitMask(maskPlane) 

1010 except pexExceptions.InvalidParameterError: 

1011 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.", 

1012 maskPlane) 

1013 

1014 @staticmethod 

1015 def setRejectedMaskMapping(statsCtrl): 

1016 """Map certain mask planes of the warps to new planes for the coadd. 

1017 

1018 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 

1019 or CLIPPED, set it to REJECTED on the coadd. 

1020 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 

1021 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 

1022 

1023 Parameters 

1024 ---------- 

1025 statsCtrl : `lsst.afw.math.StatisticsControl` 

1026 Statistics control object for coadd 

1027 

1028 Returns 

1029 ------- 

1030 maskMap : `list` of `tuple` of `int` 

1031 A list of mappings of mask planes of the warped exposures to 

1032 mask planes of the coadd. 

1033 """ 

1034 edge = afwImage.Mask.getPlaneBitMask("EDGE") 

1035 noData = afwImage.Mask.getPlaneBitMask("NO_DATA") 

1036 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED") 

1037 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped) 

1038 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")), 

1039 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")), 

1040 (clipped, clipped)] 

1041 return maskMap 

1042 

1043 def applyAltMaskPlanes(self, mask, altMaskSpans): 

1044 """Apply in place alt mask formatted as SpanSets to a mask. 

1045 

1046 Parameters 

1047 ---------- 

1048 mask : `lsst.afw.image.Mask` 

1049 Original mask. 

1050 altMaskSpans : `dict` 

1051 SpanSet lists to apply. Each element contains the new mask 

1052 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 

1053 and list of SpanSets to apply to the mask. 

1054 

1055 Returns 

1056 ------- 

1057 mask : `lsst.afw.image.Mask` 

1058 Updated mask. 

1059 """ 

1060 if self.config.doUsePsfMatchedPolygons: 

1061 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1061 ↛ 1069line 1061 didn't jump to line 1069, because the condition on line 1061 was never false

1062 # Clear away any other masks outside the validPolygons. These pixels are no longer 

1063 # contributing to inexact PSFs, and will still be rejected because of NO_DATA 

1064 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble 

1065 # This mask-clearing step must only occur *before* applying the new masks below 

1066 for spanSet in altMaskSpans['NO_DATA']: 

1067 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask()) 

1068 

1069 for plane, spanSetList in altMaskSpans.items(): 

1070 maskClipValue = mask.addMaskPlane(plane) 

1071 for spanSet in spanSetList: 

1072 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue) 

1073 return mask 

1074 

1075 def shrinkValidPolygons(self, coaddInputs): 

1076 """Shrink coaddInputs' ccds' ValidPolygons in place. 

1077 

1078 Either modify each ccd's validPolygon in place, or if CoaddInputs 

1079 does not have a validPolygon, create one from its bbox. 

1080 

1081 Parameters 

1082 ---------- 

1083 coaddInputs : `lsst.afw.image.coaddInputs` 

1084 Original mask. 

1085 

1086 """ 

1087 for ccd in coaddInputs.ccds: 

1088 polyOrig = ccd.getValidPolygon() 

1089 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox() 

1090 validPolyBBox.grow(-self.config.matchingKernelSize//2) 

1091 if polyOrig: 1091 ↛ 1092line 1091 didn't jump to line 1092, because the condition on line 1091 was never true

1092 validPolygon = polyOrig.intersectionSingle(validPolyBBox) 

1093 else: 

1094 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox)) 

1095 ccd.setValidPolygon(validPolygon) 

1096 

1097 def readBrightObjectMasks(self, dataRef): 

1098 """Retrieve the bright object masks. 

1099 

1100 Returns None on failure. 

1101 

1102 Parameters 

1103 ---------- 

1104 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1105 A Butler dataRef. 

1106 

1107 Returns 

1108 ------- 

1109 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

1110 Bright object mask from the Butler object, or None if it cannot 

1111 be retrieved. 

1112 """ 

1113 try: 

1114 return dataRef.get(datasetType="brightObjectMask", immediate=True) 

1115 except Exception as e: 

1116 self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e) 

1117 return None 

1118 

1119 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None): 

1120 """Set the bright object masks. 

1121 

1122 Parameters 

1123 ---------- 

1124 exposure : `lsst.afw.image.Exposure` 

1125 Exposure under consideration. 

1126 dataId : `lsst.daf.persistence.dataId` 

1127 Data identifier dict for patch. 

1128 brightObjectMasks : `lsst.afw.table` 

1129 Table of bright objects to mask. 

1130 """ 

1131 

1132 if brightObjectMasks is None: 

1133 self.log.warning("Unable to apply bright object mask: none supplied") 

1134 return 

1135 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId) 

1136 mask = exposure.getMaskedImage().getMask() 

1137 wcs = exposure.getWcs() 

1138 plateScale = wcs.getPixelScale().asArcseconds() 

1139 

1140 for rec in brightObjectMasks: 

1141 center = geom.PointI(wcs.skyToPixel(rec.getCoord())) 

1142 if rec["type"] == "box": 

1143 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"]) 

1144 width = rec["width"].asArcseconds()/plateScale # convert to pixels 

1145 height = rec["height"].asArcseconds()/plateScale # convert to pixels 

1146 

1147 halfSize = geom.ExtentI(0.5*width, 0.5*height) 

1148 bbox = geom.Box2I(center - halfSize, center + halfSize) 

1149 

1150 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)), 

1151 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height))) 

1152 spans = afwGeom.SpanSet(bbox) 

1153 elif rec["type"] == "circle": 

1154 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels 

1155 spans = afwGeom.SpanSet.fromShape(radius, offset=center) 

1156 else: 

1157 self.log.warning("Unexpected region type %s at %s", rec["type"], center) 

1158 continue 

1159 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask) 

1160 

1161 def setInexactPsf(self, mask): 

1162 """Set INEXACT_PSF mask plane. 

1163 

1164 If any of the input images isn't represented in the coadd (due to 

1165 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 

1166 these pixels. 

1167 

1168 Parameters 

1169 ---------- 

1170 mask : `lsst.afw.image.Mask` 

1171 Coadded exposure's mask, modified in-place. 

1172 """ 

1173 mask.addMaskPlane("INEXACT_PSF") 

1174 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF") 

1175 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous) 

1176 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd 

1177 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks 

1178 array = mask.getArray() 

1179 selected = array & (sensorEdge | clipped | rejected) > 0 

1180 array[selected] |= inexactPsf 

1181 

1182 @classmethod 

1183 def _makeArgumentParser(cls): 

1184 """Create an argument parser. 

1185 """ 

1186 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1187 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" 

1188 + cls.ConfigClass().warpType + "Warp", 

1189 help="data ID, e.g. --id tract=12345 patch=1,2", 

1190 ContainerClass=AssembleCoaddDataIdContainer) 

1191 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9", 

1192 ContainerClass=SelectDataIdContainer) 

1193 return parser 

1194 

1195 @staticmethod 

1196 def _subBBoxIter(bbox, subregionSize): 

1197 """Iterate over subregions of a bbox. 

1198 

1199 Parameters 

1200 ---------- 

1201 bbox : `lsst.geom.Box2I` 

1202 Bounding box over which to iterate. 

1203 subregionSize: `lsst.geom.Extent2I` 

1204 Size of sub-bboxes. 

1205 

1206 Yields 

1207 ------ 

1208 subBBox : `lsst.geom.Box2I` 

1209 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 

1210 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 

1211 the edges of ``bbox``, but it will never be empty. 

1212 """ 

1213 if bbox.isEmpty(): 1213 ↛ 1214line 1213 didn't jump to line 1214, because the condition on line 1213 was never true

1214 raise RuntimeError("bbox %s is empty" % (bbox,)) 

1215 if subregionSize[0] < 1 or subregionSize[1] < 1: 1215 ↛ 1216line 1215 didn't jump to line 1216, because the condition on line 1215 was never true

1216 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

1217 

1218 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

1219 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

1220 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

1221 subBBox.clip(bbox) 

1222 if subBBox.isEmpty(): 1222 ↛ 1223line 1222 didn't jump to line 1223, because the condition on line 1222 was never true

1223 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

1224 "colShift=%s, rowShift=%s" % 

1225 (bbox, subregionSize, colShift, rowShift)) 

1226 yield subBBox 

1227 

1228 def filterWarps(self, inputs, goodVisits): 

1229 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit 

1230 

1231 Parameters 

1232 ---------- 

1233 inputs : list 

1234 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit 

1235 goodVisit : `dict` 

1236 Dictionary with good visitIds as the keys. Value ignored. 

1237 

1238 Returns: 

1239 -------- 

1240 filteredInputs : `list` 

1241 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef` 

1242 """ 

1243 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs} 

1244 filteredInputs = [] 

1245 for visit in goodVisits.keys(): 

1246 filteredInputs.append(inputWarpDict[visit]) 

1247 return filteredInputs 

1248 

1249 

1250class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer): 

1251 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 

1252 """ 

1253 

1254 def makeDataRefList(self, namespace): 

1255 """Make self.refList from self.idList. 

1256 

1257 Parameters 

1258 ---------- 

1259 namespace 

1260 Results of parsing command-line (with ``butler`` and ``log`` elements). 

1261 """ 

1262 datasetType = namespace.config.coaddName + "Coadd" 

1263 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level) 

1264 

1265 for dataId in self.idList: 

1266 # tract and patch are required 

1267 for key in keysCoadd: 

1268 if key not in dataId: 

1269 raise RuntimeError("--id must include " + key) 

1270 

1271 dataRef = namespace.butler.dataRef( 

1272 datasetType=datasetType, 

1273 dataId=dataId, 

1274 ) 

1275 self.refList.append(dataRef) 

1276 

1277 

1278def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask): 

1279 """Function to count the number of pixels with a specific mask in a 

1280 footprint. 

1281 

1282 Find the intersection of mask & footprint. Count all pixels in the mask 

1283 that are in the intersection that have bitmask set but do not have 

1284 ignoreMask set. Return the count. 

1285 

1286 Parameters 

1287 ---------- 

1288 mask : `lsst.afw.image.Mask` 

1289 Mask to define intersection region by. 

1290 footprint : `lsst.afw.detection.Footprint` 

1291 Footprint to define the intersection region by. 

1292 bitmask 

1293 Specific mask that we wish to count the number of occurances of. 

1294 ignoreMask 

1295 Pixels to not consider. 

1296 

1297 Returns 

1298 ------- 

1299 result : `int` 

1300 Count of number of pixels in footprint with specified mask. 

1301 """ 

1302 bbox = footprint.getBBox() 

1303 bbox.clip(mask.getBBox(afwImage.PARENT)) 

1304 fp = afwImage.Mask(bbox) 

1305 subMask = mask.Factory(mask, bbox, afwImage.PARENT) 

1306 footprint.spans.setMask(fp, bitmask) 

1307 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0, 

1308 (subMask.getArray() & ignoreMask) == 0).sum() 

1309 

1310 

1311class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections): 

1312 """Configuration parameters for the SafeClipAssembleCoaddTask. 

1313 """ 

1314 clipDetection = pexConfig.ConfigurableField( 

1315 target=SourceDetectionTask, 

1316 doc="Detect sources on difference between unclipped and clipped coadd") 

1317 minClipFootOverlap = pexConfig.Field( 

1318 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped", 

1319 dtype=float, 

1320 default=0.6 

1321 ) 

1322 minClipFootOverlapSingle = pexConfig.Field( 

1323 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be " 

1324 "clipped when only one visit overlaps", 

1325 dtype=float, 

1326 default=0.5 

1327 ) 

1328 minClipFootOverlapDouble = pexConfig.Field( 

1329 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be " 

1330 "clipped when two visits overlap", 

1331 dtype=float, 

1332 default=0.45 

1333 ) 

1334 maxClipFootOverlapDouble = pexConfig.Field( 

1335 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when " 

1336 "considering two visits", 

1337 dtype=float, 

1338 default=0.15 

1339 ) 

1340 minBigOverlap = pexConfig.Field( 

1341 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits " 

1342 "when labeling clipped footprints", 

1343 dtype=int, 

1344 default=100 

1345 ) 

1346 

1347 def setDefaults(self): 

1348 """Set default values for clipDetection. 

1349 

1350 Notes 

1351 ----- 

1352 The numeric values for these configuration parameters were 

1353 empirically determined, future work may further refine them. 

1354 """ 

1355 AssembleCoaddConfig.setDefaults(self) 

1356 self.clipDetection.doTempLocalBackground = False 

1357 self.clipDetection.reEstimateBackground = False 

1358 self.clipDetection.returnOriginalFootprints = False 

1359 self.clipDetection.thresholdPolarity = "both" 

1360 self.clipDetection.thresholdValue = 2 

1361 self.clipDetection.nSigmaToGrow = 2 

1362 self.clipDetection.minPixels = 4 

1363 self.clipDetection.isotropicGrow = True 

1364 self.clipDetection.thresholdType = "pixel_stdev" 

1365 self.sigmaClip = 1.5 

1366 self.clipIter = 3 

1367 self.statistic = "MEAN" 

1368 

1369 def validate(self): 

1370 if self.doSigmaClip: 

1371 log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 

1372 "Ignoring doSigmaClip.") 

1373 self.doSigmaClip = False 

1374 if self.statistic != "MEAN": 

1375 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 

1376 "(%s chosen). Please set statistic to MEAN." 

1377 % (self.statistic)) 

1378 AssembleCoaddTask.ConfigClass.validate(self) 

1379 

1380 

1381class SafeClipAssembleCoaddTask(AssembleCoaddTask): 

1382 """Assemble a coadded image from a set of coadded temporary exposures, 

1383 being careful to clip & flag areas with potential artifacts. 

1384 

1385 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1386 we clip outliers). The problem with doing this is that when computing the 

1387 coadd PSF at a given location, individual visit PSFs from visits with 

1388 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1389 In this task, we correct for this behavior by creating a new 

1390 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 

1391 coaddTempExps and the final coadd where 

1392 

1393 i. difference imaging suggests that there is an outlier and 

1394 ii. this outlier appears on only one or two images. 

1395 

1396 Such regions will not contribute to the final coadd. Furthermore, any 

1397 routine to determine the coadd PSF can now be cognizant of clipped regions. 

1398 Note that the algorithm implemented by this task is preliminary and works 

1399 correctly for HSC data. Parameter modifications and or considerable 

1400 redesigning of the algorithm is likley required for other surveys. 

1401 

1402 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 

1403 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 

1404 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 

1405 if you wish. 

1406 

1407 Notes 

1408 ----- 

1409 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

1410 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 

1411 see `baseDebug` for more about ``debug.py`` files. 

1412 `SafeClipAssembleCoaddTask` has no debug variables of its own. 

1413 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 

1414 variables. See the documetation for `SourceDetectionTask` "clipDetection" 

1415 for further information. 

1416 

1417 Examples 

1418 -------- 

1419 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 

1420 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 

1421 running assembleCoadd.py *without* the flag '--legacyCoadd'. 

1422 

1423 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

1424 and filter to be coadded (specified using 

1425 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

1426 along with a list of coaddTempExps to attempt to coadd (specified using 

1427 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

1428 Only the coaddTempExps that cover the specified tract and patch will be 

1429 coadded. A list of the available optional arguments can be obtained by 

1430 calling assembleCoadd.py with the --help command line argument: 

1431 

1432 .. code-block:: none 

1433 

1434 assembleCoadd.py --help 

1435 

1436 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 

1437 context of multi-band processing, we will generate the HSC-I & -R band 

1438 coadds from HSC engineering test data provided in the ci_hsc package. 

1439 To begin, assuming that the lsst stack has been already set up, we must 

1440 set up the obs_subaru and ci_hsc packages. This defines the environment 

1441 variable $CI_HSC_DIR and points at the location of the package. The raw 

1442 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 

1443 the coadds, we must first 

1444 

1445 - ``processCcd`` 

1446 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

1447 - ``makeSkyMap`` 

1448 create a skymap that covers the area of the sky present in the raw exposures 

1449 - ``makeCoaddTempExp`` 

1450 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 

1451 

1452 We can perform all of these steps by running 

1453 

1454 .. code-block:: none 

1455 

1456 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

1457 

1458 This will produce warped coaddTempExps for each visit. To coadd the 

1459 warped data, we call ``assembleCoadd.py`` as follows: 

1460 

1461 .. code-block:: none 

1462 

1463 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

1464 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

1465 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

1466 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

1467 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

1468 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

1469 --selectId visit=903988 ccd=24 

1470 

1471 This will process the HSC-I band data. The results are written in 

1472 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

1473 

1474 You may also choose to run: 

1475 

1476 .. code-block:: none 

1477 

1478 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 

1479 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 

1480 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 

1481 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 

1482 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 

1483 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 

1484 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 

1485 --selectId visit=903346 ccd=12 

1486 

1487 to generate the coadd for the HSC-R band if you are interested in following 

1488 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 

1489 """ 

1490 ConfigClass = SafeClipAssembleCoaddConfig 

1491 _DefaultName = "safeClipAssembleCoadd" 

1492 

1493 def __init__(self, *args, **kwargs): 

1494 AssembleCoaddTask.__init__(self, *args, **kwargs) 

1495 schema = afwTable.SourceTable.makeMinimalSchema() 

1496 self.makeSubtask("clipDetection", schema=schema) 

1497 

1498 @utils.inheritDoc(AssembleCoaddTask) 

1499 @pipeBase.timeMethod 

1500 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs): 

1501 """Assemble the coadd for a region. 

1502 

1503 Compute the difference of coadds created with and without outlier 

1504 rejection to identify coadd pixels that have outlier values in some 

1505 individual visits. 

1506 Detect clipped regions on the difference image and mark these regions 

1507 on the one or two individual coaddTempExps where they occur if there 

1508 is significant overlap between the clipped region and a source. This 

1509 leaves us with a set of footprints from the difference image that have 

1510 been identified as having occured on just one or two individual visits. 

1511 However, these footprints were generated from a difference image. It 

1512 is conceivable for a large diffuse source to have become broken up 

1513 into multiple footprints acrosss the coadd difference in this process. 

1514 Determine the clipped region from all overlapping footprints from the 

1515 detected sources in each visit - these are big footprints. 

1516 Combine the small and big clipped footprints and mark them on a new 

1517 bad mask plane. 

1518 Generate the coadd using `AssembleCoaddTask.run` without outlier 

1519 removal. Clipped footprints will no longer make it into the coadd 

1520 because they are marked in the new bad mask plane. 

1521 

1522 Notes 

1523 ----- 

1524 args and kwargs are passed but ignored in order to match the call 

1525 signature expected by the parent task. 

1526 """ 

1527 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList) 

1528 mask = exp.getMaskedImage().getMask() 

1529 mask.addMaskPlane("CLIPPED") 

1530 

1531 result = self.detectClip(exp, tempExpRefList) 

1532 

1533 self.log.info('Found %d clipped objects', len(result.clipFootprints)) 

1534 

1535 maskClipValue = mask.getPlaneBitMask("CLIPPED") 

1536 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1537 # Append big footprints from individual Warps to result.clipSpans 

1538 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices, 

1539 result.detectionFootprints, maskClipValue, maskDetValue, 

1540 exp.getBBox()) 

1541 # Create mask of the current clipped footprints 

1542 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT)) 

1543 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue) 

1544 

1545 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT)) 

1546 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue) 

1547 maskClip |= maskClipBig 

1548 

1549 # Assemble coadd from base class, but ignoring CLIPPED pixels 

1550 badMaskPlanes = self.config.badMaskPlanes[:] 

1551 badMaskPlanes.append("CLIPPED") 

1552 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

1553 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

1554 result.clipSpans, mask=badPixelMask) 

1555 

1556 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList): 

1557 """Return an exposure that contains the difference between unclipped 

1558 and clipped coadds. 

1559 

1560 Generate a difference image between clipped and unclipped coadds. 

1561 Compute the difference image by subtracting an outlier-clipped coadd 

1562 from an outlier-unclipped coadd. Return the difference image. 

1563 

1564 Parameters 

1565 ---------- 

1566 skyInfo : `lsst.pipe.base.Struct` 

1567 Patch geometry information, from getSkyInfo 

1568 tempExpRefList : `list` 

1569 List of data reference to tempExp 

1570 imageScalerList : `list` 

1571 List of image scalers 

1572 weightList : `list` 

1573 List of weights 

1574 

1575 Returns 

1576 ------- 

1577 exp : `lsst.afw.image.Exposure` 

1578 Difference image of unclipped and clipped coadd wrapped in an Exposure 

1579 """ 

1580 config = AssembleCoaddConfig() 

1581 # getattr necessary because subtasks do not survive Config.toDict() 

1582 # exclude connections because the class of self.config.connections is not 

1583 # the same as AssembleCoaddConfig.connections, and the connections are not 

1584 # needed to run this task anyway. 

1585 configIntersection = {k: getattr(self.config, k) 

1586 for k, v in self.config.toDict().items() 

1587 if (k in config.keys() and k != "connections")} 

1588 configIntersection['doInputMap'] = False 

1589 configIntersection['doNImage'] = False 

1590 config.update(**configIntersection) 

1591 

1592 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign 

1593 config.statistic = 'MEAN' 

1594 task = AssembleCoaddTask(config=config) 

1595 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1596 

1597 config.statistic = 'MEANCLIP' 

1598 task = AssembleCoaddTask(config=config) 

1599 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure 

1600 

1601 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage()) 

1602 coaddDiff -= coaddClip.getMaskedImage() 

1603 exp = afwImage.ExposureF(coaddDiff) 

1604 exp.setPsf(coaddMean.getPsf()) 

1605 return exp 

1606 

1607 def detectClip(self, exp, tempExpRefList): 

1608 """Detect clipped regions on an exposure and set the mask on the 

1609 individual tempExp masks. 

1610 

1611 Detect footprints in the difference image after smoothing the 

1612 difference image with a Gaussian kernal. Identify footprints that 

1613 overlap with one or two input ``coaddTempExps`` by comparing the 

1614 computed overlap fraction to thresholds set in the config. A different 

1615 threshold is applied depending on the number of overlapping visits 

1616 (restricted to one or two). If the overlap exceeds the thresholds, 

1617 the footprint is considered "CLIPPED" and is marked as such on the 

1618 coaddTempExp. Return a struct with the clipped footprints, the indices 

1619 of the ``coaddTempExps`` that end up overlapping with the clipped 

1620 footprints, and a list of new masks for the ``coaddTempExps``. 

1621 

1622 Parameters 

1623 ---------- 

1624 exp : `lsst.afw.image.Exposure` 

1625 Exposure to run detection on. 

1626 tempExpRefList : `list` 

1627 List of data reference to tempExp. 

1628 

1629 Returns 

1630 ------- 

1631 result : `lsst.pipe.base.Struct` 

1632 Result struct with components: 

1633 

1634 - ``clipFootprints``: list of clipped footprints. 

1635 - ``clipIndices``: indices for each ``clippedFootprint`` in 

1636 ``tempExpRefList``. 

1637 - ``clipSpans``: List of dictionaries containing spanSet lists 

1638 to clip. Each element contains the new maskplane name 

1639 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 

1640 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 

1641 compressed into footprints. 

1642 """ 

1643 mask = exp.getMaskedImage().getMask() 

1644 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE") 

1645 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True) 

1646 # Merge positive and negative together footprints together 

1647 fpSet.positive.merge(fpSet.negative) 

1648 footprints = fpSet.positive 

1649 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints())) 

1650 ignoreMask = self.getBadPixelMask() 

1651 

1652 clipFootprints = [] 

1653 clipIndices = [] 

1654 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList] 

1655 

1656 # for use by detectClipBig 

1657 visitDetectionFootprints = [] 

1658 

1659 dims = [len(tempExpRefList), len(footprints.getFootprints())] 

1660 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16) 

1661 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16) 

1662 

1663 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints 

1664 for i, warpRef in enumerate(tempExpRefList): 

1665 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType), 

1666 immediate=True).getMaskedImage().getMask() 

1667 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT), 

1668 afwImage.PARENT, True) 

1669 maskVisitDet &= maskDetValue 

1670 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1)) 

1671 visitDetectionFootprints.append(visitFootprints) 

1672 

1673 for j, footprint in enumerate(footprints.getFootprints()): 

1674 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0) 

1675 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask) 

1676 

1677 # build a list of clipped spans for each visit 

1678 for j, footprint in enumerate(footprints.getFootprints()): 

1679 nPixel = footprint.getArea() 

1680 overlap = [] # hold the overlap with each visit 

1681 indexList = [] # index of visit in global list 

1682 for i in range(len(tempExpRefList)): 

1683 ignore = ignoreArr[i, j] 

1684 overlapDet = overlapDetArr[i, j] 

1685 totPixel = nPixel - ignore 

1686 

1687 # If we have more bad pixels than detection skip 

1688 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1688 ↛ 1690line 1688 didn't jump to line 1690, because the condition on line 1688 was never false

1689 continue 

1690 overlap.append(overlapDet/float(totPixel)) 

1691 indexList.append(i) 

1692 

1693 overlap = numpy.array(overlap) 

1694 if not len(overlap): 1694 ↛ 1697line 1694 didn't jump to line 1697, because the condition on line 1694 was never false

1695 continue 

1696 

1697 keep = False # Should this footprint be marked as clipped? 

1698 keepIndex = [] # Which tempExps does the clipped footprint belong to 

1699 

1700 # If footprint only has one overlap use a lower threshold 

1701 if len(overlap) == 1: 

1702 if overlap[0] > self.config.minClipFootOverlapSingle: 

1703 keep = True 

1704 keepIndex = [0] 

1705 else: 

1706 # This is the general case where only visit should be clipped 

1707 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0] 

1708 if len(clipIndex) == 1: 

1709 keep = True 

1710 keepIndex = [clipIndex[0]] 

1711 

1712 # Test if there are clipped objects that overlap two different visits 

1713 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0] 

1714 if len(clipIndex) == 2 and len(overlap) > 3: 

1715 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0] 

1716 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble: 

1717 keep = True 

1718 keepIndex = clipIndex 

1719 

1720 if not keep: 

1721 continue 

1722 

1723 for index in keepIndex: 

1724 globalIndex = indexList[index] 

1725 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans) 

1726 

1727 clipIndices.append(numpy.array(indexList)[keepIndex]) 

1728 clipFootprints.append(footprint) 

1729 

1730 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices, 

1731 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints) 

1732 

1733 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, 

1734 maskClipValue, maskDetValue, coaddBBox): 

1735 """Return individual warp footprints for large artifacts and append 

1736 them to ``clipList`` in place. 

1737 

1738 Identify big footprints composed of many sources in the coadd 

1739 difference that may have originated in a large diffuse source in the 

1740 coadd. We do this by indentifying all clipped footprints that overlap 

1741 significantly with each source in all the coaddTempExps. 

1742 

1743 Parameters 

1744 ---------- 

1745 clipList : `list` 

1746 List of alt mask SpanSets with clipping information. Modified. 

1747 clipFootprints : `list` 

1748 List of clipped footprints. 

1749 clipIndices : `list` 

1750 List of which entries in tempExpClipList each footprint belongs to. 

1751 maskClipValue 

1752 Mask value of clipped pixels. 

1753 maskDetValue 

1754 Mask value of detected pixels. 

1755 coaddBBox : `lsst.geom.Box` 

1756 BBox of the coadd and warps. 

1757 

1758 Returns 

1759 ------- 

1760 bigFootprintsCoadd : `list` 

1761 List of big footprints 

1762 """ 

1763 bigFootprintsCoadd = [] 

1764 ignoreMask = self.getBadPixelMask() 

1765 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)): 

1766 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0) 

1767 for footprint in visitFootprints.getFootprints(): 1767 ↛ 1768line 1767 didn't jump to line 1768, because the loop on line 1767 never started

1768 footprint.spans.setMask(maskVisitDet, maskDetValue) 

1769 

1770 # build a mask of clipped footprints that are in this visit 

1771 clippedFootprintsVisit = [] 

1772 for foot, clipIndex in zip(clipFootprints, clipIndices): 1772 ↛ 1773line 1772 didn't jump to line 1773, because the loop on line 1772 never started

1773 if index not in clipIndex: 

1774 continue 

1775 clippedFootprintsVisit.append(foot) 

1776 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT)) 

1777 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue) 

1778 

1779 bigFootprintsVisit = [] 

1780 for foot in visitFootprints.getFootprints(): 1780 ↛ 1781line 1780 didn't jump to line 1781, because the loop on line 1780 never started

1781 if foot.getArea() < self.config.minBigOverlap: 

1782 continue 

1783 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask) 

1784 if nCount > self.config.minBigOverlap: 

1785 bigFootprintsVisit.append(foot) 

1786 bigFootprintsCoadd.append(foot) 

1787 

1788 for footprint in bigFootprintsVisit: 1788 ↛ 1789line 1788 didn't jump to line 1789, because the loop on line 1788 never started

1789 clippedSpans["CLIPPED"].append(footprint.spans) 

1790 

1791 return bigFootprintsCoadd 

1792 

1793 

1794class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections): 

1795 psfMatchedWarps = pipeBase.connectionTypes.Input( 

1796 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 

1797 "Only PSF-Matched Warps make sense for image subtraction. " 

1798 "Therefore, they must be an additional declared input."), 

1799 name="{inputCoaddName}Coadd_psfMatchedWarp", 

1800 storageClass="ExposureF", 

1801 dimensions=("tract", "patch", "skymap", "visit"), 

1802 deferLoad=True, 

1803 multiple=True 

1804 ) 

1805 templateCoadd = pipeBase.connectionTypes.Output( 

1806 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 

1807 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"), 

1808 name="{outputCoaddName}CoaddPsfMatched", 

1809 storageClass="ExposureF", 

1810 dimensions=("tract", "patch", "skymap", "band"), 

1811 ) 

1812 

1813 def __init__(self, *, config=None): 

1814 super().__init__(config=config) 

1815 if not config.assembleStaticSkyModel.doWrite: 

1816 self.outputs.remove("templateCoadd") 

1817 config.validate() 

1818 

1819 

1820class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig, 

1821 pipelineConnections=CompareWarpAssembleCoaddConnections): 

1822 assembleStaticSkyModel = pexConfig.ConfigurableField( 

1823 target=AssembleCoaddTask, 

1824 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 

1825 " naive/first-iteration model of the static sky.", 

1826 ) 

1827 detect = pexConfig.ConfigurableField( 

1828 target=SourceDetectionTask, 

1829 doc="Detect outlier sources on difference between each psfMatched warp and static sky model" 

1830 ) 

1831 detectTemplate = pexConfig.ConfigurableField( 

1832 target=SourceDetectionTask, 

1833 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 

1834 ) 

1835 maskStreaks = pexConfig.ConfigurableField( 

1836 target=MaskStreaksTask, 

1837 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if " 

1838 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by" 

1839 "streakMaskName" 

1840 ) 

1841 streakMaskName = pexConfig.Field( 

1842 dtype=str, 

1843 default="STREAK", 

1844 doc="Name of mask bit used for streaks" 

1845 ) 

1846 maxNumEpochs = pexConfig.Field( 

1847 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 

1848 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 

1849 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 

1850 "For each footprint detected on the image difference between the psfMatched warp and static sky " 

1851 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 

1852 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 

1853 "than transient and not masked.", 

1854 dtype=int, 

1855 default=2 

1856 ) 

1857 maxFractionEpochsLow = pexConfig.RangeField( 

1858 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 

1859 "Effective maxNumEpochs = " 

1860 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1861 dtype=float, 

1862 default=0.4, 

1863 min=0., max=1., 

1864 ) 

1865 maxFractionEpochsHigh = pexConfig.RangeField( 

1866 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 

1867 "Effective maxNumEpochs = " 

1868 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)", 

1869 dtype=float, 

1870 default=0.03, 

1871 min=0., max=1., 

1872 ) 

1873 spatialThreshold = pexConfig.RangeField( 

1874 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the " 

1875 "temporal criteria. If 0, clip all. If 1, clip none.", 

1876 dtype=float, 

1877 default=0.5, 

1878 min=0., max=1., 

1879 inclusiveMin=True, inclusiveMax=True 

1880 ) 

1881 doScaleWarpVariance = pexConfig.Field( 

1882 doc="Rescale Warp variance plane using empirical noise?", 

1883 dtype=bool, 

1884 default=True, 

1885 ) 

1886 scaleWarpVariance = pexConfig.ConfigurableField( 

1887 target=ScaleVarianceTask, 

1888 doc="Rescale variance on warps", 

1889 ) 

1890 doPreserveContainedBySource = pexConfig.Field( 

1891 doc="Rescue artifacts from clipping that completely lie within a footprint detected" 

1892 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.", 

1893 dtype=bool, 

1894 default=True, 

1895 ) 

1896 doPrefilterArtifacts = pexConfig.Field( 

1897 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, " 

1898 "because they will be excluded anyway. This prevents them from contributing " 

1899 "to the outlier epoch count image and potentially being labeled as persistant." 

1900 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.", 

1901 dtype=bool, 

1902 default=True 

1903 ) 

1904 prefilterArtifactsMaskPlanes = pexConfig.ListField( 

1905 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.", 

1906 dtype=str, 

1907 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'), 

1908 ) 

1909 prefilterArtifactsRatio = pexConfig.Field( 

1910 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels", 

1911 dtype=float, 

1912 default=0.05 

1913 ) 

1914 doFilterMorphological = pexConfig.Field( 

1915 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to " 

1916 "be streaks.", 

1917 dtype=bool, 

1918 default=False 

1919 ) 

1920 

1921 def setDefaults(self): 

1922 AssembleCoaddConfig.setDefaults(self) 

1923 self.statistic = 'MEAN' 

1924 self.doUsePsfMatchedPolygons = True 

1925 

1926 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel 

1927 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling 

1928 if "EDGE" in self.badMaskPlanes: 1928 ↛ 1930line 1928 didn't jump to line 1930, because the condition on line 1928 was never false

1929 self.badMaskPlanes.remove('EDGE') 

1930 self.removeMaskPlanes.append('EDGE') 

1931 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ] 

1932 self.assembleStaticSkyModel.warpType = 'psfMatched' 

1933 self.assembleStaticSkyModel.connections.warpType = 'psfMatched' 

1934 self.assembleStaticSkyModel.statistic = 'MEANCLIP' 

1935 self.assembleStaticSkyModel.sigmaClip = 2.5 

1936 self.assembleStaticSkyModel.clipIter = 3 

1937 self.assembleStaticSkyModel.calcErrorFromInputVariance = False 

1938 self.assembleStaticSkyModel.doWrite = False 

1939 self.detect.doTempLocalBackground = False 

1940 self.detect.reEstimateBackground = False 

1941 self.detect.returnOriginalFootprints = False 

1942 self.detect.thresholdPolarity = "both" 

1943 self.detect.thresholdValue = 5 

1944 self.detect.minPixels = 4 

1945 self.detect.isotropicGrow = True 

1946 self.detect.thresholdType = "pixel_stdev" 

1947 self.detect.nSigmaToGrow = 0.4 

1948 # The default nSigmaToGrow for SourceDetectionTask is already 2.4, 

1949 # Explicitly restating because ratio with detect.nSigmaToGrow matters 

1950 self.detectTemplate.nSigmaToGrow = 2.4 

1951 self.detectTemplate.doTempLocalBackground = False 

1952 self.detectTemplate.reEstimateBackground = False 

1953 self.detectTemplate.returnOriginalFootprints = False 

1954 

1955 def validate(self): 

1956 super().validate() 

1957 if self.assembleStaticSkyModel.doNImage: 

1958 raise ValueError("No dataset type exists for a PSF-Matched Template N Image." 

1959 "Please set assembleStaticSkyModel.doNImage=False") 

1960 

1961 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType): 

1962 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 

1963 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 

1964 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 

1965 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)) 

1966 

1967 

1968class CompareWarpAssembleCoaddTask(AssembleCoaddTask): 

1969 """Assemble a compareWarp coadded image from a set of warps 

1970 by masking artifacts detected by comparing PSF-matched warps. 

1971 

1972 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 

1973 we clip outliers). The problem with doing this is that when computing the 

1974 coadd PSF at a given location, individual visit PSFs from visits with 

1975 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 

1976 In this task, we correct for this behavior by creating a new badMaskPlane 

1977 'CLIPPED' which marks pixels in the individual warps suspected to contain 

1978 an artifact. We populate this plane on the input warps by comparing 

1979 PSF-matched warps with a PSF-matched median coadd which serves as a 

1980 model of the static sky. Any group of pixels that deviates from the 

1981 PSF-matched template coadd by more than config.detect.threshold sigma, 

1982 is an artifact candidate. The candidates are then filtered to remove 

1983 variable sources and sources that are difficult to subtract such as 

1984 bright stars. This filter is configured using the config parameters 

1985 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 

1986 the maximum fraction of epochs that the deviation can appear in and still 

1987 be considered an artifact. The spatialThreshold is the maximum fraction of 

1988 pixels in the footprint of the deviation that appear in other epochs 

1989 (where other epochs is defined by the temporalThreshold). If the deviant 

1990 region meets this criteria of having a significant percentage of pixels 

1991 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 

1992 set in the mask. These regions will not contribute to the final coadd. 

1993 Furthermore, any routine to determine the coadd PSF can now be cognizant 

1994 of clipped regions. Note that the algorithm implemented by this task is 

1995 preliminary and works correctly for HSC data. Parameter modifications and 

1996 or considerable redesigning of the algorithm is likley required for other 

1997 surveys. 

1998 

1999 ``CompareWarpAssembleCoaddTask`` sub-classes 

2000 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 

2001 as a subtask to generate the TemplateCoadd (the model of the static sky). 

2002 

2003 Notes 

2004 ----- 

2005 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 

2006 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 

2007 ``baseDebug`` for more about ``debug.py`` files. 

2008 

2009 This task supports the following debug variables: 

2010 

2011 - ``saveCountIm`` 

2012 If True then save the Epoch Count Image as a fits file in the `figPath` 

2013 - ``figPath`` 

2014 Path to save the debug fits images and figures 

2015 

2016 For example, put something like: 

2017 

2018 .. code-block:: python 

2019 

2020 import lsstDebug 

2021 def DebugInfo(name): 

2022 di = lsstDebug.getInfo(name) 

2023 if name == "lsst.pipe.tasks.assembleCoadd": 

2024 di.saveCountIm = True 

2025 di.figPath = "/desired/path/to/debugging/output/images" 

2026 return di 

2027 lsstDebug.Info = DebugInfo 

2028 

2029 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 

2030 ``--debug`` flag. Some subtasks may have their own debug variables; 

2031 see individual Task documentation. 

2032 

2033 Examples 

2034 -------- 

2035 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 

2036 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 

2037 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 

2038 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 

2039 and filter to be coadded (specified using 

2040 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 

2041 along with a list of coaddTempExps to attempt to coadd (specified using 

2042 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 

2043 Only the warps that cover the specified tract and patch will be coadded. 

2044 A list of the available optional arguments can be obtained by calling 

2045 ``assembleCoadd.py`` with the ``--help`` command line argument: 

2046 

2047 .. code-block:: none 

2048 

2049 assembleCoadd.py --help 

2050 

2051 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 

2052 context of multi-band processing, we will generate the HSC-I & -R band 

2053 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 

2054 To begin, assuming that the lsst stack has been already set up, we must 

2055 set up the ``obs_subaru`` and ``ci_hsc`` packages. 

2056 This defines the environment variable ``$CI_HSC_DIR`` and points at the 

2057 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 

2058 directory. To begin assembling the coadds, we must first 

2059 

2060 - processCcd 

2061 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 

2062 - makeSkyMap 

2063 create a skymap that covers the area of the sky present in the raw exposures 

2064 - makeCoaddTempExp 

2065 warp the individual calibrated exposures to the tangent plane of the coadd 

2066 

2067 We can perform all of these steps by running 

2068 

2069 .. code-block:: none 

2070 

2071 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 

2072 

2073 This will produce warped ``coaddTempExps`` for each visit. To coadd the 

2074 warped data, we call ``assembleCoadd.py`` as follows: 

2075 

2076 .. code-block:: none 

2077 

2078 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 

2079 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 

2080 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 

2081 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 

2082 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 

2083 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 

2084 --selectId visit=903988 ccd=24 

2085 

2086 This will process the HSC-I band data. The results are written in 

2087 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 

2088 """ 

2089 ConfigClass = CompareWarpAssembleCoaddConfig 

2090 _DefaultName = "compareWarpAssembleCoadd" 

2091 

2092 def __init__(self, *args, **kwargs): 

2093 AssembleCoaddTask.__init__(self, *args, **kwargs) 

2094 self.makeSubtask("assembleStaticSkyModel") 

2095 detectionSchema = afwTable.SourceTable.makeMinimalSchema() 

2096 self.makeSubtask("detect", schema=detectionSchema) 

2097 if self.config.doPreserveContainedBySource: 2097 ↛ 2099line 2097 didn't jump to line 2099, because the condition on line 2097 was never false

2098 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema()) 

2099 if self.config.doScaleWarpVariance: 2099 ↛ 2101line 2099 didn't jump to line 2101, because the condition on line 2099 was never false

2100 self.makeSubtask("scaleWarpVariance") 

2101 if self.config.doFilterMorphological: 2101 ↛ 2102line 2101 didn't jump to line 2102, because the condition on line 2101 was never true

2102 self.makeSubtask("maskStreaks") 

2103 

2104 @utils.inheritDoc(AssembleCoaddTask) 

2105 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs): 

2106 """ 

2107 Generate a templateCoadd to use as a naive model of static sky to 

2108 subtract from PSF-Matched warps. 

2109 

2110 Returns 

2111 ------- 

2112 result : `lsst.pipe.base.Struct` 

2113 Result struct with components: 

2114 

2115 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 

2116 - ``nImage`` : N Image (``lsst.afw.image.Image``) 

2117 """ 

2118 # Ensure that psfMatchedWarps are used as input warps for template generation 

2119 staticSkyModelInputRefs = copy.deepcopy(inputRefs) 

2120 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps 

2121 

2122 # Because subtasks don't have connections we have to make one. 

2123 # The main task's `templateCoadd` is the subtask's `coaddExposure` 

2124 staticSkyModelOutputRefs = copy.deepcopy(outputRefs) 

2125 if self.config.assembleStaticSkyModel.doWrite: 

2126 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd 

2127 # Remove template coadd from both subtask's and main tasks outputs, 

2128 # because it is handled by the subtask as `coaddExposure` 

2129 del outputRefs.templateCoadd 

2130 del staticSkyModelOutputRefs.templateCoadd 

2131 

2132 # A PSF-Matched nImage does not exist as a dataset type 

2133 if 'nImage' in staticSkyModelOutputRefs.keys(): 

2134 del staticSkyModelOutputRefs.nImage 

2135 

2136 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs, 

2137 staticSkyModelOutputRefs) 

2138 if templateCoadd is None: 

2139 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2140 

2141 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2142 nImage=templateCoadd.nImage, 

2143 warpRefList=templateCoadd.warpRefList, 

2144 imageScalerList=templateCoadd.imageScalerList, 

2145 weightList=templateCoadd.weightList) 

2146 

2147 @utils.inheritDoc(AssembleCoaddTask) 

2148 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None): 

2149 """ 

2150 Generate a templateCoadd to use as a naive model of static sky to 

2151 subtract from PSF-Matched warps. 

2152 

2153 Returns 

2154 ------- 

2155 result : `lsst.pipe.base.Struct` 

2156 Result struct with components: 

2157 

2158 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 

2159 - ``nImage``: N Image (``lsst.afw.image.Image``) 

2160 """ 

2161 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList) 

2162 if templateCoadd is None: 2162 ↛ 2163line 2162 didn't jump to line 2163, because the condition on line 2162 was never true

2163 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType)) 

2164 

2165 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure, 

2166 nImage=templateCoadd.nImage, 

2167 warpRefList=templateCoadd.warpRefList, 

2168 imageScalerList=templateCoadd.imageScalerList, 

2169 weightList=templateCoadd.weightList) 

2170 

2171 def _noTemplateMessage(self, warpType): 

2172 warpName = (warpType[0].upper() + warpType[1:]) 

2173 message = """No %(warpName)s warps were found to build the template coadd which is 

2174 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 

2175 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 

2176 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 

2177 

2178 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 

2179 another algorithm like: 

2180 

2181 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 

2182 config.assemble.retarget(SafeClipAssembleCoaddTask) 

2183 """ % {"warpName": warpName} 

2184 return message 

2185 

2186 @utils.inheritDoc(AssembleCoaddTask) 

2187 @pipeBase.timeMethod 

2188 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2189 supplementaryData, *args, **kwargs): 

2190 """Assemble the coadd. 

2191 

2192 Find artifacts and apply them to the warps' masks creating a list of 

2193 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 

2194 plane. Then pass these alternative masks to the base class's `run` 

2195 method. 

2196 

2197 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 

2198 that must contain a ``templateCoadd`` that serves as the 

2199 model of the static sky. 

2200 """ 

2201 

2202 # Check and match the order of the supplementaryData 

2203 # (PSF-matched) inputs to the order of the direct inputs, 

2204 # so that the artifact mask is applied to the right warp 

2205 dataIds = [ref.dataId for ref in tempExpRefList] 

2206 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList] 

2207 

2208 if dataIds != psfMatchedDataIds: 

2209 self.log.info("Reordering and or/padding PSF-matched visit input list") 

2210 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList, 

2211 psfMatchedDataIds, dataIds) 

2212 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList, 

2213 psfMatchedDataIds, dataIds) 

2214 

2215 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts 

2216 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd, 

2217 supplementaryData.warpRefList, 

2218 supplementaryData.imageScalerList) 

2219 

2220 badMaskPlanes = self.config.badMaskPlanes[:] 

2221 badMaskPlanes.append("CLIPPED") 

2222 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) 

2223 

2224 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList, 

2225 spanSetMaskList, mask=badPixelMask) 

2226 

2227 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF 

2228 # Psf-Matching moves the real edge inwards 

2229 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList) 

2230 return result 

2231 

2232 def applyAltEdgeMask(self, mask, altMaskList): 

2233 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 

2234 

2235 Parameters 

2236 ---------- 

2237 mask : `lsst.afw.image.Mask` 

2238 Original mask. 

2239 altMaskList : `list` 

2240 List of Dicts containing ``spanSet`` lists. 

2241 Each element contains the new mask plane name (e.g. "CLIPPED 

2242 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 

2243 the mask. 

2244 """ 

2245 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"]) 

2246 for visitMask in altMaskList: 

2247 if "EDGE" in visitMask: 2247 ↛ 2246line 2247 didn't jump to line 2246, because the condition on line 2247 was never false

2248 for spanSet in visitMask['EDGE']: 

2249 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue) 

2250 

2251 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList): 

2252 """Find artifacts. 

2253 

2254 Loop through warps twice. The first loop builds a map with the count 

2255 of how many epochs each pixel deviates from the templateCoadd by more 

2256 than ``config.chiThreshold`` sigma. The second loop takes each 

2257 difference image and filters the artifacts detected in each using 

2258 count map to filter out variable sources and sources that are 

2259 difficult to subtract cleanly. 

2260 

2261 Parameters 

2262 ---------- 

2263 templateCoadd : `lsst.afw.image.Exposure` 

2264 Exposure to serve as model of static sky. 

2265 tempExpRefList : `list` 

2266 List of data references to warps. 

2267 imageScalerList : `list` 

2268 List of image scalers. 

2269 

2270 Returns 

2271 ------- 

2272 altMasks : `list` 

2273 List of dicts containing information about CLIPPED 

2274 (i.e., artifacts), NO_DATA, and EDGE pixels. 

2275 """ 

2276 

2277 self.log.debug("Generating Count Image, and mask lists.") 

2278 coaddBBox = templateCoadd.getBBox() 

2279 slateIm = afwImage.ImageU(coaddBBox) 

2280 epochCountImage = afwImage.ImageU(coaddBBox) 

2281 nImage = afwImage.ImageU(coaddBBox) 

2282 spanSetArtifactList = [] 

2283 spanSetNoDataMaskList = [] 

2284 spanSetEdgeList = [] 

2285 spanSetBadMorphoList = [] 

2286 badPixelMask = self.getBadPixelMask() 

2287 

2288 # mask of the warp diffs should = that of only the warp 

2289 templateCoadd.mask.clearAllMaskPlanes() 

2290 

2291 if self.config.doPreserveContainedBySource: 2291 ↛ 2294line 2291 didn't jump to line 2294, because the condition on line 2291 was never false

2292 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd) 

2293 else: 

2294 templateFootprints = None 

2295 

2296 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList): 

2297 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd) 

2298 if warpDiffExp is not None: 

2299 # This nImage only approximates the final nImage because it uses the PSF-matched mask 

2300 nImage.array += (numpy.isfinite(warpDiffExp.image.array) 

2301 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16) 

2302 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True) 

2303 fpSet.positive.merge(fpSet.negative) 

2304 footprints = fpSet.positive 

2305 slateIm.set(0) 

2306 spanSetList = [footprint.spans for footprint in footprints.getFootprints()] 

2307 

2308 # Remove artifacts due to defects before they contribute to the epochCountImage 

2309 if self.config.doPrefilterArtifacts: 2309 ↛ 2313line 2309 didn't jump to line 2313, because the condition on line 2309 was never false

2310 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp) 

2311 

2312 # Clear mask before adding prefiltered spanSets 

2313 self.detect.clearMask(warpDiffExp.mask) 

2314 for spans in spanSetList: 

2315 spans.setImage(slateIm, 1, doClip=True) 

2316 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED")) 

2317 epochCountImage += slateIm 

2318 

2319 if self.config.doFilterMorphological: 2319 ↛ 2320line 2319 didn't jump to line 2320, because the condition on line 2319 was never true

2320 maskName = self.config.streakMaskName 

2321 _ = self.maskStreaks.run(warpDiffExp) 

2322 streakMask = warpDiffExp.mask 

2323 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask, 

2324 streakMask.getPlaneBitMask(maskName)).split() 

2325 

2326 # PSF-Matched warps have less available area (~the matching kernel) because the calexps 

2327 # undergo a second convolution. Pixels with data in the direct warp 

2328 # but not in the PSF-matched warp will not have their artifacts detected. 

2329 # NaNs from the PSF-matched warp therefore must be masked in the direct warp 

2330 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0) 

2331 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel)) 

2332 nansMask.setXY0(warpDiffExp.getXY0()) 

2333 edgeMask = warpDiffExp.mask 

2334 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, 

2335 edgeMask.getPlaneBitMask("EDGE")).split() 

2336 else: 

2337 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist 

2338 # In this case, mask the whole epoch 

2339 nansMask = afwImage.MaskX(coaddBBox, 1) 

2340 spanSetList = [] 

2341 spanSetEdgeMask = [] 

2342 spanSetStreak = [] 

2343 

2344 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split() 

2345 

2346 spanSetNoDataMaskList.append(spanSetNoDataMask) 

2347 spanSetArtifactList.append(spanSetList) 

2348 spanSetEdgeList.append(spanSetEdgeMask) 

2349 if self.config.doFilterMorphological: 2349 ↛ 2350line 2349 didn't jump to line 2350, because the condition on line 2349 was never true

2350 spanSetBadMorphoList.append(spanSetStreak) 

2351 

2352 if lsstDebug.Info(__name__).saveCountIm: 2352 ↛ 2353line 2352 didn't jump to line 2353, because the condition on line 2352 was never true

2353 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True) 

2354 epochCountImage.writeFits(path) 

2355 

2356 for i, spanSetList in enumerate(spanSetArtifactList): 

2357 if spanSetList: 

2358 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage, 

2359 templateFootprints) 

2360 spanSetArtifactList[i] = filteredSpanSetList 

2361 if self.config.doFilterMorphological: 2361 ↛ 2362line 2361 didn't jump to line 2362, because the condition on line 2361 was never true

2362 spanSetArtifactList[i] += spanSetBadMorphoList[i] 

2363 

2364 altMasks = [] 

2365 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList): 

2366 altMasks.append({'CLIPPED': artifacts, 

2367 'NO_DATA': noData, 

2368 'EDGE': edge}) 

2369 return altMasks 

2370 

2371 def prefilterArtifacts(self, spanSetList, exp): 

2372 """Remove artifact candidates covered by bad mask plane. 

2373 

2374 Any future editing of the candidate list that does not depend on 

2375 temporal information should go in this method. 

2376 

2377 Parameters 

2378 ---------- 

2379 spanSetList : `list` 

2380 List of SpanSets representing artifact candidates. 

2381 exp : `lsst.afw.image.Exposure` 

2382 Exposure containing mask planes used to prefilter. 

2383 

2384 Returns 

2385 ------- 

2386 returnSpanSetList : `list` 

2387 List of SpanSets with artifacts. 

2388 """ 

2389 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes) 

2390 goodArr = (exp.mask.array & badPixelMask) == 0 

2391 returnSpanSetList = [] 

2392 bbox = exp.getBBox() 

2393 x0, y0 = exp.getXY0() 

2394 for i, span in enumerate(spanSetList): 

2395 y, x = span.clippedTo(bbox).indices() 

2396 yIndexLocal = numpy.array(y) - y0 

2397 xIndexLocal = numpy.array(x) - x0 

2398 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea() 

2399 if goodRatio > self.config.prefilterArtifactsRatio: 2399 ↛ 2394line 2399 didn't jump to line 2394, because the condition on line 2399 was never false

2400 returnSpanSetList.append(span) 

2401 return returnSpanSetList 

2402 

2403 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None): 

2404 """Filter artifact candidates. 

2405 

2406 Parameters 

2407 ---------- 

2408 spanSetList : `list` 

2409 List of SpanSets representing artifact candidates. 

2410 epochCountImage : `lsst.afw.image.Image` 

2411 Image of accumulated number of warpDiff detections. 

2412 nImage : `lsst.afw.image.Image` 

2413 Image of the accumulated number of total epochs contributing. 

2414 

2415 Returns 

2416 ------- 

2417 maskSpanSetList : `list` 

2418 List of SpanSets with artifacts. 

2419 """ 

2420 

2421 maskSpanSetList = [] 

2422 x0, y0 = epochCountImage.getXY0() 

2423 for i, span in enumerate(spanSetList): 

2424 y, x = span.indices() 

2425 yIdxLocal = [y1 - y0 for y1 in y] 

2426 xIdxLocal = [x1 - x0 for x1 in x] 

2427 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal] 

2428 totalN = nImage.array[yIdxLocal, xIdxLocal] 

2429 

2430 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs 

2431 effMaxNumEpochsHighN = (self.config.maxNumEpochs 

2432 + self.config.maxFractionEpochsHigh*numpy.mean(totalN)) 

2433 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN) 

2434 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN)) 

2435 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) 

2436 & (outlierN <= effectiveMaxNumEpochs)) 

2437 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN) 

2438 if percentBelowThreshold > self.config.spatialThreshold: 

2439 maskSpanSetList.append(span) 

2440 

2441 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2441 ↛ 2454line 2441 didn't jump to line 2454, because the condition on line 2441 was never false

2442 # If a candidate is contained by a footprint on the template coadd, do not clip 

2443 filteredMaskSpanSetList = [] 

2444 for span in maskSpanSetList: 

2445 doKeep = True 

2446 for footprint in footprintsToExclude.positive.getFootprints(): 

2447 if footprint.spans.contains(span): 

2448 doKeep = False 

2449 break 

2450 if doKeep: 

2451 filteredMaskSpanSetList.append(span) 

2452 maskSpanSetList = filteredMaskSpanSetList 

2453 

2454 return maskSpanSetList 

2455 

2456 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd): 

2457 """Fetch a warp from the butler and return a warpDiff. 

2458 

2459 Parameters 

2460 ---------- 

2461 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2462 Butler dataRef for the warp. 

2463 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 

2464 An image scaler object. 

2465 templateCoadd : `lsst.afw.image.Exposure` 

2466 Exposure to be substracted from the scaled warp. 

2467 

2468 Returns 

2469 ------- 

2470 warp : `lsst.afw.image.Exposure` 

2471 Exposure of the image difference between the warp and template. 

2472 """ 

2473 

2474 # If the PSF-Matched warp did not exist for this direct warp 

2475 # None is holding its place to maintain order in Gen 3 

2476 if warpRef is None: 

2477 return None 

2478 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type 

2479 warpName = self.getTempExpDatasetName('psfMatched') 

2480 if not isinstance(warpRef, DeferredDatasetHandle): 2480 ↛ 2484line 2480 didn't jump to line 2484, because the condition on line 2480 was never false

2481 if not warpRef.datasetExists(warpName): 2481 ↛ 2482line 2481 didn't jump to line 2482, because the condition on line 2481 was never true

2482 self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId) 

2483 return None 

2484 warp = warpRef.get(datasetType=warpName, immediate=True) 

2485 # direct image scaler OK for PSF-matched Warp 

2486 imageScaler.scaleMaskedImage(warp.getMaskedImage()) 

2487 mi = warp.getMaskedImage() 

2488 if self.config.doScaleWarpVariance: 2488 ↛ 2493line 2488 didn't jump to line 2493, because the condition on line 2488 was never false

2489 try: 

2490 self.scaleWarpVariance.run(mi) 

2491 except Exception as exc: 

2492 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc) 

2493 mi -= templateCoadd.getMaskedImage() 

2494 return warp 

2495 

2496 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False): 

2497 """Return a path to which to write debugging output. 

2498 

2499 Creates a hyphen-delimited string of dataId values for simple filenames. 

2500 

2501 Parameters 

2502 ---------- 

2503 prefix : `str` 

2504 Prefix for filename. 

2505 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 

2506 Butler dataRef to make the path from. 

2507 coaddLevel : `bool`, optional. 

2508 If True, include only coadd-level keys (e.g., 'tract', 'patch', 

2509 'filter', but no 'visit'). 

2510 

2511 Returns 

2512 ------- 

2513 result : `str` 

2514 Path for debugging output. 

2515 """ 

2516 if coaddLevel: 

2517 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType)) 

2518 else: 

2519 keys = warpRef.dataId.keys() 

2520 keyList = sorted(keys, reverse=True) 

2521 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "." 

2522 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList])) 

2523 return os.path.join(directory, filename)