Coverage for python/lsst/cp/pipe/defects.py: 15%

278 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-11 11:19 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.pipe.base as pipeBase 

25import lsst.pipe.base.connectionTypes as cT 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.detection as afwDetection 

33import lsst.afw.display as afwDisplay 

34from lsst.afw import cameraGeom 

35from lsst.geom import Box2I, Point2I 

36from lsst.meas.algorithms import SourceDetectionTask 

37from lsst.ip.isr import Defects, countMaskedPixels 

38from lsst.pex.exceptions import InvalidParameterError 

39 

40from ._lookupStaticCalibration import lookupStaticCalibration 

41 

42__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

43 'MergeDefectsTaskConfig', 'MergeDefectsTask', ] 

44 

45 

46class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "exposure", "detector")): 

48 inputExp = cT.Input( 

49 name="defectExps", 

50 doc="Input ISR-processed exposures to measure.", 

51 storageClass="Exposure", 

52 dimensions=("instrument", "detector", "exposure"), 

53 multiple=False 

54 ) 

55 camera = cT.PrerequisiteInput( 

56 name='camera', 

57 doc="Camera associated with this exposure.", 

58 storageClass="Camera", 

59 dimensions=("instrument", ), 

60 isCalibration=True, 

61 lookupFunction=lookupStaticCalibration, 

62 ) 

63 

64 outputDefects = cT.Output( 

65 name="singleExpDefects", 

66 doc="Output measured defects.", 

67 storageClass="Defects", 

68 dimensions=("instrument", "detector", "exposure"), 

69 ) 

70 

71 

72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=MeasureDefectsConnections): 

74 """Configuration for measuring defects from a list of exposures 

75 """ 

76 

77 nSigmaBright = pexConfig.Field( 

78 dtype=float, 

79 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be" 

80 " appropriate for some LSST sensors in DM-17490."), 

81 default=4.8, 

82 ) 

83 nSigmaDark = pexConfig.Field( 

84 dtype=float, 

85 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be" 

86 " appropriate for some LSST sensors in DM-17490."), 

87 default=-5.0, 

88 ) 

89 nPixBorderUpDown = pexConfig.Field( 

90 dtype=int, 

91 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

92 default=7, 

93 ) 

94 nPixBorderLeftRight = pexConfig.Field( 

95 dtype=int, 

96 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

97 default=7, 

98 ) 

99 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

100 dtype=int, 

101 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

102 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

103 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

104 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

105 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

106 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

107 "'goodPixelColumnGapThreshold' are not marked as bad."), 

108 default=50, 

109 ) 

110 goodPixelColumnGapThreshold = pexConfig.Field( 

111 dtype=int, 

112 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

113 "'badOnAndOffPixelColumnThreshold')."), 

114 default=30, 

115 ) 

116 

117 def validate(self): 

118 super().validate() 

119 if self.nSigmaBright < 0.0: 

120 raise ValueError("nSigmaBright must be above 0.0.") 

121 if self.nSigmaDark > 0.0: 

122 raise ValueError("nSigmaDark must be below 0.0.") 

123 

124 

125class MeasureDefectsTask(pipeBase.PipelineTask): 

126 """Measure the defects from one exposure. 

127 """ 

128 

129 ConfigClass = MeasureDefectsTaskConfig 

130 _DefaultName = 'cpDefectMeasure' 

131 

132 def run(self, inputExp, camera): 

133 """Measure one exposure for defects. 

134 

135 Parameters 

136 ---------- 

137 inputExp : `lsst.afw.image.Exposure` 

138 Exposure to examine. 

139 camera : `lsst.afw.cameraGeom.Camera` 

140 Camera to use for metadata. 

141 

142 Returns 

143 ------- 

144 results : `lsst.pipe.base.Struct` 

145 Results struct containing: 

146 

147 ``outputDefects`` 

148 The defects measured from this exposure 

149 (`lsst.ip.isr.Defects`). 

150 """ 

151 detector = inputExp.getDetector() 

152 

153 filterName = inputExp.getFilter().physicalLabel 

154 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

155 

156 if datasetType.lower() == 'dark': 

157 nSigmaList = [self.config.nSigmaBright] 

158 else: 

159 nSigmaList = [self.config.nSigmaBright, self.config.nSigmaDark] 

160 defects = self.findHotAndColdPixels(inputExp, nSigmaList) 

161 

162 msg = "Found %s defects containing %s pixels in %s" 

163 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

164 

165 defects.updateMetadataFromExposures([inputExp]) 

166 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

167 setCalibId=True, setDate=True, 

168 cpDefectGenImageType=datasetType) 

169 

170 return pipeBase.Struct( 

171 outputDefects=defects, 

172 ) 

173 

174 @staticmethod 

175 def _nPixFromDefects(defects): 

176 """Count pixels in a defect. 

177 

178 Parameters 

179 ---------- 

180 defects : `lsst.ip.isr.Defects` 

181 Defects to measure. 

182 

183 Returns 

184 ------- 

185 nPix : `int` 

186 Number of defect pixels. 

187 """ 

188 nPix = 0 

189 for defect in defects: 

190 nPix += defect.getBBox().getArea() 

191 return nPix 

192 

193 def findHotAndColdPixels(self, exp, nSigma): 

194 """Find hot and cold pixels in an image. 

195 

196 Using config-defined thresholds on a per-amp basis, mask 

197 pixels that are nSigma above threshold in dark frames (hot 

198 pixels), or nSigma away from the clipped mean in flats (hot & 

199 cold pixels). 

200 

201 Parameters 

202 ---------- 

203 exp : `lsst.afw.image.exposure.Exposure` 

204 The exposure in which to find defects. 

205 nSigma : `list` [`float`] 

206 Detection threshold to use. Positive for DETECTED pixels, 

207 negative for DETECTED_NEGATIVE pixels. 

208 

209 Returns 

210 ------- 

211 defects : `lsst.ip.isr.Defects` 

212 The defects found in the image. 

213 """ 

214 self._setEdgeBits(exp) 

215 maskedIm = exp.maskedImage 

216 

217 # the detection polarity for afwDetection, True for positive, 

218 # False for negative, and therefore True for darks as they only have 

219 # bright pixels, and both for flats, as they have bright and dark pix 

220 footprintList = [] 

221 

222 for amp in exp.getDetector(): 

223 ampImg = maskedIm[amp.getBBox()].clone() 

224 

225 # crop ampImage depending on where the amp lies in the image 

226 if self.config.nPixBorderLeftRight: 

227 if ampImg.getX0() == 0: 

228 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

229 else: 

230 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

231 if self.config.nPixBorderUpDown: 

232 if ampImg.getY0() == 0: 

233 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

234 else: 

235 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

236 

237 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

238 continue 

239 

240 # Remove a background estimate 

241 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

242 

243 mergedSet = None 

244 for sigma in nSigma: 

245 nSig = np.abs(sigma) 

246 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

247 polarity = {-1: False, 1: True}[np.sign(sigma)] 

248 

249 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

250 

251 try: 

252 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

253 except InvalidParameterError: 

254 # This occurs if the image sigma value is 0.0. 

255 # Let's mask the whole area. 

256 minValue = np.nanmin(ampImg.image.array) - 1.0 

257 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True) 

258 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

259 

260 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

261 

262 if mergedSet is None: 

263 mergedSet = footprintSet 

264 else: 

265 mergedSet.merge(footprintSet) 

266 

267 footprintList += mergedSet.getFootprints() 

268 

269 self.debugView('defectMap', ampImg, 

270 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

271 

272 defects = Defects.fromFootprintList(footprintList) 

273 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

274 

275 return defects 

276 

277 @staticmethod 

278 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

279 """Return the number of non-bad pixels in the image.""" 

280 nPixels = maskedIm.mask.array.size 

281 nBad = countMaskedPixels(maskedIm, badMaskString) 

282 return nPixels - nBad 

283 

284 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

285 """Set edge bits on an exposure or maskedImage. 

286 

287 Raises 

288 ------ 

289 TypeError 

290 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

291 """ 

292 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

293 mi = exposureOrMaskedImage.maskedImage 

294 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

295 mi = exposureOrMaskedImage 

296 else: 

297 t = type(exposureOrMaskedImage) 

298 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

299 

300 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

301 if self.config.nPixBorderLeftRight: 

302 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

303 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

304 if self.config.nPixBorderUpDown: 

305 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

306 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

307 

308 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

309 """Mask blocks in a column if there are on-and-off bad pixels 

310 

311 If there's a column with on and off bad pixels, mask all the 

312 pixels in between, except if there is a large enough gap of 

313 consecutive good pixels between two bad pixels in the column. 

314 

315 Parameters 

316 ---------- 

317 defects : `lsst.ip.isr.Defects` 

318 The defects found in the image so far 

319 

320 Returns 

321 ------- 

322 defects : `lsst.ip.isr.Defects` 

323 If the number of bad pixels in a column is not larger or 

324 equal than self.config.badPixelColumnThreshold, the input 

325 list is returned. Otherwise, the defects list returned 

326 will include boxes that mask blocks of on-and-of pixels. 

327 """ 

328 # Get the (x, y) values of each bad pixel in amp. 

329 coordinates = [] 

330 for defect in defects: 

331 bbox = defect.getBBox() 

332 x0, y0 = bbox.getMinX(), bbox.getMinY() 

333 deltaX0, deltaY0 = bbox.getDimensions() 

334 for j in np.arange(y0, y0+deltaY0): 

335 for i in np.arange(x0, x0 + deltaX0): 

336 coordinates.append((i, j)) 

337 

338 x, y = [], [] 

339 for coordinatePair in coordinates: 

340 x.append(coordinatePair[0]) 

341 y.append(coordinatePair[1]) 

342 

343 x = np.array(x) 

344 y = np.array(y) 

345 # Find the defects with same "x" (vertical) coordinate (column). 

346 unique, counts = np.unique(x, return_counts=True) 

347 multipleX = [] 

348 for (a, b) in zip(unique, counts): 

349 if b >= self.config.badOnAndOffPixelColumnThreshold: 

350 multipleX.append(a) 

351 if len(multipleX) != 0: 

352 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

353 

354 return defects 

355 

356 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

357 """Mask blocks in a column if number of on-and-off bad pixels is above 

358 threshold. 

359 

360 This function is called if the number of on-and-off bad pixels 

361 in a column is larger or equal than 

362 self.config.badOnAndOffPixelColumnThreshold. 

363 

364 Parameters 

365 --------- 

366 x : `list` 

367 Lower left x coordinate of defect box. x coordinate is 

368 along the short axis if amp. 

369 y : `list` 

370 Lower left y coordinate of defect box. x coordinate is 

371 along the long axis if amp. 

372 multipleX : list 

373 List of x coordinates in amp. with multiple bad pixels 

374 (i.e., columns with defects). 

375 defects : `lsst.ip.isr.Defects` 

376 The defcts found in the image so far 

377 

378 Returns 

379 ------- 

380 defects : `lsst.ip.isr.Defects` 

381 The defects list returned that will include boxes that 

382 mask blocks of on-and-of pixels. 

383 """ 

384 with defects.bulk_update(): 

385 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

386 for x0 in multipleX: 

387 index = np.where(x == x0) 

388 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

389 multipleY.sort() # Ensure that the y values are sorted to look for gaps. 

390 minY, maxY = np.min(multipleY), np.max(multipleY) 

391 # Next few lines: don't mask pixels in column if gap 

392 # of good pixels between two consecutive bad pixels is 

393 # larger or equal than 'goodPixelColumnGapThreshold'. 

394 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

395 if len(diffIndex) != 0: 

396 limits = [minY] # put the minimum first 

397 for gapIndex in diffIndex: 

398 limits.append(multipleY[gapIndex]) 

399 limits.append(multipleY[gapIndex+1]) 

400 limits.append(maxY) # maximum last 

401 for i in np.arange(0, len(limits)-1, 2): 

402 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

403 defects.append(s) 

404 else: # No gap is large enough 

405 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

406 defects.append(s) 

407 return defects 

408 

409 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

410 """Plot the defects found by the task. 

411 

412 Parameters 

413 ---------- 

414 stepname : `str` 

415 Debug frame to request. 

416 ampImage : `lsst.afw.image.MaskedImage` 

417 Amplifier image to display. 

418 defects : `lsst.ip.isr.Defects` 

419 The defects to plot. 

420 detector : `lsst.afw.cameraGeom.Detector` 

421 Detector holding camera geometry. 

422 """ 

423 frame = getDebugFrame(self._display, stepname) 

424 if frame: 

425 disp = afwDisplay.Display(frame=frame) 

426 disp.scale('asinh', 'zscale') 

427 disp.setMaskTransparency(80) 

428 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

429 

430 maskedIm = ampImage.clone() 

431 defects.maskPixels(maskedIm, "BAD") 

432 

433 mpDict = maskedIm.mask.getMaskPlaneDict() 

434 for plane in mpDict.keys(): 

435 if plane in ['BAD']: 

436 continue 

437 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

438 

439 disp.setImageColormap('gray') 

440 disp.mtv(maskedIm) 

441 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

442 prompt = "Press Enter to continue [c]... " 

443 while True: 

444 ans = input(prompt).lower() 

445 if ans in ('', 'c', ): 

446 break 

447 

448 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

449 """Make a histogram of the distribution of pixel values for 

450 each amp. 

451 

452 The main image data histogram is plotted in blue. Edge 

453 pixels, if masked, are in red. Note that masked edge pixels 

454 do not contribute to the underflow and overflow numbers. 

455 

456 Note that this currently only supports the 16-amp LSST 

457 detectors. 

458 

459 Parameters 

460 ---------- 

461 stepname : `str` 

462 Debug frame to request. 

463 ampImage : `lsst.afw.image.MaskedImage` 

464 Amplifier image to display. 

465 nSigmaUsed : `float` 

466 The number of sigma used for detection 

467 exp : `lsst.afw.image.exposure.Exposure` 

468 The exposure in which the defects were found. 

469 """ 

470 frame = getDebugFrame(self._display, stepname) 

471 if frame: 

472 import matplotlib.pyplot as plt 

473 

474 detector = exp.getDetector() 

475 nX = np.floor(np.sqrt(len(detector))) 

476 nY = len(detector) // nX 

477 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

478 

479 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

480 

481 for (amp, a) in zip(reversed(detector), ax.flatten()): 

482 mi = exp.maskedImage[amp.getBBox()] 

483 

484 # normalize by expTime as we plot in ADU/s and don't 

485 # always work with master calibs 

486 mi.image.array /= expTime 

487 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

488 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

489 # Get array of pixels 

490 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

491 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

492 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

493 

494 thrUpper = mean + nSigmaUsed*sigma 

495 thrLower = mean - nSigmaUsed*sigma 

496 

497 nRight = len(imgData[imgData > thrUpper]) 

498 nLeft = len(imgData[imgData < thrLower]) 

499 

500 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

501 leftEdge = mean - nsig * nSigmaUsed*sigma 

502 rightEdge = mean + nsig * nSigmaUsed*sigma 

503 nbins = np.linspace(leftEdge, rightEdge, 1000) 

504 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

505 lw=1, edgecolor='red') 

506 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

507 lw=3, edgecolor='blue') 

508 

509 # Report number of entries in over- and under-flow 

510 # bins, i.e. off the edges of the histogram 

511 nOverflow = len(imgData[imgData > rightEdge]) 

512 nUnderflow = len(imgData[imgData < leftEdge]) 

513 

514 # Put v-lines and textboxes in 

515 a.axvline(thrUpper, c='k') 

516 a.axvline(thrLower, c='k') 

517 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

518 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

519 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

520 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

521 

522 # set axis limits and scales 

523 a.set_ylim([1., 1.7*np.max(y)]) 

524 lPlot, rPlot = a.get_xlim() 

525 a.set_xlim(np.array([lPlot, rPlot])) 

526 a.set_yscale('log') 

527 a.set_xlabel("ADU/s") 

528 fig.show() 

529 prompt = "Press Enter or c to continue [chp]..." 

530 while True: 

531 ans = input(prompt).lower() 

532 if ans in ("", " ", "c",): 

533 break 

534 elif ans in ("p", ): 

535 import pdb 

536 pdb.set_trace() 

537 elif ans in ("h", ): 

538 print("[h]elp [c]ontinue [p]db") 

539 plt.close() 

540 

541 

542class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

543 dimensions=("instrument", "detector")): 

544 inputDefects = cT.Input( 

545 name="singleExpDefects", 

546 doc="Measured defect lists.", 

547 storageClass="Defects", 

548 dimensions=("instrument", "detector", "exposure"), 

549 multiple=True, 

550 ) 

551 camera = cT.PrerequisiteInput( 

552 name='camera', 

553 doc="Camera associated with these defects.", 

554 storageClass="Camera", 

555 dimensions=("instrument", ), 

556 isCalibration=True, 

557 lookupFunction=lookupStaticCalibration, 

558 ) 

559 

560 mergedDefects = cT.Output( 

561 name="defects", 

562 doc="Final merged defects.", 

563 storageClass="Defects", 

564 dimensions=("instrument", "detector"), 

565 multiple=False, 

566 isCalibration=True, 

567 ) 

568 

569 

570class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

571 pipelineConnections=MergeDefectsConnections): 

572 """Configuration for merging single exposure defects. 

573 """ 

574 

575 assertSameRun = pexConfig.Field( 

576 dtype=bool, 

577 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

578 "if the run key isn't found."), 

579 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

580 ) 

581 ignoreFilters = pexConfig.Field( 

582 dtype=bool, 

583 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

584 " images. Allows mixing of filters in the input flats. Set to False if you think" 

585 " your defects might be chromatic and want to have registry support for varying" 

586 " defects with respect to filter."), 

587 default=True, 

588 ) 

589 nullFilterName = pexConfig.Field( 

590 dtype=str, 

591 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

592 default="NONE", 

593 ) 

594 combinationMode = pexConfig.ChoiceField( 

595 doc="Which types of defects to identify", 

596 dtype=str, 

597 default="FRACTION", 

598 allowed={ 

599 "AND": "Logical AND the pixels found in each visit to form set ", 

600 "OR": "Logical OR the pixels found in each visit to form set ", 

601 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

602 } 

603 ) 

604 combinationFraction = pexConfig.RangeField( 

605 dtype=float, 

606 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

607 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

608 " mode AND to require pixel to appear in all images."), 

609 default=0.7, 

610 min=0, 

611 max=1, 

612 ) 

613 edgesAsDefects = pexConfig.Field( 

614 dtype=bool, 

615 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

616 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

617 " defect will be located there."), 

618 default=False, 

619 ) 

620 

621 

622class MergeDefectsTask(pipeBase.PipelineTask): 

623 """Merge the defects from multiple exposures. 

624 """ 

625 

626 ConfigClass = MergeDefectsTaskConfig 

627 _DefaultName = 'cpDefectMerge' 

628 

629 def run(self, inputDefects, camera): 

630 """Merge a list of single defects to find the common defect regions. 

631 

632 Parameters 

633 ---------- 

634 inputDefects : `list` [`lsst.ip.isr.Defects`] 

635 Partial defects from a single exposure. 

636 camera : `lsst.afw.cameraGeom.Camera` 

637 Camera to use for metadata. 

638 

639 Returns 

640 ------- 

641 results : `lsst.pipe.base.Struct` 

642 Results struct containing: 

643 

644 ``mergedDefects`` 

645 The defects merged from the input lists 

646 (`lsst.ip.isr.Defects`). 

647 """ 

648 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

649 if detectorId is None: 

650 raise RuntimeError("Cannot identify detector id.") 

651 detector = camera[detectorId] 

652 

653 imageTypes = set() 

654 for inDefect in inputDefects: 

655 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

656 imageTypes.add(imageType) 

657 

658 # Determine common defect pixels separately for each input image type. 

659 splitDefects = list() 

660 for imageType in imageTypes: 

661 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

662 count = 0 

663 for inDefect in inputDefects: 

664 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

665 count += 1 

666 for defect in inDefect: 

667 sumImage.image[defect.getBBox()] += 1.0 

668 sumImage /= count 

669 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

670 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

671 

672 if self.config.combinationMode == 'AND': 

673 threshold = 1.0 

674 elif self.config.combinationMode == 'OR': 

675 threshold = 0.0 

676 elif self.config.combinationMode == 'FRACTION': 

677 threshold = self.config.combinationFraction 

678 else: 

679 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

680 indices = np.where(sumImage.getImage().getArray() > threshold) 

681 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

682 sumImage.getMask().getArray()[indices] |= BADBIT 

683 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

684 partialDefect = Defects.fromMask(sumImage, 'BAD') 

685 splitDefects.append(partialDefect) 

686 

687 # Do final combination of separate image types 

688 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

689 for inDefect in splitDefects: 

690 for defect in inDefect: 

691 finalImage.image[defect.getBBox()] += 1 

692 finalImage /= len(splitDefects) 

693 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) 

694 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) 

695 

696 # This combination is the OR of all image types 

697 threshold = 0.0 

698 indices = np.where(finalImage.getImage().getArray() > threshold) 

699 BADBIT = finalImage.getMask().getPlaneBitMask('BAD') 

700 finalImage.getMask().getArray()[indices] |= BADBIT 

701 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) 

702 

703 if self.config.edgesAsDefects: 

704 self.log.info("Masking edge pixels as defects.") 

705 # Do the same as IsrTask.maskEdges() 

706 box = detector.getBBox() 

707 subImage = finalImage[box] 

708 box.grow(-self.nPixBorder) 

709 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

710 

711 merged = Defects.fromMask(finalImage, 'BAD') 

712 merged.updateMetadataFromExposures(inputDefects) 

713 merged.updateMetadata(camera=camera, detector=detector, filterName=None, 

714 setCalibId=True, setDate=True) 

715 

716 return pipeBase.Struct( 

717 mergedDefects=merged, 

718 )