Coverage for python/lsst/cp/pipe/defects.py: 19%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.pipe.base as pipeBase 

25import lsst.pipe.base.connectionTypes as cT 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.detection as afwDetection 

33import lsst.afw.display as afwDisplay 

34from lsst.afw import cameraGeom 

35from lsst.geom import Box2I, Point2I 

36from lsst.meas.algorithms import SourceDetectionTask 

37from lsst.ip.isr import IsrTask, Defects 

38from .utils import countMaskedPixels 

39from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

40from lsst.utils.timer import timeMethod 

41 

42from ._lookupStaticCalibration import lookupStaticCalibration 

43 

44__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

45 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

46 'FindDefectsTask', 'FindDefectsTaskConfig', ] 

47 

48 

49class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

50 dimensions=("instrument", "exposure", "detector")): 

51 inputExp = cT.Input( 

52 name="defectExps", 

53 doc="Input ISR-processed exposures to measure.", 

54 storageClass="Exposure", 

55 dimensions=("instrument", "detector", "exposure"), 

56 multiple=False 

57 ) 

58 camera = cT.PrerequisiteInput( 

59 name='camera', 

60 doc="Camera associated with this exposure.", 

61 storageClass="Camera", 

62 dimensions=("instrument", ), 

63 isCalibration=True, 

64 lookupFunction=lookupStaticCalibration, 

65 ) 

66 

67 outputDefects = cT.Output( 

68 name="singleExpDefects", 

69 doc="Output measured defects.", 

70 storageClass="Defects", 

71 dimensions=("instrument", "detector", "exposure"), 

72 ) 

73 

74 

75class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

76 pipelineConnections=MeasureDefectsConnections): 

77 """Configuration for measuring defects from a list of exposures 

78 """ 

79 

80 nSigmaBright = pexConfig.Field( 

81 dtype=float, 

82 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be" 

83 " appropriate for some LSST sensors in DM-17490."), 

84 default=4.8, 

85 ) 

86 nSigmaDark = pexConfig.Field( 

87 dtype=float, 

88 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be" 

89 " appropriate for some LSST sensors in DM-17490."), 

90 default=-5.0, 

91 ) 

92 nPixBorderUpDown = pexConfig.Field( 

93 dtype=int, 

94 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

95 default=7, 

96 ) 

97 nPixBorderLeftRight = pexConfig.Field( 

98 dtype=int, 

99 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

100 default=7, 

101 ) 

102 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

103 dtype=int, 

104 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

105 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

106 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

107 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

108 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

109 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

110 "'goodPixelColumnGapThreshold' are not marked as bad."), 

111 default=50, 

112 ) 

113 goodPixelColumnGapThreshold = pexConfig.Field( 

114 dtype=int, 

115 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

116 "'badOnAndOffPixelColumnThreshold')."), 

117 default=30, 

118 ) 

119 

120 def validate(self): 

121 super().validate() 

122 if self.nSigmaBright < 0.0: 

123 raise ValueError("nSigmaBright must be above 0.0.") 

124 if self.nSigmaDark > 0.0: 

125 raise ValueError("nSigmaDark must be below 0.0.") 

126 

127 

128class MeasureDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

129 """Measure the defects from one exposure. 

130 """ 

131 

132 ConfigClass = MeasureDefectsTaskConfig 

133 _DefaultName = 'cpDefectMeasure' 

134 

135 def run(self, inputExp, camera): 

136 """Measure one exposure for defects. 

137 

138 Parameters 

139 ---------- 

140 inputExp : `lsst.afw.image.Exposure` 

141 Exposure to examine. 

142 camera : `lsst.afw.cameraGeom.Camera` 

143 Camera to use for metadata. 

144 

145 Returns 

146 ------- 

147 results : `lsst.pipe.base.Struct` 

148 Results struct containing: 

149 

150 ``outputDefects`` 

151 The defects measured from this exposure 

152 (`lsst.ip.isr.Defects`). 

153 """ 

154 detector = inputExp.getDetector() 

155 

156 filterName = inputExp.getFilter().physicalLabel 

157 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

158 

159 if datasetType.lower() == 'dark': 

160 nSigmaList = [self.config.nSigmaBright] 

161 else: 

162 nSigmaList = [self.config.nSigmaBright, self.config.nSigmaDark] 

163 defects = self.findHotAndColdPixels(inputExp, nSigmaList) 

164 

165 msg = "Found %s defects containing %s pixels in %s" 

166 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

167 

168 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

169 setCalibId=True, setDate=True, 

170 cpDefectGenImageType=datasetType) 

171 

172 return pipeBase.Struct( 

173 outputDefects=defects, 

174 ) 

175 

176 @staticmethod 

177 def _nPixFromDefects(defects): 

178 """Count pixels in a defect. 

179 

180 Parameters 

181 ---------- 

182 defects : `lsst.ip.isr.Defects` 

183 Defects to measure. 

184 

185 Returns 

186 ------- 

187 nPix : `int` 

188 Number of defect pixels. 

189 """ 

190 nPix = 0 

191 for defect in defects: 

192 nPix += defect.getBBox().getArea() 

193 return nPix 

194 

195 def findHotAndColdPixels(self, exp, nSigma): 

196 """Find hot and cold pixels in an image. 

197 

198 Using config-defined thresholds on a per-amp basis, mask 

199 pixels that are nSigma above threshold in dark frames (hot 

200 pixels), or nSigma away from the clipped mean in flats (hot & 

201 cold pixels). 

202 

203 Parameters 

204 ---------- 

205 exp : `lsst.afw.image.exposure.Exposure` 

206 The exposure in which to find defects. 

207 nSigma : `list` [`float`] 

208 Detection threshold to use. Positive for DETECTED pixels, 

209 negative for DETECTED_NEGATIVE pixels. 

210 

211 Returns 

212 ------- 

213 defects : `lsst.ip.isr.Defects` 

214 The defects found in the image. 

215 """ 

216 self._setEdgeBits(exp) 

217 maskedIm = exp.maskedImage 

218 

219 # the detection polarity for afwDetection, True for positive, 

220 # False for negative, and therefore True for darks as they only have 

221 # bright pixels, and both for flats, as they have bright and dark pix 

222 footprintList = [] 

223 

224 for amp in exp.getDetector(): 

225 ampImg = maskedIm[amp.getBBox()].clone() 

226 

227 # crop ampImage depending on where the amp lies in the image 

228 if self.config.nPixBorderLeftRight: 

229 if ampImg.getX0() == 0: 

230 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

231 else: 

232 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

233 if self.config.nPixBorderUpDown: 

234 if ampImg.getY0() == 0: 

235 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

236 else: 

237 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

238 

239 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

240 continue 

241 

242 # Remove a background estimate 

243 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

244 

245 mergedSet = None 

246 for sigma in nSigma: 

247 nSig = np.abs(sigma) 

248 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

249 polarity = {-1: False, 1: True}[np.sign(sigma)] 

250 

251 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

252 

253 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

254 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

255 

256 if mergedSet is None: 

257 mergedSet = footprintSet 

258 else: 

259 mergedSet.merge(footprintSet) 

260 

261 footprintList += mergedSet.getFootprints() 

262 

263 self.debugView('defectMap', ampImg, 

264 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

265 

266 defects = Defects.fromFootprintList(footprintList) 

267 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

268 

269 return defects 

270 

271 @staticmethod 

272 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

273 """Return the number of non-bad pixels in the image.""" 

274 nPixels = maskedIm.mask.array.size 

275 nBad = countMaskedPixels(maskedIm, badMaskString) 

276 return nPixels - nBad 

277 

278 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

279 """Set edge bits on an exposure or maskedImage. 

280 

281 Raises 

282 ------ 

283 TypeError 

284 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

285 """ 

286 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

287 mi = exposureOrMaskedImage.maskedImage 

288 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

289 mi = exposureOrMaskedImage 

290 else: 

291 t = type(exposureOrMaskedImage) 

292 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

293 

294 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

295 if self.config.nPixBorderLeftRight: 

296 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

297 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

298 if self.config.nPixBorderUpDown: 

299 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

300 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

301 

302 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

303 """Mask blocks in a column if there are on-and-off bad pixels 

304 

305 If there's a column with on and off bad pixels, mask all the 

306 pixels in between, except if there is a large enough gap of 

307 consecutive good pixels between two bad pixels in the column. 

308 

309 Parameters 

310 ---------- 

311 defects : `lsst.ip.isr.Defects` 

312 The defects found in the image so far 

313 

314 Returns 

315 ------- 

316 defects : `lsst.ip.isr.Defects` 

317 If the number of bad pixels in a column is not larger or 

318 equal than self.config.badPixelColumnThreshold, the input 

319 list is returned. Otherwise, the defects list returned 

320 will include boxes that mask blocks of on-and-of pixels. 

321 """ 

322 # Get the (x, y) values of each bad pixel in amp. 

323 coordinates = [] 

324 for defect in defects: 

325 bbox = defect.getBBox() 

326 x0, y0 = bbox.getMinX(), bbox.getMinY() 

327 deltaX0, deltaY0 = bbox.getDimensions() 

328 for j in np.arange(y0, y0+deltaY0): 

329 for i in np.arange(x0, x0 + deltaX0): 

330 coordinates.append((i, j)) 

331 

332 x, y = [], [] 

333 for coordinatePair in coordinates: 

334 x.append(coordinatePair[0]) 

335 y.append(coordinatePair[1]) 

336 

337 x = np.array(x) 

338 y = np.array(y) 

339 # Find the defects with same "x" (vertical) coordinate (column). 

340 unique, counts = np.unique(x, return_counts=True) 

341 multipleX = [] 

342 for (a, b) in zip(unique, counts): 

343 if b >= self.config.badOnAndOffPixelColumnThreshold: 

344 multipleX.append(a) 

345 if len(multipleX) != 0: 

346 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

347 

348 return defects 

349 

350 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

351 """Mask blocks in a column if number of on-and-off bad pixels is above 

352 threshold. 

353 

354 This function is called if the number of on-and-off bad pixels 

355 in a column is larger or equal than 

356 self.config.badOnAndOffPixelColumnThreshold. 

357 

358 Parameters 

359 --------- 

360 x : `list` 

361 Lower left x coordinate of defect box. x coordinate is 

362 along the short axis if amp. 

363 y : `list` 

364 Lower left y coordinate of defect box. x coordinate is 

365 along the long axis if amp. 

366 multipleX : list 

367 List of x coordinates in amp. with multiple bad pixels 

368 (i.e., columns with defects). 

369 defects : `lsst.ip.isr.Defects` 

370 The defcts found in the image so far 

371 

372 Returns 

373 ------- 

374 defects : `lsst.ip.isr.Defects` 

375 The defects list returned that will include boxes that 

376 mask blocks of on-and-of pixels. 

377 """ 

378 with defects.bulk_update(): 

379 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

380 for x0 in multipleX: 

381 index = np.where(x == x0) 

382 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

383 minY, maxY = np.min(multipleY), np.max(multipleY) 

384 # Next few lines: don't mask pixels in column if gap 

385 # of good pixels between two consecutive bad pixels is 

386 # larger or equal than 'goodPixelColumnGapThreshold'. 

387 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

388 if len(diffIndex) != 0: 

389 limits = [minY] # put the minimum first 

390 for gapIndex in diffIndex: 

391 limits.append(multipleY[gapIndex]) 

392 limits.append(multipleY[gapIndex+1]) 

393 limits.append(maxY) # maximum last 

394 assert len(limits)%2 == 0, 'limits is even by design, but check anyways' 

395 for i in np.arange(0, len(limits)-1, 2): 

396 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

397 defects.append(s) 

398 else: # No gap is large enough 

399 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

400 defects.append(s) 

401 return defects 

402 

403 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

404 """Plot the defects found by the task. 

405 

406 Parameters 

407 ---------- 

408 stepname : `str` 

409 Debug frame to request. 

410 ampImage : `lsst.afw.image.MaskedImage` 

411 Amplifier image to display. 

412 defects : `lsst.ip.isr.Defects` 

413 The defects to plot. 

414 detector : `lsst.afw.cameraGeom.Detector` 

415 Detector holding camera geometry. 

416 """ 

417 frame = getDebugFrame(self._display, stepname) 

418 if frame: 

419 disp = afwDisplay.Display(frame=frame) 

420 disp.scale('asinh', 'zscale') 

421 disp.setMaskTransparency(80) 

422 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

423 

424 maskedIm = ampImage.clone() 

425 defects.maskPixels(maskedIm, "BAD") 

426 

427 mpDict = maskedIm.mask.getMaskPlaneDict() 

428 for plane in mpDict.keys(): 

429 if plane in ['BAD']: 

430 continue 

431 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

432 

433 disp.setImageColormap('gray') 

434 disp.mtv(maskedIm) 

435 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

436 prompt = "Press Enter to continue [c]... " 

437 while True: 

438 ans = input(prompt).lower() 

439 if ans in ('', 'c', ): 

440 break 

441 

442 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

443 """Make a histogram of the distribution of pixel values for 

444 each amp. 

445 

446 The main image data histogram is plotted in blue. Edge 

447 pixels, if masked, are in red. Note that masked edge pixels 

448 do not contribute to the underflow and overflow numbers. 

449 

450 Note that this currently only supports the 16-amp LSST 

451 detectors. 

452 

453 Parameters 

454 ---------- 

455 stepname : `str` 

456 Debug frame to request. 

457 ampImage : `lsst.afw.image.MaskedImage` 

458 Amplifier image to display. 

459 nSigmaUsed : `float` 

460 The number of sigma used for detection 

461 exp : `lsst.afw.image.exposure.Exposure` 

462 The exposure in which the defects were found. 

463 """ 

464 frame = getDebugFrame(self._display, stepname) 

465 if frame: 

466 import matplotlib.pyplot as plt 

467 

468 detector = exp.getDetector() 

469 nX = np.floor(np.sqrt(len(detector))) 

470 nY = len(detector) // nX 

471 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

472 

473 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

474 

475 for (amp, a) in zip(reversed(detector), ax.flatten()): 

476 mi = exp.maskedImage[amp.getBBox()] 

477 

478 # normalize by expTime as we plot in ADU/s and don't 

479 # always work with master calibs 

480 mi.image.array /= expTime 

481 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

482 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

483 # Get array of pixels 

484 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

485 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

486 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

487 

488 thrUpper = mean + nSigmaUsed*sigma 

489 thrLower = mean - nSigmaUsed*sigma 

490 

491 nRight = len(imgData[imgData > thrUpper]) 

492 nLeft = len(imgData[imgData < thrLower]) 

493 

494 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

495 leftEdge = mean - nsig * nSigmaUsed*sigma 

496 rightEdge = mean + nsig * nSigmaUsed*sigma 

497 nbins = np.linspace(leftEdge, rightEdge, 1000) 

498 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

499 lw=1, edgecolor='red') 

500 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

501 lw=3, edgecolor='blue') 

502 

503 # Report number of entries in over- and under-flow 

504 # bins, i.e. off the edges of the histogram 

505 nOverflow = len(imgData[imgData > rightEdge]) 

506 nUnderflow = len(imgData[imgData < leftEdge]) 

507 

508 # Put v-lines and textboxes in 

509 a.axvline(thrUpper, c='k') 

510 a.axvline(thrLower, c='k') 

511 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

512 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

513 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

514 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

515 

516 # set axis limits and scales 

517 a.set_ylim([1., 1.7*np.max(y)]) 

518 lPlot, rPlot = a.get_xlim() 

519 a.set_xlim(np.array([lPlot, rPlot])) 

520 a.set_yscale('log') 

521 a.set_xlabel("ADU/s") 

522 fig.show() 

523 prompt = "Press Enter or c to continue [chp]..." 

524 while True: 

525 ans = input(prompt).lower() 

526 if ans in ("", " ", "c",): 

527 break 

528 elif ans in ("p", ): 

529 import pdb 

530 pdb.set_trace() 

531 elif ans in ("h", ): 

532 print("[h]elp [c]ontinue [p]db") 

533 plt.close() 

534 

535 

536class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

537 dimensions=("instrument", "detector")): 

538 inputDefects = cT.Input( 

539 name="singleExpDefects", 

540 doc="Measured defect lists.", 

541 storageClass="Defects", 

542 dimensions=("instrument", "detector", "exposure"), 

543 multiple=True, 

544 ) 

545 camera = cT.PrerequisiteInput( 

546 name='camera', 

547 doc="Camera associated with these defects.", 

548 storageClass="Camera", 

549 dimensions=("instrument", ), 

550 isCalibration=True, 

551 lookupFunction=lookupStaticCalibration, 

552 ) 

553 

554 mergedDefects = cT.Output( 

555 name="defects", 

556 doc="Final merged defects.", 

557 storageClass="Defects", 

558 dimensions=("instrument", "detector"), 

559 multiple=False, 

560 isCalibration=True, 

561 ) 

562 

563 

564class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

565 pipelineConnections=MergeDefectsConnections): 

566 """Configuration for merging single exposure defects. 

567 """ 

568 

569 assertSameRun = pexConfig.Field( 

570 dtype=bool, 

571 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

572 "if the run key isn't found."), 

573 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

574 ) 

575 ignoreFilters = pexConfig.Field( 

576 dtype=bool, 

577 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

578 " images. Allows mixing of filters in the input flats. Set to False if you think" 

579 " your defects might be chromatic and want to have registry support for varying" 

580 " defects with respect to filter."), 

581 default=True, 

582 ) 

583 nullFilterName = pexConfig.Field( 

584 dtype=str, 

585 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

586 default="NONE", 

587 ) 

588 combinationMode = pexConfig.ChoiceField( 

589 doc="Which types of defects to identify", 

590 dtype=str, 

591 default="FRACTION", 

592 allowed={ 

593 "AND": "Logical AND the pixels found in each visit to form set ", 

594 "OR": "Logical OR the pixels found in each visit to form set ", 

595 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

596 } 

597 ) 

598 combinationFraction = pexConfig.RangeField( 

599 dtype=float, 

600 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

601 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

602 " mode AND to require pixel to appear in all images."), 

603 default=0.7, 

604 min=0, 

605 max=1, 

606 ) 

607 edgesAsDefects = pexConfig.Field( 

608 dtype=bool, 

609 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

610 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

611 " defect will be located there."), 

612 default=False, 

613 ) 

614 

615 

616class MergeDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

617 """Merge the defects from multiple exposures. 

618 """ 

619 

620 ConfigClass = MergeDefectsTaskConfig 

621 _DefaultName = 'cpDefectMerge' 

622 

623 def run(self, inputDefects, camera): 

624 """Merge a list of single defects to find the common defect regions. 

625 

626 Parameters 

627 ---------- 

628 inputDefects : `list` [`lsst.ip.isr.Defects`] 

629 Partial defects from a single exposure. 

630 camera : `lsst.afw.cameraGeom.Camera` 

631 Camera to use for metadata. 

632 

633 Returns 

634 ------- 

635 results : `lsst.pipe.base.Struct` 

636 Results struct containing: 

637 

638 ``mergedDefects`` 

639 The defects merged from the input lists 

640 (`lsst.ip.isr.Defects`). 

641 """ 

642 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

643 if detectorId is None: 

644 raise RuntimeError("Cannot identify detector id.") 

645 detector = camera[detectorId] 

646 

647 imageTypes = set() 

648 for inDefect in inputDefects: 

649 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

650 imageTypes.add(imageType) 

651 

652 # Determine common defect pixels separately for each input image type. 

653 splitDefects = list() 

654 for imageType in imageTypes: 

655 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

656 count = 0 

657 for inDefect in inputDefects: 

658 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

659 count += 1 

660 for defect in inDefect: 

661 sumImage.image[defect.getBBox()] += 1.0 

662 sumImage /= count 

663 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

664 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

665 

666 if self.config.combinationMode == 'AND': 

667 threshold = 1.0 

668 elif self.config.combinationMode == 'OR': 

669 threshold = 0.0 

670 elif self.config.combinationMode == 'FRACTION': 

671 threshold = self.config.combinationFraction 

672 else: 

673 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

674 indices = np.where(sumImage.getImage().getArray() > threshold) 

675 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

676 sumImage.getMask().getArray()[indices] |= BADBIT 

677 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

678 partialDefect = Defects.fromMask(sumImage, 'BAD') 

679 splitDefects.append(partialDefect) 

680 

681 # Do final combination of separate image types 

682 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

683 for inDefect in splitDefects: 

684 for defect in inDefect: 

685 finalImage.image[defect.getBBox()] += 1 

686 finalImage /= len(splitDefects) 

687 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) 

688 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) 

689 

690 # This combination is the OR of all image types 

691 threshold = 0.0 

692 indices = np.where(finalImage.getImage().getArray() > threshold) 

693 BADBIT = finalImage.getMask().getPlaneBitMask('BAD') 

694 finalImage.getMask().getArray()[indices] |= BADBIT 

695 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) 

696 

697 if self.config.edgesAsDefects: 

698 self.log.info("Masking edge pixels as defects.") 

699 # Do the same as IsrTask.maskEdges() 

700 box = detector.getBBox() 

701 subImage = finalImage[box] 

702 box.grow(-self.nPixBorder) 

703 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

704 

705 merged = Defects.fromMask(finalImage, 'BAD') 

706 merged.updateMetadata(camera=camera, detector=detector, filterName=None, 

707 setCalibId=True, setDate=True) 

708 

709 return pipeBase.Struct( 

710 mergedDefects=merged, 

711 ) 

712 

713 

714class FindDefectsTaskConfig(pexConfig.Config): 

715 measure = pexConfig.ConfigurableField( 

716 target=MeasureDefectsTask, 

717 doc="Task to measure single frame defects.", 

718 ) 

719 merge = pexConfig.ConfigurableField( 

720 target=MergeDefectsTask, 

721 doc="Task to merge multiple defects together.", 

722 ) 

723 

724 isrForFlats = pexConfig.ConfigurableField( 

725 target=IsrTask, 

726 doc="Task to perform instrumental signature removal", 

727 ) 

728 isrForDarks = pexConfig.ConfigurableField( 

729 target=IsrTask, 

730 doc="Task to perform instrumental signature removal", 

731 ) 

732 isrMandatoryStepsFlats = pexConfig.ListField( 

733 dtype=str, 

734 doc=("isr operations that must be performed for valid results when using flats." 

735 " Raises if any of these are False"), 

736 default=['doAssembleCcd', 'doFringe'] 

737 ) 

738 isrMandatoryStepsDarks = pexConfig.ListField( 

739 dtype=str, 

740 doc=("isr operations that must be performed for valid results when using darks. " 

741 "Raises if any of these are False"), 

742 default=['doAssembleCcd', 'doFringe'] 

743 ) 

744 isrForbiddenStepsFlats = pexConfig.ListField( 

745 dtype=str, 

746 doc=("isr operations that must NOT be performed for valid results when using flats." 

747 " Raises if any of these are True"), 

748 default=['doBrighterFatter', 'doUseOpticsTransmission', 

749 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

750 ) 

751 isrForbiddenStepsDarks = pexConfig.ListField( 

752 dtype=str, 

753 doc=("isr operations that must NOT be performed for valid results when using darks." 

754 " Raises if any of these are True"), 

755 default=['doBrighterFatter', 'doUseOpticsTransmission', 

756 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

757 ) 

758 isrDesirableSteps = pexConfig.ListField( 

759 dtype=str, 

760 doc=("isr operations that it is advisable to perform, but are not mission-critical." 

761 " WARNs are logged for any of these found to be False."), 

762 default=['doBias'] 

763 ) 

764 

765 ccdKey = pexConfig.Field( 

766 dtype=str, 

767 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'", 

768 default='ccd', 

769 ) 

770 imageTypeKey = pexConfig.Field( 

771 dtype=str, 

772 doc="The key for the butler to use by which to check whether images are darks or flats", 

773 default='imageType', 

774 ) 

775 

776 

777class FindDefectsTask(pipeBase.CmdLineTask): 

778 """Task for finding defects in sensors. 

779 

780 The task has two modes of operation, defect finding in raws and in 

781 master calibrations, which work as follows. 

782 

783 **Master calib defect finding** 

784 

785 A single visit number is supplied, for which the corresponding 

786 flat & dark will be used. This is because, at present at least, 

787 there is no way to pass a calibration exposure ID from the command 

788 line to a command line task. 

789 

790 The task retrieves the corresponding dark and flat exposures for 

791 the supplied visit. If a flat is available the task will (be able 

792 to) look for both bright and dark defects. If only a dark is found 

793 then only bright defects will be sought. 

794 

795 All pixels above/below the specified nSigma which lie with the 

796 specified borders for flats/darks are identified as defects. 

797 

798 **Raw visit defect finding** 

799 

800 A list of exposure IDs are supplied for defect finding. The task 

801 will detect bright pixels in the dark frames, if supplied, and 

802 bright & dark pixels in the flats, if supplied, i.e. if you only 

803 supply darks you will only be given bright defects. This is done 

804 automatically from the imageType of the exposure, so the input 

805 exposure list can be a mix. 

806 

807 As with the master calib detection, all pixels above/below the 

808 specified nSigma which lie with the specified borders for 

809 flats/darks are identified as defects. Then, a post-processing 

810 step is done to merge these detections, with pixels appearing in a 

811 fraction [0..1] of the images are kept as defects and those 

812 appearing below that occurrence-threshold are discarded. 

813 """ 

814 

815 ConfigClass = FindDefectsTaskConfig 

816 _DefaultName = "findDefects" 

817 

818 RunnerClass = DataRefListRunner 

819 

820 def __init__(self, **kwargs): 

821 super().__init__(**kwargs) 

822 self.makeSubtask("measure") 

823 self.makeSubtask("merge") 

824 

825 @timeMethod 

826 def runDataRef(self, dataRefList): 

827 """Run the defect finding task. 

828 

829 Find the defects, as described in the main task docstring, from a 

830 dataRef and a list of visit(s). 

831 

832 Parameters 

833 ---------- 

834 dataRefList : `list` [`lsst.daf.persistence.ButlerDataRef`] 

835 dataRefs for the data to be checked for defects. 

836 

837 Returns 

838 ------- 

839 result : `lsst.pipe.base.Struct` 

840 Result struct with Components: 

841 

842 ``defects`` 

843 The defects found by the task (`lsst.ip.isr.Defects`). 

844 ``exitStatus`` 

845 The exit code (`int`). 

846 """ 

847 dataRef = dataRefList[0] 

848 camera = dataRef.get("camera") 

849 

850 singleExpDefects = [] 

851 activeChip = None 

852 for dataRef in dataRefList: