Coverage for python/lsst/cp/pipe/defects.py: 22%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

311 statements  

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.pipe.base as pipeBase 

25import lsst.pipe.base.connectionTypes as cT 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.detection as afwDetection 

33import lsst.afw.display as afwDisplay 

34from lsst.afw import cameraGeom 

35from lsst.geom import Box2I, Point2I 

36from lsst.meas.algorithms import SourceDetectionTask 

37from lsst.ip.isr import IsrTask, Defects 

38from .utils import countMaskedPixels 

39from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

40 

41from ._lookupStaticCalibration import lookupStaticCalibration 

42 

43__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

44 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

45 'FindDefectsTask', 'FindDefectsTaskConfig', ] 

46 

47 

48class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

49 dimensions=("instrument", "exposure", "detector")): 

50 inputExp = cT.Input( 

51 name="defectExps", 

52 doc="Input ISR-processed exposures to measure.", 

53 storageClass="Exposure", 

54 dimensions=("instrument", "detector", "exposure"), 

55 multiple=False 

56 ) 

57 camera = cT.PrerequisiteInput( 

58 name='camera', 

59 doc="Camera associated with this exposure.", 

60 storageClass="Camera", 

61 dimensions=("instrument", ), 

62 isCalibration=True, 

63 lookupFunction=lookupStaticCalibration, 

64 ) 

65 

66 outputDefects = cT.Output( 

67 name="singleExpDefects", 

68 doc="Output measured defects.", 

69 storageClass="Defects", 

70 dimensions=("instrument", "detector", "exposure"), 

71 ) 

72 

73 

74class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=MeasureDefectsConnections): 

76 """Configuration for measuring defects from a list of exposures 

77 """ 

78 

79 nSigmaBright = pexConfig.Field( 

80 dtype=float, 

81 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be" 

82 " appropriate for some LSST sensors in DM-17490."), 

83 default=4.8, 

84 ) 

85 nSigmaDark = pexConfig.Field( 

86 dtype=float, 

87 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be" 

88 " appropriate for some LSST sensors in DM-17490."), 

89 default=-5.0, 

90 ) 

91 nPixBorderUpDown = pexConfig.Field( 

92 dtype=int, 

93 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

94 default=7, 

95 ) 

96 nPixBorderLeftRight = pexConfig.Field( 

97 dtype=int, 

98 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

99 default=7, 

100 ) 

101 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

102 dtype=int, 

103 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

104 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

105 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

106 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

107 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

108 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

109 "'goodPixelColumnGapThreshold' are not marked as bad."), 

110 default=50, 

111 ) 

112 goodPixelColumnGapThreshold = pexConfig.Field( 

113 dtype=int, 

114 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

115 "'badOnAndOffPixelColumnThreshold')."), 

116 default=30, 

117 ) 

118 

119 def validate(self): 

120 super().validate() 

121 if self.nSigmaBright < 0.0: 

122 raise ValueError("nSigmaBright must be above 0.0.") 

123 if self.nSigmaDark > 0.0: 

124 raise ValueError("nSigmaDark must be below 0.0.") 

125 

126 

127class MeasureDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

128 """Measure the defects from one exposure. 

129 """ 

130 

131 ConfigClass = MeasureDefectsTaskConfig 

132 _DefaultName = 'cpDefectMeasure' 

133 

134 def run(self, inputExp, camera): 

135 """Measure one exposure for defects. 

136 

137 Parameters 

138 ---------- 

139 inputExp : `lsst.afw.image.Exposure` 

140 Exposure to examine. 

141 camera : `lsst.afw.cameraGeom.Camera` 

142 Camera to use for metadata. 

143 

144 Returns 

145 ------- 

146 results : `lsst.pipe.base.Struct` 

147 Results struct containing: 

148 

149 ``outputDefects`` 

150 The defects measured from this exposure 

151 (`lsst.ip.isr.Defects`). 

152 """ 

153 detector = inputExp.getDetector() 

154 

155 filterName = inputExp.getFilterLabel().physicalLabel 

156 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

157 

158 if datasetType.lower() == 'dark': 

159 nSigmaList = [self.config.nSigmaBright] 

160 else: 

161 nSigmaList = [self.config.nSigmaBright, self.config.nSigmaDark] 

162 defects = self.findHotAndColdPixels(inputExp, nSigmaList) 

163 

164 msg = "Found %s defects containing %s pixels in %s" 

165 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

166 

167 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

168 setCalibId=True, setDate=True, 

169 cpDefectGenImageType=datasetType) 

170 

171 return pipeBase.Struct( 

172 outputDefects=defects, 

173 ) 

174 

175 @staticmethod 

176 def _nPixFromDefects(defects): 

177 """Count pixels in a defect. 

178 

179 Parameters 

180 ---------- 

181 defects : `lsst.ip.isr.Defects` 

182 Defects to measure. 

183 

184 Returns 

185 ------- 

186 nPix : `int` 

187 Number of defect pixels. 

188 """ 

189 nPix = 0 

190 for defect in defects: 

191 nPix += defect.getBBox().getArea() 

192 return nPix 

193 

194 def findHotAndColdPixels(self, exp, nSigma): 

195 """Find hot and cold pixels in an image. 

196 

197 Using config-defined thresholds on a per-amp basis, mask 

198 pixels that are nSigma above threshold in dark frames (hot 

199 pixels), or nSigma away from the clipped mean in flats (hot & 

200 cold pixels). 

201 

202 Parameters 

203 ---------- 

204 exp : `lsst.afw.image.exposure.Exposure` 

205 The exposure in which to find defects. 

206 nSigma : `list` [`float`] 

207 Detection threshold to use. Positive for DETECTED pixels, 

208 negative for DETECTED_NEGATIVE pixels. 

209 

210 Returns 

211 ------- 

212 defects : `lsst.ip.isr.Defects` 

213 The defects found in the image. 

214 """ 

215 self._setEdgeBits(exp) 

216 maskedIm = exp.maskedImage 

217 

218 # the detection polarity for afwDetection, True for positive, 

219 # False for negative, and therefore True for darks as they only have 

220 # bright pixels, and both for flats, as they have bright and dark pix 

221 footprintList = [] 

222 

223 for amp in exp.getDetector(): 

224 ampImg = maskedIm[amp.getBBox()].clone() 

225 

226 # crop ampImage depending on where the amp lies in the image 

227 if self.config.nPixBorderLeftRight: 

228 if ampImg.getX0() == 0: 

229 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

230 else: 

231 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

232 if self.config.nPixBorderUpDown: 

233 if ampImg.getY0() == 0: 

234 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

235 else: 

236 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

237 

238 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

239 continue 

240 

241 # Remove a background estimate 

242 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

243 

244 mergedSet = None 

245 for sigma in nSigma: 

246 nSig = np.abs(sigma) 

247 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

248 polarity = {-1: False, 1: True}[np.sign(sigma)] 

249 

250 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

251 

252 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

253 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

254 

255 if mergedSet is None: 

256 mergedSet = footprintSet 

257 else: 

258 mergedSet.merge(footprintSet) 

259 

260 footprintList += mergedSet.getFootprints() 

261 

262 self.debugView('defectMap', ampImg, 

263 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

264 

265 defects = Defects.fromFootprintList(footprintList) 

266 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

267 

268 return defects 

269 

270 @staticmethod 

271 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

272 """Return the number of non-bad pixels in the image.""" 

273 nPixels = maskedIm.mask.array.size 

274 nBad = countMaskedPixels(maskedIm, badMaskString) 

275 return nPixels - nBad 

276 

277 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

278 """Set edge bits on an exposure or maskedImage. 

279 

280 Raises 

281 ------ 

282 TypeError 

283 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

284 """ 

285 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

286 mi = exposureOrMaskedImage.maskedImage 

287 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

288 mi = exposureOrMaskedImage 

289 else: 

290 t = type(exposureOrMaskedImage) 

291 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

292 

293 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

294 if self.config.nPixBorderLeftRight: 

295 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

296 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

297 if self.config.nPixBorderUpDown: 

298 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

299 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

300 

301 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

302 """Mask blocks in a column if there are on-and-off bad pixels 

303 

304 If there's a column with on and off bad pixels, mask all the 

305 pixels in between, except if there is a large enough gap of 

306 consecutive good pixels between two bad pixels in the column. 

307 

308 Parameters 

309 ---------- 

310 defects : `lsst.ip.isr.Defects` 

311 The defects found in the image so far 

312 

313 Returns 

314 ------- 

315 defects : `lsst.ip.isr.Defects` 

316 If the number of bad pixels in a column is not larger or 

317 equal than self.config.badPixelColumnThreshold, the input 

318 list is returned. Otherwise, the defects list returned 

319 will include boxes that mask blocks of on-and-of pixels. 

320 """ 

321 # Get the (x, y) values of each bad pixel in amp. 

322 coordinates = [] 

323 for defect in defects: 

324 bbox = defect.getBBox() 

325 x0, y0 = bbox.getMinX(), bbox.getMinY() 

326 deltaX0, deltaY0 = bbox.getDimensions() 

327 for j in np.arange(y0, y0+deltaY0): 

328 for i in np.arange(x0, x0 + deltaX0): 

329 coordinates.append((i, j)) 

330 

331 x, y = [], [] 

332 for coordinatePair in coordinates: 

333 x.append(coordinatePair[0]) 

334 y.append(coordinatePair[1]) 

335 

336 x = np.array(x) 

337 y = np.array(y) 

338 # Find the defects with same "x" (vertical) coordinate (column). 

339 unique, counts = np.unique(x, return_counts=True) 

340 multipleX = [] 

341 for (a, b) in zip(unique, counts): 

342 if b >= self.config.badOnAndOffPixelColumnThreshold: 

343 multipleX.append(a) 

344 if len(multipleX) != 0: 

345 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

346 

347 return defects 

348 

349 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

350 """Mask blocks in a column if number of on-and-off bad pixels is above 

351 threshold. 

352 

353 This function is called if the number of on-and-off bad pixels 

354 in a column is larger or equal than 

355 self.config.badOnAndOffPixelColumnThreshold. 

356 

357 Parameters 

358 --------- 

359 x : `list` 

360 Lower left x coordinate of defect box. x coordinate is 

361 along the short axis if amp. 

362 y : `list` 

363 Lower left y coordinate of defect box. x coordinate is 

364 along the long axis if amp. 

365 multipleX : list 

366 List of x coordinates in amp. with multiple bad pixels 

367 (i.e., columns with defects). 

368 defects : `lsst.ip.isr.Defects` 

369 The defcts found in the image so far 

370 

371 Returns 

372 ------- 

373 defects : `lsst.ip.isr.Defects` 

374 The defects list returned that will include boxes that 

375 mask blocks of on-and-of pixels. 

376 """ 

377 with defects.bulk_update(): 

378 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

379 for x0 in multipleX: 

380 index = np.where(x == x0) 

381 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

382 minY, maxY = np.min(multipleY), np.max(multipleY) 

383 # Next few lines: don't mask pixels in column if gap 

384 # of good pixels between two consecutive bad pixels is 

385 # larger or equal than 'goodPixelColumnGapThreshold'. 

386 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

387 if len(diffIndex) != 0: 

388 limits = [minY] # put the minimum first 

389 for gapIndex in diffIndex: 

390 limits.append(multipleY[gapIndex]) 

391 limits.append(multipleY[gapIndex+1]) 

392 limits.append(maxY) # maximum last 

393 assert len(limits)%2 == 0, 'limits is even by design, but check anyways' 

394 for i in np.arange(0, len(limits)-1, 2): 

395 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

396 defects.append(s) 

397 else: # No gap is large enough 

398 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

399 defects.append(s) 

400 return defects 

401 

402 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

403 """Plot the defects found by the task. 

404 

405 Parameters 

406 ---------- 

407 stepname : `str` 

408 Debug frame to request. 

409 ampImage : `lsst.afw.image.MaskedImage` 

410 Amplifier image to display. 

411 defects : `lsst.ip.isr.Defects` 

412 The defects to plot. 

413 detector : `lsst.afw.cameraGeom.Detector` 

414 Detector holding camera geometry. 

415 """ 

416 frame = getDebugFrame(self._display, stepname) 

417 if frame: 

418 disp = afwDisplay.Display(frame=frame) 

419 disp.scale('asinh', 'zscale') 

420 disp.setMaskTransparency(80) 

421 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

422 

423 maskedIm = ampImage.clone() 

424 defects.maskPixels(maskedIm, "BAD") 

425 

426 mpDict = maskedIm.mask.getMaskPlaneDict() 

427 for plane in mpDict.keys(): 

428 if plane in ['BAD']: 

429 continue 

430 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

431 

432 disp.setImageColormap('gray') 

433 disp.mtv(maskedIm) 

434 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

435 prompt = "Press Enter to continue [c]... " 

436 while True: 

437 ans = input(prompt).lower() 

438 if ans in ('', 'c', ): 

439 break 

440 

441 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

442 """Make a histogram of the distribution of pixel values for 

443 each amp. 

444 

445 The main image data histogram is plotted in blue. Edge 

446 pixels, if masked, are in red. Note that masked edge pixels 

447 do not contribute to the underflow and overflow numbers. 

448 

449 Note that this currently only supports the 16-amp LSST 

450 detectors. 

451 

452 Parameters 

453 ---------- 

454 stepname : `str` 

455 Debug frame to request. 

456 ampImage : `lsst.afw.image.MaskedImage` 

457 Amplifier image to display. 

458 nSigmaUsed : `float` 

459 The number of sigma used for detection 

460 exp : `lsst.afw.image.exposure.Exposure` 

461 The exposure in which the defects were found. 

462 """ 

463 frame = getDebugFrame(self._display, stepname) 

464 if frame: 

465 import matplotlib.pyplot as plt 

466 

467 detector = exp.getDetector() 

468 nX = np.floor(np.sqrt(len(detector))) 

469 nY = len(detector) // nX 

470 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

471 

472 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

473 

474 for (amp, a) in zip(reversed(detector), ax.flatten()): 

475 mi = exp.maskedImage[amp.getBBox()] 

476 

477 # normalize by expTime as we plot in ADU/s and don't 

478 # always work with master calibs 

479 mi.image.array /= expTime 

480 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

481 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

482 # Get array of pixels 

483 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

484 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

485 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

486 

487 thrUpper = mean + nSigmaUsed*sigma 

488 thrLower = mean - nSigmaUsed*sigma 

489 

490 nRight = len(imgData[imgData > thrUpper]) 

491 nLeft = len(imgData[imgData < thrLower]) 

492 

493 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

494 leftEdge = mean - nsig * nSigmaUsed*sigma 

495 rightEdge = mean + nsig * nSigmaUsed*sigma 

496 nbins = np.linspace(leftEdge, rightEdge, 1000) 

497 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

498 lw=1, edgecolor='red') 

499 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

500 lw=3, edgecolor='blue') 

501 

502 # Report number of entries in over- and under-flow 

503 # bins, i.e. off the edges of the histogram 

504 nOverflow = len(imgData[imgData > rightEdge]) 

505 nUnderflow = len(imgData[imgData < leftEdge]) 

506 

507 # Put v-lines and textboxes in 

508 a.axvline(thrUpper, c='k') 

509 a.axvline(thrLower, c='k') 

510 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

511 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

512 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

513 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

514 

515 # set axis limits and scales 

516 a.set_ylim([1., 1.7*np.max(y)]) 

517 lPlot, rPlot = a.get_xlim() 

518 a.set_xlim(np.array([lPlot, rPlot])) 

519 a.set_yscale('log') 

520 a.set_xlabel("ADU/s") 

521 fig.show() 

522 prompt = "Press Enter or c to continue [chp]..." 

523 while True: 

524 ans = input(prompt).lower() 

525 if ans in ("", " ", "c",): 

526 break 

527 elif ans in ("p", ): 

528 import pdb 

529 pdb.set_trace() 

530 elif ans in ("h", ): 

531 print("[h]elp [c]ontinue [p]db") 

532 plt.close() 

533 

534 

535class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

536 dimensions=("instrument", "detector")): 

537 inputDefects = cT.Input( 

538 name="singleExpDefects", 

539 doc="Measured defect lists.", 

540 storageClass="Defects", 

541 dimensions=("instrument", "detector", "exposure"), 

542 multiple=True, 

543 ) 

544 camera = cT.PrerequisiteInput( 

545 name='camera', 

546 doc="Camera associated with these defects.", 

547 storageClass="Camera", 

548 dimensions=("instrument", ), 

549 isCalibration=True, 

550 lookupFunction=lookupStaticCalibration, 

551 ) 

552 

553 mergedDefects = cT.Output( 

554 name="defects", 

555 doc="Final merged defects.", 

556 storageClass="Defects", 

557 dimensions=("instrument", "detector"), 

558 multiple=False, 

559 isCalibration=True, 

560 ) 

561 

562 

563class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

564 pipelineConnections=MergeDefectsConnections): 

565 """Configuration for merging single exposure defects. 

566 """ 

567 

568 assertSameRun = pexConfig.Field( 

569 dtype=bool, 

570 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

571 "if the run key isn't found."), 

572 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

573 ) 

574 ignoreFilters = pexConfig.Field( 

575 dtype=bool, 

576 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

577 " images. Allows mixing of filters in the input flats. Set to False if you think" 

578 " your defects might be chromatic and want to have registry support for varying" 

579 " defects with respect to filter."), 

580 default=True, 

581 ) 

582 nullFilterName = pexConfig.Field( 

583 dtype=str, 

584 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

585 default="NONE", 

586 ) 

587 combinationMode = pexConfig.ChoiceField( 

588 doc="Which types of defects to identify", 

589 dtype=str, 

590 default="FRACTION", 

591 allowed={ 

592 "AND": "Logical AND the pixels found in each visit to form set ", 

593 "OR": "Logical OR the pixels found in each visit to form set ", 

594 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

595 } 

596 ) 

597 combinationFraction = pexConfig.RangeField( 

598 dtype=float, 

599 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

600 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

601 " mode AND to require pixel to appear in all images."), 

602 default=0.7, 

603 min=0, 

604 max=1, 

605 ) 

606 edgesAsDefects = pexConfig.Field( 

607 dtype=bool, 

608 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

609 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

610 " defect will be located there."), 

611 default=False, 

612 ) 

613 

614 

615class MergeDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

616 """Merge the defects from multiple exposures. 

617 """ 

618 

619 ConfigClass = MergeDefectsTaskConfig 

620 _DefaultName = 'cpDefectMerge' 

621 

622 def run(self, inputDefects, camera): 

623 """Merge a list of single defects to find the common defect regions. 

624 

625 Parameters 

626 ---------- 

627 inputDefects : `list` [`lsst.ip.isr.Defects`] 

628 Partial defects from a single exposure. 

629 camera : `lsst.afw.cameraGeom.Camera` 

630 Camera to use for metadata. 

631 

632 Returns 

633 ------- 

634 results : `lsst.pipe.base.Struct` 

635 Results struct containing: 

636 

637 ``mergedDefects`` 

638 The defects merged from the input lists 

639 (`lsst.ip.isr.Defects`). 

640 """ 

641 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

642 if detectorId is None: 

643 raise RuntimeError("Cannot identify detector id.") 

644 detector = camera[detectorId] 

645 

646 imageTypes = set() 

647 for inDefect in inputDefects: 

648 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

649 imageTypes.add(imageType) 

650 

651 # Determine common defect pixels separately for each input image type. 

652 splitDefects = list() 

653 for imageType in imageTypes: 

654 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

655 count = 0 

656 for inDefect in inputDefects: 

657 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

658 count += 1 

659 for defect in inDefect: 

660 sumImage.image[defect.getBBox()] += 1.0 

661 sumImage /= count 

662 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

663 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

664 

665 if self.config.combinationMode == 'AND': 

666 threshold = 1.0 

667 elif self.config.combinationMode == 'OR': 

668 threshold = 0.0 

669 elif self.config.combinationMode == 'FRACTION': 

670 threshold = self.config.combinationFraction 

671 else: 

672 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

673 indices = np.where(sumImage.getImage().getArray() > threshold) 

674 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

675 sumImage.getMask().getArray()[indices] |= BADBIT 

676 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

677 partialDefect = Defects.fromMask(sumImage, 'BAD') 

678 splitDefects.append(partialDefect) 

679 

680 # Do final combination of separate image types 

681 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

682 for inDefect in splitDefects: 

683 for defect in inDefect: 

684 finalImage.image[defect.getBBox()] += 1 

685 finalImage /= len(splitDefects) 

686 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) 

687 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) 

688 

689 # This combination is the OR of all image types 

690 threshold = 0.0 

691 indices = np.where(finalImage.getImage().getArray() > threshold) 

692 BADBIT = finalImage.getMask().getPlaneBitMask('BAD') 

693 finalImage.getMask().getArray()[indices] |= BADBIT 

694 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) 

695 

696 if self.config.edgesAsDefects: 

697 self.log.info("Masking edge pixels as defects.") 

698 # Do the same as IsrTask.maskEdges() 

699 box = detector.getBBox() 

700 subImage = finalImage[box] 

701 box.grow(-self.nPixBorder) 

702 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

703 

704 merged = Defects.fromMask(finalImage, 'BAD') 

705 merged.updateMetadata(camera=camera, detector=detector, filterName=None, 

706 setCalibId=True, setDate=True) 

707 

708 return pipeBase.Struct( 

709 mergedDefects=merged, 

710 ) 

711 

712 

713class FindDefectsTaskConfig(pexConfig.Config): 

714 measure = pexConfig.ConfigurableField( 

715 target=MeasureDefectsTask, 

716 doc="Task to measure single frame defects.", 

717 ) 

718 merge = pexConfig.ConfigurableField( 

719 target=MergeDefectsTask, 

720 doc="Task to merge multiple defects together.", 

721 ) 

722 

723 isrForFlats = pexConfig.ConfigurableField( 

724 target=IsrTask, 

725 doc="Task to perform instrumental signature removal", 

726 ) 

727 isrForDarks = pexConfig.ConfigurableField( 

728 target=IsrTask, 

729 doc="Task to perform instrumental signature removal", 

730 ) 

731 isrMandatoryStepsFlats = pexConfig.ListField( 

732 dtype=str, 

733 doc=("isr operations that must be performed for valid results when using flats." 

734 " Raises if any of these are False"), 

735 default=['doAssembleCcd', 'doFringe'] 

736 ) 

737 isrMandatoryStepsDarks = pexConfig.ListField( 

738 dtype=str, 

739 doc=("isr operations that must be performed for valid results when using darks. " 

740 "Raises if any of these are False"), 

741 default=['doAssembleCcd', 'doFringe'] 

742 ) 

743 isrForbiddenStepsFlats = pexConfig.ListField( 

744 dtype=str, 

745 doc=("isr operations that must NOT be performed for valid results when using flats." 

746 " Raises if any of these are True"), 

747 default=['doBrighterFatter', 'doUseOpticsTransmission', 

748 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

749 ) 

750 isrForbiddenStepsDarks = pexConfig.ListField( 

751 dtype=str, 

752 doc=("isr operations that must NOT be performed for valid results when using darks." 

753 " Raises if any of these are True"), 

754 default=['doBrighterFatter', 'doUseOpticsTransmission', 

755 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

756 ) 

757 isrDesirableSteps = pexConfig.ListField( 

758 dtype=str, 

759 doc=("isr operations that it is advisable to perform, but are not mission-critical." 

760 " WARNs are logged for any of these found to be False."), 

761 default=['doBias'] 

762 ) 

763 

764 ccdKey = pexConfig.Field( 

765 dtype=str, 

766 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'", 

767 default='ccd', 

768 ) 

769 imageTypeKey = pexConfig.Field( 

770 dtype=str, 

771 doc="The key for the butler to use by which to check whether images are darks or flats", 

772 default='imageType', 

773 ) 

774 

775 

776class FindDefectsTask(pipeBase.CmdLineTask): 

777 """Task for finding defects in sensors. 

778 

779 The task has two modes of operation, defect finding in raws and in 

780 master calibrations, which work as follows. 

781 

782 **Master calib defect finding** 

783 

784 A single visit number is supplied, for which the corresponding 

785 flat & dark will be used. This is because, at present at least, 

786 there is no way to pass a calibration exposure ID from the command 

787 line to a command line task. 

788 

789 The task retrieves the corresponding dark and flat exposures for 

790 the supplied visit. If a flat is available the task will (be able 

791 to) look for both bright and dark defects. If only a dark is found 

792 then only bright defects will be sought. 

793 

794 All pixels above/below the specified nSigma which lie with the 

795 specified borders for flats/darks are identified as defects. 

796 

797 **Raw visit defect finding** 

798 

799 A list of exposure IDs are supplied for defect finding. The task 

800 will detect bright pixels in the dark frames, if supplied, and 

801 bright & dark pixels in the flats, if supplied, i.e. if you only 

802 supply darks you will only be given bright defects. This is done 

803 automatically from the imageType of the exposure, so the input 

804 exposure list can be a mix. 

805 

806 As with the master calib detection, all pixels above/below the 

807 specified nSigma which lie with the specified borders for 

808 flats/darks are identified as defects. Then, a post-processing 

809 step is done to merge these detections, with pixels appearing in a 

810 fraction [0..1] of the images are kept as defects and those 

811 appearing below that occurrence-threshold are discarded. 

812 """ 

813 

814 ConfigClass = FindDefectsTaskConfig 

815 _DefaultName = "findDefects" 

816 

817 RunnerClass = DataRefListRunner 

818 

819 def __init__(self, **kwargs): 

820 super().__init__(**kwargs) 

821 self.makeSubtask("measure") 

822 self.makeSubtask("merge") 

823 

824 @pipeBase.timeMethod 

825 def runDataRef(self, dataRefList): 

826 """Run the defect finding task. 

827 

828 Find the defects, as described in the main task docstring, from a 

829 dataRef and a list of visit(s). 

830 

831 Parameters 

832 ---------- 

833 dataRefList : `list` [`lsst.daf.persistence.ButlerDataRef`] 

834 dataRefs for the data to be checked for defects. 

835 

836 Returns 

837 ------- 

838 result : `lsst.pipe.base.Struct` 

839 Result struct with Components: 

840 

841 ``defects`` 

842 The defects found by the task (`lsst.ip.isr.Defects`). 

843 ``exitStatus`` 

844 The exit code (`int`). 

845 """ 

846 dataRef = dataRefList[0] 

847 camera = dataRef.get("camera") 

848 

849 singleExpDefects = [] 

850 activeChip = None 

851 for dataRef in dataRefList: 

852 exposure = dataRef.get("postISRCCD") 

853 if activeChip: 

854 if exposure.getDetector().getName() != activeChip: 

855 raise RuntimeError("Too many input detectors supplied!") 

856 else: 

857 activeChip = exposure.getDetector().getName() 

858 

859 result = self.measure.run(exposure, camera) 

860 singleExpDefects.append(result.outputDefects) 

861 

862 finalResults = self.merge.run(singleExpDefects, camera) 

863 metadata = finalResults.mergedDefects.getMetadata() 

864 inputDims = {'calibDate': metadata['CALIBDATE'], 

865 'raftName': metadata['RAFTNAME'], 

866 'detectorName': metadata['SLOTNAME'], 

867 'detector': metadata['DETECTOR'], 

868 'ccd': metadata['DETECTOR'], 

869 'ccdnum': metadata['DETECTOR']} 

870 

871 butler = dataRef.getButler() 

872 butler.put(finalResults.mergedDefects, "defects", inputDims) 

873 

874 return finalResults