Coverage for python/lsst/cp/pipe/defects.py: 20%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

24 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask', 

26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ] 

27 

28import numpy as np 

29 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32 

33from lsstDebug import getDebugFrame 

34import lsst.pex.config as pexConfig 

35 

36import lsst.afw.image as afwImage 

37import lsst.afw.math as afwMath 

38import lsst.afw.detection as afwDetection 

39import lsst.afw.display as afwDisplay 

40from lsst.afw import cameraGeom 

41from lsst.geom import Box2I, Point2I 

42from lsst.meas.algorithms import SourceDetectionTask 

43from lsst.ip.isr import Defects, countMaskedPixels 

44from lsst.pex.exceptions import InvalidParameterError 

45 

46 

47class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("instrument", "exposure", "detector")): 

49 inputExp = cT.Input( 

50 name="defectExps", 

51 doc="Input ISR-processed exposures to measure.", 

52 storageClass="Exposure", 

53 dimensions=("instrument", "detector", "exposure"), 

54 multiple=False 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name='camera', 

58 doc="Camera associated with this exposure.", 

59 storageClass="Camera", 

60 dimensions=("instrument", ), 

61 isCalibration=True, 

62 ) 

63 

64 outputDefects = cT.Output( 

65 name="singleExpDefects", 

66 doc="Output measured defects.", 

67 storageClass="Defects", 

68 dimensions=("instrument", "detector", "exposure"), 

69 ) 

70 

71 

72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=MeasureDefectsConnections): 

74 """Configuration for measuring defects from a list of exposures 

75 """ 

76 

77 thresholdType = pexConfig.ChoiceField( 

78 dtype=str, 

79 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found " 

80 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found " 

81 "in flats, and hot pixels in darks."), 

82 default='STDEV', 

83 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.", 

84 'VALUE': "Use pixel value to determine detection threshold."}, 

85 ) 

86 darkCurrentThreshold = pexConfig.Field( 

87 dtype=float, 

88 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define " 

89 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."), 

90 default=5, 

91 ) 

92 fracThresholdFlat = pexConfig.Field( 

93 dtype=float, 

94 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark " 

95 "pixels in flat images (fraction of the mean value per amplifier)." 

96 "Unused if thresholdType==``STDEV``."), 

97 default=0.8, 

98 ) 

99 nSigmaBright = pexConfig.Field( 

100 dtype=float, 

101 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot " 

102 "pixel detection. The default value was found to be " 

103 "appropriate for some LSST sensors in DM-17490. " 

104 "Unused if thresholdType==``VALUE``"), 

105 default=4.8, 

106 ) 

107 nSigmaDark = pexConfig.Field( 

108 dtype=float, 

109 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel " 

110 "detection. The default value was found to be " 

111 "appropriate for some LSST sensors in DM-17490. " 

112 "Unused if thresholdType==``VALUE``"), 

113 default=-5.0, 

114 ) 

115 nPixBorderUpDown = pexConfig.Field( 

116 dtype=int, 

117 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

118 default=7, 

119 ) 

120 nPixBorderLeftRight = pexConfig.Field( 

121 dtype=int, 

122 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

123 default=7, 

124 ) 

125 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

126 dtype=int, 

127 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

128 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

129 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

130 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

131 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

132 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

133 "'goodPixelColumnGapThreshold' are not marked as bad."), 

134 default=50, 

135 ) 

136 goodPixelColumnGapThreshold = pexConfig.Field( 

137 dtype=int, 

138 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

139 "'badOnAndOffPixelColumnThreshold')."), 

140 default=30, 

141 ) 

142 

143 def validate(self): 

144 super().validate() 

145 if self.nSigmaBright < 0.0: 

146 raise ValueError("nSigmaBright must be above 0.0.") 

147 if self.nSigmaDark > 0.0: 

148 raise ValueError("nSigmaDark must be below 0.0.") 

149 

150 

151class MeasureDefectsTask(pipeBase.PipelineTask): 

152 """Measure the defects from one exposure. 

153 """ 

154 

155 ConfigClass = MeasureDefectsTaskConfig 

156 _DefaultName = 'cpDefectMeasure' 

157 

158 def run(self, inputExp, camera): 

159 """Measure one exposure for defects. 

160 

161 Parameters 

162 ---------- 

163 inputExp : `lsst.afw.image.Exposure` 

164 Exposure to examine. 

165 camera : `lsst.afw.cameraGeom.Camera` 

166 Camera to use for metadata. 

167 

168 Returns 

169 ------- 

170 results : `lsst.pipe.base.Struct` 

171 Results struct containing: 

172 

173 ``outputDefects`` 

174 The defects measured from this exposure 

175 (`lsst.ip.isr.Defects`). 

176 """ 

177 detector = inputExp.getDetector() 

178 try: 

179 filterName = inputExp.getFilter().physicalLabel 

180 except AttributeError: 

181 filterName = None 

182 

183 defects = self._findHotAndColdPixels(inputExp) 

184 

185 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

186 msg = "Found %s defects containing %s pixels in %s" 

187 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

188 

189 defects.updateMetadataFromExposures([inputExp]) 

190 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

191 setCalibId=True, setDate=True, 

192 cpDefectGenImageType=datasetType) 

193 

194 return pipeBase.Struct( 

195 outputDefects=defects, 

196 ) 

197 

198 @staticmethod 

199 def _nPixFromDefects(defects): 

200 """Count pixels in a defect. 

201 

202 Parameters 

203 ---------- 

204 defects : `lsst.ip.isr.Defects` 

205 Defects to measure. 

206 

207 Returns 

208 ------- 

209 nPix : `int` 

210 Number of defect pixels. 

211 """ 

212 nPix = 0 

213 for defect in defects: 

214 nPix += defect.getBBox().getArea() 

215 return nPix 

216 

217 def _findHotAndColdPixels(self, exp): 

218 """Find hot and cold pixels in an image. 

219 

220 Using config-defined thresholds on a per-amp basis, mask 

221 pixels that are nSigma above threshold in dark frames (hot 

222 pixels), or nSigma away from the clipped mean in flats (hot & 

223 cold pixels). 

224 

225 Parameters 

226 ---------- 

227 exp : `lsst.afw.image.exposure.Exposure` 

228 The exposure in which to find defects. 

229 

230 Returns 

231 ------- 

232 defects : `lsst.ip.isr.Defects` 

233 The defects found in the image. 

234 """ 

235 self._setEdgeBits(exp) 

236 maskedIm = exp.maskedImage 

237 

238 # the detection polarity for afwDetection, True for positive, 

239 # False for negative, and therefore True for darks as they only have 

240 # bright pixels, and both for flats, as they have bright and dark pix 

241 footprintList = [] 

242 

243 for amp in exp.getDetector(): 

244 ampImg = maskedIm[amp.getBBox()].clone() 

245 

246 # crop ampImage depending on where the amp lies in the image 

247 if self.config.nPixBorderLeftRight: 

248 if ampImg.getX0() == 0: 

249 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

250 else: 

251 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

252 if self.config.nPixBorderUpDown: 

253 if ampImg.getY0() == 0: 

254 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

255 else: 

256 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

257 

258 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

259 continue 

260 

261 # Remove a background estimate 

262 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

263 ampImg -= meanClip 

264 

265 # Determine thresholds 

266 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue() 

267 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

268 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

269 if np.isnan(expTime): 

270 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second", 

271 expTime, amp.getName(), datasetType) 

272 expTime = 1. 

273 thresholdType = self.config.thresholdType 

274 if thresholdType == 'VALUE': 

275 # LCA-128 and eoTest: bright/hot pixels in dark images are 

276 # defined as any pixel with more than 5 e-/s of dark current. 

277 # We scale by the exposure time. 

278 if datasetType.lower() == 'dark': 

279 # hot pixel threshold 

280 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain() 

281 else: 

282 # LCA-128 and eoTest: dark/cold pixels in flat images as 

283 # defined as any pixel with photoresponse <80% of 

284 # the mean (at 500nm). 

285 

286 # We subtracted the mean above, so the threshold will be 

287 # negative cold pixel threshold. 

288 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip 

289 # Find equivalent sigma values. 

290 if stDev == 0.0: 

291 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.", 

292 stDev, amp.getName(), datasetType) 

293 nSigmaList = [np.inf] 

294 else: 

295 nSigmaList = [valueThreshold/stDev] 

296 else: 

297 hotPixelThreshold = self.config.nSigmaBright 

298 coldPixelThreshold = self.config.nSigmaDark 

299 if datasetType.lower() == 'dark': 

300 nSigmaList = [hotPixelThreshold] 

301 valueThreshold = stDev*hotPixelThreshold 

302 else: 

303 nSigmaList = [hotPixelThreshold, coldPixelThreshold] 

304 valueThreshold = [x*stDev for x in nSigmaList] 

305 

306 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel" 

307 "Values (hot and cold pixels thresholds): %s, %s", 

308 datasetType, amp.getName(), thresholdType, nSigmaList, valueThreshold) 

309 mergedSet = None 

310 for sigma in nSigmaList: 

311 nSig = np.abs(sigma) 

312 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

313 polarity = {-1: False, 1: True}[np.sign(sigma)] 

314 

315 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

316 

317 try: 

318 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

319 except InvalidParameterError: 

320 # This occurs if the image sigma value is 0.0. 

321 # Let's mask the whole area. 

322 minValue = np.nanmin(ampImg.image.array) - 1.0 

323 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True) 

324 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

325 

326 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

327 

328 if mergedSet is None: 

329 mergedSet = footprintSet 

330 else: 

331 mergedSet.merge(footprintSet) 

332 

333 footprintList += mergedSet.getFootprints() 

334 

335 self.debugView('defectMap', ampImg, 

336 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

337 

338 defects = Defects.fromFootprintList(footprintList) 

339 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

340 

341 return defects 

342 

343 @staticmethod 

344 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

345 """Return the number of non-bad pixels in the image.""" 

346 nPixels = maskedIm.mask.array.size 

347 nBad = countMaskedPixels(maskedIm, badMaskString) 

348 return nPixels - nBad 

349 

350 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

351 """Set edge bits on an exposure or maskedImage. 

352 

353 Raises 

354 ------ 

355 TypeError 

356 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

357 """ 

358 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

359 mi = exposureOrMaskedImage.maskedImage 

360 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

361 mi = exposureOrMaskedImage 

362 else: 

363 t = type(exposureOrMaskedImage) 

364 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

365 

366 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

367 if self.config.nPixBorderLeftRight: 

368 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

369 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

370 if self.config.nPixBorderUpDown: 

371 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

372 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

373 

374 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

375 """Mask blocks in a column if there are on-and-off bad pixels 

376 

377 If there's a column with on and off bad pixels, mask all the 

378 pixels in between, except if there is a large enough gap of 

379 consecutive good pixels between two bad pixels in the column. 

380 

381 Parameters 

382 ---------- 

383 defects : `lsst.ip.isr.Defects` 

384 The defects found in the image so far 

385 

386 Returns 

387 ------- 

388 defects : `lsst.ip.isr.Defects` 

389 If the number of bad pixels in a column is not larger or 

390 equal than self.config.badPixelColumnThreshold, the input 

391 list is returned. Otherwise, the defects list returned 

392 will include boxes that mask blocks of on-and-of pixels. 

393 """ 

394 # Get the (x, y) values of each bad pixel in amp. 

395 coordinates = [] 

396 for defect in defects: 

397 bbox = defect.getBBox() 

398 x0, y0 = bbox.getMinX(), bbox.getMinY() 

399 deltaX0, deltaY0 = bbox.getDimensions() 

400 for j in np.arange(y0, y0+deltaY0): 

401 for i in np.arange(x0, x0 + deltaX0): 

402 coordinates.append((i, j)) 

403 

404 x, y = [], [] 

405 for coordinatePair in coordinates: 

406 x.append(coordinatePair[0]) 

407 y.append(coordinatePair[1]) 

408 

409 x = np.array(x) 

410 y = np.array(y) 

411 # Find the defects with same "x" (vertical) coordinate (column). 

412 unique, counts = np.unique(x, return_counts=True) 

413 multipleX = [] 

414 for (a, b) in zip(unique, counts): 

415 if b >= self.config.badOnAndOffPixelColumnThreshold: 

416 multipleX.append(a) 

417 if len(multipleX) != 0: 

418 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

419 

420 return defects 

421 

422 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

423 """Mask blocks in a column if number of on-and-off bad pixels is above 

424 threshold. 

425 

426 This function is called if the number of on-and-off bad pixels 

427 in a column is larger or equal than 

428 self.config.badOnAndOffPixelColumnThreshold. 

429 

430 Parameters 

431 --------- 

432 x : `list` 

433 Lower left x coordinate of defect box. x coordinate is 

434 along the short axis if amp. 

435 y : `list` 

436 Lower left y coordinate of defect box. x coordinate is 

437 along the long axis if amp. 

438 multipleX : list 

439 List of x coordinates in amp. with multiple bad pixels 

440 (i.e., columns with defects). 

441 defects : `lsst.ip.isr.Defects` 

442 The defcts found in the image so far 

443 

444 Returns 

445 ------- 

446 defects : `lsst.ip.isr.Defects` 

447 The defects list returned that will include boxes that 

448 mask blocks of on-and-of pixels. 

449 """ 

450 with defects.bulk_update(): 

451 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

452 for x0 in multipleX: 

453 index = np.where(x == x0) 

454 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

455 multipleY.sort() # Ensure that the y values are sorted to look for gaps. 

456 minY, maxY = np.min(multipleY), np.max(multipleY) 

457 # Next few lines: don't mask pixels in column if gap 

458 # of good pixels between two consecutive bad pixels is 

459 # larger or equal than 'goodPixelColumnGapThreshold'. 

460 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

461 if len(diffIndex) != 0: 

462 limits = [minY] # put the minimum first 

463 for gapIndex in diffIndex: 

464 limits.append(multipleY[gapIndex]) 

465 limits.append(multipleY[gapIndex+1]) 

466 limits.append(maxY) # maximum last 

467 for i in np.arange(0, len(limits)-1, 2): 

468 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

469 defects.append(s) 

470 else: # No gap is large enough 

471 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

472 defects.append(s) 

473 return defects 

474 

475 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

476 """Plot the defects found by the task. 

477 

478 Parameters 

479 ---------- 

480 stepname : `str` 

481 Debug frame to request. 

482 ampImage : `lsst.afw.image.MaskedImage` 

483 Amplifier image to display. 

484 defects : `lsst.ip.isr.Defects` 

485 The defects to plot. 

486 detector : `lsst.afw.cameraGeom.Detector` 

487 Detector holding camera geometry. 

488 """ 

489 frame = getDebugFrame(self._display, stepname) 

490 if frame: 

491 disp = afwDisplay.Display(frame=frame) 

492 disp.scale('asinh', 'zscale') 

493 disp.setMaskTransparency(80) 

494 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

495 

496 maskedIm = ampImage.clone() 

497 defects.maskPixels(maskedIm, "BAD") 

498 

499 mpDict = maskedIm.mask.getMaskPlaneDict() 

500 for plane in mpDict.keys(): 

501 if plane in ['BAD']: 

502 continue 

503 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

504 

505 disp.setImageColormap('gray') 

506 disp.mtv(maskedIm) 

507 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

508 prompt = "Press Enter to continue [c]... " 

509 while True: 

510 ans = input(prompt).lower() 

511 if ans in ('', 'c', ): 

512 break 

513 

514 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

515 """Make a histogram of the distribution of pixel values for 

516 each amp. 

517 

518 The main image data histogram is plotted in blue. Edge 

519 pixels, if masked, are in red. Note that masked edge pixels 

520 do not contribute to the underflow and overflow numbers. 

521 

522 Note that this currently only supports the 16-amp LSST 

523 detectors. 

524 

525 Parameters 

526 ---------- 

527 stepname : `str` 

528 Debug frame to request. 

529 ampImage : `lsst.afw.image.MaskedImage` 

530 Amplifier image to display. 

531 nSigmaUsed : `float` 

532 The number of sigma used for detection 

533 exp : `lsst.afw.image.exposure.Exposure` 

534 The exposure in which the defects were found. 

535 """ 

536 frame = getDebugFrame(self._display, stepname) 

537 if frame: 

538 import matplotlib.pyplot as plt 

539 

540 detector = exp.getDetector() 

541 nX = np.floor(np.sqrt(len(detector))) 

542 nY = len(detector) // nX 

543 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

544 

545 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

546 

547 for (amp, a) in zip(reversed(detector), ax.flatten()): 

548 mi = exp.maskedImage[amp.getBBox()] 

549 

550 # normalize by expTime as we plot in ADU/s and don't 

551 # always work with master calibs 

552 mi.image.array /= expTime 

553 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

554 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

555 # Get array of pixels 

556 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

557 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

558 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

559 

560 thrUpper = mean + nSigmaUsed*sigma 

561 thrLower = mean - nSigmaUsed*sigma 

562 

563 nRight = len(imgData[imgData > thrUpper]) 

564 nLeft = len(imgData[imgData < thrLower]) 

565 

566 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

567 leftEdge = mean - nsig * nSigmaUsed*sigma 

568 rightEdge = mean + nsig * nSigmaUsed*sigma 

569 nbins = np.linspace(leftEdge, rightEdge, 1000) 

570 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

571 lw=1, edgecolor='red') 

572 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

573 lw=3, edgecolor='blue') 

574 

575 # Report number of entries in over- and under-flow 

576 # bins, i.e. off the edges of the histogram 

577 nOverflow = len(imgData[imgData > rightEdge]) 

578 nUnderflow = len(imgData[imgData < leftEdge]) 

579 

580 # Put v-lines and textboxes in 

581 a.axvline(thrUpper, c='k') 

582 a.axvline(thrLower, c='k') 

583 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

584 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

585 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

586 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

587 

588 # set axis limits and scales 

589 a.set_ylim([1., 1.7*np.max(y)]) 

590 lPlot, rPlot = a.get_xlim() 

591 a.set_xlim(np.array([lPlot, rPlot])) 

592 a.set_yscale('log') 

593 a.set_xlabel("ADU/s") 

594 fig.show() 

595 prompt = "Press Enter or c to continue [chp]..." 

596 while True: 

597 ans = input(prompt).lower() 

598 if ans in ("", " ", "c",): 

599 break 

600 elif ans in ("p", ): 

601 import pdb 

602 pdb.set_trace() 

603 elif ans in ("h", ): 

604 print("[h]elp [c]ontinue [p]db") 

605 plt.close() 

606 

607 

608class MeasureDefectsCombinedConnections(pipeBase.PipelineTaskConnections, 

609 dimensions=("instrument", "detector")): 

610 inputExp = cT.Input( 

611 name="dark", 

612 doc="Input ISR-processed combined exposure to measure.", 

613 storageClass="ExposureF", 

614 dimensions=("instrument", "detector"), 

615 multiple=False, 

616 isCalibration=True, 

617 ) 

618 camera = cT.PrerequisiteInput( 

619 name='camera', 

620 doc="Camera associated with this exposure.", 

621 storageClass="Camera", 

622 dimensions=("instrument", ), 

623 isCalibration=True, 

624 ) 

625 

626 outputDefects = cT.Output( 

627 name="cpPartialDefectsFromDarkCombined", 

628 doc="Output measured defects.", 

629 storageClass="Defects", 

630 dimensions=("instrument", "detector"), 

631 ) 

632 

633 

634class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig, 

635 pipelineConnections=MeasureDefectsCombinedConnections): 

636 """Configuration for measuring defects from combined exposures. 

637 """ 

638 pass 

639 

640 

641class MeasureDefectsCombinedTask(MeasureDefectsTask): 

642 """Task to measure defects in combined images.""" 

643 

644 ConfigClass = MeasureDefectsCombinedTaskConfig 

645 _DefaultName = "cpDefectMeasureCombined" 

646 

647 

648class MeasureDefectsCombinedWithFilterConnections(pipeBase.PipelineTaskConnections, 

649 dimensions=("instrument", "detector", "physical_filter")): 

650 """Task to measure defects in combined flats under a certain filter.""" 

651 inputExp = cT.Input( 

652 name="flat", 

653 doc="Input ISR-processed combined exposure to measure.", 

654 storageClass="ExposureF", 

655 dimensions=("instrument", "detector", "physical_filter"), 

656 multiple=False, 

657 isCalibration=True, 

658 ) 

659 camera = cT.PrerequisiteInput( 

660 name='camera', 

661 doc="Camera associated with this exposure.", 

662 storageClass="Camera", 

663 dimensions=("instrument", ), 

664 isCalibration=True, 

665 ) 

666 

667 outputDefects = cT.Output( 

668 name="cpPartialDefectsFromFlatCombinedWithFilter", 

669 doc="Output measured defects.", 

670 storageClass="Defects", 

671 dimensions=("instrument", "detector", "physical_filter"), 

672 ) 

673 

674 

675class MeasureDefectsCombinedWithFilterTaskConfig( 

676 MeasureDefectsTaskConfig, 

677 pipelineConnections=MeasureDefectsCombinedWithFilterConnections): 

678 """Configuration for measuring defects from combined exposures. 

679 """ 

680 pass 

681 

682 

683class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask): 

684 """Task to measure defects in combined images.""" 

685 

686 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig 

687 _DefaultName = "cpDefectMeasureWithFilterCombined" 

688 

689 

690class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

691 dimensions=("instrument", "detector")): 

692 inputDefects = cT.Input( 

693 name="singleExpDefects", 

694 doc="Measured defect lists.", 

695 storageClass="Defects", 

696 dimensions=("instrument", "detector", "exposure",), 

697 multiple=True, 

698 ) 

699 camera = cT.PrerequisiteInput( 

700 name='camera', 

701 doc="Camera associated with these defects.", 

702 storageClass="Camera", 

703 dimensions=("instrument", ), 

704 isCalibration=True, 

705 ) 

706 

707 mergedDefects = cT.Output( 

708 name="defects", 

709 doc="Final merged defects.", 

710 storageClass="Defects", 

711 dimensions=("instrument", "detector"), 

712 multiple=False, 

713 isCalibration=True, 

714 ) 

715 

716 

717class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

718 pipelineConnections=MergeDefectsConnections): 

719 """Configuration for merging single exposure defects. 

720 """ 

721 

722 assertSameRun = pexConfig.Field( 

723 dtype=bool, 

724 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

725 "if the run key isn't found."), 

726 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

727 ) 

728 ignoreFilters = pexConfig.Field( 

729 dtype=bool, 

730 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

731 " images. Allows mixing of filters in the input flats. Set to False if you think" 

732 " your defects might be chromatic and want to have registry support for varying" 

733 " defects with respect to filter."), 

734 default=True, 

735 ) 

736 nullFilterName = pexConfig.Field( 

737 dtype=str, 

738 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

739 default="NONE", 

740 ) 

741 combinationMode = pexConfig.ChoiceField( 

742 doc="Which types of defects to identify", 

743 dtype=str, 

744 default="FRACTION", 

745 allowed={ 

746 "AND": "Logical AND the pixels found in each visit to form set ", 

747 "OR": "Logical OR the pixels found in each visit to form set ", 

748 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

749 } 

750 ) 

751 combinationFraction = pexConfig.RangeField( 

752 dtype=float, 

753 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

754 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

755 " mode AND to require pixel to appear in all images."), 

756 default=0.7, 

757 min=0, 

758 max=1, 

759 ) 

760 edgesAsDefects = pexConfig.Field( 

761 dtype=bool, 

762 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

763 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

764 " defect will be located there."), 

765 default=False, 

766 ) 

767 

768 

769class MergeDefectsTask(pipeBase.PipelineTask): 

770 """Merge the defects from multiple exposures. 

771 """ 

772 

773 ConfigClass = MergeDefectsTaskConfig 

774 _DefaultName = 'cpDefectMerge' 

775 

776 def run(self, inputDefects, camera): 

777 """Merge a list of single defects to find the common defect regions. 

778 

779 Parameters 

780 ---------- 

781 inputDefects : `list` [`lsst.ip.isr.Defects`] 

782 Partial defects from a single exposure. 

783 camera : `lsst.afw.cameraGeom.Camera` 

784 Camera to use for metadata. 

785 

786 Returns 

787 ------- 

788 results : `lsst.pipe.base.Struct` 

789 Results struct containing: 

790 

791 ``mergedDefects`` 

792 The defects merged from the input lists 

793 (`lsst.ip.isr.Defects`). 

794 """ 

795 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

796 if detectorId is None: 

797 raise RuntimeError("Cannot identify detector id.") 

798 detector = camera[detectorId] 

799 

800 imageTypes = set() 

801 for inDefect in inputDefects: 

802 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

803 imageTypes.add(imageType) 

804 

805 # Determine common defect pixels separately for each input image type. 

806 splitDefects = list() 

807 for imageType in imageTypes: 

808 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

809 count = 0 

810 for inDefect in inputDefects: 

811 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

812 count += 1 

813 for defect in inDefect: 

814 sumImage.image[defect.getBBox()] += 1.0 

815 sumImage /= count 

816 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

817 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

818 

819 if self.config.combinationMode == 'AND': 

820 threshold = 1.0 

821 elif self.config.combinationMode == 'OR': 

822 threshold = 0.0 

823 elif self.config.combinationMode == 'FRACTION': 

824 threshold = self.config.combinationFraction 

825 else: 

826 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

827 indices = np.where(sumImage.getImage().getArray() > threshold) 

828 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

829 sumImage.getMask().getArray()[indices] |= BADBIT 

830 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

831 partialDefect = Defects.fromMask(sumImage, 'BAD') 

832 splitDefects.append(partialDefect) 

833 

834 # Do final combination of separate image types 

835 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

836 for inDefect in splitDefects: 

837 for defect in inDefect: 

838 finalImage.image[defect.getBBox()] += 1 

839 finalImage /= len(splitDefects) 

840 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])