Coverage for python/lsst/cp/pipe/defects.py: 20%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

24 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask', 

26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ] 

27 

28import numpy as np 

29 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32 

33from lsstDebug import getDebugFrame 

34import lsst.pex.config as pexConfig 

35 

36import lsst.afw.image as afwImage 

37import lsst.afw.math as afwMath 

38import lsst.afw.detection as afwDetection 

39import lsst.afw.display as afwDisplay 

40from lsst.afw import cameraGeom 

41from lsst.geom import Box2I, Point2I 

42from lsst.meas.algorithms import SourceDetectionTask 

43from lsst.ip.isr import Defects, countMaskedPixels 

44from lsst.pex.exceptions import InvalidParameterError 

45 

46from ._lookupStaticCalibration import lookupStaticCalibration 

47 

48 

49class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

50 dimensions=("instrument", "exposure", "detector")): 

51 inputExp = cT.Input( 

52 name="defectExps", 

53 doc="Input ISR-processed exposures to measure.", 

54 storageClass="Exposure", 

55 dimensions=("instrument", "detector", "exposure"), 

56 multiple=False 

57 ) 

58 camera = cT.PrerequisiteInput( 

59 name='camera', 

60 doc="Camera associated with this exposure.", 

61 storageClass="Camera", 

62 dimensions=("instrument", ), 

63 isCalibration=True, 

64 lookupFunction=lookupStaticCalibration, 

65 ) 

66 

67 outputDefects = cT.Output( 

68 name="singleExpDefects", 

69 doc="Output measured defects.", 

70 storageClass="Defects", 

71 dimensions=("instrument", "detector", "exposure"), 

72 ) 

73 

74 

75class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

76 pipelineConnections=MeasureDefectsConnections): 

77 """Configuration for measuring defects from a list of exposures 

78 """ 

79 

80 thresholdType = pexConfig.ChoiceField( 

81 dtype=str, 

82 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found " 

83 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found " 

84 "in flats, and hot pixels in darks."), 

85 default='STDEV', 

86 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.", 

87 'VALUE': "Use pixel value to determine detection threshold."}, 

88 ) 

89 darkCurrentThreshold = pexConfig.Field( 

90 dtype=float, 

91 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define " 

92 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."), 

93 default=5, 

94 ) 

95 fracThresholdFlat = pexConfig.Field( 

96 dtype=float, 

97 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark " 

98 "pixels in flat images (fraction of the mean value per amplifier)." 

99 "Unused if thresholdType==``STDEV``."), 

100 default=0.8, 

101 ) 

102 nSigmaBright = pexConfig.Field( 

103 dtype=float, 

104 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot " 

105 "pixel detection. The default value was found to be " 

106 "appropriate for some LSST sensors in DM-17490. " 

107 "Unused if thresholdType==``VALUE``"), 

108 default=4.8, 

109 ) 

110 nSigmaDark = pexConfig.Field( 

111 dtype=float, 

112 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel " 

113 "detection. The default value was found to be " 

114 "appropriate for some LSST sensors in DM-17490. " 

115 "Unused if thresholdType==``VALUE``"), 

116 default=-5.0, 

117 ) 

118 nPixBorderUpDown = pexConfig.Field( 

119 dtype=int, 

120 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

121 default=7, 

122 ) 

123 nPixBorderLeftRight = pexConfig.Field( 

124 dtype=int, 

125 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

126 default=7, 

127 ) 

128 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

129 dtype=int, 

130 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

131 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

132 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

133 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

134 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

135 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

136 "'goodPixelColumnGapThreshold' are not marked as bad."), 

137 default=50, 

138 ) 

139 goodPixelColumnGapThreshold = pexConfig.Field( 

140 dtype=int, 

141 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

142 "'badOnAndOffPixelColumnThreshold')."), 

143 default=30, 

144 ) 

145 

146 def validate(self): 

147 super().validate() 

148 if self.nSigmaBright < 0.0: 

149 raise ValueError("nSigmaBright must be above 0.0.") 

150 if self.nSigmaDark > 0.0: 

151 raise ValueError("nSigmaDark must be below 0.0.") 

152 

153 

154class MeasureDefectsTask(pipeBase.PipelineTask): 

155 """Measure the defects from one exposure. 

156 """ 

157 

158 ConfigClass = MeasureDefectsTaskConfig 

159 _DefaultName = 'cpDefectMeasure' 

160 

161 def run(self, inputExp, camera): 

162 """Measure one exposure for defects. 

163 

164 Parameters 

165 ---------- 

166 inputExp : `lsst.afw.image.Exposure` 

167 Exposure to examine. 

168 camera : `lsst.afw.cameraGeom.Camera` 

169 Camera to use for metadata. 

170 

171 Returns 

172 ------- 

173 results : `lsst.pipe.base.Struct` 

174 Results struct containing: 

175 

176 ``outputDefects`` 

177 The defects measured from this exposure 

178 (`lsst.ip.isr.Defects`). 

179 """ 

180 detector = inputExp.getDetector() 

181 try: 

182 filterName = inputExp.getFilter().physicalLabel 

183 except AttributeError: 

184 filterName = None 

185 

186 defects = self._findHotAndColdPixels(inputExp) 

187 

188 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

189 msg = "Found %s defects containing %s pixels in %s" 

190 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

191 

192 defects.updateMetadataFromExposures([inputExp]) 

193 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

194 setCalibId=True, setDate=True, 

195 cpDefectGenImageType=datasetType) 

196 

197 return pipeBase.Struct( 

198 outputDefects=defects, 

199 ) 

200 

201 @staticmethod 

202 def _nPixFromDefects(defects): 

203 """Count pixels in a defect. 

204 

205 Parameters 

206 ---------- 

207 defects : `lsst.ip.isr.Defects` 

208 Defects to measure. 

209 

210 Returns 

211 ------- 

212 nPix : `int` 

213 Number of defect pixels. 

214 """ 

215 nPix = 0 

216 for defect in defects: 

217 nPix += defect.getBBox().getArea() 

218 return nPix 

219 

220 def _findHotAndColdPixels(self, exp): 

221 """Find hot and cold pixels in an image. 

222 

223 Using config-defined thresholds on a per-amp basis, mask 

224 pixels that are nSigma above threshold in dark frames (hot 

225 pixels), or nSigma away from the clipped mean in flats (hot & 

226 cold pixels). 

227 

228 Parameters 

229 ---------- 

230 exp : `lsst.afw.image.exposure.Exposure` 

231 The exposure in which to find defects. 

232 

233 Returns 

234 ------- 

235 defects : `lsst.ip.isr.Defects` 

236 The defects found in the image. 

237 """ 

238 self._setEdgeBits(exp) 

239 maskedIm = exp.maskedImage 

240 

241 # the detection polarity for afwDetection, True for positive, 

242 # False for negative, and therefore True for darks as they only have 

243 # bright pixels, and both for flats, as they have bright and dark pix 

244 footprintList = [] 

245 

246 for amp in exp.getDetector(): 

247 ampImg = maskedIm[amp.getBBox()].clone() 

248 

249 # crop ampImage depending on where the amp lies in the image 

250 if self.config.nPixBorderLeftRight: 

251 if ampImg.getX0() == 0: 

252 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

253 else: 

254 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

255 if self.config.nPixBorderUpDown: 

256 if ampImg.getY0() == 0: 

257 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

258 else: 

259 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

260 

261 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

262 continue 

263 

264 # Remove a background estimate 

265 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

266 ampImg -= meanClip 

267 

268 # Determine thresholds 

269 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue() 

270 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

271 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

272 if np.isnan(expTime): 

273 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second", 

274 expTime, amp.getName(), datasetType) 

275 expTime = 1. 

276 thresholdType = self.config.thresholdType 

277 if thresholdType == 'VALUE': 

278 # LCA-128 and eoTest: bright/hot pixels in dark images are 

279 # defined as any pixel with more than 5 e-/s of dark current. 

280 # We scale by the exposure time. 

281 if datasetType.lower() == 'dark': 

282 # hot pixel threshold 

283 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain() 

284 else: 

285 # LCA-128 and eoTest: dark/cold pixels in flat images as 

286 # defined as any pixel with photoresponse <80% of 

287 # the mean (at 500nm). 

288 

289 # We subtracted the mean above, so the threshold will be 

290 # negative cold pixel threshold. 

291 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip 

292 # Find equivalent sigma values. 

293 if stDev == 0.0: 

294 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.", 

295 stDev, amp.getName(), datasetType) 

296 nSigmaList = [np.inf] 

297 else: 

298 nSigmaList = [valueThreshold/stDev] 

299 else: 

300 hotPixelThreshold = self.config.nSigmaBright 

301 coldPixelThreshold = self.config.nSigmaDark 

302 if datasetType.lower() == 'dark': 

303 nSigmaList = [hotPixelThreshold] 

304 valueThreshold = stDev*hotPixelThreshold 

305 else: 

306 nSigmaList = [hotPixelThreshold, coldPixelThreshold] 

307 valueThreshold = [x*stDev for x in nSigmaList] 

308 

309 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel" 

310 "Values (hot and cold pixels thresholds): %s, %s", 

311 datasetType, amp.getName(), thresholdType, nSigmaList, valueThreshold) 

312 mergedSet = None 

313 for sigma in nSigmaList: 

314 nSig = np.abs(sigma) 

315 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

316 polarity = {-1: False, 1: True}[np.sign(sigma)] 

317 

318 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

319 

320 try: 

321 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

322 except InvalidParameterError: 

323 # This occurs if the image sigma value is 0.0. 

324 # Let's mask the whole area. 

325 minValue = np.nanmin(ampImg.image.array) - 1.0 

326 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True) 

327 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

328 

329 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

330 

331 if mergedSet is None: 

332 mergedSet = footprintSet 

333 else: 

334 mergedSet.merge(footprintSet) 

335 

336 footprintList += mergedSet.getFootprints() 

337 

338 self.debugView('defectMap', ampImg, 

339 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

340 

341 defects = Defects.fromFootprintList(footprintList) 

342 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

343 

344 return defects 

345 

346 @staticmethod 

347 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

348 """Return the number of non-bad pixels in the image.""" 

349 nPixels = maskedIm.mask.array.size 

350 nBad = countMaskedPixels(maskedIm, badMaskString) 

351 return nPixels - nBad 

352 

353 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

354 """Set edge bits on an exposure or maskedImage. 

355 

356 Raises 

357 ------ 

358 TypeError 

359 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

360 """ 

361 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

362 mi = exposureOrMaskedImage.maskedImage 

363 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

364 mi = exposureOrMaskedImage 

365 else: 

366 t = type(exposureOrMaskedImage) 

367 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

368 

369 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

370 if self.config.nPixBorderLeftRight: 

371 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

372 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

373 if self.config.nPixBorderUpDown: 

374 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

375 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

376 

377 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

378 """Mask blocks in a column if there are on-and-off bad pixels 

379 

380 If there's a column with on and off bad pixels, mask all the 

381 pixels in between, except if there is a large enough gap of 

382 consecutive good pixels between two bad pixels in the column. 

383 

384 Parameters 

385 ---------- 

386 defects : `lsst.ip.isr.Defects` 

387 The defects found in the image so far 

388 

389 Returns 

390 ------- 

391 defects : `lsst.ip.isr.Defects` 

392 If the number of bad pixels in a column is not larger or 

393 equal than self.config.badPixelColumnThreshold, the input 

394 list is returned. Otherwise, the defects list returned 

395 will include boxes that mask blocks of on-and-of pixels. 

396 """ 

397 # Get the (x, y) values of each bad pixel in amp. 

398 coordinates = [] 

399 for defect in defects: 

400 bbox = defect.getBBox() 

401 x0, y0 = bbox.getMinX(), bbox.getMinY() 

402 deltaX0, deltaY0 = bbox.getDimensions() 

403 for j in np.arange(y0, y0+deltaY0): 

404 for i in np.arange(x0, x0 + deltaX0): 

405 coordinates.append((i, j)) 

406 

407 x, y = [], [] 

408 for coordinatePair in coordinates: 

409 x.append(coordinatePair[0]) 

410 y.append(coordinatePair[1]) 

411 

412 x = np.array(x) 

413 y = np.array(y) 

414 # Find the defects with same "x" (vertical) coordinate (column). 

415 unique, counts = np.unique(x, return_counts=True) 

416 multipleX = [] 

417 for (a, b) in zip(unique, counts): 

418 if b >= self.config.badOnAndOffPixelColumnThreshold: 

419 multipleX.append(a) 

420 if len(multipleX) != 0: 

421 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

422 

423 return defects 

424 

425 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

426 """Mask blocks in a column if number of on-and-off bad pixels is above 

427 threshold. 

428 

429 This function is called if the number of on-and-off bad pixels 

430 in a column is larger or equal than 

431 self.config.badOnAndOffPixelColumnThreshold. 

432 

433 Parameters 

434 --------- 

435 x : `list` 

436 Lower left x coordinate of defect box. x coordinate is 

437 along the short axis if amp. 

438 y : `list` 

439 Lower left y coordinate of defect box. x coordinate is 

440 along the long axis if amp. 

441 multipleX : list 

442 List of x coordinates in amp. with multiple bad pixels 

443 (i.e., columns with defects). 

444 defects : `lsst.ip.isr.Defects` 

445 The defcts found in the image so far 

446 

447 Returns 

448 ------- 

449 defects : `lsst.ip.isr.Defects` 

450 The defects list returned that will include boxes that 

451 mask blocks of on-and-of pixels. 

452 """ 

453 with defects.bulk_update(): 

454 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

455 for x0 in multipleX: 

456 index = np.where(x == x0) 

457 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

458 multipleY.sort() # Ensure that the y values are sorted to look for gaps. 

459 minY, maxY = np.min(multipleY), np.max(multipleY) 

460 # Next few lines: don't mask pixels in column if gap 

461 # of good pixels between two consecutive bad pixels is 

462 # larger or equal than 'goodPixelColumnGapThreshold'. 

463 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

464 if len(diffIndex) != 0: 

465 limits = [minY] # put the minimum first 

466 for gapIndex in diffIndex: 

467 limits.append(multipleY[gapIndex]) 

468 limits.append(multipleY[gapIndex+1]) 

469 limits.append(maxY) # maximum last 

470 for i in np.arange(0, len(limits)-1, 2): 

471 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

472 defects.append(s) 

473 else: # No gap is large enough 

474 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

475 defects.append(s) 

476 return defects 

477 

478 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

479 """Plot the defects found by the task. 

480 

481 Parameters 

482 ---------- 

483 stepname : `str` 

484 Debug frame to request. 

485 ampImage : `lsst.afw.image.MaskedImage` 

486 Amplifier image to display. 

487 defects : `lsst.ip.isr.Defects` 

488 The defects to plot. 

489 detector : `lsst.afw.cameraGeom.Detector` 

490 Detector holding camera geometry. 

491 """ 

492 frame = getDebugFrame(self._display, stepname) 

493 if frame: 

494 disp = afwDisplay.Display(frame=frame) 

495 disp.scale('asinh', 'zscale') 

496 disp.setMaskTransparency(80) 

497 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

498 

499 maskedIm = ampImage.clone() 

500 defects.maskPixels(maskedIm, "BAD") 

501 

502 mpDict = maskedIm.mask.getMaskPlaneDict() 

503 for plane in mpDict.keys(): 

504 if plane in ['BAD']: 

505 continue 

506 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

507 

508 disp.setImageColormap('gray') 

509 disp.mtv(maskedIm) 

510 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

511 prompt = "Press Enter to continue [c]... " 

512 while True: 

513 ans = input(prompt).lower() 

514 if ans in ('', 'c', ): 

515 break 

516 

517 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

518 """Make a histogram of the distribution of pixel values for 

519 each amp. 

520 

521 The main image data histogram is plotted in blue. Edge 

522 pixels, if masked, are in red. Note that masked edge pixels 

523 do not contribute to the underflow and overflow numbers. 

524 

525 Note that this currently only supports the 16-amp LSST 

526 detectors. 

527 

528 Parameters 

529 ---------- 

530 stepname : `str` 

531 Debug frame to request. 

532 ampImage : `lsst.afw.image.MaskedImage` 

533 Amplifier image to display. 

534 nSigmaUsed : `float` 

535 The number of sigma used for detection 

536 exp : `lsst.afw.image.exposure.Exposure` 

537 The exposure in which the defects were found. 

538 """ 

539 frame = getDebugFrame(self._display, stepname) 

540 if frame: 

541 import matplotlib.pyplot as plt 

542 

543 detector = exp.getDetector() 

544 nX = np.floor(np.sqrt(len(detector))) 

545 nY = len(detector) // nX 

546 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

547 

548 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

549 

550 for (amp, a) in zip(reversed(detector), ax.flatten()): 

551 mi = exp.maskedImage[amp.getBBox()] 

552 

553 # normalize by expTime as we plot in ADU/s and don't 

554 # always work with master calibs 

555 mi.image.array /= expTime 

556 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

557 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

558 # Get array of pixels 

559 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

560 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

561 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

562 

563 thrUpper = mean + nSigmaUsed*sigma 

564 thrLower = mean - nSigmaUsed*sigma 

565 

566 nRight = len(imgData[imgData > thrUpper]) 

567 nLeft = len(imgData[imgData < thrLower]) 

568 

569 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

570 leftEdge = mean - nsig * nSigmaUsed*sigma 

571 rightEdge = mean + nsig * nSigmaUsed*sigma 

572 nbins = np.linspace(leftEdge, rightEdge, 1000) 

573 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

574 lw=1, edgecolor='red') 

575 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

576 lw=3, edgecolor='blue') 

577 

578 # Report number of entries in over- and under-flow 

579 # bins, i.e. off the edges of the histogram 

580 nOverflow = len(imgData[imgData > rightEdge]) 

581 nUnderflow = len(imgData[imgData < leftEdge]) 

582 

583 # Put v-lines and textboxes in 

584 a.axvline(thrUpper, c='k') 

585 a.axvline(thrLower, c='k') 

586 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

587 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

588 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

589 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

590 

591 # set axis limits and scales 

592 a.set_ylim([1., 1.7*np.max(y)]) 

593 lPlot, rPlot = a.get_xlim() 

594 a.set_xlim(np.array([lPlot, rPlot])) 

595 a.set_yscale('log') 

596 a.set_xlabel("ADU/s") 

597 fig.show() 

598 prompt = "Press Enter or c to continue [chp]..." 

599 while True: 

600 ans = input(prompt).lower() 

601 if ans in ("", " ", "c",): 

602 break 

603 elif ans in ("p", ): 

604 import pdb 

605 pdb.set_trace() 

606 elif ans in ("h", ): 

607 print("[h]elp [c]ontinue [p]db") 

608 plt.close() 

609 

610 

611class MeasureDefectsCombinedConnections(MeasureDefectsConnections, 

612 dimensions=("instrument", "detector")): 

613 inputExp = cT.Input( 

614 name="dark", 

615 doc="Input ISR-processed combined exposure to measure.", 

616 storageClass="ExposureF", 

617 dimensions=("instrument", "detector"), 

618 multiple=False, 

619 isCalibration=True, 

620 ) 

621 camera = cT.PrerequisiteInput( 

622 name='camera', 

623 doc="Camera associated with this exposure.", 

624 storageClass="Camera", 

625 dimensions=("instrument", ), 

626 isCalibration=True, 

627 lookupFunction=lookupStaticCalibration, 

628 ) 

629 

630 outputDefects = cT.Output( 

631 name="cpPartialDefectsFromDarkCombined", 

632 doc="Output measured defects.", 

633 storageClass="Defects", 

634 dimensions=("instrument", "detector"), 

635 ) 

636 

637 

638class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig, 

639 pipelineConnections=MeasureDefectsCombinedConnections): 

640 """Configuration for measuring defects from combined exposures. 

641 """ 

642 pass 

643 

644 

645class MeasureDefectsCombinedTask(MeasureDefectsTask): 

646 """Task to measure defects in combined images.""" 

647 

648 ConfigClass = MeasureDefectsCombinedTaskConfig 

649 _DefaultName = "cpDefectMeasureCombined" 

650 

651 

652class MeasureDefectsCombinedWithFilterConnections(MeasureDefectsCombinedConnections, 

653 dimensions=("instrument", "detector")): 

654 """Task to measure defects in combined flats under a certain filter.""" 

655 inputExp = cT.Input( 

656 name="flat", 

657 doc="Input ISR-processed combined exposure to measure.", 

658 storageClass="ExposureF", 

659 dimensions=("instrument", "detector", "physical_filter"), 

660 multiple=False, 

661 isCalibration=True, 

662 ) 

663 camera = cT.PrerequisiteInput( 

664 name='camera', 

665 doc="Camera associated with this exposure.", 

666 storageClass="Camera", 

667 dimensions=("instrument", ), 

668 isCalibration=True, 

669 lookupFunction=lookupStaticCalibration, 

670 ) 

671 

672 outputDefects = cT.Output( 

673 name="cpPartialDefectsFromFlatCombinedWithFilter", 

674 doc="Output measured defects.", 

675 storageClass="Defects", 

676 dimensions=("instrument", "detector", "physical_filter"), 

677 ) 

678 

679 

680class MeasureDefectsCombinedWithFilterTaskConfig( 

681 MeasureDefectsTaskConfig, 

682 pipelineConnections=MeasureDefectsCombinedWithFilterConnections): 

683 """Configuration for measuring defects from combined exposures. 

684 """ 

685 pass 

686 

687 

688class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask): 

689 """Task to measure defects in combined images.""" 

690 

691 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig 

692 _DefaultName = "cpDefectMeasureWithFilterCombined" 

693 

694 

695class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

696 dimensions=("instrument", "detector")): 

697 inputDefects = cT.Input( 

698 name="singleExpDefects", 

699 doc="Measured defect lists.", 

700 storageClass="Defects", 

701 dimensions=("instrument", "detector", "exposure",), 

702 multiple=True, 

703 ) 

704 camera = cT.PrerequisiteInput( 

705 name='camera', 

706 doc="Camera associated with these defects.", 

707 storageClass="Camera", 

708 dimensions=("instrument", ), 

709 isCalibration=True, 

710 lookupFunction=lookupStaticCalibration, 

711 ) 

712 

713 mergedDefects = cT.Output( 

714 name="defects", 

715 doc="Final merged defects.", 

716 storageClass="Defects", 

717 dimensions=("instrument", "detector"), 

718 multiple=False, 

719 isCalibration=True, 

720 ) 

721 

722 

723class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

724 pipelineConnections=MergeDefectsConnections): 

725 """Configuration for merging single exposure defects. 

726 """ 

727 

728 assertSameRun = pexConfig.Field( 

729 dtype=bool, 

730 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

731 "if the run key isn't found."), 

732 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

733 ) 

734 ignoreFilters = pexConfig.Field( 

735 dtype=bool, 

736 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

737 " images. Allows mixing of filters in the input flats. Set to False if you think" 

738 " your defects might be chromatic and want to have registry support for varying" 

739 " defects with respect to filter."), 

740 default=True, 

741 ) 

742 nullFilterName = pexConfig.Field( 

743 dtype=str, 

744 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

745 default="NONE", 

746 ) 

747 combinationMode = pexConfig.ChoiceField( 

748 doc="Which types of defects to identify", 

749 dtype=str, 

750 default="FRACTION", 

751 allowed={ 

752 "AND": "Logical AND the pixels found in each visit to form set ", 

753 "OR": "Logical OR the pixels found in each visit to form set ", 

754 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

755 } 

756 ) 

757 combinationFraction = pexConfig.RangeField( 

758 dtype=float, 

759 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

760 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

761 " mode AND to require pixel to appear in all images."), 

762 default=0.7, 

763 min=0, 

764 max=1, 

765 ) 

766 edgesAsDefects = pexConfig.Field( 

767 dtype=bool, 

768 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

769 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

770 " defect will be located there."), 

771 default=False, 

772 ) 

773 

774 

775class MergeDefectsTask(pipeBase.PipelineTask): 

776 """Merge the defects from multiple exposures. 

777 """ 

778 

779 ConfigClass = MergeDefectsTaskConfig 

780 _DefaultName = 'cpDefectMerge' 

781 

782 def run(self, inputDefects, camera): 

783 """Merge a list of single defects to find the common defect regions. 

784 

785 Parameters 

786 ---------- 

787 inputDefects : `list` [`lsst.ip.isr.Defects`] 

788 Partial defects from a single exposure. 

789 camera : `lsst.afw.cameraGeom.Camera` 

790 Camera to use for metadata. 

791 

792 Returns 

793 ------- 

794 results : `lsst.pipe.base.Struct` 

795 Results struct containing: 

796 

797 ``mergedDefects`` 

798 The defects merged from the input lists 

799 (`lsst.ip.isr.Defects`). 

800 """ 

801 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

802 if detectorId is None: 

803 raise RuntimeError("Cannot identify detector id.") 

804 detector = camera[detectorId] 

805 

806 imageTypes = set() 

807 for inDefect in inputDefects: 

808 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

809 imageTypes.add(imageType) 

810 

811 # Determine common defect pixels separately for each input image type. 

812 splitDefects = list() 

813 for imageType in imageTypes: 

814 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

815 count = 0 

816 for inDefect in inputDefects: 

817 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

818 count += 1 

819 for defect in inDefect: 

820 sumImage.image[defect.getBBox()] += 1.0 

821 sumImage /= count 

822 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

823 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

824 

825 if self.config.combinationMode == 'AND': 

826 threshold = 1.0 

827 elif self.config.combinationMode == 'OR': 

828 threshold = 0.0 

829 elif self.config.combinationMode == 'FRACTION': 

830 threshold = self.config.combinationFraction 

831 else: 

832 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

833 indices = np.where(sumImage.getImage().getArray() > threshold) 

834 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

835 sumImage.getMask().getArray()[indices] |= BADBIT 

836 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

837 partialDefect = Defects.fromMask(sumImage, 'BAD') 

838 splitDefects.append(partialDefect) 

839 

840 # Do final combination of separate image types 

841 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

842 for inDefect in splitDefects: 

843 for defect in inDefect: 

844 finalImage.image[defect.getBBox()] += 1